Infra/tiramisu
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cache in dictionary storage

Browse Source
This commit is contained in:
Emmanuel Garette
2018-06-25 21:40:16 +02:00
parent f30ca3dc46
commit b10f02a8e9
6 changed files with 306 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tiramisu/option/optiondescription.py
View File

@ -166,12 +166,15 @@ class CacheOptionDescription(BaseOption):
if option.impl_is_master_slaves('slave'):
# problem with index
raise ConfigError(_('the slave "{0}" cannot have '
'"force_store_value" property').format(option.impl_get_display_name()))
'"force_store_value" property').format(
option.impl_get_display_name()))
if option.issubdyn():
raise ConfigError(_('the dynoption "{0}" cannot have '
'"force_store_value" property').format(option.impl_get_display_name()))
'"force_store_value" property').format(
option.impl_get_display_name()))
if not values._p_.hasvalue(subpath):
config_bag = ConfigBag(config=context, option=option)
config_bag.properties = frozenset()
value = values.getvalue(subpath,
None,
config_bag)

61
tiramisu/setting.py
View File

@ -15,7 +15,6 @@
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
from time import time
from copy import copy
from logging import getLogger
import weakref
@ -337,27 +336,24 @@ class Settings(object):
# get properties and permissive methods
def get_context_properties(self):
ntime = int(time())
if self._p_.hascache(None,
None):
is_cached, props = self._p_.getcache(None,
ntime,
None)
else:
is_cached = False
if not is_cached or 'cache' not in props:
is_cached, props = self._p_.getcache(None,
None,
None,
None,
None,
'context_props')
if not is_cached:
meta = self._getcontext().cfgimpl_get_meta()
if meta is None:
props = self._p_.getproperties(None,
default_properties)
else:
props = meta.cfgimpl_get_settings().get_context_properties()
if 'cache' in props:
if 'expire' in props:
ntime = ntime + expires_time
else:
ntime = None
self._p_.setcache(None, props, ntime, None)
self._p_.setcache(None,
None,
props,
props,
None)
return props
def getproperties(self,
@ -371,19 +367,17 @@ class Settings(object):
if opt.impl_is_symlinkoption():
opt = opt.impl_getopt()
path = opt.impl_getpath(self._getcontext())
is_cached = False
if apply_requires and config_bag.setting_properties is not None:
if 'cache' in config_bag.setting_properties and \
'expire' in config_bag.setting_properties:
ntime = int(time())
else:
ntime = None
if 'cache' in config_bag.setting_properties and self._p_.hascache(path,
index):
is_cached, props = self._p_.getcache(path,
ntime,
index)
if apply_requires:
props = config_bag.setting_properties
is_cached, props = self._p_.getcache(path,
expires_time,
index,
props,
None,
'self_props')
else:
is_cached = False
if not is_cached:
meta = self._getcontext().cfgimpl_get_meta()
if meta is None:
@ -403,14 +397,12 @@ class Settings(object):
opt.impl_get_display_name())
props -= self.getpermissive(opt,
path)
if apply_requires and config_bag.setting_properties is not None and \
'cache' in config_bag.setting_properties:
if 'expire' in config_bag.setting_properties:
ntime = ntime + expires_time
if apply_requires:
self._p_.setcache(path,
index,
props,
ntime,
index)
config_bag.setting_properties,
config_bag.setting_properties)
return props
def get_context_permissive(self):
@ -599,6 +591,7 @@ class Settings(object):
"""save properties for specified path
(never save properties if same has option properties)
"""
# should have index !!!
if self._getcontext().cfgimpl_get_meta() is not None:
raise ConfigError(_('cannot change property with metaconfig'))
if path is not None and config_bag.option.impl_getrequires() is not None:

41
tiramisu/storage/dictionary/cache.py Normal file
View File

@ -0,0 +1,41 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2018 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
class Cache(object):
__slots__ = ('_cache',)
def __init__(self):
self._cache = {}
def _setcache(self, path, index, val, time):
self._cache.setdefault(path, {})[index] = (val, int(time))
def _getcache(self, path, index):
values = self._cache.get(path)
if values is None:
return
return values.get(index)
def _delcache(self, path):
del self._cache[path]
def _get_cached(self):
return self._cache
def _reset_all_cache(self):
self._cache.clear()

112
tiramisu/storage/util.py
View File

@ -15,6 +15,10 @@
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
from time import time
from .dictionary.cache import Cache as DictCache
def _display_classname(obj):
return(obj.__class__.__name__.lower())
@ -22,31 +26,78 @@ DEBUG = False
#DEBUG = True
class Cache(object):
__slots__ = ('_cache', '_storage')
key_is_path = False
class Cache(DictCache):
__slots__ = ('_storage',)
def __init__(self, storage):
self._cache = {}
self._storage = storage
super().__init__()
def setcache(self, path, val, time, index):
def setcache(self, path, index, val, props, self_props):
"""add val in cache for a specified path
if slave, add index
"""
if DEBUG:
print('setcache', path, val, _display_classname(self), id(self))
self._cache.setdefault(path, {})[index] = (val, time)
def getcache(self, path, exp, index):
value, created = self._cache[path][index]
if created is None or exp is None or exp <= created:
if props is None or 'cache' in props or \
(self_props is not None and 'cache' in self_props):
if DEBUG:
print('getcache in cache', path, value, _display_classname(self), id(self), index, exp)
return True, value
print('setcache {} with index {} and value {} in {} ({})'.format(path, index, val,
_display_classname(self),
id(self)))
self._setcache(path, index, val, time())
if DEBUG:
print('getcache not in cache')
return False, None # pragma: no cover
print('not setcache {} with index {} and value {} in {} ({})'.format(path,
index,
val,
_display_classname(self),
id(self)))
return
def getcache(self,
path,
expires_time,
index,
props,
self_props,
type_):
no_cache = False, None
if props is None or 'cache' in props:
indexed = self._getcache(path, index)
if indexed is None:
return no_cache
value, timestamp = indexed
if type_ == 'context_props':
# cached value is settings properties so value is props
props = value
elif type_ == 'self_props':
# if self_props is None, so cached value is self properties
# so value is self_props
self_props = value
# recheck "cache" value
if props is None or 'cache' in props or (self_props is not None and 'cache' in props):
if expires_time and timestamp and \
(props is not None and 'expire' in props or \
self_props is not None and 'expire' in self_props):
ntime = int(time())
if timestamp + expires_time >= ntime:
if DEBUG:
print('getcache in cache (1)', path, value, _display_classname(self),
id(self), index)
return True, value
else:
if DEBUG:
print('getcache expired value for path {} < {}'.format(
timestamp + expires_time, ntime))
# if expired, remove from cache
self.delcache(path)
else:
if DEBUG:
print('getcache in cache (2)', path, value, _display_classname(self),
id(self), index)
return True, value
if DEBUG:
print('getcache {} with index {} not in {} cache'.format(path, index,
_display_classname(self)))
return no_cache
def delcache(self, path):
"""remove cache for a specified path
@ -54,36 +105,19 @@ class Cache(object):
if DEBUG:
print('delcache', path, _display_classname(self), id(self))
if path in self._cache:
del self._cache[path]
def hascache(self, path, index):
""" path is in the cache
:param path: the path's option
"""
if DEBUG:
print('hascache', path, _display_classname(self), id(self))
return path in self._cache and index in self._cache[path]
def reset_expired_cache(self, exp):
cache_keys = list(self._cache.keys())
for key in cache_keys:
key_cache_keys = list(self._cache[key].keys())
for index in key_cache_keys:
val, created = self._cache[key][index]
if created is not None and exp > created:
del(self._cache[key][index])
if self._cache[key] == {}:
del(self._cache[key])
self._delcache(path)
def reset_all_cache(self):
"empty the cache"
if DEBUG:
print('reset_all_cache', _display_classname(self), id(self))
self._cache.clear()
self._reset_all_cache()
def get_cached(self):
"""return all values in a dictionary
please only use it in test purpose
example: {'path1': {'index1': ('value1', 'time1')}, 'path2': {'index2': ('value2', 'time2', )}}
"""
return self._cache
if DEBUG:
print('get_chached', self._cache)
return self._get_cached()

46
tiramisu/value.py
View File

@ -15,7 +15,6 @@
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
from time import time
import weakref
from .error import ConfigError, PropertiesOptionError
from .setting import owners, expires_time, undefined, forbidden_owners
@ -81,19 +80,14 @@ class Values(object):
:returns: value
"""
ntime = None
# try to retrive value in cache
setting_properties = config_bag.setting_properties
is_cached = False
if setting_properties and 'cache' in setting_properties and \
self._p_.hascache(path,
index):
if 'expire' in setting_properties or \
(config_bag.properties and 'expire' in config_bag.properties):
ntime = int(time())
is_cached, value = self._p_.getcache(path,
ntime,
index)
is_cached, value = self._p_.getcache(path,
expires_time,
index,
setting_properties,
config_bag.properties,
'value')
if not is_cached:
# no cached value so get value
@ -115,13 +109,12 @@ class Values(object):
check_error=False,
config_bag=config_bag)
# store value in cache
if not is_cached and \
setting_properties and 'cache' in setting_properties:
if 'expire' in setting_properties or 'expire' in config_bag.properties:
if ntime is None:
ntime = int(time())
ntime = ntime + expires_time
self._p_.setcache(path, value, ntime, index)
if not is_cached:
self._p_.setcache(path,
index,
value,
setting_properties,
config_bag.properties)
# and return it
return value
@ -198,10 +191,16 @@ class Values(object):
context = self._getcontext()
opt = config_bag.option
def _reset_cache(_value):
if self._p_.hascache(path, index):
is_cache, cache_value = self._p_.getcache(path, None, index)
if is_cache and cache_value == _value:
return
is_cache, cache_value = self._p_.getcache(path,
expires_time,
index,
config_bag.setting_properties,
config_bag.properties,
'value')
if is_cache and cache_value == _value:
# calculation return same value as previous value,
# so do not invalidate cache
return
# calculated value is a new value, so reset cache
context.cfgimpl_reset_cache(opt,
path,
@ -389,6 +388,7 @@ class Values(object):
config_bag,
commit=True):
self._getcontext().cfgimpl_reset_cache(config_bag.option,
path,
config_bag)