Infra/tiramisu
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

separate Cache from values and settings

Browse Source
This commit is contained in:
Emmanuel Garette
2019-08-01 18:22:18 +02:00
parent c5ce9be699
commit 1691e38db5
16 changed files with 348 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
tiramisu/config.py
View File

@ -28,7 +28,7 @@ from .error import PropertiesOptionError, ConfigError, ConflictError, \
from .option import SynDynOptionDescription, DynOptionDescription, Leadership
from .option.baseoption import BaseOption, valid_name
from .setting import OptionBag, ConfigBag, Settings, undefined
from .storage import get_storages, gen_storage_id, get_default_values_storages, list_sessions
from .storage import get_storages, gen_storage_id, get_default_values_storages, list_sessions, Cache
from .value import Values
from .i18n import _
@ -104,8 +104,6 @@ class SubConfig(object):
def reset_one_option_cache(self,
desc,
values,
settings,
resetted_opts,
option_bag):
@ -127,8 +125,6 @@ class SubConfig(object):
option_bag.index,
option_bag.config_bag)
self.reset_one_option_cache(desc,
values,
settings,
resetted_opts,
doption_bag)
elif option.issubdyn():
@ -146,8 +142,6 @@ class SubConfig(object):
option_bag.index,
option_bag.config_bag)
self.reset_one_option_cache(desc,
values,
settings,
resetted_opts,
doption_bag)
else:
@ -158,14 +152,11 @@ class SubConfig(object):
option_bag.index,
option_bag.config_bag)
self.reset_one_option_cache(desc,
values,
settings,
resetted_opts,
doption_bag)
del option
option_bag.option.reset_cache(option_bag.path,
values,
settings,
option_bag.config_bag,
resetted_opts)
def cfgimpl_reset_cache(self,
@ -178,18 +169,14 @@ class SubConfig(object):
context = self.cfgimpl_get_context()
desc = context.cfgimpl_get_description()
values = context.cfgimpl_get_values()
settings = context.cfgimpl_get_settings()
if option_bag is not None:
self.reset_one_option_cache(desc,
values,
settings,
resetted_opts,
option_bag)
else:
values._p_.reset_all_cache()
settings._p_.reset_all_cache()
context._impl_values_cache.reset_all_cache()
context._impl_properties_cache.reset_all_cache()
def cfgimpl_get_home_by_path(self,
path,
@ -608,7 +595,10 @@ class SubConfig(object):
class _CommonConfig(SubConfig):
"abstract base class for the Config, KernelGroupConfig and the KernelMetaConfig"
__slots__ = ('_impl_values',
'_impl_values_cache',
'_impl_settings',
'_impl_properties_cache',
'_impl_permissives_cache',
'_impl_meta',
'impl_type')
@ -718,7 +708,8 @@ class KernelConfig(_CommonConfig):
"main configuration management entry"
__slots__ = ('__weakref__',
'_impl_name',
'_display_name')
'_display_name',
'_impl_symlink')
impl_type = 'config'
def __init__(self,
@ -743,6 +734,7 @@ class KernelConfig(_CommonConfig):
:type persistent: `boolean`
"""
self._impl_meta = None
self._impl_symlink = []
self._display_name = display_name
if isinstance(descr, Leadership):
raise ConfigError(_('cannot set leadership object has root optiondescription'))
@ -754,7 +746,10 @@ class KernelConfig(_CommonConfig):
force_settings[1])
else:
self._impl_settings = force_settings
self._impl_permissives_cache = Cache()
self._impl_properties_cache = Cache()
self._impl_values = Values(force_values)
self._impl_values_cache = Cache()
else:
properties, permissives, values, session_id = get_storages(self,
session_id,
@ -764,7 +759,10 @@ class KernelConfig(_CommonConfig):
raise ValueError(_("invalid session ID: {0} for config").format(session_id))
self._impl_settings = Settings(properties,
permissives)
self._impl_permissives_cache = Cache()
self._impl_properties_cache = Cache()
self._impl_values = Values(values)
self._impl_values_cache = Cache()
super().__init__(descr,
weakref.ref(self),
ConfigBag(self),
@ -982,7 +980,8 @@ class KernelGroupConfig(_CommonConfig):
class KernelMixConfig(KernelGroupConfig):
__slots__ = ('_display_name',)
__slots__ = ('_display_name',
'_impl_symlink')
impl_type = 'mix'
def __init__(self,
@ -995,15 +994,8 @@ class KernelMixConfig(KernelGroupConfig):
_duplicate=False):
# FIXME _duplicate
self._display_name = display_name
self._impl_symlink = []
for child in children:
if not isinstance(child, _CommonConfig):
try:
child = child._config
except:
raise TypeError(_("{}config's children "
"should be config, not {}"
).format(self.impl_type,
type(child)))
if not isinstance(child, (KernelConfig, KernelMixConfig)):
raise TypeError(_("child must be a Config, MixConfig or MetaConfig"))
if child.cfgimpl_get_meta() is not None:
@ -1015,7 +1007,10 @@ class KernelMixConfig(KernelGroupConfig):
storage=storage)
self._impl_settings = Settings(properties,
permissives)
self._impl_permissives_cache = Cache()
self._impl_properties_cache = Cache()
self._impl_values = Values(values)
self._impl_values_cache = Cache()
super().__init__(children,
session_id=session_id,
_descr=optiondescription)
@ -1199,13 +1194,17 @@ class KernelMixConfig(KernelGroupConfig):
config.cfgimpl_reset_cache(None, None)
def pop_config(self,
session_id):
for idx, child in enumerate(self._impl_children):
if session_id == child.impl_getname():
child._impl_meta = None
child.cfgimpl_reset_cache(None, None)
return self._impl_children.pop(idx)
raise ConfigError(_('cannot find the config {}').format(session_id))
session_id,
config):
if session_id is not None:
for idx, child in enumerate(self._impl_children):
if session_id == child.impl_getname():
child._impl_meta = None
child.cfgimpl_reset_cache(None, None)
return self._impl_children.pop(idx)
raise ConfigError(_('cannot find the config {}').format(session_id))
if config is not None:
self._impl_children.pop(config)
class KernelMetaConfig(KernelMixConfig):
@ -1236,14 +1235,6 @@ class KernelMetaConfig(KernelMixConfig):
children = new_children
descr = optiondescription
for child in children:
if not isinstance(child, _CommonConfig):
try:
child = child._config
except Exception:
raise TypeError(_("{}config's children "
"should be config, not {}"
).format(self.impl_type,
type(child)))
if __debug__ and not isinstance(child, (KernelConfig,
KernelMetaConfig)):
raise TypeError(_("child must be a Config or MetaConfig"))
@ -1286,7 +1277,7 @@ class KernelMetaConfig(KernelMixConfig):
display_name=self._display_name)
# Copy context properties/permissives
if new:
config.cfgimpl_get_settings().set_context_properties(self.cfgimpl_get_settings().get_context_properties(), config)
config.cfgimpl_get_settings().set_context_properties(self.cfgimpl_get_settings().get_context_properties(config._impl_properties_cache), config)
config.cfgimpl_get_settings().set_context_permissives(self.cfgimpl_get_settings().get_context_permissives())
config.cfgimpl_get_settings().ro_append = self.cfgimpl_get_settings().ro_append
config.cfgimpl_get_settings().rw_append = self.cfgimpl_get_settings().rw_append

10
tiramisu/option/baseoption.py
View File

@ -425,13 +425,13 @@ class BaseOption(Base):
def reset_cache(self,
path: str,
values: Values,
settings: Settings,
config_bag: 'OptionBag',
resetted_opts: List[Base]) -> None:
settings._p_.delcache(path)
settings._pp_.delcache(path)
context = config_bag.context
context._impl_properties_cache.delcache(path)
context._impl_permissives_cache.delcache(path)
if not self.impl_is_optiondescription():
values._p_.delcache(path)
context._impl_values_cache.delcache(path)
def impl_is_symlinkoption(self) -> bool:
return False

18
tiramisu/option/leadership.py
View File

@ -182,36 +182,30 @@ class Leadership(OptionDescription):
def reset_cache(self,
path: str,
values: Values,
settings: Settings,
config_bag: 'ConfigBag',
resetted_opts: List[Option]) -> None:
self._reset_cache(path,
self.get_leader(),
self.get_followers(),
values,
settings,
config_bag,
resetted_opts)
def _reset_cache(self,
path: str,
leader: Option,
followers: List[Option],
values: Values,
settings: Settings,
config_bag: 'ConfigBag',
resetted_opts: List[Option]) -> None:
super().reset_cache(path,
values,
settings,
config_bag,
resetted_opts)
leader.reset_cache(leader.impl_getpath(),
values,
settings,
config_bag,
None)
for follower in followers:
spath = follower.impl_getpath()
follower.reset_cache(spath,
values,
settings,
config_bag,
None)
resetted_opts.append(spath)

6
tiramisu/option/syndynoptiondescription.py
View File

@ -117,16 +117,14 @@ class SynDynLeadership(SynDynOptionDescription):
def reset_cache(self,
path: str,
values: Values,
settings: Settings,
config_bag: 'ConfigBag',
resetted_opts: List[str]) -> None:
leader = self.get_leader()
followers = self.get_followers()
self._reset_cache(path,
leader,
followers,
values,
settings,
config_bag,
resetted_opts)
def pop(self,

54
tiramisu/setting.py
View File

@ -196,7 +196,7 @@ class ConfigBag:
def __getattr__(self, key):
if key == 'properties':
settings = self.context.cfgimpl_get_settings()
self.properties = settings.get_context_properties()
self.properties = settings.get_context_properties(self.context._impl_properties_cache)
return self.properties
if key == 'permissives':
settings = self.context.cfgimpl_get_settings()
@ -390,22 +390,23 @@ class Settings(object):
# ____________________________________________________________
# get properties and permissive methods
def get_context_properties(self):
is_cached, props, validated = self._p_.getcache(None,
None,
None,
{},
{},
'context_props')
def get_context_properties(self,
cache):
is_cached, props, validated = cache.getcache(None,
None,
None,
{},
{},
'context_props')
if not is_cached:
props = self._p_.getproperties(None,
self.default_properties)
self._p_.setcache(None,
None,
props,
{},
props,
True)
cache.setcache(None,
None,
props,
{},
props,
True)
return props
def getproperties(self,
@ -423,13 +424,14 @@ class Settings(object):
path = opt.impl_getpath()
if apply_requires:
cache = config_bag.context._impl_properties_cache
props = config_bag.properties
is_cached, props, validated = self._p_.getcache(path,
config_bag.expiration_time,
index,
props,
{},
'self_props')
is_cached, props, validated = cache.getcache(path,
config_bag.expiration_time,
index,
props,
{},
'self_props')
else:
is_cached = False
if not is_cached:
@ -443,12 +445,12 @@ class Settings(object):
path)
#if apply_requires and config_bag.properties == config_bag.true_properties:
if apply_requires and not config_bag.is_unrestraint:
self._p_.setcache(path,
index,
props,
props,
config_bag.properties,
True)
cache.setcache(path,
index,
props,
props,
config_bag.properties,
True)
return props
def get_context_permissives(self):

1
tiramisu/storage/__init__.py
View File

@ -31,6 +31,7 @@ from os.path import split
from typing import Dict
from ..error import ConfigError
from ..i18n import _
from .util import Cache
DEFAULT_STORAGE = MEMORY_STORAGE = 'dictionary'

15
tiramisu/storage/cache/dictionary.py vendored
View File

@ -16,26 +16,27 @@
# ____________________________________________________________
class Cache(object):
class Cache:
__slots__ = ('_cache',)
def __init__(self):
self._cache = {}
def _setcache(self, path, index, val, time, validated):
def setcache(self, path, index, val, time, validated):
self._cache.setdefault(path, {})[index] = (val, int(time), validated)
def _getcache(self, path, index):
def getcache(self, path, index):
values = self._cache.get(path)
if values is None:
return
return values.get(index)
def _delcache(self, path):
del self._cache[path]
def delcache(self, path):
if path in self._cache:
del self._cache[path]
def _get_cached(self):
def get_cached(self):
return self._cache
def _reset_all_cache(self):
def reset_all_cache(self):
self._cache.clear()

16
tiramisu/storage/dictionary/setting.py
View File

@ -16,19 +16,18 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
from copy import copy
from ..util import Cache
from ...log import log
class Properties(Cache):
__slots__ = ('_properties',)
class Properties:
__slots__ = ('_properties',
'_storage')
def __init__(self, storage):
# properties attribute: the name of a property enables this property
# key is None for global properties
self._properties = {}
# permissive properties
super(Properties, self).__init__(storage)
self._storage = storage
# properties
def setproperties(self, path, properties):
@ -55,13 +54,14 @@ class Properties(Cache):
self._properties = properties
class Permissives(Cache):
__slots__ = ('_permissives',)
class Permissives:
__slots__ = ('_permissives',
'_storage')
def __init__(self, storage):
# permissive properties
self._permissives = {}
super(Permissives, self).__init__(storage)
self._storage = storage
def setpermissives(self, path, permissives):
log.debug('setpermissives %s %s', path, permissives)

2
tiramisu/storage/dictionary/storage.py
View File

@ -33,7 +33,7 @@ def list_sessions():
return _list_sessions
class Storage(object):
class Storage:
__slots__ = ('session_id', 'persistent')
storage = 'dictionary'
# if object could be serializable

8
tiramisu/storage/dictionary/value.py
View File

@ -15,7 +15,6 @@
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
from ..util import Cache
from ...setting import undefined
from ...i18n import _
from ...log import log
@ -23,19 +22,18 @@ from ...log import log
from copy import deepcopy
class Values(Cache):
class Values:
__slots__ = ('_values',
'_informations',
'_storage',
'__weakref__')
def __init__(self, storage):
"""init plugin means create values storage
"""
#(('path1',), (index1,), (value1,), ('owner1'))
self._values = ([], [], [], [])
self._informations = {}
# should init cache too
super(Values, self).__init__(storage)
self._storage = storage
def commit(self):
pass

10
tiramisu/storage/sqlite3/sqlite3db.py
View File

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
"sqlite3 cache"
"sqlite3"
# Copyright (C) 2013-2019 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it
@ -19,14 +19,14 @@ try:
from cPickle import loads, dumps
except ImportError:
from pickle import loads, dumps
from ..util import Cache
class Sqlite3DB(Cache):
__slots__ = ('_session_id',)
class Sqlite3DB:
__slots__ = ('_session_id',
'_storage')
def __init__(self, storage):
self._session_id = storage.session_id
super(Sqlite3DB, self).__init__(storage)
self._storage = storage
def _sqlite_decode_path(self, path):
if path == '_none':

2
tiramisu/storage/sqlite3/storage.py
View File

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
"default plugin for cache: set it in a simple dictionary"
" with sqlite3 engine"
# Copyright (C) 2013-2019 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it

20
tiramisu/storage/util.py
View File

@ -25,12 +25,6 @@ def _display_classname(obj): # pragma: no cover
class Cache(DictCache):
__slots__ = ('_storage',)
def __init__(self, storage):
self._storage = storage
super().__init__()
def setcache(self, path, index, val, self_props, props, validated):
"""add val in cache for a specified path
if follower, add index
@ -38,7 +32,7 @@ class Cache(DictCache):
if 'cache' in props or 'cache' in self_props:
log.debug('setcache %s with index %s and value %s in %s (%s)',
path, index, val, _display_classname(self), id(self))
self._setcache(path, index, val, time(), validated)
super().setcache(path, index, val, time(), validated)
log.debug('not setcache %s with index %s and value %s and props %s and %s in %s (%s)',
path, index, val, props, self_props, _display_classname(self), id(self))
@ -51,7 +45,7 @@ class Cache(DictCache):
type_):
no_cache = False, None, False
if 'cache' in props or type_ == 'context_props':
indexed = self._getcache(path, index)
indexed = super().getcache(path, index)
if indexed is None:
return no_cache
value, timestamp, validated = indexed
@ -89,18 +83,18 @@ class Cache(DictCache):
"""remove cache for a specified path
"""
log.debug('delcache %s %s %s', path, _display_classname(self), id(self))
if path in self._cache:
self._delcache(path)
super().delcache(path)
def reset_all_cache(self):
"empty the cache"
log.debug('reset_all_cache %s %s', _display_classname(self), id(self))
self._reset_all_cache()
super().reset_all_cache()
def get_cached(self):
"""return all values in a dictionary
please only use it in test purpose
example: {'path1': {'index1': ('value1', 'time1')}, 'path2': {'index2': ('value2', 'time2', )}}
"""
log.debug('get_chached %s for %s (%s)', self._cache, _display_classname(self), id(self))
return self._get_cached()
cache = super().get_cached()
log.debug('get_chached %s for %s (%s)', cache, _display_classname(self), id(self))
return cache

38
tiramisu/value.py
View File

@ -62,12 +62,13 @@ class Values(object):
"""
# try to retrive value in cache
setting_properties = option_bag.config_bag.properties
is_cached, value, validated = self._p_.getcache(option_bag.path,
option_bag.config_bag.expiration_time,
option_bag.index,
setting_properties,
option_bag.properties,
'value')
cache = option_bag.config_bag.context._impl_values_cache
is_cached, value, validated = cache.getcache(option_bag.path,
option_bag.config_bag.expiration_time,
option_bag.index,
setting_properties,
option_bag.properties,
'value')
if not validated:
# no cached value so get value
value = self.getvalue(option_bag)
@ -79,12 +80,12 @@ class Values(object):
properties = option_bag.config_bag.properties
validator = 'validator' in properties and 'demoting_error_warning' not in properties
if not option_bag.fromconsistency and (not is_cached or validator):
self._p_.setcache(option_bag.path,
option_bag.index,
value,
option_bag.properties,
setting_properties,
validator)
cache.setcache(option_bag.path,
option_bag.index,
value,
option_bag.properties,
setting_properties,
validator)
if 'warnings' in setting_properties:
option_bag.option.impl_validate(value,
option_bag,
@ -175,12 +176,13 @@ class Values(object):
def _reset_cache(_value):
if not 'expire' in option_bag.properties:
return
is_cache, cache_value, validated = self._p_.getcache(option_bag.path,
None,
option_bag.index,
option_bag.config_bag.properties,
option_bag.properties,
'value')
cache = option_bag.config_bag.context._impl_values_cache
is_cache, cache_value, validated = cache.getcache(option_bag.path,
None,
option_bag.index,
option_bag.config_bag.properties,
option_bag.properties,
'value')
if not is_cache or cache_value == _value:
# calculation return same value as previous value,
# so do not invalidate cache