From 0ce4cc658b9abe2b1d4af859b1c1910ebf03cba7 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Thu, 21 Sep 2017 21:23:03 +0200 Subject: [PATCH] remove cache with a variable is calculated --- test/test_cache.py | 6 +- test/test_dereference.py | 179 ++++++++++++++++++++++++++- test/test_submulti.py | 18 +++ tiramisu/config.py | 66 ++++++---- tiramisu/storage/dictionary/value.py | 1 + tiramisu/value.py | 18 +-- 6 files changed, 254 insertions(+), 34 deletions(-) diff --git a/test/test_cache.py b/test/test_cache.py index 17dc30b..bc6d62d 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -573,8 +573,8 @@ def test_cache_master_callback(): 'val1': {None: (set([]), None)}, 'val1.val1': {None: (set(['empty']), None)}, 'val1.val2': {None: (set([]), None), 0: (set([]), None)}} - assert cfg.cfgimpl_get_values()._p_.get_cached(cfg) == {'val1.val1': {None: ([None], None)}, - 'val1.val2': {None: ([None], None), 0: (None, None)}} + #assert cfg.cfgimpl_get_values()._p_.get_cached(cfg) == {'val1.val1': {None: ([None], None)}, + # 'val1.val2': {None: ([None], None), 0: (None, None)}} def test_cache_requires(): @@ -665,4 +665,4 @@ def test_callback_value_incr(): assert cfg.val2 == 1 sleep(1) assert cfg.val1 == 2 - #assert cfg.val2 == 2 + assert cfg.val2 == 2 diff --git a/test/test_dereference.py b/test/test_dereference.py index a1e7652..9b4e247 100644 --- a/test/test_dereference.py +++ b/test/test_dereference.py @@ -3,13 +3,18 @@ from .autopath import do_autopath do_autopath() from tiramisu.config import Config, GroupConfig, MetaConfig -from tiramisu.option import BoolOption, IntOption, StrOption, OptionDescription, submulti +from tiramisu.option import BoolOption, IntOption, StrOption, IPOption, NetmaskOption, \ + SymLinkOption, OptionDescription, DynOptionDescription, submulti import weakref IS_DEREFABLE = True +def funcname(value): + return value + + def test_deref_storage(): b = BoolOption('b', '') o = OptionDescription('od', '', [b]) @@ -78,7 +83,7 @@ def test_deref_option_cache(): return b = BoolOption('b', '') o = OptionDescription('od', '', [b]) - o.impl_build_cache_option() + o._build_cache_option() w = weakref.ref(b) del(b) assert w() is not None @@ -91,7 +96,7 @@ def test_deref_optiondescription_cache(): return b = BoolOption('b', '') o = OptionDescription('od', '', [b]) - o.impl_build_cache_option() + o._build_cache_option() w = weakref.ref(o) del(b) assert w() is not None @@ -179,3 +184,171 @@ def test_deref_submulti(): del(m) assert w() is None assert z() is None + + +def test_deref_consistency(): + if not IS_DEREFABLE: + return + a = IPOption('a', '') + b = NetmaskOption('b', '') + od = OptionDescription('od', '', [a, b]) + b.impl_add_consistency('ip_netmask', a) + cfg = Config(od) + w = weakref.ref(a) + x = weakref.ref(b) + y = weakref.ref(od) + z = weakref.ref(cfg) + assert w() is not None + assert x() is not None + assert y() is not None + assert z() is not None + del(a) + del(b) + assert w() is not None + assert x() is not None + assert y() is not None + assert z() is not None + del(od) + assert w() is not None + assert x() is not None + assert y() is not None + assert z() is not None + del(cfg) + assert y() is None + assert z() is None + #assert w() is None + #assert x() is None + + +def test_deref_validator(): + if not IS_DEREFABLE: + return + a = StrOption('a', '', default='yes') + b = StrOption('b', '', validator=funcname, validator_params={'': ((a, False),)}, default='val') + od = OptionDescription('root', '', [a, b]) + cfg = Config(od) + w = weakref.ref(a) + x = weakref.ref(b) + y = weakref.ref(od) + z = weakref.ref(cfg) + assert w() is not None + assert x() is not None + assert w() is not None + assert x() is not None + del(a) + del(b) + assert w() is not None + assert x() is not None + assert w() is not None + assert x() is not None + del(od) + assert w() is not None + assert x() is not None + assert w() is not None + assert x() is not None + del(cfg) + assert y() is None + assert z() is None + #assert w() is None + #assert x() is None + + +def test_deref_callback(): + if not IS_DEREFABLE: + return + a = StrOption('a', "", 'val') + b = StrOption('b', "", callback=funcname, callback_params={'': ((a, False),)}) + od = OptionDescription('root', '', [a, b]) + cfg = Config(od) + w = weakref.ref(a) + x = weakref.ref(b) + y = weakref.ref(od) + z = weakref.ref(cfg) + assert w() is not None + assert x() is not None + assert w() is not None + assert x() is not None + del(a) + del(b) + assert w() is not None + assert x() is not None + assert w() is not None + assert x() is not None + del(od) + assert w() is not None + assert x() is not None + assert w() is not None + assert x() is not None + del(cfg) + assert y() is None + assert z() is None + #assert w() is None + #assert x() is None + + +def test_deref_symlink(): + if not IS_DEREFABLE: + return + a = BoolOption("a", "", default=False) + b = SymLinkOption("b", a) + od = OptionDescription('root', '', [a, b]) + cfg = Config(od) + w = weakref.ref(a) + x = weakref.ref(b) + y = weakref.ref(od) + z = weakref.ref(cfg) + assert w() is not None + assert x() is not None + assert w() is not None + assert x() is not None + del(a) + del(b) + assert w() is not None + assert x() is not None + assert w() is not None + assert x() is not None + del(od) + assert w() is not None + assert x() is not None + assert w() is not None + assert x() is not None + del(cfg) + #assert w() is None + #assert x() is None + assert y() is None + assert z() is None + + +def test_deref_dyn(): + if not IS_DEREFABLE: + return + a = StrOption('a', '', ['val1', 'val2'], multi=True) + b = StrOption('b', '') + dod = DynOptionDescription('dod', '', [b], callback=funcname, callback_params={'': ((a, False),)}) + od = OptionDescription('od', '', [dod, a]) + cfg = Config(od) + w = weakref.ref(a) + x = weakref.ref(b) + y = weakref.ref(od) + z = weakref.ref(cfg) + assert w() is not None + assert x() is not None + assert w() is not None + assert x() is not None + del(a) + del(b) + assert w() is not None + assert x() is not None + assert w() is not None + assert x() is not None + del(od) + del(dod) + assert w() is not None + assert x() is not None + assert w() is not None + assert x() is not None + del(cfg) + #assert w() is None + #assert x() is None + assert y() is None + assert z() is None diff --git a/test/test_submulti.py b/test/test_submulti.py index 321ca37..10a37ad 100644 --- a/test/test_submulti.py +++ b/test/test_submulti.py @@ -696,3 +696,21 @@ def test_multi_submulti_meta(): multi.append() assert conf1.multi == [['val', None]] assert meta.multi == [['val']] + + +def test_multi_submulti_meta_no_cache(): + multi = StrOption('multi', '', multi=submulti) + od = OptionDescription('od', '', [multi]) + conf1 = Config(od, session_id='conf1') + conf1.read_write() + conf2 = Config(od, session_id='conf2') + conf2.read_write() + meta = MetaConfig([conf1, conf2]) + meta.read_write() + meta.cfgimpl_get_settings().remove('cache') + meta.multi = [['val']] + assert meta.multi == [['val']] + multi = conf1.multi[0] + multi.append() + assert conf1.multi == [['val', None]] + assert meta.multi == [['val']] diff --git a/tiramisu/config.py b/tiramisu/config.py index edb6334..928aff5 100644 --- a/tiramisu/config.py +++ b/tiramisu/config.py @@ -22,6 +22,7 @@ import weakref import sys from time import time +from itertools import chain from .error import PropertiesOptionError, ConfigError, ConflictError @@ -81,22 +82,9 @@ class SubConfig(object): :param only_expired: if True reset only expired cached values :type only_expired: boolean """ - if orig_opts is None: - orig_opts = set() - context = self._cfgimpl_get_context() - if 'values' in only: - values = context.cfgimpl_get_values() - if 'settings' in only or 'properties' in only or 'permissives' in only: - settings = context.cfgimpl_get_settings() - if only_expired: - if 'values' in only: - values._p_.reset_expired_cache(int(time())) - if 'settings' in only or 'properties' in only: - settings._p_.reset_expired_cache(int(time())) - if 'settings' in only or 'permissives' in only: - settings._pp_.reset_expired_cache(int(time())) - elif not None in (opt, path): + def reset_one_option_cache(opt, path): if opt.__class__.__name__ == 'DynOptionDescription': + # this option is a DynOptionDescription descr = context.cfgimpl_get_description() spath = path.split('.') subpath = '.'.join(spath[:-1]) @@ -111,6 +99,7 @@ class SubConfig(object): if 'settings' in only or 'permissives' in only: settings._pp_.delcache(path) elif not isinstance(opt, DynSymLinkOption) and opt._is_subdyn(): + # this option is an instance of DynOptionDescription descr = context.cfgimpl_get_description() spath = path.split('.') try: @@ -150,7 +139,43 @@ class SubConfig(object): settings._p_.delcache(path) if 'settings' in only or 'permissives' in only: settings._pp_.delcache(path) - for option in opt._get_dependencies(self): + + def reset_expired_cache(): + # reset cache for expired cache value ony + datetime = int(time()) + if 'values' in only: + values._p_.reset_expired_cache(datetime) + if 'settings' in only or 'properties' in only: + settings._p_.reset_expired_cache(datetime) + if 'settings' in only or 'permissives' in only: + settings._pp_.reset_expired_cache(datetime) + + def reset_all_cache(): + if 'values' in only: + values._p_.reset_all_cache() + if 'settings' in only: + settings._p_.reset_all_cache() + settings._pp_.reset_all_cache() + + if orig_opts is None: + orig_opts = set() + + context = self._cfgimpl_get_context() + if 'values' in only: + values = context.cfgimpl_get_values() + if 'settings' in only or 'properties' in only or 'permissives' in only: + settings = context.cfgimpl_get_settings() + + if not None in (opt, path): + reset_one_option_cache(opt, path) + + # remove cache for option which has dependencies with this option + if not isinstance(opt, OptionDescription) and not isinstance(opt, SynDynOptionDescription) and \ + opt.impl_is_master_slaves('slave'): + slaves = opt.impl_get_master_slaves().getslaves(opt) + else: + slaves = [] + for option in chain(opt._get_dependencies(self), slaves): if option in orig_opts: continue if 'values' in only: @@ -162,12 +187,11 @@ class SubConfig(object): option.reset_cache(opt, settings, 'properties', orig_opts) if 'permissives' in only: option.reset_cache(opt, settings, 'permissives', orig_opts) + + elif only_expired: + reset_expired_cache() else: - if 'values' in only: - values._p_.reset_all_cache() - if 'settings' in only: - settings._p_.reset_all_cache() - settings._pp_.reset_all_cache() + reset_all_cache() def cfgimpl_get_home_by_path(self, path, force_permissive=False, returns_raise=False, _setting_properties=undefined): diff --git a/tiramisu/storage/dictionary/value.py b/tiramisu/storage/dictionary/value.py index 4cd9c7f..8f8b979 100644 --- a/tiramisu/storage/dictionary/value.py +++ b/tiramisu/storage/dictionary/value.py @@ -65,6 +65,7 @@ class Values(Cache): lst[idx] = tuple(lst[idx]) values.append(tuple(lst)) return vidx + # value def setvalue(self, path, value, owner, index, session, commit): """set value for a path diff --git a/tiramisu/value.py b/tiramisu/value.py index 4bd0939..75f010c 100644 --- a/tiramisu/value.py +++ b/tiramisu/value.py @@ -87,6 +87,7 @@ class Values(object): callback=callback, callback_params=callback_params, index=index, validate=validate) + _orig_context.cfgimpl_reset_cache(opt=opt, path=path, only=('values',)) if isinstance(value, list) and index is not None: #if return a list and index is set, return value only if #it's a submulti without submulti_index and without list of list @@ -682,7 +683,7 @@ class Values(object): if not 'cache' in context.cfgimpl_get_settings(): raise ConfigError(_('can force cache only if cache ' 'is actived in config')) - #FIXME properties and value should update "expired" time + context.cfgimpl_reset_cache() for path in context.cfgimpl_get_description().impl_getpaths( include_groups=True): err = context.getattr(path, returns_raise=True) @@ -731,7 +732,7 @@ class Multi(list): context, opt, path, idx)) - self[idx].submulti = weakref.ref(self) + self[idx].refmulti = weakref.ref(self) def _getcontext(self): """context could be None, we need to test it @@ -808,7 +809,7 @@ class Multi(list): if not '_index' in self.__slots__ and self.opt.impl_is_submulti(): if not isinstance(value, SubMulti): value = SubMulti(value, self.context, self.opt, self.path, index) - value.submulti = weakref.ref(self) + value.refmulti = weakref.ref(self) super(Multi, self).append(value) if setitem: self._store(force=force) @@ -911,13 +912,12 @@ class Multi(list): values = self._getcontext().cfgimpl_get_values() if not force: #FIXME could get properties an pass it - values.validate(self.opt, self, self.path, - valid_masterslave=False) + values.validate(self.opt, self, self.path, valid_masterslave=False) values._setvalue(self.opt, self.path, self, index=index) class SubMulti(Multi): - __slots__ = ('_index', 'submulti') + __slots__ = ('_index', 'refmulti') def __init__(self, value, context, opt, path, index): """ @@ -940,7 +940,11 @@ class SubMulti(Multi): #force is unused here values = self._getcontext().cfgimpl_get_values() values.validate(self.opt, self, self.path, valid_masterslave=False) - values._setvalue(self.opt, self.path, self.submulti()) + multi = self.refmulti() + if multi is None: + multi = values._get_cached_value(self.opt, path=self.path) + multi[self._index] = self + values._setvalue(self.opt, self.path, multi) def _validate(self, value, fake_context, force_index, submulti=False): if value is not None: