remove cache with a variable is calculated

This commit is contained in:
Emmanuel Garette 2017-09-21 21:23:03 +02:00
parent 3567e18256
commit 0ce4cc658b
6 changed files with 254 additions and 34 deletions

View File

@ -573,8 +573,8 @@ def test_cache_master_callback():
'val1': {None: (set([]), None)}, 'val1': {None: (set([]), None)},
'val1.val1': {None: (set(['empty']), None)}, 'val1.val1': {None: (set(['empty']), None)},
'val1.val2': {None: (set([]), None), 0: (set([]), None)}} 'val1.val2': {None: (set([]), None), 0: (set([]), None)}}
assert cfg.cfgimpl_get_values()._p_.get_cached(cfg) == {'val1.val1': {None: ([None], None)}, #assert cfg.cfgimpl_get_values()._p_.get_cached(cfg) == {'val1.val1': {None: ([None], None)},
'val1.val2': {None: ([None], None), 0: (None, None)}} # 'val1.val2': {None: ([None], None), 0: (None, None)}}
def test_cache_requires(): def test_cache_requires():
@ -665,4 +665,4 @@ def test_callback_value_incr():
assert cfg.val2 == 1 assert cfg.val2 == 1
sleep(1) sleep(1)
assert cfg.val1 == 2 assert cfg.val1 == 2
#assert cfg.val2 == 2 assert cfg.val2 == 2

View File

@ -3,13 +3,18 @@ from .autopath import do_autopath
do_autopath() do_autopath()
from tiramisu.config import Config, GroupConfig, MetaConfig from tiramisu.config import Config, GroupConfig, MetaConfig
from tiramisu.option import BoolOption, IntOption, StrOption, OptionDescription, submulti from tiramisu.option import BoolOption, IntOption, StrOption, IPOption, NetmaskOption, \
SymLinkOption, OptionDescription, DynOptionDescription, submulti
import weakref import weakref
IS_DEREFABLE = True IS_DEREFABLE = True
def funcname(value):
return value
def test_deref_storage(): def test_deref_storage():
b = BoolOption('b', '') b = BoolOption('b', '')
o = OptionDescription('od', '', [b]) o = OptionDescription('od', '', [b])
@ -78,7 +83,7 @@ def test_deref_option_cache():
return return
b = BoolOption('b', '') b = BoolOption('b', '')
o = OptionDescription('od', '', [b]) o = OptionDescription('od', '', [b])
o.impl_build_cache_option() o._build_cache_option()
w = weakref.ref(b) w = weakref.ref(b)
del(b) del(b)
assert w() is not None assert w() is not None
@ -91,7 +96,7 @@ def test_deref_optiondescription_cache():
return return
b = BoolOption('b', '') b = BoolOption('b', '')
o = OptionDescription('od', '', [b]) o = OptionDescription('od', '', [b])
o.impl_build_cache_option() o._build_cache_option()
w = weakref.ref(o) w = weakref.ref(o)
del(b) del(b)
assert w() is not None assert w() is not None
@ -179,3 +184,171 @@ def test_deref_submulti():
del(m) del(m)
assert w() is None assert w() is None
assert z() is None assert z() is None
def test_deref_consistency():
if not IS_DEREFABLE:
return
a = IPOption('a', '')
b = NetmaskOption('b', '')
od = OptionDescription('od', '', [a, b])
b.impl_add_consistency('ip_netmask', a)
cfg = Config(od)
w = weakref.ref(a)
x = weakref.ref(b)
y = weakref.ref(od)
z = weakref.ref(cfg)
assert w() is not None
assert x() is not None
assert y() is not None
assert z() is not None
del(a)
del(b)
assert w() is not None
assert x() is not None
assert y() is not None
assert z() is not None
del(od)
assert w() is not None
assert x() is not None
assert y() is not None
assert z() is not None
del(cfg)
assert y() is None
assert z() is None
#assert w() is None
#assert x() is None
def test_deref_validator():
if not IS_DEREFABLE:
return
a = StrOption('a', '', default='yes')
b = StrOption('b', '', validator=funcname, validator_params={'': ((a, False),)}, default='val')
od = OptionDescription('root', '', [a, b])
cfg = Config(od)
w = weakref.ref(a)
x = weakref.ref(b)
y = weakref.ref(od)
z = weakref.ref(cfg)
assert w() is not None
assert x() is not None
assert w() is not None
assert x() is not None
del(a)
del(b)
assert w() is not None
assert x() is not None
assert w() is not None
assert x() is not None
del(od)
assert w() is not None
assert x() is not None
assert w() is not None
assert x() is not None
del(cfg)
assert y() is None
assert z() is None
#assert w() is None
#assert x() is None
def test_deref_callback():
if not IS_DEREFABLE:
return
a = StrOption('a', "", 'val')
b = StrOption('b', "", callback=funcname, callback_params={'': ((a, False),)})
od = OptionDescription('root', '', [a, b])
cfg = Config(od)
w = weakref.ref(a)
x = weakref.ref(b)
y = weakref.ref(od)
z = weakref.ref(cfg)
assert w() is not None
assert x() is not None
assert w() is not None
assert x() is not None
del(a)
del(b)
assert w() is not None
assert x() is not None
assert w() is not None
assert x() is not None
del(od)
assert w() is not None
assert x() is not None
assert w() is not None
assert x() is not None
del(cfg)
assert y() is None
assert z() is None
#assert w() is None
#assert x() is None
def test_deref_symlink():
if not IS_DEREFABLE:
return
a = BoolOption("a", "", default=False)
b = SymLinkOption("b", a)
od = OptionDescription('root', '', [a, b])
cfg = Config(od)
w = weakref.ref(a)
x = weakref.ref(b)
y = weakref.ref(od)
z = weakref.ref(cfg)
assert w() is not None
assert x() is not None
assert w() is not None
assert x() is not None
del(a)
del(b)
assert w() is not None
assert x() is not None
assert w() is not None
assert x() is not None
del(od)
assert w() is not None
assert x() is not None
assert w() is not None
assert x() is not None
del(cfg)
#assert w() is None
#assert x() is None
assert y() is None
assert z() is None
def test_deref_dyn():
if not IS_DEREFABLE:
return
a = StrOption('a', '', ['val1', 'val2'], multi=True)
b = StrOption('b', '')
dod = DynOptionDescription('dod', '', [b], callback=funcname, callback_params={'': ((a, False),)})
od = OptionDescription('od', '', [dod, a])
cfg = Config(od)
w = weakref.ref(a)
x = weakref.ref(b)
y = weakref.ref(od)
z = weakref.ref(cfg)
assert w() is not None
assert x() is not None
assert w() is not None
assert x() is not None
del(a)
del(b)
assert w() is not None
assert x() is not None
assert w() is not None
assert x() is not None
del(od)
del(dod)
assert w() is not None
assert x() is not None
assert w() is not None
assert x() is not None
del(cfg)
#assert w() is None
#assert x() is None
assert y() is None
assert z() is None

View File

@ -696,3 +696,21 @@ def test_multi_submulti_meta():
multi.append() multi.append()
assert conf1.multi == [['val', None]] assert conf1.multi == [['val', None]]
assert meta.multi == [['val']] assert meta.multi == [['val']]
def test_multi_submulti_meta_no_cache():
multi = StrOption('multi', '', multi=submulti)
od = OptionDescription('od', '', [multi])
conf1 = Config(od, session_id='conf1')
conf1.read_write()
conf2 = Config(od, session_id='conf2')
conf2.read_write()
meta = MetaConfig([conf1, conf2])
meta.read_write()
meta.cfgimpl_get_settings().remove('cache')
meta.multi = [['val']]
assert meta.multi == [['val']]
multi = conf1.multi[0]
multi.append()
assert conf1.multi == [['val', None]]
assert meta.multi == [['val']]

View File

@ -22,6 +22,7 @@
import weakref import weakref
import sys import sys
from time import time from time import time
from itertools import chain
from .error import PropertiesOptionError, ConfigError, ConflictError from .error import PropertiesOptionError, ConfigError, ConflictError
@ -81,22 +82,9 @@ class SubConfig(object):
:param only_expired: if True reset only expired cached values :param only_expired: if True reset only expired cached values
:type only_expired: boolean :type only_expired: boolean
""" """
if orig_opts is None: def reset_one_option_cache(opt, path):
orig_opts = set()
context = self._cfgimpl_get_context()
if 'values' in only:
values = context.cfgimpl_get_values()
if 'settings' in only or 'properties' in only or 'permissives' in only:
settings = context.cfgimpl_get_settings()
if only_expired:
if 'values' in only:
values._p_.reset_expired_cache(int(time()))
if 'settings' in only or 'properties' in only:
settings._p_.reset_expired_cache(int(time()))
if 'settings' in only or 'permissives' in only:
settings._pp_.reset_expired_cache(int(time()))
elif not None in (opt, path):
if opt.__class__.__name__ == 'DynOptionDescription': if opt.__class__.__name__ == 'DynOptionDescription':
# this option is a DynOptionDescription
descr = context.cfgimpl_get_description() descr = context.cfgimpl_get_description()
spath = path.split('.') spath = path.split('.')
subpath = '.'.join(spath[:-1]) subpath = '.'.join(spath[:-1])
@ -111,6 +99,7 @@ class SubConfig(object):
if 'settings' in only or 'permissives' in only: if 'settings' in only or 'permissives' in only:
settings._pp_.delcache(path) settings._pp_.delcache(path)
elif not isinstance(opt, DynSymLinkOption) and opt._is_subdyn(): elif not isinstance(opt, DynSymLinkOption) and opt._is_subdyn():
# this option is an instance of DynOptionDescription
descr = context.cfgimpl_get_description() descr = context.cfgimpl_get_description()
spath = path.split('.') spath = path.split('.')
try: try:
@ -150,7 +139,43 @@ class SubConfig(object):
settings._p_.delcache(path) settings._p_.delcache(path)
if 'settings' in only or 'permissives' in only: if 'settings' in only or 'permissives' in only:
settings._pp_.delcache(path) settings._pp_.delcache(path)
for option in opt._get_dependencies(self):
def reset_expired_cache():
# reset cache for expired cache value ony
datetime = int(time())
if 'values' in only:
values._p_.reset_expired_cache(datetime)
if 'settings' in only or 'properties' in only:
settings._p_.reset_expired_cache(datetime)
if 'settings' in only or 'permissives' in only:
settings._pp_.reset_expired_cache(datetime)
def reset_all_cache():
if 'values' in only:
values._p_.reset_all_cache()
if 'settings' in only:
settings._p_.reset_all_cache()
settings._pp_.reset_all_cache()
if orig_opts is None:
orig_opts = set()
context = self._cfgimpl_get_context()
if 'values' in only:
values = context.cfgimpl_get_values()
if 'settings' in only or 'properties' in only or 'permissives' in only:
settings = context.cfgimpl_get_settings()
if not None in (opt, path):
reset_one_option_cache(opt, path)
# remove cache for option which has dependencies with this option
if not isinstance(opt, OptionDescription) and not isinstance(opt, SynDynOptionDescription) and \
opt.impl_is_master_slaves('slave'):
slaves = opt.impl_get_master_slaves().getslaves(opt)
else:
slaves = []
for option in chain(opt._get_dependencies(self), slaves):
if option in orig_opts: if option in orig_opts:
continue continue
if 'values' in only: if 'values' in only:
@ -162,12 +187,11 @@ class SubConfig(object):
option.reset_cache(opt, settings, 'properties', orig_opts) option.reset_cache(opt, settings, 'properties', orig_opts)
if 'permissives' in only: if 'permissives' in only:
option.reset_cache(opt, settings, 'permissives', orig_opts) option.reset_cache(opt, settings, 'permissives', orig_opts)
elif only_expired:
reset_expired_cache()
else: else:
if 'values' in only: reset_all_cache()
values._p_.reset_all_cache()
if 'settings' in only:
settings._p_.reset_all_cache()
settings._pp_.reset_all_cache()
def cfgimpl_get_home_by_path(self, path, force_permissive=False, def cfgimpl_get_home_by_path(self, path, force_permissive=False,
returns_raise=False, _setting_properties=undefined): returns_raise=False, _setting_properties=undefined):

View File

@ -65,6 +65,7 @@ class Values(Cache):
lst[idx] = tuple(lst[idx]) lst[idx] = tuple(lst[idx])
values.append(tuple(lst)) values.append(tuple(lst))
return vidx return vidx
# value # value
def setvalue(self, path, value, owner, index, session, commit): def setvalue(self, path, value, owner, index, session, commit):
"""set value for a path """set value for a path

View File

@ -87,6 +87,7 @@ class Values(object):
callback=callback, callback=callback,
callback_params=callback_params, callback_params=callback_params,
index=index, validate=validate) index=index, validate=validate)
_orig_context.cfgimpl_reset_cache(opt=opt, path=path, only=('values',))
if isinstance(value, list) and index is not None: if isinstance(value, list) and index is not None:
#if return a list and index is set, return value only if #if return a list and index is set, return value only if
#it's a submulti without submulti_index and without list of list #it's a submulti without submulti_index and without list of list
@ -682,7 +683,7 @@ class Values(object):
if not 'cache' in context.cfgimpl_get_settings(): if not 'cache' in context.cfgimpl_get_settings():
raise ConfigError(_('can force cache only if cache ' raise ConfigError(_('can force cache only if cache '
'is actived in config')) 'is actived in config'))
#FIXME properties and value should update "expired" time context.cfgimpl_reset_cache()
for path in context.cfgimpl_get_description().impl_getpaths( for path in context.cfgimpl_get_description().impl_getpaths(
include_groups=True): include_groups=True):
err = context.getattr(path, returns_raise=True) err = context.getattr(path, returns_raise=True)
@ -731,7 +732,7 @@ class Multi(list):
context, context,
opt, path, opt, path,
idx)) idx))
self[idx].submulti = weakref.ref(self) self[idx].refmulti = weakref.ref(self)
def _getcontext(self): def _getcontext(self):
"""context could be None, we need to test it """context could be None, we need to test it
@ -808,7 +809,7 @@ class Multi(list):
if not '_index' in self.__slots__ and self.opt.impl_is_submulti(): if not '_index' in self.__slots__ and self.opt.impl_is_submulti():
if not isinstance(value, SubMulti): if not isinstance(value, SubMulti):
value = SubMulti(value, self.context, self.opt, self.path, index) value = SubMulti(value, self.context, self.opt, self.path, index)
value.submulti = weakref.ref(self) value.refmulti = weakref.ref(self)
super(Multi, self).append(value) super(Multi, self).append(value)
if setitem: if setitem:
self._store(force=force) self._store(force=force)
@ -911,13 +912,12 @@ class Multi(list):
values = self._getcontext().cfgimpl_get_values() values = self._getcontext().cfgimpl_get_values()
if not force: if not force:
#FIXME could get properties an pass it #FIXME could get properties an pass it
values.validate(self.opt, self, self.path, values.validate(self.opt, self, self.path, valid_masterslave=False)
valid_masterslave=False)
values._setvalue(self.opt, self.path, self, index=index) values._setvalue(self.opt, self.path, self, index=index)
class SubMulti(Multi): class SubMulti(Multi):
__slots__ = ('_index', 'submulti') __slots__ = ('_index', 'refmulti')
def __init__(self, value, context, opt, path, index): def __init__(self, value, context, opt, path, index):
""" """
@ -940,7 +940,11 @@ class SubMulti(Multi):
#force is unused here #force is unused here
values = self._getcontext().cfgimpl_get_values() values = self._getcontext().cfgimpl_get_values()
values.validate(self.opt, self, self.path, valid_masterslave=False) values.validate(self.opt, self, self.path, valid_masterslave=False)
values._setvalue(self.opt, self.path, self.submulti()) multi = self.refmulti()
if multi is None:
multi = values._get_cached_value(self.opt, path=self.path)
multi[self._index] = self
values._setvalue(self.opt, self.path, multi)
def _validate(self, value, fake_context, force_index, submulti=False): def _validate(self, value, fake_context, force_index, submulti=False):
if value is not None: if value is not None: