From 77c1ccf40bd8d611a0af997d8599115991d9bf70 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Sat, 7 Sep 2013 17:25:22 +0200 Subject: [PATCH] add 'cache' property --- test/test_cache.py | 215 ++++++++++++++++++++++-------------- test/test_option_setting.py | 9 +- test/test_storage.py | 13 ++- tiramisu/setting.py | 23 ++-- tiramisu/storage/cache.py | 17 ++- tiramisu/value.py | 31 +++--- 6 files changed, 187 insertions(+), 121 deletions(-) diff --git a/test/test_cache.py b/test/test_cache.py index 47270ee..b65e117 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -4,7 +4,7 @@ from tiramisu import setting setting.expires_time = 1 from tiramisu.option import IntOption, OptionDescription from tiramisu.config import Config -from time import sleep +from time import sleep, time def make_description(): @@ -20,13 +20,26 @@ def test_cache(): values = c.cfgimpl_get_values() settings = c.cfgimpl_get_settings() c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.u2 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) - assert 'u2' in values._p_.get_cached('value', c) - assert 'u2' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) + assert 'u2' in values._p_.get_cached(c) + assert 'u2' in settings._p_.get_cached(c) + + +def test_get_cache(): + # force a value in cache, try if reget corrupted value + od1 = make_description() + c = Config(od1) + values = c.cfgimpl_get_values() + settings = c.cfgimpl_get_settings() + ntime = time() + 1 + settings._p_.setcache('u1', set(['inject']), ntime) + assert 'inject' in settings[od1.u1] + values._p_.setcache('u1', 100, ntime) + assert c.u1 == [100] def test_cache_reset(): @@ -36,44 +49,44 @@ def test_cache_reset(): settings = c.cfgimpl_get_settings() #when change a value c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.u2 = 1 - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) #when remove a value c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) del(c.u2) - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) #when add/del property c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.cfgimpl_get_settings()[od1.u2].append('test') - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.cfgimpl_get_settings()[od1.u2].remove('test') - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) #when enable/disabled property c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.cfgimpl_get_settings().append('test') - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.cfgimpl_get_settings().remove('test') - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) def test_cache_reset_multi(): @@ -83,32 +96,32 @@ def test_cache_reset_multi(): settings = c.cfgimpl_get_settings() #when change a value c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.u3 = [1] - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) #when append value c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.u3.append(1) - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) #when pop value c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.u3.pop(1) - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) #when remove a value c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) del(c.u3) - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) def test_reset_cache(): @@ -117,23 +130,25 @@ def test_reset_cache(): values = c.cfgimpl_get_values() settings = c.cfgimpl_get_settings() c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.cfgimpl_reset_cache() - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) + c.u1 + sleep(1) c.u1 sleep(1) c.u2 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) - assert 'u2' in values._p_.get_cached('value', c) - assert 'u2' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) + assert 'u2' in values._p_.get_cached(c) + assert 'u2' in settings._p_.get_cached(c) c.cfgimpl_reset_cache() - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) - assert 'u2' not in values._p_.get_cached('value', c) - assert 'u2' not in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) + assert 'u2' not in values._p_.get_cached(c) + assert 'u2' not in settings._p_.get_cached(c) def test_reset_cache_subconfig(): @@ -142,9 +157,9 @@ def test_reset_cache_subconfig(): c = Config(od2) values = c.cfgimpl_get_values() c.od1.u1 - assert 'od1.u1' in values._p_.get_cached('value', c) + assert 'od1.u1' in values._p_.get_cached(c) c.od1.cfgimpl_reset_cache() - assert 'od1.u1' not in values._p_.get_cached('value', c) + assert 'od1.u1' not in values._p_.get_cached(c) def test_reset_cache_only_expired(): @@ -153,22 +168,60 @@ def test_reset_cache_only_expired(): values = c.cfgimpl_get_values() settings = c.cfgimpl_get_settings() c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.cfgimpl_reset_cache(True) - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) + sleep(1) + c.u1 sleep(1) c.u2 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) - assert 'u2' in values._p_.get_cached('value', c) - assert 'u2' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) + assert 'u2' in values._p_.get_cached(c) + assert 'u2' in settings._p_.get_cached(c) c.cfgimpl_reset_cache(True) - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) - assert 'u2' in values._p_.get_cached('value', c) - assert 'u2' in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) + assert 'u2' in values._p_.get_cached(c) + assert 'u2' in settings._p_.get_cached(c) + + +def test_cache_not_expire(): + od1 = make_description() + c = Config(od1) + values = c.cfgimpl_get_values() + settings = c.cfgimpl_get_settings() + settings.remove('expire') + c.u1 + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) + c.cfgimpl_reset_cache(True) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) + sleep(1) + c.u2 + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) + assert 'u2' in values._p_.get_cached(c) + assert 'u2' in settings._p_.get_cached(c) + c.cfgimpl_reset_cache(True) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) + assert 'u2' in values._p_.get_cached(c) + assert 'u2' in settings._p_.get_cached(c) + + +def test_cache_not_cache(): + od1 = make_description() + c = Config(od1) + values = c.cfgimpl_get_values() + settings = c.cfgimpl_get_settings() + settings.remove('cache') + c.u1 + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) def test_reset_cache_only(): @@ -177,14 +230,14 @@ def test_reset_cache_only(): values = c.cfgimpl_get_values() settings = c.cfgimpl_get_settings() c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.cfgimpl_reset_cache(only=('values',)) - assert 'u1' not in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' not in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.u1 - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' in settings._p_.get_cached(c) c.cfgimpl_reset_cache(only=('settings',)) - assert 'u1' in values._p_.get_cached('value', c) - assert 'u1' not in settings._p_.get_cached('property', c) + assert 'u1' in values._p_.get_cached(c) + assert 'u1' not in settings._p_.get_cached(c) diff --git a/test/test_option_setting.py b/test/test_option_setting.py index a841b9b..2dc76ab 100644 --- a/test/test_option_setting.py +++ b/test/test_option_setting.py @@ -329,7 +329,7 @@ def test_reset_properties(): option = cfg.cfgimpl_get_description().gc.dummy assert setting._p_.get_properties(cfg) == {} setting.append('frozen') - assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator'))} + assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'cache', 'validator'))} setting.reset() assert setting._p_.get_properties(cfg) == {} setting[option].append('test') @@ -337,11 +337,11 @@ def test_reset_properties(): setting.reset() assert setting._p_.get_properties(cfg) == {'gc.dummy': set(('test',))} setting.append('frozen') - assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator')), 'gc.dummy': set(('test',))} + assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator', 'cache')), 'gc.dummy': set(('test',))} setting.reset(option) - assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator'))} + assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator', 'cache'))} setting[option].append('test') - assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator')), 'gc.dummy': set(('test',))} + assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator', 'cache')), 'gc.dummy': set(('test',))} setting.reset(all_properties=True) assert setting._p_.get_properties(cfg) == {} raises(ValueError, 'setting.reset(all_properties=True, opt=option)') @@ -350,7 +350,6 @@ def test_reset_properties(): setting[a].reset() - def test_reset_multiple(): descr = make_description() cfg = Config(descr) diff --git a/test/test_storage.py b/test/test_storage.py index c3acc70..0b2e3ac 100644 --- a/test/test_storage.py +++ b/test/test_storage.py @@ -6,9 +6,7 @@ from tiramisu.config import Config from tiramisu.option import BoolOption, OptionDescription from tiramisu.setting import owners from tiramisu.storage import list_sessions, delete_session -from tiramisu import setting -setting.expires_time = 0 def test_non_persistent(): b = BoolOption('b', '') @@ -20,6 +18,7 @@ def test_list(): b = BoolOption('b', '') o = OptionDescription('od', '', [b]) c = Config(o, session_id='test_non_persistent') + c.cfgimpl_get_settings().remove('cache') assert 'test_non_persistent' in list_sessions() del(c) assert 'test_non_persistent' not in list_sessions() @@ -66,6 +65,7 @@ def test_create_persistent_retrieve(): o = OptionDescription('od', '', [b]) try: c = Config(o, session_id='test_persistent', persistent=True) + c.cfgimpl_get_settings().remove('cache') except ValueError: # storage is not persistent pass @@ -75,10 +75,12 @@ def test_create_persistent_retrieve(): assert c.b is True del(c) c = Config(o, session_id='test_persistent', persistent=True) + c.cfgimpl_get_settings().remove('cache') assert c.b is True assert 'test_persistent' in list_sessions() delete_session('test_persistent') c = Config(o, session_id='test_persistent', persistent=True) + c.cfgimpl_get_settings().remove('cache') assert c.b is None delete_session('test_persistent') @@ -88,11 +90,13 @@ def test_two_persistent(): o = OptionDescription('od', '', [b]) try: c = Config(o, session_id='test_persistent', persistent=True) + c.cfgimpl_get_settings().remove('cache') except ValueError: # storage is not persistent pass else: c2 = Config(o, session_id='test_persistent', persistent=True) + c2.cfgimpl_get_settings().remove('cache') assert c.b is None assert c2.b is None c.b = False @@ -109,11 +113,13 @@ def test_two_persistent_owner(): o = OptionDescription('od', '', [b]) try: c = Config(o, session_id='test_persistent', persistent=True) + c.cfgimpl_get_settings().remove('cache') except ValueError: # storage is not persistent pass else: c2 = Config(o, session_id='test_persistent', persistent=True) + c2.cfgimpl_get_settings().remove('cache') owners.addowner('persistent') assert c.getowner(b) == owners.default assert c2.getowner(b) == owners.default @@ -131,6 +137,7 @@ def test_two_persistent_information(): o = OptionDescription('od', '', [b]) try: c = Config(o, session_id='test_persistent', persistent=True) + c.cfgimpl_get_settings().remove('cache') except ValueError: # storage is not persistent pass @@ -138,5 +145,7 @@ def test_two_persistent_information(): c.impl_set_information('info', 'string') assert c.impl_get_information('info') == 'string' c2 = Config(o, session_id='test_persistent', persistent=True) + c2.cfgimpl_get_settings().remove('cache') + c2.cfgimpl_get_settings().remove('cache') assert c2.impl_get_information('info') == 'string' delete_session('test_persistent') diff --git a/tiramisu/setting.py b/tiramisu/setting.py index 3c88c1c..7946212 100644 --- a/tiramisu/setting.py +++ b/tiramisu/setting.py @@ -35,7 +35,7 @@ ro_append = set(['frozen', 'disabled', 'validator', 'everything_frozen', 'mandatory']) rw_remove = set(['permissive', 'everything_frozen', 'mandatory']) rw_append = set(['frozen', 'disabled', 'validator', 'hidden']) -default_properties = ('expire', 'validator') +default_properties = ('cache', 'expire', 'validator') class _NameSpace: @@ -242,18 +242,21 @@ class Settings(object): raise ValueError(_('if opt is not None, path should not be' ' None in _getproperties')) ntime = None - if self._p_.hascache('property', path): - ntime = time() - is_cached, props = self._p_.getcache('property', path, ntime) + if 'cache' in self and self._p_.hascache(path): + if 'expire' in self: + ntime = int(time()) + is_cached, props = self._p_.getcache(path, ntime) if is_cached: return props props = self._p_.getproperties(path, opt._properties) if is_apply_req: props |= self.apply_requires(opt, path) - if 'expire' in self: - if ntime is None: - ntime = time() - self._p_.setcache('property', path, props, ntime + expires_time) + if 'cache' in self: + if 'expire' in self: + if ntime is None: + ntime = int(time()) + ntime = ntime + expires_time + self._p_.setcache(path, props, ntime) return props def append(self, propname): @@ -387,9 +390,9 @@ class Settings(object): def reset_cache(self, only_expired): if only_expired: - self._p_.reset_expired_cache('property', time()) + self._p_.reset_expired_cache(int(time())) else: - self._p_.reset_all_cache('property') + self._p_.reset_all_cache() def apply_requires(self, opt, path): """carries out the jit (just in time) requirements between options diff --git a/tiramisu/storage/cache.py b/tiramisu/storage/cache.py index 664990d..98db066 100644 --- a/tiramisu/storage/cache.py +++ b/tiramisu/storage/cache.py @@ -27,34 +27,33 @@ class Cache(object): self._cache = {} self.storage = storage - def setcache(self, cache_type, path, val, time): + def setcache(self, path, val, time): self._cache[path] = (val, time) - def getcache(self, cache_type, path, exp): + def getcache(self, path, exp): value, created = self._cache[path] - if exp < created: + if exp <= created: return True, value return False, None - def hascache(self, cache_type, path): + def hascache(self, path): """ path is in the cache - :param cache_type: value | property :param path: the path's option """ return path in self._cache - def reset_expired_cache(self, cache_type, exp): + def reset_expired_cache(self, exp): for key in tuple(self._cache.keys()): val, created = self._cache[key] - if exp > created: + if created is not None and exp > created: del(self._cache[key]) - def reset_all_cache(self, cache_type): + def reset_all_cache(self): "empty the cache" self._cache.clear() - def get_cached(self, cache_type, context): + def get_cached(self, context): """return all values in a dictionary example: {'path1': ('value1', 'time1'), 'path2': ('value2', 'time2')} """ diff --git a/tiramisu/value.py b/tiramisu/value.py index f35c16d..ffd34d6 100644 --- a/tiramisu/value.py +++ b/tiramisu/value.py @@ -147,25 +147,28 @@ class Values(object): def getitem(self, opt, path=None, validate=True, force_permissive=False, force_properties=None, validate_properties=True): - ntime = None if path is None: path = self._get_opt_path(opt) - if self._p_.hascache('value', path): - ntime = time() - is_cached, value = self._p_.getcache('value', path, ntime) + ntime = None + setting = self.context().cfgimpl_get_settings() + if 'cache' in setting and self._p_.hascache(path): + if 'expire' in setting: + ntime = int(time()) + is_cached, value = self._p_.getcache(path, ntime) if is_cached: if opt.impl_is_multi() and not isinstance(value, Multi): #load value so don't need to validate if is not a Multi value = Multi(value, self.context, opt, path, validate=False) return value - val = self._getitem(opt, path, validate, force_permissive, force_properties, - validate_properties) - if 'expire' in self.context().cfgimpl_get_settings() and validate and \ - validate_properties and force_permissive is False and \ - force_properties is None: - if ntime is None: - ntime = time() - self._p_.setcache('value', path, val, ntime + expires_time) + val = self._getitem(opt, path, validate, force_permissive, + force_properties, validate_properties) + if 'cache' in setting and validate and validate_properties and \ + force_permissive is False and force_properties is None: + if 'expire' in setting: + if ntime is None: + ntime = int(time()) + ntime = ntime + expires_time + self._p_.setcache(path, val, ntime) return val @@ -300,9 +303,9 @@ class Values(object): clears the cache if necessary """ if only_expired: - self._p_.reset_expired_cache('value', time()) + self._p_.reset_expired_cache(int(time())) else: - self._p_.reset_all_cache('value') + self._p_.reset_all_cache() def _get_opt_path(self, opt): """