Infra/tiramisu
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add 'cache' property

Browse Source
This commit is contained in:
Emmanuel Garette 2013-09-07 17:25:22 +02:00
parent f8b0a53c3f
commit 77c1ccf40b
6 changed files with 187 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

215
test/test_cache.py
View File

@ -4,7 +4,7 @@ from tiramisu import setting
setting.expires_time = 1
from tiramisu.option import IntOption, OptionDescription
from tiramisu.config import Config
from time import sleep
from time import sleep, time
def make_description():
@ -20,13 +20,26 @@ def test_cache():
values = c.cfgimpl_get_values()
settings = c.cfgimpl_get_settings()
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.u2
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u2' in values._p_.get_cached('value', c)
assert 'u2' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
assert 'u2' in values._p_.get_cached(c)
assert 'u2' in settings._p_.get_cached(c)
def test_get_cache():
# force a value in cache, try if reget corrupted value
od1 = make_description()
c = Config(od1)
values = c.cfgimpl_get_values()
settings = c.cfgimpl_get_settings()
ntime = time() + 1
settings._p_.setcache('u1', set(['inject']), ntime)
assert 'inject' in settings[od1.u1]
values._p_.setcache('u1', 100, ntime)
assert c.u1 == [100]
def test_cache_reset():
@ -36,44 +49,44 @@ def test_cache_reset():
settings = c.cfgimpl_get_settings()
#when change a value
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.u2 = 1
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
#when remove a value
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
del(c.u2)
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
#when add/del property
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.cfgimpl_get_settings()[od1.u2].append('test')
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.cfgimpl_get_settings()[od1.u2].remove('test')
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
#when enable/disabled property
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.cfgimpl_get_settings().append('test')
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.cfgimpl_get_settings().remove('test')
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
def test_cache_reset_multi():
@ -83,32 +96,32 @@ def test_cache_reset_multi():
settings = c.cfgimpl_get_settings()
#when change a value
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.u3 = [1]
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
#when append value
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.u3.append(1)
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
#when pop value
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.u3.pop(1)
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
#when remove a value
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
del(c.u3)
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
def test_reset_cache():
@ -117,23 +130,25 @@ def test_reset_cache():
values = c.cfgimpl_get_values()
settings = c.cfgimpl_get_settings()
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.cfgimpl_reset_cache()
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
c.u1
sleep(1)
c.u1
sleep(1)
c.u2
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u2' in values._p_.get_cached('value', c)
assert 'u2' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
assert 'u2' in values._p_.get_cached(c)
assert 'u2' in settings._p_.get_cached(c)
c.cfgimpl_reset_cache()
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u2' not in values._p_.get_cached('value', c)
assert 'u2' not in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
assert 'u2' not in values._p_.get_cached(c)
assert 'u2' not in settings._p_.get_cached(c)
def test_reset_cache_subconfig():
@ -142,9 +157,9 @@ def test_reset_cache_subconfig():
c = Config(od2)
values = c.cfgimpl_get_values()
c.od1.u1
assert 'od1.u1' in values._p_.get_cached('value', c)
assert 'od1.u1' in values._p_.get_cached(c)
c.od1.cfgimpl_reset_cache()
assert 'od1.u1' not in values._p_.get_cached('value', c)
assert 'od1.u1' not in values._p_.get_cached(c)
def test_reset_cache_only_expired():
@ -153,22 +168,60 @@ def test_reset_cache_only_expired():
values = c.cfgimpl_get_values()
settings = c.cfgimpl_get_settings()
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.cfgimpl_reset_cache(True)
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
sleep(1)
c.u1
sleep(1)
c.u2
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u2' in values._p_.get_cached('value', c)
assert 'u2' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
assert 'u2' in values._p_.get_cached(c)
assert 'u2' in settings._p_.get_cached(c)
c.cfgimpl_reset_cache(True)
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u2' in values._p_.get_cached('value', c)
assert 'u2' in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
assert 'u2' in values._p_.get_cached(c)
assert 'u2' in settings._p_.get_cached(c)
def test_cache_not_expire():
od1 = make_description()
c = Config(od1)
values = c.cfgimpl_get_values()
settings = c.cfgimpl_get_settings()
settings.remove('expire')
c.u1
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.cfgimpl_reset_cache(True)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
sleep(1)
c.u2
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
assert 'u2' in values._p_.get_cached(c)
assert 'u2' in settings._p_.get_cached(c)
c.cfgimpl_reset_cache(True)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
assert 'u2' in values._p_.get_cached(c)
assert 'u2' in settings._p_.get_cached(c)
def test_cache_not_cache():
od1 = make_description()
c = Config(od1)
values = c.cfgimpl_get_values()
settings = c.cfgimpl_get_settings()
settings.remove('cache')
c.u1
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)
def test_reset_cache_only():
@ -177,14 +230,14 @@ def test_reset_cache_only():
values = c.cfgimpl_get_values()
settings = c.cfgimpl_get_settings()
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.cfgimpl_reset_cache(only=('values',))
assert 'u1' not in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' not in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.u1
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' in settings._p_.get_cached(c)
c.cfgimpl_reset_cache(only=('settings',))
assert 'u1' in values._p_.get_cached('value', c)
assert 'u1' not in settings._p_.get_cached('property', c)
assert 'u1' in values._p_.get_cached(c)
assert 'u1' not in settings._p_.get_cached(c)

9
test/test_option_setting.py
View File

@ -329,7 +329,7 @@ def test_reset_properties():
option = cfg.cfgimpl_get_description().gc.dummy
assert setting._p_.get_properties(cfg) == {}
setting.append('frozen')
assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator'))}
assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'cache', 'validator'))}
setting.reset()
assert setting._p_.get_properties(cfg) == {}
setting[option].append('test')
@ -337,11 +337,11 @@ def test_reset_properties():
setting.reset()
assert setting._p_.get_properties(cfg) == {'gc.dummy': set(('test',))}
setting.append('frozen')
assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator')), 'gc.dummy': set(('test',))}
assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator', 'cache')), 'gc.dummy': set(('test',))}
setting.reset(option)
assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator'))}
assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator', 'cache'))}
setting[option].append('test')
assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator')), 'gc.dummy': set(('test',))}
assert setting._p_.get_properties(cfg) == {None: set(('frozen', 'expire', 'validator', 'cache')), 'gc.dummy': set(('test',))}
setting.reset(all_properties=True)
assert setting._p_.get_properties(cfg) == {}
raises(ValueError, 'setting.reset(all_properties=True, opt=option)')
@ -350,7 +350,6 @@ def test_reset_properties():
setting[a].reset()
def test_reset_multiple():
descr = make_description()
cfg = Config(descr)

13
test/test_storage.py
View File

@ -6,9 +6,7 @@ from tiramisu.config import Config
from tiramisu.option import BoolOption, OptionDescription
from tiramisu.setting import owners
from tiramisu.storage import list_sessions, delete_session
from tiramisu import setting
setting.expires_time = 0
def test_non_persistent():
b = BoolOption('b', '')
@ -20,6 +18,7 @@ def test_list():
b = BoolOption('b', '')
o = OptionDescription('od', '', [b])
c = Config(o, session_id='test_non_persistent')
c.cfgimpl_get_settings().remove('cache')
assert 'test_non_persistent' in list_sessions()
del(c)
assert 'test_non_persistent' not in list_sessions()
@ -66,6 +65,7 @@ def test_create_persistent_retrieve():
o = OptionDescription('od', '', [b])
try:
c = Config(o, session_id='test_persistent', persistent=True)
c.cfgimpl_get_settings().remove('cache')
except ValueError:
# storage is not persistent
pass
@ -75,10 +75,12 @@ def test_create_persistent_retrieve():
assert c.b is True
del(c)
c = Config(o, session_id='test_persistent', persistent=True)
c.cfgimpl_get_settings().remove('cache')
assert c.b is True
assert 'test_persistent' in list_sessions()
delete_session('test_persistent')
c = Config(o, session_id='test_persistent', persistent=True)
c.cfgimpl_get_settings().remove('cache')
assert c.b is None
delete_session('test_persistent')
@ -88,11 +90,13 @@ def test_two_persistent():
o = OptionDescription('od', '', [b])
try:
c = Config(o, session_id='test_persistent', persistent=True)
c.cfgimpl_get_settings().remove('cache')
except ValueError:
# storage is not persistent
pass
else:
c2 = Config(o, session_id='test_persistent', persistent=True)
c2.cfgimpl_get_settings().remove('cache')
assert c.b is None
assert c2.b is None
c.b = False
@ -109,11 +113,13 @@ def test_two_persistent_owner():
o = OptionDescription('od', '', [b])
try:
c = Config(o, session_id='test_persistent', persistent=True)
c.cfgimpl_get_settings().remove('cache')
except ValueError:
# storage is not persistent
pass
else:
c2 = Config(o, session_id='test_persistent', persistent=True)
c2.cfgimpl_get_settings().remove('cache')
owners.addowner('persistent')
assert c.getowner(b) == owners.default
assert c2.getowner(b) == owners.default
@ -131,6 +137,7 @@ def test_two_persistent_information():
o = OptionDescription('od', '', [b])
try:
c = Config(o, session_id='test_persistent', persistent=True)
c.cfgimpl_get_settings().remove('cache')
except ValueError:
# storage is not persistent
pass
@ -138,5 +145,7 @@ def test_two_persistent_information():
c.impl_set_information('info', 'string')
assert c.impl_get_information('info') == 'string'
c2 = Config(o, session_id='test_persistent', persistent=True)
c2.cfgimpl_get_settings().remove('cache')
c2.cfgimpl_get_settings().remove('cache')
assert c2.impl_get_information('info') == 'string'
delete_session('test_persistent')

23
tiramisu/setting.py
View File

@ -35,7 +35,7 @@ ro_append = set(['frozen', 'disabled', 'validator', 'everything_frozen',
'mandatory'])
rw_remove = set(['permissive', 'everything_frozen', 'mandatory'])
rw_append = set(['frozen', 'disabled', 'validator', 'hidden'])
default_properties = ('expire', 'validator')
default_properties = ('cache', 'expire', 'validator')
class _NameSpace:
@ -242,18 +242,21 @@ class Settings(object):
raise ValueError(_('if opt is not None, path should not be'
' None in _getproperties'))
ntime = None
if self._p_.hascache('property', path):
ntime = time()
is_cached, props = self._p_.getcache('property', path, ntime)
if 'cache' in self and self._p_.hascache(path):
if 'expire' in self:
ntime = int(time())
is_cached, props = self._p_.getcache(path, ntime)
if is_cached:
return props
props = self._p_.getproperties(path, opt._properties)
if is_apply_req:
props |= self.apply_requires(opt, path)
if 'expire' in self:
if ntime is None:
ntime = time()
self._p_.setcache('property', path, props, ntime + expires_time)
if 'cache' in self:
if 'expire' in self:
if ntime is None:
ntime = int(time())
ntime = ntime + expires_time
self._p_.setcache(path, props, ntime)
return props
def append(self, propname):
@ -387,9 +390,9 @@ class Settings(object):
def reset_cache(self, only_expired):
if only_expired:
self._p_.reset_expired_cache('property', time())
self._p_.reset_expired_cache(int(time()))
else:
self._p_.reset_all_cache('property')
self._p_.reset_all_cache()
def apply_requires(self, opt, path):
"""carries out the jit (just in time) requirements between options

17
tiramisu/storage/cache.py
View File

@ -27,34 +27,33 @@ class Cache(object):
self._cache = {}
self.storage = storage
def setcache(self, cache_type, path, val, time):
def setcache(self, path, val, time):
self._cache[path] = (val, time)
def getcache(self, cache_type, path, exp):
def getcache(self, path, exp):
value, created = self._cache[path]
if exp < created:
if exp <= created:
return True, value
return False, None
def hascache(self, cache_type, path):
def hascache(self, path):
""" path is in the cache
:param cache_type: value | property
:param path: the path's option
"""
return path in self._cache
def reset_expired_cache(self, cache_type, exp):
def reset_expired_cache(self, exp):
for key in tuple(self._cache.keys()):
val, created = self._cache[key]
if exp > created:
if created is not None and exp > created:
del(self._cache[key])
def reset_all_cache(self, cache_type):
def reset_all_cache(self):
"empty the cache"
self._cache.clear()
def get_cached(self, cache_type, context):
def get_cached(self, context):
"""return all values in a dictionary
example: {'path1': ('value1', 'time1'), 'path2': ('value2', 'time2')}
"""

31
tiramisu/value.py
View File

@ -147,25 +147,28 @@ class Values(object):
def getitem(self, opt, path=None, validate=True, force_permissive=False,
force_properties=None, validate_properties=True):
ntime = None
if path is None:
path = self._get_opt_path(opt)
if self._p_.hascache('value', path):
ntime = time()
is_cached, value = self._p_.getcache('value', path, ntime)
ntime = None
setting = self.context().cfgimpl_get_settings()
if 'cache' in setting and self._p_.hascache(path):
if 'expire' in setting:
ntime = int(time())
is_cached, value = self._p_.getcache(path, ntime)
if is_cached:
if opt.impl_is_multi() and not isinstance(value, Multi):
#load value so don't need to validate if is not a Multi
value = Multi(value, self.context, opt, path, validate=False)
return value
val = self._getitem(opt, path, validate, force_permissive, force_properties,
validate_properties)
if 'expire' in self.context().cfgimpl_get_settings() and validate and \
validate_properties and force_permissive is False and \
force_properties is None:
if ntime is None:
ntime = time()
self._p_.setcache('value', path, val, ntime + expires_time)
val = self._getitem(opt, path, validate, force_permissive,
force_properties, validate_properties)
if 'cache' in setting and validate and validate_properties and \
force_permissive is False and force_properties is None:
if 'expire' in setting:
if ntime is None:
ntime = int(time())
ntime = ntime + expires_time
self._p_.setcache(path, val, ntime)
return val
@ -300,9 +303,9 @@ class Values(object):
clears the cache if necessary
"""
if only_expired:
self._p_.reset_expired_cache('value', time())
self._p_.reset_expired_cache(int(time()))
else:
self._p_.reset_all_cache('value')
self._p_.reset_all_cache()
def _get_opt_path(self, opt):
"""