From 313b03b2462106fe54b603d2f46395823e8fd4a7 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Sat, 1 Feb 2014 16:26:23 +0100 Subject: [PATCH] cache in sql works --- test/test_dereference.py | 48 ++++++------ tiramisu/config.py | 38 +++------- tiramisu/option.py | 66 +++++------------ tiramisu/setting.py | 4 +- tiramisu/storage/sqlalchemy/option.py | 101 ++++++++++++++++++++++---- 5 files changed, 143 insertions(+), 114 deletions(-) diff --git a/test/test_dereference.py b/test/test_dereference.py index e24e1e8..4ea1240 100644 --- a/test/test_dereference.py +++ b/test/test_dereference.py @@ -65,17 +65,19 @@ def test_deref_optiondescription(): #assert w() is None -def test_deref_option_cache(): - b = BoolOption('b', '') - o = OptionDescription('od', '', [b]) - o.impl_build_cache_option() - w = weakref.ref(b) - del(b) - assert w() is not None - del(o) - #FIXME l'objet n'est plus en mémoire mais par contre reste dans la base - #Voir comment supprimer (et quand) - #assert w() is None +#def test_deref_option_cache(): +# FIXME quand c'est un dico, il faut garder la reference +# mais la comme c'est dans la base, forcement c'est dereference +# b = BoolOption('b', '') +# o = OptionDescription('od', '', [b]) +# o.impl_build_cache_option() +# w = weakref.ref(b) +# del(b) +# assert w() is not None +# del(o) +# #FIXME l'objet n'est plus en mémoire mais par contre reste dans la base +# #Voir comment supprimer (et quand) +# #assert w() is None def test_deref_optiondescription_cache(): @@ -90,18 +92,18 @@ def test_deref_optiondescription_cache(): #assert w() is None -def test_deref_option_config(): - b = BoolOption('b', '') - o = OptionDescription('od', '', [b]) - c = Config(o) - w = weakref.ref(b) - del(b) - assert w() is not None - del(o) - assert w() is not None - del(c) - #FIXME meme chose - #assert w() is None +#def test_deref_option_config(): +# b = BoolOption('b', '') +# o = OptionDescription('od', '', [b]) +# c = Config(o) +# w = weakref.ref(b) +# del(b) +# assert w() is not None +# del(o) +# assert w() is not None +# del(c) +# #FIXME meme chose +# #assert w() is None #FIXME rien a voir mais si je fais un config.impl_get_path_by_opt() ca me retourne la methode ! diff --git a/tiramisu/config.py b/tiramisu/config.py index 5153a2e..962fb0a 100644 --- a/tiramisu/config.py +++ b/tiramisu/config.py @@ -300,15 +300,6 @@ class SubConfig(object): :param first: return only one option if True, a list otherwise :return: find list or an exception if nothing has been found """ - def _filter_by_name(): - try: - if byname is None or path == byname or \ - path.endswith('.' + byname): - return True - except IndexError: - pass - return False - def _filter_by_value(): if byvalue is None: return True @@ -322,30 +313,21 @@ class SubConfig(object): # upon the access of the value return False - def _filter_by_type(): - if bytype is None: - return True - if isinstance(option, bytype): - return True - return False - if type_ not in ('option', 'path', 'value'): raise ValueError(_('unknown type_ type {0}' 'for _find').format(type_)) find_results = [] - paths = self.cfgimpl_get_description()._cache_paths[1] - for path in paths: - option = self.cfgimpl_get_description().impl_get_opt_by_path(path) - if isinstance(option, OptionDescription): - continue - if _subpath is not None and not path.startswith(_subpath + '.'): - continue - if not _filter_by_name(): - continue + # if value and/or check_properties are set, need all avalaible option + # If first one has no good value or not good property check second one + # and so on + #FIXME + #only_first = first == True and value is None and check_properties is None + only_first = first + options = self.cfgimpl_get_description().impl_get_options_paths( + bytype, byname, _subpath, only_first) + for path, option in options: if not _filter_by_value(): continue - if not _filter_by_type(): - continue #remove option with propertyerror, ... if byvalue is None and check_properties: try: @@ -422,7 +404,7 @@ class SubConfig(object): "option")) if withoption is not None: mypath = self.cfgimpl_get_path() - for path in self._cfgimpl_get_context()._find(bytype=Option, + for path in self._cfgimpl_get_context()._find(bytype=None, byname=withoption, byvalue=withvalue, first=False, diff --git a/tiramisu/option.py b/tiramisu/option.py index e793ee5..875149b 100644 --- a/tiramisu/option.py +++ b/tiramisu/option.py @@ -289,7 +289,11 @@ class BaseOption(Base): return self._name -class Option(BaseOption): +class OnlyOption(BaseOption): + pass + + +class Option(OnlyOption): """ Abstract base class for configuration option's. @@ -584,7 +588,6 @@ class Option(BaseOption): opts[idx_inf].impl_getname(), opts[idx_inf + idx_sup + 1].impl_getname())) def _impl_convert_callbacks(self, descr, load=False): - #FIXME if not load and self._callback is None: self._state_callback = None elif load and self._state_callback is None: @@ -770,7 +773,7 @@ else: raise ValueError(_('invalid unicode')) -class SymLinkOption(BaseOption): +class SymLinkOption(OnlyOption): #__slots__ = ('_name', '_opt', '_state_opt', '_readonly', '_parent') #not return _opt consistencies #_consistencies = None @@ -864,9 +867,9 @@ class PortOption(Option): properties=None, allow_range=False, allow_zero=False, allow_wellknown=True, allow_registred=True, allow_private=False, warnings_only=False): - self._allow_range = allow_range - self._min_value = None - self._max_value = None + extra = {'_allow_range': allow_range, + '_min_value': None, + '_max_value': None} ports_min = [0, 1, 1024, 49152] ports_max = [0, 1023, 49151, 65535] is_finally = False @@ -874,17 +877,17 @@ class PortOption(Option): allow_wellknown, allow_registred, allow_private]): - if self._min_value is None: + if extra['_min_value'] is None: if allowed: - self._min_value = ports_min[index] + extra['_min_value'] = ports_min[index] elif not allowed: is_finally = True elif allowed and is_finally: raise ValueError(_('inconsistency in allowed range')) if allowed: - self._max_value = ports_max[index] + extra['_max_value'] = ports_max[index] - if self._max_value is None: + if extra['_max_value'] is None: raise ValueError(_('max value is empty')) super(PortOption, self).__init__(name, doc, default=default, @@ -897,9 +900,10 @@ class PortOption(Option): validator_params=validator_params, properties=properties, warnings_only=warnings_only) + self._extra = extra def _validate(self, value): - if self._allow_range and ":" in str(value): + if self._extra['_allow_range'] and ":" in str(value): value = str(value).split(':') if len(value) != 2: raise ValueError('invalid part, range must have two values ' @@ -911,9 +915,9 @@ class PortOption(Option): value = [value] for val in value: - if not self._min_value <= int(val) <= self._max_value: + if not self._extra['_min_value'] <= int(val) <= self._extra['_max_value']: raise ValueError('invalid port, must be an between {0} and {1}' - ''.format(self._min_value, self._max_value)) + ''.format(self._extra['_min_value'], self._extra['_max_value'])) class NetworkOption(Option): @@ -1168,16 +1172,12 @@ class OptionDescription(BaseOption, StorageOptionDescription): raise ConflictError(_('duplicate option: ' '{0}').format(child)) self._children.append(child) # = (tuple(child_names), tuple(children)) - self._cache_paths = None + #FIXME pour dico ! + #self._cache_paths = None self._cache_consistencies = None # the group_type is useful for filtering OptionDescriptions in a config self._group_type = groups.default - #def impl_getproperties(self): - # #FIXME - # for prop in self._properties: - # yield(prop.name) - def impl_getrequires(self): #FIXME return self._requires @@ -1288,31 +1288,6 @@ class OptionDescription(BaseOption, StorageOptionDescription): if init: self._readonly = True - def impl_already_build_caches(self): - return self._cache_paths is not None - - def impl_build_cache_option(self, cache_path=None, cache_option=None, - _currpath=None): - if _currpath is None: - save = True - _currpath = [] - cache_path = [] - cache_option = [] - else: - save = False - for option in self.impl_getchildren(): - attr = option.impl_getname() - #FIXME specifique sqlachemy... - cache_option.append(option.id) - cache_path.append(str('.'.join(_currpath + [attr]))) - if isinstance(option, OptionDescription): - _currpath.append(attr) - option.impl_build_cache_option(cache_path, - cache_option, - _currpath) - _currpath.pop() - if save: - self._cache_paths = (tuple(cache_option), tuple(cache_path)) # ____________________________________________________________ def impl_set_group_type(self, group_type): @@ -1370,9 +1345,6 @@ class OptionDescription(BaseOption, StorageOptionDescription): raise ValueError(_('group_type: {0}' ' not allowed').format(group_type)) - def impl_get_group_type(self): - return getattr(groups, self._group_type) - def _valid_consistency(self, option, value, context, index): if self._cache_consistencies is None: return True diff --git a/tiramisu/setting.py b/tiramisu/setting.py index e483041..9e9926a 100644 --- a/tiramisu/setting.py +++ b/tiramisu/setting.py @@ -384,7 +384,6 @@ class Settings(object): is_cached, props = self._p_.getcache(path, ntime) if is_cached: return props - #FIXME props = self._p_.getproperties(path, opt._properties) if is_apply_req: props |= self.apply_requires(opt, path) @@ -606,8 +605,7 @@ class Settings(object): " '{0}' with requirement on: " "'{1}'").format(path, reqpath)) try: - value = context._getattr(reqpath, - force_permissive=True) + value = context._getattr(reqpath, force_permissive=True) except PropertiesOptionError as err: if not transitive: continue diff --git a/tiramisu/storage/sqlalchemy/option.py b/tiramisu/storage/sqlalchemy/option.py index 82339d1..0990248 100644 --- a/tiramisu/storage/sqlalchemy/option.py +++ b/tiramisu/storage/sqlalchemy/option.py @@ -18,7 +18,9 @@ # # ____________________________________________________________ from tiramisu.i18n import _ +from tiramisu.setting import groups +from sqlalchemy import not_, or_ from sqlalchemy.ext.declarative import declarative_base, declared_attr from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy import create_engine, Column, Integer, String, Boolean, \ @@ -40,9 +42,6 @@ SqlAlchemyBase = declarative_base() # require -#_Base : object dans la base de donnée -# => _PropertyOption => liste des propriétés -# => _CallbackParam avec des Options def load_requires(collection_type, proxy): def getter(obj): if obj is None: @@ -274,8 +273,10 @@ class _Base(SqlAlchemyBase): 'polymorphic_identity': 'option', 'polymorphic_on': _type } + _extra = Column(PickleType) #FIXME devrait etre une table _group_type = Column(String) + _is_build_cache = Column(Boolean, default=False) def __init__(self): self.commit() @@ -324,21 +325,95 @@ class _Base(SqlAlchemyBase): key)) +class Cache(SqlAlchemyBase): + __tablename__ = 'cache' + id = Column(Integer, primary_key=True) + #FIXME indexer ... les 3 + path = Column(String, nullable=False) + descr = Column(Integer, nullable=False) + option = Column(Integer, nullable=False) + opt_type = Column(String, nullable=False) + + def __init__(self, descr, option, path): + self.descr = descr.id + self.option = option.id + self.path = path + self.opt_type = option.__class__.__name__ + + class StorageOptionDescription(object): + def impl_already_build_caches(self): + return self._is_build_cache + def impl_get_opt_by_path(self, path): - try: - #FIXME - idx = self._cache_paths[1].index(path) - opt_id = self._cache_paths[0][idx] - return session.query(_Base).filter_by(id=opt_id).first() - except ValueError: + ret = session.query(Cache).filter_by(descr=self.id, path=path).first() + if ret is None: raise AttributeError(_('no option for path {0}').format(path)) + return session.query(_Base).filter_by(id=ret.option).first() def impl_get_path_by_opt(self, opt): - try: - return self._cache_paths[1][self._cache_paths[0].index(opt.id)] - except ValueError: + ret = session.query(Cache).filter_by(descr=self.id, + option=opt.id).first() + if ret is None: raise AttributeError(_('no option {0} found').format(opt)) + return ret.path + + def impl_get_group_type(self): + return getattr(groups, self._group_type) + + def impl_build_cache_option(self, descr=None, _currpath=None): + if descr is None: + save = True + descr = self + _currpath = [] + else: + save = False + for option in self.impl_getchildren(): + attr = option.impl_getname() + session.add(Cache(descr, option, + str('.'.join(_currpath + [attr])))) + if isinstance(option, StorageOptionDescription): + _currpath.append(attr) + option.impl_build_cache_option(descr, + _currpath) + _currpath.pop() + if save: + self._is_build_cache = True + session.commit() + + def impl_get_options_paths(self, bytype, byname, _subpath, only_first): + #FIXME tester si 1er est un descr ... + #FAIRE UN JOIN pour only_first + sqlquery = session.query(Cache).filter_by(descr=self.id) + if bytype is None: + sqlquery = sqlquery.filter(not_(Cache.opt_type == 'OptionDescription')) + else: + sqlquery = sqlquery.filter_by(opt_type=bytype.__name__) + + query = '' + or_query = '' + if _subpath is not None: + query += _subpath + '.' + if byname is not None: + or_query = query + byname + query += '%.' + byname + if query != '': + filter_query = Cache.path.like(query) + if or_query != '': + filter_query = or_(Cache.path == or_query, filter_query) + sqlquery = sqlquery.filter(filter_query) + if only_first: + opt = sqlquery.first() + if opt is None: + return tuple() + option = session.query(_Base).filter_by(id=opt.option).first() + return ((opt.path, option),) + else: + ret = [] + for opt in sqlquery.all(): + option = session.query(_Base).filter_by(id=opt.option).first() + ret.append((opt.path, option)) + return ret class StorageBase(_Base): @@ -347,7 +422,7 @@ class StorageBase(_Base): return {'polymorphic_identity': self.__name__.lower()} -#FIXME +#engine.echo = True SqlAlchemyBase.metadata.create_all(engine) Session = sessionmaker(bind=engine) session = Session()