Infra
/
tiramisu
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cache in sql works

This commit is contained in:
Emmanuel Garette 2014-02-01 16:26:23 +01:00
parent 02a987b39d
commit 313b03b246
5 changed files with 143 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
test/test_dereference.py
View File

@ -65,17 +65,19 @@ def test_deref_optiondescription():
#assert w() is None #assert w() is None
def test_deref_option_cache(): #def test_deref_option_cache():
b = BoolOption('b', '') # FIXME quand c'est un dico, il faut garder la reference
o = OptionDescription('od', '', [b]) # mais la comme c'est dans la base, forcement c'est dereference
o.impl_build_cache_option() # b = BoolOption('b', '')
w = weakref.ref(b) # o = OptionDescription('od', '', [b])
del(b) # o.impl_build_cache_option()
assert w() is not None # w = weakref.ref(b)
del(o) # del(b)
#FIXME l'objet n'est plus en mémoire mais par contre reste dans la base # assert w() is not None
#Voir comment supprimer (et quand) # del(o)
#assert w() is None # #FIXME l'objet n'est plus en mémoire mais par contre reste dans la base
# #Voir comment supprimer (et quand)
# #assert w() is None
def test_deref_optiondescription_cache(): def test_deref_optiondescription_cache():
@ -90,18 +92,18 @@ def test_deref_optiondescription_cache():
#assert w() is None #assert w() is None
def test_deref_option_config(): #def test_deref_option_config():
b = BoolOption('b', '') # b = BoolOption('b', '')
o = OptionDescription('od', '', [b]) # o = OptionDescription('od', '', [b])
c = Config(o) # c = Config(o)
w = weakref.ref(b) # w = weakref.ref(b)
del(b) # del(b)
assert w() is not None # assert w() is not None
del(o) # del(o)
assert w() is not None # assert w() is not None
del(c) # del(c)
#FIXME meme chose # #FIXME meme chose
#assert w() is None # #assert w() is None
#FIXME rien a voir mais si je fais un config.impl_get_path_by_opt() ca me retourne la methode ! #FIXME rien a voir mais si je fais un config.impl_get_path_by_opt() ca me retourne la methode !

38
tiramisu/config.py
View File

@ -300,15 +300,6 @@ class SubConfig(object):
:param first: return only one option if True, a list otherwise :param first: return only one option if True, a list otherwise
:return: find list or an exception if nothing has been found :return: find list or an exception if nothing has been found
""" """
def _filter_by_name():
try:
if byname is None or path == byname or \
path.endswith('.' + byname):
return True
except IndexError:
pass
return False
def _filter_by_value(): def _filter_by_value():
if byvalue is None: if byvalue is None:
return True return True
@ -322,30 +313,21 @@ class SubConfig(object):
# upon the access of the value # upon the access of the value
return False return False
def _filter_by_type():
if bytype is None:
return True
if isinstance(option, bytype):
return True
return False
if type_ not in ('option', 'path', 'value'): if type_ not in ('option', 'path', 'value'):
raise ValueError(_('unknown type_ type {0}' raise ValueError(_('unknown type_ type {0}'
'for _find').format(type_)) 'for _find').format(type_))
find_results = [] find_results = []
paths = self.cfgimpl_get_description()._cache_paths[1] # if value and/or check_properties are set, need all avalaible option
for path in paths: # If first one has no good value or not good property check second one
option = self.cfgimpl_get_description().impl_get_opt_by_path(path) # and so on
if isinstance(option, OptionDescription): #FIXME
continue #only_first = first == True and value is None and check_properties is None
if _subpath is not None and not path.startswith(_subpath + '.'): only_first = first
continue options = self.cfgimpl_get_description().impl_get_options_paths(
if not _filter_by_name(): bytype, byname, _subpath, only_first)
continue for path, option in options:
if not _filter_by_value(): if not _filter_by_value():
continue continue
if not _filter_by_type():
continue
#remove option with propertyerror, ... #remove option with propertyerror, ...
if byvalue is None and check_properties: if byvalue is None and check_properties:
try: try:
@ -422,7 +404,7 @@ class SubConfig(object):
"option")) "option"))
if withoption is not None: if withoption is not None:
mypath = self.cfgimpl_get_path() mypath = self.cfgimpl_get_path()
for path in self._cfgimpl_get_context()._find(bytype=Option, for path in self._cfgimpl_get_context()._find(bytype=None,
byname=withoption, byname=withoption,
byvalue=withvalue, byvalue=withvalue,
first=False, first=False,

66
tiramisu/option.py
View File

@ -289,7 +289,11 @@ class BaseOption(Base):
return self._name return self._name
class Option(BaseOption): class OnlyOption(BaseOption):
pass
class Option(OnlyOption):
""" """
Abstract base class for configuration option's. Abstract base class for configuration option's.
@ -584,7 +588,6 @@ class Option(BaseOption):
opts[idx_inf].impl_getname(), opts[idx_inf + idx_sup + 1].impl_getname())) opts[idx_inf].impl_getname(), opts[idx_inf + idx_sup + 1].impl_getname()))
def _impl_convert_callbacks(self, descr, load=False): def _impl_convert_callbacks(self, descr, load=False):
#FIXME
if not load and self._callback is None: if not load and self._callback is None:
self._state_callback = None self._state_callback = None
elif load and self._state_callback is None: elif load and self._state_callback is None:
@ -770,7 +773,7 @@ else:
raise ValueError(_('invalid unicode')) raise ValueError(_('invalid unicode'))
class SymLinkOption(BaseOption): class SymLinkOption(OnlyOption):
#__slots__ = ('_name', '_opt', '_state_opt', '_readonly', '_parent') #__slots__ = ('_name', '_opt', '_state_opt', '_readonly', '_parent')
#not return _opt consistencies #not return _opt consistencies
#_consistencies = None #_consistencies = None
@ -864,9 +867,9 @@ class PortOption(Option):
properties=None, allow_range=False, allow_zero=False, properties=None, allow_range=False, allow_zero=False,
allow_wellknown=True, allow_registred=True, allow_wellknown=True, allow_registred=True,
allow_private=False, warnings_only=False): allow_private=False, warnings_only=False):
self._allow_range = allow_range extra = {'_allow_range': allow_range,
self._min_value = None '_min_value': None,
self._max_value = None '_max_value': None}
ports_min = [0, 1, 1024, 49152] ports_min = [0, 1, 1024, 49152]
ports_max = [0, 1023, 49151, 65535] ports_max = [0, 1023, 49151, 65535]
is_finally = False is_finally = False
@ -874,17 +877,17 @@ class PortOption(Option):
allow_wellknown, allow_wellknown,
allow_registred, allow_registred,
allow_private]): allow_private]):
if self._min_value is None: if extra['_min_value'] is None:
if allowed: if allowed:
self._min_value = ports_min[index] extra['_min_value'] = ports_min[index]
elif not allowed: elif not allowed:
is_finally = True is_finally = True
elif allowed and is_finally: elif allowed and is_finally:
raise ValueError(_('inconsistency in allowed range')) raise ValueError(_('inconsistency in allowed range'))
if allowed: if allowed:
self._max_value = ports_max[index] extra['_max_value'] = ports_max[index]
if self._max_value is None: if extra['_max_value'] is None:
raise ValueError(_('max value is empty')) raise ValueError(_('max value is empty'))
super(PortOption, self).__init__(name, doc, default=default, super(PortOption, self).__init__(name, doc, default=default,
@ -897,9 +900,10 @@ class PortOption(Option):
validator_params=validator_params, validator_params=validator_params,
properties=properties, properties=properties,
warnings_only=warnings_only) warnings_only=warnings_only)
self._extra = extra
def _validate(self, value): def _validate(self, value):
if self._allow_range and ":" in str(value): if self._extra['_allow_range'] and ":" in str(value):
value = str(value).split(':') value = str(value).split(':')
if len(value) != 2: if len(value) != 2:
raise ValueError('invalid part, range must have two values ' raise ValueError('invalid part, range must have two values '
@ -911,9 +915,9 @@ class PortOption(Option):
value = [value] value = [value]
for val in value: for val in value:
if not self._min_value <= int(val) <= self._max_value: if not self._extra['_min_value'] <= int(val) <= self._extra['_max_value']:
raise ValueError('invalid port, must be an between {0} and {1}' raise ValueError('invalid port, must be an between {0} and {1}'
''.format(self._min_value, self._max_value)) ''.format(self._extra['_min_value'], self._extra['_max_value']))
class NetworkOption(Option): class NetworkOption(Option):
@ -1168,16 +1172,12 @@ class OptionDescription(BaseOption, StorageOptionDescription):
raise ConflictError(_('duplicate option: ' raise ConflictError(_('duplicate option: '
'{0}').format(child)) '{0}').format(child))
self._children.append(child) # = (tuple(child_names), tuple(children)) self._children.append(child) # = (tuple(child_names), tuple(children))
self._cache_paths = None #FIXME pour dico !
#self._cache_paths = None
self._cache_consistencies = None self._cache_consistencies = None
# the group_type is useful for filtering OptionDescriptions in a config # the group_type is useful for filtering OptionDescriptions in a config
self._group_type = groups.default self._group_type = groups.default
#def impl_getproperties(self):
# #FIXME
# for prop in self._properties:
# yield(prop.name)
def impl_getrequires(self): def impl_getrequires(self):
#FIXME #FIXME
return self._requires return self._requires
@ -1288,31 +1288,6 @@ class OptionDescription(BaseOption, StorageOptionDescription):
if init: if init:
self._readonly = True self._readonly = True
def impl_already_build_caches(self):
return self._cache_paths is not None
def impl_build_cache_option(self, cache_path=None, cache_option=None,
_currpath=None):
if _currpath is None:
save = True
_currpath = []
cache_path = []
cache_option = []
else:
save = False
for option in self.impl_getchildren():
attr = option.impl_getname()
#FIXME specifique sqlachemy...
cache_option.append(option.id)
cache_path.append(str('.'.join(_currpath + [attr])))
if isinstance(option, OptionDescription):
_currpath.append(attr)
option.impl_build_cache_option(cache_path,
cache_option,
_currpath)
_currpath.pop()
if save:
self._cache_paths = (tuple(cache_option), tuple(cache_path))
# ____________________________________________________________ # ____________________________________________________________
def impl_set_group_type(self, group_type): def impl_set_group_type(self, group_type):
@ -1370,9 +1345,6 @@ class OptionDescription(BaseOption, StorageOptionDescription):
raise ValueError(_('group_type: {0}' raise ValueError(_('group_type: {0}'
' not allowed').format(group_type)) ' not allowed').format(group_type))
def impl_get_group_type(self):
return getattr(groups, self._group_type)
def _valid_consistency(self, option, value, context, index): def _valid_consistency(self, option, value, context, index):
if self._cache_consistencies is None: if self._cache_consistencies is None:
return True return True

4
tiramisu/setting.py
View File

@ -384,7 +384,6 @@ class Settings(object):
is_cached, props = self._p_.getcache(path, ntime) is_cached, props = self._p_.getcache(path, ntime)
if is_cached: if is_cached:
return props return props
#FIXME
props = self._p_.getproperties(path, opt._properties) props = self._p_.getproperties(path, opt._properties)
if is_apply_req: if is_apply_req:
props |= self.apply_requires(opt, path) props |= self.apply_requires(opt, path)
@ -606,8 +605,7 @@ class Settings(object):
" '{0}' with requirement on: " " '{0}' with requirement on: "
"'{1}'").format(path, reqpath)) "'{1}'").format(path, reqpath))
try: try:
value = context._getattr(reqpath, value = context._getattr(reqpath, force_permissive=True)
force_permissive=True)
except PropertiesOptionError as err: except PropertiesOptionError as err:
if not transitive: if not transitive:
continue continue

101
tiramisu/storage/sqlalchemy/option.py
View File

@ -18,7 +18,9 @@
# #
# ____________________________________________________________ # ____________________________________________________________
from tiramisu.i18n import _ from tiramisu.i18n import _
from tiramisu.setting import groups
from sqlalchemy import not_, or_
from sqlalchemy.ext.declarative import declarative_base, declared_attr from sqlalchemy.ext.declarative import declarative_base, declared_attr
from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy import create_engine, Column, Integer, String, Boolean, \ from sqlalchemy import create_engine, Column, Integer, String, Boolean, \
@ -40,9 +42,6 @@ SqlAlchemyBase = declarative_base()
# require # require
#_Base : object dans la base de donnée
# => _PropertyOption => liste des propriétés
# => _CallbackParam avec des Options
def load_requires(collection_type, proxy): def load_requires(collection_type, proxy):
def getter(obj): def getter(obj):
if obj is None: if obj is None:
@ -274,8 +273,10 @@ class _Base(SqlAlchemyBase):
'polymorphic_identity': 'option', 'polymorphic_identity': 'option',
'polymorphic_on': _type 'polymorphic_on': _type
} }
_extra = Column(PickleType)
#FIXME devrait etre une table #FIXME devrait etre une table
_group_type = Column(String) _group_type = Column(String)
_is_build_cache = Column(Boolean, default=False)
def __init__(self): def __init__(self):
self.commit() self.commit()
@ -324,21 +325,95 @@ class _Base(SqlAlchemyBase):
key)) key))
class Cache(SqlAlchemyBase):
__tablename__ = 'cache'
id = Column(Integer, primary_key=True)
#FIXME indexer ... les 3
path = Column(String, nullable=False)
descr = Column(Integer, nullable=False)
option = Column(Integer, nullable=False)
opt_type = Column(String, nullable=False)
def __init__(self, descr, option, path):
self.descr = descr.id
self.option = option.id
self.path = path
self.opt_type = option.__class__.__name__
class StorageOptionDescription(object): class StorageOptionDescription(object):
def impl_already_build_caches(self):
return self._is_build_cache
def impl_get_opt_by_path(self, path): def impl_get_opt_by_path(self, path):
try: ret = session.query(Cache).filter_by(descr=self.id, path=path).first()
#FIXME if ret is None:
idx = self._cache_paths[1].index(path)
opt_id = self._cache_paths[0][idx]
return session.query(_Base).filter_by(id=opt_id).first()
except ValueError:
raise AttributeError(_('no option for path {0}').format(path)) raise AttributeError(_('no option for path {0}').format(path))
return session.query(_Base).filter_by(id=ret.option).first()
def impl_get_path_by_opt(self, opt): def impl_get_path_by_opt(self, opt):
try: ret = session.query(Cache).filter_by(descr=self.id,
return self._cache_paths[1][self._cache_paths[0].index(opt.id)] option=opt.id).first()
except ValueError: if ret is None:
raise AttributeError(_('no option {0} found').format(opt)) raise AttributeError(_('no option {0} found').format(opt))
return ret.path
def impl_get_group_type(self):
return getattr(groups, self._group_type)
def impl_build_cache_option(self, descr=None, _currpath=None):
if descr is None:
save = True
descr = self
_currpath = []
else:
save = False
for option in self.impl_getchildren():
attr = option.impl_getname()
session.add(Cache(descr, option,
str('.'.join(_currpath + [attr]))))
if isinstance(option, StorageOptionDescription):
_currpath.append(attr)
option.impl_build_cache_option(descr,
_currpath)
_currpath.pop()
if save:
self._is_build_cache = True
session.commit()
def impl_get_options_paths(self, bytype, byname, _subpath, only_first):
#FIXME tester si 1er est un descr ...
#FAIRE UN JOIN pour only_first
sqlquery = session.query(Cache).filter_by(descr=self.id)
if bytype is None:
sqlquery = sqlquery.filter(not_(Cache.opt_type == 'OptionDescription'))
else:
sqlquery = sqlquery.filter_by(opt_type=bytype.__name__)
query = ''
or_query = ''
if _subpath is not None:
query += _subpath + '.'
if byname is not None:
or_query = query + byname
query += '%.' + byname
if query != '':
filter_query = Cache.path.like(query)
if or_query != '':
filter_query = or_(Cache.path == or_query, filter_query)
sqlquery = sqlquery.filter(filter_query)
if only_first:
opt = sqlquery.first()
if opt is None:
return tuple()
option = session.query(_Base).filter_by(id=opt.option).first()
return ((opt.path, option),)
else:
ret = []
for opt in sqlquery.all():
option = session.query(_Base).filter_by(id=opt.option).first()
ret.append((opt.path, option))
return ret
class StorageBase(_Base): class StorageBase(_Base):
@ -347,7 +422,7 @@ class StorageBase(_Base):
return {'polymorphic_identity': self.__name__.lower()} return {'polymorphic_identity': self.__name__.lower()}
#FIXME #engine.echo = True
SqlAlchemyBase.metadata.create_all(engine) SqlAlchemyBase.metadata.create_all(engine)
Session = sessionmaker(bind=engine) Session = sessionmaker(bind=engine)
session = Session() session = Session()