From c81a2bcdbfd06e08904268d0cf286ea2da92b4ec Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Sat, 1 Oct 2016 20:15:08 +0200 Subject: [PATCH] better sqlalchemy integration --- test/test_config.py | 8 +- test/test_config_api.py | 93 ++++++++---- test/test_dereference.py | 10 ++ test/test_state.py | 2 +- tiramisu/option/option.py | 25 +++- tiramisu/storage/dictionary/option.py | 2 +- tiramisu/storage/sqlalchemy/option.py | 207 ++++++++++++++++++++++++-- 7 files changed, 297 insertions(+), 50 deletions(-) diff --git a/test/test_config.py b/test/test_config.py index 9dfb809..80b1b66 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -192,8 +192,12 @@ def test_config_impl_get_opt_by_path(): config = Config(descr) dummy = config.unwrap_from_path('gc.dummy') boo = config.unwrap_from_path('bool') - assert config.cfgimpl_get_description().impl_get_opt_by_path('bool') == boo - assert config.cfgimpl_get_description().impl_get_opt_by_path('gc.dummy') == dummy + if 'id' in dir(boo): + assert config.cfgimpl_get_description().impl_get_opt_by_path('bool').id == boo.id + assert config.cfgimpl_get_description().impl_get_opt_by_path('gc.dummy').id == dummy.id + else: + assert config.cfgimpl_get_description().impl_get_opt_by_path('bool') == boo + assert config.cfgimpl_get_description().impl_get_opt_by_path('gc.dummy') == dummy raises(AttributeError, "config.cfgimpl_get_description().impl_get_opt_by_path('gc.unknown')") diff --git a/test/test_config_api.py b/test/test_config_api.py index 4b7bb4d..6950581 100644 --- a/test/test_config_api.py +++ b/test/test_config_api.py @@ -37,6 +37,13 @@ def make_description(): return descr +def _is_same_opt(opt1, opt2): + if "id" in dir(opt1): + assert opt1.id == opt2.id + else: + assert opt1 == opt2 + + def test_iter_config(): "iteration on config object" s = StrOption("string", "", default="string") @@ -133,38 +140,70 @@ def test_find_in_config(): conf = Config(descr) conf.read_only() conf.cfgimpl_get_settings().setpermissive(('hidden',)) - assert conf.find(byname='dummy') == [conf.unwrap_from_path('gc.dummy')] - assert conf.find(byname='float') == [conf.unwrap_from_path('gc.float'), conf.unwrap_from_path('float')] - assert conf.find_first(byname='bool') == conf.unwrap_from_path('gc.gc2.bool') - assert conf.find_first(byname='bool', byvalue=True) == conf.unwrap_from_path('bool') - assert conf.find_first(byname='dummy') == conf.unwrap_from_path('gc.dummy') - assert conf.find_first(byname='float') == conf.unwrap_from_path('gc.float') - assert conf.find(bytype=ChoiceOption) == [conf.unwrap_from_path('gc.name'), conf.unwrap_from_path('objspace')] - assert conf.find_first(bytype=ChoiceOption) == conf.unwrap_from_path('gc.name') - assert conf.find(byvalue='ref') == [conf.unwrap_from_path('gc.name')] - assert conf.find_first(byvalue='ref') == conf.unwrap_from_path('gc.name') - assert conf.find(byname='prop') == [conf.unwrap_from_path('gc.prop')] + ret = conf.find(byname='dummy') + assert len(ret) == 1 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.dummy')) + ret = conf.find(byname='float') + assert len(ret) == 2 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.float')) + _is_same_opt(ret[1], conf.unwrap_from_path('float')) + _is_same_opt(conf.find_first(byname='bool'), conf.unwrap_from_path('gc.gc2.bool')) + _is_same_opt(conf.find_first(byname='bool', byvalue=True), conf.unwrap_from_path('bool')) + _is_same_opt(conf.find_first(byname='dummy'), conf.unwrap_from_path('gc.dummy')) + _is_same_opt(conf.find_first(byname='float'), conf.unwrap_from_path('gc.float')) + ret = conf.find(bytype=ChoiceOption) + assert len(ret) == 2 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.name')) + _is_same_opt(ret[1], conf.unwrap_from_path('objspace')) + _is_same_opt(conf.find_first(bytype=ChoiceOption), conf.unwrap_from_path('gc.name')) + ret = conf.find(byvalue='ref') + assert len(ret) == 1 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.name')) + _is_same_opt(conf.find_first(byvalue='ref'), conf.unwrap_from_path('gc.name')) + ret = conf.find(byname='prop') + assert len(ret) == 1 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.prop')) conf.read_write() raises(AttributeError, "assert conf.find(byname='prop')") - assert conf.find(byname='prop', check_properties=False) == [conf.unwrap_from_path('gc.gc2.prop'), conf.unwrap_from_path('gc.prop')] - assert conf.find(byname='prop', force_permissive=True) == [conf.unwrap_from_path('gc.prop')] - assert conf.find_first(byname='prop', force_permissive=True) == conf.unwrap_from_path('gc.prop') + ret = conf.find(byname='prop', check_properties=False) + assert len(ret) == 2 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.gc2.prop')) + _is_same_opt(ret[1], conf.unwrap_from_path('gc.prop')) + ret = conf.find(byname='prop', force_permissive=True) + assert len(ret) == 1 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.prop')) + _is_same_opt(conf.find_first(byname='prop', force_permissive=True), conf.unwrap_from_path('gc.prop')) #assert conf.find_first(byname='prop') == conf.unwrap_from_path('gc.prop') # combinaison of filters - assert conf.find(bytype=BoolOption, byname='dummy') == [conf.unwrap_from_path('gc.dummy')] - assert conf.find_first(bytype=BoolOption, byname='dummy') == conf.unwrap_from_path('gc.dummy') - assert conf.find(byvalue=False, byname='dummy') == [conf.unwrap_from_path('gc.dummy')] - assert conf.find_first(byvalue=False, byname='dummy') == conf.unwrap_from_path('gc.dummy') + ret = conf.find(bytype=BoolOption, byname='dummy') + assert len(ret) == 1 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.dummy')) + _is_same_opt(conf.find_first(bytype=BoolOption, byname='dummy'), conf.unwrap_from_path('gc.dummy')) + ret = conf.find(byvalue=False, byname='dummy') + assert len(ret) == 1 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.dummy')) + _is_same_opt(conf.find_first(byvalue=False, byname='dummy'), conf.unwrap_from_path('gc.dummy')) #subconfig - assert conf.gc.find(byname='dummy') == [conf.unwrap_from_path('gc.dummy')] - assert conf.gc.find(byname='float') == [conf.unwrap_from_path('gc.float')] - assert conf.gc.find(byname='bool') == [conf.unwrap_from_path('gc.gc2.bool')] - assert conf.gc.find_first(byname='bool', byvalue=False) == conf.unwrap_from_path('gc.gc2.bool') + ret = conf.gc.find(byname='dummy') + assert len(ret) == 1 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.dummy')) + ret = conf.gc.find(byname='float') + assert len(ret) == 1 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.float')) + ret = conf.gc.find(byname='bool') + assert len(ret) == 1 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.gc2.bool')) + _is_same_opt(conf.gc.find_first(byname='bool', byvalue=False), conf.unwrap_from_path('gc.gc2.bool')) raises(AttributeError, "assert conf.gc.find_first(byname='bool', byvalue=True)") raises(AttributeError, "conf.gc.find(byname='wantref').first()") - assert conf.gc.find(byname='prop', check_properties=False) == [conf.unwrap_from_path('gc.gc2.prop'), conf.unwrap_from_path('gc.prop')] + ret = conf.gc.find(byname='prop', check_properties=False) + assert len(ret) == 2 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.gc2.prop')) + _is_same_opt(ret[1], conf.unwrap_from_path('gc.prop')) conf.read_only() - assert conf.gc.find(byname='prop') == [conf.unwrap_from_path('gc.prop')] + ret = conf.gc.find(byname='prop') + assert len(ret) == 1 + _is_same_opt(ret[0], conf.unwrap_from_path('gc.prop')) # not OptionDescription raises(AttributeError, "conf.find_first(byname='gc')") raises(AttributeError, "conf.gc.find_first(byname='gc2')") @@ -184,8 +223,10 @@ def test_find_multi(): raises(AttributeError, "conf.find(byvalue=True)") raises(AttributeError, "conf.find_first(byvalue=True)") conf.bool.append(True) - assert conf.find(byvalue=True) == [b] - assert conf.find_first(byvalue=True) == b + ret = conf.find(byvalue=True) + assert len(ret) == 1 + _is_same_opt(ret[0], b) + _is_same_opt(conf.find_first(byvalue=True), b) def test_does_not_find_in_config(): diff --git a/test/test_dereference.py b/test/test_dereference.py index f8776b5..00e824d 100644 --- a/test/test_dereference.py +++ b/test/test_dereference.py @@ -87,6 +87,8 @@ def test_deref_option_cache(): def test_deref_optiondescription_cache(): + if not IS_DEREFABLE: + return b = BoolOption('b', '') o = OptionDescription('od', '', [b]) o.impl_build_cache_option() @@ -113,6 +115,8 @@ def test_deref_option_config(): def test_deref_optiondescription_config(): + if not IS_DEREFABLE: + return b = BoolOption('b', '') o = OptionDescription('od', '', [b]) c = Config(o) @@ -126,6 +130,8 @@ def test_deref_optiondescription_config(): def test_deref_groupconfig(): + if not IS_DEREFABLE: + return i1 = IntOption('i1', '') od1 = OptionDescription('od1', '', [i1]) od2 = OptionDescription('od2', '', [od1]) @@ -140,6 +146,8 @@ def test_deref_groupconfig(): def test_deref_metaconfig(): + if not IS_DEREFABLE: + return i1 = IntOption('i1', '') od1 = OptionDescription('od1', '', [i1]) od2 = OptionDescription('od2', '', [od1]) @@ -154,6 +162,8 @@ def test_deref_metaconfig(): def test_deref_submulti(): + if not IS_DEREFABLE: + return multi = StrOption('multi', '', multi=submulti) od = OptionDescription('od', '', [multi]) cfg = Config(od) diff --git a/test/test_state.py b/test/test_state.py index 824e368..75e23dd 100644 --- a/test/test_state.py +++ b/test/test_state.py @@ -20,7 +20,7 @@ def return_value(value=None): def _get_slots(opt): slots = set() for subclass in opt.__class__.__mro__: - if subclass is not object: + if subclass is not object and '__slots__' in dir(subclass): slots.update(subclass.__slots__) return slots diff --git a/tiramisu/option/option.py b/tiramisu/option/option.py index b92a382..e90fb77 100644 --- a/tiramisu/option/option.py +++ b/tiramisu/option/option.py @@ -36,7 +36,7 @@ class ChoiceOption(Option): The option can also have the value ``None`` """ - __slots__ = tuple() + __slots__ = tuple('_init') display_name = _('choice') def __init__(self, name, doc, values, default=None, @@ -55,8 +55,10 @@ class ChoiceOption(Option): if not isinstance(values, tuple): # pragma: optional cover raise TypeError(_('values must be a tuple or a function for {0}' ).format(name)) - self.impl_set_choice_values_params(values, values_params) - + session = self.getsession() + #cannot add values and values_params in database before add option + #set in _init temporary + self._init = (values, values_params) super(ChoiceOption, self).__init__(name, doc, default=default, default_multi=default_multi, callback=callback, @@ -66,19 +68,30 @@ class ChoiceOption(Option): validator=validator, validator_params=validator_params, properties=properties, - warnings_only=warnings_only) + warnings_only=warnings_only, + session=session) + self.impl_set_choice_values_params(values, values_params, session) + session.commit() + del(self._init) def impl_get_values(self, context, current_opt=undefined, returns_raise=False): if current_opt is undefined: current_opt = self + params = undefined #FIXME cache? but in context... - values = self._choice_values + if '_init' in dir(self): + values, params = self._init + else: + values = self._choice_values if isinstance(values, FunctionType): if context is None: values = [] else: - values_params = self.impl_get_choice_values_params() + if params is not undefined: + values_params = params + else: + values_params = self.impl_get_choice_values_params() values = carry_out_calculation(current_opt, context=context, callback=values, callback_params=values_params, diff --git a/tiramisu/storage/dictionary/option.py b/tiramisu/storage/dictionary/option.py index ea49b50..e9b49a8 100644 --- a/tiramisu/storage/dictionary/option.py +++ b/tiramisu/storage/dictionary/option.py @@ -420,7 +420,7 @@ class StorageOptionDescription(StorageBase): def impl_build_cache_option(self, _currpath=None, cache_path=None, cache_option=None): - if _currpath is None and getattr(self, '_cache_paths', None) is not None: + if self.impl_is_readonly() or (_currpath is None and getattr(self, '_cache_paths', None) is not None): # cache already set return if _currpath is None: diff --git a/tiramisu/storage/sqlalchemy/option.py b/tiramisu/storage/sqlalchemy/option.py index e03a934..9d5e137 100644 --- a/tiramisu/storage/sqlalchemy/option.py +++ b/tiramisu/storage/sqlalchemy/option.py @@ -198,6 +198,152 @@ class _CallbackParam(SqlAlchemyBase): self.params.append(_CallbackParamOption(value=param)) +#____________________________________________________________ +# +# choice +class _ChoiceParamOption(SqlAlchemyBase): + __tablename__ = 'choice_param_option' + id = Column(Integer, primary_key=True) + choice = Column(Integer, index=True) + option = Column(Integer) + force_permissive = Column(Boolean) + value = Column(PickleType) + + def __init__(self, choice, option=undefined, force_permissive=undefined, value=undefined): + self.choice = choice.id + if value is not undefined: + self.value = value + elif option is not undefined: + self.option = option.id + self.force_permissive = force_permissive + + +class _ChoiceParam(SqlAlchemyBase): + __tablename__ = 'choice_param' + id = Column(Integer, primary_key=True) + option = Column(Integer, index=True) + key = Column(String) + + def __init__(self, option, key): + self.option = option.id + self.key = key + + +#def load_choice_parm(collection_type, proxy): +# def getter(obj): +# if obj is None: +# return None +# ret = [] +# requires = getattr(obj, proxy.value_attr) +# session = util.Session() +# for require in requires: +# if require.value is not None: +# ret.append(require.value) +# else: +# option = session.query(_Base).filter_by(id=require.option).first() +# ret.append((option, require.force_permissive)) +# return tuple(ret) +# +# def setter(obj, value): +# setattr(obj, proxy.value_attr, value) +# return getter, setter +# +# +#class _ChoiceParamOption(SqlAlchemyBase): +# __tablename__ = 'choice_param_option' +# id = Column(Integer, primary_key=True) +# valid_param = Column(Integer, ForeignKey('choice_param.id')) +# option = Column(Integer) +# force_permissive = Column(Boolean) +# value = Column(PickleType) +# +# def __init__(self, option=undefined, force_permissive=undefined, value=undefined): +# if value is not undefined: +# self.value = value +# elif option is not undefined: +# self.option = option.id +# self.force_permissive = force_permissive +# +# +#class _ChoiceParam(SqlAlchemyBase): +# __tablename__ = 'choice_param' +# id = Column(Integer, primary_key=True) +# choice = Column(Integer, ForeignKey('baseoption.id')) +# key = Column(String) +# params = relationship('_ChoiceParamOption') +# +# def __init__(self, key, params): +# self.key = key +# for param in params: +# if isinstance(param, tuple): +# if param == (None,): +# self.params.append(_ChoiceParamOption()) +# else: +# self.params.append(_ChoiceParamOption(option=param[0], +# force_permissive=param[1])) +# else: +# self.params.append(_ChoiceParamOption(value=param)) + + +#____________________________________________________________ +# +# validator +def load_validator_parm(collection_type, proxy): + def getter(obj): + if obj is None: + return None + ret = [] + requires = getattr(obj, proxy.value_attr) + session = util.Session() + for require in requires: + if require.value is not None: + ret.append(require.value) + else: + option = session.query(_Base).filter_by(id=require.option).first() + ret.append((option, require.force_permissive)) + return tuple(ret) + + def setter(obj, value): + setattr(obj, proxy.value_attr, value) + return getter, setter + + +class _ValidatorParamOption(SqlAlchemyBase): + __tablename__ = 'validator_param_option' + id = Column(Integer, primary_key=True) + validator_param = Column(Integer, ForeignKey('validator_param.id')) + option = Column(Integer) + force_permissive = Column(Boolean) + value = Column(PickleType) + + def __init__(self, option=undefined, force_permissive=undefined, value=undefined): + if value is not undefined: + self.value = value + elif option is not undefined: + self.option = option.id + self.force_permissive = force_permissive + + +class _ValidatorParam(SqlAlchemyBase): + __tablename__ = 'validator_param' + id = Column(Integer, primary_key=True) + validator = Column(Integer, ForeignKey('baseoption.id')) + key = Column(String) + params = relationship('_ValidatorParamOption') + + def __init__(self, key, params): + self.key = key + for param in params: + if isinstance(param, tuple): + if param == (None,): + self.params.append(_ValidatorParamOption()) + else: + self.params.append(_ValidatorParamOption(option=param[0], + force_permissive=param[1])) + else: + self.params.append(_ValidatorParamOption(value=param)) + + #____________________________________________________________ # # consistency @@ -253,10 +399,10 @@ class _Base(SqlAlchemyBase): _opt = Column(Integer) _master_slaves = Column(Integer) _choice_values = Column(PickleType) - _cho_params = relationship('_CallbackParam', - collection_class=attribute_mapped_collection('key')) - _choice_values_params = association_proxy("_cho_params", "params", - getset_factory=load_callback_parm) + #_cho_params = relationship('_ChoiceParam', + # collection_class=attribute_mapped_collection('key')) + #_choice_values_params = association_proxy("_cho_params", "params", + # getset_factory=load_choice_parm) _reqs = relationship("_Require", collection_class=list) _requires = association_proxy("_reqs", "requires", getset_factory=load_requires) _multi = Column(Integer) @@ -267,10 +413,10 @@ class _Base(SqlAlchemyBase): _callback_params = association_proxy("_call_params", "params", getset_factory=load_callback_parm) _validator = Column(PickleType) - _val_params = relationship('_CallbackParam', + _val_params = relationship('_ValidatorParam', collection_class=attribute_mapped_collection('key')) _validator_params = association_proxy("_val_params", "params", - getset_factory=load_callback_parm) + getset_factory=load_validator_parm) ###### #FIXME not autoload _props = relationship("_PropertyOption", collection_class=set) @@ -348,9 +494,6 @@ class _Base(SqlAlchemyBase): return (None, {}) return ret, self._callback_params - def impl_get_choice_values_params(self): - return self._choice_values_params - def impl_get_validator(self): ret = self._validator if ret is None: @@ -407,6 +550,10 @@ class _Base(SqlAlchemyBase): self.commit(session) def _set_readonly(self, has_extra): + session = self.getsession() + opt = session.query(_Base).filter_by(id=self.id).first() + opt._readonly = True + session.commit() self._readonly = True def _set_callback(self, callback, callback_params): @@ -414,10 +561,38 @@ class _Base(SqlAlchemyBase): if callback_params is not None: self._callback_params = callback_params - def impl_set_choice_values_params(self, values, values_params): + def impl_set_choice_values_params(self, values, values_params, session): self._choice_values = values if values_params is not None: - self._choice_values_params = values_params + for key, params in values_params.items(): + choice = _ChoiceParam(self, key) + session.add(choice) + session.commit() + for param in params: + if isinstance(param, tuple): + if param == (None,): + session.add(_ChoiceParamOption(choice)) + else: + session.add(_ChoiceParamOption(choice, option=param[0], force_permissive=param[1])) + else: + session.add(_ChoiceParamOption(choice, value=param)) + session.commit() + + def impl_get_choice_values_params(self): + session = self.getsession() + params = {} + for param in session.query(_ChoiceParam).filter_by(option=self.id).all(): + _params = [] + for _param in session.query(_ChoiceParamOption).filter_by(choice=param.id).all(): + if _param.value: + _params.append(_param.value) + elif _param.option: + _params.append((session.query(_Base).filter_by(id=_param.option).first(), + _param.force_permissive)) + else: + _params.append((None,)) + params[param.key] = _params + return params def _set_validator(self, validator, validator_params): self._validator = validator @@ -425,10 +600,11 @@ class _Base(SqlAlchemyBase): self._validator_params = validator_params def impl_is_readonly(self): - try: - return self._readonly - except AttributeError: + session = self.getsession() + opt = session.query(_Base).filter_by(id=self.id).first() + if opt is None or opt._readonly is None: return False + return opt._readonly def impl_is_multi(self): return self._multi == 0 or self._multi == 2 @@ -559,6 +735,9 @@ class StorageOptionDescription(object): def impl_build_cache_option(self, descr=None, _currpath=None, subdyn_path=None, session=None): + if self.impl_is_readonly() or (_currpath is None and getattr(self, '_cache_paths', None) is not None): + # cache already set + return if descr is None: save = True descr = self