better sqlalchemy integration

test/test_config.py

@@ -192,6 +192,10 @@ def test_config_impl_get_opt_by_path():
     config = Config(descr)
     dummy = config.unwrap_from_path('gc.dummy')
     boo = config.unwrap_from_path('bool')
+    if 'id' in dir(boo):
+        assert config.cfgimpl_get_description().impl_get_opt_by_path('bool').id == boo.id
+        assert config.cfgimpl_get_description().impl_get_opt_by_path('gc.dummy').id == dummy.id
+    else:
         assert config.cfgimpl_get_description().impl_get_opt_by_path('bool') == boo
         assert config.cfgimpl_get_description().impl_get_opt_by_path('gc.dummy') == dummy
     raises(AttributeError, "config.cfgimpl_get_description().impl_get_opt_by_path('gc.unknown')")

test/test_config_api.py

@@ -37,6 +37,13 @@ def make_description():
     return descr
 
 
+def _is_same_opt(opt1, opt2):
+    if "id" in dir(opt1):
+        assert opt1.id == opt2.id
+    else:
+        assert opt1 == opt2
+
+
 def test_iter_config():
     "iteration on config object"
     s = StrOption("string", "", default="string")
@@ -133,38 +140,70 @@ def test_find_in_config():
     conf = Config(descr)
     conf.read_only()
     conf.cfgimpl_get_settings().setpermissive(('hidden',))
-    assert conf.find(byname='dummy') == [conf.unwrap_from_path('gc.dummy')]
+    ret = conf.find(byname='dummy')
-    assert conf.find(byname='float') == [conf.unwrap_from_path('gc.float'), conf.unwrap_from_path('float')]
+    assert len(ret) == 1
-    assert conf.find_first(byname='bool') == conf.unwrap_from_path('gc.gc2.bool')
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.dummy'))
-    assert conf.find_first(byname='bool', byvalue=True) == conf.unwrap_from_path('bool')
+    ret = conf.find(byname='float')
-    assert conf.find_first(byname='dummy') == conf.unwrap_from_path('gc.dummy')
+    assert len(ret) == 2
-    assert conf.find_first(byname='float') == conf.unwrap_from_path('gc.float')
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.float'))
-    assert conf.find(bytype=ChoiceOption) == [conf.unwrap_from_path('gc.name'), conf.unwrap_from_path('objspace')]
+    _is_same_opt(ret[1], conf.unwrap_from_path('float'))
-    assert conf.find_first(bytype=ChoiceOption) == conf.unwrap_from_path('gc.name')
+    _is_same_opt(conf.find_first(byname='bool'), conf.unwrap_from_path('gc.gc2.bool'))
-    assert conf.find(byvalue='ref') == [conf.unwrap_from_path('gc.name')]
+    _is_same_opt(conf.find_first(byname='bool', byvalue=True), conf.unwrap_from_path('bool'))
-    assert conf.find_first(byvalue='ref') == conf.unwrap_from_path('gc.name')
+    _is_same_opt(conf.find_first(byname='dummy'), conf.unwrap_from_path('gc.dummy'))
-    assert conf.find(byname='prop') == [conf.unwrap_from_path('gc.prop')]
+    _is_same_opt(conf.find_first(byname='float'), conf.unwrap_from_path('gc.float'))
+    ret = conf.find(bytype=ChoiceOption)
+    assert len(ret) == 2
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.name'))
+    _is_same_opt(ret[1], conf.unwrap_from_path('objspace'))
+    _is_same_opt(conf.find_first(bytype=ChoiceOption), conf.unwrap_from_path('gc.name'))
+    ret = conf.find(byvalue='ref')
+    assert len(ret) == 1
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.name'))
+    _is_same_opt(conf.find_first(byvalue='ref'), conf.unwrap_from_path('gc.name'))
+    ret = conf.find(byname='prop')
+    assert len(ret) == 1
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.prop'))
     conf.read_write()
     raises(AttributeError, "assert conf.find(byname='prop')")
-    assert conf.find(byname='prop', check_properties=False) == [conf.unwrap_from_path('gc.gc2.prop'), conf.unwrap_from_path('gc.prop')]
+    ret = conf.find(byname='prop', check_properties=False)
-    assert conf.find(byname='prop', force_permissive=True) == [conf.unwrap_from_path('gc.prop')]
+    assert len(ret) == 2
-    assert conf.find_first(byname='prop', force_permissive=True) == conf.unwrap_from_path('gc.prop')
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.gc2.prop'))
+    _is_same_opt(ret[1], conf.unwrap_from_path('gc.prop'))
+    ret = conf.find(byname='prop', force_permissive=True)
+    assert len(ret) == 1
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.prop'))
+    _is_same_opt(conf.find_first(byname='prop', force_permissive=True), conf.unwrap_from_path('gc.prop'))
     #assert conf.find_first(byname='prop') == conf.unwrap_from_path('gc.prop')
     # combinaison of filters
-    assert conf.find(bytype=BoolOption, byname='dummy') == [conf.unwrap_from_path('gc.dummy')]
+    ret = conf.find(bytype=BoolOption, byname='dummy')
-    assert conf.find_first(bytype=BoolOption, byname='dummy') == conf.unwrap_from_path('gc.dummy')
+    assert len(ret) == 1
-    assert conf.find(byvalue=False, byname='dummy') == [conf.unwrap_from_path('gc.dummy')]
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.dummy'))
-    assert conf.find_first(byvalue=False, byname='dummy') == conf.unwrap_from_path('gc.dummy')
+    _is_same_opt(conf.find_first(bytype=BoolOption, byname='dummy'), conf.unwrap_from_path('gc.dummy'))
+    ret = conf.find(byvalue=False, byname='dummy')
+    assert len(ret) == 1
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.dummy'))
+    _is_same_opt(conf.find_first(byvalue=False, byname='dummy'), conf.unwrap_from_path('gc.dummy'))
     #subconfig
-    assert conf.gc.find(byname='dummy') == [conf.unwrap_from_path('gc.dummy')]
+    ret = conf.gc.find(byname='dummy')
-    assert conf.gc.find(byname='float') == [conf.unwrap_from_path('gc.float')]
+    assert len(ret) == 1
-    assert conf.gc.find(byname='bool') == [conf.unwrap_from_path('gc.gc2.bool')]
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.dummy'))
-    assert conf.gc.find_first(byname='bool', byvalue=False) == conf.unwrap_from_path('gc.gc2.bool')
+    ret = conf.gc.find(byname='float')
+    assert len(ret) == 1
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.float'))
+    ret = conf.gc.find(byname='bool')
+    assert len(ret) == 1
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.gc2.bool'))
+    _is_same_opt(conf.gc.find_first(byname='bool', byvalue=False), conf.unwrap_from_path('gc.gc2.bool'))
     raises(AttributeError, "assert conf.gc.find_first(byname='bool', byvalue=True)")
     raises(AttributeError, "conf.gc.find(byname='wantref').first()")
-    assert conf.gc.find(byname='prop', check_properties=False) == [conf.unwrap_from_path('gc.gc2.prop'), conf.unwrap_from_path('gc.prop')]
+    ret = conf.gc.find(byname='prop', check_properties=False)
+    assert len(ret) == 2
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.gc2.prop'))
+    _is_same_opt(ret[1], conf.unwrap_from_path('gc.prop'))
     conf.read_only()
-    assert conf.gc.find(byname='prop') == [conf.unwrap_from_path('gc.prop')]
+    ret = conf.gc.find(byname='prop')
+    assert len(ret) == 1
+    _is_same_opt(ret[0], conf.unwrap_from_path('gc.prop'))
     # not OptionDescription
     raises(AttributeError, "conf.find_first(byname='gc')")
     raises(AttributeError, "conf.gc.find_first(byname='gc2')")
@@ -184,8 +223,10 @@ def test_find_multi():
     raises(AttributeError, "conf.find(byvalue=True)")
     raises(AttributeError, "conf.find_first(byvalue=True)")
     conf.bool.append(True)
-    assert conf.find(byvalue=True) == [b]
+    ret = conf.find(byvalue=True)
-    assert conf.find_first(byvalue=True) == b
+    assert len(ret) == 1
+    _is_same_opt(ret[0], b)
+    _is_same_opt(conf.find_first(byvalue=True), b)
 
 
 def test_does_not_find_in_config():

test/test_dereference.py

@@ -87,6 +87,8 @@ def test_deref_option_cache():
 
 
 def test_deref_optiondescription_cache():
+    if not IS_DEREFABLE:
+        return
     b = BoolOption('b', '')
     o = OptionDescription('od', '', [b])
     o.impl_build_cache_option()
@@ -113,6 +115,8 @@ def test_deref_option_config():
 
 
 def test_deref_optiondescription_config():
+    if not IS_DEREFABLE:
+        return
     b = BoolOption('b', '')
     o = OptionDescription('od', '', [b])
     c = Config(o)
@@ -126,6 +130,8 @@ def test_deref_optiondescription_config():
 
 
 def test_deref_groupconfig():
+    if not IS_DEREFABLE:
+        return
     i1 = IntOption('i1', '')
     od1 = OptionDescription('od1', '', [i1])
     od2 = OptionDescription('od2', '', [od1])
@@ -140,6 +146,8 @@ def test_deref_groupconfig():
 
 
 def test_deref_metaconfig():
+    if not IS_DEREFABLE:
+        return
     i1 = IntOption('i1', '')
     od1 = OptionDescription('od1', '', [i1])
     od2 = OptionDescription('od2', '', [od1])
@@ -154,6 +162,8 @@ def test_deref_metaconfig():
 
 
 def test_deref_submulti():
+    if not IS_DEREFABLE:
+        return
     multi = StrOption('multi', '', multi=submulti)
     od = OptionDescription('od', '', [multi])
     cfg = Config(od)

test/test_state.py

@@ -20,7 +20,7 @@ def return_value(value=None):
 def _get_slots(opt):
     slots = set()
     for subclass in opt.__class__.__mro__:
-        if subclass is not object:
+        if subclass is not object and '__slots__' in dir(subclass):
             slots.update(subclass.__slots__)
     return slots
 

tiramisu/option/option.py

@@ -36,7 +36,7 @@ class ChoiceOption(Option):
 
     The option can also have the value ``None``
     """
-    __slots__ = tuple()
+    __slots__ = tuple('_init')
     display_name = _('choice')
 
     def __init__(self, name, doc, values, default=None,
@@ -55,8 +55,10 @@ class ChoiceOption(Option):
             if not isinstance(values, tuple):  # pragma: optional cover
                 raise TypeError(_('values must be a tuple or a function for {0}'
                                   ).format(name))
-        self.impl_set_choice_values_params(values, values_params)
+        session = self.getsession()
-
+        #cannot add values and values_params in database before add option
+        #set in _init temporary
+        self._init = (values, values_params)
         super(ChoiceOption, self).__init__(name, doc, default=default,
                                            default_multi=default_multi,
                                            callback=callback,
@@ -66,17 +68,28 @@ class ChoiceOption(Option):
                                            validator=validator,
                                            validator_params=validator_params,
                                            properties=properties,
-                                           warnings_only=warnings_only)
+                                           warnings_only=warnings_only,
+                                           session=session)
+        self.impl_set_choice_values_params(values, values_params, session)
+        session.commit()
+        del(self._init)
 
     def impl_get_values(self, context, current_opt=undefined,
                         returns_raise=False):
         if current_opt is undefined:
             current_opt = self
+        params = undefined
         #FIXME cache? but in context...
+        if '_init' in dir(self):
+            values, params = self._init
+        else:
             values = self._choice_values
         if isinstance(values, FunctionType):
             if context is None:
                 values = []
+            else:
+                if params is not undefined:
+                    values_params = params
                 else:
                     values_params = self.impl_get_choice_values_params()
                 values = carry_out_calculation(current_opt, context=context,

tiramisu/storage/dictionary/option.py

@@ -420,7 +420,7 @@ class StorageOptionDescription(StorageBase):
     def impl_build_cache_option(self, _currpath=None, cache_path=None,
                                 cache_option=None):
 
-        if _currpath is None and getattr(self, '_cache_paths', None) is not None:
+        if self.impl_is_readonly() or (_currpath is None and getattr(self, '_cache_paths', None) is not None):
             # cache already set
             return
         if _currpath is None:

tiramisu/storage/sqlalchemy/option.py

@@ -198,6 +198,152 @@ class _CallbackParam(SqlAlchemyBase):
                 self.params.append(_CallbackParamOption(value=param))
 
 
+#____________________________________________________________
+#
+# choice
+class _ChoiceParamOption(SqlAlchemyBase):
+    __tablename__ = 'choice_param_option'
+    id = Column(Integer, primary_key=True)
+    choice = Column(Integer, index=True)
+    option = Column(Integer)
+    force_permissive = Column(Boolean)
+    value = Column(PickleType)
+
+    def __init__(self, choice, option=undefined, force_permissive=undefined,  value=undefined):
+        self.choice = choice.id
+        if value is not undefined:
+            self.value = value
+        elif option is not undefined:
+            self.option = option.id
+            self.force_permissive = force_permissive
+
+
+class _ChoiceParam(SqlAlchemyBase):
+    __tablename__ = 'choice_param'
+    id = Column(Integer, primary_key=True)
+    option = Column(Integer, index=True)
+    key = Column(String)
+
+    def __init__(self, option, key):
+        self.option = option.id
+        self.key = key
+
+
+#def load_choice_parm(collection_type, proxy):
+#    def getter(obj):
+#        if obj is None:
+#            return None
+#        ret = []
+#        requires = getattr(obj, proxy.value_attr)
+#        session = util.Session()
+#        for require in requires:
+#            if require.value is not None:
+#                ret.append(require.value)
+#            else:
+#                option = session.query(_Base).filter_by(id=require.option).first()
+#                ret.append((option, require.force_permissive))
+#        return tuple(ret)
+#
+#    def setter(obj, value):
+#        setattr(obj, proxy.value_attr, value)
+#    return getter, setter
+#
+#
+#class _ChoiceParamOption(SqlAlchemyBase):
+#    __tablename__ = 'choice_param_option'
+#    id = Column(Integer, primary_key=True)
+#    valid_param = Column(Integer, ForeignKey('choice_param.id'))
+#    option = Column(Integer)
+#    force_permissive = Column(Boolean)
+#    value = Column(PickleType)
+#
+#    def __init__(self, option=undefined, force_permissive=undefined,  value=undefined):
+#        if value is not undefined:
+#            self.value = value
+#        elif option is not undefined:
+#            self.option = option.id
+#            self.force_permissive = force_permissive
+#
+#
+#class _ChoiceParam(SqlAlchemyBase):
+#    __tablename__ = 'choice_param'
+#    id = Column(Integer, primary_key=True)
+#    choice = Column(Integer, ForeignKey('baseoption.id'))
+#    key = Column(String)
+#    params = relationship('_ChoiceParamOption')
+#
+#    def __init__(self, key, params):
+#        self.key = key
+#        for param in params:
+#            if isinstance(param, tuple):
+#                if param == (None,):
+#                    self.params.append(_ChoiceParamOption())
+#                else:
+#                    self.params.append(_ChoiceParamOption(option=param[0],
+#                                                            force_permissive=param[1]))
+#            else:
+#                self.params.append(_ChoiceParamOption(value=param))
+
+
+#____________________________________________________________
+#
+# validator
+def load_validator_parm(collection_type, proxy):
+    def getter(obj):
+        if obj is None:
+            return None
+        ret = []
+        requires = getattr(obj, proxy.value_attr)
+        session = util.Session()
+        for require in requires:
+            if require.value is not None:
+                ret.append(require.value)
+            else:
+                option = session.query(_Base).filter_by(id=require.option).first()
+                ret.append((option, require.force_permissive))
+        return tuple(ret)
+
+    def setter(obj, value):
+        setattr(obj, proxy.value_attr, value)
+    return getter, setter
+
+
+class _ValidatorParamOption(SqlAlchemyBase):
+    __tablename__ = 'validator_param_option'
+    id = Column(Integer, primary_key=True)
+    validator_param = Column(Integer, ForeignKey('validator_param.id'))
+    option = Column(Integer)
+    force_permissive = Column(Boolean)
+    value = Column(PickleType)
+
+    def __init__(self, option=undefined, force_permissive=undefined,  value=undefined):
+        if value is not undefined:
+            self.value = value
+        elif option is not undefined:
+            self.option = option.id
+            self.force_permissive = force_permissive
+
+
+class _ValidatorParam(SqlAlchemyBase):
+    __tablename__ = 'validator_param'
+    id = Column(Integer, primary_key=True)
+    validator = Column(Integer, ForeignKey('baseoption.id'))
+    key = Column(String)
+    params = relationship('_ValidatorParamOption')
+
+    def __init__(self, key, params):
+        self.key = key
+        for param in params:
+            if isinstance(param, tuple):
+                if param == (None,):
+                    self.params.append(_ValidatorParamOption())
+                else:
+                    self.params.append(_ValidatorParamOption(option=param[0],
+                                                            force_permissive=param[1]))
+            else:
+                self.params.append(_ValidatorParamOption(value=param))
+
+
 #____________________________________________________________
 #
 # consistency
@@ -253,10 +399,10 @@ class _Base(SqlAlchemyBase):
     _opt = Column(Integer)
     _master_slaves = Column(Integer)
     _choice_values = Column(PickleType)
-    _cho_params = relationship('_CallbackParam',
+    #_cho_params = relationship('_ChoiceParam',
-                               collection_class=attribute_mapped_collection('key'))
+    #                           collection_class=attribute_mapped_collection('key'))
-    _choice_values_params = association_proxy("_cho_params", "params",
+    #_choice_values_params = association_proxy("_cho_params", "params",
-                                       getset_factory=load_callback_parm)
+    #                                   getset_factory=load_choice_parm)
     _reqs = relationship("_Require", collection_class=list)
     _requires = association_proxy("_reqs", "requires", getset_factory=load_requires)
     _multi = Column(Integer)
@@ -267,10 +413,10 @@ class _Base(SqlAlchemyBase):
     _callback_params = association_proxy("_call_params", "params",
                                          getset_factory=load_callback_parm)
     _validator = Column(PickleType)
-    _val_params = relationship('_CallbackParam',
+    _val_params = relationship('_ValidatorParam',
                                collection_class=attribute_mapped_collection('key'))
     _validator_params = association_proxy("_val_params", "params",
-                                          getset_factory=load_callback_parm)
+                                          getset_factory=load_validator_parm)
     ######
     #FIXME not autoload
     _props = relationship("_PropertyOption", collection_class=set)
@@ -348,9 +494,6 @@ class _Base(SqlAlchemyBase):
             return (None, {})
         return ret, self._callback_params
 
-    def impl_get_choice_values_params(self):
-        return self._choice_values_params
-
     def impl_get_validator(self):
         ret = self._validator
         if ret is None:
@@ -407,6 +550,10 @@ class _Base(SqlAlchemyBase):
         self.commit(session)
 
     def _set_readonly(self, has_extra):
+        session = self.getsession()
+        opt = session.query(_Base).filter_by(id=self.id).first()
+        opt._readonly = True
+        session.commit()
         self._readonly = True
 
     def _set_callback(self, callback, callback_params):
@@ -414,10 +561,38 @@ class _Base(SqlAlchemyBase):
         if callback_params is not None:
             self._callback_params = callback_params
 
-    def impl_set_choice_values_params(self, values, values_params):
+    def impl_set_choice_values_params(self, values, values_params, session):
         self._choice_values = values
         if values_params is not None:
-            self._choice_values_params = values_params
+            for key, params in values_params.items():
+                choice = _ChoiceParam(self, key)
+                session.add(choice)
+                session.commit()
+                for param in params:
+                    if isinstance(param, tuple):
+                        if param == (None,):
+                            session.add(_ChoiceParamOption(choice))
+                        else:
+                            session.add(_ChoiceParamOption(choice, option=param[0], force_permissive=param[1]))
+                    else:
+                        session.add(_ChoiceParamOption(choice, value=param))
+        session.commit()
+
+    def impl_get_choice_values_params(self):
+        session = self.getsession()
+        params = {}
+        for param in session.query(_ChoiceParam).filter_by(option=self.id).all():
+            _params = []
+            for _param in session.query(_ChoiceParamOption).filter_by(choice=param.id).all():
+                if _param.value:
+                    _params.append(_param.value)
+                elif _param.option:
+                    _params.append((session.query(_Base).filter_by(id=_param.option).first(),
+                                         _param.force_permissive))
+                else:
+                    _params.append((None,))
+            params[param.key] = _params
+        return params
 
     def _set_validator(self, validator, validator_params):
         self._validator = validator
@@ -425,10 +600,11 @@ class _Base(SqlAlchemyBase):
             self._validator_params = validator_params
 
     def impl_is_readonly(self):
-        try:
+        session = self.getsession()
-            return self._readonly
+        opt = session.query(_Base).filter_by(id=self.id).first()
-        except AttributeError:
+        if opt is None or opt._readonly is None:
             return False
+        return opt._readonly
 
     def impl_is_multi(self):
         return self._multi == 0 or self._multi == 2
@@ -559,6 +735,9 @@ class StorageOptionDescription(object):
 
     def impl_build_cache_option(self, descr=None, _currpath=None,
                                 subdyn_path=None, session=None):
+        if self.impl_is_readonly() or (_currpath is None and getattr(self, '_cache_paths', None) is not None):
+            # cache already set
+            return
         if descr is None:
             save = True
             descr = self