Merge remote-tracking branch 'official/master' into develop

tests/test_dyn_optiondescription.py

@@ -260,9 +260,31 @@ async def test_prop_dyndescription_force_store_value():
     dod = DynOptionDescription('dod', '', [st], suffixes=Calculation(return_list))
     od = OptionDescription('od', '', [dod])
     od2 = OptionDescription('od', '', [od])
-    with pytest.raises(ConfigError):
+    async with await Config(od2) as cfg:
-        await Config(od2, session_id='error')
+        await cfg.property.read_write()
-    await delete_session('error')
+        assert await cfg.value.dict() == {'od.dodval1.stval1': None, 'od.dodval2.stval2': None}
+    assert not await list_sessions()
+
+
+@pytest.mark.asyncio
+async def test_prop_dyndescription_force_store_value_calculation_prefix():
+    lst = StrOption('lst', '', ['val1', 'val2'], multi=True)
+    st = StrOption('st', '', Calculation(return_list, Params(ParamSuffix())) , properties=('force_store_value',))
+    dod = DynOptionDescription('dod', '', [st], suffixes=Calculation(return_list, Params(ParamOption(lst))))
+    od = OptionDescription('od', '', [dod, lst])
+    od2 = OptionDescription('od', '', [od])
+    async with await Config(od2) as cfg:
+        await cfg.property.read_write()
+        assert await cfg.option('od.dodval1.stval1').owner.isdefault() == False
+        assert await cfg.option('od.dodval2.stval2').owner.isdefault() == False
+        assert await cfg.value.dict() == {'od.lst': ['val1', 'val2'], 'od.dodval1.stval1': 'val1', 'od.dodval2.stval2': 'val2'}
+        #
+        await cfg.option('od.lst').value.set(['val1', 'val2', 'val3'])
+        assert await cfg.option('od.dodval3.stval3').owner.isdefault() == False
+        assert await cfg.option('od.dodval1.stval1').owner.isdefault() == False
+        assert await cfg.option('od.dodval2.stval2').owner.isdefault() == False
+        assert await cfg.value.dict() == {'od.lst': ['val1', 'val2', 'val3'], 'od.dodval1.stval1': 'val1', 'od.dodval2.stval2': 'val2', 'od.dodval3.stval3': 'val3'}
+
     assert not await list_sessions()
 
 

tests/test_metaconfig.py

@@ -1130,6 +1130,39 @@ async def test_meta_properties_meta_deepcopy():
     await delete_sessions([meta, meta2])
 
 
+@pytest.mark.asyncio
+async def test_meta_properties_meta_deepcopy_multi_parent():
+    ip_admin_eth0 = NetworkOption('ip_admin_eth0', "ip")
+    netmask_admin_eth0 = NetmaskOption('netmask_admin_eth0', "mask")
+    interface1 = OptionDescription('ip_admin_eth0', '', [ip_admin_eth0, netmask_admin_eth0])
+    conf1 = await Config(interface1, session_id='conf1')
+    conf2 = await Config(interface1, session_id='conf2')
+    await conf1.property.read_write()
+    await conf2.property.read_write()
+    meta1 = await MetaConfig([conf1, conf2], session_id='meta1')
+    await meta1.permissive.add('hidden')
+    await meta1.property.read_write()
+
+    meta2 = await MetaConfig(['name1', 'name2'], optiondescription=interface1, session_id='meta2')
+    await meta2.config.add(conf1)
+
+    await meta1.option('ip_admin_eth0').value.set('192.168.1.1')
+    await meta2.option('netmask_admin_eth0').value.set('255.255.255.0')
+
+    assert await meta1.value.dict() == {'ip_admin_eth0': '192.168.1.1', 'netmask_admin_eth0': None}
+    assert await meta2.value.dict() == {'ip_admin_eth0': None, 'netmask_admin_eth0': '255.255.255.0'}
+    assert await conf1.value.dict() == {'ip_admin_eth0': '192.168.1.1', 'netmask_admin_eth0': '255.255.255.0'}
+    assert await conf2.value.dict() == {'ip_admin_eth0': '192.168.1.1', 'netmask_admin_eth0': None}
+
+    copy_meta2 = await conf1.config.deepcopy(session_id='copy_conf1', metaconfig_prefix='copy_')
+    assert await copy_meta2.config.path() == 'copy_meta2'
+    copy_meta1 = await copy_meta2.config('copy_meta1')
+    copy_conf1 = await copy_meta1.config('copy_conf1')
+    assert await copy_meta2.value.dict() == {'ip_admin_eth0': None, 'netmask_admin_eth0': '255.255.255.0'}
+    assert await copy_conf1.value.dict() == {'ip_admin_eth0': '192.168.1.1', 'netmask_admin_eth0': '255.255.255.0'}
+    await delete_sessions([conf1, conf2, meta1, meta2, copy_conf1, copy_meta1, copy_meta2])
+
+
 @pytest.mark.asyncio
 async def test_meta_properties_submeta_deepcopy():
     ip_admin_eth0 = NetworkOption('ip_admin_eth0', "ip", multi=True, default=['192.168.1.1'])

tests/test_option.py

@@ -31,6 +31,18 @@ async def test_option_valid_name():
     with pytest.raises(ValueError):
         SymLinkOption(1, i)
     i = SymLinkOption("test1", i)
+#
+#
+#@pytest.mark.asyncio
+#async def test_option_unvalid_name():
+#    with pytest.raises(ValueError):
+#        IntOption('test.', '')
+#    with pytest.raises(ValueError):
+#        IntOption('test.val', '')
+#    with pytest.raises(ValueError):
+#        IntOption('.test', '')
+#    with pytest.raises(ValueError):
+#        OptionDescription('.test', '', [])
 
 
 @pytest.mark.asyncio

tiramisu/api.py

@@ -484,7 +484,8 @@ class TiramisuOptionInformation(CommonTiramisuOption):
 
     @option_and_connection
     async def reset(self,
-              key):
+                    key: str,
+                    ) -> None:
         """Remove information"""
         path = self._option_bag.path
         values = self._option_bag.config_bag.context.cfgimpl_get_values()
@@ -921,6 +922,16 @@ class TiramisuContextInformation(TiramisuConfig):
         """List information's keys"""
         return await self._config_bag.context.impl_list_information(self._config_bag.connection)
 
+    @connection
+    async def exportation(self):
+        """Export all informations"""
+        return await self._config_bag.context.cfgimpl_get_values()._p_.exportation_informations(self._config_bag.connection)
+
+    @connection
+    async def importation(self, informations):
+        """Import informations"""
+        return await self._config_bag.context.cfgimpl_get_values()._p_.importation_informations(self._config_bag.connection, informations)
+
 
 class TiramisuContextValue(TiramisuConfig):
     """Manage config value"""

tiramisu/autolib.py

@@ -358,10 +358,11 @@ async def manager_callback(callbk: Param,
                 suffix = callbk.suffix
             else:
                 if not option.impl_is_dynsymlinkoption():
-                    msg = 'option "{}" is not dynamic in callback of the option "{}"'
+                    msg = 'option "{}" is not dynamic but is an argument of the dynamic option "{}" in a callback'
-                    raise ConfigError(_(msg).format(callbk_option.impl_get_display_name(),
+                    raise ConfigError(_(msg).format(option.impl_get_display_name(),
-                                                    option.impl_get_display_name(),
+                                                    callbk_option.impl_get_display_name(),
                                                     ))
+                #FIXME in same dynamic option?
                 rootpath = option.rootpath
                 suffix = option.impl_getsuffix()
                 subdyn = callbk_option.getsubdyn()

tiramisu/config.py

@@ -173,7 +173,8 @@ class SubConfig:
 
     async def cfgimpl_reset_cache(self,
                                   option_bag,
-                                  resetted_opts=None):
+                                  resetted_opts=None,
+                                  ):
         """reset all settings in cache
         """
         if resetted_opts is None:

tiramisu/function.py

@@ -333,14 +333,39 @@ class CalcValue:
                         value = value[index]
                     else:
                         value = None
-        elif None in value and not allow_none:
+        else:
-            value = []
+            if join is not None:
-        elif remove_duplicate_value:
+                if None not in value:
-            new_value = []
+                    length_val = None
-            for val in value:
+                    for val in value:
-                if val not in new_value:
+                        if isinstance(val, list):
-                    new_value.append(val)
+                            if None in val:
-            value = new_value
+                                length_val = None
+                                break
+                            lval = len(val)
+                            if length_val is not None and length_val != lval:
+                                raise ValueError(_(f'unexpected value in calc_value with join attribute "{val}" with invalid length "{length_val}"'))
+                            length_val = lval
+                    new_value = []
+                    for idx in range(length_val):
+                        idx_val = []
+                        for val in value:
+                            if isinstance(val, list):
+                                idx_val.append(val[idx])
+                            else:
+                                idx_val.append(val)
+                        new_value.append(join.join(idx_val))
+                    value = new_value
+                else:
+                    value = []
+            elif None in value and not allow_none:
+                value = []
+            elif remove_duplicate_value:
+                new_value = []
+                for val in value:
+                    if val not in new_value:
+                        new_value.append(val)
+                value = new_value
         return value
 
     def value_from_kwargs(self,

tiramisu/option/baseoption.py

@@ -40,6 +40,8 @@ submulti = 2
 def valid_name(name):
     if not isinstance(name, str):
         return False
+#    if '.' in name:
+#        return False
     return True
 
 
@@ -55,6 +57,7 @@ class Base:
                  '_properties',
                  '_has_dependency',
                  '_dependencies',
+                 '_suffixes_dependencies',
                  '__weakref__'
                 )
 
@@ -99,18 +102,29 @@ class Base:
         return hasattr(self, '_dependencies')
 
     def _get_dependencies(self,
-                          context_od) -> Set[str]:
+                          context_od,
+                          ) -> Set[str]:
         ret = set(getattr(self, '_dependencies', STATIC_TUPLE))
         if context_od and hasattr(context_od, '_dependencies'):
             # add options that have context is set in calculation
             return set(context_od._dependencies) | ret
         return ret
 
+    def _get_suffixes_dependencies(self) -> Set[str]:
+        return getattr(self, '_suffixes_dependencies', STATIC_TUPLE)
+
     def _add_dependency(self,
-                        option) -> None:
+                        option,
+                        is_suffix: bool=False,
+                        ) -> None:
+        woption = weakref.ref(option)
         options = self._get_dependencies(None)
         options.add(weakref.ref(option))
         self._dependencies = tuple(options)
+        if is_suffix:
+            options = list(self._get_suffixes_dependencies())
+            options.append(weakref.ref(option))
+            self._suffixes_dependencies = tuple(options)
 
     def _impl_set_callback(self,
                            callback: Callable,

tiramisu/option/dynoptiondescription.py

@@ -66,7 +66,9 @@ class DynOptionDescription(OptionDescription):
             raise ConfigError(_('suffixes in dynoptiondescription has to be a calculation'))
         for param in chain(suffixes.params.args, suffixes.params.kwargs.values()):
             if isinstance(param, ParamOption):
-                param.option._add_dependency(self)
+                param.option._add_dependency(self,
+                                             is_suffix=True,
+                                             )
         self._suffixes = suffixes
 
     def convert_suffix_to_path(self,

tiramisu/option/optiondescription.py

@@ -87,11 +87,6 @@ class CacheOptionDescription(BaseOption):
                 if not option.impl_is_symlinkoption():
                     properties = option.impl_getproperties()
                     if 'force_store_value' in properties:
-                        if __debug__:
-                            if option.issubdyn():
-                                raise ConfigError(_('the dynoption "{0}" cannot have '
-                                                    '"force_store_value" property').format(
-                                                        option.impl_get_display_name()))
                         force_store_values.append((subpath, option))
                     if __debug__ and ('force_default_on_freeze' in properties or \
                             'force_metaconfig_on_freeze' in properties) and \
@@ -115,7 +110,8 @@ class CacheOptionDescription(BaseOption):
             self._set_readonly()
 
     async def impl_build_force_store_values(self,
-                                            config_bag: ConfigBag) -> None:
+                                            config_bag: ConfigBag,
+                                            ) -> None:
         if 'force_store_value' not in config_bag.properties:
             return
         values = config_bag.context.cfgimpl_get_values()
@@ -143,17 +139,37 @@ class CacheOptionDescription(BaseOption):
                                                   index,
                                                   False)
                 else:
-                    option_bag = OptionBag()
+                    option_bags = []
-                    option_bag.set_option(option,
+                    if option.issubdyn():
-                                          None,
+                        dynopt = option.getsubdyn()
-                                          config_bag)
+                        rootpath = dynopt.impl_getpath()
-                    option_bag.properties = frozenset()
+                        subpaths = [rootpath] + option.impl_getpath()[len(rootpath) + 1:].split('.')[1:]
-                    await values._p_.setvalue(config_bag.connection,
+                        for suffix in await dynopt.get_suffixes(config_bag):
-                                              subpath,
+                            path_suffix = dynopt.convert_suffix_to_path(suffix)
-                                              await values.getvalue(option_bag),
+                            subpath = '.'.join([subp + path_suffix for subp in subpaths])
-                                              owners.forced,
+                            doption = option.to_dynoption(subpath,
+                                                          suffix,
+                                                          option)
+                            doption_bag = OptionBag()
+                            doption_bag.set_option(doption,
+                                                   None,
+                                                   config_bag)
+                            option_bags.append(doption_bag)
+                    else:
+                        option_bag = OptionBag()
+                        option_bag.set_option(option,
                                               None,
-                                              False)
+                                              config_bag)
+                        option_bags.append(option_bag)
+                    for option_bag in option_bags:
+                        option_bag.properties = frozenset()
+                        await values._p_.setvalue(config_bag.connection,
+                                                  option_bag.path,
+                                                  await values.getvalue(option_bag),
+                                                  owners.forced,
+                                                  None,
+                                                  False,
+                                                  )
 
 
 class OptionDescriptionWalk(CacheOptionDescription):

tiramisu/storage/postgres/value.py

@@ -240,7 +240,9 @@ class Values:
                                        connection,
                                        informations,
                                        ):
+        await self.del_informations(connection)
         for path, path_infos in informations.items():
+            path = self._storage.convert_path(path)
             for key, value in path_infos.items():
                 await connection.execute("INSERT INTO information(key, value, session_id, path) VALUES "
                                          "($1, $2, $3, $4)", key, dumps(value), self._storage.database_id, path)

tiramisu/value.py

@@ -242,6 +242,8 @@ class Values:
             return
         # calculated value is a new value, so reset cache
         await option_bag.config_bag.context.cfgimpl_reset_cache(option_bag)
+        # and manage force_store_value
+        await self._set_force_value_suffix(option_bag)
 
     async def calculate_value(self,
                               option_bag: OptionBag,
@@ -361,15 +363,58 @@ class Values:
                                     check_error=False)
 
     async def _setvalue(self,
-                        option_bag,
+                        option_bag: OptionBag,
-                        value,
+                        value: Any,
-                        owner):
+                        owner: str,
+                        ) -> None:
         await option_bag.config_bag.context.cfgimpl_reset_cache(option_bag)
         await self._p_.setvalue(option_bag.config_bag.connection,
                                 option_bag.path,
                                 value,
                                 owner,
                                 option_bag.index)
+        await self._set_force_value_suffix(option_bag)
+
+    async def _set_force_value_suffix(self,
+                                      option_bag: OptionBag,
+                                      ) -> None:
+        if 'force_store_value' not in option_bag.config_bag.properties:
+            return
+        for woption in option_bag.option._get_suffixes_dependencies():
+            option = woption()
+            force_store_options = []
+            async for coption in option.get_children_recursively(None,
+                                                                 None,
+                                                                 option_bag.config_bag,
+                                                                 ):
+                if 'force_store_value' in coption.impl_getproperties():
+                    force_store_options.append(coption)
+            if not force_store_options:
+                continue
+            rootpath = option.impl_getpath()
+            settings = option_bag.config_bag.context.cfgimpl_get_settings()
+            for suffix in await option.get_suffixes(option_bag.config_bag):
+                for coption in force_store_options:
+                    subpaths = [rootpath] + coption.impl_getpath()[len(rootpath) + 1:].split('.')[:-1]
+                    path_suffix = option.convert_suffix_to_path(suffix)
+                    subpath = '.'.join([subp + path_suffix for subp in subpaths])
+                    doption = coption.to_dynoption(subpath,
+                                                   suffix,
+                                                   coption,
+                                                   )
+                    coption_bag = OptionBag()
+                    coption_bag.set_option(doption,
+                                           None,
+                                           option_bag.config_bag,
+                                           )
+                    coption_bag.properties = await settings.getproperties(coption_bag)
+                    await self._p_.setvalue(coption_bag.config_bag.connection,
+                                            coption_bag.path,
+                                            await self.getvalue(coption_bag),
+                                            owners.forced,
+                                            None,
+                                            False,
+                                            )
 
     async def _get_modified_parent(self,
                                    option_bag: OptionBag) -> Optional[OptionBag]: