Merge remote-tracking branch 'official/master' into develop

tests/test_duplicate_config.py

@@ -59,6 +59,16 @@ async def test_copy():
     assert not await list_sessions()
 
 
+@pytest.mark.asyncio
+async def test_copy_information():
+    od = make_description()
+    async with await Config(od) as cfg:
+        await cfg.information.set('key', 'value')
+        async with await cfg.config.copy() as ncfg:
+            assert await ncfg.information.get('key') == 'value'
+    assert not await list_sessions()
+
+
 @pytest.mark.asyncio
 async def test_copy_force_store_value():
     od = make_description()

tests/test_option_callback.py

@@ -10,7 +10,7 @@ from tiramisu.setting import groups, owners
 from tiramisu import ChoiceOption, BoolOption, IntOption, FloatOption, \
     StrOption, OptionDescription, SymLinkOption, IPOption, NetmaskOption, Leadership, \
     undefined, Calculation, Params, ParamOption, ParamValue, ParamIndex, calc_value, \
-    valid_ip_netmask, ParamSelfOption
+    valid_ip_netmask, ParamSelfOption, ParamInformation
 from tiramisu.error import PropertiesOptionError, ConflictError, LeadershipError, ConfigError
 from tiramisu.i18n import _
 from tiramisu.storage import list_sessions
@@ -43,6 +43,10 @@ def return_value(value=None):
     return value
 
 
+async def return_async_value(value=None):
+    return value
+
+
 def return_value2(*args, **kwargs):
     value = list(args)
     value.extend(kwargs.values())
@@ -333,6 +337,50 @@ async def test_callback_value(config_type):
     assert not await list_sessions()
 
 
+@pytest.mark.asyncio
+async def test_callback_async_value(config_type):
+    val1 = StrOption('val1', "", 'val')
+    val2 = StrOption('val2', "", Calculation(return_async_value, Params(ParamOption(val1))))
+    val3 = StrOption('val3', "", Calculation(return_async_value, Params(ParamValue('yes'))))
+    val4 = StrOption('val4', "", Calculation(return_async_value, Params(kwargs={'value': ParamOption(val1)})))
+    val5 = StrOption('val5', "", Calculation(return_async_value, Params(ParamValue('yes'))))
+    maconfig = OptionDescription('rootconfig', '', [val1, val2, val3, val4, val5])
+    async with await Config(maconfig) as cfg:
+        await cfg.property.read_write()
+        cfg = await get_config(cfg, config_type)
+        assert await cfg.option('val1').value.get() == 'val'
+        assert await cfg.option('val2').value.get() == 'val'
+        assert await cfg.option('val4').value.get() == 'val'
+        await cfg.option('val1').value.set('new-val')
+        assert await cfg.option('val1').value.get() == 'new-val'
+        assert await cfg.option('val2').value.get() == 'new-val'
+        assert await cfg.option('val4').value.get() == 'new-val'
+        await cfg.option('val1').value.reset()
+        assert await cfg.option('val1').value.get() == 'val'
+        assert await cfg.option('val2').value.get() == 'val'
+        assert await cfg.option('val3').value.get() == 'yes'
+        assert await cfg.option('val4').value.get() == 'val'
+        assert await cfg.option('val5').value.get() == 'yes'
+    assert not await list_sessions()
+
+
+@pytest.mark.asyncio
+async def test_callback_information(config_type):
+    val1 = StrOption('val1', "", Calculation(return_value, Params(ParamInformation('information', 'no_value'))))
+    val2 = StrOption('val2', "", Calculation(return_value, Params(ParamInformation('information'))))
+    maconfig = OptionDescription('rootconfig', '', [val1, val2])
+    async with await Config(maconfig) as cfg:
+        await cfg.property.read_write()
+        cfg = await get_config(cfg, config_type)
+        assert await cfg.option('val1').value.get() == 'no_value'
+        with pytest.raises(ConfigError):
+            await cfg.option('val2').value.get()
+        await cfg.information.set('information', 'new_value')
+        assert await cfg.option('val1').value.get() == 'new_value'
+        assert await cfg.option('val2').value.get() == 'new_value'
+    assert not await list_sessions()
+
+
 @pytest.mark.asyncio
 async def test_callback_value_tuple(config_type):
     val1 = StrOption('val1', "", 'val1')

tiramisu/__init__.py

@@ -18,7 +18,7 @@ from .function import calc_value, calc_value_property_help, valid_ip_netmask, \
                       valid_network_netmask, valid_in_network, valid_broadcast, \
                       valid_not_equal
 from .autolib import Calculation, Params, ParamOption, ParamDynOption, ParamSelfOption, \
-                     ParamValue, ParamIndex, ParamSuffix
+                     ParamValue, ParamIndex, ParamSuffix, ParamInformation
 from .option import *
 from .error import APIError
 from .api import Config, MetaConfig, GroupConfig, MixConfig
@@ -36,6 +36,7 @@ allfuncs = ['Calculation',
             'ParamValue',
             'ParamIndex',
             'ParamSuffix',
+            'ParamInformation',
             'MetaConfig',
             'MixConfig',
             'GroupConfig',

tiramisu/api.py

@@ -87,8 +87,8 @@ class CommonTiramisu(TiramisuHelp):
 
     async def _get_option(self,
                           connection) -> Any:
+        config_bag = self._option_bag.config_bag
         if not self._subconfig:
-            config_bag = self._option_bag.config_bag
             try:
                 subconfig, name = await config_bag.context.cfgimpl_get_home_by_path(self._option_bag.path,
                                                                                     config_bag,
@@ -101,7 +101,7 @@ class CommonTiramisu(TiramisuHelp):
             self._name = name
         option = self._option_bag.option
         if option is None:
-            option = await self._subconfig.cfgimpl_get_description().get_child(name,
+            option = await self._subconfig.cfgimpl_get_description().get_child(self._name,
                                                                                config_bag,
                                                                                self._subconfig.cfgimpl_get_path())
             self._option_bag.option = option
@@ -885,24 +885,35 @@ def connection(func):
 class TiramisuContextInformation(TiramisuConfig):
     """Manage config informations"""
     @connection
-    async def get(self, name, default=undefined):
+    async def get(self,
+                  name,
+                  default=undefined,
+                  ):
         """Get an information"""
         return await self._config_bag.context.impl_get_information(self._config_bag.connection,
                                                                    name,
-                                                                   default)
+                                                                   default,
+                                                                   )
 
     @connection
-    async def set(self, name, value):
+    async def set(self,
+                  name,
+                  value,
+                  ):
         """Set an information"""
-        await self._config_bag.context.impl_set_information(self._config_bag.connection,
+        await self._config_bag.context.impl_set_information(self._config_bag,
                                                             name,
-                                                            value)
+                                                            value,
+                                                            )
 
     @connection
-    async def reset(self, name):
+    async def reset(self,
+                    name,
+                    ):
         """Remove an information"""
         await self._config_bag.context.impl_del_information(self._config_bag.connection,
-                                                            name)
+                                                            name,
+                                                            )
 
     @connection
     async def list(self):

tiramisu/autolib.py

@@ -19,6 +19,7 @@
 # ____________________________________________________________
 "enables us to carry out a calculation and return an option's value"
 from typing import Any, Optional, Union, Callable, Dict, List
+from types import CoroutineType
 from itertools import chain
 
 from .error import PropertiesOptionError, ConfigError, LeadershipError, ValueWarning
@@ -82,8 +83,7 @@ class ParamOption(Param):
 
 
 class ParamDynOption(ParamOption):
-    __slots__ = ('suffix',
+    __slots__ = ('suffix',)
-                 )
     def __init__(self,
                  option: 'Option',
                  suffix: str,
@@ -118,6 +118,16 @@ class ParamValue(Param):
         self.value = value
 
 
+class ParamInformation(Param):
+    __slots__ = ('information_name',)
+    def __init__(self,
+                 information_name: str,
+                 default_value: Any=undefined,
+                 ) -> None:
+        self.information_name = information_name
+        self.default_value = default_value
+
+
 class ParamIndex(Param):
     __slots__ = tuple()
 
@@ -199,7 +209,7 @@ class Break(Exception):
     pass
 
 
-async def manager_callback(callbk: Union[ParamOption, ParamValue],
+async def manager_callback(callbk: Param,
                            option,
                            index: Optional[int],
                            orig_value,
@@ -286,12 +296,23 @@ async def manager_callback(callbk: Union[ParamOption, ParamValue],
     if isinstance(callbk, ParamValue):
         return callbk.value
 
+    if isinstance(callbk, ParamInformation):
+        try:
+            return await config_bag.context.impl_get_information(config_bag.connection,
+                                                                 callbk.information_name,
+                                                                 callbk.default_value,
+                                                                 )
+        except ValueError as err:
+            raise ConfigError(_('option "{}" cannot be calculated: {}').format(option.impl_get_display_name(),
+                                                                               str(err),
+                                                                               ))
+
     if isinstance(callbk, ParamIndex):
         return index
 
     if isinstance(callbk, ParamSuffix):
         if not option.issubdyn():
-            raise ConfigError('option "{}" is not in a dynoptiondescription'.format(option.impl_get_display_name()))
+            raise ConfigError(_('option "{}" is not in a dynoptiondescription').format(option.impl_get_display_name()))
         return option.impl_getsuffix()
 
     if isinstance(callbk, ParamSelfOption):
@@ -413,12 +434,12 @@ async def carry_out_calculation(option,
                         kwargs[key] = {'propertyerror': str(err)}
             except Break:
                 continue
-    ret = calculate(option,
+    ret = await calculate(option,
-                    callback,
+                          callback,
-                    allow_value_error,
+                          allow_value_error,
-                    force_value_warning,
+                          force_value_warning,
-                    args,
+                          args,
-                    kwargs)
+                          kwargs)
     if isinstance(ret, list) and not option.impl_is_dynoptiondescription() and \
             option.impl_is_follower() and not option.impl_is_submulti():
         if args or kwargs:
@@ -439,13 +460,13 @@ async def carry_out_calculation(option,
     return ret
 
 
-def calculate(option,
+async def calculate(option,
-              callback: Callable,
+                    callback: Callable,
-              allow_value_error: bool,
+                    allow_value_error: bool,
-              force_value_warning: bool,
+                    force_value_warning: bool,
-              args,
+                    args,
-              kwargs,
+                    kwargs,
-              ):
+                    ):
     """wrapper that launches the 'callback'
 
     :param callback: callback function
@@ -454,7 +475,10 @@ def calculate(option,
 
     """
     try:
-        return callback(*args, **kwargs)
+        ret = callback(*args, **kwargs)
+        if isinstance(ret, CoroutineType):
+            ret = await ret
+        return ret
     except (ValueError, ValueWarning) as err:
         if allow_value_error:
             if force_value_warning:

tiramisu/config.py

@@ -64,11 +64,11 @@ class SubConfig:
                 (not isinstance(descr, (BaseOption, SynDynOptionDescription)) or
                  not descr.impl_is_optiondescription()):
             try:
-                msg = descr.impl_get_displayname()
+                msg = descr.impl_get_display_name()
             except AttributeError:
                 msg = descr
-            raise TypeError(_('"{0}" must be an optiondescription, not an {1}'
+            raise TypeError(_('cannot create a sub config for "{0}" this is a "{1}", not an "OptionDescription"'
-                              ).format(msg, type(descr)))
+                              ).format(msg, descr.__class__.__name__))
         self._impl_descr = descr
         self._impl_context = context
         self._impl_path = subpath
@@ -562,17 +562,24 @@ class _CommonConfig(SubConfig):
 
     # information
     async def impl_set_information(self,
-                                   connection,
+                                   config_bag,
                                    key,
-                                   value):
+                                   value,
+                                   ):
         """updates the information's attribute
 
         :param key: information's key (ex: "help", "doc"
         :param value: information's value (ex: "the help string")
         """
-        await self._impl_values.set_information(connection,
+        await self._impl_values.set_information(config_bag.connection,
                                                 key,
                                                 value)
+        for option in config_bag.context.cfgimpl_get_description()._cache_dependencies_information.get(key, []):
+            option_bag = OptionBag()
+            option_bag.set_option(option,
+                                  None,
+                                  config_bag)
+            await config_bag.context.cfgimpl_reset_cache(option_bag)
 
     async def impl_get_information(self,
                                    connection,
@@ -647,6 +654,9 @@ class _CommonConfig(SubConfig):
         duplicated_settings = duplicated_config.cfgimpl_get_settings()
         await duplicated_values._p_.importation(connection,
                                                 await self.cfgimpl_get_values()._p_.exportation(connection))
+        await duplicated_values._p_.importation_informations(connection,
+                                                             await self.cfgimpl_get_values()._p_.exportation_informations(connection),
+                                                             )
         properties = await self.cfgimpl_get_settings()._p_.exportation(connection)
         await duplicated_settings._p_.importation(connection,
                                                   properties)

tiramisu/option/choiceoption.py

@@ -76,17 +76,18 @@ class ChoiceOption(Option):
         if isinstance(self._choice_values, Calculation):
             return
         values = self._choice_values
-        if values is not undefined and value not in values:
+        self.validate_values(value, values)
-            if len(values) == 1:
-                raise ValueError(_('only "{0}" is allowed'
-                                   '').format(values[0]))
-            raise ValueError(_('only {0} are allowed'
-                               '').format(display_list(values, add_quote=True)))
 
     async def validate_with_option(self,
                                    value: Any,
                                    option_bag: OptionBag) -> None:
         values = await self.impl_get_values(option_bag)
+        self.validate_values(value, values)
+
+    def validate_values(self,
+                        value,
+                        values,
+                        ) -> None:
         if values is not undefined and value not in values:
             if len(values) == 1:
                 raise ValueError(_('only "{0}" is allowed'

tiramisu/option/option.py

@@ -27,7 +27,7 @@ from itertools import chain
 from .baseoption import BaseOption, submulti, STATIC_TUPLE
 from ..i18n import _
 from ..setting import undefined, OptionBag, Undefined
-from ..autolib import Calculation, Params, ParamValue, ParamOption
+from ..autolib import Calculation, Params, ParamOption, ParamInformation
 from ..error import (ConfigError, ValueWarning, ValueErrorWarning, PropertiesOptionError,
                      ValueOptionError, display_list)
 from .syndynoption import SynDynOption
@@ -50,6 +50,7 @@ class Option(BaseOption):
                  #
                  '_validators',
                  #
+                 '_dependencies_information',
                  '_leadership',
                  '_choice_values',
                  '_choice_values_params',
@@ -66,6 +67,7 @@ class Option(BaseOption):
                  warnings_only: bool=False,
                  extra: Optional[Dict]=None):
         _setattr = object.__setattr__
+        _dependencies_information = []
         if not multi and default_multi is not None:
             raise ValueError(_("default_multi is set whereas multi is False"
                                " in option: {0}").format(name))
@@ -105,6 +107,8 @@ class Option(BaseOption):
                     if isinstance(param, ParamOption):
                         param.option._add_dependency(self)
                         self._has_dependency = True
+                    elif isinstance(param, ParamInformation):
+                        _dependencies_information.append(param.information_name)
 
                 self._validators = tuple(validators)
         if extra is not None and extra != {}:
@@ -155,29 +159,37 @@ class Option(BaseOption):
         self.sync_impl_validate(default,
                                 option_bag,
                                 check_error=False)
-        self.value_dependencies(default)
+        self.value_dependencies(default, _dependencies_information)
         if (is_multi and default != []) or \
                 (not is_multi and default is not None):
             if is_multi and isinstance(default, list):
                 default = tuple(default)
             _setattr(self, '_default', default)
+        if _dependencies_information:
+            self._dependencies_information = _dependencies_information
 
     def value_dependencies(self,
-                           value: Any) -> Any:
+                           value: Any,
+                           _dependencies_information: List[str],
+                           ) -> Any:
         if isinstance(value, list):
             for val in value:
                 if isinstance(value, list):
-                    self.value_dependencies(val)
+                    self.value_dependencies(val, _dependencies_information)
                 elif isinstance(value, Calculation):
-                    self.value_dependency(val)
+                    self.value_dependency(val, _dependencies_information)
         elif isinstance(value, Calculation):
-            self.value_dependency(value)
+            self.value_dependency(value, _dependencies_information)
 
     def value_dependency(self,
-                         value: Any) -> Any:
+                         value: Any,
+                         _dependencies_information: List[str],
+                         ) -> Any:
         for param in chain(value.params.args, value.params.kwargs.values()):
             if isinstance(param, ParamOption):
                 param.option._add_dependency(self)
+            elif isinstance(param, ParamInformation):
+                _dependencies_information.append(param.information_name)
 
     #__________________________________________________________________________
     # option's information
@@ -191,6 +203,9 @@ class Option(BaseOption):
     def impl_is_dynsymlinkoption(self) -> bool:
         return False
 
+    def get_dependencies_information(self) -> List[str]:
+        return getattr(self, '_dependencies_information', [])
+
     def get_type(self) -> str:
         # _display_name for compatibility with older version than 3.0rc3
         return getattr(self, '_type', self._display_name)

tiramisu/option/optiondescription.py

@@ -30,7 +30,9 @@ from ..error import ConfigError, ConflictError
 
 
 class CacheOptionDescription(BaseOption):
-    __slots__ = ('_cache_force_store_values',)
+    __slots__ = ('_cache_force_store_values',
+                 '_cache_dependencies_information',
+                 )
 
     def impl_already_build_caches(self) -> bool:
         return self.impl_is_readonly()
@@ -42,7 +44,9 @@ class CacheOptionDescription(BaseOption):
                            currpath: List[str]=None,
                            cache_option=None,
                            force_store_values=None,
-                           display_name=None) -> None:
+                           dependencies_information=None,
+                           display_name=None,
+                           ) -> None:
         """validate options and set option has readonly option
         """
         # _consistencies is None only when we start to build cache
@@ -52,6 +56,7 @@ class CacheOptionDescription(BaseOption):
             if __debug__:
                 cache_option = []
             force_store_values = []
+            dependencies_information = {}
             currpath = []
         else:
             init = False
@@ -73,8 +78,11 @@ class CacheOptionDescription(BaseOption):
                                           sub_currpath,
                                           cache_option,
                                           force_store_values,
+                                          dependencies_information,
                                           display_name)
             else:
+                for information in option.get_dependencies_information():
+                    dependencies_information.setdefault(information, []).append(option)
                 is_multi = option.impl_is_multi()
                 if not option.impl_is_symlinkoption():
                     properties = option.impl_getproperties()
@@ -102,6 +110,7 @@ class CacheOptionDescription(BaseOption):
             option._set_readonly()
         if init:
             self._cache_force_store_values = force_store_values
+            self._cache_dependencies_information = dependencies_information
             self._path = self._name
             self._set_readonly()
 

tiramisu/storage/dictionary/value.py

@@ -25,14 +25,11 @@ from copy import deepcopy
 
 class Values:
     __slots__ = ('_values',
-                 '_informations',
                  '_storage',
                  '__weakref__')
     def __init__(self, storage):
         """init plugin means create values storage
         """
-        #self._values = ([], [], [], [])
-        #self._informations = {}
         self._storage = storage
 
     def _setvalue_info(self, nb, idx, value, index, follower_idx=None):
@@ -296,6 +293,18 @@ class Values:
                                connection):
         self._storage.set_informations({})
 
+    async def exportation_informations(self,
+                                       connection,
+                                       ):
+        return deepcopy(self._storage.get_informations())
+
+    async def importation_informations(self,
+                                       connection,
+                                       informations,
+                                       ):
+        #deepcopy(informations)
+        return self._storage.set_informations(informations)
+
     async def exportation(self,
                           connection):
         return deepcopy(self._storage.get_values())

tiramisu/storage/postgres/value.py

@@ -231,6 +231,24 @@ class Values:
         await connection.execute("DELETE FROM information WHERE session_id = $1",
                                  self._storage.database_id)
 
+    async def exportation_informations(self,
+                                       connection,
+                                       ):
+        informations = {}
+        for path, key, value in await connection.fetch("SELECT path, key, value FROM information WHERE session_id = $1", self._storage.database_id):
+            path = self._storage.load_path(path)
+            informations.setdefault(path, {})[key] = loads(value)
+        return informations
+
+    async def importation_informations(self,
+                                       connection,
+                                       informations,
+                                       ):
+        for path, path_infos in informations.items():
+            for key, value in path_infos.items():
+                await connection.execute("INSERT INTO information(key, value, session_id, path) VALUES "
+                                         "($1, $2, $3, $4)", key, dumps(value), self._storage.database_id, path)
+
     async def exportation(self,
                           connection):
         # log.debug('exportation')