From d71018a88e0acaee5702918a8a65ae31a69217d4 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Wed, 1 Apr 2020 11:49:18 +0200 Subject: [PATCH 01/12] release 3.0rc16 --- ChangeLog | 5 +++++ tiramisu/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 72d7b45..afe7f18 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +Mon Apr 1 11:47:30 2019 +0200 Emmanuel Garette + * version 3.0 rc16 + * tiramisu is now async + * add postgresql storage + Mon Sep 2 14:10:40 2019 +0200 Emmanuel Garette * version 3.0 rc15 * add parents method to MetaConfig diff --git a/tiramisu/__init__.py b/tiramisu/__init__.py index 359fd51..5156885 100644 --- a/tiramisu/__init__.py +++ b/tiramisu/__init__.py @@ -58,4 +58,4 @@ allfuncs.extend(all_options) del(all_options) __all__ = tuple(allfuncs) del(allfuncs) -__version__ = "3.0rc15" +__version__ = "3.0rc16" From f7bd6e3a471b957ebe829c601a0ef805221bd47e Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Sat, 11 Apr 2020 13:13:35 +0200 Subject: [PATCH 02/12] add support in calculation when option is in a dynoptiondescription --- tests/test_dyn_optiondescription.py | 58 +++++++++++- tests/test_metaconfig.py | 21 +++++ tiramisu/__init__.py | 5 +- tiramisu/api.py | 24 ++++- tiramisu/autolib.py | 113 +++++++++++++++++------- tiramisu/config.py | 35 +++++++- tiramisu/option/baseoption.py | 26 ++++-- tiramisu/option/dynoptiondescription.py | 10 ++- tiramisu/option/option.py | 21 +++-- tiramisu/option/syndynoption.py | 4 +- 10 files changed, 252 insertions(+), 65 deletions(-) diff --git a/tests/test_dyn_optiondescription.py b/tests/test_dyn_optiondescription.py index 66865c1..481f7dc 100644 --- a/tests/test_dyn_optiondescription.py +++ b/tests/test_dyn_optiondescription.py @@ -9,7 +9,9 @@ from tiramisu import BoolOption, StrOption, ChoiceOption, IPOption, \ StrOption, PortOption, BroadcastOption, DomainnameOption, \ EmailOption, URLOption, UsernameOption, FilenameOption, SymLinkOption, \ OptionDescription, DynOptionDescription, SynDynOption, submulti, Leadership, \ - Config, Params, ParamOption, ParamValue, ParamSuffix, ParamSelfOption, ParamIndex, Calculation, calc_value, \ + Config, \ + Params, ParamOption, ParamValue, ParamSuffix, ParamSelfOption, ParamDynOption, ParamIndex, \ + Calculation, calc_value, \ delete_session from tiramisu.error import PropertiesOptionError, ConfigError, ConflictError from tiramisu.storage import list_sessions @@ -145,8 +147,8 @@ async def test_getdoc_dyndescription(): assert await cfg.option('od.dodval2.stval2').option.name() == 'stval2' assert await cfg.option('od.dodval1').option.name() == 'dodval1' assert await cfg.option('od.dodval2').option.name() == 'dodval2' - assert await cfg.option('od.dodval1.stval1').option.doc() == 'doc1' - assert await cfg.option('od.dodval2.stval2').option.doc() == 'doc1' + assert await cfg.option('od.dodval1.stval1').option.doc() == 'doc1val1' + assert await cfg.option('od.dodval2.stval2').option.doc() == 'doc1val2' assert await cfg.option('od.dodval1').option.doc() == 'doc2val1' assert await cfg.option('od.dodval2').option.doc() == 'doc2val2' assert not await list_sessions() @@ -289,6 +291,56 @@ async def test_callback_dyndescription(): assert not await list_sessions() +@pytest.mark.asyncio +async def test_callback_dyndescription_outside_wrong_param(): + lst = StrOption('lst', '', ['val1', 'val2'], multi=True) + st = StrOption('st', '', Calculation(return_dynval)) + dod = DynOptionDescription('dod', '', [st], suffixes=Calculation(return_list, Params(ParamOption(lst)))) + out = StrOption('out', '', Calculation(return_dynval, Params(ParamOption(st)))) + od = OptionDescription('od', '', [dod, out]) + od2 = OptionDescription('od', '', [od, lst]) + async with await Config(od2) as cfg: + with pytest.raises(ConfigError): + await cfg.value.dict() + assert not await list_sessions() + + +@pytest.mark.asyncio +async def test_callback_dyndescription_outside1(): + lst = StrOption('lst', '', ['val1', 'val2'], multi=True) + st = StrOption('st', '', Calculation(return_dynval)) + dod = DynOptionDescription('dod', '', [st], suffixes=Calculation(return_list, Params(ParamOption(lst)))) + out = StrOption('out', '', Calculation(return_dynval, Params(ParamDynOption(st, 'val1', dod)))) + od = OptionDescription('od', '', [dod, out]) + od2 = OptionDescription('od', '', [od, lst]) + async with await Config(od2) as cfg: + assert await cfg.value.dict() == {'od.dodval1.stval1': 'val', 'od.dodval2.stval2': 'val', 'od.out': 'val', 'lst': ['val1', 'val2']} + await cfg.option('od.dodval1.stval1').value.set('val1') + await cfg.option('od.dodval2.stval2').value.set('val2') + assert await cfg.value.dict() == {'od.dodval1.stval1': 'val1', 'od.dodval2.stval2': 'val2', 'od.out': 'val1', 'lst': ['val1', 'val2']} + await cfg.option('lst').value.set(['val2']) + with pytest.raises(ConfigError): + await cfg.value.dict() + await cfg.option('lst').value.set(['val1']) + assert await cfg.value.dict() == {'od.dodval1.stval1': 'val1', 'od.out': 'val1', 'lst': ['val1']} + assert not await list_sessions() + + +@pytest.mark.asyncio +async def test_callback_dyndescription_outside2(): + lst = StrOption('lst', '', ['val1', 'val2'], multi=True) + out = StrOption('out', '') + st = StrOption('st', '', Calculation(return_dynval, Params(ParamOption(out)))) + dod = DynOptionDescription('dod', '', [st], suffixes=Calculation(return_list, Params(ParamOption(lst)))) + od = OptionDescription('od', '', [dod, out]) + od2 = OptionDescription('od', '', [od, lst]) + async with await Config(od2) as cfg: + assert await cfg.value.dict() == {'od.dodval1.stval1': None, 'od.dodval2.stval2': None, 'od.out': None, 'lst': ['val1', 'val2']} + await cfg.option('od.out').value.set('val1') + assert await cfg.value.dict() == {'od.dodval1.stval1': 'val1', 'od.dodval2.stval2': 'val1', 'od.out': 'val1', 'lst': ['val1', 'val2']} + assert not await list_sessions() + + @pytest.mark.asyncio async def test_callback_list_dyndescription(): st = StrOption('st', '', Calculation(return_list2, Params(ParamSuffix())), multi=True, properties=('notunique',)) diff --git a/tests/test_metaconfig.py b/tests/test_metaconfig.py index ceb54f7..cf216f6 100644 --- a/tests/test_metaconfig.py +++ b/tests/test_metaconfig.py @@ -358,6 +358,27 @@ async def test_meta_new_config_wrong_name(): await delete_sessions(meta) +@pytest.mark.asyncio +async def test_meta_load_config(): + od = make_description() + meta = await MetaConfig(['name1', 'name2'], optiondescription=od) + assert len(list(await meta.config.list())) == 2 + await meta.config.load('name1') + assert len(list(await meta.config.list())) == 3 + await delete_sessions(meta) + + +@pytest.mark.asyncio +async def test_meta_load_config_wrong_name(): + od = make_description() + meta = await MetaConfig(['name1', 'name2'], optiondescription=od) + assert len(list(await meta.config.list())) == 2 + with pytest.raises(ConfigError): + await meta.config.load('name3') + assert len(list(await meta.config.list())) == 2 + await delete_sessions(meta) + + @pytest.mark.asyncio async def test_meta_meta_set(): meta = await make_metaconfig(double=True) diff --git a/tiramisu/__init__.py b/tiramisu/__init__.py index 5156885..793e890 100644 --- a/tiramisu/__init__.py +++ b/tiramisu/__init__.py @@ -17,8 +17,8 @@ from .function import calc_value, calc_value_property_help, valid_ip_netmask, \ valid_network_netmask, valid_in_network, valid_broadcast, \ valid_not_equal -from .autolib import Calculation, Params, ParamOption, ParamSelfOption, ParamValue, \ - ParamIndex, ParamSuffix +from .autolib import Calculation, Params, ParamOption, ParamDynOption, ParamSelfOption, \ + ParamValue, ParamIndex, ParamSuffix from .option import * from .error import APIError from .api import Config, MetaConfig, GroupConfig, MixConfig @@ -31,6 +31,7 @@ from .storage import default_storage, Storage, list_sessions, \ allfuncs = ['Calculation', 'Params', 'ParamOption', + 'ParamDynOption', 'ParamSelfOption', 'ParamValue', 'ParamIndex', diff --git a/tiramisu/api.py b/tiramisu/api.py index 366d689..2399e1b 100644 --- a/tiramisu/api.py +++ b/tiramisu/api.py @@ -1533,8 +1533,7 @@ class _TiramisuContextMixConfig(_TiramisuContextGroupConfig, _TiramisuContextCon async def new(self, session_id, storage=None, - type='config', - new=None): + type='config'): """Create and add a new config""" config = self._config_bag.context if storage is None: @@ -1545,7 +1544,26 @@ class _TiramisuContextMixConfig(_TiramisuContextGroupConfig, _TiramisuContextCon session_id=session_id, storage=storage, type_=type, - new=new) + ) + return await self._return_config(new_config, + storage) + + async def load(self, + session_id, + storage=None, + type='config', + ): + """Create and add a new config""" + config = self._config_bag.context + if storage is None: + storage = config._storage + storage_obj = await storage.get() + async with storage_obj.Connection() as connection: + new_config = await config.load_config(connection, + session_id=session_id, + storage=storage, + type_=type, + ) return await self._return_config(new_config, storage) diff --git a/tiramisu/autolib.py b/tiramisu/autolib.py index 9361518..d4ea3df 100644 --- a/tiramisu/autolib.py +++ b/tiramisu/autolib.py @@ -81,6 +81,26 @@ class ParamOption(Param): self.raisepropertyerror = raisepropertyerror +class ParamDynOption(ParamOption): + __slots__ = ('suffix', + ) + def __init__(self, + option: 'Option', + suffix: str, + dynoptiondescription: 'DynOptionDescription', + notraisepropertyerror: bool=False, + raisepropertyerror: bool=False, + todict: bool=False, + ) -> None: + super().__init__(option, + notraisepropertyerror, + raisepropertyerror, + todict, + ) + self.suffix = suffix + self.dynoptiondescription = dynoptiondescription + + class ParamSelfOption(Param): __slots__ = ('todict', 'whole') def __init__(self, @@ -214,7 +234,10 @@ async def manager_callback(callbk: Union[ParamOption, ParamValue], value = value[apply_index] return value - async def get_value(callbk, option_bag, path): + async def get_value(callbk, + option_bag, + path, + ): try: # get value value = await config_bag.context.getattr(path, @@ -227,6 +250,10 @@ async def manager_callback(callbk: Union[ParamOption, ParamValue], ', {}').format(option.impl_get_display_name(), err), err) except ValueError as err: raise ValueError(_('the option "{0}" is used in a calculation but is invalid ({1})').format(option_bag.option.impl_get_display_name(), err)) + except AttributeError as err: + raise ConfigError(_('impossible to calculate "{0}", {1}').format(option_bag.option.impl_get_display_name(), + err, + )) return value async def get_option_bag(config_bag, @@ -272,41 +299,59 @@ async def manager_callback(callbk: Union[ParamOption, ParamValue], return {'name': option.impl_get_display_name(), 'value': value} - # it's ParamOption - callbk_option = callbk.option - if callbk_option.issubdyn(): - callbk_option = callbk_option.to_dynoption(option.rootpath, - option.impl_getsuffix(), - callbk_option.getsubdyn()) - if leadership_must_have_index and callbk_option.impl_get_leadership() and index is None: - raise Break() - if config_bag is undefined: - return undefined - if index is not None and callbk_option.impl_get_leadership() and \ - callbk_option.impl_get_leadership().in_same_group(option): - if not callbk_option.impl_is_follower(): - # leader - index_ = None - with_index = True + if isinstance(callbk, ParamOption): + callbk_option = callbk.option + if callbk_option.issubdyn(): + if isinstance(callbk, ParamDynOption): + subdyn = callbk.dynoptiondescription + rootpath = subdyn.impl_getpath() + callbk.suffix + suffix = callbk.suffix + else: + if not option.impl_is_dynsymlinkoption(): + msg = 'option "{}" is not dynamic in callback of the option "{}"' + raise ConfigError(_(msg).format(callbk_option.impl_get_display_name(), + option.impl_get_display_name(), + )) + rootpath = option.rootpath + suffix = option.impl_getsuffix() + subdyn = callbk_option.getsubdyn() + callbk_option = callbk_option.to_dynoption(rootpath, + suffix, + subdyn) + if leadership_must_have_index and callbk_option.impl_get_leadership() and index is None: + raise Break() + if config_bag is undefined: + return undefined + if index is not None and callbk_option.impl_get_leadership() and \ + callbk_option.impl_get_leadership().in_same_group(option): + if not callbk_option.impl_is_follower(): + # leader + index_ = None + with_index = True + else: + # follower + index_ = index + with_index = False else: - # follower - index_ = index + index_ = None with_index = False - else: - index_ = None - with_index = False - path = callbk_option.impl_getpath() - option_bag = await get_option_bag(config_bag, - callbk_option, - index_, - False) - value = await get_value(callbk, option_bag, path) - if with_index: - value = value[index] - if not callbk.todict: - return value - return {'name': callbk_option.impl_get_display_name(), - 'value': value} + path = callbk_option.impl_getpath() + option_bag = await get_option_bag(config_bag, + callbk_option, + index_, + False) + value = await get_value(callbk, + option_bag, + path, + ) + if with_index: + value = value[index] + if not callbk.todict: + return value + return {'name': callbk_option.impl_get_display_name(), + 'value': value} + raise ConfigError(_('unknown callback type {} in option {}').format(callbk, + option.impl_get_display_name())) async def carry_out_calculation(option, diff --git a/tiramisu/config.py b/tiramisu/config.py index 8a8ed0d..d33d3cd 100644 --- a/tiramisu/config.py +++ b/tiramisu/config.py @@ -124,6 +124,18 @@ class SubConfig: await self.reset_one_option_cache(desc, resetted_opts, doption_bag) + async for coption in self.cfgimpl_get_description().get_children_recursively(None, + None, + option_bag.config_bag): + coption_bag = OptionBag() + coption_bag.set_option(coption, + option_bag.index, + option_bag.config_bag) + coption_bag.properties = await self.cfgimpl_get_settings().getproperties(coption_bag) + await self.reset_one_option_cache(option, + resetted_opts, + coption_bag, + ) elif option.issubdyn(): # it's an option in dynoptiondescription, remove cache for all generated option dynopt = option.getsubdyn() @@ -1176,13 +1188,28 @@ class KernelMixConfig(KernelGroupConfig): session_id, type_='config', storage=None, - new=None, ): - if new is None: - new = session_id not in await list_sessions() - if new and session_id in [child.impl_getname() for child in self._impl_children]: + if session_id in [child.impl_getname() for child in self._impl_children]: raise ConflictError(_('config name must be uniq in ' 'groupconfig for {0}').format(session_id)) + return await self.load_config(connection, + session_id, + type_, + storage, + new=True, + ) + + async def load_config(self, + connection, + session_id, + type_='config', + storage=None, + new=False, + ): + if not new: + if session_id not in [child.impl_getname() for child in self._impl_children]: + raise ConfigError(_('cannot find existing config with session_id to "{}"').format(session_id)) + assert type_ in ('config', 'metaconfig', 'mixconfig'), _('unknown type {}').format(type_) if type_ == 'config': config = await KernelConfig(self._impl_descr, diff --git a/tiramisu/option/baseoption.py b/tiramisu/option/baseoption.py index 902a60d..8440b2f 100644 --- a/tiramisu/option/baseoption.py +++ b/tiramisu/option/baseoption.py @@ -270,20 +270,36 @@ class BaseOption(Base): return self.impl_get_callback()[0] is not None def _impl_get_display_name(self, - dyn_name: Base=None) -> str: + dyn_name: Base=None, + suffix: str=None, + ) -> str: name = self.impl_get_information('doc', None) if name is None or name == '': if dyn_name is not None: name = dyn_name else: name = self.impl_getname() + elif suffix: + name += suffix return name - def impl_get_display_name(self, - dyn_name: Base=None) -> str: + def _get_display_name(self, + dyn_name, + suffix, + ): if hasattr(self, '_display_name_function'): - return self._display_name_function(self, dyn_name) - return self._impl_get_display_name(dyn_name) + return self._display_name_function(self, + dyn_name, + suffix, + ) + return self._impl_get_display_name(dyn_name, + suffix, + ) + + def impl_get_display_name(self) -> str: + return self._get_display_name(None, + None, + ) def reset_cache(self, path: str, diff --git a/tiramisu/option/dynoptiondescription.py b/tiramisu/option/dynoptiondescription.py index 324b031..7b04750 100644 --- a/tiramisu/option/dynoptiondescription.py +++ b/tiramisu/option/dynoptiondescription.py @@ -20,6 +20,8 @@ # ____________________________________________________________ import re from typing import List, Callable +from itertools import chain +from ..autolib import ParamOption from ..i18n import _ @@ -60,8 +62,12 @@ class DynOptionDescription(OptionDescription): 'dynoptiondescription')) child._setsubdyn(self) # add suffixes - if __debug__ and isinstance(suffixes, Calculation): - self._suffixes = suffixes + if __debug__ and not isinstance(suffixes, Calculation): + raise ConfigError(_('suffixes in dynoptiondescription has to be a calculation')) + for param in chain(suffixes.params.args, suffixes.params.kwargs.values()): + if isinstance(param, ParamOption): + param.option._add_dependency(self) + self._suffixes = suffixes def convert_suffix_to_path(self, suffix): diff --git a/tiramisu/option/option.py b/tiramisu/option/option.py index d28092f..7b42d6e 100644 --- a/tiramisu/option/option.py +++ b/tiramisu/option/option.py @@ -93,17 +93,16 @@ class Option(BaseOption): doc, properties=properties, is_multi=is_multi) - if __debug__: - if validators is not None: - if not isinstance(validators, list): - raise ValueError(_('validators must be a list of Calculation for "{}"').format(name)) - for validator in validators: - if not isinstance(validator, Calculation): - raise ValueError(_('validators must be a Calculation for "{}"').format(name)) - for param in chain(validator.params.args, validator.params.kwargs.values()): - if isinstance(param, ParamOption): - param.option._add_dependency(self) - self._has_dependency = True + if validators is not None: + if __debug__ and not isinstance(validators, list): + raise ValueError(_('validators must be a list of Calculation for "{}"').format(name)) + for validator in validators: + if __debug__ and not isinstance(validator, Calculation): + raise ValueError(_('validators must be a Calculation for "{}"').format(name)) + for param in chain(validator.params.args, validator.params.kwargs.values()): + if isinstance(param, ParamOption): + param.option._add_dependency(self) + self._has_dependency = True self._validators = tuple(validators) if extra is not None and extra != {}: diff --git a/tiramisu/option/syndynoption.py b/tiramisu/option/syndynoption.py index 2a5073a..37204a4 100644 --- a/tiramisu/option/syndynoption.py +++ b/tiramisu/option/syndynoption.py @@ -59,7 +59,9 @@ class SynDynOption: return self.opt.impl_getname() + self.suffix def impl_get_display_name(self) -> str: - return self.opt.impl_get_display_name(dyn_name=self.impl_getname()) + self.suffix + return self.opt._get_display_name(dyn_name=self.impl_getname(), + suffix=self.suffix, + ) def impl_getsuffix(self) -> str: return self.suffix From f437bb78f3ffb1ac9f322a8833b10c144dd274cd Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Tue, 21 Apr 2020 09:16:29 +0200 Subject: [PATCH 03/12] add a test with default_multi for a submulti follower --- tests/test_submulti.py | 46 +++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/tests/test_submulti.py b/tests/test_submulti.py index 2479aaf..92d5a24 100644 --- a/tests/test_submulti.py +++ b/tests/test_submulti.py @@ -5,7 +5,6 @@ import pytest import warnings -from tiramisu.api import TIRAMISU_VERSION from tiramisu.setting import groups, owners from tiramisu import StrOption, IntOption, OptionDescription, submulti, Leadership, Config, \ MetaConfig, undefined, Params, ParamOption, Calculation @@ -38,11 +37,7 @@ async def test_unknown_multi(): @pytest.mark.asyncio async def test_submulti(): multi = StrOption('multi', '', multi=submulti) - if TIRAMISU_VERSION == 2: - default_multi = 'yes' - else: - default_multi = ['yes'] - multi2 = StrOption('multi2', '', default_multi=default_multi, multi=submulti) + multi2 = StrOption('multi2', '', default_multi=['yes'], multi=submulti) multi3 = StrOption('multi3', '', default=[['yes']], multi=submulti) od = OptionDescription('od', '', [multi, multi2, multi3]) async with await Config(od) as cfg: @@ -66,11 +61,7 @@ async def test_submulti_default_multi_not_list(): @pytest.mark.asyncio async def test_append_submulti(): multi = StrOption('multi', '', multi=submulti) - if TIRAMISU_VERSION == 2: - default_multi = 'yes' - else: - default_multi = ['yes'] - multi2 = StrOption('multi2', '', default_multi=default_multi, multi=submulti) + multi2 = StrOption('multi2', '', default_multi=['yes'], multi=submulti) multi3 = StrOption('multi3', '', default=[['yes']], multi=submulti) od = OptionDescription('od', '', [multi, multi2, multi3]) async with await Config(od) as cfg: @@ -104,11 +95,7 @@ async def test_append_submulti(): @pytest.mark.asyncio async def test_append_unvalide_submulti(): multi = StrOption('multi', '', multi=submulti) - if TIRAMISU_VERSION == 2: - default_multi = 'yes' - else: - default_multi = ['yes'] - multi2 = StrOption('multi2', '', default_multi=default_multi, multi=submulti) + multi2 = StrOption('multi2', '', default_multi=['yes'], multi=submulti) multi3 = StrOption('multi3', '', default=[['yes']], multi=submulti) od = OptionDescription('od', '', [multi, multi2, multi3]) async with await Config(od) as cfg: @@ -137,11 +124,7 @@ async def test_append_unvalide_submulti(): @pytest.mark.asyncio async def test_pop_submulti(): multi = StrOption('multi', '', multi=submulti) - if TIRAMISU_VERSION == 2: - default_multi = 'yes' - else: - default_multi = ['yes'] - multi2 = StrOption('multi2', '', default_multi=default_multi, multi=submulti) + multi2 = StrOption('multi2', '', default_multi=['yes'], multi=submulti) multi3 = StrOption('multi3', '', default=[['yes']], multi=submulti) od = OptionDescription('od', '', [multi, multi2, multi3]) async with await Config(od) as cfg: @@ -267,6 +250,27 @@ async def test_values_with_leader_and_followers_submulti(): assert not await list_sessions() +@pytest.mark.asyncio +async def test_values_with_leader_and_followers_submulti_default_multi(): + ip_admin_eth0 = StrOption('ip_admin_eth0', "ip réseau autorisé", multi=True) + netmask_admin_eth0 = StrOption('netmask_admin_eth0', "masque du sous-réseau", multi=submulti, default_multi=['255.255.0.0', '0.0.0.0']) + interface1 = Leadership('ip_admin_eth0', '', [ip_admin_eth0, netmask_admin_eth0]) + maconfig = OptionDescription('toto', '', [interface1]) + async with await Config(maconfig) as cfg: + await cfg.property.read_write() + owner = await cfg.owner.get() + assert interface1.impl_get_group_type() == groups.leadership + assert await cfg.option('ip_admin_eth0.ip_admin_eth0').owner.get() == owners.default + await cfg.option('ip_admin_eth0.ip_admin_eth0').value.set(["192.168.230.145"]) + assert await cfg.option('ip_admin_eth0.ip_admin_eth0').value.get() == ["192.168.230.145"] + assert await cfg.option('ip_admin_eth0.netmask_admin_eth0', 0).value.get() == ['255.255.0.0', '0.0.0.0'] + await cfg.option('ip_admin_eth0.ip_admin_eth0').value.set(["192.168.230.145", "192.168.230.147"]) + await cfg.option('ip_admin_eth0.netmask_admin_eth0', 0).value.set(['255.255.255.0']) + assert await cfg.option('ip_admin_eth0.netmask_admin_eth0', 0).value.get() == ['255.255.255.0'] + assert await cfg.option('ip_admin_eth0.netmask_admin_eth0', 1).value.get() == ['255.255.0.0', '0.0.0.0'] + assert not await list_sessions() + + @pytest.mark.asyncio async def test_reset_values_with_leader_and_followers_submulti(): ip_admin_eth0 = StrOption('ip_admin_eth0', "ip réseau autorisé", multi=True) From 86a48ce9f1acfdd7a798551951b87fa5f60c0b11 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Tue, 21 Apr 2020 17:13:25 +0200 Subject: [PATCH 04/12] support callback with submulti --- tests/test_submulti.py | 14 ++++++++++++++ tiramisu/autolib.py | 2 +- tiramisu/value.py | 30 +++++++++++++++++++++--------- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/tests/test_submulti.py b/tests/test_submulti.py index 92d5a24..ac880b7 100644 --- a/tests/test_submulti.py +++ b/tests/test_submulti.py @@ -433,6 +433,20 @@ async def test_callback_submulti(): assert not await list_sessions() +@pytest.mark.asyncio +async def test_callback_submulti_follower(): + multi = StrOption('multi', '', multi=True) + multi2 = StrOption('multi2', '', Calculation(return_list), multi=submulti) + od = Leadership('multi', '', [multi, multi2]) + od = OptionDescription('multi', '', [od]) + async with await Config(od) as cfg: + await cfg.property.read_write() + assert await cfg.option('multi.multi').value.get() == [] + await cfg.option('multi.multi').value.set(['val']) + assert await cfg.option('multi.multi2', 0).value.get() == ['val', 'val'] + assert not await list_sessions() + + @pytest.mark.asyncio async def test_submulti_unique(): i = IntOption('int', '', multi=submulti, properties=('unique',)) diff --git a/tiramisu/autolib.py b/tiramisu/autolib.py index d4ea3df..bc1bf86 100644 --- a/tiramisu/autolib.py +++ b/tiramisu/autolib.py @@ -415,7 +415,7 @@ async def carry_out_calculation(option, args, kwargs) if isinstance(ret, list) and not option.impl_is_dynoptiondescription() and \ - option.impl_is_follower(): + option.impl_is_follower() and not option.impl_is_submulti(): if args or kwargs: raise LeadershipError(_('the "{}" function with positional arguments "{}" ' 'and keyword arguments "{}" must not return ' diff --git a/tiramisu/value.py b/tiramisu/value.py index 9de473b..73c81ce 100644 --- a/tiramisu/value.py +++ b/tiramisu/value.py @@ -198,15 +198,27 @@ class Values: # now try to get default value: value = await self.calc_value(option_bag, option_bag.option.impl_getdefault()) - if option_bag.option.impl_is_multi() and option_bag.index is not None and isinstance(value, (list, tuple)): - # if index, must return good value for this index - if len(value) > option_bag.index: - value = value[option_bag.index] - else: - # no value for this index, retrieve default multi value - # default_multi is already a list for submulti - value = await self.calc_value(option_bag, - option_bag.option.impl_getdefault_multi()) + if option_bag.index is not None and isinstance(value, (list, tuple)): + if value and option_bag.option.impl_is_submulti(): + # first index is a list, assume other data are list too + if isinstance(value[0], list): + # if index, must return good value for this index + if len(value) > option_bag.index: + value = value[option_bag.index] + else: + # no value for this index, retrieve default multi value + # default_multi is already a list for submulti + value = await self.calc_value(option_bag, + option_bag.option.impl_getdefault_multi()) + elif option_bag.option.impl_is_multi(): + # if index, must return good value for this index + if len(value) > option_bag.index: + value = value[option_bag.index] + else: + # no value for this index, retrieve default multi value + # default_multi is already a list for submulti + value = await self.calc_value(option_bag, + option_bag.option.impl_getdefault_multi()) return value async def calculate_reset_cache(self, From 2d7729612b0985a08f956a4b0cb34bbf25d2b517 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Tue, 21 Apr 2020 19:19:04 +0200 Subject: [PATCH 05/12] can load old session --- tiramisu/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tiramisu/config.py b/tiramisu/config.py index d33d3cd..f350408 100644 --- a/tiramisu/config.py +++ b/tiramisu/config.py @@ -1206,8 +1206,10 @@ class KernelMixConfig(KernelGroupConfig): storage=None, new=False, ): + if storage is None: + storage = self._storage if not new: - if session_id not in [child.impl_getname() for child in self._impl_children]: + if session_id not in await list_sessions(storage=storage): raise ConfigError(_('cannot find existing config with session_id to "{}"').format(session_id)) assert type_ in ('config', 'metaconfig', 'mixconfig'), _('unknown type {}').format(type_) From 2f125cfc8c3439136b87c87b944156db82968c5d Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Tue, 4 Aug 2020 16:33:52 +0200 Subject: [PATCH 06/12] better error for IP like 192.168.001.1 => 192.168.1.1 --- tiramisu/option/ipoption.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tiramisu/option/ipoption.py b/tiramisu/option/ipoption.py index cde3283..0ccdc20 100644 --- a/tiramisu/option/ipoption.py +++ b/tiramisu/option/ipoption.py @@ -62,7 +62,9 @@ class IPOption(StrOption): def _validate_ip(self, value): try: - ip_address(value) + new_value = str(ip_address(value)) + if value != new_value: + raise ValueError(f'should be {new_value}') except ValueError: raise ValueError() From c4572ec4090d351b5114e8e5d11a220d60f75891 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Tue, 4 Aug 2020 16:34:43 +0200 Subject: [PATCH 07/12] remove codeset to i18n --- tiramisu/i18n.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tiramisu/i18n.py b/tiramisu/i18n.py index f4d4e0e..d021f0e 100644 --- a/tiramisu/i18n.py +++ b/tiramisu/i18n.py @@ -55,7 +55,8 @@ def get_translation() -> str: trans = translation(domain=app_name, localedir=translations_path, languages=[user_locale], - codeset='UTF-8') + ) +# codeset='UTF-8') except FileNotFoundError: log.debug('cannot found translation file for langage {} in localedir {}'.format(user_locale, translations_path)) From 5c1d4afd567242c8107a7891ea7019e84d92bd67 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Tue, 4 Aug 2020 16:35:10 +0200 Subject: [PATCH 08/12] remove unconsistent test --- tests/test_freeze.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/test_freeze.py b/tests/test_freeze.py index edd2ab0..0bbaa66 100644 --- a/tests/test_freeze.py +++ b/tests/test_freeze.py @@ -187,26 +187,6 @@ async def test_force_store_value(): assert not await list_sessions() -@pytest.mark.asyncio -async def test_force_store_value_leadership_follower(): - b = IntOption('int', 'Test int option', multi=True) - c = StrOption('str', 'Test string option', multi=True, properties=('force_store_value',)) - descr = Leadership("int", "", [b, c]) - with pytest.raises(ConfigError): - cfg = await Config(descr, session_id='error') - await delete_session('error') - assert not await list_sessions() - - -#@pytest.mark.asyncio -#async def test_force_store_value_leadership(): -# b = IntOption('int', 'Test int option', multi=True, properties=('force_store_value',)) -# c = StrOption('str', 'Test string option', multi=True) -# descr = Leadership("int", "", [b, c]) -# cfg = await Config(descr) -# assert await cfg.value.get() == {'int': ('forced', ())} - - @pytest.mark.asyncio async def test_force_store_value_leadership_sub(): b = IntOption('int', 'Test int option', multi=True, properties=('force_store_value',)) From 96c76286dbfa17976b593ed360a8bb0c457a9fe4 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Tue, 4 Aug 2020 16:35:40 +0200 Subject: [PATCH 09/12] get dependencies in API --- tiramisu/api.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tiramisu/api.py b/tiramisu/api.py index 2399e1b..3d8734e 100644 --- a/tiramisu/api.py +++ b/tiramisu/api.py @@ -52,7 +52,7 @@ class TiramisuHelp: if module_name in ['forcepermissive', 'unrestraint']: force = True max_len = max(max_len, len('forcepermissive')) - elif module_name is not 'help' and not module_name.startswith('_'): + elif module_name != 'help' and not module_name.startswith('_'): modules.append(module_name) max_len = max(max_len, len(module_name)) modules.sort() @@ -212,6 +212,17 @@ class _TiramisuOptionOptionDescription(CommonTiramisuOption): """Test if option has dependency""" return self._option_bag.option.impl_has_dependency(self_is_dep) + @option_and_connection + async def dependencies(self): + """Get dependencies from this option""" + options = [] + for option in self._option_bag.option._get_dependencies(self._option_bag.config_bag.context): + options.append(TiramisuOption(option().impl_getpath(), + None, + self._option_bag.config_bag, + )) + return options + @option_and_connection async def isoptiondescription(self): """Test if option is an optiondescription""" From dc8010f0af83c66b5f0b2fb035dec240c20cff44 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Tue, 4 Aug 2020 16:43:40 +0200 Subject: [PATCH 10/12] calc_value with join parameter now work if an option is empty --- tests/test_option_callback.py | 9 ++++++--- tiramisu/function.py | 5 ++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/test_option_callback.py b/tests/test_option_callback.py index 491aafe..a0c402c 100644 --- a/tests/test_option_callback.py +++ b/tests/test_option_callback.py @@ -1433,11 +1433,14 @@ async def test_calc_value_remove_duplicate(config_type): async def test_calc_value_join(config_type): val1 = StrOption('val1', "", 'val1') val2 = StrOption('val2', "", 'val2') - val3 = StrOption('val3', "", Calculation(calc_value, Params((ParamOption(val1), ParamOption(val2)), join=ParamValue('.')))) - od = OptionDescription('root', '', [val1, val2, val3]) + val3 = StrOption('val3', "") + val4 = StrOption('val4', "", Calculation(calc_value, Params((ParamOption(val1), ParamOption(val2), ParamOption(val3)), join=ParamValue('.')))) + od = OptionDescription('root', '', [val1, val2, val3, val4]) async with await Config(od) as cfg: cfg = await get_config(cfg, config_type) - assert await cfg.value.dict() == {'val1': 'val1', 'val2': 'val2', 'val3': 'val1.val2'} + assert await cfg.value.dict() == {'val1': 'val1', 'val2': 'val2', 'val3': None, 'val4': None} + await cfg.option('val3').value.set('val3') + assert await cfg.value.dict() == {'val1': 'val1', 'val2': 'val2', 'val3': 'val3', 'val4': 'val1.val2.val3'} assert not await list_sessions() diff --git a/tiramisu/function.py b/tiramisu/function.py index f96ea5d..1779bcb 100644 --- a/tiramisu/function.py +++ b/tiramisu/function.py @@ -311,7 +311,10 @@ class CalcValue: min_args_len) if not multi: if join is not None: - value = join.join(value) + if None not in value: + value = join.join(value) + else: + value = None elif value and operator: new_value = value[0] op = {'mul': mul, From 659243ba8f4d29d82caa801da78b1543a06e9892 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Tue, 4 Aug 2020 16:49:21 +0200 Subject: [PATCH 11/12] better support for warnings_only --- tiramisu/autolib.py | 35 +++++++++++++++++++++-------------- tiramisu/option/baseoption.py | 2 +- tiramisu/option/option.py | 21 +++++++-------------- tiramisu/value.py | 3 ++- 4 files changed, 31 insertions(+), 30 deletions(-) diff --git a/tiramisu/autolib.py b/tiramisu/autolib.py index bc1bf86..8108849 100644 --- a/tiramisu/autolib.py +++ b/tiramisu/autolib.py @@ -156,7 +156,9 @@ class Calculation: option_bag: OptionBag, leadership_must_have_index: bool=False, orig_value: Any=undefined, - allow_raises=False) -> Any: + allow_value_error=False, + force_value_warning=False, + ) -> Any: return await carry_out_calculation(option_bag.option, callback=self.function, callback_params=self.params, @@ -164,7 +166,9 @@ class Calculation: config_bag=option_bag.config_bag, leadership_must_have_index=leadership_must_have_index, orig_value=orig_value, - allow_raises=allow_raises) + allow_value_error=allow_value_error, + force_value_warning=force_value_warning, + ) async def help(self, option_bag: OptionBag, @@ -361,18 +365,18 @@ async def carry_out_calculation(option, config_bag: Optional[ConfigBag], orig_value=undefined, leadership_must_have_index: bool=False, - allow_raises: int=False): + allow_value_error: bool=False, + force_value_warning: bool=False, + ): """a function that carries out a calculation for an option's value :param option: the option :param callback: the name of the callback function - :type callback: str :param callback_params: the callback's parameters (only keyword parameters are allowed) - :type callback_params: dict :param index: if an option is multi, only calculates the nth value - :type index: int - :param allow_raises: to know if carry_out_calculation is used to validate a value + :param allow_value_error: to know if carry_out_calculation can return ValueError or ValueWarning (for example if it's a validation) + :param force_value_warning: transform valueError to ValueWarning object The callback_params is a dict. Key is used to build args (if key is '') and kwargs (otherwise). Values are tuple of: @@ -411,7 +415,8 @@ async def carry_out_calculation(option, continue ret = calculate(option, callback, - allow_raises, + allow_value_error, + force_value_warning, args, kwargs) if isinstance(ret, list) and not option.impl_is_dynoptiondescription() and \ @@ -436,9 +441,11 @@ async def carry_out_calculation(option, def calculate(option, callback: Callable, - allow_raises: bool, + allow_value_error: bool, + force_value_warning: bool, args, - kwargs): + kwargs, + ): """wrapper that launches the 'callback' :param callback: callback function @@ -448,12 +455,12 @@ def calculate(option, """ try: return callback(*args, **kwargs) - except ValueError as err: - if allow_raises: + except (ValueError, ValueWarning) as err: + if allow_value_error: + if force_value_warning: + raise ValueWarning(str(err)) raise err error = err - except ValueWarning as err: - raise err except Exception as err: # import traceback # traceback.print_exc() diff --git a/tiramisu/option/baseoption.py b/tiramisu/option/baseoption.py index 8440b2f..40a3909 100644 --- a/tiramisu/option/baseoption.py +++ b/tiramisu/option/baseoption.py @@ -106,7 +106,7 @@ class Base: context_od) -> Set[str]: ret = set(getattr(self, '_dependencies', STATIC_TUPLE)) if context_od and hasattr(context_od, '_dependencies'): - # if context is set in options, add those options + # add options that have context is set in calculation return set(context_od._dependencies) | ret return ret diff --git a/tiramisu/option/option.py b/tiramisu/option/option.py index 7b42d6e..fde3fa0 100644 --- a/tiramisu/option/option.py +++ b/tiramisu/option/option.py @@ -84,7 +84,9 @@ class Option(BaseOption): is_multi = True _multi = submulti else: - raise ValueError(_('invalid multi type "{}"').format(multi)) + raise ValueError(_('invalid multi type "{}" for "{}"').format(multi, + name, + )) if _multi != 1: _setattr(self, '_multi', _multi) if multi is not False and default is None: @@ -335,7 +337,9 @@ class Option(BaseOption): if ((check_error and not calc_is_warnings_only) or (not check_error and calc_is_warnings_only)): try: - kwargs = {'allow_raises': True} + kwargs = {'allow_value_error': True, + 'force_value_warning': calc_is_warnings_only, + } if _index is not None and option_bag.index == _index: soption_bag = option_bag else: @@ -346,17 +350,6 @@ class Option(BaseOption): await validator.execute(soption_bag, leadership_must_have_index=True, **kwargs) - except ValueError as err: - if calc_is_warnings_only: - warnings.warn_explicit(ValueWarning(val, - self._display_name, - self, - '{0}'.format(err), - _index), - ValueWarning, - self.__class__.__name__, 306) - else: - raise err except ValueWarning as warn: warnings.warn_explicit(ValueWarning(val, self._display_name, @@ -364,7 +357,7 @@ class Option(BaseOption): '{0}'.format(warn), _index), ValueWarning, - self.__class__.__name__, 316) + self.__class__.__name__, 356) async def do_validation(_value, _index): diff --git a/tiramisu/value.py b/tiramisu/value.py index 73c81ce..f492d0f 100644 --- a/tiramisu/value.py +++ b/tiramisu/value.py @@ -241,7 +241,8 @@ class Values: await option_bag.config_bag.context.cfgimpl_reset_cache(option_bag) async def calculate_value(self, - option_bag: OptionBag) -> Any: + option_bag: OptionBag, + ) -> Any: # if value has callback, calculate value callback, callback_params = option_bag.option.impl_get_callback() From 50d42624ccb6ba0f55184ea2ee78fe297e2b2d5f Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Wed, 12 Aug 2020 11:55:55 +0200 Subject: [PATCH 12/12] better support of PACKAGE_DST environment variable --- setup.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index a52538c..9812bac 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,17 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from setuptools import setup, find_packages +from setuptools import setup import os from tiramisu import __version__ -PACKAGE_NAME = os.environ.get('PACKAGE_DST', 'tiramisu') +ORI_PACKAGE_NAME = 'tiramisu' +PACKAGE_NAME = os.environ.get('PACKAGE_DST', ORI_PACKAGE_NAME) + +if PACKAGE_NAME != ORI_PACKAGE_NAME: + package_dir = {PACKAGE_NAME: ORI_PACKAGE_NAME} +else: + package_dir = None setup( version=__version__, @@ -40,5 +46,6 @@ producing flexible and fast options access. This version requires Python 3.5 or later. """, include_package_data=True, - packages=find_packages(include=['tiramisu']) + package_dir=package_dir, + packages=[PACKAGE_NAME], )