Merge branch 'develop' into dist/risotto/risotto-2.8.0/develop

src/rougail/annotator.py

@@ -34,6 +34,33 @@ def mode_factory():
 modes = mode_factory()
 
 
+CONVERT_OPTION = {'number': dict(opttype="IntOption", func=int),
+                  'float': dict(opttype="FloatOption", func=float),
+                  'choice': dict(opttype="ChoiceOption"),
+                  'string': dict(opttype="StrOption"),
+                  'password': dict(opttype="PasswordOption"),
+                  'mail': dict(opttype="EmailOption"),
+                  'boolean': dict(opttype="BoolOption"),
+                  'symlink': dict(opttype="SymLinkOption"),
+                  'filename': dict(opttype="FilenameOption"),
+                  'date': dict(opttype="DateOption"),
+                  'unix_user': dict(opttype="UsernameOption"),
+                  'ip': dict(opttype="IPOption", initkwargs={'allow_reserved': True}),
+                  'local_ip': dict(opttype="IPOption", initkwargs={'private_only': True, 'warnings_only': True}),
+                  'netmask': dict(opttype="NetmaskOption"),
+                  'network': dict(opttype="NetworkOption"),
+                  'broadcast': dict(opttype="BroadcastOption"),
+                  'netbios': dict(opttype="DomainnameOption", initkwargs={'type': 'netbios', 'warnings_only': True}),
+                  'domain': dict(opttype="DomainnameOption", initkwargs={'type': 'domainname', 'allow_ip': False}),
+                  'hostname': dict(opttype="DomainnameOption", initkwargs={'type': 'hostname', 'allow_ip': False}),
+                  'web_address': dict(opttype="URLOption", initkwargs={'allow_ip': True, 'allow_without_dot': True}),
+                  'port': dict(opttype="PortOption", initkwargs={'allow_private': True}),
+                  'mac': dict(opttype="MACOption"),
+                  'cidr': dict(opttype="IPOption", initkwargs={'cidr': True}),
+                  'network_cidr': dict(opttype="NetworkOption", initkwargs={'cidr': True}),
+                  }
+
+
 # a CreoleObjSpace's attribute has some annotations
 # that shall not be present in the exported (flatened) XML
 ERASED_ATTRIBUTES = ('redefine', 'exists', 'fallback', 'optional', 'remove_check', 'namespace',
@@ -56,8 +83,6 @@ KEY_TYPE = {'variable': 'symlink',
             'URLOption': 'web_address',
             'FilenameOption': 'filename'}
 
-CONVERSION = {'number': int}
-
 FREEZE_AUTOFREEZE_VARIABLE = 'module_instancie'
 
 PROPERTIES = ('hidden', 'frozen', 'auto_freeze', 'auto_save', 'force_default_on_freeze',
@@ -108,7 +133,7 @@ class GroupAnnotator:
                     self.manage_follower(namespace,
                                          leader_family_name,
                                          variable,
-                                         leader_name,
+                                         leadership_name,
                                          follower_names,
                                          leader_space,
                                          leader_is_hidden,
@@ -126,8 +151,13 @@ class GroupAnnotator:
                         #     leader_is_hidden = True
                     else:
                         leader_space = self.objectspace.Leadership(variable.xmlfiles)
+                        if hasattr(group, 'name'):
+                            leadership_name = group.name
+                        else:
+                            leadership_name = leader_name
                         leader_is_hidden = self.manage_leader(leader_space,
                                                               leader_family_name,
+                                                              leadership_name,
                                                               leader_name,
                                                               namespace,
                                                               variable,
@@ -142,6 +172,7 @@ class GroupAnnotator:
     def manage_leader(self,
                       leader_space: 'Leadership',
                       leader_family_name: str,
+                      leadership_name: str,
                       leader_name: str,
                       namespace: str,
                       variable: 'Variable',
@@ -152,7 +183,7 @@ class GroupAnnotator:
         if variable.multi is not True:
             raise DictConsistencyError(_('the variable {} in a group must be multi').format(variable.name))
         leader_space.variable = []
-        leader_space.name = leader_name
+        leader_space.name = leadership_name
         leader_space.hidden = variable.hidden
         if variable.hidden:
             leader_is_hidden = True
@@ -167,9 +198,9 @@ class GroupAnnotator:
             leader_space.doc = variable.description
         else:
             leader_space.doc = variable.name
-        leader_path = namespace + '.' + leader_family_name + '.' + leader_name
+        leadership_path = namespace + '.' + leader_family_name + '.' + leadership_name
         self.objectspace.paths.add_family(namespace,
-                                          leader_path,
+                                          leadership_path,
                                           leader_space,
                                           )
         leader_family = self.objectspace.space.variables[namespace].family[leader_family_name]
@@ -459,7 +490,7 @@ class VariableAnnotator:
                 for value in variable.value:
                     if not hasattr(value, 'type'):
                         value.type = variable.type
-                    value.name = CONVERSION.get(value.type, str)(value.name)
+                    value.name = CONVERT_OPTION.get(value.type, {}).get('func', str)(value.name)
             for key, value in RENAME_ATTIBUTES.items():
                 setattr(variable, value, getattr(variable, key))
                 setattr(variable, key, None)
@@ -491,6 +522,11 @@ class VariableAnnotator:
                 self.objectspace.space.constraints.check.append(check)
                 variable.type = 'string'
 
+        def _valid_type(variable):
+            if variable.type not in CONVERT_OPTION:
+                xmlfiles = self.objectspace.display_xmlfiles(variable.xmlfiles)
+                raise DictConsistencyError(_(f'unvalid type "{variable.type}" for variable "{variable.name}" in {xmlfiles}'))
+
         if not hasattr(self.objectspace.space, 'variables'):
             return
         for families in self.objectspace.space.variables.values():
@@ -520,6 +556,7 @@ class VariableAnnotator:
                                                         follower,
                                                         path,
                                                         )
+                                    _valid_type(follower)
                             else:
                                 path = '{}.{}.{}'.format(namespace, normalize_family(family.name), variable.name)
                                 _convert_variable(variable,
@@ -529,6 +566,7 @@ class VariableAnnotator:
                                                     variable,
                                                     path,
                                                     )
+                                _valid_type(variable)
 
     def convert_helps(self):
         if not hasattr(self.objectspace.space, 'help'):
@@ -935,7 +973,7 @@ class ConstraintAnnotator:
                 choice = self.objectspace.choice(variable.xmlfiles)
                 try:
                     if value is not None:
-                        choice.name = CONVERSION.get(type_, str)(value)
+                        choice.name = CONVERT_OPTION[type_].get('func', str)(value)
                     else:
                         choice.name = value
                 except:
@@ -950,7 +988,7 @@ class ConstraintAnnotator:
                 for value in variable.value:
                     value.type = type_
                     try:
-                        cvalue = CONVERSION.get(type_, str)(value.name)
+                        cvalue = CONVERT_OPTION[type_].get('func', str)(value.name)
                     except:
                         raise DictConsistencyError(_(f'unable to change type of value "{value}" is not a valid "{type_}" for "{variable.name}"'))
                     if cvalue not in choices:

src/rougail/data/rougail.dtd

@@ -131,6 +131,7 @@
 
 <!ELEMENT group (follower+)>
 <!ATTLIST group leader CDATA #REQUIRED>
+<!ATTLIST group name CDATA #IMPLIED>
 <!ATTLIST group description CDATA #IMPLIED>
 
 <!ELEMENT param (#PCDATA)>

src/rougail/template.py

@@ -9,8 +9,8 @@ from shutil import copy
 import logging
 from typing import Dict, Any
 from subprocess import call
-from os import listdir, makedirs
-from os.path import dirname, join, isfile
+from os import listdir, makedirs, getcwd, chdir
+from os.path import dirname, join, isfile, abspath, normpath, relpath
 
 from Cheetah.Template import Template as ChtTemplate
 from Cheetah.NameMapper import NotFound as CheetahNotFound
@@ -79,7 +79,8 @@ class CheetahTemplate(ChtTemplate):
                  context,
                  eosfunc: Dict,
                  destfilename,
-                 variable):
+                 variable,
+                 ):
         """Initialize Creole CheetahTemplate
         """
         extra_context = {'is_defined' : IsDefined(context),
@@ -92,6 +93,24 @@ class CheetahTemplate(ChtTemplate):
                              file=filename,
                              searchList=[context, eosfunc, extra_context])
 
+    # FORK of Cheetah fonction, do not replace '\\' by '/'
+    def serverSidePath(self,
+                       path=None,
+                       normpath=normpath,
+                       abspath=abspath
+                       ):
+
+        # strange...
+        if path is None and isinstance(self, str):
+            path = self
+        if path:
+            return normpath(abspath(path))
+#            return normpath(abspath(path.replace("\\", '/')))
+        elif hasattr(self, '_filePath') and self._filePath:
+            return normpath(abspath(self._filePath))
+        else:
+            return None
+
 
 class CreoleLeader:
     def __init__(self, value, follower=None, index=None):
@@ -287,37 +306,43 @@ class CreoleTemplateEngine:
         return CreoleExtra(families)
 
     def patch_template(self,
-                       filename: str):
+                       filename: str,
+                       tmp_dir: str,
+                       patch_dir: str,
+                       ) -> None:
         """Apply patch to a template
         """
-        patch_cmd = ['patch', '-d', self.tmp_dir, '-N', '-p1']
+        patch_cmd = ['patch', '-d', tmp_dir, '-N', '-p1']
         patch_no_debug = ['-s', '-r', '-', '--backup-if-mismatch']
 
-        # patches variante + locaux
-        for directory in [join(Config['patch_dir'], 'variante'), Config['patch_dir']]:
-            patch_file = join(directory, f'{filename}.patch')
-            if isfile(patch_file):
-                log.info(_("Patching template '{filename}' with '{patch_file}'"))
-                ret = call(patch_cmd + patch_no_debug + ['-i', patch_file])
-                if ret:
-                    patch_cmd_err = ' '.join(patch_cmd + ['-i', patch_file])
-                    log.error(_(f"Error applying patch: '{patch_file}'\nTo reproduce and fix this error {patch_cmd_err}"))
-                    copy(filename, self.tmp_dir)
+        patch_file = join(patch_dir, f'{filename}.patch')
+        if isfile(patch_file):
+            log.info(_("Patching template '{filename}' with '{patch_file}'"))
+            rel_patch_file = relpath(patch_file, tmp_dir)
+            ret = call(patch_cmd + patch_no_debug + ['-i', rel_patch_file])
+            if ret:
+                patch_cmd_err = ' '.join(patch_cmd + ['-i', rel_patch_file])
+                log.error(_(f"Error applying patch: '{rel_patch_file}'\nTo reproduce and fix this error {patch_cmd_err}"))
+                copy(join(self.distrib_dir, filename), tmp_dir)
 
     def prepare_template(self,
-                         filename: str):
+                         filename: str,
+                         tmp_dir: str,
+                         patch_dir: str,
+                         ) -> None:
         """Prepare template source file
         """
-        log.info(_("Copy template: '{filename}' -> '{self.tmp_dir}'"))
-        copy(filename, self.tmp_dir)
-        self.patch_template(filename)
+        log.info(_("Copy template: '{filename}' -> '{tmp_dir}'"))
+        copy(filename, tmp_dir)
+        self.patch_template(filename, tmp_dir, patch_dir)
 
     def process(self,
                 source: str,
                 true_destfilename: str,
                 destfilename: str,
                 filevar: Dict,
-                variable: Any):
+                variable: Any,
+                ):
         """Process a cheetah template
         """
         # full path of the destination file
@@ -332,16 +357,19 @@ class CreoleTemplateEngine:
             data = str(cheetah_template)
         except CheetahNotFound as err:
             varname = err.args[0][13:-1]
-            raise TemplateError(_(f"Error: unknown variable used in template {destfilename} : {varname}"))
+            raise TemplateError(_(f"Error: unknown variable used in template {source} to {destfilename} : {varname}"))
         except Exception as err:
-            raise TemplateError(_(f"Error while instantiating template {destfilename}: {err}"))
+            raise TemplateError(_(f"Error while instantiating template {source} to {destfilename}: {err}"))
 
         with open(destfilename, 'w') as file_h:
             file_h.write(data)
 
     def instance_file(self,
                       filevar: Dict,
-                      service_name: str) -> None:
+                      service_name: str,
+                      tmp_dir: str,
+                      dest_dir: str,
+                      ) -> None:
         """Run templatisation on one file
         """
         log.info(_("Instantiating file '{filename}'"))
@@ -355,13 +383,13 @@ class CreoleTemplateEngine:
             if variable:
                 variable = [variable]
         for idx, filename in enumerate(filenames):
-            destfilename = join(self.dest_dir, filename[1:])
+            destfilename = join(dest_dir, filename[1:])
             makedirs(dirname(destfilename), exist_ok=True)
             if variable:
                 var = variable[idx]
             else:
                 var = None
-            source = join(self.tmp_dir, filevar['source'])
+            source = join(tmp_dir, filevar['source'])
             if filevar['templating']:
                 self.process(source,
                              filename,
@@ -374,6 +402,11 @@ class CreoleTemplateEngine:
     async def instance_files(self) -> None:
         """Run templatisation on all files
         """
+        ori_dir = getcwd()
+        tmp_dir = relpath(self.tmp_dir, self.distrib_dir)
+        dest_dir = relpath(self.dest_dir, self.distrib_dir)
+        patch_dir = relpath(Config['patch_dir'], self.distrib_dir)
+        chdir(self.distrib_dir)
         for option in await self.config.option.list(type='all'):
             namespace = await option.option.name()
             if namespace == Config['variable_namespace']:
@@ -382,8 +415,8 @@ class CreoleTemplateEngine:
                 families = await self.load_eole_variables(namespace,
                                                           option)
                 self.rougail_variables_dict[namespace] = families
-        for template in listdir(self.distrib_dir):
-            self.prepare_template(join(self.distrib_dir, template))
+        for template in listdir('.'):
+            self.prepare_template(template, tmp_dir, patch_dir)
         for service_obj in await self.config.option('services').list('all'):
             service_name = await service_obj.option.doc()
             for fills in await service_obj.list('all'):
@@ -391,15 +424,17 @@ class CreoleTemplateEngine:
                     for fill_obj in await fills.list('all'):
                         fill = await fill_obj.value.dict()
                         filename = fill['source']
-                        distib_file = join(self.distrib_dir, filename)
-                        if not isfile(distib_file):
-                            raise FileNotFound(_(f"File {distib_file} does not exist."))
+                        if not isfile(filename):
+                            raise FileNotFound(_(f"File {filename} does not exist."))
                         if fill.get('activate', False):
                             self.instance_file(fill,
                                                service_name,
+                                               tmp_dir,
+                                               dest_dir,
                                                )
                         else:
                             log.debug(_("Instantiation of file '{filename}' disabled"))
+        chdir(ori_dir)
 
 
 async def generate(config: Config,

src/rougail/tiramisureflector.py

@@ -4,7 +4,7 @@ flattened XML specific
 from .config import Config
 from .i18n import _
 from .error import LoaderError
-from .annotator import ERASED_ATTRIBUTES
+from .annotator import ERASED_ATTRIBUTES, CONVERT_OPTION
 
 
 FUNC_TO_DICT = ['valid_not_equal']
@@ -12,32 +12,6 @@ FORCE_INFORMATIONS = ['help', 'test', 'separator', 'manage']
 ATTRIBUTES_ORDER = ('name', 'doc', 'default', 'multi')
 
 
-CONVERT_OPTION = {'number': dict(opttype="IntOption", func=int),
-                  'choice': dict(opttype="ChoiceOption"),
-                  'string': dict(opttype="StrOption"),
-                  'password': dict(opttype="PasswordOption"),
-                  'mail': dict(opttype="EmailOption"),
-                  'boolean': dict(opttype="BoolOption"),
-                  'symlink': dict(opttype="SymLinkOption"),
-                  'filename': dict(opttype="FilenameOption"),
-                  'date': dict(opttype="DateOption"),
-                  'unix_user': dict(opttype="UsernameOption"),
-                  'ip': dict(opttype="IPOption", initkwargs={'allow_reserved': True}),
-                  'local_ip': dict(opttype="IPOption", initkwargs={'private_only': True, 'warnings_only': True}),
-                  'netmask': dict(opttype="NetmaskOption"),
-                  'network': dict(opttype="NetworkOption"),
-                  'broadcast': dict(opttype="BroadcastOption"),
-                  'netbios': dict(opttype="DomainnameOption", initkwargs={'type': 'netbios', 'warnings_only': True}),
-                  'domain': dict(opttype="DomainnameOption", initkwargs={'type': 'domainname', 'allow_ip': False}),
-                  'hostname': dict(opttype="DomainnameOption", initkwargs={'type': 'hostname', 'allow_ip': False}),
-                  'web_address': dict(opttype="URLOption", initkwargs={'allow_ip': True, 'allow_without_dot': True}),
-                  'port': dict(opttype="PortOption", initkwargs={'allow_private': True}),
-                  'mac': dict(opttype="MACOption"),
-                  'cidr': dict(opttype="IPOption", initkwargs={'cidr': True}),
-                  'network_cidr': dict(opttype="NetworkOption", initkwargs={'cidr': True}),
-                  }
-
-
 class TiramisuReflector:
     def __init__(self,
                  xmlroot,
@@ -344,6 +318,8 @@ class Variable(Common):
                     value = value.split(',')
                     if self.object_type == 'IntOption':
                         value = [int(v) for v in value]
+                    elif self.object_type == 'FloatOption':
+                        value = [float(v) for v in value]
                 self.informations[key] = value
             else:
                 self.attrib[key] = value
@@ -397,7 +373,7 @@ class Variable(Common):
             function = child.name
             if hasattr(child, 'param'):
                 for param in child.param:
-                    value = self.populate_param(param)
+                    value = self.populate_param(function, param)
                     if not hasattr(param, 'name'):
                         args.append(str(value))
                     else:
@@ -411,6 +387,7 @@ class Variable(Common):
         return ret + ')'
 
     def populate_param(self,
+                       function: str,
                        param,
                        ):
         if param.type == 'string':
@@ -420,7 +397,7 @@ class Variable(Common):
         elif param.type == 'variable':
             value = {'option': param.text,
                      'notraisepropertyerror': param.notraisepropertyerror,
-                     'todict': param.text in FUNC_TO_DICT,
+                     'todict': function in FUNC_TO_DICT,
                      }
             if hasattr(param, 'suffix'):
                 value['suffix'] = param.suffix
@@ -443,7 +420,7 @@ class Variable(Common):
             self.attrib['default'].append(value)
             if not self.is_leader:
                 self.attrib['default_multi'] = value
-        elif isinstance(value, int):
+        elif isinstance(value, (int, float)):
             self.attrib['default'].append(value)
         else:
             self.attrib['default'].append("'" + value + "'")

tests/dictionaries/01base_file_include/00-base.xml

@@ -0,0 +1,20 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail>
+
+    <services>
+        <service name="test">
+            <file name="/etc/file"/>
+        </service>
+    </services>
+
+    <variables>
+        <family name="general">
+            <variable name="mode_conteneur_actif" type="oui/non" description="Description">
+                <value>non</value>
+            </variable>
+        </family>
+        <separators/>
+    </variables>
+</rougail>
+<!-- vim: ts=4 sw=4 expandtab
+-->

tests/dictionaries/01base_file_include/makedict/base.json

@@ -0,0 +1 @@
+{"rougail.general.mode_conteneur_actif": "non", "services.test.files.file.group": "root", "services.test.files.file.mode": "0644", "services.test.files.file.name": "/etc/file", "services.test.files.file.owner": "root", "services.test.files.file.source": "file", "services.test.files.file.templating": true, "services.test.files.file.activate": true}

tests/dictionaries/01base_file_include/result/etc/file

@@ -0,0 +1 @@
+non

tests/dictionaries/01base_file_include/tiramisu/base.py

@@ -0,0 +1,26 @@
+import imp
+func = imp.load_source('func', 'tests/dictionaries/../eosfunc/test.py')
+for key, value in dict(locals()).items():
+    if key != ['imp', 'func']:
+        setattr(func, key, value)
+try:
+    from tiramisu3 import *
+except:
+    from tiramisu import *
+from rougail.tiramisu import ConvertDynOptionDescription
+option_3 = ChoiceOption(properties=frozenset({'mandatory', 'normal'}), name='mode_conteneur_actif', doc='Description', multi=False, default='non', values=('oui', 'non'))
+option_2 = OptionDescription(name='general', doc='general', properties=frozenset({'normal'}), children=[option_3])
+option_1 = OptionDescription(name='rougail', doc='rougail', children=[option_2])
+option_8 = StrOption(name='group', doc='group', multi=False, default='root')
+option_9 = StrOption(name='mode', doc='mode', multi=False, default='0644')
+option_10 = StrOption(name='name', doc='name', multi=False, default='/etc/file')
+option_11 = StrOption(name='owner', doc='owner', multi=False, default='root')
+option_12 = StrOption(name='source', doc='source', multi=False, default='file')
+option_13 = BoolOption(name='templating', doc='templating', multi=False, default=True)
+option_14 = BoolOption(name='activate', doc='activate', multi=False, default=True)
+option_7 = OptionDescription(name='file', doc='file', children=[option_8, option_9, option_10, option_11, option_12, option_13, option_14])
+option_6 = OptionDescription(name='files', doc='files', children=[option_7])
+option_5 = OptionDescription(name='test', doc='test', children=[option_6])
+option_5.impl_set_information("manage", True)
+option_4 = OptionDescription(name='services', doc='services', properties=frozenset({'hidden'}), children=[option_5])
+option_0 = OptionDescription(name='baseoption', doc='baseoption', children=[option_1, option_4])

tests/dictionaries/01base_file_include/tmpl/file

@@ -0,0 +1 @@
+%include "incfile"

tests/dictionaries/01base_file_include/tmpl/incfile

@@ -0,0 +1 @@
+%%mode_conteneur_actif

tests/dictionaries/01base_file_patch/00-base.xml

@@ -0,0 +1,20 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail>
+
+    <services>
+        <service name="test">
+            <file name="/etc/file"/>
+        </service>
+    </services>
+
+    <variables>
+        <family name="general">
+            <variable name="mode_conteneur_actif" type="oui/non" description="Description">
+                <value>non</value>
+            </variable>
+        </family>
+        <separators/>
+    </variables>
+</rougail>
+<!-- vim: ts=4 sw=4 expandtab
+-->

tests/dictionaries/01base_file_patch/makedict/base.json

@@ -0,0 +1 @@
+{"rougail.general.mode_conteneur_actif": "non", "services.test.files.file.group": "root", "services.test.files.file.mode": "0644", "services.test.files.file.name": "/etc/file", "services.test.files.file.owner": "root", "services.test.files.file.source": "file", "services.test.files.file.templating": true, "services.test.files.file.activate": true}

tests/dictionaries/01base_file_patch/patches/file.patch

@@ -0,0 +1,5 @@
+--- tmpl/file	2020-11-20 07:44:38.588472784 +0100
++++ dest/file	2020-11-20 07:44:54.588536011 +0100
+@@ -1 +1 @@
+-unpatched
++patched

tests/dictionaries/01base_file_patch/result/etc/file

@@ -0,0 +1 @@
+patched

tests/dictionaries/01base_file_patch/tiramisu/base.py

@@ -0,0 +1,26 @@
+import imp
+func = imp.load_source('func', 'tests/dictionaries/../eosfunc/test.py')
+for key, value in dict(locals()).items():
+    if key != ['imp', 'func']:
+        setattr(func, key, value)
+try:
+    from tiramisu3 import *
+except:
+    from tiramisu import *
+from rougail.tiramisu import ConvertDynOptionDescription
+option_3 = ChoiceOption(properties=frozenset({'mandatory', 'normal'}), name='mode_conteneur_actif', doc='Description', multi=False, default='non', values=('oui', 'non'))
+option_2 = OptionDescription(name='general', doc='general', properties=frozenset({'normal'}), children=[option_3])
+option_1 = OptionDescription(name='rougail', doc='rougail', children=[option_2])
+option_8 = StrOption(name='group', doc='group', multi=False, default='root')
+option_9 = StrOption(name='mode', doc='mode', multi=False, default='0644')
+option_10 = StrOption(name='name', doc='name', multi=False, default='/etc/file')
+option_11 = StrOption(name='owner', doc='owner', multi=False, default='root')
+option_12 = StrOption(name='source', doc='source', multi=False, default='file')
+option_13 = BoolOption(name='templating', doc='templating', multi=False, default=True)
+option_14 = BoolOption(name='activate', doc='activate', multi=False, default=True)
+option_7 = OptionDescription(name='file', doc='file', children=[option_8, option_9, option_10, option_11, option_12, option_13, option_14])
+option_6 = OptionDescription(name='files', doc='files', children=[option_7])
+option_5 = OptionDescription(name='test', doc='test', children=[option_6])
+option_5.impl_set_information("manage", True)
+option_4 = OptionDescription(name='services', doc='services', properties=frozenset({'hidden'}), children=[option_5])
+option_0 = OptionDescription(name='baseoption', doc='baseoption', children=[option_1, option_4])

tests/dictionaries/01base_file_patch/tmpl/file

@@ -0,0 +1 @@
+unpatched

tests/dictionaries/01base_file_utfchar/00-base.xml

@@ -0,0 +1,20 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail>
+
+    <services>
+        <service name="test">
+            <file name="/etc/systemd-makefs@dev-disk-by\\x2dpartlabel"/>
+        </service>
+    </services>
+
+    <variables>
+        <family name="general">
+            <variable name="mode_conteneur_actif" type="oui/non" description="Description">
+                <value>non</value>
+            </variable>
+        </family>
+        <separators/>
+    </variables>
+</rougail>
+<!-- vim: ts=4 sw=4 expandtab
+-->

tests/dictionaries/01base_file_utfchar/makedict/base.json

@@ -0,0 +1 @@
+{"rougail.general.mode_conteneur_actif": "non", "services.test.files.systemd_makefs@dev_disk_by\\x2dpartlabel.group": "root", "services.test.files.systemd_makefs@dev_disk_by\\x2dpartlabel.mode": "0644", "services.test.files.systemd_makefs@dev_disk_by\\x2dpartlabel.name": "/etc/systemd-makefs@dev-disk-by\\x2dpartlabel", "services.test.files.systemd_makefs@dev_disk_by\\x2dpartlabel.owner": "root", "services.test.files.systemd_makefs@dev_disk_by\\x2dpartlabel.source": "systemd-makefs@dev-disk-by\\x2dpartlabel", "services.test.files.systemd_makefs@dev_disk_by\\x2dpartlabel.templating": true, "services.test.files.systemd_makefs@dev_disk_by\\x2dpartlabel.activate": true}

tests/dictionaries/01base_file_utfchar/result/etc/systemd-makefs@dev-disk-by\x2dpartlabel

@@ -0,0 +1 @@
+non

tests/dictionaries/01base_file_utfchar/tiramisu/base.py

@@ -0,0 +1,26 @@
+import imp
+func = imp.load_source('func', 'tests/dictionaries/../eosfunc/test.py')
+for key, value in dict(locals()).items():
+    if key != ['imp', 'func']:
+        setattr(func, key, value)
+try:
+    from tiramisu3 import *
+except:
+    from tiramisu import *
+from rougail.tiramisu import ConvertDynOptionDescription
+option_3 = ChoiceOption(properties=frozenset({'mandatory', 'normal'}), name='mode_conteneur_actif', doc='Description', multi=False, default='non', values=('oui', 'non'))
+option_2 = OptionDescription(name='general', doc='general', properties=frozenset({'normal'}), children=[option_3])
+option_1 = OptionDescription(name='rougail', doc='rougail', children=[option_2])
+option_8 = StrOption(name='group', doc='group', multi=False, default='root')
+option_9 = StrOption(name='mode', doc='mode', multi=False, default='0644')
+option_10 = StrOption(name='name', doc='name', multi=False, default='/etc/systemd-makefs@dev-disk-by\\x2dpartlabel')
+option_11 = StrOption(name='owner', doc='owner', multi=False, default='root')
+option_12 = StrOption(name='source', doc='source', multi=False, default='systemd-makefs@dev-disk-by\\x2dpartlabel')
+option_13 = BoolOption(name='templating', doc='templating', multi=False, default=True)
+option_14 = BoolOption(name='activate', doc='activate', multi=False, default=True)
+option_7 = OptionDescription(name='systemd_makefs@dev_disk_by\\x2dpartlabel', doc='systemd-makefs@dev-disk-by\\x2dpartlabel', children=[option_8, option_9, option_10, option_11, option_12, option_13, option_14])
+option_6 = OptionDescription(name='files', doc='files', children=[option_7])
+option_5 = OptionDescription(name='test', doc='test', children=[option_6])
+option_5.impl_set_information("manage", True)
+option_4 = OptionDescription(name='services', doc='services', properties=frozenset({'hidden'}), children=[option_5])
+option_0 = OptionDescription(name='baseoption', doc='baseoption', children=[option_1, option_4])

tests/dictionaries/01base_file_utfchar/tmpl/systemd-makefs@dev-disk-by\x2dpartlabel

@@ -0,0 +1 @@
+%%mode_conteneur_actif

tests/dictionaries/01base_float/00-base.xml

@@ -0,0 +1,16 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail>
+    <variables>
+        <family name="general">
+            <variable name="float" type="float" description="Description">
+                <value>0.527</value>
+            </variable>
+            <variable name="float_multi" type="float" description="Description" multi="True">
+                <value>0.527</value>
+            </variable>
+        </family>
+        <separators/>
+    </variables>
+</rougail>
+<!-- vim: ts=4 sw=4 expandtab
+-->

tests/dictionaries/01base_float/makedict/base.json

@@ -0,0 +1 @@
+{"rougail.general.float": 0.527, "rougail.general.float_multi": [0.527]}

tests/dictionaries/01base_float/tiramisu/base.py

@@ -0,0 +1,15 @@
+import imp
+func = imp.load_source('func', 'tests/dictionaries/../eosfunc/test.py')
+for key, value in dict(locals()).items():
+    if key != ['imp', 'func']:
+        setattr(func, key, value)
+try:
+    from tiramisu3 import *
+except:
+    from tiramisu import *
+from rougail.tiramisu import ConvertDynOptionDescription
+option_3 = FloatOption(properties=frozenset({'mandatory', 'normal'}), name='float', doc='Description', multi=False, default=0.527)
+option_4 = FloatOption(properties=frozenset({'mandatory', 'normal'}), name='float_multi', doc='Description', multi=True, default=[0.527], default_multi=0.527)
+option_2 = OptionDescription(name='general', doc='general', properties=frozenset({'normal'}), children=[option_3, option_4])
+option_1 = OptionDescription(name='rougail', doc='rougail', children=[option_2])
+option_0 = OptionDescription(name='baseoption', doc='baseoption', children=[option_1])

tests/dictionaries/10check_optional/tiramisu/base.py

@@ -10,7 +10,7 @@ except:
 from rougail.tiramisu import ConvertDynOptionDescription
 option_3 = StrOption(properties=frozenset({'mandatory', 'normal'}), name='mode_conteneur_actif', doc='No change', multi=False, default='b')
 option_5 = IntOption(properties=frozenset({'normal'}), name='int2', doc='No change', multi=False)
-option_4 = IntOption(properties=frozenset({'normal'}), validators=[Calculation(func.valid_not_equal, Params((ParamSelfOption(), ParamOption(option_5, notraisepropertyerror=False, todict=False)), kwargs={}), warnings_only=False)], name='int', doc='No change', multi=False)
+option_4 = IntOption(properties=frozenset({'normal'}), validators=[Calculation(func.valid_not_equal, Params((ParamSelfOption(), ParamOption(option_5, notraisepropertyerror=False, todict=True)), kwargs={}), warnings_only=False)], name='int', doc='No change', multi=False)
 option_2 = OptionDescription(name='general', doc='general', properties=frozenset({'normal'}), children=[option_3, option_4, option_5])
 option_1 = OptionDescription(name='rougail', doc='rougail', children=[option_2])
 option_0 = OptionDescription(name='baseoption', doc='baseoption', children=[option_1])

tests/dictionaries/10check_valid_differ/tiramisu/base.py

@@ -9,7 +9,7 @@ except:
     from tiramisu import *
 from rougail.tiramisu import ConvertDynOptionDescription
 option_4 = ChoiceOption(properties=frozenset({'mandatory', 'normal'}), name='mode_conteneur_actif1', doc='No change', multi=False, default='non', values=('oui', 'non'))
-option_3 = ChoiceOption(properties=frozenset({'mandatory', 'normal'}), validators=[Calculation(func.valid_not_equal, Params((ParamSelfOption(), ParamOption(option_4, notraisepropertyerror=False, todict=False)), kwargs={}), warnings_only=False)], name='mode_conteneur_actif', doc='No change', multi=False, default='oui', values=('oui', 'non'))
+option_3 = ChoiceOption(properties=frozenset({'mandatory', 'normal'}), validators=[Calculation(func.valid_not_equal, Params((ParamSelfOption(), ParamOption(option_4, notraisepropertyerror=False, todict=True)), kwargs={}), warnings_only=False)], name='mode_conteneur_actif', doc='No change', multi=False, default='oui', values=('oui', 'non'))
 option_2 = OptionDescription(name='general', doc='general', properties=frozenset({'normal'}), children=[option_3, option_4])
 option_1 = OptionDescription(name='rougail', doc='rougail', children=[option_2])
 option_0 = OptionDescription(name='baseoption', doc='baseoption', children=[option_1])

tests/dictionaries/10check_valid_differ_add/tiramisu/base.py

@@ -11,7 +11,7 @@ from rougail.tiramisu import ConvertDynOptionDescription
 option_3 = ChoiceOption(properties=frozenset({'mandatory', 'normal'}), name='mode_conteneur_actif', doc='No change', multi=False, default='oui', values=('oui', 'non'))
 option_4 = ChoiceOption(properties=frozenset({'mandatory', 'normal'}), name='mode_conteneur_actif1', doc='No change', multi=False, default='non', values=('oui', 'non'))
 option_5 = ChoiceOption(properties=frozenset({'mandatory', 'normal'}), name='mode_conteneur_actif2', doc='No change', multi=False, default='non', values=('oui', 'non'))
-option_6 = StrOption(properties=frozenset({'mandatory', 'normal'}), validators=[Calculation(func.valid_not_equal, Params((ParamSelfOption(), ParamOption(option_4, notraisepropertyerror=False, todict=False)), kwargs={}), warnings_only=False), Calculation(func.valid_not_equal, Params((ParamSelfOption(), ParamOption(option_5, notraisepropertyerror=False, todict=False)), kwargs={}), warnings_only=False)], name='mode_conteneur_actif3', doc='No change', multi=False, default='oui')
+option_6 = StrOption(properties=frozenset({'mandatory', 'normal'}), validators=[Calculation(func.valid_not_equal, Params((ParamSelfOption(), ParamOption(option_4, notraisepropertyerror=False, todict=True)), kwargs={}), warnings_only=False), Calculation(func.valid_not_equal, Params((ParamSelfOption(), ParamOption(option_5, notraisepropertyerror=False, todict=True)), kwargs={}), warnings_only=False)], name='mode_conteneur_actif3', doc='No change', multi=False, default='oui')
 option_2 = OptionDescription(name='general', doc='general', properties=frozenset({'normal'}), children=[option_3, option_4, option_5, option_6])
 option_1 = OptionDescription(name='rougail', doc='rougail', children=[option_2])
 option_0 = OptionDescription(name='baseoption', doc='baseoption', children=[option_1])

tests/dictionaries/10check_valid_differ_removecheck/tiramisu/base.py

@@ -11,7 +11,7 @@ from rougail.tiramisu import ConvertDynOptionDescription
 option_3 = ChoiceOption(properties=frozenset({'mandatory', 'normal'}), name='mode_conteneur_actif', doc='No change', multi=False, default='oui', values=('oui', 'non'))
 option_4 = ChoiceOption(properties=frozenset({'mandatory', 'normal'}), name='mode_conteneur_actif1', doc='No change', multi=False, default='non', values=('oui', 'non'))
 option_5 = ChoiceOption(properties=frozenset({'mandatory', 'normal'}), name='mode_conteneur_actif2', doc='No change', multi=False, default='non', values=('oui', 'non'))
-option_6 = StrOption(properties=frozenset({'mandatory', 'normal'}), validators=[Calculation(func.valid_not_equal, Params((ParamSelfOption(), ParamOption(option_4, notraisepropertyerror=False, todict=False)), kwargs={}), warnings_only=False), Calculation(func.valid_not_equal, Params((ParamSelfOption(), ParamOption(option_5, notraisepropertyerror=False, todict=False)), kwargs={}), warnings_only=False)], name='mode_conteneur_actif3', doc='No change', multi=False, default='oui')
+option_6 = StrOption(properties=frozenset({'mandatory', 'normal'}), validators=[Calculation(func.valid_not_equal, Params((ParamSelfOption(), ParamOption(option_4, notraisepropertyerror=False, todict=True)), kwargs={}), warnings_only=False), Calculation(func.valid_not_equal, Params((ParamSelfOption(), ParamOption(option_5, notraisepropertyerror=False, todict=True)), kwargs={}), warnings_only=False)], name='mode_conteneur_actif3', doc='No change', multi=False, default='oui')
 option_2 = OptionDescription(name='general', doc='general', properties=frozenset({'normal'}), children=[option_3, option_4, option_5, option_6])
 option_1 = OptionDescription(name='rougail', doc='rougail', children=[option_2])
 option_0 = OptionDescription(name='baseoption', doc='baseoption', children=[option_1])

tests/dictionaries/10load_leadership_name/00-base.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<rougail>
+    <variables>
+        <family name='general'>
+            <variable name='mode_conteneur_actif' type='oui/non' description="No change">
+                <value>non</value>
+            </variable>
+        </family>
+        <family name='general1'>
+            <variable name='leader' type='string' description="leader" multi="True"/>
+            <variable name='follower1' type='string' description="follower1"/>
+            <variable name='follower2' type='string' description="follower2"/>
+        </family>
+    </variables>
+    <constraints>
+        <fill name='calc_val' target='follower1'>
+            <param name='valeur'>valfill</param>
+        </fill>
+        <fill name='calc_val' target='follower2'>
+            <param type='variable'>follower1</param>
+        </fill>
+        <group leader='leader' name="other_name">
+            <follower>follower1</follower>
+            <follower>follower2</follower>
+        </group>
+    </constraints>
+</rougail>

tests/dictionaries/10load_leadership_name/makedict/base.json

@@ -0,0 +1 @@
+{"rougail.general.mode_conteneur_actif": "non", "rougail.general1.other_name.leader": [], "rougail.general1.other_name.follower1": [], "rougail.general1.other_name.follower2": []}

tests/dictionaries/10load_leadership_name/tiramisu/base.py

@@ -0,0 +1,19 @@
+import imp
+func = imp.load_source('func', 'tests/dictionaries/../eosfunc/test.py')
+for key, value in dict(locals()).items():
+    if key != ['imp', 'func']:
+        setattr(func, key, value)
+try:
+    from tiramisu3 import *
+except:
+    from tiramisu import *
+from rougail.tiramisu import ConvertDynOptionDescription
+option_3 = ChoiceOption(properties=frozenset({'mandatory', 'normal'}), name='mode_conteneur_actif', doc='No change', multi=False, default='non', values=('oui', 'non'))
+option_2 = OptionDescription(name='general', doc='general', properties=frozenset({'normal'}), children=[option_3])
+option_6 = StrOption(name='leader', doc='leader', multi=True)
+option_7 = StrOption(properties=frozenset({'normal'}), name='follower1', doc='follower1', multi=True, default=Calculation(func.calc_val, Params((), kwargs={'valeur': ParamValue("valfill")})))
+option_8 = StrOption(properties=frozenset({'normal'}), name='follower2', doc='follower2', multi=True, default=Calculation(func.calc_val, Params((ParamOption(option_7, notraisepropertyerror=False, todict=False)), kwargs={})))
+option_5 = Leadership(name='other_name', doc='leader', properties=frozenset({'normal'}), children=[option_6, option_7, option_8])
+option_4 = OptionDescription(name='general1', doc='general1', properties=frozenset({'normal'}), children=[option_5])
+option_1 = OptionDescription(name='rougail', doc='rougail', children=[option_2, option_4])
+option_0 = OptionDescription(name='baseoption', doc='baseoption', children=[option_1])

tests/dictionaries/40empty_param/makedict/base.json

@@ -1,2 +1 @@
 {"rougail.proxy_authentifie.toto1": null, "rougail.proxy_authentifie.toto2": "3127"}
-

tests/dictionaries/40empty_param2/makedict/base.json

@@ -1,2 +1 @@
 {"rougail.proxy_authentifie.toto1": null, "rougail.proxy_authentifie.toto2": "3127"}
-

tests/dictionaries/80unknown_type/00-base.xml

@@ -0,0 +1,10 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail>
+    <variables>
+        <family name="general">
+            <variable name="mode_conteneur_actif" type="unknown" description="Description"/>
+        </family>
+    </variables>
+</rougail>
+<!-- vim: ts=4 sw=4 expandtab
+-->

tests/test_1_flattener.py

@@ -1,4 +1,5 @@
 from lxml import etree
+from os import getcwd
 from os.path import isfile, join, isdir
 from pytest import fixture, raises
 from os import listdir
@@ -25,11 +26,14 @@ for test in listdir(dico_dirs):
             test_raise.add(test)
 
 excludes = set([])
+#excludes = set(['01base_file_utfchar'])
 test_ok -= excludes
 test_raise -= excludes
 #test_ok = ['10leadership_autoleader']
 #test_raise = []
 
+ORI_DIR = getcwd()
+
 debug = False
 #debug = True
 
@@ -80,6 +84,7 @@ def launch_flattener(test_dir, test_ok=False):
     if isdir(subfolder):
         eolobj.create_or_populate_from_xml('extra1', [subfolder])
     eosfunc = join(dico_dirs, '../eosfunc/test.py')
+    Config['patch_dir'] = join(test_dir, 'patches')
     eolobj.space_visitor(eosfunc)
     tiramisu_objects = eolobj.save()
     tiramisu_dir = join(test_dir, 'tiramisu')
@@ -109,14 +114,18 @@ def teardown_module(module):
 
 
 def test_dictionary(test_dir):
+    assert getcwd() == ORI_DIR
     test_dir = join(dico_dirs, test_dir)
     launch_flattener(test_dir, True)
+    assert getcwd() == ORI_DIR
 
 
 def test_error_dictionary(test_dir_error):
+    assert getcwd() == ORI_DIR
     test_dir = join(dico_dirs, test_dir_error)
     with raises(DictConsistencyError):
         launch_flattener(test_dir)
+    assert getcwd() == ORI_DIR
 
 
 def test_no_dtd():

tests/test_2_makedict.py

@@ -17,8 +17,10 @@ for test in listdir(dico_dirs):
         if isdir(join(dico_dirs, test, 'tiramisu')):
             test_ok.add(test)
 
+debug = False
+#debug = True
 excludes = set([])
-#excludes = set(['70container_services'])
+#excludes = set(['01base_file_utfchar'])
 test_ok -= excludes
 #test_ok = ['10check_valid_ipnetmask']
 
@@ -33,8 +35,6 @@ def test_dir(request):
 
 
 async def launch_flattener(test_dir):
-    debug = False
-    #debug = True
     makedict_dir = join(test_dir, 'makedict')
     makedict_file = join(makedict_dir, 'base.json')
 

tests/test_3_template.py

@@ -9,7 +9,10 @@ from rougail import template
 
 
 template_dirs = 'tests/dictionaries'
-test_ok = [f for f in listdir(template_dirs) if not f.startswith('_') and isdir(join(template_dirs, f, 'tmpl'))]
+excludes = set([])
+#excludes = set(['01base_file_utfchar'])
+test_ok = {f for f in listdir(template_dirs) if not f.startswith('_') and isdir(join(template_dirs, f, 'tmpl'))}
+test_ok -= excludes
 #test_ok = ['60extra_group']
 
 
@@ -55,6 +58,7 @@ async def test_dictionary(test_dir):
     if isdir(dest_dir):
         rmtree(dest_dir)
     mkdir(dest_dir)
+    template.Config['patch_dir'] = join(test_dir, 'patches')
     await template.generate(config,
                             funcs_file,
                             distrib_dir,