Merge branch 'develop' into dist/risotto/risotto-2.8.0/develop

debian/control

@@ -9,7 +9,11 @@ Homepage: https://forge.cadoles.com/Infra/risotto
 Package: python3-risotto
 Architecture: any
 Pre-Depends: dpkg, python3, ${misc:Pre-Depends}
-Depends: ${python:Depends}, ${misc:Depends}, python3-asyncpg, python3-rougail, python3-aiohttp
+Depends: ${python:Depends}, ${misc:Depends},
+	python3-asyncpg,
+	python3-rougail,
+	python3-aiohttp,
+	python3-sdnotify
 Description: configuration manager libraries
 
 Package: risotto

script/risotto-server

@@ -1,13 +1,16 @@
 #!/usr/bin/env python3
+from sdnotify import SystemdNotifier
 from asyncio import get_event_loop
 from risotto import get_app
 
 
 if __name__ == '__main__':
+    notifier = SystemdNotifier()
     loop = get_event_loop()
     loop.run_until_complete(get_app(loop))
-    try:
     print('HTTP server ready')
+    notifier.notify("READY=1")
+    try:
         loop.run_forever()
     except KeyboardInterrupt:
         pass

sql/risotto.sql

@@ -1,8 +1,16 @@
-CREATE TABLE log(
+CREATE TABLE RisottoLog(
+  LogId SERIAL PRIMARY KEY,
+  ContextId INTEGER,
   Msg VARCHAR(255) NOT NULL,
-  Level VARCHAR(10) NOT NULL,
-  Path VARCHAR(255),
-  Username VARCHAR(100) NOT NULL,
-  Data JSON,
-  Date timestamp DEFAULT current_timestamp
+  URI VARCHAR(255),
+  URIS VARCHAR(255),
+  UserLogin VARCHAR(100) NOT NULL,
+  Status INTEGER NOT NULL,
+  Kwargs JSON,
+  Returns JSON,
+  StartDate timestamp DEFAULT current_timestamp,
+  StopDate timestamp
 );
+CREATE INDEX RisottoLog_ContextId_index ON RisottoLog(ContextId);
+CREATE INDEX RisottoLog_Login_index ON RisottoLog(UserLogin);
+CREATE INDEX RisottoLog_URI_index ON RisottoLog(URI);

src/risotto/config.py

@@ -1,6 +1,7 @@
 from os import environ
 from os.path import isfile
 from configobj import ConfigObj
+from uuid import uuid4
 
 
 CONFIG_FILE = environ.get('CONFIG_FILE', '/etc/risotto/risotto.conf')
@@ -16,6 +17,10 @@ if 'RISOTTO_PORT' in environ:
     RISOTTO_PORT = environ['RISOTTO_PORT']
 else:
     RISOTTO_PORT = config.get('RISOTTO_PORT', 8080)
+if 'RISOTTO_URL' in environ:
+    RISOTTO_URL = environ['RISOTTO_URL']
+else:
+    RISOTTO_URL = config.get('RISOTTO_URL', 'http://localhost:8080/')
 if 'CONFIGURATION_DIR' in environ:
     CONFIGURATION_DIR = environ['CONFIGURATION_DIR']
 else:
@@ -60,6 +65,18 @@ if 'CELERYRISOTTO_DB_USER' in environ:
     CELERYRISOTTO_DB_USER = environ['CELERYRISOTTO_DB_USER']
 else:
     CELERYRISOTTO_DB_USER = config.get('CELERYRISOTTO_DB_USER', None)
+if 'LEMUR_DB_NAME' in environ:
+    LEMUR_DB_NAME = environ['LEMUR_DB_NAME']
+else:
+    LEMUR_DB_NAME = config.get('LEMUR_DB_NAME', None)
+if 'LEMUR_DB_PASSWORD' in environ:
+    LEMUR_DB_PASSWORD = environ['LEMUR_DB_PASSWORD']
+else:
+    LEMUR_DB_PASSWORD = config.get('LEMUR_DB_PASSWORD', None)
+if 'LEMUR_DB_USER' in environ:
+    LEMUR_DB_USER = environ['LEMUR_DB_USER']
+else:
+    LEMUR_DB_USER = config.get('LEMUR_DB_USER', None)
 if 'DB_ADDRESS' in environ:
     DB_ADDRESS = environ['DB_ADDRESS']
 else:
@@ -88,6 +105,45 @@ if 'IMAGE_PATH' in environ:
     IMAGE_PATH = environ['IMAGE_PATH']
 else:
     IMAGE_PATH = config.get('IMAGE_PATH', '/tmp')
+if 'PASSWORD_ADMIN_USERNAME' in environ:
+    PASSWORD_ADMIN_USERNAME = environ['PASSWORD_ADMIN_USERNAME']
+else:
+    PASSWORD_ADMIN_USERNAME = config.get('PASSWORD_ADMIN_USERNAME', 'risotto')
+if 'PASSWORD_ADMIN_EMAIL' in environ:
+    PASSWORD_ADMIN_EMAIL = environ['PASSWORD_ADMIN_EMAIL']
+else:
+    # this parameter is mandatory
+    PASSWORD_ADMIN_EMAIL = config['PASSWORD_ADMIN_EMAIL']
+if 'PASSWORD_ADMIN_PASSWORD' in environ:
+    PASSWORD_ADMIN_PASSWORD = environ['PASSWORD_ADMIN_PASSWORD']
+else:
+    # this parameter is mandatory
+    PASSWORD_ADMIN_PASSWORD = config['PASSWORD_ADMIN_PASSWORD']
+if 'PASSWORD_DEVICE_IDENTIFIER' in environ:
+    PASSWORD_DEVICE_IDENTIFIER = environ['PASSWORD_DEVICE_IDENTIFIER']
+else:
+    PASSWORD_DEVICE_IDENTIFIER = config.get('PASSWORD_DEVICE_IDENTIFIER', uuid4())
+if 'PASSWORD_URL' in environ:
+    PASSWORD_URL = environ['PASSWORD_URL']
+else:
+    PASSWORD_URL = config.get('PASSWORD_URL', 'https://localhost:8001/')
+
+if 'PASSWORD_LENGTH' in environ:
+    PASSWORD_LENGTH = int(environ['PASSWORD_LENGTH'])
+else:
+    PASSWORD_LENGTH = int(config.get('PASSWORD_LENGTH', 20))
+if 'PKI_ADMIN_PASSWORD' in environ:
+    PKI_ADMIN_PASSWORD = environ['PKI_ADMIN_PASSWORD']
+else:
+    PKI_ADMIN_PASSWORD = config['PKI_ADMIN_PASSWORD']
+if 'PKI_ADMIN_EMAIL' in environ:
+    PKI_ADMIN_EMAIL = environ['PKI_ADMIN_EMAIL']
+else:
+    PKI_ADMIN_EMAIL = config['PKI_ADMIN_EMAIL']
+if 'PKI_URL' in environ:
+    PKI_URL = environ['PKI_URL']
+else:
+    PKI_URL = config.get('PKI_URL', 'http://localhost:8002')
 
 
 def dsn_factory(database, user, password, address=DB_ADDRESS):
@@ -97,10 +153,12 @@ def dsn_factory(database, user, password, address=DB_ADDRESS):
 
 _config = {'database': {'dsn': dsn_factory(RISOTTO_DB_NAME, RISOTTO_DB_USER, RISOTTO_DB_PASSWORD),
                         'tiramisu_dsn': dsn_factory(TIRAMISU_DB_NAME, TIRAMISU_DB_USER, TIRAMISU_DB_PASSWORD),
-                        'celery_dsn': dsn_factory(CELERYRISOTTO_DB_NAME, CELERYRISOTTO_DB_USER, CELERYRISOTTO_DB_PASSWORD)
+                        'celery_dsn': dsn_factory(CELERYRISOTTO_DB_NAME, CELERYRISOTTO_DB_USER, CELERYRISOTTO_DB_PASSWORD),
+                        'lemur_dns': dsn_factory(LEMUR_DB_NAME, LEMUR_DB_USER, LEMUR_DB_PASSWORD),
                         },
            'http_server': {'port': RISOTTO_PORT,
-                           'default_user': DEFAULT_USER},
+                           'default_user': DEFAULT_USER,
+                           'url': RISOTTO_URL},
            'global': {'message_root_path': MESSAGE_PATH,
                       'configurations_dir': CONFIGURATION_DIR,
                       'debug': True,
@@ -110,6 +168,17 @@ _config = {'database': {'dsn': dsn_factory(RISOTTO_DB_NAME, RISOTTO_DB_USER, RIS
                       'sql_dir': SQL_DIR,
                       'tmp_dir': TMP_DIR,
                       },
+           'password': {'admin_username': PASSWORD_ADMIN_USERNAME,
+                        'admin_email': PASSWORD_ADMIN_EMAIL,
+                        'admin_password': PASSWORD_ADMIN_PASSWORD,
+                        'device_identifier': PASSWORD_DEVICE_IDENTIFIER,
+                        'service_url': PASSWORD_URL,
+                        'length': PASSWORD_LENGTH,
+                        },
+           'pki': {'admin_password': PKI_ADMIN_PASSWORD,
+                   'owner': PKI_ADMIN_EMAIL,
+                   'url': PKI_URL,
+                   },
            'cache': {'root_path': CACHE_ROOT_PATH},
            'servermodel': {'internal_source_path': SRV_SEED_PATH,
                            'internal_source': 'internal'},

src/risotto/context.py

@@ -1,3 +1,13 @@
 class Context:
     def __init__(self):
         self.paths = []
+        self.context_id = None
+        self.start_id = None
+
+    def copy(self):
+        context = Context()
+        for key, value in self.__dict__.items():
+            if key.startswith('__'):
+                continue
+            setattr(context, key, value)
+        return context

src/risotto/controller.py

@@ -1,6 +1,22 @@
+from os import listdir, makedirs
+from os.path import join, isdir, isfile
+from shutil import rmtree
+from traceback import print_exc
+from typing import Dict
+from rougail import RougailConvert, RougailConfig, RougailUpgrade
+try:
+    from tiramisu3 import Storage, Config
+except:
+    from tiramisu import Storage, Config
+
+from .config import get_config
+from .utils import _, tiramisu_display_name
+from .logger import log
 from .dispatcher import dispatcher
 from .context import Context
-from .utils import _
+
+
+RougailConfig['variable_namespace'] = 'configuration'
 
 
 class Controller:
@@ -8,7 +24,7 @@ class Controller:
     """
     def __init__(self,
                  test: bool,
-                 ):
+                 ) -> None:
         pass
 
     async def call(self,
@@ -48,7 +64,268 @@ class Controller:
                                  **kwargs,
                                  )
 
+    @staticmethod
+    async def check_role(uri: str,
+                         username: str,
+                         **kwargs: dict,
+                         ) -> None:
+        # create a new config
+        async with await Config(dispatcher.option) as config:
+            await config.property.read_write()
+            await config.option('message').value.set(uri)
+            subconfig = config.option(uri)
+            for key, value in kwargs.items():
+                try:
+                    await subconfig.option(key).value.set(value)
+                except AttributeError:
+                    if get_config()['global']['debug']:
+                        print_exc()
+                    raise ValueError(_(f'unknown parameter in "{uri}": "{key}"'))
+                except ValueOptionError as err:
+                    raise ValueError(_(f'invalid parameter in "{uri}": {err}'))
+            await dispatcher.check_role(subconfig,
+                                        username,
+                                        uri,
+                                        )
+
     async def on_join(self,
                       risotto_context,
                       ):
         pass
+
+
+class TiramisuController(Controller):
+    def __init__(self,
+                 test: bool,
+                 ) -> None:
+        if not 'dataset_name' in vars(self):
+            raise Exception(f'please specify "dataset_name" to "{self.__class__.__name__}"')
+        self.tiramisu_cache_root_path = join(get_config()['cache']['root_path'], self.dataset_name)
+        if not test:
+            db_conf = get_config()['database']['tiramisu_dsn']
+            self.save_storage = Storage(engine='postgres')
+            self.save_storage.setting(dsn=db_conf)
+        if self.dataset_name != 'servermodel':
+            self.optiondescription = None
+            dispatcher.set_function('v1.setting.dataset.updated',
+                                    None,
+                                    TiramisuController.dataset_updated,
+                                    self.__class__.__module__,
+                                    )
+
+    async def on_join(self,
+                      risotto_context: Context,
+                      ) -> None:
+        if isdir(self.tiramisu_cache_root_path):
+            await self.load_datas(risotto_context)
+
+    async def dataset_updated(self,
+                              risotto_context: Context,
+                              ) -> Dict:
+        await self.gen_dictionaries(risotto_context)
+        await self.load_datas(risotto_context)
+
+    async def gen_dictionaries(self,
+                               risotto_context: Context,
+                               ) -> None:
+        sources = await self.get_sources(risotto_context)
+        self._aggregate_tiramisu_funcs(sources)
+        self._convert_dictionaries_to_tiramisu(sources)
+
+    async def get_sources(self,
+                          risotto_context: Context,
+                          ) -> None:
+        return await self.call('v1.setting.source.list',
+                               risotto_context,
+                               )
+
+    def _aggregate_tiramisu_funcs(self,
+                                  sources: list,
+                                  ) -> None:
+        dest_file = join(self.tiramisu_cache_root_path, 'funcs.py')
+        if not isdir(self.tiramisu_cache_root_path):
+            makedirs(self.tiramisu_cache_root_path)
+        with open(dest_file, 'wb') as funcs:
+            funcs.write(b"""try:
+    from tiramisu3 import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value
+except:
+    from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value
+
+""")
+            for source in sources:
+                root_path = join(source['source_directory'],
+                                 self.dataset_name,
+                                 )
+                if not isdir(root_path):
+                    continue
+                for service in listdir(root_path):
+                    path = join(root_path,
+                                service,
+                                'funcs',
+                                )
+                    if not isdir(path):
+                        continue
+                    for filename in listdir(path):
+                        if not filename.endswith('.py'):
+                            continue
+                        filename_path = join(path, filename)
+                        with open(filename_path, 'rb') as fh:
+                            funcs.write(f'# {filename_path}\n'.encode())
+                            funcs.write(fh.read())
+                            funcs.write(b'\n')
+
+    def _convert_dictionaries_to_tiramisu(self, sources: list) -> None:
+        funcs_file = join(self.tiramisu_cache_root_path, 'funcs.py')
+        tiramisu_file = join(self.tiramisu_cache_root_path, 'tiramisu.py')
+        dictionaries_dir = join(self.tiramisu_cache_root_path, 'dictionaries')
+        extras_dictionaries_dir = join(self.tiramisu_cache_root_path, 'extra_dictionaries')
+        if isdir(dictionaries_dir):
+            rmtree(dictionaries_dir)
+        makedirs(dictionaries_dir)
+        if isdir(extras_dictionaries_dir):
+            rmtree(extras_dictionaries_dir)
+        makedirs(extras_dictionaries_dir)
+        extras = []
+        upgrade = RougailUpgrade()
+        for source in sources:
+            root_path = join(source['source_directory'],
+                             self.dataset_name,
+                             )
+            if not isdir(root_path):
+                continue
+            for service in listdir(root_path):
+                # upgrade dictionaries
+                path = join(root_path,
+                            service,
+                            'dictionaries',
+                            )
+                if not isdir(path):
+                    continue
+                upgrade.load_xml_from_folders(path,
+                                              dictionaries_dir,
+                                              RougailConfig['variable_namespace'],
+                                              )
+            for service in listdir(root_path):
+                # upgrade extra dictionaries
+                path = join(root_path,
+                            service,
+                            'extras',
+                            )
+                if not isdir(path):
+                    continue
+                for namespace in listdir(path):
+                    extra_dir = join(path,
+                                     namespace,
+                                     )
+                    if not isdir(extra_dir):
+                        continue
+                    extra_dictionaries_dir = join(extras_dictionaries_dir,
+                                                  namespace,
+                                                  )
+                    if not isdir(extra_dictionaries_dir):
+                        makedirs(extra_dictionaries_dir)
+                        extras.append((namespace, [extra_dictionaries_dir]))
+                    upgrade.load_xml_from_folders(extra_dir,
+                                                  extra_dictionaries_dir,
+                                                  namespace,
+                                                  )
+        del upgrade
+        config = RougailConfig.copy()
+        config['functions_file'] = funcs_file
+        config['dictionaries_dir'] = [dictionaries_dir]
+        config['extra_dictionaries'] = {}
+        for extra in extras:
+            config['extra_dictionaries'][extra[0]] = extra[1]
+        eolobj = RougailConvert(rougailconfig=config)
+        eolobj.save(tiramisu_file)
+
+    async def load(self,
+                   risotto_context: Context,
+                   name: str,
+                   to_deploy: bool=False,
+                   ) -> Config:
+        if self.optiondescription is None:
+            # use file in cache
+            tiramisu_file = join(self.tiramisu_cache_root_path, 'tiramisu.py')
+            if not isfile(tiramisu_file):
+                raise Exception(_(f'unable to load the "{self.dataset_name}" configuration, is dataset loaded?'))
+            with open(tiramisu_file) as fileio:
+                tiramisu_locals = {}
+                try:
+                    exec(fileio.read(), None, tiramisu_locals)
+                except Exception as err:
+                    raise Exception(_(f'unable to load tiramisu file {tiramisu_file}: {err}'))
+
+            self.optiondescription = tiramisu_locals['option_0']
+            del tiramisu_locals
+        try:
+            letter = self.dataset_name[0]
+            if not to_deploy:
+                session_id = f'{letter}_{name}'
+            else:
+                session_id = f'{letter}td_{name}'
+            config = await Config(self.optiondescription,
+                                  session_id=session_id,
+                                  storage=self.save_storage,
+                                  display_name=tiramisu_display_name,
+                                  )
+            # change default rights
+            await config.property.read_only()
+            await config.permissive.add('basic')
+            await config.permissive.add('normal')
+            await config.permissive.add('expert')
+
+            # set information and owner
+            await config.owner.set(session_id)
+            await config.information.set(f'{self.dataset_name}_name', name)
+        except Exception as err:
+            if get_config()['global']['debug']:
+                print_exc()
+            msg = _(f'unable to load config for {self.dataset_name} "{name}": {err}')
+            await log.error_msg(risotto_context,
+                                None,
+                                msg,
+                                )
+        return config
+
+    async def _deploy_configuration(self,
+                                    dico: dict,
+                                    ) -> None:
+        config_std = dico['config_to_deploy']
+        config = dico['config']
+        # when deploy, calculate force_store_value
+        ro = await config_std.property.getdefault('read_only', 'append')
+        if 'force_store_value' not in ro:
+            await config_std.property.read_write()
+            if self.dataset_name == 'servermodel':
+                # server_deployed should be hidden
+                await config_std.forcepermissive.option('configuration.general.server_deployed').value.set(True)
+            ro = frozenset(list(ro) + ['force_store_value'])
+            rw = await config_std.property.getdefault('read_write', 'append')
+            rw = frozenset(list(rw) + ['force_store_value'])
+            await config_std.property.setdefault(ro, 'read_only', 'append')
+            await config_std.property.setdefault(rw, 'read_write', 'append')
+        await config_std.property.read_only()
+
+        # copy informations from 'to deploy' configuration to configuration
+        await config.information.importation(await config_std.information.exportation())
+        await config.value.importation(await config_std.value.exportation())
+        await config.permissive.importation(await config_std.permissive.exportation())
+        await config.property.importation(await config_std.property.exportation())
+
+    async def build_configuration(self,
+                                  config: Config,
+                                  ) -> dict:
+        configuration = {}
+        for option in await config.option.list('optiondescription'):
+            name = await option.option.name()
+            if name == 'services':
+                continue
+            if name == RougailConfig['variable_namespace']:
+                fullpath = False
+                flatten = True
+            else:
+                fullpath = True
+                flatten = False
+            configuration.update(await option.value.dict(leader_to_list=True, fullpath=fullpath, flatten=flatten))
+        return configuration

src/risotto/dispatcher.py

@@ -30,14 +30,11 @@ class CallDispatcher:
         if response.impl_get_information('multi'):
             if not isinstance(returns, list):
                 err = _(f'function {module_name}.{function_name} has to return a list')
-                await log.error_msg(risotto_context, kwargs, err)
-                raise CallError(str(err))
+                raise CallError(err)
         else:
             if not isinstance(returns, dict):
-                await log.error_msg(risotto_context, kwargs, returns)
                 err = _(f'function {module_name}.{function_name} has to return a dict')
-                await log.error_msg(risotto_context, kwargs, err)
-                raise CallError(str(err))
+                raise CallError(err)
             returns = [returns]
         if response is None:
             raise Exception('hu?')
@@ -45,17 +42,22 @@ class CallDispatcher:
             for ret in returns:
                 async with await Config(response, display_name=lambda self, dyn_name, suffix: self.impl_getname()) as config:
                     await config.property.read_write()
+                    key = None
                     try:
                         for key, value in ret.items():
                             await config.option(key).value.set(value)
-                    except AttributeError:
+                    except AttributeError as err:
+                        if key is not None:
                             err = _(f'function {module_name}.{function_name} return the unknown parameter "{key}" for the uri "{risotto_context.version}.{risotto_context.message}"')
-                        await log.error_msg(risotto_context, kwargs, err)
-                        raise CallError(str(err))
-                    except ValueError:
-                        err = _(f'function {module_name}.{function_name} return the parameter "{key}" with an unvalid value "{value}" for the uri "{risotto_context.version}.{risotto_context.message}"')
-                        await log.error_msg(risotto_context, kwargs, err)
-                        raise CallError(str(err))
+                        else:
+                            err = _(f'function {module_name}.{function_name} return unconsistency data "{err}" for the uri "{risotto_context.version}.{risotto_context.message}"')
+                        raise CallError(err)
+                    except ValueError as err:
+                        if key is not None:
+                            err = _(f'function {module_name}.{function_name} return the invalid parameter "{key}" for the uri "{risotto_context.version}.{risotto_context.message}": {err}')
+                        else:
+                            err = _(f'function {module_name}.{function_name} return unconsistency error for the uri "{risotto_context.version}.{risotto_context.message}": {err}')
+                        raise CallError(err)
                     await config.property.read_only()
                     mandatories = await config.value.mandatory()
                     if mandatories:
@@ -65,8 +67,7 @@ class CallDispatcher:
                         await config.value.dict()
                     except Exception as err:
                         err = _(f'function {module_name}.{function_name} return an invalid response {err} for the uri "{risotto_context.version}.{risotto_context.message}"')
-                        await log.error_msg(risotto_context, kwargs, err)
-                        raise CallError(str(err))
+                        raise CallError(err)
 
     async def call(self,
                    version: str,
@@ -89,9 +90,16 @@ class CallDispatcher:
         if message not in self.messages[version]:
             raise CallError(_(f'cannot find message "{version}.{message}"'))
         function_obj = self.messages[version][message]
-        # do not start a new database connection
+        # log
+        function_name = function_obj['function'].__name__
+        info_msg = _(f"call function {function_obj['full_module_name']}.{function_name}")
         if hasattr(old_risotto_context, 'connection'):
+            # do not start a new database connection
             risotto_context.connection = old_risotto_context.connection
+            await log.start(risotto_context,
+                            kwargs,
+                            info_msg,
+                            )
             await self.check_message_type(risotto_context,
                                           kwargs,
                                           )
@@ -101,22 +109,23 @@ class CallDispatcher:
                                                                 check_role,
                                                                 internal,
                                                                 )
-            return await self.launch(risotto_context,
+            try:
+                ret = await self.launch(risotto_context,
                                         kwargs,
                                         config_arguments,
                                         function_obj,
                                         )
+                await log.success(risotto_context,
+                                  ret,
+                                  )
+            except Exception as err:
+                await log.failed(risotto_context,
+                                 str(err),
+                                 )
+                raise CallError(err) from err
         else:
+            error = None
             try:
-                await self.check_message_type(risotto_context,
-                                              kwargs,
-                                              )
-                config_arguments = await self.load_kwargs_to_config(risotto_context,
-                                                                    f'{version}.{message}',
-                                                                    kwargs,
-                                                                    check_role,
-                                                                    internal,
-                                                                    )
                 async with self.pool.acquire() as connection:
                     await connection.set_type_codec(
                         'json',
@@ -126,28 +135,56 @@ class CallDispatcher:
                     )
                     risotto_context.connection = connection
                     async with connection.transaction():
-                        return await self.launch(risotto_context,
+                        try:
+                            await log.start(risotto_context,
+                                            kwargs,
+                                            info_msg,
+                                            )
+                            await self.check_message_type(risotto_context,
+                                                          kwargs,
+                                                          )
+                            config_arguments = await self.load_kwargs_to_config(risotto_context,
+                                                                                f'{version}.{message}',
+                                                                                kwargs,
+                                                                                check_role,
+                                                                                internal,
+                                                                                )
+                            ret = await self.launch(risotto_context,
                                                     kwargs,
                                                     config_arguments,
                                                     function_obj,
                                                     )
+                            # log the success
+                            await log.success(risotto_context,
+                                              ret,
+                                              )
+                            if not internal and isinstance(ret, dict):
+                                ret['context_id'] = risotto_context.context_id
                         except CallError as err:
-                raise err
+                            if get_config()['global']['debug']:
+                                print_exc()
+                            await log.failed(risotto_context,
+                                             str(err),
+                                             )
+                            raise err from err
+            except CallError as err:
+                error = err
             except Exception as err:
                 # if there is a problem with arguments, just send an error and do nothing
                 if get_config()['global']['debug']:
                     print_exc()
-                async with self.pool.acquire() as connection:
-                    await connection.set_type_codec(
-                        'json',
-                        encoder=dumps,
-                        decoder=loads,
-                        schema='pg_catalog'
+                await log.failed(risotto_context,
+                                 str(err),
                                  )
-                    risotto_context.connection = connection
-                    async with connection.transaction():
-                        await log.error_msg(risotto_context, kwargs, err)
-                raise err
+                error = err
+            if error:
+                if not internal:
+                    err = CallError(str(error))
+                    err.context_id = risotto_context.context_id
+                else:
+                    err = error
+                raise err from error
+        return ret
 
 
 class PublishDispatcher:
@@ -159,8 +196,6 @@ class PublishDispatcher:
             for message, message_infos in messages.items():
                 # event not emit locally
                 if message_infos['pattern'] == 'event' and 'functions' in message_infos and message_infos['functions']:
-#                    module, submodule, submessage = message.split('.', 2)
-#                    if f'{module}.{submodule}' not in self.injected_self:
                     uri = f'{version}.{message}'
                     print(f'  - {uri}')
                     await self.listened_connection.add_listener(uri,
@@ -180,6 +215,7 @@ class PublishDispatcher:
         remote_kw = dumps({'kwargs': kwargs,
                            'context': {'username': risotto_context.username,
                                        'paths': risotto_context.paths,
+                                       'context_id': risotto_context.context_id,
                                        }
                           })
         # FIXME should be better :/
@@ -195,21 +231,34 @@ class PublishDispatcher:
         version, message = uri.split('.', 1)
         loop = get_event_loop()
         remote_kw = loads(payload)
+        for function_obj in self.messages[version][message]['functions']:
             risotto_context = self.build_new_context(remote_kw['context'],
                                                      version,
                                                      message,
                                                      'event',
                                                      )
-        callback = lambda: ensure_future(self._publish(version,
-                                                       message,
-                                                       risotto_context,
-                                                       **remote_kw['kwargs'],
-                                                       ))
+            callback = self.get_callback(version, message, function_obj, risotto_context, remote_kw['kwargs'],)
             loop.call_soon(callback)
 
+    def get_callback(self,
+                     version,
+                     message,
+                     function_obj,
+                     risotto_context,
+                     kwargs,
+                     ):
+        return lambda: ensure_future(self._publish(version,
+                                                   message,
+                                                   function_obj,
+                                                   risotto_context,
+                                                   **kwargs,
+                                                   ))
+
+
     async def _publish(self,
                        version: str,
                        message: str,
+                       function_obj,
                        risotto_context: Context,
                        **kwargs,
                        ) -> None:
@@ -219,12 +268,7 @@ class PublishDispatcher:
                                                             False,
                                                             False,
                                                             )
-        for function_obj in self.messages[version][message]['functions']:
         async with self.pool.acquire() as connection:
-                try:
-                    await self.check_message_type(risotto_context,
-                                                  kwargs,
-                                                  )
             await connection.set_type_codec(
                 'json',
                 encoder=dumps,
@@ -232,33 +276,46 @@ class PublishDispatcher:
                 schema='pg_catalog'
             )
             risotto_context.connection = connection
+            function_name = function_obj['function'].__name__
+            info_msg = _(f"call function {function_obj['full_module_name']}.{function_name}")
+            try:
                 async with connection.transaction():
+                    try:
+                        await log.start(risotto_context,
+                                        kwargs,
+                                        info_msg,
+                                        )
+                        await self.check_message_type(risotto_context,
+                                                      kwargs,
+                                                      )
                         await self.launch(risotto_context,
                                           kwargs,
                                           config_arguments,
                                           function_obj,
                                           )
+                        # log the success
+                        await log.success(risotto_context)
                     except CallError as err:
+                        if get_config()['global']['debug']:
+                            print_exc()
+                        await log.failed(risotto_context,
+                                         str(err),
+                                         )
+            except CallError:
                 pass
             except Exception as err:
                 # if there is a problem with arguments, log and do nothing
                 if get_config()['global']['debug']:
                     print_exc()
-                    async with self.pool.acquire() as connection:
-                        await connection.set_type_codec(
-                            'json',
-                            encoder=dumps,
-                            decoder=loads,
-                            schema='pg_catalog'
+                await log.failed(risotto_context,
+                                 str(err),
                                  )
-                        risotto_context.connection = connection
-                        async with connection.transaction():
-                            await log.error_msg(risotto_context, kwargs, err)
 
 
 class Dispatcher(register.RegisterDispatcher,
                  CallDispatcher,
-                 PublishDispatcher):
+                 PublishDispatcher,
+                 ):
     """ Manage message (call or publish)
     so launch a function when a message is called
     """
@@ -274,11 +331,13 @@ class Dispatcher(register.RegisterDispatcher,
         risotto_context = Context()
         risotto_context.username = context['username']
         risotto_context.paths = copy(context['paths'])
+        risotto_context.context_id = context['context_id']
         risotto_context.paths.append(uri)
         risotto_context.uri = uri
         risotto_context.type = type
         risotto_context.message = message
         risotto_context.version = version
+        risotto_context.pool = self.pool
         return risotto_context
 
     async def check_message_type(self,
@@ -287,7 +346,6 @@ class Dispatcher(register.RegisterDispatcher,
                                  ) -> None:
         if self.messages[risotto_context.version][risotto_context.message]['pattern'] != risotto_context.type:
             msg = _(f'{risotto_context.uri} is not a {risotto_context.type} message')
-            await log.error_msg(risotto_context, kwargs, msg)
             raise CallError(msg)
 
     async def load_kwargs_to_config(self,
@@ -342,14 +400,15 @@ class Dispatcher(register.RegisterDispatcher,
     async def check_role(self,
                          config: Config,
                          user_login: str,
-                         uri: str) -> None:
+                         uri: str,
+                         ) -> None:
         async with self.pool.acquire() as connection:
             async with connection.transaction():
                 # Verify if user exists and get ID
                 sql = '''
                 SELECT UserId
                 FROM UserUser
-                WHERE UserLogin = $1
+                WHERE Login = $1
                 '''
                 user_id = await connection.fetchval(sql,
                                                     user_login)
@@ -395,8 +454,6 @@ class Dispatcher(register.RegisterDispatcher,
         # so send the message
         function = function_obj['function']
         risotto_context.module = function_obj['module'].split('.', 1)[0]
-        function_name = function.__name__
-        info_msg = _(f"in function {function_obj['full_module_name']}.{function_name}")
         # build argument for this function
         if risotto_context.type == 'rpc':
             kw = config_arguments
@@ -407,7 +464,8 @@ class Dispatcher(register.RegisterDispatcher,
                     kw[key] = value
 
         kw['risotto_context'] = risotto_context
-        returns = await function(self.injected_self[function_obj['module']], **kw)
+        # launch
+        returns = await function(self.get_service(function_obj['module']), **kw)
         if risotto_context.type == 'rpc':
             # valid returns
             await self.valid_call_returns(risotto_context,
@@ -415,14 +473,10 @@ class Dispatcher(register.RegisterDispatcher,
                                           returns,
                                           kwargs,
                                           )
-        # log the success
-        await log.info_msg(risotto_context,
-                           {'arguments': kwargs,
-                            'returns': returns},
-                           info_msg,
-                           )
         # notification
         if function_obj.get('notification'):
+            if returns is None:
+                raise Exception(_(f'function "{function_obj["full_module_name"]}.{function_obj["function"].__name__}" must returns something for {function_obj["notification"]}!'))
             notif_version, notif_message = function_obj['notification'].split('.', 1)
             if not isinstance(returns, list):
                 send_returns = [returns]

src/risotto/http.py

@@ -1,4 +1,4 @@
-from aiohttp.web import Application, Response, get, post, HTTPBadRequest, HTTPInternalServerError, HTTPNotFound
+from aiohttp.web import Application, Response, get, post, HTTPBadRequest, HTTPInternalServerError, HTTPNotFound, static
 from json import dumps
 from traceback import print_exc
 try:
@@ -12,36 +12,53 @@ from .utils import _
 from .context import Context
 from .error import CallError, NotAllowedError, RegistrationError
 from .message import get_messages
-from .logger import log
+#from .logger import log
 from .config import get_config
 from . import services
 
 
 extra_routes = {}
+extra_statics = {}
 
 
 def create_context(request):
     risotto_context = Context()
-    risotto_context.username = request.match_info.get('username',
-                                                      get_config()['http_server']['default_user'],
-                                                      )
+    if 'username' in dict(request.match_info):
+        username = request.match_info['username']
+    elif 'username' in request.headers:
+        username = request.headers['username']
+    else:
+        username = get_config()['http_server']['default_user']
+    risotto_context.username = username
     return risotto_context
 
 
 def register(version: str,
-             path: str):
+             path: str,
+             ):
     """ Decorator to register function to the http route
     """
     def decorator(function):
         if path in extra_routes:
-            raise RegistrationError(f'the route {path} is already registered')
+            raise RegistrationError(f'the route "{path}" is already registered')
         extra_routes[path] = {'function': function,
-                              'version': version}
+                              'version': version,
+                              }
     return decorator
 
 
+def register_static(path: str,
+                    directory: str,
+                    ) -> None:
+    if path in extra_statics:
+        raise RegistrationError(f'the static path "{path}" is already registered')
+    extra_statics[path] = directory
+
+
 class extra_route_handler:
-    async def __new__(cls, request):
+    async def __new__(cls,
+                      request,
+                      ):
         kwargs = dict(request.match_info)
         kwargs['request'] = request
         kwargs['risotto_context'] = create_context(request)
@@ -67,7 +84,8 @@ class extra_route_handler:
         # await log.info_msg(kwargs['risotto_context'],
         #                    dict(request.match_info))
         return Response(text=dumps(returns),
-                        content_type='application/json')
+                        content_type='application/json',
+                        )
 
 
 async def handle(request):
@@ -87,20 +105,38 @@ async def handle(request):
                             internal=False,
                             **kwargs,
                             )
-    except NotAllowedError as err:
-        raise HTTPNotFound(reason=str(err))
-    except CallError as err:
-        raise HTTPBadRequest(reason=str(err).replace('\n', ' '))
     except Exception as err:
+        context_id = None
+        if isinstance(err, NotAllowedError):
+            error_type = HTTPNotFound
+        elif isinstance(err, CallError):
+            error_type = HTTPBadRequest
+            context_id = err.context_id
+        else:
             if get_config()['global']['debug']:
                 print_exc()
-        raise HTTPInternalServerError(reason=str(err))
-    return Response(text=dumps({'response': text}),
-                    content_type='application/json')
+            error_type = HTTPInternalServerError
+        response = {'type': 'error',
+                    'reason': str(err).replace('\n', ' '),
+                    }
+        if context_id is not None:
+            response['context_id'] = context_id
+        err = dumps({'response': response,
+                     'type': 'error',
+                     })
+        raise error_type(text=err,
+                         content_type='application/json',
+                         )
+    return Response(text=dumps({'response': text,
+                                'type': 'success',
+                                }),
+                    content_type='application/json',
+                    )
 
 
 async def api(request,
-              risotto_context):
+              risotto_context,
+              ):
     global TIRAMISU
     if not TIRAMISU:
         # check all URI that have an associated role
@@ -130,7 +166,7 @@ async def api(request,
 async def get_app(loop):
     """ build all routes
     """
-    global extra_routes
+    global extra_routes, extra_statics
     services.link_to_dispatcher(dispatcher)
     app = Application(loop=loop)
     routes = []
@@ -152,7 +188,8 @@ async def get_app(loop):
     for version in versions:
         api_route = {'function': api,
                      'version': version,
-                     'path': f'/api/{version}'}
+                     'path': f'/api/{version}',
+                     }
         extra_handler = type(api_route['path'], (extra_route_handler,), api_route)
         routes.append(get(api_route['path'], extra_handler))
         print(f'  - {api_route["path"]} (http_get)')
@@ -169,12 +206,22 @@ async def get_app(loop):
             extra_handler = type(path, (extra_route_handler,), extra)
             routes.append(get(path, extra_handler))
             print(f'  - {path} (http_get)')
+    if extra_statics:
+        if not extra_routes:
+            print(_('======== Registered static routes ========'))
+        for path, directory in extra_statics.items():
+            routes.append(static(path, directory))
+            print(f'  - {path} (static)')
     del extra_routes
+    del extra_statics
     app.router.add_routes(routes)
     await dispatcher.register_remote()
     print()
     await dispatcher.on_join()
-    return await loop.create_server(app.make_handler(), '*', get_config()['http_server']['port'])
+    return await loop.create_server(app.make_handler(),
+                                    '*',
+                                    get_config()['http_server']['port'],
+                                    )
 
 
 TIRAMISU = None

src/risotto/image.py

@@ -0,0 +1,278 @@
+from os import listdir, walk, makedirs
+from os.path import isfile, isdir, join, dirname
+from yaml import load, SafeLoader
+from json import load as jload, dump as jdump
+from time import time
+from shutil import copy2, rmtree, move
+from hashlib import sha512
+from subprocess import Popen
+
+from rougail import RougailConvert, RougailConfig, RougailUpgrade
+try:
+    from tiramisu3 import Config
+except:
+    from tiramisu import Config
+
+from .utils import _
+
+
+DATASET_PATH = ''
+TMP_DIRECTORY = '/tmp'
+PACKER_TMP_DIRECTORY = join(TMP_DIRECTORY, 'packer')
+PACKER_FILE_NAME = 'recipe.json'
+IMAGES_DIRECTORY = join(TMP_DIRECTORY, 'images')
+
+
+FUNCTIONS = b"""from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value
+# =============================================================
+# fork of risotto-setting/src/risotto_setting/config/config.py
+def get_password(**kwargs):
+    return 'password'
+
+
+def get_ip(**kwargs):
+    return '1.1.1.1'
+
+
+def get_chain(**kwargs):
+    return 'chain'
+
+
+def get_certificates(**kwargs):
+    return []
+
+
+def get_certificate(**kwargs):
+    return 'certificate'
+
+
+def get_private_key(**kwargs):
+    return 'private_key'
+
+
+def get_linked_configuration(**kwargs):
+    if 'test' in kwargs and kwargs['test']:
+        return kwargs['test'][0]
+    return 'configuration'
+
+
+def zone_information(**kwargs):
+    return 'zone'
+# =============================================================
+
+"""
+
+
+class Image:
+    def __init__(self):
+        self.parse_applications()
+
+    def parse_applications(self) -> None:
+        self.builds = []
+        self.applications = {}
+        for distrib in listdir(join(DATASET_PATH, 'seed')):
+            distrib_dir = join(DATASET_PATH, 'seed', distrib, 'applicationservice')
+            if not isdir(distrib_dir):
+                continue
+            for release in listdir(distrib_dir):
+                release_dir = join(distrib_dir, release)
+                if not isdir(release_dir):
+                    continue
+                for applicationservice in listdir(release_dir):
+                    applicationservice_dir = join(release_dir, applicationservice)
+                    if not isdir(applicationservice_dir):
+                        continue
+                    if applicationservice in self.applications:
+                        raise Exception('multi applicationservice')
+                    with open(join(applicationservice_dir, 'applicationservice.yml')) as yaml:
+                        app = load(yaml, Loader=SafeLoader)
+                    self.applications[applicationservice] = {'path': applicationservice_dir,
+                                                             'yml': app,
+                                                             }
+                    if 'service' in app and app['service']:
+                        self.builds.append(applicationservice)
+
+    def calc_depends(self,
+                     dependencies: list,
+                     appname,
+                     ):
+        app = self.applications[appname]['yml']
+        if not 'depends' in app or not app['depends']:
+            return
+        for dependency in app['depends']:
+            dependency_path = self.applications[dependency]['path']
+            if dependency_path not in dependencies:
+                dependencies.insert(0, dependency_path)
+            self.calc_depends(dependencies, dependency)
+
+
+    def list_images(self):
+        for build in self.builds:
+            dependencies = [self.applications[build]['path']]
+            self.calc_depends(dependencies, build)
+            yield build, dependencies
+
+    def copy_files(self,
+                   src_path: str,
+                   dst_path: str,
+                   ) -> None:
+        root_len = len(src_path) + 1
+        for dir_name, subdir_names, filenames in walk(src_path):
+            subdir = join(dst_path, dir_name[root_len:])
+            if not isdir(subdir):
+                makedirs(subdir)
+            for filename in filenames:
+                path = join(dir_name, filename)   
+                sub_dst_path = join(subdir, filename)  
+                if isfile(sub_dst_path):
+                    raise Exception(_(f'Try to copy {sub_dst_path} which is already exists'))
+                copy2(path, sub_dst_path)
+
+    def load_configuration(self,
+                           dependencies_path: list,
+                           packer_tmp_directory: str,
+                           ) -> dict:
+        config = RougailConfig.copy()
+        dictionaries = [join(dependency_path, 'dictionaries') for dependency_path in dependencies_path if isdir(join(dependency_path, 'dictionaries'))]
+        upgrade = RougailUpgrade()
+        dest_dictionaries = join(packer_tmp_directory, 'dictionaries')
+        makedirs(dest_dictionaries)
+        dest_dictionaries_extras = join(packer_tmp_directory, 'dictionaries_extras')
+        makedirs(dest_dictionaries_extras)
+        for dependency_path in dependencies_path:
+            dictionaries_dir = join(dependency_path, 'dictionaries')
+            if isdir(dictionaries_dir):
+                upgrade.load_xml_from_folders(dictionaries_dir,
+                                              dest_dictionaries,
+                                              RougailConfig['variable_namespace'],
+                                              )
+            extra_dir = join(dependency_path, 'extras', 'packer')
+            if isdir(extra_dir):
+                upgrade.load_xml_from_folders(extra_dir,
+                                              dest_dictionaries_extras,
+                                              'packer',
+                                              )
+        config['dictionaries_dir'] = [dest_dictionaries]
+        config['extra_dictionaries'] = {'packer': [dest_dictionaries_extras]}
+        return config
+
+
+    def merge_funcs(self,
+                    config: RougailConfig,
+                    dependencies_path: list,
+                    packer_tmp_directory: str,
+                    ):
+        functions = FUNCTIONS
+        for dependency_path in dependencies_path:
+            funcs_dir = join(dependency_path, 'funcs')
+            if not isdir(funcs_dir):
+                continue
+            for func in listdir(funcs_dir):
+                print(join(funcs_dir, func))
+                with open(join(funcs_dir, func), 'rb') as fh:
+                    functions += fh.read()
+        func_name = join(packer_tmp_directory, 'func.py')
+        with open(func_name, 'wb') as fh:
+            fh.write(functions)
+        config['functions_file'] = func_name
+
+    async def get_packer_information(self,
+                                     config: RougailConfig,
+                                     packer_tmp_directory: str,
+                                     ) -> dict:
+        eolobj = RougailConvert(config)
+        xml = eolobj.save(join(packer_tmp_directory, 'tiramisu.py'))
+        optiondescription = {}
+        exec(xml, None, optiondescription)
+        config = await Config(optiondescription['option_0'])
+        return await config.option('packer').value.dict(flatten=True)
+
+    def do_recipe_checksum(self,
+                           path: str,
+                           ) -> str:
+        files = []
+        root_len = len(path) + 1
+        for dir_name, subdir_names, filenames in walk(path):
+            subpath = dir_name[root_len:]
+            for filename in filenames:
+                with open(join(dir_name, filename), 'rb') as fh: 
+                    ctl_sum = sha512(fh.read()).hexdigest()
+                files.append(f'{subpath}/{filename}/ctl_sum')
+        return sha512('\n'.join(files).encode()).hexdigest()
+
+    async def build(self) -> None:
+        if isdir(PACKER_TMP_DIRECTORY):
+            rmtree(PACKER_TMP_DIRECTORY)
+        for application, dependencies_path in self.list_images():
+            packer_tmp_directory = join(PACKER_TMP_DIRECTORY,
+                                        application + '_' + str(time()),
+                                        )
+            makedirs(packer_tmp_directory)
+            packer_tmp_os_directory = join(packer_tmp_directory, 'os')
+            makedirs(packer_tmp_os_directory)
+            packer_tmp_img_directory = join(packer_tmp_directory, 'image')
+            makedirs(packer_tmp_img_directory)
+            config = self.load_configuration(dependencies_path, packer_tmp_directory)
+            self.merge_funcs(config, dependencies_path, packer_tmp_directory)
+            packer_configuration = await self.get_packer_information(config, packer_tmp_directory)
+            # OS image needed ?
+            packer_dst_os_filename = join(IMAGES_DIRECTORY,
+                                          'os',
+                                          packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img',
+                                          )
+            for dependency_path in dependencies_path:
+                packer_directory = join(dependency_path,
+                                        'packer',
+                                        'os',
+                                        )
+                self.copy_files(packer_directory,
+                                packer_tmp_os_directory,
+                                )
+                packer_directory = join(dependency_path,
+                                        'packer',
+                                        'image',
+                                        )
+                self.copy_files(packer_directory,
+                                packer_tmp_img_directory,
+                                )
+            if not isfile(packer_dst_os_filename):
+                self.build_image(packer_dst_os_filename,
+                                 packer_tmp_os_directory,
+                                 packer_configuration,
+                                 )
+            recipe_checksum = self.do_recipe_checksum(packer_tmp_img_directory)
+            packer_dst_filename = join(IMAGES_DIRECTORY,
+                                       f'{recipe_checksum}.img',
+                                       )
+            self.build_image(packer_dst_filename,
+                             packer_tmp_img_directory,
+                             packer_configuration,
+                             )
+
+    def build_image(self,
+                    packer_dst_filename: str,
+                    packer_tmp_directory: str,
+                    packer_configuration: dict,
+                    ) -> None:
+        tmp_directory = join(packer_tmp_directory, 'tmp')
+        makedirs(tmp_directory)
+        packer_configuration['tmp_directory'] = tmp_directory
+        recipe = {'variables': packer_configuration}
+        packer_filename = join(packer_tmp_directory, PACKER_FILE_NAME)
+        with open(packer_filename, 'r') as recipe_fd:
+            for key, value in jload(recipe_fd).items():
+                recipe[key] = value
+        with open(packer_filename, 'w') as recipe_fd:
+            jdump(recipe, recipe_fd, indent=2)
+        proc = Popen(['packer', 'build', packer_filename],
+                     #stdout=PIPE,
+                     #stderr=PIPE,
+                     cwd=tmp_directory,
+                     )
+        proc.wait()
+        if proc.returncode:
+            raise Exception(_(f'cannot build {application} with {packer_filename}'))
+        move(join(tmp_directory, 'image.img'), packer_dst_filename)
+        move(join(tmp_directory, 'image.sha256'), f'{packer_dst_filename}.sha256')
+        rmtree(tmp_directory)
+        print(_(f'Image {packer_dst_filename} created'))

src/risotto/logger.py

@@ -1,38 +1,87 @@
-from typing import Dict, Any
-from json import dumps
+from typing import Dict, Any, Optional
+from json import dumps, loads
 from asyncpg.exceptions import UndefinedTableError
-
+from datetime import datetime
+from asyncio import Lock
 
 from .context import Context
 from .utils import _
 from .config import get_config
 
+database_lock = Lock()
+
+
+LEVELS = ['Error', 'Info', 'Success', 'Started', 'Failure']
+
 
 class Logger:
     """ An object to manager log
     """
+    def __init__(self) -> None:
+        self.log_connection = None
+
+    async def get_connection(self,
+                             risotto_context: Context,
+                             ):
+        if not self.log_connection:
+            self.log_connection = await risotto_context.pool.acquire()
+            await self.log_connection.set_type_codec(
+                'json',
+                encoder=dumps,
+                decoder=loads,
+                schema='pg_catalog'
+            )
+        return self.log_connection
+
     async def insert(self,
                      msg: str,
-                     path: str,
-                     risotto_context: str,
+                     risotto_context: Context,
                      level: str,
-                     data: Any= None) -> None:
-        insert = 'INSERT INTO log(Msg, Path, Username, Level'
-        values = 'VALUES($1,$2,$3,$4'
-        args = [msg, path, risotto_context.username, level]
-        if data:
-            insert += ', Data'
-            values += ',$5'
-            args.append(dumps(data))
+                     kwargs: Any=None,
+                     start: bool=False,
+                     ) -> None:
+        uri = self._get_last_uri(risotto_context)
+        uris = " ".join(risotto_context.paths)
+        insert = 'INSERT INTO RisottoLog(Msg, URI, URIS, UserLogin, Status'
+        values = 'VALUES($1,$2,$3,$4,$5'
+        args = [msg, uri, uris, risotto_context.username, LEVELS.index(level)]
+        if kwargs:
+            insert += ', Kwargs'
+            values += ',$6'
+            args.append(dumps(kwargs))
+        context_id = risotto_context.context_id
+        if context_id is not None:
+            insert += ', ContextId'
+            if kwargs:
+                values += ',$7'
+            else:
+                values += ',$6'
+            args.append(context_id)
 
-        sql = insert + ') ' + values + ')'
+        sql = insert + ') ' + values + ') RETURNING LogId'
         try:
-            await risotto_context.connection.fetch(sql, *args)
+            async with database_lock:
+                connection = await self.get_connection(risotto_context)
+                log_id = await connection.fetchval(sql, *args)
+            if context_id is None and start:
+                risotto_context.context_id = log_id
+            if start:
+                risotto_context.start_id = log_id
         except UndefinedTableError as err:
             raise Exception(_(f'cannot access to database ({err}), was the database really created?'))
 
+    def _get_last_uri(self,
+                      risotto_context: Context,
+                      ) -> str:
+        if risotto_context.paths:
+            return risotto_context.paths[-1]
+        return ''
+
     def _get_message_paths(self,
-                           risotto_context: Context):
+                           risotto_context: Context,
+                           ) -> str:
+        if not risotto_context.paths:
+            return ''
         paths = risotto_context.paths
         if risotto_context.type:
             paths_msg = f' {risotto_context.type} '
@@ -49,44 +98,114 @@ class Logger:
                         risotto_context: Context,
                         arguments,
                         error: str,
-                        msg: str=''):
+                        msg: str='',
+                        ):
         """ send message when an error append
         """
         paths_msg = self._get_message_paths(risotto_context)
         print(_(f'{risotto_context.username}: ERROR: {error} ({paths_msg} with arguments "{arguments}": {msg})'))
         await self.insert(msg,
-                          paths_msg,
                           risotto_context,
                           'Error',
-                          arguments)
+                          arguments,
+                          )
 
     async def info_msg(self,
                        risotto_context: Context,
                        arguments: Dict,
-                       msg: str=''):
+                       msg: str='',
+                       ) -> None:
         """ send message with common information
         """
-        if risotto_context.paths:
         paths_msg = self._get_message_paths(risotto_context)
-        else:
-            paths_msg = ''
         if get_config()['global']['debug']:
             print(_(f'{risotto_context.username}: INFO:{paths_msg}: {msg}'))
         await self.insert(msg,
-                          paths_msg,
                           risotto_context,
                           'Info',
-                          arguments)
+                          arguments,
+                          )
+
+    async def start(self,
+                    risotto_context: Context,
+                    arguments: dict,
+                    msg: str,
+                    ) -> None:
+        paths_msg = self._get_message_paths(risotto_context)
+        if get_config()['global']['debug']:
+            if risotto_context.context_id != None:
+                context = f'({risotto_context.context_id})'
+            else:
+                context = ''
+            print(_(f'{risotto_context.username}: START{context}:{paths_msg}: {msg}'))
+        await self.insert(msg,
+                          risotto_context,
+                          'Started',
+                          arguments,
+                          start=True,
+                          )
+
+    async def success(self,
+                      risotto_context: Context,
+                      returns: Optional[dict]=None,
+                      ) -> None:
+        if get_config()['global']['debug']:
+            paths_msg = self._get_message_paths(risotto_context)
+            print(_(f'{risotto_context.username}: SUCCESS({risotto_context.context_id}):{paths_msg}'))
+        sql = """UPDATE RisottoLog
+        SET StopDate = $2,
+        Status = $3
+        """
+        args = [datetime.now(), LEVELS.index('Success')]
+        if returns:
+            sql += """, Returns = $4
+            """
+            args.append(dumps(returns))
+        sql += """WHERE LogId = $1
+        """
+        async with database_lock:
+            connection = await self.get_connection(risotto_context)
+            await connection.execute(sql,
+                                     risotto_context.start_id,
+                                     *args,
+                                     )
+
+    async def failed(self,
+                     risotto_context: Context,
+                     err: str,
+                     ) -> None:
+        if get_config()['global']['debug']:
+            paths_msg = self._get_message_paths(risotto_context)
+            if risotto_context.context_id != None:
+                context = f'({risotto_context.context_id})'
+            else:
+                context = ''
+            print(_(f'{risotto_context.username}: FAILED({risotto_context.context_id}):{paths_msg}: {err}'))
+        sql = """UPDATE RisottoLog
+        SET StopDate = $2,
+        Status = $4,
+        Msg = $3
+        WHERE LogId = $1
+        """
+        async with database_lock:
+            connection = await self.get_connection(risotto_context)
+            await connection.execute(sql,
+                                     risotto_context.start_id,
+                                     datetime.now(),
+                                     err[:254],
+                                     LEVELS.index('Failure'),
+                                     )
 
     async def info(self,
                    risotto_context,
-                   msg):
+                   msg,
+                   ):
         if get_config()['global']['debug']:
             print(msg)
         await self.insert(msg,
-                          None,
                           risotto_context,
-                          'Info')
+                          'Info',
+                          )
 
 
 log = Logger()

src/risotto/message.py

@@ -313,6 +313,7 @@ class CustomParam:
                  'string': 'String',
                  'number': 'Number',
                  'object': 'Dict',
+                 'any': 'Any',
                  'array': 'Array',
                  'file': 'File',
                  'float': 'Float'}
@@ -448,6 +449,7 @@ def _get_option(name,
                                                 'reverse_condition': ParamValue(True)}),
                                  calc_value_property_help))
 
+    props.append('notunique')
     description = arg.description.strip().rstrip()
     kwargs = {'name': name,
               'doc': _get_description(description, name),
@@ -523,6 +525,7 @@ def _parse_responses(message_def,
                   'Number': IntOption,
                   'Boolean': BoolOption,
                   'Dict': DictOption,
+                  'Any': AnyOption,
                   'Float': FloatOption,
                   # FIXME
                   'File': StrOption}.get(type_)
@@ -530,8 +533,9 @@ def _parse_responses(message_def,
             raise Exception(f'unknown param type {obj.type} in responses of message {message_def.message}')
         if hasattr(obj, 'default'):
             kwargs['default'] = obj.default
+            kwargs['properties'] = ('notunique',)
         else:
-            kwargs['properties'] = ('mandatory',)
+            kwargs['properties'] = ('mandatory', 'notunique')
         options.append(option(**kwargs))
     od = OptionDescription(uri,
                            message_def.response.description,
@@ -600,7 +604,7 @@ def get_messages(current_module_names,
     select_option = ChoiceOption('message',
                                  'Nom du message.',
                                  tuple(messages),
-                                 properties=frozenset(['mandatory', 'positional']))
+                                 properties=frozenset(['mandatory', 'positional', 'notunique']))
     for uri in messages:
         message_def = get_message(uri,
                                   current_module_names,

src/risotto/register.py

@@ -7,6 +7,7 @@ from typing import Callable, Optional, List
 from asyncpg import create_pool
 from json import dumps, loads
 from pkg_resources import iter_entry_points
+from traceback import print_exc
 import risotto
 from .utils import _
 from .error import RegistrationError
@@ -23,7 +24,7 @@ class Services():
 
     def load_services(self):
         for entry_point in iter_entry_points(group='risotto_services'):
-            self.services.setdefault(entry_point.name, [])
+            self.services.setdefault(entry_point.name, {})
         self.services_loaded = True
 
     def load_modules(self,
@@ -32,21 +33,20 @@ class Services():
         for entry_point in iter_entry_points(group='risotto_modules'):
             service_name, module_name = entry_point.name.split('.')
             if limit_services is None or service_name in limit_services:
-                setattr(self, module_name, entry_point.load())
-                self.services[service_name].append(module_name)
+                self.services[service_name][module_name] = entry_point.load()
         self.modules_loaded = True
-
-    def get_services(self):
-        if not self.services_loaded:
-            self.load_services()
-        return [(s, getattr(self, s)) for s in self.services]
+#
+#    def get_services(self):
+#        if not self.services_loaded:
+#            self.load_services()
+#        return [(service, getattr(self, service)) for service in self.services]
 
     def get_modules(self,
                     limit_services: Optional[List[str]]=None,
                     ) -> List[str]:
         if not self.modules_loaded:
             self.load_modules(limit_services=limit_services)
-        return [(module + '.' + submodule, getattr(self, submodule)) for module, submodules in self.services.items() for submodule in submodules]
+        return [(module + '.' + submodule, entry_point) for module, submodules in self.services.items() for submodule, entry_point in submodules.items()]
 
     def get_services_list(self):
         return self.services.keys()
@@ -84,11 +84,11 @@ def register(uris: str,
 
     def decorator(function):
         for uri in uris:
-            version, message = uri.split('.', 1)
-            dispatcher.set_function(version,
-                                    message,
+            dispatcher.set_function(uri,
                                     notification,
-                                    function)
+                                    function,
+                                    function.__module__
+                                    )
     return decorator
 
 
@@ -107,6 +107,7 @@ class RegisterDispatcher:
             version = obj['version']
             if version not in self.messages:
                 self.messages[version] = {}
+            obj['message'] = tiramisu_message
             self.messages[version][tiramisu_message] = obj
 
     def get_function_args(self,
@@ -185,21 +186,20 @@ class RegisterDispatcher:
             raise RegistrationError(_(f'error with {module_name}.{function_name} arguments: {msg}'))
 
     def set_function(self,
-                     version: str,
-                     message: str,
+                     uri: str,
                      notification: str,
                      function: Callable,
+                     full_module_name: str,
                      ):
         """ register a function to an URI
         URI is a message
         """
-
+        version, message = uri.split('.', 1)
         # check if message exists
         if message not in self.messages[version]:
             raise RegistrationError(_(f'the message {message} not exists'))
 
         # xxx submodule can only be register with v1.yyy.xxx..... message
-        full_module_name = function.__module__
         risotto_module_name, submodule_name = full_module_name.split('.')[-3:-1]
         module_name = risotto_module_name.split('_')[-1]
         message_module, message_submodule, message_name = message.split('.', 2)
@@ -215,7 +215,7 @@ class RegisterDispatcher:
 
         # check if already register
         if 'function' in self.messages[version][message]:
-            raise RegistrationError(_(f'uri {version}.{message} already registered'))
+            raise RegistrationError(_(f'uri {uri} already registered'))
 
         # register
         if self.messages[version][message]['pattern'] == 'rpc':
@@ -278,7 +278,7 @@ class RegisterDispatcher:
         try:
             self.injected_self[submodule_name] = module.Risotto(test)
         except AttributeError as err:
-            raise RegistrationError(_(f'unable to register the module {submodule_name}, this module must have Risotto class'))
+            print(_(f'unable to register the module {submodule_name}, this module must have Risotto class'))
 
     def validate(self):
         """ check if all messages have a function
@@ -288,9 +288,9 @@ class RegisterDispatcher:
             for message, message_obj in messages.items():
                 if not 'functions' in message_obj and not 'function' in message_obj:
                     if message_obj['pattern'] == 'event':
-                        print(f'{message} prêche dans le désert')
+                        print(f'{version}.{message} prêche dans le désert')
                     else:
-                        missing_messages.append(message)
+                        missing_messages.append(f'{version}.{message}')
         if missing_messages:
             raise RegistrationError(_(f'no matching function for uri {missing_messages}'))
 
@@ -314,13 +314,20 @@ class RegisterDispatcher:
                     risotto_context.username = internal_user
                     risotto_context.paths.append(f'internal.{submodule_name}.on_join')
                     risotto_context.type = None
+                    risotto_context.pool = self.pool
                     risotto_context.connection = connection
                     risotto_context.module = submodule_name.split('.', 1)[0]
                     info_msg = _(f'in function risotto_{submodule_name}.on_join')
                     await log.info_msg(risotto_context,
                                        None,
                                        info_msg)
+                    try:
                         await module.on_join(risotto_context)
+                    except Exception as err:
+                        if get_config()['global']['debug']:
+                            print_exc()
+                        msg = _(f'on_join returns an error in module {submodule_name}: {err}')
+                        await log.error_msg(risotto_context, {}, msg)
 
     async def load(self):
         # valid function's arguments

src/risotto/utils.py

@@ -1,9 +1,27 @@
 class Undefined:
     pass
+undefined = Undefined()
 
 
 def _(s):
     return s
 
 
-undefined = Undefined()
+def tiramisu_display_name(kls,
+                          dyn_name: 'Base'=None,
+                          suffix: str=None,
+                          ) -> str:
+    if dyn_name is not None:
+        name = dyn_name
+    else:
+        name = kls.impl_getname()
+    doc = kls.impl_get_information('doc', None)
+    if doc:
+        doc = str(doc)
+        if doc.endswith('.'):
+            doc = doc[:-1]
+        if suffix:
+            doc += suffix
+        if name != doc:
+            name += f'" "{doc}'
+    return name

tests/test_config.py

@@ -392,7 +392,6 @@ async def test_server_created_base():
                              release_distribution='last',
                              site_name='site_1',
                              zones_name=['zones'],
-                             zones_ip=['1.1.1.1'],
                              )
     assert list(config_module.server) == [server_name]
     assert set(config_module.server[server_name]) == {'server', 'server_to_deploy', 'funcs_file'}
@@ -420,7 +419,6 @@ async def test_server_created_own_sm():
                              release_distribution='last',
                              site_name='site_1',
                              zones_name=['zones'],
-                             zones_ip=['1.1.1.1'],
                              )
     assert list(config_module.server) == [server_name]
     assert set(config_module.server[server_name]) == {'server', 'server_to_deploy', 'funcs_file'}
@@ -469,7 +467,6 @@ async def test_server_configuration_get():
                              release_distribution='last',
                              site_name='site_1',
                              zones_name=['zones'],
-                             zones_ip=['1.1.1.1'],
                              )
     #
     await config_module.server[server_name]['server'].property.read_write()
@@ -515,7 +512,6 @@ async def test_server_configuration_deployed():
                              release_distribution='last',
                              site_name='site_1',
                              zones_name=['zones'],
-                             zones_ip=['1.1.1.1'],
                              )
     #
     await config_module.server[server_name]['server'].property.read_write()