Merge branch 'develop' into dist/risotto/risotto-2.8.0/develop

debian/changelog

@@ -1,5 +0,0 @@
-risotto (0.1) unstable; urgency=low
-
-  * first version
-
- -- Cadoles <contact@cadoles.com>  Fri, 20 Mar 2020 15:18:25 +0100

debian/control

@@ -2,13 +2,19 @@ Source: risotto
 Section: admin
 Priority: extra
 Maintainer: Cadoles <contact@cadoles.com>
-Build-depends: debhelper (>=11), python3-all, python3-setuptools
+Build-depends: debhelper (>=11), python3-all, python3-setuptools, dh-python
 Standards-Version: 3.9.4
 Homepage: https://forge.cadoles.com/Infra/risotto
 
+Package: python3-risotto
+Architecture: any
+Pre-Depends: dpkg, python3, ${misc:Pre-Depends}
+Depends: ${python:Depends}, ${misc:Depends}, python3-asyncpg, python3-rougail, python3-aiohttp
+Description: configuration manager libraries
+
 Package: risotto
 Architecture: any
 Pre-Depends: dpkg, python3, ${misc:Pre-Depends}
-Depends: ${python:Depends}, ${misc:Depends}
+Depends: ${python:Depends}, ${misc:Depends}, python3-risotto
 Description: configuration manager
 

debian/risotto.install

@@ -0,0 +1,2 @@
+script/risotto-server usr/bin/
+sql/risotto.sql usr/share/eole/db/eole-risotto/gen/

script/risotto-server

@@ -1,16 +1,13 @@
 #!/usr/bin/env python3
-from sdnotify import SystemdNotifier
 from asyncio import get_event_loop
 from risotto import get_app
 
 
 if __name__ == '__main__':
-    notifier = SystemdNotifier()
     loop = get_event_loop()
     loop.run_until_complete(get_app(loop))
-    print('HTTP server ready')
-    notifier.notify("READY=1")
     try:
+        print('HTTP server ready')
         loop.run_forever()
     except KeyboardInterrupt:
         pass

sql/risotto.sql

@@ -1,16 +1,8 @@
-CREATE TABLE RisottoLog(
+CREATE TABLE log(
-  LogId SERIAL PRIMARY KEY,
   Msg VARCHAR(255) NOT NULL,
-  URI VARCHAR(255),
-  URIS VARCHAR(255),
-  UserLogin VARCHAR(100) NOT NULL,
   Level VARCHAR(10) NOT NULL,
-  ContextId INTEGER,
+  Path VARCHAR(255),
+  Username VARCHAR(100) NOT NULL,
   Data JSON,
-  Returns JSON,
+  Date timestamp DEFAULT current_timestamp
-  StartDate timestamp DEFAULT current_timestamp,
-  StopDate timestamp
 );
-CREATE INDEX RisottoLog_ContextId_index ON RisottoLog(ContextId);
-CREATE INDEX RisottoLog_Login_index ON RisottoLog(UserLogin);
-CREATE INDEX RisottoLog_URI_index ON RisottoLog(URI);

src/risotto/config.py

@@ -1,7 +1,6 @@
 from os import environ
 from os.path import isfile
 from configobj import ConfigObj
-from uuid import uuid4
 
 
 CONFIG_FILE = environ.get('CONFIG_FILE', '/etc/risotto/risotto.conf')
@@ -17,10 +16,6 @@ if 'RISOTTO_PORT' in environ:
     RISOTTO_PORT = environ['RISOTTO_PORT']
 else:
     RISOTTO_PORT = config.get('RISOTTO_PORT', 8080)
-if 'RISOTTO_URL' in environ:
-    RISOTTO_URL = environ['RISOTTO_URL']
-else:
-    RISOTTO_URL = config.get('RISOTTO_URL', 'http://localhost:8080/')
 if 'CONFIGURATION_DIR' in environ:
     CONFIGURATION_DIR = environ['CONFIGURATION_DIR']
 else:
@@ -89,44 +84,6 @@ if 'TMP_DIR' in environ:
     TMP_DIR = environ['TMP_DIR']
 else:
     TMP_DIR = config.get('TMP_DIR', '/tmp')
-if 'IMAGE_PATH' in environ:
-    IMAGE_PATH = environ['IMAGE_PATH']
-else:
-    IMAGE_PATH = config.get('IMAGE_PATH', '/tmp')
-if 'PASSWORD_ADMIN_USERNAME' in environ:
-    PASSWORD_ADMIN_USERNAME = environ['PASSWORD_ADMIN_USERNAME']
-else:
-    PASSWORD_ADMIN_USERNAME = config.get('PASSWORD_ADMIN_USERNAME', 'risotto')
-if 'PASSWORD_ADMIN_EMAIL' in environ:
-    PASSWORD_ADMIN_EMAIL = environ['PASSWORD_ADMIN_EMAIL']
-else:
-    # this parameter is mandatory
-    PASSWORD_ADMIN_EMAIL = config['PASSWORD_ADMIN_EMAIL']
-if 'PASSWORD_ADMIN_PASSWORD' in environ:
-    PASSWORD_ADMIN_PASSWORD = environ['PASSWORD_ADMIN_PASSWORD']
-else:
-    # this parameter is mandatory
-    PASSWORD_ADMIN_PASSWORD = config['PASSWORD_ADMIN_PASSWORD']
-if 'PASSWORD_DEVICE_IDENTIFIER' in environ:
-    PASSWORD_DEVICE_IDENTIFIER = environ['PASSWORD_DEVICE_IDENTIFIER']
-else:
-    PASSWORD_DEVICE_IDENTIFIER = config.get('PASSWORD_DEVICE_IDENTIFIER', uuid4())
-if 'PASSWORD_URL' in environ:
-    PASSWORD_URL = environ['PASSWORD_URL']
-else:
-    PASSWORD_URL = config.get('PASSWORD_URL', 'https://localhost:8001/')
-if 'PKI_ADMIN_PASSWORD' in environ:
-    PKI_ADMIN_PASSWORD = environ['PKI_ADMIN_PASSWORD']
-else:
-    PKI_ADMIN_PASSWORD = config['PKI_ADMIN_PASSWORD']
-if 'PKI_ADMIN_EMAIL' in environ:
-    PKI_ADMIN_EMAIL = environ['PKI_ADMIN_EMAIL']
-else:
-    PKI_ADMIN_EMAIL = config['PKI_ADMIN_EMAIL']
-if 'PKI_URL' in environ:
-    PKI_URL = environ['PKI_URL']
-else:
-    PKI_URL = config.get('PKI_URL', 'http://localhost:8002')
 
 
 def dsn_factory(database, user, password, address=DB_ADDRESS):
@@ -139,8 +96,7 @@ _config = {'database': {'dsn': dsn_factory(RISOTTO_DB_NAME, RISOTTO_DB_USER, RIS
                         'celery_dsn': dsn_factory(CELERYRISOTTO_DB_NAME, CELERYRISOTTO_DB_USER, CELERYRISOTTO_DB_PASSWORD)
                         },
            'http_server': {'port': RISOTTO_PORT,
-                           'default_user': DEFAULT_USER,
+                           'default_user': DEFAULT_USER},
-                           'url': RISOTTO_URL},
            'global': {'message_root_path': MESSAGE_PATH,
                       'configurations_dir': CONFIGURATION_DIR,
                       'debug': True,
@@ -150,24 +106,13 @@ _config = {'database': {'dsn': dsn_factory(RISOTTO_DB_NAME, RISOTTO_DB_USER, RIS
                       'sql_dir': SQL_DIR,
                       'tmp_dir': TMP_DIR,
                       },
-           'password': {'admin_username': PASSWORD_ADMIN_USERNAME,
-                        'admin_email': PASSWORD_ADMIN_EMAIL,
-                        'admin_password': PASSWORD_ADMIN_PASSWORD,
-                        'device_identifier': PASSWORD_DEVICE_IDENTIFIER,
-                        'service_url': PASSWORD_URL,
-                        },
-           'pki': {'admin_password': PKI_ADMIN_PASSWORD,
-                   'owner': PKI_ADMIN_EMAIL,
-                   'url': PKI_URL,
-                   },
            'cache': {'root_path': CACHE_ROOT_PATH},
            'servermodel': {'internal_source_path': SRV_SEED_PATH,
                            'internal_source': 'internal'},
            'submodule': {'allow_insecure_https': False,
                          'pki': '192.168.56.112'},
            'provider': {'factory_configuration_filename': 'infra.json',
-                        'packer_filename': 'recipe.json',
+                        'packer_filename': 'recipe.json'},
-                        'risotto_images_dir': IMAGE_PATH},
            }
 
 

src/risotto/context.py

@@ -1,5 +1,3 @@
 class Context:
     def __init__(self):
         self.paths = []
-        self.context_id = None
-        self.start_id = None

src/risotto/controller.py

@@ -1,22 +1,6 @@
-from os import listdir, makedirs
-from os.path import join, isdir, isfile
-from shutil import rmtree
-from traceback import print_exc
-from typing import Dict
-from rougail import RougailConvert, RougailConfig, RougailUpgrade
-try:
-    from tiramisu3 import Storage, Config
-except:
-    from tiramisu import Storage, Config
-
-from .config import get_config
-from .utils import _, tiramisu_display_name
-from .logger import log
 from .dispatcher import dispatcher
 from .context import Context
-
+from .utils import _
-
-RougailConfig['variable_namespace'] = 'configuration'
 
 
 class Controller:
@@ -24,7 +8,7 @@ class Controller:
     """
     def __init__(self,
                  test: bool,
-                 ) -> None:
+                 ):
         pass
 
     async def call(self,
@@ -64,268 +48,7 @@ class Controller:
                                  **kwargs,
                                  )
 
-    @staticmethod
-    async def check_role(self,
-                         uri: str,
-                         username: str,
-                         **kwargs: dict,
-                         ) -> None:
-        # create a new config
-        async with await Config(dispatcher.option) as config:
-            await config.property.read_write()
-            await config.option('message').value.set(uri)
-            subconfig = config.option(uri)
-            for key, value in kwargs.items():
-                try:
-                    await subconfig.option(key).value.set(value)
-                except AttributeError:
-                    if get_config()['global']['debug']:
-                        print_exc()
-                    raise ValueError(_(f'unknown parameter in "{uri}": "{key}"'))
-                except ValueOptionError as err:
-                    raise ValueError(_(f'invalid parameter in "{uri}": {err}'))
-            await dispatcher.check_role(subconfig,
-                                        username,
-                                        uri,
-                                        )
-
     async def on_join(self,
                       risotto_context,
                       ):
         pass
-
-
-class TiramisuController(Controller):
-    def __init__(self,
-                 test: bool,
-                 ) -> None:
-        if not 'dataset_name' in vars(self):
-            raise Exception(f'please specify "dataset_name" to "{self.__class__.__name__}"')
-        self.tiramisu_cache_root_path = join(get_config()['cache']['root_path'], self.dataset_name)
-        if not test:
-            db_conf = get_config()['database']['tiramisu_dsn']
-            self.save_storage = Storage(engine='postgres')
-            self.save_storage.setting(dsn=db_conf)
-        if self.dataset_name != 'servermodel':
-            self.optiondescription = None
-            dispatcher.set_function('v1.setting.dataset.updated',
-                                    None,
-                                    TiramisuController.dataset_updated,
-                                    self.__class__.__module__,
-                                    )
-
-    async def on_join(self,
-                      risotto_context: Context,
-                      ) -> None:
-        if isdir(self.tiramisu_cache_root_path):
-            await self.load_datas(risotto_context)
-
-    async def dataset_updated(self,
-                              risotto_context: Context,
-                              ) -> Dict:
-        await self.gen_dictionaries(risotto_context)
-        await self.load_datas(risotto_context)
-
-    async def gen_dictionaries(self,
-                               risotto_context: Context,
-                               ) -> None:
-        sources = await self.get_sources(risotto_context)
-        self._aggregate_tiramisu_funcs(sources)
-        self._convert_dictionaries_to_tiramisu(sources)
-
-    async def get_sources(self,
-                          risotto_context: Context,
-                          ) -> None:
-        return await self.call('v1.setting.source.list',
-                               risotto_context,
-                               )
-
-    def _aggregate_tiramisu_funcs(self,
-                                  sources: list,
-                                  ) -> None:
-        dest_file = join(self.tiramisu_cache_root_path, 'funcs.py')
-        if not isdir(self.tiramisu_cache_root_path):
-            makedirs(self.tiramisu_cache_root_path)
-        with open(dest_file, 'wb') as funcs:
-            funcs.write(b"""try:
-    from tiramisu3 import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value
-except:
-    from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value
-
-""")
-            for source in sources:
-                root_path = join(source['source_directory'],
-                                 self.dataset_name,
-                                 )
-                if not isdir(root_path):
-                    continue
-                for service in listdir(root_path):
-                    path = join(root_path,
-                                service,
-                                'funcs',
-                                )
-                    if not isdir(path):
-                        continue
-                    for filename in listdir(path):
-                        if not filename.endswith('.py'):
-                            continue
-                        filename_path = join(path, filename)
-                        with open(filename_path, 'rb') as fh:
-                            funcs.write(f'# {filename_path}\n'.encode())
-                            funcs.write(fh.read())
-                            funcs.write(b'\n')
-
-    def _convert_dictionaries_to_tiramisu(self, sources: list) -> None:
-        funcs_file = join(self.tiramisu_cache_root_path, 'funcs.py')
-        tiramisu_file = join(self.tiramisu_cache_root_path, 'tiramisu.py')
-        dictionaries_dir = join(self.tiramisu_cache_root_path, 'dictionaries')
-        extras_dictionaries_dir = join(self.tiramisu_cache_root_path, 'extra_dictionaries')
-        if isdir(dictionaries_dir):
-            rmtree(dictionaries_dir)
-        makedirs(dictionaries_dir)
-        if isdir(extras_dictionaries_dir):
-            rmtree(extras_dictionaries_dir)
-        makedirs(extras_dictionaries_dir)
-        extras = []
-        upgrade = RougailUpgrade()
-        for source in sources:
-            root_path = join(source['source_directory'],
-                             self.dataset_name,
-                             )
-            if not isdir(root_path):
-                continue
-            for service in listdir(root_path):
-                # upgrade dictionaries
-                path = join(root_path,
-                            service,
-                            'dictionaries',
-                            )
-                if not isdir(path):
-                    continue
-                upgrade.load_xml_from_folders(path,
-                                              dictionaries_dir,
-                                              RougailConfig['variable_namespace'],
-                                              )
-            for service in listdir(root_path):
-                # upgrade extra dictionaries
-                path = join(root_path,
-                            service,
-                            'extras',
-                            )
-                if not isdir(path):
-                    continue
-                for namespace in listdir(path):
-                    extra_dir = join(path,
-                                     namespace,
-                                     )
-                    if not isdir(extra_dir):
-                        continue
-                    extra_dictionaries_dir = join(extras_dictionaries_dir,
-                                                  namespace,
-                                                  )
-                    if not isdir(extra_dictionaries_dir):
-                        makedirs(extra_dictionaries_dir)
-                        extras.append((namespace, [extra_dictionaries_dir]))
-                    upgrade.load_xml_from_folders(extra_dir,
-                                                  extra_dictionaries_dir,
-                                                  namespace,
-                                                  )
-        del upgrade
-        config = RougailConfig.copy()
-        config['functions_file'] = funcs_file
-        config['dictionaries_dir'] = [dictionaries_dir]
-        config['extra_dictionaries'] = {}
-        for extra in extras:
-            config['extra_dictionaries'][extra[0]] = extra[1]
-        eolobj = RougailConvert(rougailconfig=config)
-        eolobj.save(tiramisu_file)
-
-    async def load(self,
-                   risotto_context: Context,
-                   name: str,
-                   to_deploy: bool=False,
-                   ) -> Config:
-        if self.optiondescription is None:
-            # use file in cache
-            tiramisu_file = join(self.tiramisu_cache_root_path, 'tiramisu.py')
-            if not isfile(tiramisu_file):
-                raise Exception(_(f'unable to load the "{self.dataset_name}" configuration, is dataset loaded?'))
-            with open(tiramisu_file) as fileio:
-                tiramisu_locals = {}
-                try:
-                    exec(fileio.read(), None, tiramisu_locals)
-                except Exception as err:
-                    raise Exception(_(f'unable to load tiramisu file {tiramisu_file}: {err}'))
-
-            self.optiondescription = tiramisu_locals['option_0']
-            del tiramisu_locals
-        try:
-            letter = self.dataset_name[0]
-            if not to_deploy:
-                session_id = f'{letter}_{name}'
-            else:
-                session_id = f'{letter}td_{name}'
-            config = await Config(self.optiondescription,
-                                  session_id=session_id,
-                                  storage=self.save_storage,
-                                  display_name=tiramisu_display_name,
-                                  )
-            # change default rights
-            await config.property.read_only()
-            await config.permissive.add('basic')
-            await config.permissive.add('normal')
-            await config.permissive.add('expert')
-
-            # set information and owner
-            await config.owner.set(session_id)
-            await config.information.set(f'{self.dataset_name}_name', name)
-        except Exception as err:
-            if get_config()['global']['debug']:
-                print_exc()
-            msg = _(f'unable to load config for {self.dataset_name} "{name}": {err}')
-            await log.error_msg(risotto_context,
-                                None,
-                                msg,
-                                )
-        return config
-
-    async def _deploy_configuration(self,
-                                    dico: dict,
-                                    ) -> None:
-        config_std = dico['config_to_deploy']
-        config = dico['config']
-        # when deploy, calculate force_store_value
-        ro = await config_std.property.getdefault('read_only', 'append')
-        if 'force_store_value' not in ro:
-            await config_std.property.read_write()
-            if self.dataset_name == 'servermodel':
-                # server_deployed should be hidden
-                await config_std.forcepermissive.option('configuration.general.server_deployed').value.set(True)
-            ro = frozenset(list(ro) + ['force_store_value'])
-            rw = await config_std.property.getdefault('read_write', 'append')
-            rw = frozenset(list(rw) + ['force_store_value'])
-            await config_std.property.setdefault(ro, 'read_only', 'append')
-            await config_std.property.setdefault(rw, 'read_write', 'append')
-        await config_std.property.read_only()
-
-        # copy informations from 'to deploy' configuration to configuration
-        await config.value.importation(await config_std.value.exportation())
-        await config.permissive.importation(await config_std.permissive.exportation())
-        await config.property.importation(await config_std.property.exportation())
-
-    async def build_configuration(self,
-                                  config: Config,
-                                  ) -> dict:
-        configuration = {}
-        for option in await config.option.list('optiondescription'):
-            name = await option.option.name()
-            if name == 'services':
-                continue
-            if name == RougailConfig['variable_namespace']:
-                fullpath = False
-                flatten = True
-            else:
-                fullpath = True
-                flatten = False
-            configuration.update(await option.value.dict(leader_to_list=True, fullpath=fullpath, flatten=flatten))
-        return configuration

src/risotto/dispatcher.py

@@ -30,11 +30,14 @@ class CallDispatcher:
         if response.impl_get_information('multi'):
             if not isinstance(returns, list):
                 err = _(f'function {module_name}.{function_name} has to return a list')
-                raise CallError(err)
+                await log.error_msg(risotto_context, kwargs, err)
+                raise CallError(str(err))
         else:
             if not isinstance(returns, dict):
+                await log.error_msg(risotto_context, kwargs, returns)
                 err = _(f'function {module_name}.{function_name} has to return a dict')
-                raise CallError(err)
+                await log.error_msg(risotto_context, kwargs, err)
+                raise CallError(str(err))
             returns = [returns]
         if response is None:
             raise Exception('hu?')
@@ -47,10 +50,12 @@ class CallDispatcher:
                             await config.option(key).value.set(value)
                     except AttributeError:
                         err = _(f'function {module_name}.{function_name} return the unknown parameter "{key}" for the uri "{risotto_context.version}.{risotto_context.message}"')
-                        raise CallError(err)
+                        await log.error_msg(risotto_context, kwargs, err)
+                        raise CallError(str(err))
                     except ValueError:
                         err = _(f'function {module_name}.{function_name} return the parameter "{key}" with an unvalid value "{value}" for the uri "{risotto_context.version}.{risotto_context.message}"')
-                        raise CallError(err)
+                        await log.error_msg(risotto_context, kwargs, err)
+                        raise CallError(str(err))
                     await config.property.read_only()
                     mandatories = await config.value.mandatory()
                     if mandatories:
@@ -60,7 +65,8 @@ class CallDispatcher:
                         await config.value.dict()
                     except Exception as err:
                         err = _(f'function {module_name}.{function_name} return an invalid response {err} for the uri "{risotto_context.version}.{risotto_context.message}"')
-                        raise CallError(err)
+                        await log.error_msg(risotto_context, kwargs, err)
+                        raise CallError(str(err))
 
     async def call(self,
                    version: str,
@@ -83,16 +89,9 @@ class CallDispatcher:
         if message not in self.messages[version]:
             raise CallError(_(f'cannot find message "{version}.{message}"'))
         function_obj = self.messages[version][message]
-        # log
+        # do not start a new database connection
-        function_name = function_obj['function'].__name__
-        info_msg = _(f"call function {function_obj['full_module_name']}.{function_name}")
         if hasattr(old_risotto_context, 'connection'):
-            # do not start a new database connection
             risotto_context.connection = old_risotto_context.connection
-            await log.start(risotto_context,
-                            kwargs,
-                            info_msg,
-                            )
             await self.check_message_type(risotto_context,
                                           kwargs,
                                           )
@@ -102,19 +101,22 @@ class CallDispatcher:
                                                                 check_role,
                                                                 internal,
                                                                 )
-            try:
+            return await self.launch(risotto_context,
-                ret = await self.launch(risotto_context,
+                                     kwargs,
-                                        kwargs,
+                                     config_arguments,
-                                        config_arguments,
+                                     function_obj,
-                                        function_obj,
+                                     )
-                                        )
-            except CallError as err:
-                await log.failed(risotto_context,
-                                 str(err),
-                                 )
-                raise err from err
         else:
             try:
+                await self.check_message_type(risotto_context,
+                                              kwargs,
+                                              )
+                config_arguments = await self.load_kwargs_to_config(risotto_context,
+                                                                    f'{version}.{message}',
+                                                                    kwargs,
+                                                                    check_role,
+                                                                    internal,
+                                                                    )
                 async with self.pool.acquire() as connection:
                     await connection.set_type_codec(
                         'json',
@@ -124,40 +126,13 @@ class CallDispatcher:
                     )
                     risotto_context.connection = connection
                     async with connection.transaction():
-                        try:
+                        return await self.launch(risotto_context,
-                            await log.start(risotto_context,
+                                                 kwargs,
-                                            kwargs,
+                                                 config_arguments,
-                                            info_msg,
+                                                 function_obj,
-                                            )
+                                                 )
-                            await self.check_message_type(risotto_context,
-                                                          kwargs,
-                                                          )
-                            config_arguments = await self.load_kwargs_to_config(risotto_context,
-                                                                                f'{version}.{message}',
-                                                                                kwargs,
-                                                                                check_role,
-                                                                                internal,
-                                                                                )
-                            ret = await self.launch(risotto_context,
-                                                    kwargs,
-                                                    config_arguments,
-                                                    function_obj,
-                                                    )
-                            # log the success
-                            await log.success(risotto_context,
-                                              ret,
-                                              )
-                            if not internal and isinstance(ret, dict):
-                                ret['context_id'] = risotto_context.context_id
-                        except CallError as err:
-                            if get_config()['global']['debug']:
-                                print_exc()
-                            await log.failed(risotto_context,
-                                             str(err),
-                                             )
-                            raise err from err
             except CallError as err:
-                raise err from err
+                raise err
             except Exception as err:
                 # if there is a problem with arguments, just send an error and do nothing
                 if get_config()['global']['debug']:
@@ -171,11 +146,8 @@ class CallDispatcher:
                     )
                     risotto_context.connection = connection
                     async with connection.transaction():
-                        await log.failed(risotto_context,
+                        await log.error_msg(risotto_context, kwargs, err)
-                                         str(err),
+                raise err
-                                         )
-                raise err from err
-        return ret
 
 
 class PublishDispatcher:
@@ -208,7 +180,6 @@ class PublishDispatcher:
         remote_kw = dumps({'kwargs': kwargs,
                            'context': {'username': risotto_context.username,
                                        'paths': risotto_context.paths,
-                                       'context_id': risotto_context.context_id,
                                        }
                           })
         # FIXME should be better :/
@@ -248,41 +219,26 @@ class PublishDispatcher:
                                                             False,
                                                             False,
                                                             )
-        async with self.pool.acquire() as connection:
+        for function_obj in self.messages[version][message]['functions']:
-            await connection.set_type_codec(
+            async with self.pool.acquire() as connection:
-                'json',
-                encoder=dumps,
-                decoder=loads,
-                schema='pg_catalog'
-            )
-            risotto_context.connection = connection
-            for function_obj in self.messages[version][message]['functions']:
-                function_name = function_obj['function'].__name__
-                info_msg = _(f"call function {function_obj['full_module_name']}.{function_name}")
                 try:
+                    await self.check_message_type(risotto_context,
+                                                  kwargs,
+                                                  )
+                    await connection.set_type_codec(
+                        'json',
+                        encoder=dumps,
+                        decoder=loads,
+                        schema='pg_catalog'
+                    )
+                    risotto_context.connection = connection
                     async with connection.transaction():
-                        try:
+                        await self.launch(risotto_context,
-                            await log.start(risotto_context,
+                                          kwargs,
-                                            kwargs,
+                                          config_arguments,
-                                            info_msg,
+                                          function_obj,
-                                            )
+                                          )
-                            await self.check_message_type(risotto_context,
+                except CallError as err:
-                                                          kwargs,
-                                                          )
-                            await self.launch(risotto_context,
-                                              kwargs,
-                                              config_arguments,
-                                              function_obj,
-                                              )
-                            # log the success
-                            await log.success(risotto_context)
-                        except CallError as err:
-                            if get_config()['global']['debug']:
-                                print_exc()
-                            await log.failed(risotto_context,
-                                             str(err),
-                                             )
-                except CallError:
                     pass
                 except Exception as err:
                     # if there is a problem with arguments, log and do nothing
@@ -297,15 +253,12 @@ class PublishDispatcher:
                         )
                         risotto_context.connection = connection
                         async with connection.transaction():
-                            await log.failed(risotto_context,
+                            await log.error_msg(risotto_context, kwargs, err)
-                                             str(err),
-                                             )
 
 
 class Dispatcher(register.RegisterDispatcher,
                  CallDispatcher,
-                 PublishDispatcher,
+                 PublishDispatcher):
-                 ):
     """ Manage message (call or publish)
     so launch a function when a message is called
     """
@@ -321,7 +274,6 @@ class Dispatcher(register.RegisterDispatcher,
         risotto_context = Context()
         risotto_context.username = context['username']
         risotto_context.paths = copy(context['paths'])
-        risotto_context.context_id = context['context_id']
         risotto_context.paths.append(uri)
         risotto_context.uri = uri
         risotto_context.type = type
@@ -335,6 +287,7 @@ class Dispatcher(register.RegisterDispatcher,
                                  ) -> None:
         if self.messages[risotto_context.version][risotto_context.message]['pattern'] != risotto_context.type:
             msg = _(f'{risotto_context.uri} is not a {risotto_context.type} message')
+            await log.error_msg(risotto_context, kwargs, msg)
             raise CallError(msg)
 
     async def load_kwargs_to_config(self,
@@ -380,7 +333,7 @@ class Dispatcher(register.RegisterDispatcher,
             parameters = await subconfig.value.dict()
             if extra_parameters:
                 parameters.update(extra_parameters)
-        return parameters
+            return parameters
 
     def get_service(self,
                     name: str):
@@ -389,15 +342,14 @@ class Dispatcher(register.RegisterDispatcher,
     async def check_role(self,
                          config: Config,
                          user_login: str,
-                         uri: str,
+                         uri: str) -> None:
-                         ) -> None:
         async with self.pool.acquire() as connection:
             async with connection.transaction():
                 # Verify if user exists and get ID
                 sql = '''
                 SELECT UserId
                 FROM UserUser
-                WHERE Login = $1
+                WHERE UserLogin = $1
                 '''
                 user_id = await connection.fetchval(sql,
                                                     user_login)
@@ -442,7 +394,10 @@ class Dispatcher(register.RegisterDispatcher,
                      ) -> Optional[Dict]:
         # so send the message
         function = function_obj['function']
-        risotto_context.module = function_obj['module'].split('.', 1)[0]
+        submodule_name = function_obj['module']
+        function_name = function.__name__
+        risotto_context.module = submodule_name.split('.', 1)[0]
+        info_msg = _(f'in module {submodule_name}.{function_name}')
         # build argument for this function
         if risotto_context.type == 'rpc':
             kw = config_arguments
@@ -453,8 +408,7 @@ class Dispatcher(register.RegisterDispatcher,
                     kw[key] = value
 
         kw['risotto_context'] = risotto_context
-        # launch
+        returns = await function(self.injected_self[function_obj['module']], **kw)
-        returns = await function(self.get_service(function_obj['module']), **kw)
         if risotto_context.type == 'rpc':
             # valid returns
             await self.valid_call_returns(risotto_context,
@@ -462,6 +416,12 @@ class Dispatcher(register.RegisterDispatcher,
                                           returns,
                                           kwargs,
                                           )
+        # log the success
+        await log.info_msg(risotto_context,
+                           {'arguments': kwargs,
+                            'returns': returns},
+                           info_msg,
+                           )
         # notification
         if function_obj.get('notification'):
             notif_version, notif_message = function_obj['notification'].split('.', 1)

src/risotto/http.py

@@ -1,4 +1,4 @@
-from aiohttp.web import Application, Response, get, post, HTTPBadRequest, HTTPInternalServerError, HTTPNotFound, static
+from aiohttp.web import Application, Response, get, post, HTTPBadRequest, HTTPInternalServerError, HTTPNotFound
 from json import dumps
 from traceback import print_exc
 try:
@@ -12,13 +12,12 @@ from .utils import _
 from .context import Context
 from .error import CallError, NotAllowedError, RegistrationError
 from .message import get_messages
-#from .logger import log
+from .logger import log
 from .config import get_config
 from . import services
 
 
 extra_routes = {}
-extra_statics = {}
 
 
 def create_context(request):
@@ -30,31 +29,19 @@ def create_context(request):
 
 
 def register(version: str,
-             path: str,
+             path: str):
-             ):
     """ Decorator to register function to the http route
     """
     def decorator(function):
         if path in extra_routes:
-            raise RegistrationError(f'the route "{path}" is already registered')
+            raise RegistrationError(f'the route {path} is already registered')
         extra_routes[path] = {'function': function,
-                              'version': version,
+                              'version': version}
-                              }
     return decorator
 
 
-def register_static(path: str,
-                    directory: str,
-                    ) -> None:
-    if path in extra_statics:
-        raise RegistrationError(f'the static path "{path}" is already registered')
-    extra_statics[path] = directory
-
-
 class extra_route_handler:
-    async def __new__(cls,
+    async def __new__(cls, request):
-                      request,
-                      ):
         kwargs = dict(request.match_info)
         kwargs['request'] = request
         kwargs['risotto_context'] = create_context(request)
@@ -80,8 +67,7 @@ class extra_route_handler:
         # await log.info_msg(kwargs['risotto_context'],
         #                    dict(request.match_info))
         return Response(text=dumps(returns),
-                        content_type='application/json',
+                        content_type='application/json')
-                        )
 
 
 async def handle(request):
@@ -110,13 +96,11 @@ async def handle(request):
             print_exc()
         raise HTTPInternalServerError(reason=str(err))
     return Response(text=dumps({'response': text}),
-                    content_type='application/json',
+                    content_type='application/json')
-                    )
 
 
 async def api(request,
-              risotto_context,
+              risotto_context):
-              ):
     global TIRAMISU
     if not TIRAMISU:
         # check all URI that have an associated role
@@ -146,7 +130,7 @@ async def api(request,
 async def get_app(loop):
     """ build all routes
     """
-    global extra_routes, extra_statics
+    global extra_routes
     services.link_to_dispatcher(dispatcher)
     app = Application(loop=loop)
     routes = []
@@ -168,8 +152,7 @@ async def get_app(loop):
     for version in versions:
         api_route = {'function': api,
                      'version': version,
-                     'path': f'/api/{version}',
+                     'path': f'/api/{version}'}
-                     }
         extra_handler = type(api_route['path'], (extra_route_handler,), api_route)
         routes.append(get(api_route['path'], extra_handler))
         print(f'  - {api_route["path"]} (http_get)')
@@ -186,22 +169,12 @@ async def get_app(loop):
             extra_handler = type(path, (extra_route_handler,), extra)
             routes.append(get(path, extra_handler))
             print(f'  - {path} (http_get)')
-    if extra_statics:
-        if not extra_routes:
-            print(_('======== Registered static routes ========'))
-        for path, directory in extra_statics.items():
-            routes.append(static(path, directory))
-            print(f'  - {path} (static)')
     del extra_routes
-    del extra_statics
     app.router.add_routes(routes)
     await dispatcher.register_remote()
     print()
     await dispatcher.on_join()
-    return await loop.create_server(app.make_handler(),
+    return await loop.create_server(app.make_handler(), '*', get_config()['http_server']['port'])
-                                    '*',
-                                    get_config()['http_server']['port'],
-                                    )
 
 
 TIRAMISU = None

src/risotto/logger.py

@@ -1,7 +1,7 @@
-from typing import Dict, Any, Optional
+from typing import Dict, Any
-from json import dumps, loads
+from json import dumps
 from asyncpg.exceptions import UndefinedTableError
-from datetime import datetime
+
 
 from .context import Context
 from .utils import _
@@ -13,79 +13,26 @@ class Logger:
     """
     async def insert(self,
                      msg: str,
-                     uri: str,
+                     path: str,
-                     uris: str,
+                     risotto_context: str,
-                     risotto_context: Context,
                      level: str,
-                     data: Any=None,
+                     data: Any= None) -> None:
-                     start: bool=False,
+        insert = 'INSERT INTO log(Msg, Path, Username, Level'
-                     ) -> None:
+        values = 'VALUES($1,$2,$3,$4'
-        insert = 'INSERT INTO RisottoLog(Msg, URI, URIS, UserLogin, Level'
+        args = [msg, path, risotto_context.username, level]
-        values = 'VALUES($1,$2,$3,$4,$5'
-        args = [msg, uri, uris, risotto_context.username, level]
         if data:
             insert += ', Data'
-            values += ',$6'
+            values += ',$5'
             args.append(dumps(data))
-        context_id = risotto_context.context_id
-        if context_id is not None:
-            insert += ', ContextId'
-            if data:
-                values += ',$7'
-            else:
-                values += ',$6'
-            args.append(context_id)
 
-        sql = insert + ') ' + values + ') RETURNING LogId'
+        sql = insert + ') ' + values + ')'
         try:
-            log_id = await risotto_context.connection.fetchval(sql, *args)
+            await risotto_context.connection.fetch(sql, *args)
-            if context_id is None and start:
-                risotto_context.context_id = log_id
-            if start:
-                risotto_context.start_id = log_id
         except UndefinedTableError as err:
             raise Exception(_(f'cannot access to database ({err}), was the database really created?'))
 
-    async def query(self,
-                    risotto_context: Context,
-                    context_id: int,
-                    uri: Optional[str],
-                    ) -> list:
-        sql = '''SELECT Msg as msg, URI as uri_name, URIS as uris, UserLogin as user_login, Level as level, Data as data, StartDate as start_date, StopDate as stop_date
-        FROM RisottoLog
-        WHERE UserLogin = $1 AND (LogId = $2 OR ContextId = $2)
-        '''
-        args = [sql, risotto_context.username, context_id]
-        if uri is not None:
-            sql += ' AND URI = $3'
-            args.append(uri)
-        ret = []
-        for row in await risotto_context.connection.fetch(*args):
-            d = {}
-            for key, value in row.items():
-                if key == 'data':
-                    if not value:
-                        value = {}
-                    else:
-                        value = loads(value)
-                elif key in ['start_date', 'stop_date']:
-                    value = str(value)
-                d[key] = value
-            ret.append(d)
-        return ret
-
-    def _get_last_uri(self,
-                      risotto_context: Context,
-                      ) -> str:
-        if risotto_context.paths:
-            return risotto_context.paths[-1]
-        return ''
-
     def _get_message_paths(self,
-                           risotto_context: Context,
+                           risotto_context: Context):
-                           ) -> str:
-        if not risotto_context.paths:
-            return ''
         paths = risotto_context.paths
         if risotto_context.type:
             paths_msg = f' {risotto_context.type} '
@@ -102,109 +49,44 @@ class Logger:
                         risotto_context: Context,
                         arguments,
                         error: str,
-                        msg: str='',
+                        msg: str=''):
-                        ):
         """ send message when an error append
         """
         paths_msg = self._get_message_paths(risotto_context)
         print(_(f'{risotto_context.username}: ERROR: {error} ({paths_msg} with arguments "{arguments}": {msg})'))
         await self.insert(msg,
-                          self._get_last_uri(risotto_context),
                           paths_msg,
                           risotto_context,
                           'Error',
-                          arguments,
+                          arguments)
-                          )
 
     async def info_msg(self,
                        risotto_context: Context,
                        arguments: Dict,
-                       msg: str='',
+                       msg: str=''):
-                       ) -> None:
         """ send message with common information
         """
-        paths_msg = self._get_message_paths(risotto_context)
+        if risotto_context.paths:
+            paths_msg = self._get_message_paths(risotto_context)
+        else:
+            paths_msg = ''
         if get_config()['global']['debug']:
             print(_(f'{risotto_context.username}: INFO:{paths_msg}: {msg}'))
         await self.insert(msg,
-                          self._get_last_uri(risotto_context),
                           paths_msg,
                           risotto_context,
                           'Info',
-                          arguments,
+                          arguments)
-                          )
-
-    async def start(self,
-                    risotto_context: Context,
-                    arguments: dict,
-                    msg: str,
-                    ) -> None:
-        paths_msg = self._get_message_paths(risotto_context)
-        if get_config()['global']['debug']:
-            print(_(f'{risotto_context.username}: START:{paths_msg}: {msg}'))
-        await self.insert(msg,
-                          self._get_last_uri(risotto_context),
-                          paths_msg,
-                          risotto_context,
-                          'Start',
-                          arguments,
-                          start=True,
-                          )
-
-    async def success(self,
-                      risotto_context: Context,
-                      returns: Optional[dict]=None,
-                      ) -> None:
-        if get_config()['global']['debug']:
-            paths_msg = self._get_message_paths(risotto_context)
-            print(_(f'{risotto_context.username}: SUCCESS:{paths_msg}({risotto_context.context_id})'))
-        sql = """UPDATE RisottoLog
-        SET StopDate = $2,
-        Level = 'SUCCESS'
-        """
-        args = [datetime.now()]
-        if returns:
-            sql += """, Returns = $3
-            """
-            args.append(dumps(returns))
-        sql += """WHERE LogId = $1
-        """
-        await risotto_context.connection.execute(sql,
-                                                 risotto_context.start_id,
-                                                 *args,
-                                                 )
-
-    async def failed(self,
-                     risotto_context: Context,
-                     err: str,
-                     ) -> None:
-        if get_config()['global']['debug']:
-            paths_msg = self._get_message_paths(risotto_context)
-            print(_(f'{risotto_context.username}: FAILED:{paths_msg}({risotto_context.context_id}): err'))
-        sql = """UPDATE RisottoLog
-        SET StopDate = $2,
-        Level = 'FAILED',
-        Msg = $3
-        WHERE LogId = $1
-        """
-        await risotto_context.connection.execute(sql,
-                                                 risotto_context.start_id,
-                                                 datetime.now(),
-                                                 err,
-                                                 )
 
     async def info(self,
                    risotto_context,
-                   msg,
+                   msg):
-                   ):
         if get_config()['global']['debug']:
             print(msg)
         await self.insert(msg,
-                          '',
                           None,
                           risotto_context,
-                          'Info',
+                          'Info')
-                          )
 
 
 log = Logger()

src/risotto/register.py

@@ -7,7 +7,6 @@ from typing import Callable, Optional, List
 from asyncpg import create_pool
 from json import dumps, loads
 from pkg_resources import iter_entry_points
-from traceback import print_exc
 import risotto
 from .utils import _
 from .error import RegistrationError
@@ -24,7 +23,7 @@ class Services():
 
     def load_services(self):
         for entry_point in iter_entry_points(group='risotto_services'):
-            self.services.setdefault(entry_point.name, {})
+            self.services.setdefault(entry_point.name, [])
         self.services_loaded = True
 
     def load_modules(self,
@@ -33,20 +32,21 @@ class Services():
         for entry_point in iter_entry_points(group='risotto_modules'):
             service_name, module_name = entry_point.name.split('.')
             if limit_services is None or service_name in limit_services:
-                self.services[service_name][module_name] = entry_point.load()
+                setattr(self, module_name, entry_point.load())
+                self.services[service_name].append(module_name)
         self.modules_loaded = True
-#
+
-#    def get_services(self):
+    def get_services(self):
-#        if not self.services_loaded:
+        if not self.services_loaded:
-#            self.load_services()
+            self.load_services()
-#        return [(service, getattr(self, service)) for service in self.services]
+        return [(s, getattr(self, s)) for s in self.services]
 
     def get_modules(self,
                     limit_services: Optional[List[str]]=None,
                     ) -> List[str]:
         if not self.modules_loaded:
             self.load_modules(limit_services=limit_services)
-        return [(module + '.' + submodule, entry_point) for module, submodules in self.services.items() for submodule, entry_point in submodules.items()]
+        return [(module + '.' + submodule, getattr(self, submodule)) for module, submodules in self.services.items() for submodule in submodules]
 
     def get_services_list(self):
         return self.services.keys()
@@ -84,11 +84,11 @@ def register(uris: str,
 
     def decorator(function):
         for uri in uris:
-            dispatcher.set_function(uri,
+            version, message = uri.split('.', 1)
+            dispatcher.set_function(version,
+                                    message,
                                     notification,
-                                    function,
+                                    function)
-                                    function.__module__
-                                    )
     return decorator
 
 
@@ -185,21 +185,21 @@ class RegisterDispatcher:
             raise RegistrationError(_(f'error with {module_name}.{function_name} arguments: {msg}'))
 
     def set_function(self,
-                     uri: str,
+                     version: str,
+                     message: str,
                      notification: str,
                      function: Callable,
-                     full_module_name: str,
                      ):
         """ register a function to an URI
         URI is a message
         """
-        version, message = uri.split('.', 1)
+
         # check if message exists
         if message not in self.messages[version]:
             raise RegistrationError(_(f'the message {message} not exists'))
 
         # xxx submodule can only be register with v1.yyy.xxx..... message
-        risotto_module_name, submodule_name = full_module_name.split('.')[-3:-1]
+        risotto_module_name, submodule_name = function.__module__.split('.')[-3:-1]
         module_name = risotto_module_name.split('_')[-1]
         message_module, message_submodule, message_name = message.split('.', 2)
         if message_module not in self.risotto_modules:
@@ -214,7 +214,7 @@ class RegisterDispatcher:
 
         # check if already register
         if 'function' in self.messages[version][message]:
-            raise RegistrationError(_(f'uri {uri} already registered'))
+            raise RegistrationError(_(f'uri {version}.{message} already registered'))
 
         # register
         if self.messages[version][message]['pattern'] == 'rpc':
@@ -224,7 +224,6 @@ class RegisterDispatcher:
         register(version,
                  message,
                  f'{module_name}.{submodule_name}',
-                 full_module_name,
                  function,
                  function_args,
                  notification,
@@ -234,13 +233,11 @@ class RegisterDispatcher:
                      version: str,
                      message: str,
                      module_name: str,
-                     full_module_name: str,
                      function: Callable,
                      function_args: list,
                      notification: Optional[str],
                      ):
         self.messages[version][message]['module'] = module_name
-        self.messages[version][message]['full_module_name'] = full_module_name
         self.messages[version][message]['function'] = function
         self.messages[version][message]['arguments'] = function_args
         if notification:
@@ -250,7 +247,6 @@ class RegisterDispatcher:
                        version: str,
                        message: str,
                        module_name: str,
-                       full_module_name: str,
                        function: Callable,
                        function_args: list,
                        notification: Optional[str],
@@ -259,10 +255,8 @@ class RegisterDispatcher:
             self.messages[version][message]['functions'] = []
 
         dico = {'module': module_name,
-                'full_module_name': full_module_name,
                 'function': function,
-                'arguments': function_args,
+                'arguments': function_args}
-                }
         if notification and notification:
             dico['notification'] = notification
         self.messages[version][message]['functions'].append(dico)
@@ -277,7 +271,7 @@ class RegisterDispatcher:
         try:
             self.injected_self[submodule_name] = module.Risotto(test)
         except AttributeError as err:
-            print(_(f'unable to register the module {submodule_name}, this module must have Risotto class'))
+            raise RegistrationError(_(f'unable to register the module {submodule_name}, this module must have Risotto class'))
 
     def validate(self):
         """ check if all messages have a function
@@ -287,9 +281,9 @@ class RegisterDispatcher:
             for message, message_obj in messages.items():
                 if not 'functions' in message_obj and not 'function' in message_obj:
                     if message_obj['pattern'] == 'event':
-                        print(f'{version}.{message} prêche dans le désert')
+                        print(f'{message} prêche dans le désert')
                     else:
-                        missing_messages.append(f'{version}.{message}')
+                        missing_messages.append(message)
         if missing_messages:
             raise RegistrationError(_(f'no matching function for uri {missing_messages}'))
 
@@ -315,17 +309,11 @@ class RegisterDispatcher:
                     risotto_context.type = None
                     risotto_context.connection = connection
                     risotto_context.module = submodule_name.split('.', 1)[0]
-                    info_msg = _(f'in function risotto_{submodule_name}.on_join')
+                    info_msg = _(f'in module risotto_{submodule_name}.on_join')
                     await log.info_msg(risotto_context,
                                        None,
                                        info_msg)
-                    try:
+                    await module.on_join(risotto_context)
-                        await module.on_join(risotto_context)
-                    except Exception as err:
-                        if get_config()['global']['debug']:
-                            print_exc()
-                        msg = _(f'on_join returns an error in module {submodule_name}: {err}')
-                        await log.error_msg(risotto_context, {}, msg)
 
     async def load(self):
         # valid function's arguments

src/risotto/utils.py

@@ -1,27 +1,9 @@
 class Undefined:
     pass
-undefined = Undefined()
 
 
 def _(s):
     return s
 
 
-def tiramisu_display_name(kls,
+undefined = Undefined()
-                          dyn_name: 'Base'=None,
-                          suffix: str=None,
-                          ) -> str:
-    if dyn_name is not None:
-        name = dyn_name
-    else:
-        name = kls.impl_getname()
-    doc = kls.impl_get_information('doc', None)
-    if doc:
-        doc = str(doc)
-        if doc.endswith('.'):
-            doc = doc[:-1]
-        if suffix:
-            doc += suffix
-        if name != doc:
-            name += f' ({doc})'
-    return name

tests/test_config.py

@@ -392,6 +392,7 @@ async def test_server_created_base():
                              release_distribution='last',
                              site_name='site_1',
                              zones_name=['zones'],
+                             zones_ip=['1.1.1.1'],
                              )
     assert list(config_module.server) == [server_name]
     assert set(config_module.server[server_name]) == {'server', 'server_to_deploy', 'funcs_file'}
@@ -419,6 +420,7 @@ async def test_server_created_own_sm():
                              release_distribution='last',
                              site_name='site_1',
                              zones_name=['zones'],
+                             zones_ip=['1.1.1.1'],
                              )
     assert list(config_module.server) == [server_name]
     assert set(config_module.server[server_name]) == {'server', 'server_to_deploy', 'funcs_file'}
@@ -467,6 +469,7 @@ async def test_server_configuration_get():
                              release_distribution='last',
                              site_name='site_1',
                              zones_name=['zones'],
+                             zones_ip=['1.1.1.1'],
                              )
     #
     await config_module.server[server_name]['server'].property.read_write()
@@ -512,6 +515,7 @@ async def test_server_configuration_deployed():
                              release_distribution='last',
                              site_name='site_1',
                              zones_name=['zones'],
+                             zones_ip=['1.1.1.1'],
                              )
     #
     await config_module.server[server_name]['server'].property.read_write()