Merge branch 'develop' into dist/risotto/risotto-2.8.0/develop

sql/risotto.sql

@@ -1,12 +1,12 @@
 CREATE TABLE RisottoLog(
   LogId SERIAL PRIMARY KEY,
+  ContextId INTEGER,
   Msg VARCHAR(255) NOT NULL,
   URI VARCHAR(255),
   URIS VARCHAR(255),
   UserLogin VARCHAR(100) NOT NULL,
-  Level VARCHAR(10) NOT NULL,
+  Status INTEGER NOT NULL,
-  ContextId INTEGER,
+  Kwargs JSON,
-  Data JSON,
   Returns JSON,
   StartDate timestamp DEFAULT current_timestamp,
   StopDate timestamp

src/risotto/config.py

@@ -65,6 +65,18 @@ if 'CELERYRISOTTO_DB_USER' in environ:
     CELERYRISOTTO_DB_USER = environ['CELERYRISOTTO_DB_USER']
 else:
     CELERYRISOTTO_DB_USER = config.get('CELERYRISOTTO_DB_USER', None)
+if 'LEMUR_DB_NAME' in environ:
+    LEMUR_DB_NAME = environ['LEMUR_DB_NAME']
+else:
+    LEMUR_DB_NAME = config.get('LEMUR_DB_NAME', None)
+if 'LEMUR_DB_PASSWORD' in environ:
+    LEMUR_DB_PASSWORD = environ['LEMUR_DB_PASSWORD']
+else:
+    LEMUR_DB_PASSWORD = config.get('LEMUR_DB_PASSWORD', None)
+if 'LEMUR_DB_USER' in environ:
+    LEMUR_DB_USER = environ['LEMUR_DB_USER']
+else:
+    LEMUR_DB_USER = config.get('LEMUR_DB_USER', None)
 if 'DB_ADDRESS' in environ:
     DB_ADDRESS = environ['DB_ADDRESS']
 else:
@@ -115,6 +127,11 @@ if 'PASSWORD_URL' in environ:
     PASSWORD_URL = environ['PASSWORD_URL']
 else:
     PASSWORD_URL = config.get('PASSWORD_URL', 'https://localhost:8001/')
+
+if 'PASSWORD_LENGTH' in environ:
+    PASSWORD_LENGTH = int(environ['PASSWORD_LENGTH'])
+else:
+    PASSWORD_LENGTH = int(config.get('PASSWORD_LENGTH', 20))
 if 'PKI_ADMIN_PASSWORD' in environ:
     PKI_ADMIN_PASSWORD = environ['PKI_ADMIN_PASSWORD']
 else:
@@ -136,7 +153,8 @@ def dsn_factory(database, user, password, address=DB_ADDRESS):
 
 _config = {'database': {'dsn': dsn_factory(RISOTTO_DB_NAME, RISOTTO_DB_USER, RISOTTO_DB_PASSWORD),
                         'tiramisu_dsn': dsn_factory(TIRAMISU_DB_NAME, TIRAMISU_DB_USER, TIRAMISU_DB_PASSWORD),
-                        'celery_dsn': dsn_factory(CELERYRISOTTO_DB_NAME, CELERYRISOTTO_DB_USER, CELERYRISOTTO_DB_PASSWORD)
+                        'celery_dsn': dsn_factory(CELERYRISOTTO_DB_NAME, CELERYRISOTTO_DB_USER, CELERYRISOTTO_DB_PASSWORD),
+                        'lemur_dns': dsn_factory(LEMUR_DB_NAME, LEMUR_DB_USER, LEMUR_DB_PASSWORD),
                         },
            'http_server': {'port': RISOTTO_PORT,
                            'default_user': DEFAULT_USER,
@@ -155,6 +173,7 @@ _config = {'database': {'dsn': dsn_factory(RISOTTO_DB_NAME, RISOTTO_DB_USER, RIS
                         'admin_password': PASSWORD_ADMIN_PASSWORD,
                         'device_identifier': PASSWORD_DEVICE_IDENTIFIER,
                         'service_url': PASSWORD_URL,
+                        'length': PASSWORD_LENGTH,
                         },
            'pki': {'admin_password': PKI_ADMIN_PASSWORD,
                    'owner': PKI_ADMIN_EMAIL,

src/risotto/context.py

@@ -3,3 +3,11 @@ class Context:
         self.paths = []
         self.context_id = None
         self.start_id = None
+
+    def copy(self):
+        context = Context()
+        for key, value in self.__dict__.items():
+            if key.startswith('__'):
+                continue
+            setattr(context, key, value)
+        return context

src/risotto/controller.py

@@ -65,8 +65,7 @@ class Controller:
                                  )
 
     @staticmethod
-    async def check_role(self,
+    async def check_role(uri: str,
-                         uri: str,
                          username: str,
                          **kwargs: dict,
                          ) -> None:
@@ -309,6 +308,7 @@ except:
         await config_std.property.read_only()
 
         # copy informations from 'to deploy' configuration to configuration
+        await config.information.importation(await config_std.information.exportation())
         await config.value.importation(await config_std.value.exportation())
         await config.permissive.importation(await config_std.permissive.exportation())
         await config.property.importation(await config_std.property.exportation())

src/risotto/dispatcher.py

@@ -42,14 +42,21 @@ class CallDispatcher:
             for ret in returns:
                 async with await Config(response, display_name=lambda self, dyn_name, suffix: self.impl_getname()) as config:
                     await config.property.read_write()
+                    key = None
                     try:
                         for key, value in ret.items():
                             await config.option(key).value.set(value)
-                    except AttributeError:
+                    except AttributeError as err:
+                        if key is not None:
                             err = _(f'function {module_name}.{function_name} return the unknown parameter "{key}" for the uri "{risotto_context.version}.{risotto_context.message}"')
+                        else:
+                            err = _(f'function {module_name}.{function_name} return unconsistency data "{err}" for the uri "{risotto_context.version}.{risotto_context.message}"')
                         raise CallError(err)
                     except ValueError as err:
+                        if key is not None:
                             err = _(f'function {module_name}.{function_name} return the invalid parameter "{key}" for the uri "{risotto_context.version}.{risotto_context.message}": {err}')
+                        else:
+                            err = _(f'function {module_name}.{function_name} return unconsistency error for the uri "{risotto_context.version}.{risotto_context.message}": {err}')
                         raise CallError(err)
                     await config.property.read_only()
                     mandatories = await config.value.mandatory()
@@ -89,7 +96,6 @@ class CallDispatcher:
         if hasattr(old_risotto_context, 'connection'):
             # do not start a new database connection
             risotto_context.connection = old_risotto_context.connection
-            risotto_context.log_connection = old_risotto_context.log_connection
             await log.start(risotto_context,
                             kwargs,
                             info_msg,
@@ -119,16 +125,7 @@ class CallDispatcher:
                 raise CallError(err) from err
         else:
             error = None
-            async with self.pool.acquire() as log_connection:
-                await log_connection.set_type_codec(
-                    'json',
-                    encoder=dumps,
-                    decoder=loads,
-                    schema='pg_catalog'
-                )
-                async with log_connection.transaction():
             try:
-                        risotto_context.log_connection = log_connection
                 async with self.pool.acquire() as connection:
                     await connection.set_type_codec(
                         'json',
@@ -199,8 +196,6 @@ class PublishDispatcher:
             for message, message_infos in messages.items():
                 # event not emit locally
                 if message_infos['pattern'] == 'event' and 'functions' in message_infos and message_infos['functions']:
-#                    module, submodule, submessage = message.split('.', 2)
-#                    if f'{module}.{submodule}' not in self.injected_self:
                     uri = f'{version}.{message}'
                     print(f'  - {uri}')
                     await self.listened_connection.add_listener(uri,
@@ -236,21 +231,34 @@ class PublishDispatcher:
         version, message = uri.split('.', 1)
         loop = get_event_loop()
         remote_kw = loads(payload)
+        for function_obj in self.messages[version][message]['functions']:
             risotto_context = self.build_new_context(remote_kw['context'],
                                                      version,
                                                      message,
                                                      'event',
                                                      )
-        callback = lambda: ensure_future(self._publish(version,
+            callback = self.get_callback(version, message, function_obj, risotto_context, remote_kw['kwargs'],)
-                                                       message,
-                                                       risotto_context,
-                                                       **remote_kw['kwargs'],
-                                                       ))
             loop.call_soon(callback)
 
+    def get_callback(self,
+                     version,
+                     message,
+                     function_obj,
+                     risotto_context,
+                     kwargs,
+                     ):
+        return lambda: ensure_future(self._publish(version,
+                                                   message,
+                                                   function_obj,
+                                                   risotto_context,
+                                                   **kwargs,
+                                                   ))
+
+
     async def _publish(self,
                        version: str,
                        message: str,
+                       function_obj,
                        risotto_context: Context,
                        **kwargs,
                        ) -> None:
@@ -260,15 +268,6 @@ class PublishDispatcher:
                                                             False,
                                                             False,
                                                             )
-        async with self.pool.acquire() as log_connection:
-            await log_connection.set_type_codec(
-                'json',
-                encoder=dumps,
-                decoder=loads,
-                schema='pg_catalog'
-            )
-            async with log_connection.transaction():
-                risotto_context.log_connection = log_connection
         async with self.pool.acquire() as connection:
             await connection.set_type_codec(
                 'json',
@@ -277,7 +276,6 @@ class PublishDispatcher:
                 schema='pg_catalog'
             )
             risotto_context.connection = connection
-                    for function_obj in self.messages[version][message]['functions']:
             function_name = function_obj['function'].__name__
             info_msg = _(f"call function {function_obj['full_module_name']}.{function_name}")
             try:
@@ -309,15 +307,6 @@ class PublishDispatcher:
                 # if there is a problem with arguments, log and do nothing
                 if get_config()['global']['debug']:
                     print_exc()
-                            async with self.pool.acquire() as connection:
-                                await connection.set_type_codec(
-                                    'json',
-                                    encoder=dumps,
-                                    decoder=loads,
-                                    schema='pg_catalog'
-                                )
-                                risotto_context.connection = connection
-                                async with connection.transaction():
                 await log.failed(risotto_context,
                                  str(err),
                                  )
@@ -348,6 +337,7 @@ class Dispatcher(register.RegisterDispatcher,
         risotto_context.type = type
         risotto_context.message = message
         risotto_context.version = version
+        risotto_context.pool = self.pool
         return risotto_context
 
     async def check_message_type(self,
@@ -485,6 +475,8 @@ class Dispatcher(register.RegisterDispatcher,
                                           )
         # notification
         if function_obj.get('notification'):
+            if returns is None:
+                raise Exception(_(f'function "{function_obj["full_module_name"]}.{function_obj["function"].__name__}" must returns something for {function_obj["notification"]}!'))
             notif_version, notif_message = function_obj['notification'].split('.', 1)
             if not isinstance(returns, list):
                 send_returns = [returns]

src/risotto/http.py

@@ -23,9 +23,13 @@ extra_statics = {}
 
 def create_context(request):
     risotto_context = Context()
-    risotto_context.username = request.match_info.get('username',
+    if 'username' in dict(request.match_info):
-                                                      get_config()['http_server']['default_user'],
+        username = request.match_info['username']
-                                                      )
+    elif 'username' in request.headers:
+        username = request.headers['username']
+    else:
+        username = get_config()['http_server']['default_user']
+    risotto_context.username = username
     return risotto_context
 
 

src/risotto/logger.py

@@ -2,35 +2,57 @@ from typing import Dict, Any, Optional
 from json import dumps, loads
 from asyncpg.exceptions import UndefinedTableError
 from datetime import datetime
+from asyncio import Lock
 
 from .context import Context
 from .utils import _
 from .config import get_config
 
+database_lock = Lock()
+
+
+LEVELS = ['Error', 'Info', 'Success', 'Started', 'Failure']
+
 
 class Logger:
     """ An object to manager log
     """
+    def __init__(self) -> None:
+        self.log_connection = None
+
+    async def get_connection(self,
+                             risotto_context: Context,
+                             ):
+        if not self.log_connection:
+            self.log_connection = await risotto_context.pool.acquire()
+            await self.log_connection.set_type_codec(
+                'json',
+                encoder=dumps,
+                decoder=loads,
+                schema='pg_catalog'
+            )
+        return self.log_connection
+
     async def insert(self,
                      msg: str,
-                     uri: str,
-                     uris: str,
                      risotto_context: Context,
                      level: str,
-                     data: Any=None,
+                     kwargs: Any=None,
                      start: bool=False,
                      ) -> None:
-        insert = 'INSERT INTO RisottoLog(Msg, URI, URIS, UserLogin, Level'
+        uri = self._get_last_uri(risotto_context)
+        uris = " ".join(risotto_context.paths)
+        insert = 'INSERT INTO RisottoLog(Msg, URI, URIS, UserLogin, Status'
         values = 'VALUES($1,$2,$3,$4,$5'
-        args = [msg, uri, uris, risotto_context.username, level]
+        args = [msg, uri, uris, risotto_context.username, LEVELS.index(level)]
-        if data:
+        if kwargs:
-            insert += ', Data'
+            insert += ', Kwargs'
             values += ',$6'
-            args.append(dumps(data))
+            args.append(dumps(kwargs))
         context_id = risotto_context.context_id
         if context_id is not None:
             insert += ', ContextId'
-            if data:
+            if kwargs:
                 values += ',$7'
             else:
                 values += ',$6'
@@ -38,7 +60,9 @@ class Logger:
 
         sql = insert + ') ' + values + ') RETURNING LogId'
         try:
-            log_id = await risotto_context.log_connection.fetchval(sql, *args)
+            async with database_lock:
+                connection = await self.get_connection(risotto_context)
+                log_id = await connection.fetchval(sql, *args)
             if context_id is None and start:
                 risotto_context.context_id = log_id
             if start:
@@ -46,36 +70,6 @@ class Logger:
         except UndefinedTableError as err:
             raise Exception(_(f'cannot access to database ({err}), was the database really created?'))
 
-    async def query(self,
-                    risotto_context: Context,
-                    context_id: int,
-                    uri: Optional[str],
-                    ) -> list:
-        sql = '''SELECT Msg as msg, URI as uri_name, URIS as uris, UserLogin as user_login, Level as level, Data as data, StartDate as start_date, StopDate as stop_date
-        FROM RisottoLog
-        WHERE UserLogin = $1 AND (LogId = $2 OR ContextId = $2)
-        '''
-        args = [sql, risotto_context.username, context_id]
-        if uri is not None:
-            sql += ' AND URI = $3'
-            args.append(uri)
-        ret = []
-        for row in await risotto_context.log_connection.fetch(*args):
-            d = {}
-            for key, value in row.items():
-                if key == 'data':
-                    if isinstance(value, dict):
-                        pass
-                    elif not value:
-                        value = {}
-                    else:
-                        value = loads(value)
-                elif key in ['start_date', 'stop_date']:
-                    value = str(value)
-                d[key] = value
-            ret.append(d)
-        return ret
-
     def _get_last_uri(self,
                       risotto_context: Context,
                       ) -> str:
@@ -111,8 +105,6 @@ class Logger:
         paths_msg = self._get_message_paths(risotto_context)
         print(_(f'{risotto_context.username}: ERROR: {error} ({paths_msg} with arguments "{arguments}": {msg})'))
         await self.insert(msg,
-                          self._get_last_uri(risotto_context),
-                          paths_msg,
                           risotto_context,
                           'Error',
                           arguments,
@@ -129,8 +121,6 @@ class Logger:
         if get_config()['global']['debug']:
             print(_(f'{risotto_context.username}: INFO:{paths_msg}: {msg}'))
         await self.insert(msg,
-                          self._get_last_uri(risotto_context),
-                          paths_msg,
                           risotto_context,
                           'Info',
                           arguments,
@@ -149,10 +139,8 @@ class Logger:
                 context = ''
             print(_(f'{risotto_context.username}: START{context}:{paths_msg}: {msg}'))
         await self.insert(msg,
-                          self._get_last_uri(risotto_context),
-                          paths_msg,
                           risotto_context,
-                          'Start',
+                          'Started',
                           arguments,
                           start=True,
                           )
@@ -166,16 +154,18 @@ class Logger:
             print(_(f'{risotto_context.username}: SUCCESS({risotto_context.context_id}):{paths_msg}'))
         sql = """UPDATE RisottoLog
         SET StopDate = $2,
-        Level = 'SUCCESS'
+        Status = $3
         """
-        args = [datetime.now()]
+        args = [datetime.now(), LEVELS.index('Success')]
         if returns:
-            sql += """, Returns = $3
+            sql += """, Returns = $4
             """
             args.append(dumps(returns))
         sql += """WHERE LogId = $1
         """
-        await risotto_context.log_connection.execute(sql,
+        async with database_lock:
+            connection = await self.get_connection(risotto_context)
+            await connection.execute(sql,
                                      risotto_context.start_id,
                                      *args,
                                      )
@@ -190,17 +180,20 @@ class Logger:
                 context = f'({risotto_context.context_id})'
             else:
                 context = ''
-            print(_(f'{risotto_context.username}: FAILED({risotto_context.context_id}):{paths_msg}: err'))
+            print(_(f'{risotto_context.username}: FAILED({risotto_context.context_id}):{paths_msg}: {err}'))
         sql = """UPDATE RisottoLog
         SET StopDate = $2,
-        Level = 'FAILED',
+        Status = $4,
         Msg = $3
         WHERE LogId = $1
         """
-        await risotto_context.log_connection.execute(sql,
+        async with database_lock:
+            connection = await self.get_connection(risotto_context)
+            await connection.execute(sql,
                                      risotto_context.start_id,
                                      datetime.now(),
-                                                     err,
+                                     err[:254],
+                                     LEVELS.index('Failure'),
                                      )
 
     async def info(self,
@@ -210,8 +203,6 @@ class Logger:
         if get_config()['global']['debug']:
             print(msg)
         await self.insert(msg,
-                          '',
-                          None,
                           risotto_context,
                           'Info',
                           )

src/risotto/message.py

@@ -313,6 +313,7 @@ class CustomParam:
                  'string': 'String',
                  'number': 'Number',
                  'object': 'Dict',
+                 'any': 'Any',
                  'array': 'Array',
                  'file': 'File',
                  'float': 'Float'}
@@ -448,6 +449,7 @@ def _get_option(name,
                                                 'reverse_condition': ParamValue(True)}),
                                  calc_value_property_help))
 
+    props.append('notunique')
     description = arg.description.strip().rstrip()
     kwargs = {'name': name,
               'doc': _get_description(description, name),
@@ -523,6 +525,7 @@ def _parse_responses(message_def,
                   'Number': IntOption,
                   'Boolean': BoolOption,
                   'Dict': DictOption,
+                  'Any': AnyOption,
                   'Float': FloatOption,
                   # FIXME
                   'File': StrOption}.get(type_)
@@ -530,8 +533,9 @@ def _parse_responses(message_def,
             raise Exception(f'unknown param type {obj.type} in responses of message {message_def.message}')
         if hasattr(obj, 'default'):
             kwargs['default'] = obj.default
+            kwargs['properties'] = ('notunique',)
         else:
-            kwargs['properties'] = ('mandatory',)
+            kwargs['properties'] = ('mandatory', 'notunique')
         options.append(option(**kwargs))
     od = OptionDescription(uri,
                            message_def.response.description,
@@ -600,7 +604,7 @@ def get_messages(current_module_names,
     select_option = ChoiceOption('message',
                                  'Nom du message.',
                                  tuple(messages),
-                                 properties=frozenset(['mandatory', 'positional']))
+                                 properties=frozenset(['mandatory', 'positional', 'notunique']))
     for uri in messages:
         message_def = get_message(uri,
                                   current_module_names,

src/risotto/register.py

@@ -107,6 +107,7 @@ class RegisterDispatcher:
             version = obj['version']
             if version not in self.messages:
                 self.messages[version] = {}
+            obj['message'] = tiramisu_message
             self.messages[version][tiramisu_message] = obj
 
     def get_function_args(self,
@@ -297,8 +298,6 @@ class RegisterDispatcher:
                       truncate: bool=False,
                       ) -> None:
         internal_user = get_config()['global']['internal_user']
-        async with self.pool.acquire() as log_connection:
-            async with log_connection.transaction():
         async with self.pool.acquire() as connection:
             await connection.set_type_codec(
                 'json',
@@ -315,7 +314,7 @@ class RegisterDispatcher:
                     risotto_context.username = internal_user
                     risotto_context.paths.append(f'internal.{submodule_name}.on_join')
                     risotto_context.type = None
-                            risotto_context.log_connection = log_connection
+                    risotto_context.pool = self.pool
                     risotto_context.connection = connection
                     risotto_context.module = submodule_name.split('.', 1)[0]
                     info_msg = _(f'in function risotto_{submodule_name}.on_join')

src/risotto/utils.py

@@ -23,5 +23,5 @@ def tiramisu_display_name(kls,
         if suffix:
             doc += suffix
         if name != doc:
-            name += f' ({doc})'
+            name += f'" "{doc}'
     return name