tiramisu in postgres database

README.md

@@ -44,6 +44,9 @@ su - postgres
 psql -U postgres risotto
 drop table log; drop table userrole; drop table release; drop table source; drop table server; drop table servermodel; drop table applicationservice; drop table roleuri;  drop table risottouser; drop table uri;
 
+psql -U postgres tiramisu
+drop table value; drop table property; drop table permissive; drop table information; drop table session;
+
 # Import EOLE
 ./script/cucchiaiata source.create -n eole -u http://localhost
 ./script/cucchiaiata source.release.create -s eole -n 2.7.1.1 -d last

docker/docker-compose.yaml

@@ -15,6 +15,9 @@ services:
     #command: tail -F /var/log
     command: python -u /srv/src/risotto/script/server.py
     restart: on-failure
+    environment:
+        RISOTTO_DSN: ${RISOTTO_DSN:-postgres://risotto:risotto@postgres:5432/risotto}
+        RISOTTO_TIRAMISU_DSN: ${RISOTTO_TIRAMISU_DSN:-postgres://risotto:risotto@postgres:5432/tiramisu}
   postgres:
     image: postgres:11-alpine
     environment:
@@ -25,4 +28,4 @@ services:
       - ./postgres-init/:/docker-entrypoint-initdb.d/
     ports:
       - "5432:5432"
-    restart: unless-stopped
+    restart: unless-stopped

script/database_manager.py

@@ -96,16 +96,8 @@ CREATE TABLE log(
 """
 
 async def main():
-    db_conf = get_config().get('database')
+    db_conf = get_config()['database']['dsn']
-    #asyncpg.connect('postgresql://postgres@localhost/test')
+    pool = await asyncpg.create_pool(db_conf)
-    engine = db_conf.get('engine')
-    host = db_conf.get('host')
-    dbname = db_conf.get('dbname')
-    dbuser = db_conf.get('user')
-    dbpassword = db_conf.get('password')
-    dbport = db_conf.get('port')
-    cfg = "{}://{}:{}@{}:{}/{}".format(engine, dbuser, dbpassword, host, dbport, dbname)
-    pool = await asyncpg.create_pool(cfg)
     async with pool.acquire() as connection:
         async with connection.transaction():
             returns = await connection.execute(VERSION_INIT)

script/server.py

@@ -1,6 +1,7 @@
 from asyncio import get_event_loop
 from risotto import get_app
 
+
 if __name__ == '__main__':
     loop = get_event_loop()
     loop.run_until_complete(get_app(loop))
@@ -8,5 +9,3 @@ if __name__ == '__main__':
         loop.run_forever()
     except KeyboardInterrupt:
         pass
-
-

src/risotto/config.py

@@ -1,23 +1,20 @@
 MESSAGE_ROOT_PATH = 'messages'
-DATABASE_DIR = '/var/cache/risotto/database'
 INTERNAL_USER = 'internal'
 CONFIGURATION_DIR = 'configurations'
 TEMPLATE_DIR = 'templates'
 TMP_DIR = 'tmp'
 ROUGAIL_DTD_PATH = '../rougail/data/creole.dtd'
 DEFAULT_USER = 'Anonymous'
+DEFAULT_DSN = 'postgres:///risotto?host=/var/run/postgresql/&user=risotto'
+DEFAULT_TIRAMISU_DSN = 'postgres:///tiramisu?host=/var/run/postgresql/&user=tiramisu'
 
-import os
+from os import environ
 from pathlib import PurePosixPath
 CURRENT_PATH = PurePosixPath(__file__)
 
 def get_config():
-    return {'database': {'engine': 'postgres',
+    return {'database': {'dsn': environ.get('RISOTTO_DSN', DEFAULT_DSN),
-                         'host': 'postgres',
+                         'tiramisu_dsn': environ.get('RISOTTO_TIRAMISU_DSN', DEFAULT_TIRAMISU_DSN),
-                         'port': 5432,
-                         'dbname': 'risotto',
-                         'user': 'risotto',
-                         'password': 'risotto',
                          },
             'http_server': {'port': 8080,
                             #'default_user': "gnunux"},

src/risotto/dispatcher.py

@@ -38,31 +38,30 @@ class CallDispatcher:
             raise Exception('hu?')
         else:
             for ret in returns:
-                config = await Config(response,
+                async with await Config(response, display_name=lambda self, dyn_name: self.impl_getname()) as config:
-                                      display_name=lambda self, dyn_name: self.impl_getname())
+                    await config.property.read_write()
-                await config.property.read_write()
+                    try:
-                try:
+                        for key, value in ret.items():
-                    for key, value in ret.items():
+                            await config.option(key).value.set(value)
-                        await config.option(key).value.set(value)
+                    except AttributeError:
-                except AttributeError:
+                        err = _(f'function {module_name}.{function_name} return the unknown parameter "{key}"')
-                    err = _(f'function {module_name}.{function_name} return the unknown parameter "{key}"')
+                        await log.error_msg(risotto_context, kwargs, err)
-                    await log.error_msg(risotto_context, kwargs, err)
+                        raise CallError(str(err))
-                    raise CallError(str(err))
+                    except ValueError:
-                except ValueError:
+                        err = _(f'function {module_name}.{function_name} return the parameter "{key}" with an unvalid value "{value}"')
-                    err = _(f'function {module_name}.{function_name} return the parameter "{key}" with an unvalid value "{value}"')
+                        await log.error_msg(risotto_context, kwargs, err)
-                    await log.error_msg(risotto_context, kwargs, err)
+                        raise CallError(str(err))
-                    raise CallError(str(err))
+                    await config.property.read_only()
-                await config.property.read_only()
+                    mandatories = await config.value.mandatory()
-                mandatories = await config.value.mandatory()
+                    if mandatories:
-                if mandatories:
+                        mand = [mand.split('.')[-1] for mand in mandatories]
-                    mand = [mand.split('.')[-1] for mand in mandatories]
+                        raise ValueError(_(f'missing parameters in response: {mand} in message "{risotto_context.message}"'))
-                    raise ValueError(_(f'missing parameters in response: {mand} in message "{risotto_context.message}"'))
+                    try:
-                try:
+                        await config.value.dict()
-                    await config.value.dict()
+                    except Exception as err:
-                except Exception as err:
+                        err = _(f'function {module_name}.{function_name} return an invalid response {err}')
-                    err = _(f'function {module_name}.{function_name} return an invalid response {err}')
+                        await log.error_msg(risotto_context, kwargs, err)
-                    await log.error_msg(risotto_context, kwargs, err)
+                        raise CallError(str(err))
-                    raise CallError(str(err))
 
     async def call(self,
                    version: str,
@@ -182,31 +181,31 @@ class Dispatcher(register.RegisterDispatcher, CallDispatcher, PublishDispatcher)
         """ create a new Config et set values to it
         """
         # create a new config
-        config = await Config(self.option)
+        async with await Config(self.option) as config:
-        await config.property.read_write()
+            await config.property.read_write()
-        # set message's option
+            # set message's option
-        await config.option('message').value.set(risotto_context.message)
+            await config.option('message').value.set(risotto_context.message)
-        # store values
+            # store values
-        subconfig = config.option(risotto_context.message)
+            subconfig = config.option(risotto_context.message)
-        for key, value in kwargs.items():
+            for key, value in kwargs.items():
-            try:
+                try:
-                await subconfig.option(key).value.set(value)
+                    await subconfig.option(key).value.set(value)
-            except AttributeError:
+                except AttributeError:
-                if get_config()['global']['debug']:
+                    if get_config()['global']['debug']:
-                    print_exc()
+                        print_exc()
-                raise ValueError(_(f'unknown parameter in "{uri}": "{key}"'))
+                    raise ValueError(_(f'unknown parameter in "{uri}": "{key}"'))
-        # check mandatories options
+            # check mandatories options
-        if check_role and get_config().get('global').get('check_role'):
+            if check_role and get_config().get('global').get('check_role'):
-            await self.check_role(subconfig,
+                await self.check_role(subconfig,
-                                  risotto_context.username,
+                                      risotto_context.username,
-                                  uri)
+                                      uri)
-        await config.property.read_only()
+            await config.property.read_only()
-        mandatories = await config.value.mandatory()
+            mandatories = await config.value.mandatory()
-        if mandatories:
+            if mandatories:
-            mand = [mand.split('.')[-1] for mand in mandatories]
+                mand = [mand.split('.')[-1] for mand in mandatories]
-            raise ValueError(_(f'missing parameters in "{uri}": {mand}'))
+                raise ValueError(_(f'missing parameters in "{uri}": {mand}'))
-        # return complete an validated kwargs
+            # return complete an validated kwargs
-        return await subconfig.value.dict()
+            return await subconfig.value.dict()
 
     def get_service(self,
                     name: str):

src/risotto/http.py

@@ -1,7 +1,7 @@
 from aiohttp.web import Application, Response, get, post, HTTPBadRequest, HTTPInternalServerError, HTTPNotFound
 from json import dumps
 from traceback import print_exc
-from tiramisu import Config
+from tiramisu import Config, default_storage
 
 
 from .dispatcher import dispatcher
@@ -102,9 +102,9 @@ async def api(request, risotto_context):
                 WHERE RoleURI.URIId = URI.URIId
                 '''
                 uris = [uri['uriname'] for uri in await connection.fetch(sql)]
-        config = await Config(get_messages(load_shortarg=True, uris=uris)[1])
+        async with await Config(get_messages(load_shortarg=True, uris=uris)[1]) as config:
-        await config.property.read_write()
+            await config.property.read_write()
-        tiramisu = await config.option.dict(remotable='none')
+            tiramisu = await config.option.dict(remotable='none')
     return tiramisu
 
 
@@ -119,6 +119,7 @@ async def get_app(loop):
     load_services()
     app = Application(loop=loop)
     routes = []
+    default_storage.engine('dictionary')
     await dispatcher.load()
     for version, messages in dispatcher.messages.items():
         print()

src/risotto/register.py

@@ -59,13 +59,13 @@ class RegisterDispatcher:
         """
         async def get_message_args():
             # load config
-            config = await Config(self.option)
+            async with await Config(self.option) as config:
-            await config.property.read_write()
+                await config.property.read_write()
-            # set message to the uri name
+                # set message to the uri name
-            await config.option('message').value.set(message)
+                await config.option('message').value.set(message)
-            # get message argument
+                # get message argument
-            dico = await config.option(message).value.dict()
+                dico = await config.option(message).value.dict()
-            return set(dico.keys())
+                return set(dico.keys())
 
         def get_function_args():
             function_args = self.get_function_args(function)
@@ -101,13 +101,13 @@ class RegisterDispatcher:
         """
         async def get_message_args():
             # load config
-            config = await Config(self.option)
+            async with await Config(self.option) as config:
-            await config.property.read_write()
+                await config.property.read_write()
-            # set message to the message name
+                # set message to the message name
-            await config.option('message').value.set(message)
+                await config.option('message').value.set(message)
-            # get message argument
+                # get message argument
-            dico = await config.option(message).value.dict()
+                dico = await config.option(message).value.dict()
-            return set(dico.keys())
+                return set(dico.keys())
 
         def get_function_args():
             function_args = self.get_function_args(function)
@@ -247,16 +247,8 @@ class RegisterDispatcher:
 
     async def load(self):
         # valid function's arguments
-        db_conf = get_config().get('database')
+        db_conf = get_config()['database']['dsn']
-
+        self.pool = await asyncpg.create_pool(db_conf)
-        engine = db_conf.get('engine')
-        host = db_conf.get('host')
-        dbname = db_conf.get('dbname')
-        dbuser = db_conf.get('user')
-        dbpassword = db_conf.get('password')
-        dbport = db_conf.get('port')
-        cfg = "{}://{}:{}@{}:{}/{}".format(engine, dbuser, dbpassword, host, dbport, dbname)
-        self.pool = await asyncpg.create_pool(cfg)
         async with self.pool.acquire() as connection:
             async with connection.transaction():
                 for version, messages in self.messages.items():

src/risotto/services/config/config.py

@@ -10,7 +10,7 @@ from rougail import load as rougail_load
 
 from ...controller import Controller
 from ...register import register
-from ...config import DATABASE_DIR, ROUGAIL_DTD_PATH, get_config
+from ...config import ROUGAIL_DTD_PATH, get_config
 from ...context import Context
 from ...utils import _
 from ...error import CallError, RegistrationError
@@ -23,11 +23,12 @@ class Risotto(Controller):
                  test) -> None:
         global conf_storage
         self.cache_root_path = join(get_config().get('cache').get('root_path'), 'servermodel')
-        for dirname in [self.cache_root_path, DATABASE_DIR]:
+        if not isdir(self.cache_root_path):
-            if not isdir(dirname):
+            raise RegistrationError(_(f'unable to find the cache dir "{self.cache_root_path}"'))
-                raise RegistrationError(_(f'unable to find the cache dir "{dirname}"'))
         if not test:
-            self.save_storage = Storage(engine='sqlite3', dir_database=DATABASE_DIR)
+            db_conf = get_config()['database']['tiramisu_dsn']
+            self.save_storage = Storage(engine='postgres')
+            self.save_storage.setting(dsn=db_conf)
         self.servermodel = {}
         self.server = {}
         super().__init__(test)
@@ -115,12 +116,10 @@ class Risotto(Controller):
         # build servermodel metaconfig (v_xxx.m_v_xxx)
         metaconfig = await MetaConfig([],
                                       optiondescription=optiondescription,
-                                      persistent=True,
                                       session_id=session_id,
                                       storage=self.save_storage)
         mixconfig = await MixConfig(children=[],
                                     optiondescription=optiondescription,
-                                    persistent=True,
                                     session_id='m_' + session_id,
                                     storage=self.save_storage)
         await metaconfig.config.add(mixconfig)
@@ -248,8 +247,7 @@ class Risotto(Controller):
         """ build server's config
         """
         config = await metaconfig.config.new(session_id,
-                                             storage=self.save_storage,
+                                             storage=self.save_storage)
-                                             persistent=True)
         await config.information.set('server_id', server_id)
         await config.information.set('server_name', server_name)
         await config.owner.set(server_name)

src/risotto/services/servermodel/servermodel.py

@@ -5,7 +5,6 @@ from yaml import load, SafeLoader
 from traceback import print_exc
 from typing import Dict, List, Optional
 from rougail import CreoleObjSpace
-from rougail.config import dtdfilename
 from ...controller import Controller
 from ...register import register
 from ...utils import _
@@ -68,7 +67,7 @@ class Risotto(Controller):
         as_names_str = '", "'.join(as_names)
         await log.info(risotto_context,
                        _(f'gen funcs for "{servermodel_name}" with application services "{as_names_str}"'))
-        eolobj = CreoleObjSpace(dtdfilename)
+        #eolobj = CreoleObjSpace(get_config()['global']['rougail_dtd_path'])
 
     async def servermodel_gen_schema(self,
                                      servermodel_name: str,
@@ -106,7 +105,7 @@ class Risotto(Controller):
                         continue
                     as_names.add(applicationservice_name)
                     extras.append((namespace, [extra_dir]))
-        eolobj = CreoleObjSpace(dtdfilename)
+        eolobj = CreoleObjSpace(get_config()['global']['rougail_dtd_path'])
         as_names_str = '", "'.join(as_names)
         await log.info(risotto_context,
                        _(f'gen schema for "{servermodel_name}" with application services "{as_names_str}"'))

src/risotto/services/session/session.py

@@ -293,7 +293,7 @@ class Risotto(Controller):
             modif_config = session['config']
             await config.value.importation(await modif_config.value.exportation())
             await config.permissive.importation(await modif_config.permissive.exportation())
-        storage.del_session(session_id)
+        await storage.del_session(session_id)
         return self.format_session(session_id, session)
 
     @register_http('v1', '/config/server/{session_id}')

src/risotto/services/session/storage.py

@@ -103,8 +103,21 @@ class Storage(object):
             raise NotAllowedError()
         return session
 
-    def del_session(self,
+    async def del_session(self,
-                    id: int):
+                          id: int):
+        config = self.sessions[id]['meta']
+        while True:
+            try:
+                children = list(await config.config.list())
+                if not children:
+                    # it's an empty metaconfig
+                    break
+                config = children[0]
+                await config.session.reset()
+            except:
+                # it's a config, so no "list" method
+                break
+        await self.sessions[id]['meta'].session.reset()
         del self.sessions[id]