From 6bdf21d1aca746e4f13db6e0fb5976f42fe582f8 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Thu, 30 Jan 2020 16:22:06 +0100 Subject: [PATCH] tiramisu in postgres database --- README.md | 3 + docker/docker-compose.yaml | 5 +- script/database_manager.py | 12 +-- script/server.py | 3 +- src/risotto/config.py | 13 +-- src/risotto/dispatcher.py | 99 +++++++++---------- src/risotto/http.py | 9 +- src/risotto/register.py | 40 +++----- src/risotto/services/config/config.py | 16 ++- .../services/servermodel/servermodel.py | 5 +- src/risotto/services/session/session.py | 2 +- src/risotto/services/session/storage.py | 17 +++- 12 files changed, 110 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index 0676113..b7ddc5b 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,9 @@ su - postgres psql -U postgres risotto drop table log; drop table userrole; drop table release; drop table source; drop table server; drop table servermodel; drop table applicationservice; drop table roleuri; drop table risottouser; drop table uri; +psql -U postgres tiramisu +drop table value; drop table property; drop table permissive; drop table information; drop table session; + # Import EOLE ./script/cucchiaiata source.create -n eole -u http://localhost ./script/cucchiaiata source.release.create -s eole -n 2.7.1.1 -d last diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index dce1a1f..ab9be9f 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -15,6 +15,9 @@ services: #command: tail -F /var/log command: python -u /srv/src/risotto/script/server.py restart: on-failure + environment: + RISOTTO_DSN: ${RISOTTO_DSN:-postgres://risotto:risotto@postgres:5432/risotto} + RISOTTO_TIRAMISU_DSN: ${RISOTTO_TIRAMISU_DSN:-postgres://risotto:risotto@postgres:5432/tiramisu} postgres: image: postgres:11-alpine environment: @@ -25,4 +28,4 @@ services: - ./postgres-init/:/docker-entrypoint-initdb.d/ ports: - "5432:5432" - restart: unless-stopped \ No newline at end of file + restart: unless-stopped diff --git a/script/database_manager.py b/script/database_manager.py index 56017ce..cd5ee05 100644 --- a/script/database_manager.py +++ b/script/database_manager.py @@ -96,16 +96,8 @@ CREATE TABLE log( """ async def main(): - db_conf = get_config().get('database') - #asyncpg.connect('postgresql://postgres@localhost/test') - engine = db_conf.get('engine') - host = db_conf.get('host') - dbname = db_conf.get('dbname') - dbuser = db_conf.get('user') - dbpassword = db_conf.get('password') - dbport = db_conf.get('port') - cfg = "{}://{}:{}@{}:{}/{}".format(engine, dbuser, dbpassword, host, dbport, dbname) - pool = await asyncpg.create_pool(cfg) + db_conf = get_config()['database']['dsn'] + pool = await asyncpg.create_pool(db_conf) async with pool.acquire() as connection: async with connection.transaction(): returns = await connection.execute(VERSION_INIT) diff --git a/script/server.py b/script/server.py index 8d3e618..a94dc78 100644 --- a/script/server.py +++ b/script/server.py @@ -1,6 +1,7 @@ from asyncio import get_event_loop from risotto import get_app + if __name__ == '__main__': loop = get_event_loop() loop.run_until_complete(get_app(loop)) @@ -8,5 +9,3 @@ if __name__ == '__main__': loop.run_forever() except KeyboardInterrupt: pass - - diff --git a/src/risotto/config.py b/src/risotto/config.py index bcff151..90e7712 100644 --- a/src/risotto/config.py +++ b/src/risotto/config.py @@ -1,23 +1,20 @@ MESSAGE_ROOT_PATH = 'messages' -DATABASE_DIR = '/var/cache/risotto/database' INTERNAL_USER = 'internal' CONFIGURATION_DIR = 'configurations' TEMPLATE_DIR = 'templates' TMP_DIR = 'tmp' ROUGAIL_DTD_PATH = '../rougail/data/creole.dtd' DEFAULT_USER = 'Anonymous' +DEFAULT_DSN = 'postgres:///risotto?host=/var/run/postgresql/&user=risotto' +DEFAULT_TIRAMISU_DSN = 'postgres:///tiramisu?host=/var/run/postgresql/&user=tiramisu' -import os +from os import environ from pathlib import PurePosixPath CURRENT_PATH = PurePosixPath(__file__) def get_config(): - return {'database': {'engine': 'postgres', - 'host': 'postgres', - 'port': 5432, - 'dbname': 'risotto', - 'user': 'risotto', - 'password': 'risotto', + return {'database': {'dsn': environ.get('RISOTTO_DSN', DEFAULT_DSN), + 'tiramisu_dsn': environ.get('RISOTTO_TIRAMISU_DSN', DEFAULT_TIRAMISU_DSN), }, 'http_server': {'port': 8080, #'default_user': "gnunux"}, diff --git a/src/risotto/dispatcher.py b/src/risotto/dispatcher.py index c3eff4f..0c86897 100644 --- a/src/risotto/dispatcher.py +++ b/src/risotto/dispatcher.py @@ -38,31 +38,30 @@ class CallDispatcher: raise Exception('hu?') else: for ret in returns: - config = await Config(response, - display_name=lambda self, dyn_name: self.impl_getname()) - await config.property.read_write() - try: - for key, value in ret.items(): - await config.option(key).value.set(value) - except AttributeError: - err = _(f'function {module_name}.{function_name} return the unknown parameter "{key}"') - await log.error_msg(risotto_context, kwargs, err) - raise CallError(str(err)) - except ValueError: - err = _(f'function {module_name}.{function_name} return the parameter "{key}" with an unvalid value "{value}"') - await log.error_msg(risotto_context, kwargs, err) - raise CallError(str(err)) - await config.property.read_only() - mandatories = await config.value.mandatory() - if mandatories: - mand = [mand.split('.')[-1] for mand in mandatories] - raise ValueError(_(f'missing parameters in response: {mand} in message "{risotto_context.message}"')) - try: - await config.value.dict() - except Exception as err: - err = _(f'function {module_name}.{function_name} return an invalid response {err}') - await log.error_msg(risotto_context, kwargs, err) - raise CallError(str(err)) + async with await Config(response, display_name=lambda self, dyn_name: self.impl_getname()) as config: + await config.property.read_write() + try: + for key, value in ret.items(): + await config.option(key).value.set(value) + except AttributeError: + err = _(f'function {module_name}.{function_name} return the unknown parameter "{key}"') + await log.error_msg(risotto_context, kwargs, err) + raise CallError(str(err)) + except ValueError: + err = _(f'function {module_name}.{function_name} return the parameter "{key}" with an unvalid value "{value}"') + await log.error_msg(risotto_context, kwargs, err) + raise CallError(str(err)) + await config.property.read_only() + mandatories = await config.value.mandatory() + if mandatories: + mand = [mand.split('.')[-1] for mand in mandatories] + raise ValueError(_(f'missing parameters in response: {mand} in message "{risotto_context.message}"')) + try: + await config.value.dict() + except Exception as err: + err = _(f'function {module_name}.{function_name} return an invalid response {err}') + await log.error_msg(risotto_context, kwargs, err) + raise CallError(str(err)) async def call(self, version: str, @@ -182,31 +181,31 @@ class Dispatcher(register.RegisterDispatcher, CallDispatcher, PublishDispatcher) """ create a new Config et set values to it """ # create a new config - config = await Config(self.option) - await config.property.read_write() - # set message's option - await config.option('message').value.set(risotto_context.message) - # store values - subconfig = config.option(risotto_context.message) - for key, value in kwargs.items(): - try: - await subconfig.option(key).value.set(value) - except AttributeError: - if get_config()['global']['debug']: - print_exc() - raise ValueError(_(f'unknown parameter in "{uri}": "{key}"')) - # check mandatories options - if check_role and get_config().get('global').get('check_role'): - await self.check_role(subconfig, - risotto_context.username, - uri) - await config.property.read_only() - mandatories = await config.value.mandatory() - if mandatories: - mand = [mand.split('.')[-1] for mand in mandatories] - raise ValueError(_(f'missing parameters in "{uri}": {mand}')) - # return complete an validated kwargs - return await subconfig.value.dict() + async with await Config(self.option) as config: + await config.property.read_write() + # set message's option + await config.option('message').value.set(risotto_context.message) + # store values + subconfig = config.option(risotto_context.message) + for key, value in kwargs.items(): + try: + await subconfig.option(key).value.set(value) + except AttributeError: + if get_config()['global']['debug']: + print_exc() + raise ValueError(_(f'unknown parameter in "{uri}": "{key}"')) + # check mandatories options + if check_role and get_config().get('global').get('check_role'): + await self.check_role(subconfig, + risotto_context.username, + uri) + await config.property.read_only() + mandatories = await config.value.mandatory() + if mandatories: + mand = [mand.split('.')[-1] for mand in mandatories] + raise ValueError(_(f'missing parameters in "{uri}": {mand}')) + # return complete an validated kwargs + return await subconfig.value.dict() def get_service(self, name: str): diff --git a/src/risotto/http.py b/src/risotto/http.py index dbef434..686e131 100644 --- a/src/risotto/http.py +++ b/src/risotto/http.py @@ -1,7 +1,7 @@ from aiohttp.web import Application, Response, get, post, HTTPBadRequest, HTTPInternalServerError, HTTPNotFound from json import dumps from traceback import print_exc -from tiramisu import Config +from tiramisu import Config, default_storage from .dispatcher import dispatcher @@ -102,9 +102,9 @@ async def api(request, risotto_context): WHERE RoleURI.URIId = URI.URIId ''' uris = [uri['uriname'] for uri in await connection.fetch(sql)] - config = await Config(get_messages(load_shortarg=True, uris=uris)[1]) - await config.property.read_write() - tiramisu = await config.option.dict(remotable='none') + async with await Config(get_messages(load_shortarg=True, uris=uris)[1]) as config: + await config.property.read_write() + tiramisu = await config.option.dict(remotable='none') return tiramisu @@ -119,6 +119,7 @@ async def get_app(loop): load_services() app = Application(loop=loop) routes = [] + default_storage.engine('dictionary') await dispatcher.load() for version, messages in dispatcher.messages.items(): print() diff --git a/src/risotto/register.py b/src/risotto/register.py index fd6d3cf..45829bf 100644 --- a/src/risotto/register.py +++ b/src/risotto/register.py @@ -59,13 +59,13 @@ class RegisterDispatcher: """ async def get_message_args(): # load config - config = await Config(self.option) - await config.property.read_write() - # set message to the uri name - await config.option('message').value.set(message) - # get message argument - dico = await config.option(message).value.dict() - return set(dico.keys()) + async with await Config(self.option) as config: + await config.property.read_write() + # set message to the uri name + await config.option('message').value.set(message) + # get message argument + dico = await config.option(message).value.dict() + return set(dico.keys()) def get_function_args(): function_args = self.get_function_args(function) @@ -101,13 +101,13 @@ class RegisterDispatcher: """ async def get_message_args(): # load config - config = await Config(self.option) - await config.property.read_write() - # set message to the message name - await config.option('message').value.set(message) - # get message argument - dico = await config.option(message).value.dict() - return set(dico.keys()) + async with await Config(self.option) as config: + await config.property.read_write() + # set message to the message name + await config.option('message').value.set(message) + # get message argument + dico = await config.option(message).value.dict() + return set(dico.keys()) def get_function_args(): function_args = self.get_function_args(function) @@ -247,16 +247,8 @@ class RegisterDispatcher: async def load(self): # valid function's arguments - db_conf = get_config().get('database') - - engine = db_conf.get('engine') - host = db_conf.get('host') - dbname = db_conf.get('dbname') - dbuser = db_conf.get('user') - dbpassword = db_conf.get('password') - dbport = db_conf.get('port') - cfg = "{}://{}:{}@{}:{}/{}".format(engine, dbuser, dbpassword, host, dbport, dbname) - self.pool = await asyncpg.create_pool(cfg) + db_conf = get_config()['database']['dsn'] + self.pool = await asyncpg.create_pool(db_conf) async with self.pool.acquire() as connection: async with connection.transaction(): for version, messages in self.messages.items(): diff --git a/src/risotto/services/config/config.py b/src/risotto/services/config/config.py index e7a7044..6fcf2bb 100644 --- a/src/risotto/services/config/config.py +++ b/src/risotto/services/config/config.py @@ -10,7 +10,7 @@ from rougail import load as rougail_load from ...controller import Controller from ...register import register -from ...config import DATABASE_DIR, ROUGAIL_DTD_PATH, get_config +from ...config import ROUGAIL_DTD_PATH, get_config from ...context import Context from ...utils import _ from ...error import CallError, RegistrationError @@ -23,11 +23,12 @@ class Risotto(Controller): test) -> None: global conf_storage self.cache_root_path = join(get_config().get('cache').get('root_path'), 'servermodel') - for dirname in [self.cache_root_path, DATABASE_DIR]: - if not isdir(dirname): - raise RegistrationError(_(f'unable to find the cache dir "{dirname}"')) + if not isdir(self.cache_root_path): + raise RegistrationError(_(f'unable to find the cache dir "{self.cache_root_path}"')) if not test: - self.save_storage = Storage(engine='sqlite3', dir_database=DATABASE_DIR) + db_conf = get_config()['database']['tiramisu_dsn'] + self.save_storage = Storage(engine='postgres') + self.save_storage.setting(dsn=db_conf) self.servermodel = {} self.server = {} super().__init__(test) @@ -115,12 +116,10 @@ class Risotto(Controller): # build servermodel metaconfig (v_xxx.m_v_xxx) metaconfig = await MetaConfig([], optiondescription=optiondescription, - persistent=True, session_id=session_id, storage=self.save_storage) mixconfig = await MixConfig(children=[], optiondescription=optiondescription, - persistent=True, session_id='m_' + session_id, storage=self.save_storage) await metaconfig.config.add(mixconfig) @@ -248,8 +247,7 @@ class Risotto(Controller): """ build server's config """ config = await metaconfig.config.new(session_id, - storage=self.save_storage, - persistent=True) + storage=self.save_storage) await config.information.set('server_id', server_id) await config.information.set('server_name', server_name) await config.owner.set(server_name) diff --git a/src/risotto/services/servermodel/servermodel.py b/src/risotto/services/servermodel/servermodel.py index 143232e..1a0f6b6 100644 --- a/src/risotto/services/servermodel/servermodel.py +++ b/src/risotto/services/servermodel/servermodel.py @@ -5,7 +5,6 @@ from yaml import load, SafeLoader from traceback import print_exc from typing import Dict, List, Optional from rougail import CreoleObjSpace -from rougail.config import dtdfilename from ...controller import Controller from ...register import register from ...utils import _ @@ -68,7 +67,7 @@ class Risotto(Controller): as_names_str = '", "'.join(as_names) await log.info(risotto_context, _(f'gen funcs for "{servermodel_name}" with application services "{as_names_str}"')) - eolobj = CreoleObjSpace(dtdfilename) + #eolobj = CreoleObjSpace(get_config()['global']['rougail_dtd_path']) async def servermodel_gen_schema(self, servermodel_name: str, @@ -106,7 +105,7 @@ class Risotto(Controller): continue as_names.add(applicationservice_name) extras.append((namespace, [extra_dir])) - eolobj = CreoleObjSpace(dtdfilename) + eolobj = CreoleObjSpace(get_config()['global']['rougail_dtd_path']) as_names_str = '", "'.join(as_names) await log.info(risotto_context, _(f'gen schema for "{servermodel_name}" with application services "{as_names_str}"')) diff --git a/src/risotto/services/session/session.py b/src/risotto/services/session/session.py index c8def59..461fd6d 100644 --- a/src/risotto/services/session/session.py +++ b/src/risotto/services/session/session.py @@ -293,7 +293,7 @@ class Risotto(Controller): modif_config = session['config'] await config.value.importation(await modif_config.value.exportation()) await config.permissive.importation(await modif_config.permissive.exportation()) - storage.del_session(session_id) + await storage.del_session(session_id) return self.format_session(session_id, session) @register_http('v1', '/config/server/{session_id}') diff --git a/src/risotto/services/session/storage.py b/src/risotto/services/session/storage.py index 984d62c..fe71432 100644 --- a/src/risotto/services/session/storage.py +++ b/src/risotto/services/session/storage.py @@ -103,8 +103,21 @@ class Storage(object): raise NotAllowedError() return session - def del_session(self, - id: int): + async def del_session(self, + id: int): + config = self.sessions[id]['meta'] + while True: + try: + children = list(await config.config.list()) + if not children: + # it's an empty metaconfig + break + config = children[0] + await config.session.reset() + except: + # it's a config, so no "list" method + break + await self.sessions[id]['meta'].session.reset() del self.sessions[id]