Infra/risotto
9
0
Fork 1
Code Issues 1 Releases Wiki Activity

tiramisu in postgres database

Browse Source
This commit is contained in:
Emmanuel Garette 2020-01-30 16:22:06 +01:00
parent 5666c01bdc
commit 6bdf21d1ac
12 changed files with 110 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -44,6 +44,9 @@ su - postgres
psql -U postgres risotto
drop table log; drop table userrole; drop table release; drop table source; drop table server; drop table servermodel; drop table applicationservice; drop table roleuri; drop table risottouser; drop table uri;
psql -U postgres tiramisu
drop table value; drop table property; drop table permissive; drop table information; drop table session;
# Import EOLE
./script/cucchiaiata source.create -n eole -u http://localhost
./script/cucchiaiata source.release.create -s eole -n 2.7.1.1 -d last

5
docker/docker-compose.yaml
View File

@ -15,6 +15,9 @@ services:
#command: tail -F /var/log
command: python -u /srv/src/risotto/script/server.py
restart: on-failure
environment:
RISOTTO_DSN: ${RISOTTO_DSN:-postgres://risotto:risotto@postgres:5432/risotto}
RISOTTO_TIRAMISU_DSN: ${RISOTTO_TIRAMISU_DSN:-postgres://risotto:risotto@postgres:5432/tiramisu}
postgres:
image: postgres:11-alpine
environment:
@ -25,4 +28,4 @@ services:
- ./postgres-init/:/docker-entrypoint-initdb.d/
ports:
- "5432:5432"
restart: unless-stopped
restart: unless-stopped

12
script/database_manager.py
View File

@ -96,16 +96,8 @@ CREATE TABLE log(
"""
async def main():
db_conf = get_config().get('database')
#asyncpg.connect('postgresql://postgres@localhost/test')
engine = db_conf.get('engine')
host = db_conf.get('host')
dbname = db_conf.get('dbname')
dbuser = db_conf.get('user')
dbpassword = db_conf.get('password')
dbport = db_conf.get('port')
cfg = "{}://{}:{}@{}:{}/{}".format(engine, dbuser, dbpassword, host, dbport, dbname)
pool = await asyncpg.create_pool(cfg)
db_conf = get_config()['database']['dsn']
pool = await asyncpg.create_pool(db_conf)
async with pool.acquire() as connection:
async with connection.transaction():
returns = await connection.execute(VERSION_INIT)

3
script/server.py
View File

@ -1,6 +1,7 @@
from asyncio import get_event_loop
from risotto import get_app
if __name__ == '__main__':
loop = get_event_loop()
loop.run_until_complete(get_app(loop))
@ -8,5 +9,3 @@ if __name__ == '__main__':
loop.run_forever()
except KeyboardInterrupt:
pass

13
src/risotto/config.py
View File

@ -1,23 +1,20 @@
MESSAGE_ROOT_PATH = 'messages'
DATABASE_DIR = '/var/cache/risotto/database'
INTERNAL_USER = 'internal'
CONFIGURATION_DIR = 'configurations'
TEMPLATE_DIR = 'templates'
TMP_DIR = 'tmp'
ROUGAIL_DTD_PATH = '../rougail/data/creole.dtd'
DEFAULT_USER = 'Anonymous'
DEFAULT_DSN = 'postgres:///risotto?host=/var/run/postgresql/&user=risotto'
DEFAULT_TIRAMISU_DSN = 'postgres:///tiramisu?host=/var/run/postgresql/&user=tiramisu'
import os
from os import environ
from pathlib import PurePosixPath
CURRENT_PATH = PurePosixPath(__file__)
def get_config():
return {'database': {'engine': 'postgres',
'host': 'postgres',
'port': 5432,
'dbname': 'risotto',
'user': 'risotto',
'password': 'risotto',
return {'database': {'dsn': environ.get('RISOTTO_DSN', DEFAULT_DSN),
'tiramisu_dsn': environ.get('RISOTTO_TIRAMISU_DSN', DEFAULT_TIRAMISU_DSN),
},
'http_server': {'port': 8080,
#'default_user': "gnunux"},

99
src/risotto/dispatcher.py
View File

@ -38,31 +38,30 @@ class CallDispatcher:
raise Exception('hu?')
else:
for ret in returns:
config = await Config(response,
display_name=lambda self, dyn_name: self.impl_getname())
await config.property.read_write()
try:
for key, value in ret.items():
await config.option(key).value.set(value)
except AttributeError:
err = _(f'function {module_name}.{function_name} return the unknown parameter "{key}"')
await log.error_msg(risotto_context, kwargs, err)
raise CallError(str(err))
except ValueError:
err = _(f'function {module_name}.{function_name} return the parameter "{key}" with an unvalid value "{value}"')
await log.error_msg(risotto_context, kwargs, err)
raise CallError(str(err))
await config.property.read_only()
mandatories = await config.value.mandatory()
if mandatories:
mand = [mand.split('.')[-1] for mand in mandatories]
raise ValueError(_(f'missing parameters in response: {mand} in message "{risotto_context.message}"'))
try:
await config.value.dict()
except Exception as err:
err = _(f'function {module_name}.{function_name} return an invalid response {err}')
await log.error_msg(risotto_context, kwargs, err)
raise CallError(str(err))
async with await Config(response, display_name=lambda self, dyn_name: self.impl_getname()) as config:
await config.property.read_write()
try:
for key, value in ret.items():
await config.option(key).value.set(value)
except AttributeError:
err = _(f'function {module_name}.{function_name} return the unknown parameter "{key}"')
await log.error_msg(risotto_context, kwargs, err)
raise CallError(str(err))
except ValueError:
err = _(f'function {module_name}.{function_name} return the parameter "{key}" with an unvalid value "{value}"')
await log.error_msg(risotto_context, kwargs, err)
raise CallError(str(err))
await config.property.read_only()
mandatories = await config.value.mandatory()
if mandatories:
mand = [mand.split('.')[-1] for mand in mandatories]
raise ValueError(_(f'missing parameters in response: {mand} in message "{risotto_context.message}"'))
try:
await config.value.dict()
except Exception as err:
err = _(f'function {module_name}.{function_name} return an invalid response {err}')
await log.error_msg(risotto_context, kwargs, err)
raise CallError(str(err))
async def call(self,
version: str,
@ -182,31 +181,31 @@ class Dispatcher(register.RegisterDispatcher, CallDispatcher, PublishDispatcher)
""" create a new Config et set values to it
"""
# create a new config
config = await Config(self.option)
await config.property.read_write()
# set message's option
await config.option('message').value.set(risotto_context.message)
# store values
subconfig = config.option(risotto_context.message)
for key, value in kwargs.items():
try:
await subconfig.option(key).value.set(value)
except AttributeError:
if get_config()['global']['debug']:
print_exc()
raise ValueError(_(f'unknown parameter in "{uri}": "{key}"'))
# check mandatories options
if check_role and get_config().get('global').get('check_role'):
await self.check_role(subconfig,
risotto_context.username,
uri)
await config.property.read_only()
mandatories = await config.value.mandatory()
if mandatories:
mand = [mand.split('.')[-1] for mand in mandatories]
raise ValueError(_(f'missing parameters in "{uri}": {mand}'))
# return complete an validated kwargs
return await subconfig.value.dict()
async with await Config(self.option) as config:
await config.property.read_write()
# set message's option
await config.option('message').value.set(risotto_context.message)
# store values
subconfig = config.option(risotto_context.message)
for key, value in kwargs.items():
try:
await subconfig.option(key).value.set(value)
except AttributeError:
if get_config()['global']['debug']:
print_exc()
raise ValueError(_(f'unknown parameter in "{uri}": "{key}"'))
# check mandatories options
if check_role and get_config().get('global').get('check_role'):
await self.check_role(subconfig,
risotto_context.username,
uri)
await config.property.read_only()
mandatories = await config.value.mandatory()
if mandatories:
mand = [mand.split('.')[-1] for mand in mandatories]
raise ValueError(_(f'missing parameters in "{uri}": {mand}'))
# return complete an validated kwargs
return await subconfig.value.dict()
def get_service(self,
name: str):

9
src/risotto/http.py
View File

@ -1,7 +1,7 @@
from aiohttp.web import Application, Response, get, post, HTTPBadRequest, HTTPInternalServerError, HTTPNotFound
from json import dumps
from traceback import print_exc
from tiramisu import Config
from tiramisu import Config, default_storage
from .dispatcher import dispatcher
@ -102,9 +102,9 @@ async def api(request, risotto_context):
WHERE RoleURI.URIId = URI.URIId
'''
uris = [uri['uriname'] for uri in await connection.fetch(sql)]
config = await Config(get_messages(load_shortarg=True, uris=uris)[1])
await config.property.read_write()
tiramisu = await config.option.dict(remotable='none')
async with await Config(get_messages(load_shortarg=True, uris=uris)[1]) as config:
await config.property.read_write()
tiramisu = await config.option.dict(remotable='none')
return tiramisu
@ -119,6 +119,7 @@ async def get_app(loop):
load_services()
app = Application(loop=loop)
routes = []
default_storage.engine('dictionary')
await dispatcher.load()
for version, messages in dispatcher.messages.items():
print()

40
src/risotto/register.py
View File

@ -59,13 +59,13 @@ class RegisterDispatcher:
"""
async def get_message_args():
# load config
config = await Config(self.option)
await config.property.read_write()
# set message to the uri name
await config.option('message').value.set(message)
# get message argument
dico = await config.option(message).value.dict()
return set(dico.keys())
async with await Config(self.option) as config:
await config.property.read_write()
# set message to the uri name
await config.option('message').value.set(message)
# get message argument
dico = await config.option(message).value.dict()
return set(dico.keys())
def get_function_args():
function_args = self.get_function_args(function)
@ -101,13 +101,13 @@ class RegisterDispatcher:
"""
async def get_message_args():
# load config
config = await Config(self.option)
await config.property.read_write()
# set message to the message name
await config.option('message').value.set(message)
# get message argument
dico = await config.option(message).value.dict()
return set(dico.keys())
async with await Config(self.option) as config:
await config.property.read_write()
# set message to the message name
await config.option('message').value.set(message)
# get message argument
dico = await config.option(message).value.dict()
return set(dico.keys())
def get_function_args():
function_args = self.get_function_args(function)
@ -247,16 +247,8 @@ class RegisterDispatcher:
async def load(self):
# valid function's arguments
db_conf = get_config().get('database')
engine = db_conf.get('engine')
host = db_conf.get('host')
dbname = db_conf.get('dbname')
dbuser = db_conf.get('user')
dbpassword = db_conf.get('password')
dbport = db_conf.get('port')
cfg = "{}://{}:{}@{}:{}/{}".format(engine, dbuser, dbpassword, host, dbport, dbname)
self.pool = await asyncpg.create_pool(cfg)
db_conf = get_config()['database']['dsn']
self.pool = await asyncpg.create_pool(db_conf)
async with self.pool.acquire() as connection:
async with connection.transaction():
for version, messages in self.messages.items():

16
src/risotto/services/config/config.py
View File

@ -10,7 +10,7 @@ from rougail import load as rougail_load
from ...controller import Controller
from ...register import register
from ...config import DATABASE_DIR, ROUGAIL_DTD_PATH, get_config
from ...config import ROUGAIL_DTD_PATH, get_config
from ...context import Context
from ...utils import _
from ...error import CallError, RegistrationError
@ -23,11 +23,12 @@ class Risotto(Controller):
test) -> None:
global conf_storage
self.cache_root_path = join(get_config().get('cache').get('root_path'), 'servermodel')
for dirname in [self.cache_root_path, DATABASE_DIR]:
if not isdir(dirname):
raise RegistrationError(_(f'unable to find the cache dir "{dirname}"'))
if not isdir(self.cache_root_path):
raise RegistrationError(_(f'unable to find the cache dir "{self.cache_root_path}"'))
if not test:
self.save_storage = Storage(engine='sqlite3', dir_database=DATABASE_DIR)
db_conf = get_config()['database']['tiramisu_dsn']
self.save_storage = Storage(engine='postgres')
self.save_storage.setting(dsn=db_conf)
self.servermodel = {}
self.server = {}
super().__init__(test)
@ -115,12 +116,10 @@ class Risotto(Controller):
# build servermodel metaconfig (v_xxx.m_v_xxx)
metaconfig = await MetaConfig([],
optiondescription=optiondescription,
persistent=True,
session_id=session_id,
storage=self.save_storage)
mixconfig = await MixConfig(children=[],
optiondescription=optiondescription,
persistent=True,
session_id='m_' + session_id,
storage=self.save_storage)
await metaconfig.config.add(mixconfig)
@ -248,8 +247,7 @@ class Risotto(Controller):
""" build server's config
"""
config = await metaconfig.config.new(session_id,
storage=self.save_storage,
persistent=True)
storage=self.save_storage)
await config.information.set('server_id', server_id)
await config.information.set('server_name', server_name)
await config.owner.set(server_name)

5
src/risotto/services/servermodel/servermodel.py
View File

@ -5,7 +5,6 @@ from yaml import load, SafeLoader
from traceback import print_exc
from typing import Dict, List, Optional
from rougail import CreoleObjSpace
from rougail.config import dtdfilename
from ...controller import Controller
from ...register import register
from ...utils import _
@ -68,7 +67,7 @@ class Risotto(Controller):
as_names_str = '", "'.join(as_names)
await log.info(risotto_context,
_(f'gen funcs for "{servermodel_name}" with application services "{as_names_str}"'))
eolobj = CreoleObjSpace(dtdfilename)
#eolobj = CreoleObjSpace(get_config()['global']['rougail_dtd_path'])
async def servermodel_gen_schema(self,
servermodel_name: str,
@ -106,7 +105,7 @@ class Risotto(Controller):
continue
as_names.add(applicationservice_name)
extras.append((namespace, [extra_dir]))
eolobj = CreoleObjSpace(dtdfilename)
eolobj = CreoleObjSpace(get_config()['global']['rougail_dtd_path'])
as_names_str = '", "'.join(as_names)
await log.info(risotto_context,
_(f'gen schema for "{servermodel_name}" with application services "{as_names_str}"'))

2
src/risotto/services/session/session.py
View File

@ -293,7 +293,7 @@ class Risotto(Controller):
modif_config = session['config']
await config.value.importation(await modif_config.value.exportation())
await config.permissive.importation(await modif_config.permissive.exportation())
storage.del_session(session_id)
await storage.del_session(session_id)
return self.format_session(session_id, session)
@register_http('v1', '/config/server/{session_id}')

17
src/risotto/services/session/storage.py
View File

@ -103,8 +103,21 @@ class Storage(object):
raise NotAllowedError()
return session
def del_session(self,
id: int):
async def del_session(self,
id: int):
config = self.sessions[id]['meta']
while True:
try:
children = list(await config.config.list())
if not children:
# it's an empty metaconfig
break
config = children[0]
await config.session.reset()
except:
# it's a config, so no "list" method
break
await self.sessions[id]['meta'].session.reset()
del self.sessions[id]