This commit is contained in:
Benjamin Bohard 2019-12-13 13:55:30 +01:00
parent 7dc6ce7845
commit a7934e37d7
22 changed files with 431 additions and 78 deletions

View File

@ -29,4 +29,9 @@ psql -U postgres -h localhost -c "GRANT ALL ON DATABASE risotto TO risotto;"
psql -U postgres -h localhost -c "CREATE EXTENSION hstore;" risotto psql -U postgres -h localhost -c "CREATE EXTENSION hstore;" risotto
``` ```
Gestion de la base de données avec Sqitch
```
cpanm --quiet --notest App::Sqitch
sqitch init risotto --uri https://forge.cadoles.com/Infra/risotto --engine pg
```

View File

@ -0,0 +1,46 @@
---
uri: applicationservice.create
description: |
Créé un service applicatif.
pattern: rpc
public: true
parameters:
applicationservice_name:
type: String
shortarg: n
description: |
Nom du service applicatif à créer.
applicationservice_description:
type: String
shortarg: d
description: |
Description du service applicatif à créer.
release_id:
type: Number
shortarg: r
description: |
Identifiant de la version associée au service applicatif.
response:
type: ApplicationService
description: Informations sur le modèle de serveur créé.
errors:
- uri: servermodel.create.error.database_not_available
- uri: servermodel.create.error.duplicate_servermodel
- uri: servermodel.create.error.invalid_parentservermodel_id
- uri: servermodel.create.error.invalid_source_id
- uri: servermodel.create.error.unknown_parentservermodel_id
- uri: servermodel.create.error.unknown_source_id
- uri: servermodel.create.error.servermodelname_not_provided
related:
- servermodel.list
- servermodel.describe
- servermodel.update
- servermodel.delete
- servermodel.event

View File

@ -0,0 +1,24 @@
uri: applicationservices.dataset.updated
description: |
Initialise la table pour les services applicatifs.
pattern: rpc
public: true
domain: applicationservices-domain
parameters:
release_path:
type: String
shortarg: s
description: Nom de la source.
release_id:
type: Number
shortarg: r
description: Nom de la version.
response:
type: ReturnStatus
description: Code de retour sur linjection des services applicatifs en base.

View File

@ -0,0 +1,24 @@
uri: servermodel.dataset.updated
description: |
Initialise la table pour les modèles de serveur.
pattern: rpc
public: true
domain: servermodel-domain
parameters:
release_path:
type: String
shortarg: s
description: Nom de la source.
release_id:
type: Number
shortarg: r
description: Nom de la version.
response:
type: ReturnStatus
description: Code de retour sur linjection des modèles de serveur en base.

View File

@ -1,27 +0,0 @@
uri: servermodel.init
description: |
Initialise la table pour les modèles de serveur.
sampleuse: |
zephir-client servermodel.init
pattern: rpc
public: true
domain: servermodel-domain
response:
type: ReturnStatus
description: Liste des modèles de serveur disponibles.
errors:
- uri: servermodel.list.error.database_engine_not_available
related:
- servermodel.describe
- servermodel.create
- servermodel.update
- servermodel.delete
- servermodel.event

View File

@ -0,0 +1,28 @@
uri: source.dataset.updated
description: |
Initialise la table pour les versions.
pattern: rpc
public: true
domain: source-domain
parameters:
source_name:
type: String
shortarg: s
description: Nom de la source.
source_url:
type: String
shortarg: u
description: URL de la source.
release_name:
type: String
shortarg: r
description: Nom de la version.
response:
type: Release
description: Informations sur la version injectée en base.

View File

@ -0,0 +1,15 @@
---
uri: source.release.list
description: |
Retourne la liste des versions.
pattern: rpc
public: true
domain: source-domain
response:
type: '[]Release'
description: Liste des versions disponibles.

View File

@ -0,0 +1,31 @@
---
title: ApplicationService
type: object
description: Description d'un modèle de serveur.
properties:
applicationservice_id:
type: number
description: ID du service applicatif.
applicationservice_name:
type: string
description: Nom du service applicatif.
applicationservice_description:
type: string
description: Description du service applicatif.
release_id:
type: number
ref: Version.ReleaseId
description: Version du service applicatif.
applicationservice_dependencies:
type: array
items:
type: integer
description: Liste des services applicatifs déclarés en dépendance de ce service applicatif.
required:
- servermodelid
- servermodelname
- servermodeldescription
- servermodelsubreleaseid
- sourceid
- subreleasename

View File

@ -6,9 +6,9 @@ properties:
retcode: retcode:
type: number type: number
description: Code de retour de la commande. description: Code de retour de la commande.
return: returns:
type: string type: string
description: Retour de la commande. description: Retour de la commande.
required: required:
- retcode - retcode
- return - returns

View File

@ -0,0 +1,24 @@
---
title: Release
type: object
description: Description de la version.
properties:
release_id:
type: number
description: Identifiant de la version.
release_name:
type: string
description: Le nom de la version.
source_url:
type: string
description: URL de la source.
ref: Source.ReleaseId
source_name:
type: string
description: Le nom de la source.
required:
- release_id
- release_name
- source_name
- source_url

View File

@ -0,0 +1,84 @@
import asyncpg
import asyncio
from risotto.config import get_config
VERSION_INIT = """
-- Création de la table Source
CREATE TABLE Source (
SourceId SERIAL PRIMARY KEY,
SourceName VARCHAR(255) NOT NULL UNIQUE,
SourceURL TEXT
);
-- Création de la table Release
CREATE TABLE Release (
ReleaseId SERIAL PRIMARY KEY,
ReleaseName VARCHAR(255) NOT NULL,
ReleaseSourceId INTEGER NOT NULL,
UNIQUE (ReleaseName, ReleaseSourceId),
FOREIGN KEY (ReleaseSourceId) REFERENCES Source(SourceId)
);
-- Création de la table ServerModel
CREATE TABLE ServerModel (
ServerModelId SERIAL PRIMARY KEY,
ServerModelName VARCHAR(255) NOT NULL,
ServerModelDescription VARCHAR(255) NOT NULL,
ServerModelParentsId INTEGER [] DEFAULT '{}',
ServerModelReleaseId INTEGER NOT NULL,
ServerModelApplicationServiceId INTEGER NOT NULL,
ServerModelUsers hstore,
UNIQUE (ServerModelName, ServerModelReleaseId)
);
-- Création de la table ApplicationService
CREATE TABLE ApplicationService (
ApplicationServiceId SERIAL PRIMARY KEY,
ApplicationServiceName VARCHAR(255) NOT NULL,
ApplicationServiceDescription VARCHAR(255) NOT NULL,
ApplicationServiceReleaseId INTEGER NOT NULL,
ApplicationServiceDependencies JSON,
UNIQUE (ApplicationServiceName, ApplicationServiceReleaseId)
);
-- Création de la table de jointure ApplicationServiceProvides
CREATE TABLE ApplicationServiceProvides (
ApplicationServiceId INTEGER NOT NULL,
VirtualApplicationServiceId INTEGER NOT NULL,
FOREIGN KEY (ApplicationServiceId) REFERENCES ApplicationService(ApplicationServiceId),
FOREIGN KEY (VirtualApplicationServiceId) REFERENCES ApplicationService(ApplicationServiceId),
PRIMARY KEY (ApplicationServiceId, VirtualApplicationServiceId)
);
-- Création de la table Package
CREATE TABLE Package (
PackageId SERIAL PRIMARY KEY,
PackageApplicationServiceId INTEGER,
PackageName VARCHAR(255) NOT NULL,
FOREIGN KEY (PackageApplicationServiceId) REFERENCES ApplicationService(ApplicationServiceId)
);
-- Création de la table Document
CREATE TABLE Document (
DocumentId SERIAL PRIMARY KEY,
DocumentServiceId INTEGER,
DocumentName VARCHAR(255) NOT NULL,
DocumentPath TEXT,
DocumentOwner VARCHAR(255) DEFAULT 'root',
DocumentGroup VARCHAR(255) DEFAULT 'root',
DocumentMode VARCHAR(10) DEFAULT '0644',
DocumentType VARCHAR(100) CHECK ( DocumentType IN ('probes', 'aggregated_dico', 'dico', 'template', 'pretemplate', 'posttemplate', 'preservice', 'postservice', 'creolefuncs', 'file') ),
DocumentSHASUM VARCHAR(255),
DocumentContent BYTEA,
FOREIGN KEY (DocumentServiceId) REFERENCES ApplicationService(ApplicationServiceId)
);
"""
async def main():
db_conf = get_config().get('database')
pool = await asyncpg.create_pool(database=db_conf.get('dbname'), user=db_conf.get('user'))
async with pool.acquire() as connection:
async with connection.transaction():
returns = await connection.execute(VERSION_INIT)
asyncio.run(main())

View File

@ -1,7 +1,7 @@
HTTP_PORT = 8080 HTTP_PORT = 8080
MESSAGE_ROOT_PATH = 'messages' MESSAGE_ROOT_PATH = 'messages'
ROOT_CACHE_DIR = 'cache' ROOT_CACHE_DIR = 'cache'
DEBUG = False DEBUG = True
DATABASE_DIR = 'database' DATABASE_DIR = 'database'
INTERNAL_USER = 'internal' INTERNAL_USER = 'internal'
CONFIGURATION_DIR = 'configurations' CONFIGURATION_DIR = 'configurations'
@ -20,7 +20,7 @@ def get_config():
}, },
'http_server': {'port': 8080}, 'http_server': {'port': 8080},
'global': {'message_root_path': 'messages', 'global': {'message_root_path': 'messages',
'debug': False, 'debug': DEBUG,
'internal_user': 'internal', 'internal_user': 'internal',
'rougail_dtd_path': '../rougail/data/creole.dtd'} 'rougail_dtd_path': '../rougail/data/creole.dtd'}
} }

View File

@ -92,20 +92,20 @@ class CallDispatcher:
try: try:
tiramisu_config = self.load_kwargs_to_config(risotto_context, tiramisu_config = self.load_kwargs_to_config(risotto_context,
kwargs) kwargs)
obj = self.messages[version][message] function_obj = self.messages[version][message]
kw = tiramisu_config.option(message).value.dict() kw = tiramisu_config.option(message).value.dict()
risotto_context.function = obj['function'] risotto_context.function = function_obj['function']
if obj['risotto_context']: if function_obj['risotto_context']:
kw['risotto_context'] = risotto_context kw['risotto_context'] = risotto_context
if 'database' in obj and obj['database']: if function_obj['database']:
db_conf = get_config().get('database') db_conf = get_config().get('database')
pool = await asyncpg.create_pool(database=db_conf.get('dbname'), user=db_conf.get('user')) pool = await asyncpg.create_pool(database=db_conf.get('dbname'), user=db_conf.get('user'))
async with pool.acquire() as connection: async with pool.acquire() as connection:
risotto_context.connection = connection risotto_context.connection = connection
async with connection.transaction(): async with connection.transaction():
returns = await risotto_context.function(self.injected_self[obj['module']], **kw) returns = await risotto_context.function(self.injected_self[function_obj['module']], **kw)
else: else:
returns = await risotto_context.function(self.injected_self[obj['module']], **kw) returns = await risotto_context.function(self.injected_self[function_obj['module']], **kw)
except CallError as err: except CallError as err:
raise err raise err
except Exception as err: except Exception as err:
@ -124,8 +124,8 @@ class CallDispatcher:
kwargs, kwargs,
_(f'returns {returns}')) _(f'returns {returns}'))
# notification # notification
if obj.get('notification'): if function_obj.get('notification'):
notif_version, notif_message = obj['notification'].split('.', 1) notif_version, notif_message = function_obj['notification'].split('.', 1)
if not isinstance(returns, list): if not isinstance(returns, list):
send_returns = [returns] send_returns = [returns]
else: else:
@ -174,6 +174,14 @@ class PublishDispatcher:
if function_obj['risotto_context']: if function_obj['risotto_context']:
kw['risotto_context'] = risotto_context kw['risotto_context'] = risotto_context
# send event # send event
if function_obj['database']:
db_conf = get_config().get('database')
pool = await asyncpg.create_pool(database=db_conf.get('dbname'), user=db_conf.get('user'))
async with pool.acquire() as connection:
risotto_context.connection = connection
async with connection.transaction():
returns = await function(self.injected_self[function_obj['module']], **kw)
else:
returns = await function(self.injected_self[function_obj['module']], **kw) returns = await function(self.injected_self[function_obj['module']], **kw)
except Exception as err: except Exception as err:
if DEBUG: if DEBUG:

View File

@ -8,3 +8,7 @@ class CallError(Exception):
class NotAllowedError(Exception): class NotAllowedError(Exception):
pass pass
class ExecutionError(Exception):
pass

View File

@ -61,19 +61,24 @@ class extra_route_handler:
async def handle(request): async def handle(request):
version, uri = request.match_info.get_info()['path'].rsplit('/', 2)[-2:] version, message = request.match_info.get_info()['path'].rsplit('/', 2)[-2:]
risotto_context = create_context(request) risotto_context = create_context(request)
kwargs = await request.json() kwargs = await request.json()
try: try:
text = await dispatcher.call(version, pattern = dispatcher.messages[version][message]['pattern']
uri, if pattern == 'rpc':
method = dispatcher.call
else:
method = dispatcher.publish
text = await method(version,
message,
risotto_context, risotto_context,
public_only=True, public_only=True,
**kwargs) **kwargs)
except NotAllowedError as err: except NotAllowedError as err:
raise HTTPNotFound(reason=str(err)) raise HTTPNotFound(reason=str(err))
except CallError as err: except CallError as err:
raise HTTPBadRequest(reason=str(err)) raise HTTPBadRequest(reason=str(err).replace('\n', ' '))
except Exception as err: except Exception as err:
if DEBUG: if DEBUG:
print_exc() print_exc()

View File

@ -51,7 +51,7 @@ class MessageDefinition:
'related', 'related',
'response') 'response')
def __init__(self, raw_def): def __init__(self, raw_def, message):
# default value for non mandatory key # default value for non mandatory key
self.version = u'' self.version = u''
self.parameters = OrderedDict() self.parameters = OrderedDict()
@ -91,6 +91,8 @@ class MessageDefinition:
# message with pattern = error must be public # message with pattern = error must be public
if self.public is False and self.pattern == 'error': if self.public is False and self.pattern == 'error':
raise Exception(_('Error message must be public : {}').format(self.uri)) raise Exception(_('Error message must be public : {}').format(self.uri))
if self.uri != message:
raise Exception(_(f'yaml file name "{message}.yml" does not match uri "{self.uri}"'))
class ParameterDefinition: class ParameterDefinition:
@ -217,8 +219,8 @@ def _parse_parameters(raw_defs):
return parameters return parameters
def parse_definition(filename: str): def parse_definition(filecontent: bytes, message: str):
return MessageDefinition(load(filename, Loader=SafeLoader)) return MessageDefinition(load(filecontent, Loader=SafeLoader), message)
def is_message_defined(uri): def is_message_defined(uri):
version, message = split_message_uri(uri) version, message = split_message_uri(uri)
@ -231,7 +233,7 @@ def get_message(uri):
version, message = split_message_uri(uri) version, message = split_message_uri(uri)
path = get_message_file_path(version, message) path = get_message_file_path(version, message)
with open(path, "r") as message_file: with open(path, "r") as message_file:
message_content = parse_definition(message_file.read()) message_content = parse_definition(message_file.read(), message)
message_content.version = version message_content.version = version
return message_content return message_content
except Exception as err: except Exception as err:
@ -478,13 +480,7 @@ def _parse_responses(message_def,
"""build option with returns """build option with returns
""" """
if message_def.response.parameters is None: if message_def.response.parameters is None:
raise Exception('not implemented yet') raise Exception('uri "{}" did not returned any valid parameters.'.format(message_def.uri))
#name = 'response'
#keys['']['columns'][name] = {'description': message_def.response.description,
# 'type': message_def.response.type}
#responses = {}
#responses['keys'] = keys
#return responses
options = [] options = []
names = [] names = []

View File

@ -218,6 +218,7 @@ class RegisterDispatcher:
dico = {'module': module_name, dico = {'module': module_name,
'function': function, 'function': function,
'arguments': function_args, 'arguments': function_args,
'database': database,
'risotto_context': inject_risotto_context} 'risotto_context': inject_risotto_context}
if notification and notification is not undefined: if notification and notification is not undefined:
dico['notification'] = notification dico['notification'] = notification
@ -238,6 +239,9 @@ class RegisterDispatcher:
for version, messages in self.messages.items(): for version, messages in self.messages.items():
for message, message_obj in messages.items(): for message, message_obj in messages.items():
if not 'functions' in message_obj and not 'function' in message_obj: if not 'functions' in message_obj and not 'function' in message_obj:
if message_obj['pattern'] == 'event':
print(f'{message} prêche dans le désert')
else:
missing_messages.append(message) missing_messages.append(message)
if missing_messages: if missing_messages:
raise RegistrationError(_(f'missing uri {missing_messages}')) raise RegistrationError(_(f'missing uri {missing_messages}'))

View File

@ -0,0 +1 @@
from .application_services import Risotto

View File

@ -0,0 +1,31 @@
from ...controller import Controller
from ...register import register
class Risotto(Controller):
@register('v1.applicationservice.create', None, database=True)
async def applicationservice_create(self, risotto_context, applicationservice_name, applicationservice_description, release_id):
applicationservice_update_query = """INSERT INTO ApplicationService(ApplicationServiceName, ApplicationServiceDescription, ApplicationServiceReleaseId) VALUES ($1,$2,$3)
RETURNING ApplicationServiceId
"""
applicationservice_id = await risotto_context.connection.fetchval(applicationservice_update_query, applicationservice_description['name'], applicationservice_description['description'], release_id)
return {'applicationservice_name': applicationservice_name, 'applicationservice_description': applicationservice_description, 'applicationservice_id': applicationservice_id}
@register('v1.applicationservice.dataset.updated', None, database=True)
async def applicationservice_update(self, risotto_context, release_path, release_id):
applicationservice_path = os.path.join(release_path, 'applicationservice')
for service in os.listdir(applicationservice_path):
try:
applicationservice_description_path = os.path.join(applicationservice_path, service, 'applicationservice.yml')
with open(applicationservice_description_path, 'r') as applicationservice_yml:
applicationservice_description = yaml.load(applicationservice_yml, Loader=yaml.SafeLoader)
except Exception as err:
if get_config().get('global').get('debug'):
print_exc()
raise ExecutionError(_(f'Error while reading {applicationservice_description_path}: {err}'))
try:
await self.applicationservice_create(risotto_context, applicationservice_description['name'], applicationservice_description['description'], release_id)
except Exception as err:
if get_config().get('global').get('debug'):
print_exc()
raise ExecutionError(_(f"Error while injecting application service {applicationservice_description['name']} in database: {err}"))
return {'retcode': 0, 'returns': _('Application Services successfully loaded')}

View File

@ -1,27 +1,35 @@
from ...controller import Controller from ...controller import Controller
from ...register import register from ...register import register
from ...utils import _
sql_init = """ import os
-- Création de la table ServerModel import yaml
CREATE TABLE ServerModel ( from traceback import print_exc
ServerModelId SERIAL PRIMARY KEY, from ...config import get_config
ServerModelName VARCHAR(255) NOT NULL, from ...error import ExecutionError
ServerModelDescription VARCHAR(255) NOT NULL,
ServerModelSourceId INTEGER NOT NULL,
ServerModelParentId INTEGER,
ServerModelSubReleaseId INTEGER NOT NULL,
ServerModelSubReleaseName VARCHAR(255) NOT NULL,
UNIQUE (ServerModelName, ServerModelSubReleaseId)
);
"""
class Risotto(Controller): class Risotto(Controller):
@register('v1.servermodel.init', None, database=True) @register('v1.servermodel.dataset.updated', None, database=True)
async def servermodel_init(self, risotto_context): async def servermodel_update(self, risotto_context, release_path, release_id, applicationservice_įd):
result = await risotto_context.connection.execute(sql_init) applicationservice_update = """INSERT INTO ApplicationService(ApplicationServiceName, ApplicationServiceDescription, ApplicationServiceReleaseId) VALUES ($1,$2,$3)
return {'retcode': 0, 'return': result} RETURNING ApplicationServiceId
"""
servermodel_path = os.path.join(release_path, 'servermodel')
for servermodel in os.listdir(servermodel_path):
try:
with open(os.path.join(servermodel_path, servermodel), 'r') as applicationservice_yml:
applicationservice_description = yaml.load(applicationservice_yml, Loader=yaml.SafeLoader)
except Exception as err:
if get_config().get('global').get('debug'):
print_exc()
raise ExecutionError(_(f'Error while reading {applicationservice_description_path}: {err}'))
try:
await risotto_context.connection.fetch(applicationservice_update, applicationservice_description['name'], applicationservice_description['description'], release_id)
except Exception as err:
if get_config().get('global').get('debug'):
print_exc()
raise ExecutionError(_(f"Error while injecting application service {applicationservice_description['name']} in database: {err}"))
return {'retcode': 0, 'returns': _('Application Services successfully loaded')}
@register('v1.servermodel.list', None, database=True) @register('v1.servermodel.list', None, database=True)
async def servermodel_list(self, risotto_context, sourceid): async def servermodel_list(self, risotto_context, sourceid):

View File

@ -0,0 +1 @@
from .source import Risotto

View File

@ -0,0 +1,41 @@
from ...controller import Controller
from ...register import register
VERSION_INIT = """
-- Création de la table Source
CREATE TABLE Source (
SourceId SERIAL PRIMARY KEY,
SourceName VARCHAR(255) NOT NULL UNIQUE,
SourceURL TEXT
);
-- Création de la table Release
CREATE TABLE Release (
ReleaseId SERIAL PRIMARY KEY,
ReleaseName VARCHAR(255) NOT NULL,
ReleaseSourceId INTEGER NOT NULL,
UNIQUE (ReleaseName, ReleaseSourceId),
FOREIGN KEY (ReleaseSourceId) REFERENCES Source(SourceId)
);
"""
RELEASE_QUERY = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name FROM Release, Source WHERE Source.SourceId=Release.ReleaseSourceId"""
class Risotto(Controller):
@register('v1.source.dataset.updated', None, database=True)
async def version_update(self, risotto_context, source_name, source_url, release_name):
source_upsert = """INSERT INTO Source(SourceName, SourceURL) VALUES ($1, $2)
ON CONFLICT (SourceName) DO UPDATE SET SourceURL = $2
RETURNING SourceId
"""
release_insert = """INSERT INTO Release(ReleaseName, ReleaseSourceId) VALUES ($1, $2)
RETURNING ReleaseId
"""
source_id = await risotto_context.connection.fetchval(source_upsert, source_name, source_url)
result = await risotto_context.connection.fetchval(release_insert, release_name, source_id)
return {'release_id': result, 'source_name': source_name, 'source_url': source_url, 'release_name': release_name}
@register('v1.source.release.list', None, database=True)
async def release_list(self, risotto_context):
result = await risotto_context.connection.fetch(RELEASE_QUERY)
return [dict(r) for r in result]