Infra
/
risotto
9
0
Fork 1
Code Issues 1 Releases Wiki Activity

WIP

This commit is contained in:
Benjamin Bohard 2019-12-13 13:55:30 +01:00
parent 7dc6ce7845
commit a7934e37d7
22 changed files with 431 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -29,4 +29,9 @@ psql -U postgres -h localhost -c "GRANT ALL ON DATABASE risotto TO risotto;"
psql -U postgres -h localhost -c "CREATE EXTENSION hstore;" risotto
```
Gestion de la base de données avec Sqitch
```
cpanm --quiet --notest App::Sqitch
sqitch init risotto --uri https://forge.cadoles.com/Infra/risotto --engine pg
```

46
messages/v1/messages/applicationservices.create.yml Normal file
View File

@ -0,0 +1,46 @@
---
uri: applicationservice.create
description: |
Créé un service applicatif.
pattern: rpc
public: true
parameters:
applicationservice_name:
type: String
shortarg: n
description: |
Nom du service applicatif à créer.
applicationservice_description:
type: String
shortarg: d
description: |
Description du service applicatif à créer.
release_id:
type: Number
shortarg: r
description: |
Identifiant de la version associée au service applicatif.
response:
type: ApplicationService
description: Informations sur le modèle de serveur créé.
errors:
- uri: servermodel.create.error.database_not_available
- uri: servermodel.create.error.duplicate_servermodel
- uri: servermodel.create.error.invalid_parentservermodel_id
- uri: servermodel.create.error.invalid_source_id
- uri: servermodel.create.error.unknown_parentservermodel_id
- uri: servermodel.create.error.unknown_source_id
- uri: servermodel.create.error.servermodelname_not_provided
related:
- servermodel.list
- servermodel.describe
- servermodel.update
- servermodel.delete
- servermodel.event

24
messages/v1/messages/applicationservices.dataset.updated.yml Normal file
View File

@ -0,0 +1,24 @@
uri: applicationservices.dataset.updated
description: |
Initialise la table pour les services applicatifs.
pattern: rpc
public: true
domain: applicationservices-domain
parameters:
release_path:
type: String
shortarg: s
description: Nom de la source.
release_id:
type: Number
shortarg: r
description: Nom de la version.
response:
type: ReturnStatus
description: Code de retour sur l’injection des services applicatifs en base.

24
messages/v1/messages/servermodel.dataset.updated.yml Normal file
View File

@ -0,0 +1,24 @@
uri: servermodel.dataset.updated
description: |
Initialise la table pour les modèles de serveur.
pattern: rpc
public: true
domain: servermodel-domain
parameters:
release_path:
type: String
shortarg: s
description: Nom de la source.
release_id:
type: Number
shortarg: r
description: Nom de la version.
response:
type: ReturnStatus
description: Code de retour sur l’injection des modèles de serveur en base.

27
messages/v1/messages/servermodel.init.yml
View File

@ -1,27 +0,0 @@
uri: servermodel.init
description: |
Initialise la table pour les modèles de serveur.
sampleuse: |
zephir-client servermodel.init
pattern: rpc
public: true
domain: servermodel-domain
response:
type: ReturnStatus
description: Liste des modèles de serveur disponibles.
errors:
- uri: servermodel.list.error.database_engine_not_available
related:
- servermodel.describe
- servermodel.create
- servermodel.update
- servermodel.delete
- servermodel.event

28
messages/v1/messages/source.dataset.updated.yml Normal file
View File

@ -0,0 +1,28 @@
uri: source.dataset.updated
description: |
Initialise la table pour les versions.
pattern: rpc
public: true
domain: source-domain
parameters:
source_name:
type: String
shortarg: s
description: Nom de la source.
source_url:
type: String
shortarg: u
description: URL de la source.
release_name:
type: String
shortarg: r
description: Nom de la version.
response:
type: Release
description: Informations sur la version injectée en base.

15
messages/v1/messages/source.release.list.yml Normal file
View File

@ -0,0 +1,15 @@
---
uri: source.release.list
description: |
Retourne la liste des versions.
pattern: rpc
public: true
domain: source-domain
response:
type: '[]Release'
description: Liste des versions disponibles.

31
messages/v1/types/applicationservice.yml Normal file
View File

@ -0,0 +1,31 @@
---
title: ApplicationService
type: object
description: Description d'un modèle de serveur.
properties:
applicationservice_id:
type: number
description: ID du service applicatif.
applicationservice_name:
type: string
description: Nom du service applicatif.
applicationservice_description:
type: string
description: Description du service applicatif.
release_id:
type: number
ref: Version.ReleaseId
description: Version du service applicatif.
applicationservice_dependencies:
type: array
items:
type: integer
description: Liste des services applicatifs déclarés en dépendance de ce service applicatif.
required:
- servermodelid
- servermodelname
- servermodeldescription
- servermodelsubreleaseid
- sourceid
- subreleasename

4
messages/v1/types/global.returnstatus.yml
View File

@ -6,9 +6,9 @@ properties:
retcode:
type: number
description: Code de retour de la commande.
return:
returns:
type: string
description: Retour de la commande.
required:
- retcode
- return
- returns

24
messages/v1/types/release.yml Normal file
View File

@ -0,0 +1,24 @@
---
title: Release
type: object
description: Description de la version.
properties:
release_id:
type: number
description: Identifiant de la version.
release_name:
type: string
description: Le nom de la version.
source_url:
type: string
description: URL de la source.
ref: Source.ReleaseId
source_name:
type: string
description: Le nom de la source.
required:
- release_id
- release_name
- source_name
- source_url

84
script/database_manager.py Normal file
View File

@ -0,0 +1,84 @@
import asyncpg
import asyncio
from risotto.config import get_config
VERSION_INIT = """
-- Création de la table Source
CREATE TABLE Source (
SourceId SERIAL PRIMARY KEY,
SourceName VARCHAR(255) NOT NULL UNIQUE,
SourceURL TEXT
);
-- Création de la table Release
CREATE TABLE Release (
ReleaseId SERIAL PRIMARY KEY,
ReleaseName VARCHAR(255) NOT NULL,
ReleaseSourceId INTEGER NOT NULL,
UNIQUE (ReleaseName, ReleaseSourceId),
FOREIGN KEY (ReleaseSourceId) REFERENCES Source(SourceId)
);
-- Création de la table ServerModel
CREATE TABLE ServerModel (
ServerModelId SERIAL PRIMARY KEY,
ServerModelName VARCHAR(255) NOT NULL,
ServerModelDescription VARCHAR(255) NOT NULL,
ServerModelParentsId INTEGER [] DEFAULT '{}',
ServerModelReleaseId INTEGER NOT NULL,
ServerModelApplicationServiceId INTEGER NOT NULL,
ServerModelUsers hstore,
UNIQUE (ServerModelName, ServerModelReleaseId)
);
-- Création de la table ApplicationService
CREATE TABLE ApplicationService (
ApplicationServiceId SERIAL PRIMARY KEY,
ApplicationServiceName VARCHAR(255) NOT NULL,
ApplicationServiceDescription VARCHAR(255) NOT NULL,
ApplicationServiceReleaseId INTEGER NOT NULL,
ApplicationServiceDependencies JSON,
UNIQUE (ApplicationServiceName, ApplicationServiceReleaseId)
);
-- Création de la table de jointure ApplicationServiceProvides
CREATE TABLE ApplicationServiceProvides (
ApplicationServiceId INTEGER NOT NULL,
VirtualApplicationServiceId INTEGER NOT NULL,
FOREIGN KEY (ApplicationServiceId) REFERENCES ApplicationService(ApplicationServiceId),
FOREIGN KEY (VirtualApplicationServiceId) REFERENCES ApplicationService(ApplicationServiceId),
PRIMARY KEY (ApplicationServiceId, VirtualApplicationServiceId)
);
-- Création de la table Package
CREATE TABLE Package (
PackageId SERIAL PRIMARY KEY,
PackageApplicationServiceId INTEGER,
PackageName VARCHAR(255) NOT NULL,
FOREIGN KEY (PackageApplicationServiceId) REFERENCES ApplicationService(ApplicationServiceId)
);
-- Création de la table Document
CREATE TABLE Document (
DocumentId SERIAL PRIMARY KEY,
DocumentServiceId INTEGER,
DocumentName VARCHAR(255) NOT NULL,
DocumentPath TEXT,
DocumentOwner VARCHAR(255) DEFAULT 'root',
DocumentGroup VARCHAR(255) DEFAULT 'root',
DocumentMode VARCHAR(10) DEFAULT '0644',
DocumentType VARCHAR(100) CHECK ( DocumentType IN ('probes', 'aggregated_dico', 'dico', 'template', 'pretemplate', 'posttemplate', 'preservice', 'postservice', 'creolefuncs', 'file') ),
DocumentSHASUM VARCHAR(255),
DocumentContent BYTEA,
FOREIGN KEY (DocumentServiceId) REFERENCES ApplicationService(ApplicationServiceId)
);
"""
async def main():
db_conf = get_config().get('database')
pool = await asyncpg.create_pool(database=db_conf.get('dbname'), user=db_conf.get('user'))
async with pool.acquire() as connection:
async with connection.transaction():
returns = await connection.execute(VERSION_INIT)
asyncio.run(main())

4
src/risotto/config.py
View File

@ -1,7 +1,7 @@
HTTP_PORT = 8080
MESSAGE_ROOT_PATH = 'messages'
ROOT_CACHE_DIR = 'cache'
DEBUG = False
DEBUG = True
DATABASE_DIR = 'database'
INTERNAL_USER = 'internal'
CONFIGURATION_DIR = 'configurations'
@ -20,7 +20,7 @@ def get_config():
},
'http_server': {'port': 8080},
'global': {'message_root_path': 'messages',
'debug': False,
'debug': DEBUG,
'internal_user': 'internal',
'rougail_dtd_path': '../rougail/data/creole.dtd'}
}

24
src/risotto/dispatcher.py
View File

@ -92,20 +92,20 @@ class CallDispatcher:
try:
tiramisu_config = self.load_kwargs_to_config(risotto_context,
kwargs)
obj = self.messages[version][message]
function_obj = self.messages[version][message]
kw = tiramisu_config.option(message).value.dict()
risotto_context.function = obj['function']
if obj['risotto_context']:
risotto_context.function = function_obj['function']
if function_obj['risotto_context']:
kw['risotto_context'] = risotto_context
if 'database' in obj and obj['database']:
if function_obj['database']:
db_conf = get_config().get('database')
pool = await asyncpg.create_pool(database=db_conf.get('dbname'), user=db_conf.get('user'))
async with pool.acquire() as connection:
risotto_context.connection = connection
async with connection.transaction():
returns = await risotto_context.function(self.injected_self[obj['module']], **kw)
returns = await risotto_context.function(self.injected_self[function_obj['module']], **kw)
else:
returns = await risotto_context.function(self.injected_self[obj['module']], **kw)
returns = await risotto_context.function(self.injected_self[function_obj['module']], **kw)
except CallError as err:
raise err
except Exception as err:
@ -124,8 +124,8 @@ class CallDispatcher:
kwargs,
_(f'returns {returns}'))
# notification
if obj.get('notification'):
notif_version, notif_message = obj['notification'].split('.', 1)
if function_obj.get('notification'):
notif_version, notif_message = function_obj['notification'].split('.', 1)
if not isinstance(returns, list):
send_returns = [returns]
else:
@ -174,6 +174,14 @@ class PublishDispatcher:
if function_obj['risotto_context']:
kw['risotto_context'] = risotto_context
# send event
if function_obj['database']:
db_conf = get_config().get('database')
pool = await asyncpg.create_pool(database=db_conf.get('dbname'), user=db_conf.get('user'))
async with pool.acquire() as connection:
risotto_context.connection = connection
async with connection.transaction():
returns = await function(self.injected_self[function_obj['module']], **kw)
else:
returns = await function(self.injected_self[function_obj['module']], **kw)
except Exception as err:
if DEBUG:

4
src/risotto/error.py
View File

@ -8,3 +8,7 @@ class CallError(Exception):
class NotAllowedError(Exception):
pass
class ExecutionError(Exception):
pass

13
src/risotto/http.py
View File

@ -61,19 +61,24 @@ class extra_route_handler:
async def handle(request):
version, uri = request.match_info.get_info()['path'].rsplit('/', 2)[-2:]
version, message = request.match_info.get_info()['path'].rsplit('/', 2)[-2:]
risotto_context = create_context(request)
kwargs = await request.json()
try:
text = await dispatcher.call(version,
uri,
pattern = dispatcher.messages[version][message]['pattern']
if pattern == 'rpc':
method = dispatcher.call
else:
method = dispatcher.publish
text = await method(version,
message,
risotto_context,
public_only=True,
**kwargs)
except NotAllowedError as err:
raise HTTPNotFound(reason=str(err))
except CallError as err:
raise HTTPBadRequest(reason=str(err))
raise HTTPBadRequest(reason=str(err).replace('\n', ' '))
except Exception as err:
if DEBUG:
print_exc()

18
src/risotto/message/message.py
View File

@ -51,7 +51,7 @@ class MessageDefinition:
'related',
'response')
def __init__(self, raw_def):
def __init__(self, raw_def, message):
# default value for non mandatory key
self.version = u''
self.parameters = OrderedDict()
@ -91,6 +91,8 @@ class MessageDefinition:
# message with pattern = error must be public
if self.public is False and self.pattern == 'error':
raise Exception(_('Error message must be public : {}').format(self.uri))
if self.uri != message:
raise Exception(_(f'yaml file name "{message}.yml" does not match uri "{self.uri}"'))
class ParameterDefinition:
@ -217,8 +219,8 @@ def _parse_parameters(raw_defs):
return parameters
def parse_definition(filename: str):
return MessageDefinition(load(filename, Loader=SafeLoader))
def parse_definition(filecontent: bytes, message: str):
return MessageDefinition(load(filecontent, Loader=SafeLoader), message)
def is_message_defined(uri):
version, message = split_message_uri(uri)
@ -231,7 +233,7 @@ def get_message(uri):
version, message = split_message_uri(uri)
path = get_message_file_path(version, message)
with open(path, "r") as message_file:
message_content = parse_definition(message_file.read())
message_content = parse_definition(message_file.read(), message)
message_content.version = version
return message_content
except Exception as err:
@ -478,13 +480,7 @@ def _parse_responses(message_def,
"""build option with returns
"""
if message_def.response.parameters is None:
raise Exception('not implemented yet')
#name = 'response'
#keys['']['columns'][name] = {'description': message_def.response.description,
# 'type': message_def.response.type}
#responses = {}
#responses['keys'] = keys
#return responses
raise Exception('uri "{}" did not returned any valid parameters.'.format(message_def.uri))
options = []
names = []

4
src/risotto/register.py
View File

@ -218,6 +218,7 @@ class RegisterDispatcher:
dico = {'module': module_name,
'function': function,
'arguments': function_args,
'database': database,
'risotto_context': inject_risotto_context}
if notification and notification is not undefined:
dico['notification'] = notification
@ -238,6 +239,9 @@ class RegisterDispatcher:
for version, messages in self.messages.items():
for message, message_obj in messages.items():
if not 'functions' in message_obj and not 'function' in message_obj:
if message_obj['pattern'] == 'event':
print(f'{message} prêche dans le désert')
else:
missing_messages.append(message)
if missing_messages:
raise RegistrationError(_(f'missing uri {missing_messages}'))

1
src/risotto/services/applicationservices/__init__.py Normal file
View File

@ -0,0 +1 @@
from .application_services import Risotto

31
src/risotto/services/applicationservices/applicationservices.py Normal file
View File

@ -0,0 +1,31 @@
from ...controller import Controller
from ...register import register
class Risotto(Controller):
@register('v1.applicationservice.create', None, database=True)
async def applicationservice_create(self, risotto_context, applicationservice_name, applicationservice_description, release_id):
applicationservice_update_query = """INSERT INTO ApplicationService(ApplicationServiceName, ApplicationServiceDescription, ApplicationServiceReleaseId) VALUES ($1,$2,$3)
RETURNING ApplicationServiceId
"""
applicationservice_id = await risotto_context.connection.fetchval(applicationservice_update_query, applicationservice_description['name'], applicationservice_description['description'], release_id)
return {'applicationservice_name': applicationservice_name, 'applicationservice_description': applicationservice_description, 'applicationservice_id': applicationservice_id}
@register('v1.applicationservice.dataset.updated', None, database=True)
async def applicationservice_update(self, risotto_context, release_path, release_id):
applicationservice_path = os.path.join(release_path, 'applicationservice')
for service in os.listdir(applicationservice_path):
try:
applicationservice_description_path = os.path.join(applicationservice_path, service, 'applicationservice.yml')
with open(applicationservice_description_path, 'r') as applicationservice_yml:
applicationservice_description = yaml.load(applicationservice_yml, Loader=yaml.SafeLoader)
except Exception as err:
if get_config().get('global').get('debug'):
print_exc()
raise ExecutionError(_(f'Error while reading {applicationservice_description_path}: {err}'))
try:
await self.applicationservice_create(risotto_context, applicationservice_description['name'], applicationservice_description['description'], release_id)
except Exception as err:
if get_config().get('global').get('debug'):
print_exc()
raise ExecutionError(_(f"Error while injecting application service {applicationservice_description['name']} in database: {err}"))
return {'retcode': 0, 'returns': _('Application Services successfully loaded')}

46
src/risotto/services/servermodel/servermodel.py
View File

@ -1,27 +1,35 @@
from ...controller import Controller
from ...register import register
sql_init = """
-- Création de la table ServerModel
CREATE TABLE ServerModel (
ServerModelId SERIAL PRIMARY KEY,
ServerModelName VARCHAR(255) NOT NULL,
ServerModelDescription VARCHAR(255) NOT NULL,
ServerModelSourceId INTEGER NOT NULL,
ServerModelParentId INTEGER,
ServerModelSubReleaseId INTEGER NOT NULL,
ServerModelSubReleaseName VARCHAR(255) NOT NULL,
UNIQUE (ServerModelName, ServerModelSubReleaseId)
);
"""
from ...utils import _
import os
import yaml
from traceback import print_exc
from ...config import get_config
from ...error import ExecutionError
class Risotto(Controller):
@register('v1.servermodel.init', None, database=True)
async def servermodel_init(self, risotto_context):
result = await risotto_context.connection.execute(sql_init)
return {'retcode': 0, 'return': result}
@register('v1.servermodel.dataset.updated', None, database=True)
async def servermodel_update(self, risotto_context, release_path, release_id, applicationservice_įd):
applicationservice_update = """INSERT INTO ApplicationService(ApplicationServiceName, ApplicationServiceDescription, ApplicationServiceReleaseId) VALUES ($1,$2,$3)
RETURNING ApplicationServiceId
"""
servermodel_path = os.path.join(release_path, 'servermodel')
for servermodel in os.listdir(servermodel_path):
try:
with open(os.path.join(servermodel_path, servermodel), 'r') as applicationservice_yml:
applicationservice_description = yaml.load(applicationservice_yml, Loader=yaml.SafeLoader)
except Exception as err:
if get_config().get('global').get('debug'):
print_exc()
raise ExecutionError(_(f'Error while reading {applicationservice_description_path}: {err}'))
try:
await risotto_context.connection.fetch(applicationservice_update, applicationservice_description['name'], applicationservice_description['description'], release_id)
except Exception as err:
if get_config().get('global').get('debug'):
print_exc()
raise ExecutionError(_(f"Error while injecting application service {applicationservice_description['name']} in database: {err}"))
return {'retcode': 0, 'returns': _('Application Services successfully loaded')}
@register('v1.servermodel.list', None, database=True)
async def servermodel_list(self, risotto_context, sourceid):

1
src/risotto/services/source/__init__.py Normal file
View File

@ -0,0 +1 @@
from .source import Risotto

41
src/risotto/services/source/source.py Normal file
View File

@ -0,0 +1,41 @@
from ...controller import Controller
from ...register import register
VERSION_INIT = """
-- Création de la table Source
CREATE TABLE Source (
SourceId SERIAL PRIMARY KEY,
SourceName VARCHAR(255) NOT NULL UNIQUE,
SourceURL TEXT
);
-- Création de la table Release
CREATE TABLE Release (
ReleaseId SERIAL PRIMARY KEY,
ReleaseName VARCHAR(255) NOT NULL,
ReleaseSourceId INTEGER NOT NULL,
UNIQUE (ReleaseName, ReleaseSourceId),
FOREIGN KEY (ReleaseSourceId) REFERENCES Source(SourceId)
);
"""
RELEASE_QUERY = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name FROM Release, Source WHERE Source.SourceId=Release.ReleaseSourceId"""
class Risotto(Controller):
@register('v1.source.dataset.updated', None, database=True)
async def version_update(self, risotto_context, source_name, source_url, release_name):
source_upsert = """INSERT INTO Source(SourceName, SourceURL) VALUES ($1, $2)
ON CONFLICT (SourceName) DO UPDATE SET SourceURL = $2
RETURNING SourceId
"""
release_insert = """INSERT INTO Release(ReleaseName, ReleaseSourceId) VALUES ($1, $2)
RETURNING ReleaseId
"""
source_id = await risotto_context.connection.fetchval(source_upsert, source_name, source_url)
result = await risotto_context.connection.fetchval(release_insert, release_name, source_id)
return {'release_id': result, 'source_name': source_name, 'source_url': source_url, 'release_name': release_name}
@register('v1.source.release.list', None, database=True)
async def release_list(self, risotto_context):
result = await risotto_context.connection.fetch(RELEASE_QUERY)
return [dict(r) for r in result]