diff --git a/messages/v1/messages/old/servermodel.describe.yml b/messages/v1/messages/old/servermodel.describe.yml new file mode 100644 index 0000000..588ccbc --- /dev/null +++ b/messages/v1/messages/old/servermodel.describe.yml @@ -0,0 +1,32 @@ +--- +uri: servermodel.describe + +description: | + Retourne les attributs détaillés d'un modèle de serveur. + +pattern: rpc + +public: true + +parameters: + servermodel_id: + type: Number + shortarg: s + description: Identifiant du modèle de serveur à récupérer. + ref: Servermodel.ServermodelId + +response: + type: Servermodel + description: Description du modèle de serveur. + +errors: + - uri: servermodel.describe.error.database_not_available + - uri: servermodel.describe.error.invalid_servermodel_id + - uri: servermodel.describe.error.unknown_servermodel_id + +related: + - servermodel.list + - servermodel.create + - servermodel.update + - servermodel.delete + - servermodel.event diff --git a/messages/v1/messages/servermodel.dataset.updated.yml b/messages/v1/messages/servermodel.dataset.updated.yml index d0efe3e..24237b5 100644 --- a/messages/v1/messages/servermodel.dataset.updated.yml +++ b/messages/v1/messages/servermodel.dataset.updated.yml @@ -8,14 +8,14 @@ pattern: rpc public: true parameters: - release_path: + source_name: type: String shortarg: s description: Nom de la source. - release_id: - type: Number + release_distribution: + type: String shortarg: r - description: Nom de la version. + description: Distribution de la version. response: type: ReturnStatus diff --git a/messages/v1/messages/servermodel.describe.yml b/messages/v1/messages/servermodel.describe.yml deleted file mode 100644 index b8d5672..0000000 --- a/messages/v1/messages/servermodel.describe.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- -uri: servermodel.describe - -description: | - Retourne les attributs détaillés d'un modèle de serveur. - -pattern: rpc - -public: true - -parameters: - servermodel_id: - type: Number - shortarg: s - description: Identifiant du modèle de serveur à récupérer. - ref: Servermodel.ServermodelId - inheritance: - type: Boolean - shortarg: i - description: Inclure les données héritées des modèles de serveur parents. - default: true - resolvdepends: - type: Boolean - shortarg: r - description: Résoudre les dépendances de services. - default: true - schema: - type: Boolean - shortarg: c - description: Inclure le schema de configuration (réaggrège les données provenant du datasource). - default: false - probes: - type: Boolean - shortarg: p - description: Inclure les informations sur les sondes de la configuration. - default: false - creolefuncs: - type: Boolean - shortarg: o - description: Inclure les fonctions Creole. - default: false - conffiles: - type: Boolean - shortarg: f - description: Inclure les fichier creole au format tar encodé en base64 - default: false - -response: - type: Servermodel - description: Description du modèle de serveur. - -errors: - - uri: servermodel.describe.error.database_not_available - - uri: servermodel.describe.error.invalid_servermodel_id - - uri: servermodel.describe.error.unknown_servermodel_id - -related: - - servermodel.list - - servermodel.create - - servermodel.update - - servermodel.delete - - servermodel.event diff --git a/messages/v1/messages/source.get.yml b/messages/v1/messages/source.describe.yml similarity index 92% rename from messages/v1/messages/source.get.yml rename to messages/v1/messages/source.describe.yml index c38f556..336506f 100644 --- a/messages/v1/messages/source.get.yml +++ b/messages/v1/messages/source.describe.yml @@ -1,5 +1,5 @@ --- -uri: source.get +uri: source.describe description: | Retourne une source. diff --git a/messages/v1/messages/source.release.get_by_distribution.yml b/messages/v1/messages/source.release.get_by_distribution.yml new file mode 100644 index 0000000..acc13d7 --- /dev/null +++ b/messages/v1/messages/source.release.get_by_distribution.yml @@ -0,0 +1,24 @@ +--- +uri: source.release.get_by_distribution + +description: | + Retourne version suivant le nom de la distribution. + +pattern: rpc + +public: true + +parameters: + source_id: + type: Number + shortarg: s + description: ID de la source. + release_distribution: + type: String + shortarg: r + description: Distribution de la version. + +response: + type: 'Release' + description: La version disponibles. + diff --git a/messages/v1/messages/source.release.get_by_id.yml b/messages/v1/messages/source.release.get_by_id.yml new file mode 100644 index 0000000..0aab4e5 --- /dev/null +++ b/messages/v1/messages/source.release.get_by_id.yml @@ -0,0 +1,19 @@ +--- +uri: source.release.get_by_id + +description: | + Retourne version suivant l'identifiant. + +pattern: rpc + +public: true + +parameters: + release_id: + type: Number + shortarg: r + description: ID de la version. + +response: + type: 'Release' + description: La version disponibles. diff --git a/script/database_manager.py b/script/database_manager.py index 9cba22c..a886622 100644 --- a/script/database_manager.py +++ b/script/database_manager.py @@ -17,6 +17,7 @@ CREATE TABLE Release ( ReleaseSourceId INTEGER NOT NULL, ReleaseDistribution VARCHAR(20) CONSTRAINT releasedistribution_choice CHECK (ReleaseDistribution IN ('dev', 'stable', 'maintained', 'deprecation-warning', 'deprecated')), UNIQUE (ReleaseName, ReleaseSourceId), + UNIQUE (ReleaseDistribution, ReleaseSourceId), FOREIGN KEY (ReleaseSourceId) REFERENCES Source(SourceId) ); diff --git a/src/risotto/config.py b/src/risotto/config.py index 71bcacb..81fc16a 100644 --- a/src/risotto/config.py +++ b/src/risotto/config.py @@ -23,5 +23,6 @@ def get_config(): 'debug': DEBUG, 'internal_user': 'internal', 'rougail_dtd_path': '../rougail/data/creole.dtd'}, - 'source': {'root_path': '/srv/seed'} + 'source': {'root_path': '/srv/seed'}, + 'cache': {'root_path': '/var/cache/risotto'} } diff --git a/src/risotto/services/config/config.py b/src/risotto/services/config/config.py index 088796b..82b7087 100644 --- a/src/risotto/services/config/config.py +++ b/src/risotto/services/config/config.py @@ -82,28 +82,29 @@ class Risotto(Controller): f'Load servermodel {servermodel_name} ({servermodel_id})') # use file in cache if found, otherwise retrieve it in servermodel context - if isfile(cache_file): - fileio = open(cache_file) - else: - servermodel = await self.call('v1.servermodel.describe', - risotto_context, - servermodel_id=servermodel_id, - inheritance=False, - resolvdepends=False, - schema=True, - creolefuncs=True) - fileio = BytesIO() - fileio.write(servermodel['schema'].encode()) - fileio.seek(0) - - with open(cache_file, 'w') as cache: - cache.write(servermodel['schema']) - with open(funcs_file, 'w') as cache: - cache.write(servermodel['creolefuncs']) - del servermodel - - # loads tiramisu config and store it - xmlroot = parse(fileio).getroot() +# if isfile(cache_file): +# fileio = open(cache_file) +# else: +# servermodel = await self.call('v1.servermodel.describe', +# risotto_context, +# servermodel_id=servermodel_id, +# inheritance=False, +# resolvdepends=False, +# schema=True, +# creolefuncs=True) +# fileio = BytesIO() +# fileio.write(servermodel['schema'].encode()) +# fileio.seek(0) +# +# with open(cache_file, 'w') as cache: +# cache.write(servermodel['schema']) +# with open(funcs_file, 'w') as cache: +# cache.write(servermodel['creolefuncs']) +# del servermodel +# +# # loads tiramisu config and store it + with open(cache_file) as fileio: + xmlroot = parse(fileio).getroot() self.servermodel[servermodel_id] = self.build_metaconfig(servermodel_id, servermodel_name, xmlroot, diff --git a/src/risotto/services/servermodel/servermodel.py b/src/risotto/services/servermodel/servermodel.py index cea60bb..a311936 100644 --- a/src/risotto/services/servermodel/servermodel.py +++ b/src/risotto/services/servermodel/servermodel.py @@ -1,8 +1,10 @@ -from os import listdir -from os.path import join +from os import listdir, makedirs +from os.path import join, isdir from yaml import load, SafeLoader from traceback import print_exc from typing import Dict, List +from rougail import CreoleObjSpace +from rougail.config import dtdfilename from ...controller import Controller from ...register import register from ...utils import _ @@ -12,24 +14,61 @@ from ...error import ExecutionError class Risotto(Controller): + def __init__(self): + self.source_root_path = get_config().get('source').get('root_path') + self.cache_root_path = join(get_config().get('cache').get('root_path'), 'servermodel') + if not isdir(self.cache_root_path): + makedirs(join(self.cache_root_path)) + async def on_join(self, risotto_context: Context) -> None: internal_source = await self.call('v1.source.create', risotto_context, source_name='internal', source_url='none') - self.internal_release_id = await self.call('v1.source.release.create', - risotto_context, - source_id=internal_source['source_id'], - release_name='none') + internal_release = await self.call('v1.source.release.create', + risotto_context, + source_id=internal_source['source_id'], + release_name='none') + self.internal_release_id = internal_release['release_id'] + + async def servermodel_gen_schema(self, + risotto_context: Context, + servermodel_id: int, + applicationservice_id: int) -> None: + dependencies = await self.get_applicationservices(risotto_context, + applicationservice_id) + if release_cache is None: + release_cache = {} + dict_paths = [] + for applicationservice_id, applicationservice_infos in dependencies.items(): + applicationservice_name, as_release_id = applicationservice_infos + if as_release_id not in release_cache: + release_cache[as_release_id] = await self.call('v1.source.release.get_by_id', + release_id=as_release_id) + dict_paths.append(join(self.source_root_path, + release_cache[as_release_id]['source_name'], + release_cache[as_release_id]['release_distribution'], + 'applicationservice', + applicationservice_name, + 'dictionaries')) + eolobj = CreoleObjSpace(dtdfilename) + eolobj.create_or_populate_from_xml('creole', dict_paths) + # FIXME extra + # FIXME eosfunc + eosfunc = '/eosfunc.py' + eolobj.space_visitor(eosfunc) + eolobj.save(join(self.cache_root_path, f'{servermodel_id}.xml')) async def _servermodel_create(self, risotto_context: Context, servermodel_name: str, servermodel_description: str, - servermodel_parents_id, - release_id): - servermodel_update = """INSERT INTO Servermodel(ServermodelName, ServermodelDescription, ServermodelParentsId, ServermodelReleaseId, ServermodelApplicationServiceId) VALUES ($1,$2,$3,$4,$5) + servermodel_parents_id: List[int], + release_id: int, + release_cache: Dict=None) -> Dict: + servermodel_update = """INSERT INTO Servermodel(ServermodelName, ServermodelDescription, ServermodelParentsId, ServermodelReleaseId, ServermodelApplicationServiceId) + VALUES ($1,$2,$3,$4,$5) RETURNING ServermodelId """ as_name = f"local_{servermodel_name}" @@ -39,28 +78,32 @@ class Risotto(Controller): applicationservice_name=as_name, applicationservice_description=as_description, release_id=self.internal_release_id) - servermodel = await risotto_context.connection.fetchval(servermodel_update, - sm_name, - sm_description, - sm_parentsid, - release_id, - applicationservice['applicationservice_id']) + applicationservice_id = applicationservice['applicationservice_id'] + servermodel_id = await risotto_context.connection.fetchval(servermodel_update, + servermodel_name, + servermodel_description, + servermodel_parents_id, + release_id, + applicationservice_id) + self.servermodel_gen_schema(risotto_context, + servermodel_id, + applicationservice_id) return {'servermodel_name': servermodel_name, 'servermodel_description': servermodel_description, 'servermodel_parents_id': servermodel_parents_id, 'release_id': release_id, - 'servermodel_id': servermodel['servermodel_id']} + 'servermodel_id': servermodel_id} def parse_parents(self, servermodels: Dict, servermodel: Dict, parents: List=None) -> List: if parents is None: - parents = [servermodel] - parent = servermodels[servermodel]['parent'] + parents = [servermodel['name']] + parent = servermodel['parent'] if parent in servermodels: parents.append(parent) - self.parse_parents(servermodels, parent, parents) + self.parse_parents(servermodels, servermodels[parent], parents) return parents async def get_servermodel_id_by_name(self, @@ -75,11 +118,22 @@ class Risotto(Controller): @register('v1.servermodel.dataset.updated', None, database=True) async def servermodel_update(self, risotto_context: Context, - release_path: str, - release_id: int): - servermodel_path = join(release_path, 'servermodel') + source_name: str, + release_distribution: int): + servermodel_path = join(self.source_root_path, + source_name, + release_distribution, + 'servermodel') servermodels = {} # for dependencies reason load all servermodels + source = await self.call('v1.source.describe', + risotto_context, + source_name=source_name) + release = await self.call('v1.source.release.get_by_distribution', + risotto_context, + source_id=source['source_id'], + release_distribution=release_distribution) + release_id = release['release_id'] for servermodel in listdir(servermodel_path): if not servermodel.endswith('.yml'): continue @@ -92,10 +146,11 @@ class Risotto(Controller): if get_config().get('global').get('debug'): print_exc() raise ExecutionError(_(f'Error while reading {servermodel_description_path}: {err}')) - servermodels[servermodel_yml['name']] = servermodel_description - servermodels[servermodel_yml['name']]['done'] = False + servermodels[servermodel_description['name']] = servermodel_description + servermodels[servermodel_description['name']]['done'] = False - for servermodel in servermodels: + release_cache = {release['release_id']: release} + for servermodel in servermodels.values(): if not servermodel['done']: # parent needs to create before child, so retrieve all parents parents = self.parse_parents(servermodels, @@ -114,10 +169,13 @@ class Risotto(Controller): sm_name = servermodel_description['name'] sm_description = servermodel_description['description'] try: - servermodel_id = await self._servermodel_create(sm_name, + servermodel_ob = await self._servermodel_create(risotto_context, + sm_name, sm_description, servermodelparent_id, - release_id)['servermodel_id'] + release_id, + release_cache) + servermodel_id = servermodel_ob['servermodel_id'] except Exception as err: if get_config().get('global').get('debug'): print_exc() @@ -136,56 +194,59 @@ class Risotto(Controller): servermodels = await risotto_context.connection.fetch(sql) return [dict(r) for r in servermodels] - @register('v1.servermodel.describe', None) - async def servermodel_describe(self, inheritance, creolefuncs, servermodel_id, schema, conffiles, resolvdepends, probes): - schema = """ - - - - - - - False - - - /etc/mailname - - - False - - - mailname - - - True - - - - basic - - - - - normal - - oui - non - mandatory - normal - non - - - normal - - - normal - - - normal - - - - - -""" - return {'servermodel_id': 1, 'servermodel_name': 'name', 'servermodel_description': 'description', 'release_id': 1, 'schema': schema, 'creolefuncs': ''} + async def _parse_depends(self, + risotto_context, + applicationservice_id: int, + or_depends: list, + ids: list) -> None: + applicationservice_query = """ + SELECT ApplicationServiceName, ApplicationServiceDependencies, ApplicationServiceReleaseId + FROM applicationservice + WHERE applicationserviceid=$1""" + applicationservice = await risotto_context.connection.fetchval(servermodel_update, + applicationservice_id) + ids[applicationservice_id] = (applicationserverice['ApplicationServiceName'], + applicationserverice['ApplicationServiceReleaseId']) + if applicationserverice['ApplicationServiceDependencies']: + for depend in pplicationserverice['ApplicationServiceDependencies']: + if isinstance(depend, dict): + or_depends.append(depend['or']) + elif depend not in ids: + await self._parse_depends(risotto_context, + depend, + or_depends, + ids) + + async def _parse_or_depends(self, + risotto_context, + or_depends: list, + ids: list) -> None: + new_or_depends = [] + set_ids = set(ids) + for or_depend in or_depends: + if not set(or_depend) & set_ids: + applicationservice_id= or_depend[0] + await self._parse_depends(risotto_context, + applicationservice_id, + new_or_depends, + ids) + if new_or_depends: + await self._parse_or_depends(risotto_context, + new_or_depends, + ids) + + async def get_applicationservices(self, + risotto_context, + applicationservice_id: int) -> list: + """Return consolidated dependencies or raise. + """ + or_depends = [] + ids = [] + await self._parse_depends(risotto_context, + applicationservice_id, + or_depends, + ids) + await self._parse_or_depends(risotto_context, + or_depends, + ids) + return ids diff --git a/src/risotto/services/source/source.py b/src/risotto/services/source/source.py index b3280d2..bf050c9 100644 --- a/src/risotto/services/source/source.py +++ b/src/risotto/services/source/source.py @@ -43,16 +43,19 @@ class Risotto(Controller): 'source_url': source_url, 'source_id': source_id} - @register('v1.source.get', None, database=True) - async def source_get(self, - risotto_context: Context, - source_name: str) -> Dict: + @register('v1.source.describe', None, database=True) + async def source_describe(self, + risotto_context: Context, + source_name: str) -> Dict: source_get = """SELECT SourceId as source_id, SourceName as source_name, SourceURL as source_url FROM Source WHERE SourceName = $1 """ - return dict(await risotto_context.connection.fetchrow(source_get, - source_name)) + source = await risotto_context.connection.fetchrow(source_get, + source_name) + if not source: + raise Exception(f'unknown source with name {source_name}') + return dict(source) @register('v1.source.list', None, database=True) async def source_list(self, @@ -81,7 +84,7 @@ class Risotto(Controller): @register('v1.source.release.create', None, database=True) async def source_release_create(self, risotto_context: Context, - source_id: str, + source_id: int, release_name: str) -> Dict: source_get = """SELECT SourceId as source_id, SourceName as source_name, SourceURL as source_url FROM Source @@ -104,6 +107,34 @@ class Risotto(Controller): @register('v1.source.release.list', None, database=True) async def release_list(self, risotto_context): - release_query = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name FROM Release, Source WHERE Source.SourceId=Release.ReleaseSourceId""" - result = await risotto_context.connection.fetch(release_name) + release_query = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name + FROM Release, Source + WHERE Source.SourceId=Release.ReleaseSourceId""" + result = await risotto_context.connection.fetch(release_query) return [dict(r) for r in result] + + @register('v1.source.release.get_by_id', None, database=True) + async def release_get_by_id(self, + risotto_context: Context, + release_id: int) -> Dict: + release_query = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name + FROM Release, Source + WHERE Release.ReleaseId = $1""" + result = await risotto_context.connection.fetchrow(release_query, + release_id) + return dict(result) + + @register('v1.source.release.get_by_distribution', None, database=True) + async def release_get_by_distribution(self, + risotto_context: Context, + source_id: int, + release_distribution: str) -> Dict: + release_query = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name + FROM Release, Source + WHERE Source.SourceId = $1 AND Release.ReleaseName = $2""" + result = await risotto_context.connection.fetchrow(release_query, + source_id, + release_distribution) + if not result: + raise Exception(f'unknown distribution {release_distribution} with source {source_id}') + return dict(result) diff --git a/tests/fake_services/servermodel/servermodel.py b/tests/fake_services/servermodel/servermodel.py index 57d1b70..a85415e 100644 --- a/tests/fake_services/servermodel/servermodel.py +++ b/tests/fake_services/servermodel/servermodel.py @@ -14,56 +14,62 @@ class Risotto(Controller): 'servermodel_description': 'description2', 'servermodel_parents_id': [1]}] - @register('v1.servermodel.describe', None) - async def servermodel_describe(self, inheritance, creolefuncs, servermodel_id, schema, conffiles, resolvdepends, probes): - schema = """ - - - - - - - False - - - /etc/mailname - - - False - - - mailname - - - True - - - - basic - - - - - normal - - oui - non - mandatory - normal - non - - - normal - - - normal - - - normal - - - - - -""" - return {'servermodel_id': 1, 'servermodel_name': 'name', 'servermodel_description': 'description', 'release_id': 1, 'schema': schema, 'creolefuncs': ''} +# @register('v1.servermodel.describe', None) +# async def servermodel_describe(self, +# servermodel_id, +# inheritance, +# resolv_depends): +# schema = """ +# +# +# +# +# +# +# False +# +# +# /etc/mailname +# +# +# False +# +# +# mailname +# +# +# True +# +# +# +# basic +# +# +# +# +# normal +# +# oui +# non +# mandatory +# normal +# non +# +# +# normal +# +# +# normal +# +# +# normal +# +# +# +# +# +#""" +# return {'servermodel_id': 1, +# 'servermodel_name': 'name', +# 'servermodel_description': 'description', +# 'release_id': 1}