From 4c775c21e499e355b9aaa8143552e31a456ae388 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Sun, 23 Feb 2020 16:53:29 +0100 Subject: [PATCH] add v1.applicationservice.dependency.add message --- README.md | 16 +- .../messages/applicationservice.updated.yml | 10 + messages/v1/messages/server.create.yml | 5 - .../v1/messages/servermodel.get_by_id.yml | 17 - script/database_manager.py | 43 +-- src/risotto/config.py | 3 +- .../applicationservice/applicationservice.py | 129 ++++++-- src/risotto/services/server/server.py | 8 +- src/risotto/services/servermodel/generator.py | 172 ++++++++++ .../services/servermodel/servermodel.py | 300 ++++-------------- src/risotto/services/uri/uri.py | 3 +- 11 files changed, 395 insertions(+), 311 deletions(-) create mode 100644 messages/v1/messages/applicationservice.updated.yml delete mode 100644 messages/v1/messages/servermodel.get_by_id.yml create mode 100644 src/risotto/services/servermodel/generator.py diff --git a/README.md b/README.md index 8bbe5fa..cf0077b 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ docker exec -ti postgres bash psql -U postgres -h localhost -c "CREATE ROLE risotto WITH LOGIN PASSWORD 'risotto';" psql -U postgres -h localhost -c "CREATE DATABASE risotto;" psql -U postgres -h localhost -c "GRANT ALL ON DATABASE risotto TO risotto;" -psql -U postgres -h localhost -c "CREATE EXTENSION hstore;" risotto +#psql -U postgres -h localhost -c "CREATE EXTENSION hstore;" risotto ``` Gestion de la base de données avec Sqitch @@ -100,7 +100,7 @@ S=xxxxxxxxxxxxxxxxxxxxxx XXXXX # Create a server -./script/cucchiaiata server.create -s test -d description -m unbound_etab1 -n internal -r last +./script/cucchiaiata server.create -s test -d description -m unbound_etab1 -r last ./script/cucchiaiata session.server.start -s test S=xxxxxxxxxxxxxxxxxxxxxx @@ -113,5 +113,13 @@ S=xxxxxxxxxxxxxxxxxxxxxx ./script/cucchiaiata template.generate -s test -# OpenSSH: -./script/cucchiaiata server.create -s test -d description -m eolebase -n internal -r last +# OpenSSH +#add servermodel openssh link to eolebase +#add servermodel openssh2 lint to openssh +#link openssh applicationservice to this servermodel + +./script/cucchiaiata servermodel.create -n openssh_1 -d description -p eolebase -s eole -r last +./script/cucchiaiata servermodel.create -n openssh_2 -d openssh_2 -p openssh_1 -s internal -r last +./script/cucchiaiata applicationservice.dependency.add -n local_openssh_1 -a openssh -s eole -r last + +#./script/cucchiaiata server.create -s test -d description -m eolebase -r last diff --git a/messages/v1/messages/applicationservice.updated.yml b/messages/v1/messages/applicationservice.updated.yml new file mode 100644 index 0000000..ddc05a0 --- /dev/null +++ b/messages/v1/messages/applicationservice.updated.yml @@ -0,0 +1,10 @@ +--- +uri: applicationservice.updated + +description: Un service application a été modifié. + +pattern: event + +parameters: + type: 'ApplicationService' + description: Informations sur le service applicatif modifié. diff --git a/messages/v1/messages/server.create.yml b/messages/v1/messages/server.create.yml index 187ee4d..245afd1 100644 --- a/messages/v1/messages/server.create.yml +++ b/messages/v1/messages/server.create.yml @@ -19,11 +19,6 @@ parameters: shortarg: m ref: Servermodel.ServermodelName description: Nom du modèle de serveur. - source_name: - type: String - shortarg: n - ref: Source.SourceName - description: Nom de la source. release_distribution: type: String shortarg: r diff --git a/messages/v1/messages/servermodel.get_by_id.yml b/messages/v1/messages/servermodel.get_by_id.yml deleted file mode 100644 index 4f11666..0000000 --- a/messages/v1/messages/servermodel.get_by_id.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -uri: servermodel.get_by_id - -description: Retourne les attributs détaillés d'un modèle de serveur suivant son identifiant. - -pattern: rpc - -parameters: - servermodel_id: - type: Number - shortarg: s - description: Identifiant du modèle de serveur à récupérer. - ref: Servermodel.ServermodelId - -response: - type: Servermodel - description: Description du modèle de serveur. diff --git a/script/database_manager.py b/script/database_manager.py index cd5ee05..9de09ca 100644 --- a/script/database_manager.py +++ b/script/database_manager.py @@ -3,14 +3,14 @@ import asyncio from risotto.config import get_config VERSION_INIT = """ --- Création de la table Source +-- Source CREATE TABLE Source ( SourceId SERIAL PRIMARY KEY, SourceName VARCHAR(255) NOT NULL UNIQUE, SourceURL TEXT ); --- Création de la table Release +-- Release CREATE TABLE Release ( ReleaseId SERIAL PRIMARY KEY, ReleaseName VARCHAR(255) NOT NULL, @@ -21,30 +21,35 @@ CREATE TABLE Release ( FOREIGN KEY (ReleaseSourceId) REFERENCES Source(SourceId) ); - --- Création de la table Servermodel +-- Servermodel CREATE TABLE Servermodel ( ServermodelId SERIAL PRIMARY KEY, ServermodelName VARCHAR(255) NOT NULL, ServermodelDescription VARCHAR(255) NOT NULL, ServermodelParentsId INTEGER [] DEFAULT '{}', ServermodelReleaseId INTEGER NOT NULL, - ServermodelApplicationServiceId INTEGER NOT NULL, - ServermodelUsers hstore, + ServermodelApplicationserviceId INTEGER NOT NULL, UNIQUE (ServermodelName, ServermodelReleaseId) ); +CREATE INDEX ServermodelApplicationserviceId_index ON Servermodel (ServermodelApplicationserviceId); --- Création de la table ApplicationService -CREATE TABLE ApplicationService ( - ApplicationServiceId SERIAL PRIMARY KEY, - ApplicationServiceName VARCHAR(255) NOT NULL, - ApplicationServiceDescription VARCHAR(255) NOT NULL, - ApplicationServiceReleaseId INTEGER NOT NULL, - ApplicationServiceDependencies JSON, - UNIQUE (ApplicationServiceName, ApplicationServiceReleaseId) +-- Applicationservice +CREATE TABLE Applicationservice ( + ApplicationserviceId SERIAL PRIMARY KEY, + ApplicationserviceName VARCHAR(255) NOT NULL, + ApplicationserviceDescription VARCHAR(255) NOT NULL, + ApplicationserviceReleaseId INTEGER NOT NULL, + UNIQUE (ApplicationserviceName, ApplicationserviceReleaseId) +); +CREATE TABLE ApplicationserviceDependency ( + ApplicationserviceId INTEGER NOT NULL, + ApplicationserviceDependencyId INTEGER NOT NULL, + UNIQUE(ApplicationserviceId, ApplicationserviceDependencyId), + FOREIGN KEY (ApplicationserviceId) REFERENCES Applicationservice(ApplicationserviceId), + FOREIGN KEY (ApplicationserviceDependencyId) REFERENCES Applicationservice(ApplicationserviceId) ); --- Server table creation +-- Server CREATE TABLE Server ( ServerId SERIAL PRIMARY KEY, ServerName VARCHAR(255) NOT NULL UNIQUE, @@ -52,8 +57,7 @@ CREATE TABLE Server ( ServerServermodelId INTEGER NOT NULL ); --- User, Role and ACL table creation - +-- User, Role and ACL CREATE TABLE RisottoUser ( UserId SERIAL PRIMARY KEY, UserLogin VARCHAR(100) NOT NULL UNIQUE, @@ -82,8 +86,7 @@ CREATE TABLE RoleURI ( PRIMARY KEY (RoleName, URIId) ); --- Log table creation - +-- Log CREATE TABLE log( Msg VARCHAR(255) NOT NULL, Level VARCHAR(10) NOT NULL, @@ -92,7 +95,6 @@ CREATE TABLE log( Data JSON, Date timestamp DEFAULT current_timestamp ); - """ async def main(): @@ -106,4 +108,3 @@ if __name__ == '__main__': loop = asyncio.get_event_loop() loop.run_until_complete(main()) # asyncio.run(main()) - diff --git a/src/risotto/config.py b/src/risotto/config.py index 3482828..3180360 100644 --- a/src/risotto/config.py +++ b/src/risotto/config.py @@ -26,6 +26,7 @@ def get_config(): 'source': {'root_path': '/srv/seed'}, 'cache': {'root_path': '/var/cache/risotto'}, 'servermodel': {'internal_source': 'internal', - 'internal_distribution': 'last'}, + 'internal_distribution': 'last', + 'internal_release_name': 'none'}, } diff --git a/src/risotto/services/applicationservice/applicationservice.py b/src/risotto/services/applicationservice/applicationservice.py index aadd54f..467ee9b 100644 --- a/src/risotto/services/applicationservice/applicationservice.py +++ b/src/risotto/services/applicationservice/applicationservice.py @@ -2,7 +2,7 @@ from os import listdir from os.path import join from traceback import print_exc from yaml import load, SafeLoader -from typing import Dict, List +from typing import Dict, List, Set from ...controller import Controller from ...register import register @@ -17,6 +17,7 @@ class Risotto(Controller): self.source_root_path = get_config().get('source').get('root_path') self.internal_source_name = get_config()['servermodel']['internal_source'] self.internal_distribution_name = get_config()['servermodel']['internal_distribution'] + self.internal_release_name = get_config()['servermodel']['internal_release_name'] async def on_join(self, risotto_context: Context) -> None: @@ -27,7 +28,7 @@ class Risotto(Controller): internal_release = await self.call('v1.source.release.create', risotto_context, source_name=self.internal_source_name, - release_name='none', + release_name=self.internal_release_name, release_distribution=self.internal_distribution_name) self.internal_release_id = internal_release['release_id'] @@ -37,19 +38,33 @@ class Risotto(Controller): applicationservice_description: str, applicationservice_dependencies: List[int], release_id: int) -> Dict: - applicationservice_update_query = """INSERT INTO ApplicationService(ApplicationServiceName, ApplicationServiceDescription, ApplicationServiceDependencies, ApplicationServiceReleaseId) VALUES ($1,$2,$3,$4) - RETURNING ApplicationServiceId - """ + applicationservice_update_query = '''INSERT INTO Applicationservice(ApplicationserviceName, ApplicationserviceDescription, ApplicationserviceReleaseId) + VALUES ($1,$2,$3) + RETURNING ApplicationserviceId + ''' applicationservice_id = await risotto_context.connection.fetchval(applicationservice_update_query, applicationservice_name, applicationservice_description, - applicationservice_dependencies, release_id) + await self.insert_dependency(risotto_context, + applicationservice_id, + applicationservice_dependencies) return {'applicationservice_name': applicationservice_name, 'applicationservice_description': applicationservice_description, 'applicationservice_release_id': release_id, 'applicationservice_id': applicationservice_id} + async def insert_dependency(self, + risotto_context: Context, + applicationservice_id: int, + dependencies: list) -> None: + sql = '''INSERT INTO ApplicationserviceDependency(ApplicationserviceId, ApplicationserviceDependencyId) + VALUES ($1, $2)''' + for dependency in dependencies: + await risotto_context.connection.execute(sql, + applicationservice_id, + dependency) + @register('v1.applicationservice.dependency.add') async def applicationservice_dependency_add(self, risotto_context: Context, @@ -65,15 +80,59 @@ class Risotto(Controller): applicationservice_name, self.internal_release_id) dependency_descr = await self._applicationservice_describe(risotto_context, - dependency, + applicationservice_dependency, release['release_id']) - as_descr['applicationservice_dependencies'].append(dependency_descr['applicationservice_id']) - sql = "UPDATE ApplicationService SET ApplicationServiceDependencies = $1 WHERE ApplicationServiceId = $2" - await risotto_context.connection.execute(sql, - as_descr['applicationservice_dependencies'], - as_descr['applicationserviceid']) + sql = '''SELECT ApplicationserviceDependencyId + FROM ApplicationserviceDependency + WHERE ApplicationserviceId = $1 AND ApplicationserviceDependencyId = $2''' + if await risotto_context.connection.fetchrow(sql, + as_descr['applicationservice_id'], + dependency_descr['applicationservice_id']): + raise Exception(_(f'{applicationservice_name} has already a dependency to {applicationservice_dependency}')) + else: + await self.insert_dependency(risotto_context, + as_descr['applicationservice_id'], + [dependency_descr['applicationservice_id']]) + await self.publish('v1.applicationservice.updated', + risotto_context, + **as_descr) + await self.updated_related_applicationservice(risotto_context, + as_descr['applicationservice_id']) return as_descr + async def updated_related_applicationservice(self, + risotto_context: Context, + applicationservice_id: int) -> None: + sql = """ + SELECT ApplicationserviceId as applicationservice_id + FROM ApplicationserviceDependency + WHERE ApplicationserviceDependencyId = $1""" + for dependency in await risotto_context.connection.fetch(sql, + applicationservice_id): + dependency_id = dependency['applicationservice_id'] + applicationservice = await self._applicationservice_get_by_id(risotto_context, + dependency_id) + await self.publish('v1.applicationservice.updated', + risotto_context, + **applicationservice) + await self.updated_related_applicationservice(risotto_context, + dependency_id) + + async def get_dependencies(self, + risotto_context: Context, + applicationservice_id: int) -> List[int]: + dependencies = set() + sql = """ + SELECT ApplicationserviceDependencyId as applicationservice_dependency_id + FROM ApplicationserviceDependency + WHERE ApplicationserviceId = $1""" + for dependency in await risotto_context.connection.fetch(sql, + applicationservice_id): + dependencies.add(dependency['applicationservice_dependency_id']) + for dependency in dependencies.copy(): + dependencies |= await self.get_dependencies(risotto_context, + dependency) + return dependencies @register('v1.applicationservice.create') async def applicationservice_create(self, @@ -81,11 +140,15 @@ class Risotto(Controller): applicationservice_name: str, applicationservice_description: str, applicationservice_dependencies: List[int]) -> Dict: - return await self._applicationservice_create(risotto_context, - applicationservice_name, - applicationservice_description, - applicationservice_dependencies, - self.internal_release_id) + applicationservice = await self._applicationservice_create(risotto_context, + applicationservice_name, + applicationservice_description, + applicationservice_dependencies, + self.internal_release_id) + dependencies = list(await self.get_dependencies(risotto_context, + applicationservice['applicationservice_id'])) + applicationservice['applicationservice_dependencies'] = dependencies + return applicationservice @register('v1.applicationservice.dataset.updated') async def applicationservice_update(self, @@ -130,24 +193,34 @@ class Risotto(Controller): async def applicationservice_get_by_id(self, risotto_context: Context, applicationservice_id: int) -> Dict: + return await self._applicationservice_get_by_id(risotto_context, + applicationservice_id) + + async def _applicationservice_get_by_id(self, + risotto_context: Context, + applicationservice_id: int) -> Dict: applicationservice_query = """ - SELECT ApplicationServiceId as applicationservice_id, ApplicationServiceName as applicationservice_name, ApplicationServiceDependencies as applicationservice_dependencies, ApplicationServiceReleaseId as applicationservice_release_id + SELECT ApplicationserviceId as applicationservice_id, ApplicationserviceName as applicationservice_name, ApplicationserviceReleaseId as applicationservice_release_id FROM applicationservice - WHERE applicationserviceid=$1""" + WHERE ApplicationserviceId=$1""" applicationservice = await risotto_context.connection.fetchrow(applicationservice_query, applicationservice_id) if applicationservice is None: raise Exception(_(f'unknown service with ID {applicationservice_id}')) - return dict(applicationservice) + dependencies = list(await self.get_dependencies(risotto_context, + applicationservice['applicationservice_id'])) + applicationservice = dict(applicationservice) + applicationservice['applicationservice_dependencies'] = dependencies + return applicationservice async def _applicationservice_describe(self, risotto_context: Context, applicationservice_name, release_id): applicationservice_query = """ - SELECT ApplicationServiceId as applicationservice_id, ApplicationServiceName as applicationservice_name, ApplicationServiceDependencies as applicationservice_dependencies, ApplicationServiceReleaseId as applicationservice_release_id - FROM ApplicationService - WHERE ApplicationServiceName=$1 AND ApplicationServiceReleaseId=$2""" + SELECT ApplicationserviceId as applicationservice_id, ApplicationserviceName as applicationservice_name, ApplicationserviceReleaseId as applicationservice_release_id + FROM Applicationservice + WHERE ApplicationserviceName=$1 AND ApplicationserviceReleaseId=$2""" applicationservice = await risotto_context.connection.fetchrow(applicationservice_query, applicationservice_name, release_id) @@ -166,6 +239,10 @@ class Risotto(Controller): risotto_context, source_name=source_name, release_distribution=release_distribution) - return await self._applicationservice_describe(risotto_context, - applicationservice_name, - release['release_id']) + applicationservice = await self._applicationservice_describe(risotto_context, + applicationservice_name, + release['release_id']) + dependencies = list(await self.get_dependencies(risotto_context, + applicationservice['applicationservice_id'])) + applicationservice['applicationservice_dependencies'] = dependencies + return applicationservice diff --git a/src/risotto/services/server/server.py b/src/risotto/services/server/server.py index c5c768e..3f2fd9b 100644 --- a/src/risotto/services/server/server.py +++ b/src/risotto/services/server/server.py @@ -3,10 +3,15 @@ from typing import Dict from ...controller import Controller from ...register import register from ...context import Context +from ...config import get_config from ...utils import _ class Risotto(Controller): + def __init__(self, + test: bool) -> None: + self.internal_source_name = get_config()['servermodel']['internal_source'] + @register('v1.server.list') async def server_list(self, risotto_context: Context) -> Dict: @@ -23,12 +28,11 @@ class Risotto(Controller): server_name: str, server_description: str, servermodel_name: str, - source_name: str, release_distribution: str) -> Dict: servermodel = await self.call('v1.servermodel.describe', risotto_context, servermodel_name=servermodel_name, - source_name=source_name, + source_name=self.internal_source_name, release_distribution=release_distribution) server_insert = """INSERT INTO Server(ServerName, ServerDescription, ServerServermodelId) VALUES ($1,$2,$3) diff --git a/src/risotto/services/servermodel/generator.py b/src/risotto/services/servermodel/generator.py new file mode 100644 index 0000000..91a0044 --- /dev/null +++ b/src/risotto/services/servermodel/generator.py @@ -0,0 +1,172 @@ +from os.path import join, isdir, isfile +from os import listdir, makedirs +from shutil import rmtree, copyfile +from typing import Dict, List, Optional +from rougail import CreoleObjSpace +from rougail.config import dtdfilename +from ...controller import Controller +from ...context import Context +from ...logger import log +from ...utils import _ + + +class Generator(Controller): + async def generate(self, + risotto_context: Context, + servermodel_name: str, + servermodel_id: int, + dependencies: List[int], + generate_cache: Optional[Dict]=None) -> None: + if generate_cache is None: + generate_cache = {'applicationservice': {}, + 'release_id': {}} + await self.servermodel_gen_funcs(servermodel_name, + servermodel_id, + dependencies, + generate_cache, + risotto_context) + await self.servermodel_gen_schema(servermodel_name, + servermodel_id, + dependencies, + generate_cache, + risotto_context) + await self.servermodel_copy_templates(servermodel_name, + servermodel_id, + dependencies, + generate_cache, + risotto_context) + + async def servermodel_gen_funcs(self, + servermodel_name: str, + servermodel_id: int, + dependencies: Dict, + generate_cache: Dict, + risotto_context: Context) -> None: + as_names = [] + dest_file = self.get_servermodel_cache(servermodel_id, 'funcs.py') + with open(dest_file, 'wb') as funcs: + funcs.write(b'from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value\n\n') + for dependency in dependencies: + if dependency not in generate_cache['applicationservice']: + applicationservice = await self.call('v1.applicationservice.get_by_id', + risotto_context, + applicationservice_id=dependency) + generate_cache['applicationservice'][dependency] = (applicationservice['applicationservice_name'], + applicationservice['applicationservice_release_id']) + applicationservice_name, release_id = generate_cache['applicationservice'][dependency] + if release_id not in generate_cache['release_id']: + release = await self.call('v1.source.release.get_by_id', + risotto_context, + release_id=release_id) + generate_cache['release_id'][release_id] = (release['source_name'], + release['release_name']) + source_name, release_name = generate_cache['release_id'][release_id] + path = join(self.source_root_path, + source_name, + release_name, + 'applicationservice', + applicationservice_name, + 'funcs') + if isdir(path): + as_names.append(applicationservice_name) + for fil in listdir(path): + if not fil.endswith('.py'): + continue + fil_path = join(path, fil) + with open(fil_path, 'rb') as fh: + funcs.write(f'# {fil_path}\n'.encode()) + funcs.write(fh.read()) + funcs.write(b'\n') + + as_names_str = '", "'.join(as_names) + await log.info(risotto_context, + _(f'gen funcs for "{servermodel_name}" with application services "{as_names_str}"')) + + async def servermodel_gen_schema(self, + servermodel_name: str, + servermodel_id: int, + dependencies: Dict, + generate_cache: Dict, + risotto_context: Context) -> None: + paths = [] + extras = [] + as_names = set() + for dependency in dependencies: + applicationservice_name, release_id = generate_cache['applicationservice'][dependency] + source_name, release_name = generate_cache['release_id'][release_id] + # load creole dictionaries + path = join(self.source_root_path, + source_name, + release_name, + 'applicationservice', + applicationservice_name, + 'dictionaries') + if isdir(path): + as_names.add(applicationservice_name) + paths.append(path) + + # load extra dictionaries + path = join(self.source_root_path, + source_name, + release_name, + 'applicationservice', + applicationservice_name, + 'extras') + if isdir(path): + for namespace in listdir(path): + extra_dir = join(path, namespace) + if not isdir(extra_dir): + continue + as_names.add(applicationservice_name) + extras.append((namespace, [extra_dir])) + eolobj = CreoleObjSpace(dtdfilename) + as_names_str = '", "'.join(as_names) + await log.info(risotto_context, + _(f'gen schema for "{servermodel_name}" with application services "{as_names_str}"')) + eolobj.create_or_populate_from_xml('creole', paths) + for extra in extras: + eolobj.create_or_populate_from_xml(extra[0], extra[1]) + # FIXME extra + funcs_file = self.get_servermodel_cache(servermodel_id, 'funcs.py') + eolobj.space_visitor(funcs_file) + dest_dir = self.get_servermodel_cache(servermodel_id, 'dictionaries.xml') + eolobj.save(dest_dir) + + def get_servermodel_cache(self, + servermodel_id: int, + subdir: Optional[str]=None) -> str: + if subdir: + return join(self.cache_root_path, str(servermodel_id), subdir) + return join(self.cache_root_path, str(servermodel_id)) + + async def servermodel_copy_templates(self, + servermodel_name: str, + servermodel_id: int, + dependencies: Dict, + generate_cache: Dict, + risotto_context: Context) -> None: + as_names = [] + dest_dir = self.get_servermodel_cache(servermodel_id, 'templates') + if isdir(dest_dir): + rmtree(dest_dir) + makedirs(dest_dir) + for dependency in dependencies: + applicationservice_name, release_id = generate_cache['applicationservice'][dependency] + source_name, release_name = generate_cache['release_id'][release_id] + path = join(self.source_root_path, + source_name, + release_name, + 'applicationservice', + applicationservice_name, + 'templates') + if isdir(path): + for template in listdir(path): + template_path = join(dest_dir, template) + if isfile(template_path): + as_names_str = '", "'.join(as_names) + raise Exception(_(f'duplicate "{template}" when copying template from "{applicationservice_name}" to "{dest_dir}" for servermodel "{servermodel_name}" (previous application services was "{as_names_str}"')) + copyfile(join(path, template), template_path) + as_names.append(applicationservice_name) + as_names_str = '", "'.join(as_names) + await log.info(risotto_context, + _(f'copy templates for "{servermodel_name}" with application services "{as_names_str}"')) diff --git a/src/risotto/services/servermodel/servermodel.py b/src/risotto/services/servermodel/servermodel.py index 12a9562..412a556 100644 --- a/src/risotto/services/servermodel/servermodel.py +++ b/src/risotto/services/servermodel/servermodel.py @@ -1,12 +1,10 @@ -from shutil import rmtree, copyfile +from shutil import rmtree from os import listdir, makedirs -from os.path import join, isdir, isfile +from os.path import join, isdir from yaml import load, SafeLoader from traceback import print_exc from typing import Dict, List, Optional -from rougail import CreoleObjSpace -from rougail.config import dtdfilename -from ...controller import Controller +from .generator import Generator from ...register import register from ...utils import _ from ...context import Context @@ -15,147 +13,25 @@ from ...error import ExecutionError from ...logger import log -class Risotto(Controller): +class Risotto(Generator): def __init__(self, test: bool) -> None: self.source_root_path = get_config()['source']['root_path'] self.cache_root_path = join(get_config()['cache']['root_path'], 'servermodel') self.internal_source_name = get_config()['servermodel']['internal_source'] self.internal_distribution_name = get_config()['servermodel']['internal_distribution'] + self.internal_release_name = get_config()['servermodel']['internal_release_name'] if not isdir(self.cache_root_path): makedirs(join(self.cache_root_path)) async def on_join(self, risotto_context: Context) -> None: - internal_source = await self.call('v1.source.create', - risotto_context, - source_name=self.internal_source_name, - source_url='none') - internal_release = await self.call('v1.source.release.create', + internal_release = await self.call('v1.source.release.describe', risotto_context, source_name=self.internal_source_name, - release_name='none', release_distribution=self.internal_distribution_name) self.internal_release_id = internal_release['release_id'] - async def servermodel_gen_funcs(self, - servermodel_name: str, - servermodel_id: int, - dependencies: Dict, - release_cache: Dict, - risotto_context: Context) -> None: - as_names = [] - dest_file = self.get_servermodel_cache(servermodel_id, 'funcs.py') - with open(dest_file, 'wb') as funcs: - funcs.write(b'from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value\n\n') - for applicationservice_id, applicationservice_infos in dependencies.items(): - applicationservice_name, as_release_id = applicationservice_infos - path = join(self.source_root_path, - release_cache[as_release_id]['source_name'], - release_cache[as_release_id]['release_name'], - 'applicationservice', - applicationservice_name, - 'funcs') - if isdir(path): - as_names.append(applicationservice_name) - for fil in listdir(path): - if not fil.endswith('.py'): - continue - fil_path = join(path, fil) - with open(fil_path, 'rb') as fh: - funcs.write(f'# {fil_path}\n'.encode()) - funcs.write(fh.read()) - funcs.write(b'\n') - - as_names_str = '", "'.join(as_names) - await log.info(risotto_context, - _(f'gen funcs for "{servermodel_name}" with application services "{as_names_str}"')) - - async def servermodel_gen_schema(self, - servermodel_name: str, - servermodel_id: int, - dependencies: Dict, - release_cache: Dict, - risotto_context: Context) -> None: - paths = [] - extras = [] - as_names = set() - for applicationservice_id, applicationservice_infos in dependencies.items(): - applicationservice_name, as_release_id = applicationservice_infos - # load creole dictionaries - path = join(self.source_root_path, - release_cache[as_release_id]['source_name'], - release_cache[as_release_id]['release_name'], - 'applicationservice', - applicationservice_name, - 'dictionaries') - if isdir(path): - as_names.add(applicationservice_name) - paths.append(path) - - # load extra dictionaries - path = join(self.source_root_path, - release_cache[as_release_id]['source_name'], - release_cache[as_release_id]['release_name'], - 'applicationservice', - applicationservice_name, - 'extras') - if isdir(path): - for namespace in listdir(path): - extra_dir = join(path, namespace) - if not isdir(extra_dir): - continue - as_names.add(applicationservice_name) - extras.append((namespace, [extra_dir])) - eolobj = CreoleObjSpace(dtdfilename) - as_names_str = '", "'.join(as_names) - await log.info(risotto_context, - _(f'gen schema for "{servermodel_name}" with application services "{as_names_str}"')) - eolobj.create_or_populate_from_xml('creole', paths) - for extra in extras: - eolobj.create_or_populate_from_xml(extra[0], extra[1]) - # FIXME extra - funcs_file = self.get_servermodel_cache(servermodel_id, 'funcs.py') - eolobj.space_visitor(funcs_file) - dest_dir = self.get_servermodel_cache(servermodel_id, 'dictionaries.xml') - eolobj.save(dest_dir) - - def get_servermodel_cache(self, - servermodel_id: int, - subdir: Optional[str]=None) -> str: - if subdir: - return join(self.cache_root_path, str(servermodel_id), subdir) - return join(self.cache_root_path, str(servermodel_id)) - - async def servermodel_copy_templates(self, - servermodel_name: str, - servermodel_id: int, - dependencies: Dict, - release_cache: Dict, - risotto_context: Context) -> None: - as_names = [] - dest_dir = self.get_servermodel_cache(servermodel_id, 'templates') - makedirs(dest_dir) - for applicationservice_id, applicationservice_infos in dependencies.items(): - applicationservice_name, as_release_id = applicationservice_infos - path = join(self.source_root_path, - release_cache[as_release_id]['source_name'], - release_cache[as_release_id]['release_name'], - 'applicationservice', - applicationservice_name, - 'templates') - if isdir(path): - for template in listdir(path): - template_path = join(dest_dir, template) - if isfile(template_path): - as_names_str = '", "'.join(as_names) - raise Exception(_(f'duplicate "{template}" when copying template from "{applicationservice_name}" to "{dest_dir}" for servermodel "{servermodel_name}" (previous application services was "{as_names_str}"')) - copyfile(join(path, template), template_path) - as_names.append(applicationservice_name) - as_names_str = '", "'.join(as_names) - await log.info(risotto_context, - _(f'copy templates for "{servermodel_name}" with application services "{as_names_str}"')) - async def _servermodel_create(self, risotto_context: Context, servermodel_name: str, @@ -163,7 +39,10 @@ class Risotto(Controller): servermodel_parents: List[Dict], dependencies: List[int], release_id: int, - release_cache: Dict=None) -> Dict: + generate_cache: Dict=None) -> Dict: + if generate_cache is None: + generate_cache = {'applicationservice': {}, + 'release_id': {}} servermodel_insert = """INSERT INTO Servermodel(ServermodelName, ServermodelDescription, ServermodelParentsId, ServermodelReleaseId, ServermodelApplicationServiceId) VALUES ($1,$2,$3,$4,$5) RETURNING ServermodelId @@ -180,6 +59,12 @@ class Risotto(Controller): applicationservice_description=as_description, applicationservice_dependencies=dependencies) applicationservice_id = applicationservice['applicationservice_id'] + generate_cache['applicationservice'][applicationservice_id] = (as_name, + self.internal_release_id) + if self.internal_release_id not in generate_cache['release_id']: + generate_cache['release_id'][self.internal_release_id] = (self.internal_source_name, + self.internal_release_name) + servermodel_id = await risotto_context.connection.fetchval(servermodel_insert, servermodel_name, servermodel_description, @@ -190,34 +75,18 @@ class Risotto(Controller): if isdir(dest_dir): rmtree(dest_dir) makedirs(dest_dir) - # get all dependencies for this application service - dependencies = await self.get_applicationservices(risotto_context, - applicationservice_id) - # build cache to have all release informations - if release_cache is None: - release_cache = {} - for applicationservice_infos in dependencies.values(): - applicationservice_name, as_release_id = applicationservice_infos - if as_release_id not in release_cache: - release_cache[as_release_id] = await self.call('v1.source.release.get_by_id', - risotto_context, - release_id=as_release_id) - - await self.servermodel_gen_funcs(servermodel_name, - servermodel_id, - dependencies, - release_cache, - risotto_context) - await self.servermodel_gen_schema(servermodel_name, - servermodel_id, - dependencies, - release_cache, - risotto_context) - await self.servermodel_copy_templates(servermodel_name, - servermodel_id, - dependencies, - release_cache, - risotto_context) + dependencies = applicationservice['applicationservice_dependencies'] + # for as_release_id in dependencies.values(): + # applicationservice_name, as_release_id = applicationservice_infos + # if as_release_id not in release_cache: + # release_cache[as_release_id] = await self.call('v1.source.release.get_by_id', + # risotto_context, + # release_id=as_release_id) + await self.generate(risotto_context, + servermodel_name, + servermodel_id, + dependencies, + generate_cache) sm_dict = {'servermodel_name': servermodel_name, 'servermodel_description': servermodel_description, 'servermodel_parents_id': servermodel_parents_id, @@ -238,16 +107,48 @@ class Risotto(Controller): self.parse_parents(servermodels, servermodels[parent], parents) return parents + @register('v1.applicationservice.updated') + async def applicationservice_updated(self, + risotto_context: Context, + applicationservice_id): + # FIXME applicationservices qui depend de ce services => updated + sql = ''' + SELECT ServermodelId as servermodel_id, ServermodelName as servermodel_name, ServermodelDescription as servermodel_description, ServermodelParentsId as servermodel_parents_id, ServermodelReleaseId as release_id, ServermodelApplicationServiceId as servermodel_applicationservice_id + FROM Servermodel + WHERE ServermodelApplicationServiceId = $1 + ''' + servermodel = await risotto_context.connection.fetchrow(sql, + applicationservice_id) + if servermodel is not None: + servermodel_name = servermodel['servermodel_name'] + servermodel_id = servermodel['servermodel_id'] + release_id = servermodel['release_id'] + applicationservice = await self.call('v1.applicationservice.get_by_id', + risotto_context, + applicationservice_id=applicationservice_id) + dependencies = applicationservice['applicationservice_dependencies'] + await self.generate(risotto_context, + servermodel_name, + servermodel_id, + dependencies, + None) + await self.publish('v1.servermodel.updated', + risotto_context, + **servermodel) + @register('v1.servermodel.dataset.updated') - async def servermodel_update(self, - risotto_context: Context, - source_name: str, - release_distribution: int): + async def servermodel_dataset_updated(self, + risotto_context: Context, + source_name: str, + release_distribution: int): release = await self.call('v1.source.release.describe', risotto_context, source_name=source_name, release_distribution=release_distribution) release_id = release['release_id'] + generate_cache = {'applicationservice': {}, + 'release_id': {release['release_id']: (release['source_name'], + release['release_name'])}} servermodel_path = join(self.source_root_path, source_name, release['release_name'], @@ -268,7 +169,6 @@ class Risotto(Controller): servermodels[servermodel_description['name']] = servermodel_description servermodels[servermodel_description['name']]['done'] = False - release_cache = {release['release_id']: release} for servermodel in servermodels.values(): if not servermodel['done']: # parent needs to create before child, so retrieve all parents @@ -302,7 +202,7 @@ class Risotto(Controller): servermodel_parent, dependencies, release_id, - release_cache) + generate_cache) await self.publish('v1.servermodel.created', risotto_context, **servermodel_ob) @@ -379,72 +279,4 @@ class Risotto(Controller): servermodel_parents, [], self.internal_release_id, - {}) - - @register('v1.servermodel.get_by_id') - async def servermodel_get_by_id(self, - risotto_context: Context, - servermodel_id: int) -> Dict: - sql = ''' - SELECT ServermodelId as servermodel_id, ServermodelName as servermodel_name, ServermodelDescription as servermodel_description, ServermodelParentsId as servermodel_parents_id, ServermodelReleaseId as release_id, ServermodelApplicationServiceId as servermodel_applicationservice_id - FROM Servermodel - WHERE ServermodelId=$1 - ''' - servermodel = await risotto_context.connection.fetchrow(sql, - servermodel_id) - if not servermodel: - raise Exception(_(f'{servermodel_id} is not a valid ID for a servermodel')) - return dict(servermodel) - - async def _parse_depends(self, - risotto_context: Context, - applicationservice_id: int, - or_depends: list, - ids: list) -> None: - applicationservice = await self.call('v1.applicationservice.get_by_id', - risotto_context, - applicationservice_id=applicationservice_id) - ids[applicationservice_id] = (applicationservice['applicationservice_name'], - applicationservice['applicationservice_release_id']) - for depend in applicationservice['applicationservice_dependencies']: - if isinstance(depend, dict): - or_depends.append(depend['or']) - elif depend not in ids: - await self._parse_depends(risotto_context, - depend, - or_depends, - ids) - - async def _parse_or_depends(self, - risotto_context: Context, - or_depends: list, - ids: list) -> None: - new_or_depends = [] - set_ids = set(ids) - for or_depend in or_depends: - if not set(or_depend) & set_ids: - applicationservice_id= or_depend[0] - await self._parse_depends(risotto_context, - applicationservice_id, - new_or_depends, - ids) - if new_or_depends: - await self._parse_or_depends(risotto_context, - new_or_depends, - ids) - - async def get_applicationservices(self, - risotto_context: Context, - applicationservice_id: int) -> list: - """Return consolidated dependencies or raise. - """ - or_depends = [] - ids = {} - await self._parse_depends(risotto_context, - applicationservice_id, - or_depends, - ids) - await self._parse_or_depends(risotto_context, - or_depends, - ids) - return ids + None) diff --git a/src/risotto/services/uri/uri.py b/src/risotto/services/uri/uri.py index da7ae93..c0fcc40 100644 --- a/src/risotto/services/uri/uri.py +++ b/src/risotto/services/uri/uri.py @@ -11,6 +11,7 @@ class Risotto(Controller): risotto_context): for uri in ['v1.applicationservice.create', 'v1.applicationservice.dataset.updated', + 'v1.applicationservice.dependency.add', 'v1.server.create', 'v1.servermodel.create', 'v1.servermodel.dataset.updated', @@ -34,7 +35,6 @@ class Risotto(Controller): except: pass for uri in ['v1.applicationservice.describe', - 'v1.applicationservice.get_by_id', 'v1.server.describe', 'v1.server.list', 'v1.servermodel.list', @@ -61,6 +61,7 @@ class Risotto(Controller): except: pass for uri in ['v1.server.describe', + 'v1.applicationservice.dependency.add', 'v1.config.configuration.server.get', 'v1.config.configuration.server.deploy', 'v1.session.server.start',