From 3b3cefa38a448d3b56155d961d65017cba002313 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Fri, 31 Jan 2020 11:23:18 +0100 Subject: [PATCH] add v1.servermodel.create message --- README.md | 5 + .../v1/messages/session.servermodel.start.yml | 10 -- messages/v1/types/servermodel.yml | 5 + src/risotto/config.py | 4 +- src/risotto/dispatcher.py | 117 +++++++++++------- .../services/servermodel/servermodel.py | 100 +++++++++------ src/risotto/services/session/session.py | 11 +- src/risotto/services/template/template.py | 56 +++++---- src/risotto/services/uri/uri.py | 1 + 9 files changed, 187 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index b7ddc5b..fbb2007 100644 --- a/README.md +++ b/README.md @@ -73,3 +73,8 @@ S=xxxxxxxxxxxxxxxxxxxxxx # Create a new user and set role 'server_rw' for this server ./script/cucchiaiata user.create -l gnunux -n gnunux -s gnunux ./script/cucchiaiata user.role.create -u gnunux -n 'server_rw' -a 'Server.ServerName' -v test + +# Heritage +./script/cucchiaiata servermodel.create -n aca -d Aca -p eolebase -s eole -r last +./script/cucchiaiata session.servermodel.start -s aca +./script/cucchiaiata session.servermodel.configure -s $S --creole.reseau.unbound_domain_name test.cadoles.com diff --git a/messages/v1/messages/session.servermodel.start.yml b/messages/v1/messages/session.servermodel.start.yml index 8335017..e6c91a5 100644 --- a/messages/v1/messages/session.servermodel.start.yml +++ b/messages/v1/messages/session.servermodel.start.yml @@ -11,16 +11,6 @@ parameters: ref: Servermodel.ServermodelName shortarg: s description: Nom du serveurmodel. - source_name: - type: String - shortarg: n - description: Nom de la source. - ref: Source.SourceName - release_distribution: - type: String - shortarg: r - description: Nom de la distribution. - ref: Source.ReleaseDistribution response: type: Session diff --git a/messages/v1/types/servermodel.yml b/messages/v1/types/servermodel.yml index 83512e7..f86de29 100644 --- a/messages/v1/types/servermodel.yml +++ b/messages/v1/types/servermodel.yml @@ -23,8 +23,13 @@ properties: type: number ref: Servermodel.SubreleaseId description: Version du modèle de serveur. + servermodel_applicationservice_id: + type: number + ref: Applicationservice.ApplicationserviceId + description: Identifiant de l'application service local. required: - servermodel_id - servermodel_name - servermodel_description - release_id + - servermodel_applicationservice_id diff --git a/src/risotto/config.py b/src/risotto/config.py index 90e7712..086f5c1 100644 --- a/src/risotto/config.py +++ b/src/risotto/config.py @@ -26,6 +26,8 @@ def get_config(): 'rougail_dtd_path': '../rougail/data/creole.dtd', 'admin_user': DEFAULT_USER}, 'source': {'root_path': '/srv/seed'}, - 'cache': {'root_path': '/var/cache/risotto'} + 'cache': {'root_path': '/var/cache/risotto'}, + 'servermodel': {'internal_source': 'internal', + 'internal_distribution': 'last'}, } diff --git a/src/risotto/dispatcher.py b/src/risotto/dispatcher.py index 0c86897..b2e2e08 100644 --- a/src/risotto/dispatcher.py +++ b/src/risotto/dispatcher.py @@ -87,21 +87,39 @@ class CallDispatcher: kwargs, function_objs) else: - async with self.pool.acquire() as connection: - await connection.set_type_codec( - 'json', - encoder=dumps, - decoder=loads, - schema='pg_catalog' - ) - risotto_context.connection = connection - async with connection.transaction(): - return await self.launch(version, - message, - risotto_context, - check_role, - kwargs, - function_objs) + try: + async with self.pool.acquire() as connection: + await connection.set_type_codec( + 'json', + encoder=dumps, + decoder=loads, + schema='pg_catalog' + ) + risotto_context.connection = connection + async with connection.transaction(): + return await self.launch(version, + message, + risotto_context, + check_role, + kwargs, + function_objs) + except CallError as err: + raise err + except Exception as err: + # if there is a problem with arguments, just send an error and do nothing + if get_config()['global']['debug']: + print_exc() + async with self.pool.acquire() as connection: + await connection.set_type_codec( + 'json', + encoder=dumps, + decoder=loads, + schema='pg_catalog' + ) + risotto_context.connection = connection + async with connection.transaction(): + await log.error_msg(risotto_context, kwargs, err) + raise err class PublishDispatcher: @@ -126,21 +144,39 @@ class PublishDispatcher: kwargs, function_objs) else: - async with self.pool.acquire() as connection: - await connection.set_type_codec( - 'json', - encoder=dumps, - decoder=loads, - schema='pg_catalog' - ) - risotto_context.connection = connection - async with connection.transaction(): - return await self.launch(version, - message, - risotto_context, - check_role, - kwargs, - function_objs) + try: + async with self.pool.acquire() as connection: + await connection.set_type_codec( + 'json', + encoder=dumps, + decoder=loads, + schema='pg_catalog' + ) + risotto_context.connection = connection + async with connection.transaction(): + return await self.launch(version, + message, + risotto_context, + check_role, + kwargs, + function_objs) + except CallError as err: + raise err + except Exception as err: + # if there is a problem with arguments, just send an error and do nothing + if get_config()['global']['debug']: + print_exc() + async with self.pool.acquire() as connection: + await connection.set_type_codec( + 'json', + encoder=dumps, + decoder=loads, + schema='pg_catalog' + ) + risotto_context.connection = connection + async with connection.transaction(): + await log.error_msg(risotto_context, kwargs, err) + raise err class Dispatcher(register.RegisterDispatcher, CallDispatcher, PublishDispatcher): @@ -267,19 +303,10 @@ class Dispatcher(register.RegisterDispatcher, CallDispatcher, PublishDispatcher) function_objs: List) -> Optional[Dict]: await self.check_message_type(risotto_context, kwargs) - try: - config_arguments = await self.load_kwargs_to_config(risotto_context, - f'{version}.{message}', - kwargs, - check_role) - except Exception as err: - # if there is a problem with arguments, just send an error and do nothing - if get_config()['global']['debug']: - print_exc() - await log.error_msg(risotto_context, kwargs, err) - if risotto_context.type == 'rpc': - raise err - return + config_arguments = await self.load_kwargs_to_config(risotto_context, + f'{version}.{message}', + kwargs, + check_role) # config is ok, so send the message for function_obj in function_objs: function = function_obj['function'] @@ -305,13 +332,13 @@ class Dispatcher(register.RegisterDispatcher, CallDispatcher, PublishDispatcher) raise err continue except Exception as err: + if risotto_context.type == 'rpc': + raise err if get_config().get('global').get('debug'): print_exc() await log.error_msg(risotto_context, kwargs, err) - if risotto_context.type == 'rpc': - raise CallError(str(err)) continue else: if risotto_context.type == 'rpc': diff --git a/src/risotto/services/servermodel/servermodel.py b/src/risotto/services/servermodel/servermodel.py index 1a0f6b6..319d836 100644 --- a/src/risotto/services/servermodel/servermodel.py +++ b/src/risotto/services/servermodel/servermodel.py @@ -17,8 +17,10 @@ from ...logger import log class Risotto(Controller): def __init__(self, test: bool) -> None: - self.source_root_path = get_config().get('source').get('root_path') - self.cache_root_path = join(get_config().get('cache').get('root_path'), 'servermodel') + self.source_root_path = get_config()['source']['root_path'] + self.cache_root_path = join(get_config()['cache']['root_path'], 'servermodel') + self.internal_source_name = get_config()['servermodel']['internal_source'] + self.internal_distribution_name = get_config()['servermodel']['internal_distribution'] if not isdir(self.cache_root_path): makedirs(join(self.cache_root_path)) @@ -26,13 +28,13 @@ class Risotto(Controller): risotto_context: Context) -> None: internal_source = await self.call('v1.source.create', risotto_context, - source_name='internal', + source_name=self.internal_source_name, source_url='none') internal_release = await self.call('v1.source.release.create', risotto_context, - source_name='internal', + source_name=self.internal_source_name, release_name='none', - release_distribution='last') + release_distribution=self.internal_distribution_name) self.internal_release_id = internal_release['release_id'] async def servermodel_gen_funcs(self, @@ -158,16 +160,20 @@ class Risotto(Controller): risotto_context: Context, servermodel_name: str, servermodel_description: str, - servermodel_parents_id: List[int], + servermodel_parents: List[Dict], dependencies: List[int], release_id: int, release_cache: Dict=None) -> Dict: - servermodel_update = """INSERT INTO Servermodel(ServermodelName, ServermodelDescription, ServermodelParentsId, ServermodelReleaseId, ServermodelApplicationServiceId) + servermodel_insert = """INSERT INTO Servermodel(ServermodelName, ServermodelDescription, ServermodelParentsId, ServermodelReleaseId, ServermodelApplicationServiceId) VALUES ($1,$2,$3,$4,$5) RETURNING ServermodelId """ as_name = f"local_{servermodel_name}" as_description = f'local application service for {servermodel_name}' + servermodel_parents_id = [] + for servermodel_parent in servermodel_parents: + servermodel_parents_id.append(servermodel_parent['servermodel_id']) + dependencies.append(servermodel_parent['servermodel_applicationservice_id']) applicationservice = await self.call('v1.applicationservice.create', risotto_context, applicationservice_name=as_name, @@ -175,7 +181,7 @@ class Risotto(Controller): applicationservice_dependencies=dependencies, release_id=self.internal_release_id) applicationservice_id = applicationservice['applicationservice_id'] - servermodel_id = await risotto_context.connection.fetchval(servermodel_update, + servermodel_id = await risotto_context.connection.fetchval(servermodel_insert, servermodel_name, servermodel_description, servermodel_parents_id, @@ -191,7 +197,7 @@ class Risotto(Controller): # build cache to have all release informations if release_cache is None: release_cache = {} - for applicationservice_id, applicationservice_infos in dependencies.items(): + for applicationservice_infos in dependencies.values(): applicationservice_name, as_release_id = applicationservice_infos if as_release_id not in release_cache: release_cache[as_release_id] = await self.call('v1.source.release.get_by_id', @@ -216,11 +222,9 @@ class Risotto(Controller): sm_dict = {'servermodel_name': servermodel_name, 'servermodel_description': servermodel_description, 'servermodel_parents_id': servermodel_parents_id, + 'servermodel_applicationservice_id': applicationservice_id, 'release_id': release_id, 'servermodel_id': servermodel_id} - await self.publish('v1.servermodel.created', - risotto_context, - **sm_dict) return sm_dict def parse_parents(self, @@ -235,15 +239,6 @@ class Risotto(Controller): self.parse_parents(servermodels, servermodels[parent], parents) return parents - async def get_servermodel_id_by_name(self, - risotto_context: Context, - servermodel_name: str, - release_id: int): - sql = 'SELECT ServermodelId as servermodel_id FROM Servermodel WHERE ServermodelName = $1 AND ServermodelReleaseId = $2', - return await risotto_context.connection.fetchval(sql, - servermodel_name, - release_id)['servermodel_id'] - @register('v1.servermodel.dataset.updated') async def servermodel_update(self, risotto_context: Context, @@ -284,16 +279,15 @@ class Risotto(Controller): parents = self.parse_parents(servermodels, servermodel) parents.reverse() - servermodelparent_id = [] + servermodel_parent = [] for new_servermodel in parents: if not servermodels[new_servermodel]['done']: servermodel_description = servermodels[new_servermodel] parent = servermodel_description['parent'] - if not servermodelparent_id and parent is not None: - # parent is a str, so get ID - servermodelparent_id = [await self.get_servermodel_id_by_name(risotto_context, - parent, - release_id)] + if not servermodel_parent and parent is not None: + servermodel_parent = [await self._servermodel_describe(risotto_context, + parent, + release_id)] # link application service with this servermodel dependencies = [] for depend in servermodels[new_servermodel]['applicationservices']: @@ -308,16 +302,18 @@ class Risotto(Controller): servermodel_ob = await self._servermodel_create(risotto_context, sm_name, sm_description, - servermodelparent_id, + servermodel_parent, dependencies, release_id, release_cache) - servermodel_id = servermodel_ob['servermodel_id'] + await self.publish('v1.servermodel.created', + risotto_context, + **servermodel_ob) except Exception as err: if get_config().get('global').get('debug'): print_exc() raise ExecutionError(_(f"Error while injecting servermodel {sm_name} in database: {err}")) - servermodelparent_id = [servermodel_id] + servermodel_parent = [servermodel_ob] servermodel_description['done'] = True return {'retcode': 0, 'returns': _('Servermodels successfully loaded')} @@ -326,7 +322,7 @@ class Risotto(Controller): risotto_context: Context, source_id: int): sql = ''' - SELECT ServermodelId as servermodel_id, ServermodelName as servermodel_name, ServermodelDescription as servermodel_description, ServermodelParentsId as servermodel_parents_id, ServermodelReleaseId as release_id + SELECT ServermodelId as servermodel_id, ServermodelName as servermodel_name, ServermodelDescription as servermodel_description, ServermodelParentsId as servermodel_parents_id, ServermodelReleaseId as release_id, ServermodelApplicationServiceId as servermodel_applicationservice_id FROM Servermodel ''' servermodels = await risotto_context.connection.fetch(sql) @@ -342,24 +338,58 @@ class Risotto(Controller): risotto_context, source_name=source_name, release_distribution=release_distribution) + return await self._servermodel_describe(risotto_context, + servermodel_name, + release['release_id']) + + async def _servermodel_describe(self, + risotto_context, + servermodel_name, + release_id): sql = ''' - SELECT ServermodelId as servermodel_id, ServermodelName as servermodel_name, ServermodelDescription as servermodel_description, ServermodelParentsId as servermodel_parents_id, ServermodelReleaseId as release_id + SELECT ServermodelId as servermodel_id, ServermodelName as servermodel_name, ServermodelDescription as servermodel_description, ServermodelParentsId as servermodel_parents_id, ServermodelReleaseId as release_id, ServermodelApplicationServiceId as servermodel_applicationservice_id FROM Servermodel WHERE ServermodelName=$1 AND ServermodelReleaseId=$2 ''' servermodel = await risotto_context.connection.fetchrow(sql, servermodel_name, - release['release_id']) + release_id) if not servermodel: - raise Exception(_(f'{servermodel_id} is not a valid ID for a servermodel')) + raise Exception(_(f'"{servermodel_name}" is not a valid name for a servermodel in source "{source_name}" and release "{release_distribution}"')) return dict(servermodel) + @register('v1.servermodel.create', notification='v1.servermodel.created') + async def create_servermodel(self, + risotto_context: Context, + servermodel_name: str, + servermodel_description: str, + servermodel_parents_name: List[int], + servermodel_parents_source_name: str, + servermodel_parents_release_distribution: str) -> Dict: + release = await self.call('v1.source.release.describe', + risotto_context, + source_name=servermodel_parents_source_name, + release_distribution=servermodel_parents_release_distribution) + release_id = release['release_id'] + servermodel_parents = [] + for servermodel_parent_name in servermodel_parents_name: + servermodel_parents.append(await self._servermodel_describe(risotto_context, + servermodel_parent_name, + release_id)) + return await self._servermodel_create(risotto_context, + servermodel_name, + servermodel_description, + servermodel_parents, + [], + self.internal_release_id, + {}) + @register('v1.servermodel.get_by_id') async def servermodel_get_by_id(self, risotto_context: Context, servermodel_id: int) -> Dict: sql = ''' - SELECT ServermodelId as servermodel_id, ServermodelName as servermodel_name, ServermodelDescription as servermodel_description, ServermodelParentsId as servermodel_parents_id, ServermodelReleaseId as release_id + SELECT ServermodelId as servermodel_id, ServermodelName as servermodel_name, ServermodelDescription as servermodel_description, ServermodelParentsId as servermodel_parents_id, ServermodelReleaseId as release_id, ServermodelApplicationServiceId as servermodel_applicationservice_id FROM Servermodel WHERE ServermodelId=$1 ''' diff --git a/src/risotto/services/session/session.py b/src/risotto/services/session/session.py index 461fd6d..2b27efe 100644 --- a/src/risotto/services/session/session.py +++ b/src/risotto/services/session/session.py @@ -12,12 +12,15 @@ from .storage import storage_server, storage_servermodel from ...controller import Controller from ...register import register from ...dispatcher import dispatcher +from ...config import get_config class Risotto(Controller): def __init__(self, test): self.modify_storage = Storage(engine='dictionary') + self.internal_source_name = get_config()['servermodel']['internal_source'] + self.internal_distribution_name = get_config()['servermodel']['internal_distribution'] def get_storage(self, type: str): @@ -110,17 +113,15 @@ class Risotto(Controller): @register('v1.session.servermodel.start') async def start_session_servermodel(self, risotto_context: Context, - servermodel_name: str, - source_name: str, - release_distribution: str) -> Dict: + servermodel_name: str) -> Dict: """ start a new config session for a server or a servermodel """ config_module = dispatcher.get_service('config') servermodel = await self.call('v1.servermodel.describe', risotto_context, servermodel_name=servermodel_name, - source_name=source_name, - release_distribution=release_distribution) + source_name=self.internal_source_name, + release_distribution=self.internal_distribution_name) if not servermodel or servermodel['servermodel_id'] not in config_module.servermodel: raise Exception(_(f'cannot find servermodel with name {servermodel_name}')) id = servermodel['servermodel_id'] diff --git a/src/risotto/services/template/template.py b/src/risotto/services/template/template.py index 0502b62..7fcb931 100644 --- a/src/risotto/services/template/template.py +++ b/src/risotto/services/template/template.py @@ -21,41 +21,45 @@ class Risotto(Controller): async def template_get(self, risotto_context, server_name: str) -> Dict: + # get informations for server server = await self.call('v1.server.describe', risotto_context, server_name=server_name) server_id = server['server_id'] servermodel_id = server['server_servermodel_id'] + # verify if server has deployed configuration config_module = dispatcher.get_service('config') server = config_module.server[server_id] export = await server['server'].value.exportation() if not export[0]: raise Exception(_(f'configuration for server "{server_name}" is empty, you should deploy it first')) - config = meta = await server['server'].config.deepcopy(storage=self.storage) - while True: - try: - children = list(await config.config.list()) - except: - break - if children: - config = children[0] - else: - break - configurations_dir = join(CONFIGURATION_DIR, - str(server_id)) - if isdir(configurations_dir): - rmtree(configurations_dir) - mkdir(configurations_dir) - tmp_dir = join(TMP_DIR, str(server_id)) - if isdir(tmp_dir): - rmtree(tmp_dir) - mkdir(tmp_dir) - templates_dir = join(self.cache_root_path, str(servermodel_id), 'templates') - await generate(config, - server['funcs_file'], - templates_dir, - tmp_dir, - configurations_dir) - + # copy deployed configuration + async with await server['server'].config.deepcopy(storage=self.storage) as config: + meta = config + while True: + try: + children = list(await config.config.list()) + except: + break + if children: + config = children[0] + else: + break + configurations_dir = join(CONFIGURATION_DIR, + str(server_id)) + if isdir(configurations_dir): + rmtree(configurations_dir) + mkdir(configurations_dir) + tmp_dir = join(TMP_DIR, str(server_id)) + if isdir(tmp_dir): + rmtree(tmp_dir) + mkdir(tmp_dir) + templates_dir = join(self.cache_root_path, str(servermodel_id), 'templates') + await generate(config, + server['funcs_file'], + templates_dir, + tmp_dir, + configurations_dir) + del meta return {'server_name': server_name, 'template_dir': configurations_dir} diff --git a/src/risotto/services/uri/uri.py b/src/risotto/services/uri/uri.py index 2904d34..d5fcfe8 100644 --- a/src/risotto/services/uri/uri.py +++ b/src/risotto/services/uri/uri.py @@ -12,6 +12,7 @@ class Risotto(Controller): for uri in ['v1.applicationservice.create', 'v1.applicationservice.dataset.updated', 'v1.server.create', + 'v1.servermodel.create', 'v1.servermodel.dataset.updated', 'v1.session.server.start', 'v1.source.create',