From d1cf1df4b18acbefe4b2a374bb249021a4b72b52 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Sat, 7 Mar 2020 18:35:23 +0100 Subject: [PATCH] better version support --- src/risotto/config.py | 3 +- src/risotto/http.py | 27 +++- src/risotto/message/message.py | 242 +++++++++++++++++++-------------- src/risotto/register.py | 40 ++---- src/risotto/remote.py | 12 +- 5 files changed, 181 insertions(+), 143 deletions(-) diff --git a/src/risotto/config.py b/src/risotto/config.py index f7375a90..df23d716 100644 --- a/src/risotto/config.py +++ b/src/risotto/config.py @@ -23,8 +23,7 @@ def get_config(): 'check_role': True, 'admin_user': DEFAULT_USER, 'module_name': MODULE_NAME, - 'sql_filename': SQL_FILENAME, - 'version': 'v1'}, + 'sql_filename': SQL_FILENAME}, 'source': {'root_path': '/srv/seed'}, 'cache': {'root_path': '/var/cache/risotto'}, 'servermodel': {'internal_source': 'internal', diff --git a/src/risotto/http.py b/src/risotto/http.py index aba434cc..58ead740 100644 --- a/src/risotto/http.py +++ b/src/risotto/http.py @@ -14,6 +14,9 @@ from .config import get_config from .services import load_services +extra_routes = {} + + def create_context(request): risotto_context = Context() risotto_context.username = request.match_info.get('username', @@ -89,7 +92,8 @@ async def handle(request): content_type='application/json')) -async def api(request, risotto_context): +async def api(request, + risotto_context): global tiramisu if not tiramisu: # check all URI that have an associated role @@ -105,16 +109,13 @@ async def api(request, risotto_context): ''' uris = [uri['uriname'] for uri in await connection.fetch(sql)] async with await Config(get_messages(load_shortarg=True, + current_version=risotto_context.version, uris=uris)[1]) as config: await config.property.read_write() tiramisu = await config.option.dict(remotable='none') return tiramisu -extra_routes = {'': {'function': api, - 'version': 'v1'}} - - async def get_app(loop): """ build all routes """ @@ -124,7 +125,10 @@ async def get_app(loop): routes = [] default_storage.engine('dictionary') await dispatcher.load() + versions = [] for version, messages in dispatcher.messages.items(): + if version not in versions: + versions.append(version) print() print(_('======== Registered messages ========')) for message in messages: @@ -132,7 +136,17 @@ async def get_app(loop): pattern = dispatcher.messages[version][message]['pattern'] print(f' - {web_message} ({pattern})') routes.append(post(web_message, handle)) - print() + print() + print(_('======== Registered api routes ========')) + for version in versions: + api_route = {'function': api, + 'version': version, + 'path': f'/api/{version}'} + extra_handler = type(api_route['path'], (extra_route_handler,), api_route) + routes.append(get(api_route['path'], extra_handler)) + print(f' - {api_route["path"]} (http_get)') + print() + if extra_routes: print(_('======== Registered extra routes ========')) for path, extra in extra_routes.items(): version = extra['version'] @@ -141,7 +155,6 @@ async def get_app(loop): extra_handler = type(path, (extra_route_handler,), extra) routes.append(get(path, extra_handler)) print(f' - {path} (http_get)') - # routes.append(get(f'/api/{version}', api)) print() del extra_routes app.add_routes(routes) diff --git a/src/risotto/message/message.py b/src/risotto/message/message.py index d49600f5..25ce4192 100644 --- a/src/risotto/message/message.py +++ b/src/risotto/message/message.py @@ -52,9 +52,12 @@ class MessageDefinition: 'related', 'response') - def __init__(self, raw_def, message): + def __init__(self, + raw_def, + version, + message): # default value for non mandatory key - self.version = u'' + self.version = version self.parameters = OrderedDict() self.errors = [] self.related = [] @@ -74,11 +77,13 @@ class MessageDefinition: elif key == 'parameters': if 'type' in value and isinstance(value['type'], str): # should be a customtype - value = CUSTOMTYPES[value['type']].properties + value = CUSTOMTYPES[self.version][value['type']].properties else: - value = _parse_parameters(value) + value = _parse_parameters(value, + self.version) elif key == 'response': - value = ResponseDefinition(value) + value = ResponseDefinition(value, + self.version) elif key == 'errors': value = _parse_error_definition(value) setattr(self, key, value) @@ -99,7 +104,10 @@ class ParameterDefinition: 'ref', 'shortarg') - def __init__(self, name, raw_def): + def __init__(self, + name, + version, + raw_def): self.name = name # default value for non mandatory key self.help = None @@ -115,11 +123,11 @@ class ParameterDefinition: tvalue = value[2:] else: tvalue = value - if tvalue in CUSTOMTYPES: + if tvalue in CUSTOMTYPES[version]: if value.startswith('[]'): - value = '[]{}'.format(CUSTOMTYPES[tvalue].type) + value = '[]{}'.format(CUSTOMTYPES[version][tvalue].type) else: - value = CUSTOMTYPES[value].type + value = CUSTOMTYPES[version][value].type else: self._valid_type(value) #self._valid_type(value) @@ -150,7 +158,9 @@ class ResponseDefinition: 'required', 'multi') - def __init__(self, responses): + def __init__(self, + responses, + version): self.ref = None self.parameters = None self.multi = False @@ -164,13 +174,13 @@ class ResponseDefinition: self.multi = True else: tvalue = value - if tvalue in CUSTOMTYPES: - self.parameters = CUSTOMTYPES[tvalue].properties - self.required = CUSTOMTYPES[tvalue].required + if tvalue in CUSTOMTYPES[version]: + self.parameters = CUSTOMTYPES[version][tvalue].properties + self.required = CUSTOMTYPES[version][tvalue].required if value.startswith('[]'): - value = '[]{}'.format(CUSTOMTYPES[tvalue].type) + value = '[]{}'.format(CUSTOMTYPES[version][tvalue].type) else: - value = CUSTOMTYPES[value].type + value = CUSTOMTYPES[version][value].type else: self._valid_type(value) setattr(self, key, value) @@ -208,60 +218,66 @@ def _parse_error_definition(raw_defs): return new_value -def _parse_parameters(raw_defs): +def _parse_parameters(raw_defs, + version): parameters = OrderedDict() for name, raw_def in raw_defs.items(): - parameters[name] = ParameterDefinition(name, raw_def) + parameters[name] = ParameterDefinition(name, + version, + raw_def) return parameters -def parse_definition(filecontent: bytes, - message: str): - return MessageDefinition(load(filecontent, Loader=SafeLoader), message) - - -def is_message_defined(uri): - version, message = split_message_uri(uri) - path = get_message_file_path(version, message) - return isfile(path) - - -def get_message(uri): - load_customtypes() +def get_message(uri: str, + current_module_name: str): try: - version, message = split_message_uri(uri) - path = get_message_file_path(version, message) + version, message = uri.split('.', 1) + path = get_message_file_path(version, + message, + current_module_name) with open(path, "r") as message_file: - message_content = parse_definition(message_file.read(), message) - message_content.version = version - return message_content + return MessageDefinition(load(message_file.read(), Loader=SafeLoader), + version, + message) except Exception as err: import traceback traceback.print_exc() - raise Exception(_('cannot parse message {}: {}').format(uri, str(err))) + raise Exception(_(f'cannot parse message {uri}: {err}')) -def split_message_uri(uri): - return uri.split('.', 1) - - -def get_message_file_path(version, message): +def get_message_file_path(version, + message, + current_module_name): module_name, filename = message.split('.', 1) - if module_name != MODULE_NAME: - raise Exception(f'should only load message for {MODULE_NAME}, not {message}') - return join(MESSAGE_ROOT_PATH, version, MODULE_NAME, 'messages', filename + '.yml') + if current_module_name and module_name != current_module_name: + raise Exception(f'should only load message for {current_module_name}, not {message}') + return join(MESSAGE_ROOT_PATH, version, module_name, 'messages', filename + '.yml') -def list_messages(uris): - versions = listdir(join(MESSAGE_ROOT_PATH)) +def list_messages(uris, + current_module_name, + current_version): + def get_module_paths(): + if current_module_name is not None: + yield current_module_name, join(MESSAGE_ROOT_PATH, version, current_module_name, 'messages') + else: + for module_name in listdir(join(MESSAGE_ROOT_PATH, version)): + yield module_name, join(MESSAGE_ROOT_PATH, version, module_name, 'messages') + + if current_version: + versions = [current_version] + else: + versions = listdir(join(MESSAGE_ROOT_PATH)) versions.sort() for version in versions: - for message in listdir(join(MESSAGE_ROOT_PATH, version, MODULE_NAME, 'messages')): - if message.endswith('.yml'): - uri = version + '.' + MODULE_NAME + '.' + message.rsplit('.', 1)[0] - if uris is not None and uri not in uris: - continue - yield uri + for module_name, message_path in get_module_paths(): + for message in listdir(message_path): + if message.endswith('.yml'): + uri = version + '.' + module_name + '.' + message.rsplit('.', 1)[0] + # if uris is not None, return only is in uris' list + if uris is not None and uri not in uris: + continue + yield uri class CustomParam: @@ -374,38 +390,48 @@ class CustomType: return self.title -def load_customtypes(): - if not CUSTOMTYPES: - versions = listdir(MESSAGE_ROOT_PATH) - versions.sort() - for version in versions: - for message in listdir(join(MESSAGE_ROOT_PATH, version, MODULE_NAME, 'types')): - if message.endswith('.yml'): - path = join(MESSAGE_ROOT_PATH, version, MODULE_NAME, 'types', message) - message = message.rsplit('.', 1)[0] - with open(path, "r") as message_file: - try: - ret = CustomType(load(message_file, Loader=SafeLoader)) - CUSTOMTYPES[ret.getname()] = ret - except Exception as err: - import traceback - traceback.print_exc() - raise Exception('{} for {}'.format(err, message)) - for customtype in CUSTOMTYPES.values(): - properties = {} - for key, value in customtype.properties.items(): - type_ = value.type - if type_.startswith('[]'): - ttype_ = type_[2:] - else: - ttype_ = type_ - if ttype_ in CUSTOMTYPES: - if type_.startswith('[]'): - raise Exception(_('cannot have []CustomType')) - properties[key] = CUSTOMTYPES[ttype_] +def load_customtypes(current_module_name: str) -> None: + versions = listdir(MESSAGE_ROOT_PATH) + versions.sort() + def convert_properties(customtype: str, + version: str) -> None: + """ if properties include an other customtype, replace it + """ + properties = {} + for key, value in customtype.properties.items(): + type_ = value.type + if type_.startswith('[]'): + if type_ in CUSTOMTYPES[version]: + raise Exception(_('cannot have []CustomType')) + properties[key] = value + else: + if type_ in CUSTOMTYPES[version]: + print('====== ca existe') + properties[key] = CUSTOMTYPES[version][ttype_] else: properties[key] = value - customtype.properties = properties + customtype.properties = properties + for version in versions: + if version not in CUSTOMTYPES: + CUSTOMTYPES[version] = {} + types_path = join(MESSAGE_ROOT_PATH, + version, + current_module_name, + 'types') + for message in listdir(types_path): + if message.endswith('.yml'): + path = join(types_path, message) + # remove extension + message = message.rsplit('.', 1)[0] + with open(path, "r") as message_file: + try: + custom_type = CustomType(load(message_file, Loader=SafeLoader)) + convert_properties(custom_type, + version) + CUSTOMTYPES[version][custom_type.getname()] = custom_type + except Exception as err: + raise Exception(_(f'enable to load type {err}: {message}')) + def _get_description(description, @@ -465,8 +491,7 @@ def _get_option(name, return obj -def _parse_args(message_def, - options, +def get_options(message_def, file_path, needs, select_option, @@ -474,12 +499,8 @@ def _parse_args(message_def, load_shortarg): """build option with args/kwargs """ - new_options = OrderedDict() + options =[] for name, arg in message_def.parameters.items(): - #new_options[name] = arg - # if arg.ref: - # needs.setdefault(message_def.uri, {}).setdefault(arg.ref, []).append(name) - #for name, arg in new_options.items(): current_opt = _get_option(name, arg, file_path, @@ -488,6 +509,7 @@ def _parse_args(message_def, options.append(current_opt) if hasattr(arg, 'shortarg') and arg.shortarg and load_shortarg: options.append(SymLinkOption(arg.shortarg, current_opt)) + return options def _parse_responses(message_def, @@ -531,7 +553,6 @@ def _parse_responses(message_def, def _getoptions_from_yml(message_def, - version, optiondescriptions, file_path, needs, @@ -541,12 +562,12 @@ def _getoptions_from_yml(message_def, raise Exception('event with response?: {}'.format(file_path)) if message_def.pattern == 'rpc' and not message_def.response: print('rpc without response?: {}'.format(file_path)) - options = [] - # options = [StrOption('version', - # 'version', - # version, - # properties=frozenset(['hidden']))] - _parse_args(message_def, options, file_path, needs, select_option, message_def.uri, load_shortarg) + options = get_options(message_def, + file_path, + needs, + select_option, + message_def.uri, + load_shortarg) name = message_def.uri description = message_def.description.strip().rstrip() optiondescriptions[name] = (description, options) @@ -592,30 +613,45 @@ def _get_root_option(select_option, optiondescriptions): def get_messages(load_shortarg=False, - uris=None): + current_version=None, + uris=None, + current_module_name=MODULE_NAME): """generate description from yml files """ + global CUSTOMTYPES optiondescriptions = {} optiondescriptions_info = {} needs = {} - messages = list(list_messages(uris)) + messages = list(list_messages(uris, + current_module_name, + current_version)) messages.sort() optiondescriptions_name = [message_name.split('.', 1)[1] for message_name in messages] select_option = ChoiceOption('message', 'Nom du message.', tuple(optiondescriptions_name), properties=frozenset(['mandatory', 'positional'])) + if current_module_name is None: + CUSTOMTYPES = {} + if not CUSTOMTYPES: + if current_module_name is None: + for version in listdir(MESSAGE_ROOT_PATH): + for module_name in listdir(join(MESSAGE_ROOT_PATH, version)): + load_customtypes(module_name) + else: + load_customtypes(current_module_name) for message_name in messages: - message_def = get_message(message_name) - optiondescriptions_info[message_def.uri] = {'pattern': message_def.pattern} + message_def = get_message(message_name, + current_module_name) + optiondescriptions_info[message_def.uri] = {'pattern': message_def.pattern, + 'default_roles': message_def.default_roles, + 'version': message_name.split('.')[0]} if message_def.pattern == 'rpc': optiondescriptions_info[message_def.uri]['response'] = _parse_responses(message_def, message_name) elif message_def.response: raise Exception(f'response not allowed for {message_def.uri}') - version = message_name.split('.')[0] _getoptions_from_yml(message_def, - version, optiondescriptions, message_name, needs, @@ -623,4 +659,6 @@ def get_messages(load_shortarg=False, load_shortarg) root = _get_root_option(select_option, optiondescriptions) + if current_module_name is None: + CUSTOMTYPES = {} return optiondescriptions_info, root diff --git a/src/risotto/register.py b/src/risotto/register.py index 75c8eea9..9b348f7b 100644 --- a/src/risotto/register.py +++ b/src/risotto/register.py @@ -35,13 +35,15 @@ class RegisterDispatcher: self.injected_self = {} # postgresql pool self.pool = None - # list of uris with informations: {"v1": {"module_name.xxxxx": yyyyyy}} - self.messages = {} # load tiramisu objects messages, self.option = get_messages() - version = get_config()['global']['version'] - self.messages[version] = {} + # list of uris with informations: {"v1": {"module_name.xxxxx": yyyyyy}} + version = 'v1' + self.messages = {} for tiramisu_message, obj in messages.items(): + version = obj['version'] + if version not in self.messages: + self.messages[version] = {} self.messages[version][tiramisu_message] = obj self.risotto_module = get_config()['global']['module_name'] @@ -245,15 +247,6 @@ class RegisterDispatcher: info_msg) await module.on_join(risotto_context) - async def insert_message(self, - connection, - uri): - sql = """INSERT INTO URI(URIName) VALUES ($1) - ON CONFLICT (URIName) DO NOTHING - """ - await connection.fetchval(sql, - uri) - async def load(self): # valid function's arguments db_conf = get_config()['database']['dsn'] @@ -269,15 +262,12 @@ class RegisterDispatcher: message, function, module_name) - else: - if 'functions' in message_infos: - for function_infos in message_infos['functions']: - module_name = function_infos['module'] - function = function_infos['function'] - await self.valid_event_params(version, - message, - function, - module_name) - await self.insert_message(connection, - f'{version}.{message}') - + elif 'functions' in message_infos: + # event with functions + for function_infos in message_infos['functions']: + module_name = function_infos['module'] + function = function_infos['function'] + await self.valid_event_params(version, + message, + function, + module_name) diff --git a/src/risotto/remote.py b/src/risotto/remote.py index 0d56a03a..a95d6ca2 100644 --- a/src/risotto/remote.py +++ b/src/risotto/remote.py @@ -30,23 +30,21 @@ class Remote: self.submodules[submodule] = json return Config(self.submodules[submodule]) - async def call_or_publish(self, - submodule: str, - version: str, - message: str, - payload) -> dict: + async def remove_call(self, + submodule: str, + version: str, + message: str, + payload) -> dict: domain_name = get_config()['submodule'][submodule] remote_url = f'http://{domain_name}:8080/api/{version}' message_url = f'{remote_url}/{message}' config = await self._get_config(submodule, remote_url) - print(config) for key, value in payload.items(): path = message + '.' + key config.option(path).value.set(value) session = ClientSession() - print(message_url) async with session.post(message_url, data=dumps(payload)) as resp: response = await resp.json() if 'error' in response: