Infra
/
risotto
9
0
Fork 1
Code Issues 1 Releases Wiki Activity

better version support

This commit is contained in:
Emmanuel Garette 2020-03-07 18:35:23 +01:00
parent 5fdca08c4d
commit d1cf1df4b1
5 changed files with 181 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/risotto/config.py
View File

@ -23,8 +23,7 @@ def get_config():
'check_role': True, 'check_role': True,
'admin_user': DEFAULT_USER, 'admin_user': DEFAULT_USER,
'module_name': MODULE_NAME, 'module_name': MODULE_NAME,
'sql_filename': SQL_FILENAME, 'sql_filename': SQL_FILENAME},
'version': 'v1'},
'source': {'root_path': '/srv/seed'}, 'source': {'root_path': '/srv/seed'},
'cache': {'root_path': '/var/cache/risotto'}, 'cache': {'root_path': '/var/cache/risotto'},
'servermodel': {'internal_source': 'internal', 'servermodel': {'internal_source': 'internal',

27
src/risotto/http.py
View File

@ -14,6 +14,9 @@ from .config import get_config
from .services import load_services from .services import load_services
extra_routes = {}
def create_context(request): def create_context(request):
risotto_context = Context() risotto_context = Context()
risotto_context.username = request.match_info.get('username', risotto_context.username = request.match_info.get('username',
@ -89,7 +92,8 @@ async def handle(request):
content_type='application/json')) content_type='application/json'))
async def api(request, risotto_context): async def api(request,
risotto_context):
global tiramisu global tiramisu
if not tiramisu: if not tiramisu:
# check all URI that have an associated role # check all URI that have an associated role
@ -105,16 +109,13 @@ async def api(request, risotto_context):
''' '''
uris = [uri['uriname'] for uri in await connection.fetch(sql)] uris = [uri['uriname'] for uri in await connection.fetch(sql)]
async with await Config(get_messages(load_shortarg=True, async with await Config(get_messages(load_shortarg=True,
current_version=risotto_context.version,
uris=uris)[1]) as config: uris=uris)[1]) as config:
await config.property.read_write() await config.property.read_write()
tiramisu = await config.option.dict(remotable='none') tiramisu = await config.option.dict(remotable='none')
return tiramisu return tiramisu
extra_routes = {'': {'function': api,
'version': 'v1'}}
async def get_app(loop): async def get_app(loop):
""" build all routes """ build all routes
""" """
@ -124,7 +125,10 @@ async def get_app(loop):
routes = [] routes = []
default_storage.engine('dictionary') default_storage.engine('dictionary')
await dispatcher.load() await dispatcher.load()
versions = []
for version, messages in dispatcher.messages.items(): for version, messages in dispatcher.messages.items():
if version not in versions:
versions.append(version)
print() print()
print(_('======== Registered messages ========')) print(_('======== Registered messages ========'))
for message in messages: for message in messages:
@ -132,7 +136,17 @@ async def get_app(loop):
pattern = dispatcher.messages[version][message]['pattern'] pattern = dispatcher.messages[version][message]['pattern']
print(f' - {web_message} ({pattern})') print(f' - {web_message} ({pattern})')
routes.append(post(web_message, handle)) routes.append(post(web_message, handle))
print() print()
print(_('======== Registered api routes ========'))
for version in versions:
api_route = {'function': api,
'version': version,
'path': f'/api/{version}'}
extra_handler = type(api_route['path'], (extra_route_handler,), api_route)
routes.append(get(api_route['path'], extra_handler))
print(f' - {api_route["path"]} (http_get)')
print()
if extra_routes:
print(_('======== Registered extra routes ========')) print(_('======== Registered extra routes ========'))
for path, extra in extra_routes.items(): for path, extra in extra_routes.items():
version = extra['version'] version = extra['version']
@ -141,7 +155,6 @@ async def get_app(loop):
extra_handler = type(path, (extra_route_handler,), extra) extra_handler = type(path, (extra_route_handler,), extra)
routes.append(get(path, extra_handler)) routes.append(get(path, extra_handler))
print(f' - {path} (http_get)') print(f' - {path} (http_get)')
# routes.append(get(f'/api/{version}', api))
print() print()
del extra_routes del extra_routes
app.add_routes(routes) app.add_routes(routes)

242
src/risotto/message/message.py
View File

@ -52,9 +52,12 @@ class MessageDefinition:
'related', 'related',
'response') 'response')
def __init__(self, raw_def, message): def __init__(self,
raw_def,
version,
message):
# default value for non mandatory key # default value for non mandatory key
self.version = u'' self.version = version
self.parameters = OrderedDict() self.parameters = OrderedDict()
self.errors = [] self.errors = []
self.related = [] self.related = []
@ -74,11 +77,13 @@ class MessageDefinition:
elif key == 'parameters': elif key == 'parameters':
if 'type' in value and isinstance(value['type'], str): if 'type' in value and isinstance(value['type'], str):
# should be a customtype # should be a customtype
value = CUSTOMTYPES[value['type']].properties value = CUSTOMTYPES[self.version][value['type']].properties
else: else:
value = _parse_parameters(value) value = _parse_parameters(value,
self.version)
elif key == 'response': elif key == 'response':
value = ResponseDefinition(value) value = ResponseDefinition(value,
self.version)
elif key == 'errors': elif key == 'errors':
value = _parse_error_definition(value) value = _parse_error_definition(value)
setattr(self, key, value) setattr(self, key, value)
@ -99,7 +104,10 @@ class ParameterDefinition:
'ref', 'ref',
'shortarg') 'shortarg')
def __init__(self, name, raw_def): def __init__(self,
name,
version,
raw_def):
self.name = name self.name = name
# default value for non mandatory key # default value for non mandatory key
self.help = None self.help = None
@ -115,11 +123,11 @@ class ParameterDefinition:
tvalue = value[2:] tvalue = value[2:]
else: else:
tvalue = value tvalue = value
if tvalue in CUSTOMTYPES: if tvalue in CUSTOMTYPES[version]:
if value.startswith('[]'): if value.startswith('[]'):
value = '[]{}'.format(CUSTOMTYPES[tvalue].type) value = '[]{}'.format(CUSTOMTYPES[version][tvalue].type)
else: else:
value = CUSTOMTYPES[value].type value = CUSTOMTYPES[version][value].type
else: else:
self._valid_type(value) self._valid_type(value)
#self._valid_type(value) #self._valid_type(value)
@ -150,7 +158,9 @@ class ResponseDefinition:
'required', 'required',
'multi') 'multi')
def __init__(self, responses): def __init__(self,
responses,
version):
self.ref = None self.ref = None
self.parameters = None self.parameters = None
self.multi = False self.multi = False
@ -164,13 +174,13 @@ class ResponseDefinition:
self.multi = True self.multi = True
else: else:
tvalue = value tvalue = value
if tvalue in CUSTOMTYPES: if tvalue in CUSTOMTYPES[version]:
self.parameters = CUSTOMTYPES[tvalue].properties self.parameters = CUSTOMTYPES[version][tvalue].properties
self.required = CUSTOMTYPES[tvalue].required self.required = CUSTOMTYPES[version][tvalue].required
if value.startswith('[]'): if value.startswith('[]'):
value = '[]{}'.format(CUSTOMTYPES[tvalue].type) value = '[]{}'.format(CUSTOMTYPES[version][tvalue].type)
else: else:
value = CUSTOMTYPES[value].type value = CUSTOMTYPES[version][value].type
else: else:
self._valid_type(value) self._valid_type(value)
setattr(self, key, value) setattr(self, key, value)
@ -208,60 +218,66 @@ def _parse_error_definition(raw_defs):
return new_value return new_value
def _parse_parameters(raw_defs): def _parse_parameters(raw_defs,
version):
parameters = OrderedDict() parameters = OrderedDict()
for name, raw_def in raw_defs.items(): for name, raw_def in raw_defs.items():
parameters[name] = ParameterDefinition(name, raw_def) parameters[name] = ParameterDefinition(name,
version,
raw_def)
return parameters return parameters
def parse_definition(filecontent: bytes, def get_message(uri: str,
message: str): current_module_name: str):
return MessageDefinition(load(filecontent, Loader=SafeLoader), message)
def is_message_defined(uri):
version, message = split_message_uri(uri)
path = get_message_file_path(version, message)
return isfile(path)
def get_message(uri):
load_customtypes()
try: try:
version, message = split_message_uri(uri) version, message = uri.split('.', 1)
path = get_message_file_path(version, message) path = get_message_file_path(version,
message,
current_module_name)
with open(path, "r") as message_file: with open(path, "r") as message_file:
message_content = parse_definition(message_file.read(), message) return MessageDefinition(load(message_file.read(), Loader=SafeLoader),
message_content.version = version version,
return message_content message)
except Exception as err: except Exception as err:
import traceback import traceback
traceback.print_exc() traceback.print_exc()
raise Exception(_('cannot parse message {}: {}').format(uri, str(err))) raise Exception(_(f'cannot parse message {uri}: {err}'))
def split_message_uri(uri): def get_message_file_path(version,
return uri.split('.', 1) message,
current_module_name):
def get_message_file_path(version, message):
module_name, filename = message.split('.', 1) module_name, filename = message.split('.', 1)
if module_name != MODULE_NAME: if current_module_name and module_name != current_module_name:
raise Exception(f'should only load message for {MODULE_NAME}, not {message}') raise Exception(f'should only load message for {current_module_name}, not {message}')
return join(MESSAGE_ROOT_PATH, version, MODULE_NAME, 'messages', filename + '.yml') return join(MESSAGE_ROOT_PATH, version, module_name, 'messages', filename + '.yml')
def list_messages(uris): def list_messages(uris,
versions = listdir(join(MESSAGE_ROOT_PATH)) current_module_name,
current_version):
def get_module_paths():
if current_module_name is not None:
yield current_module_name, join(MESSAGE_ROOT_PATH, version, current_module_name, 'messages')
else:
for module_name in listdir(join(MESSAGE_ROOT_PATH, version)):
yield module_name, join(MESSAGE_ROOT_PATH, version, module_name, 'messages')
if current_version:
versions = [current_version]
else:
versions = listdir(join(MESSAGE_ROOT_PATH))
versions.sort() versions.sort()
for version in versions: for version in versions:
for message in listdir(join(MESSAGE_ROOT_PATH, version, MODULE_NAME, 'messages')): for module_name, message_path in get_module_paths():
if message.endswith('.yml'): for message in listdir(message_path):
uri = version + '.' + MODULE_NAME + '.' + message.rsplit('.', 1)[0] if message.endswith('.yml'):
if uris is not None and uri not in uris: uri = version + '.' + module_name + '.' + message.rsplit('.', 1)[0]
continue # if uris is not None, return only is in uris' list
yield uri if uris is not None and uri not in uris:
continue
yield uri
class CustomParam: class CustomParam:
@ -374,38 +390,48 @@ class CustomType:
return self.title return self.title
def load_customtypes(): def load_customtypes(current_module_name: str) -> None:
if not CUSTOMTYPES: versions = listdir(MESSAGE_ROOT_PATH)
versions = listdir(MESSAGE_ROOT_PATH) versions.sort()
versions.sort() def convert_properties(customtype: str,
for version in versions: version: str) -> None:
for message in listdir(join(MESSAGE_ROOT_PATH, version, MODULE_NAME, 'types')): """ if properties include an other customtype, replace it
if message.endswith('.yml'): """
path = join(MESSAGE_ROOT_PATH, version, MODULE_NAME, 'types', message) properties = {}
message = message.rsplit('.', 1)[0] for key, value in customtype.properties.items():
with open(path, "r") as message_file: type_ = value.type
try: if type_.startswith('[]'):
ret = CustomType(load(message_file, Loader=SafeLoader)) if type_ in CUSTOMTYPES[version]:
CUSTOMTYPES[ret.getname()] = ret raise Exception(_('cannot have []CustomType'))
except Exception as err: properties[key] = value
import traceback else:
traceback.print_exc() if type_ in CUSTOMTYPES[version]:
raise Exception('{} for {}'.format(err, message)) print('====== ca existe')
for customtype in CUSTOMTYPES.values(): properties[key] = CUSTOMTYPES[version][ttype_]
properties = {}
for key, value in customtype.properties.items():
type_ = value.type
if type_.startswith('[]'):
ttype_ = type_[2:]
else:
ttype_ = type_
if ttype_ in CUSTOMTYPES:
if type_.startswith('[]'):
raise Exception(_('cannot have []CustomType'))
properties[key] = CUSTOMTYPES[ttype_]
else: else:
properties[key] = value properties[key] = value
customtype.properties = properties customtype.properties = properties
for version in versions:
if version not in CUSTOMTYPES:
CUSTOMTYPES[version] = {}
types_path = join(MESSAGE_ROOT_PATH,
version,
current_module_name,
'types')
for message in listdir(types_path):
if message.endswith('.yml'):
path = join(types_path, message)
# remove extension
message = message.rsplit('.', 1)[0]
with open(path, "r") as message_file:
try:
custom_type = CustomType(load(message_file, Loader=SafeLoader))
convert_properties(custom_type,
version)
CUSTOMTYPES[version][custom_type.getname()] = custom_type
except Exception as err:
raise Exception(_(f'enable to load type {err}: {message}'))
def _get_description(description, def _get_description(description,
@ -465,8 +491,7 @@ def _get_option(name,
return obj return obj
def _parse_args(message_def, def get_options(message_def,
options,
file_path, file_path,
needs, needs,
select_option, select_option,
@ -474,12 +499,8 @@ def _parse_args(message_def,
load_shortarg): load_shortarg):
"""build option with args/kwargs """build option with args/kwargs
""" """
new_options = OrderedDict() options =[]
for name, arg in message_def.parameters.items(): for name, arg in message_def.parameters.items():
#new_options[name] = arg
# if arg.ref:
# needs.setdefault(message_def.uri, {}).setdefault(arg.ref, []).append(name)
#for name, arg in new_options.items():
current_opt = _get_option(name, current_opt = _get_option(name,
arg, arg,
file_path, file_path,
@ -488,6 +509,7 @@ def _parse_args(message_def,
options.append(current_opt) options.append(current_opt)
if hasattr(arg, 'shortarg') and arg.shortarg and load_shortarg: if hasattr(arg, 'shortarg') and arg.shortarg and load_shortarg:
options.append(SymLinkOption(arg.shortarg, current_opt)) options.append(SymLinkOption(arg.shortarg, current_opt))
return options
def _parse_responses(message_def, def _parse_responses(message_def,
@ -531,7 +553,6 @@ def _parse_responses(message_def,
def _getoptions_from_yml(message_def, def _getoptions_from_yml(message_def,
version,
optiondescriptions, optiondescriptions,
file_path, file_path,
needs, needs,
@ -541,12 +562,12 @@ def _getoptions_from_yml(message_def,
raise Exception('event with response?: {}'.format(file_path)) raise Exception('event with response?: {}'.format(file_path))
if message_def.pattern == 'rpc' and not message_def.response: if message_def.pattern == 'rpc' and not message_def.response:
print('rpc without response?: {}'.format(file_path)) print('rpc without response?: {}'.format(file_path))
options = [] options = get_options(message_def,
# options = [StrOption('version', file_path,
# 'version', needs,
# version, select_option,
# properties=frozenset(['hidden']))] message_def.uri,
_parse_args(message_def, options, file_path, needs, select_option, message_def.uri, load_shortarg) load_shortarg)
name = message_def.uri name = message_def.uri
description = message_def.description.strip().rstrip() description = message_def.description.strip().rstrip()
optiondescriptions[name] = (description, options) optiondescriptions[name] = (description, options)
@ -592,30 +613,45 @@ def _get_root_option(select_option, optiondescriptions):
def get_messages(load_shortarg=False, def get_messages(load_shortarg=False,
uris=None): current_version=None,
uris=None,
current_module_name=MODULE_NAME):
"""generate description from yml files """generate description from yml files
""" """
global CUSTOMTYPES
optiondescriptions = {} optiondescriptions = {}
optiondescriptions_info = {} optiondescriptions_info = {}
needs = {} needs = {}
messages = list(list_messages(uris)) messages = list(list_messages(uris,
current_module_name,
current_version))
messages.sort() messages.sort()
optiondescriptions_name = [message_name.split('.', 1)[1] for message_name in messages] optiondescriptions_name = [message_name.split('.', 1)[1] for message_name in messages]
select_option = ChoiceOption('message', select_option = ChoiceOption('message',
'Nom du message.', 'Nom du message.',
tuple(optiondescriptions_name), tuple(optiondescriptions_name),
properties=frozenset(['mandatory', 'positional'])) properties=frozenset(['mandatory', 'positional']))
if current_module_name is None:
CUSTOMTYPES = {}
if not CUSTOMTYPES:
if current_module_name is None:
for version in listdir(MESSAGE_ROOT_PATH):
for module_name in listdir(join(MESSAGE_ROOT_PATH, version)):
load_customtypes(module_name)
else:
load_customtypes(current_module_name)
for message_name in messages: for message_name in messages:
message_def = get_message(message_name) message_def = get_message(message_name,
optiondescriptions_info[message_def.uri] = {'pattern': message_def.pattern} current_module_name)
optiondescriptions_info[message_def.uri] = {'pattern': message_def.pattern,
'default_roles': message_def.default_roles,
'version': message_name.split('.')[0]}
if message_def.pattern == 'rpc': if message_def.pattern == 'rpc':
optiondescriptions_info[message_def.uri]['response'] = _parse_responses(message_def, optiondescriptions_info[message_def.uri]['response'] = _parse_responses(message_def,
message_name) message_name)
elif message_def.response: elif message_def.response:
raise Exception(f'response not allowed for {message_def.uri}') raise Exception(f'response not allowed for {message_def.uri}')
version = message_name.split('.')[0]
_getoptions_from_yml(message_def, _getoptions_from_yml(message_def,
version,
optiondescriptions, optiondescriptions,
message_name, message_name,
needs, needs,
@ -623,4 +659,6 @@ def get_messages(load_shortarg=False,
load_shortarg) load_shortarg)
root = _get_root_option(select_option, optiondescriptions) root = _get_root_option(select_option, optiondescriptions)
if current_module_name is None:
CUSTOMTYPES = {}
return optiondescriptions_info, root return optiondescriptions_info, root

40
src/risotto/register.py
View File

@ -35,13 +35,15 @@ class RegisterDispatcher:
self.injected_self = {} self.injected_self = {}
# postgresql pool # postgresql pool
self.pool = None self.pool = None
# list of uris with informations: {"v1": {"module_name.xxxxx": yyyyyy}}
self.messages = {}
# load tiramisu objects # load tiramisu objects
messages, self.option = get_messages() messages, self.option = get_messages()
version = get_config()['global']['version'] # list of uris with informations: {"v1": {"module_name.xxxxx": yyyyyy}}
self.messages[version] = {} version = 'v1'
self.messages = {}
for tiramisu_message, obj in messages.items(): for tiramisu_message, obj in messages.items():
version = obj['version']
if version not in self.messages:
self.messages[version] = {}
self.messages[version][tiramisu_message] = obj self.messages[version][tiramisu_message] = obj
self.risotto_module = get_config()['global']['module_name'] self.risotto_module = get_config()['global']['module_name']
@ -245,15 +247,6 @@ class RegisterDispatcher:
info_msg) info_msg)
await module.on_join(risotto_context) await module.on_join(risotto_context)
async def insert_message(self,
connection,
uri):
sql = """INSERT INTO URI(URIName) VALUES ($1)
ON CONFLICT (URIName) DO NOTHING
"""
await connection.fetchval(sql,
uri)
async def load(self): async def load(self):
# valid function's arguments # valid function's arguments
db_conf = get_config()['database']['dsn'] db_conf = get_config()['database']['dsn']
@ -269,15 +262,12 @@ class RegisterDispatcher:
message, message,
function, function,
module_name) module_name)
else: elif 'functions' in message_infos:
if 'functions' in message_infos: # event with functions
for function_infos in message_infos['functions']: for function_infos in message_infos['functions']:
module_name = function_infos['module'] module_name = function_infos['module']
function = function_infos['function'] function = function_infos['function']
await self.valid_event_params(version, await self.valid_event_params(version,
message, message,
function, function,
module_name) module_name)
await self.insert_message(connection,
f'{version}.{message}')

12
src/risotto/remote.py
View File

@ -30,23 +30,21 @@ class Remote:
self.submodules[submodule] = json self.submodules[submodule] = json
return Config(self.submodules[submodule]) return Config(self.submodules[submodule])
async def call_or_publish(self, async def remove_call(self,
submodule: str, submodule: str,
version: str, version: str,
message: str, message: str,
payload) -> dict: payload) -> dict:
domain_name = get_config()['submodule'][submodule] domain_name = get_config()['submodule'][submodule]
remote_url = f'http://{domain_name}:8080/api/{version}' remote_url = f'http://{domain_name}:8080/api/{version}'
message_url = f'{remote_url}/{message}' message_url = f'{remote_url}/{message}'
config = await self._get_config(submodule, config = await self._get_config(submodule,
remote_url) remote_url)
print(config)
for key, value in payload.items(): for key, value in payload.items():
path = message + '.' + key path = message + '.' + key
config.option(path).value.set(value) config.option(path).value.set(value)
session = ClientSession() session = ClientSession()
print(message_url)
async with session.post(message_url, data=dumps(payload)) as resp: async with session.post(message_url, data=dumps(payload)) as resp:
response = await resp.json() response = await resp.json()
if 'error' in response: if 'error' in response: