Infra
/
risotto
9
0
Fork 1
Code Issues 1 Releases Wiki Activity

better version support

This commit is contained in:
Emmanuel Garette 2020-03-07 18:35:23 +01:00
parent 5fdca08c4d
commit d1cf1df4b1
5 changed files with 181 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/risotto/config.py
View File

@ -23,8 +23,7 @@ def get_config():
'check_role': True,
'admin_user': DEFAULT_USER,
'module_name': MODULE_NAME,
'sql_filename': SQL_FILENAME,
'version': 'v1'},
'sql_filename': SQL_FILENAME},
'source': {'root_path': '/srv/seed'},
'cache': {'root_path': '/var/cache/risotto'},
'servermodel': {'internal_source': 'internal',

25
src/risotto/http.py
View File

@ -14,6 +14,9 @@ from .config import get_config
from .services import load_services
extra_routes = {}
def create_context(request):
risotto_context = Context()
risotto_context.username = request.match_info.get('username',
@ -89,7 +92,8 @@ async def handle(request):
content_type='application/json'))
async def api(request, risotto_context):
async def api(request,
risotto_context):
global tiramisu
if not tiramisu:
# check all URI that have an associated role
@ -105,16 +109,13 @@ async def api(request, risotto_context):
'''
uris = [uri['uriname'] for uri in await connection.fetch(sql)]
async with await Config(get_messages(load_shortarg=True,
current_version=risotto_context.version,
uris=uris)[1]) as config:
await config.property.read_write()
tiramisu = await config.option.dict(remotable='none')
return tiramisu
extra_routes = {'': {'function': api,
'version': 'v1'}}
async def get_app(loop):
""" build all routes
"""
@ -124,7 +125,10 @@ async def get_app(loop):
routes = []
default_storage.engine('dictionary')
await dispatcher.load()
versions = []
for version, messages in dispatcher.messages.items():
if version not in versions:
versions.append(version)
print()
print(_('======== Registered messages ========'))
for message in messages:
@ -133,6 +137,16 @@ async def get_app(loop):
print(f' - {web_message} ({pattern})')
routes.append(post(web_message, handle))
print()
print(_('======== Registered api routes ========'))
for version in versions:
api_route = {'function': api,
'version': version,
'path': f'/api/{version}'}
extra_handler = type(api_route['path'], (extra_route_handler,), api_route)
routes.append(get(api_route['path'], extra_handler))
print(f' - {api_route["path"]} (http_get)')
print()
if extra_routes:
print(_('======== Registered extra routes ========'))
for path, extra in extra_routes.items():
version = extra['version']
@ -141,7 +155,6 @@ async def get_app(loop):
extra_handler = type(path, (extra_route_handler,), extra)
routes.append(get(path, extra_handler))
print(f' - {path} (http_get)')
# routes.append(get(f'/api/{version}', api))
print()
del extra_routes
app.add_routes(routes)

216
src/risotto/message/message.py
View File

@ -52,9 +52,12 @@ class MessageDefinition:
'related',
'response')
def __init__(self, raw_def, message):
def __init__(self,
raw_def,
version,
message):
# default value for non mandatory key
self.version = u''
self.version = version
self.parameters = OrderedDict()
self.errors = []
self.related = []
@ -74,11 +77,13 @@ class MessageDefinition:
elif key == 'parameters':
if 'type' in value and isinstance(value['type'], str):
# should be a customtype
value = CUSTOMTYPES[value['type']].properties
value = CUSTOMTYPES[self.version][value['type']].properties
else:
value = _parse_parameters(value)
value = _parse_parameters(value,
self.version)
elif key == 'response':
value = ResponseDefinition(value)
value = ResponseDefinition(value,
self.version)
elif key == 'errors':
value = _parse_error_definition(value)
setattr(self, key, value)
@ -99,7 +104,10 @@ class ParameterDefinition:
'ref',
'shortarg')
def __init__(self, name, raw_def):
def __init__(self,
name,
version,
raw_def):
self.name = name
# default value for non mandatory key
self.help = None
@ -115,11 +123,11 @@ class ParameterDefinition:
tvalue = value[2:]
else:
tvalue = value
if tvalue in CUSTOMTYPES:
if tvalue in CUSTOMTYPES[version]:
if value.startswith('[]'):
value = '[]{}'.format(CUSTOMTYPES[tvalue].type)
value = '[]{}'.format(CUSTOMTYPES[version][tvalue].type)
else:
value = CUSTOMTYPES[value].type
value = CUSTOMTYPES[version][value].type
else:
self._valid_type(value)
#self._valid_type(value)
@ -150,7 +158,9 @@ class ResponseDefinition:
'required',
'multi')
def __init__(self, responses):
def __init__(self,
responses,
version):
self.ref = None
self.parameters = None
self.multi = False
@ -164,13 +174,13 @@ class ResponseDefinition:
self.multi = True
else:
tvalue = value
if tvalue in CUSTOMTYPES:
self.parameters = CUSTOMTYPES[tvalue].properties
self.required = CUSTOMTYPES[tvalue].required
if tvalue in CUSTOMTYPES[version]:
self.parameters = CUSTOMTYPES[version][tvalue].properties
self.required = CUSTOMTYPES[version][tvalue].required
if value.startswith('[]'):
value = '[]{}'.format(CUSTOMTYPES[tvalue].type)
value = '[]{}'.format(CUSTOMTYPES[version][tvalue].type)
else:
value = CUSTOMTYPES[value].type
value = CUSTOMTYPES[version][value].type
else:
self._valid_type(value)
setattr(self, key, value)
@ -208,57 +218,63 @@ def _parse_error_definition(raw_defs):
return new_value
def _parse_parameters(raw_defs):
def _parse_parameters(raw_defs,
version):
parameters = OrderedDict()
for name, raw_def in raw_defs.items():
parameters[name] = ParameterDefinition(name, raw_def)
parameters[name] = ParameterDefinition(name,
version,
raw_def)
return parameters
def parse_definition(filecontent: bytes,
message: str):
return MessageDefinition(load(filecontent, Loader=SafeLoader), message)
def is_message_defined(uri):
version, message = split_message_uri(uri)
path = get_message_file_path(version, message)
return isfile(path)
def get_message(uri):
load_customtypes()
def get_message(uri: str,
current_module_name: str):
try:
version, message = split_message_uri(uri)
path = get_message_file_path(version, message)
version, message = uri.split('.', 1)
path = get_message_file_path(version,
message,
current_module_name)
with open(path, "r") as message_file:
message_content = parse_definition(message_file.read(), message)
message_content.version = version
return message_content
return MessageDefinition(load(message_file.read(), Loader=SafeLoader),
version,
message)
except Exception as err:
import traceback
traceback.print_exc()
raise Exception(_('cannot parse message {}: {}').format(uri, str(err)))
raise Exception(_(f'cannot parse message {uri}: {err}'))
def split_message_uri(uri):
return uri.split('.', 1)
def get_message_file_path(version, message):
def get_message_file_path(version,
message,
current_module_name):
module_name, filename = message.split('.', 1)
if module_name != MODULE_NAME:
raise Exception(f'should only load message for {MODULE_NAME}, not {message}')
return join(MESSAGE_ROOT_PATH, version, MODULE_NAME, 'messages', filename + '.yml')
if current_module_name and module_name != current_module_name:
raise Exception(f'should only load message for {current_module_name}, not {message}')
return join(MESSAGE_ROOT_PATH, version, module_name, 'messages', filename + '.yml')
def list_messages(uris):
def list_messages(uris,
current_module_name,
current_version):
def get_module_paths():
if current_module_name is not None:
yield current_module_name, join(MESSAGE_ROOT_PATH, version, current_module_name, 'messages')
else:
for module_name in listdir(join(MESSAGE_ROOT_PATH, version)):
yield module_name, join(MESSAGE_ROOT_PATH, version, module_name, 'messages')
if current_version:
versions = [current_version]
else:
versions = listdir(join(MESSAGE_ROOT_PATH))
versions.sort()
for version in versions:
for message in listdir(join(MESSAGE_ROOT_PATH, version, MODULE_NAME, 'messages')):
for module_name, message_path in get_module_paths():
for message in listdir(message_path):
if message.endswith('.yml'):
uri = version + '.' + MODULE_NAME + '.' + message.rsplit('.', 1)[0]
uri = version + '.' + module_name + '.' + message.rsplit('.', 1)[0]
# if uris is not None, return only is in uris' list
if uris is not None and uri not in uris:
continue
yield uri
@ -374,38 +390,48 @@ class CustomType:
return self.title
def load_customtypes():
if not CUSTOMTYPES:
def load_customtypes(current_module_name: str) -> None:
versions = listdir(MESSAGE_ROOT_PATH)
versions.sort()
for version in versions:
for message in listdir(join(MESSAGE_ROOT_PATH, version, MODULE_NAME, 'types')):
if message.endswith('.yml'):
path = join(MESSAGE_ROOT_PATH, version, MODULE_NAME, 'types', message)
message = message.rsplit('.', 1)[0]
with open(path, "r") as message_file:
try:
ret = CustomType(load(message_file, Loader=SafeLoader))
CUSTOMTYPES[ret.getname()] = ret
except Exception as err:
import traceback
traceback.print_exc()
raise Exception('{} for {}'.format(err, message))
for customtype in CUSTOMTYPES.values():
def convert_properties(customtype: str,
version: str) -> None:
""" if properties include an other customtype, replace it
"""
properties = {}
for key, value in customtype.properties.items():
type_ = value.type
if type_.startswith('[]'):
ttype_ = type_[2:]
else:
ttype_ = type_
if ttype_ in CUSTOMTYPES:
if type_.startswith('[]'):
if type_ in CUSTOMTYPES[version]:
raise Exception(_('cannot have []CustomType'))
properties[key] = CUSTOMTYPES[ttype_]
properties[key] = value
else:
if type_ in CUSTOMTYPES[version]:
print('====== ca existe')
properties[key] = CUSTOMTYPES[version][ttype_]
else:
properties[key] = value
customtype.properties = properties
for version in versions:
if version not in CUSTOMTYPES:
CUSTOMTYPES[version] = {}
types_path = join(MESSAGE_ROOT_PATH,
version,
current_module_name,
'types')
for message in listdir(types_path):
if message.endswith('.yml'):
path = join(types_path, message)
# remove extension
message = message.rsplit('.', 1)[0]
with open(path, "r") as message_file:
try:
custom_type = CustomType(load(message_file, Loader=SafeLoader))
convert_properties(custom_type,
version)
CUSTOMTYPES[version][custom_type.getname()] = custom_type
except Exception as err:
raise Exception(_(f'enable to load type {err}: {message}'))
def _get_description(description,
@ -465,8 +491,7 @@ def _get_option(name,
return obj
def _parse_args(message_def,
options,
def get_options(message_def,
file_path,
needs,
select_option,
@ -474,12 +499,8 @@ def _parse_args(message_def,
load_shortarg):
"""build option with args/kwargs
"""
new_options = OrderedDict()
options =[]
for name, arg in message_def.parameters.items():
#new_options[name] = arg
# if arg.ref:
# needs.setdefault(message_def.uri, {}).setdefault(arg.ref, []).append(name)
#for name, arg in new_options.items():
current_opt = _get_option(name,
arg,
file_path,
@ -488,6 +509,7 @@ def _parse_args(message_def,
options.append(current_opt)
if hasattr(arg, 'shortarg') and arg.shortarg and load_shortarg:
options.append(SymLinkOption(arg.shortarg, current_opt))
return options
def _parse_responses(message_def,
@ -531,7 +553,6 @@ def _parse_responses(message_def,
def _getoptions_from_yml(message_def,
version,
optiondescriptions,
file_path,
needs,
@ -541,12 +562,12 @@ def _getoptions_from_yml(message_def,
raise Exception('event with response?: {}'.format(file_path))
if message_def.pattern == 'rpc' and not message_def.response:
print('rpc without response?: {}'.format(file_path))
options = []
# options = [StrOption('version',
# 'version',
# version,
# properties=frozenset(['hidden']))]
_parse_args(message_def, options, file_path, needs, select_option, message_def.uri, load_shortarg)
options = get_options(message_def,
file_path,
needs,
select_option,
message_def.uri,
load_shortarg)
name = message_def.uri
description = message_def.description.strip().rstrip()
optiondescriptions[name] = (description, options)
@ -592,30 +613,45 @@ def _get_root_option(select_option, optiondescriptions):
def get_messages(load_shortarg=False,
uris=None):
current_version=None,
uris=None,
current_module_name=MODULE_NAME):
"""generate description from yml files
"""
global CUSTOMTYPES
optiondescriptions = {}
optiondescriptions_info = {}
needs = {}
messages = list(list_messages(uris))
messages = list(list_messages(uris,
current_module_name,
current_version))
messages.sort()
optiondescriptions_name = [message_name.split('.', 1)[1] for message_name in messages]
select_option = ChoiceOption('message',
'Nom du message.',
tuple(optiondescriptions_name),
properties=frozenset(['mandatory', 'positional']))
if current_module_name is None:
CUSTOMTYPES = {}
if not CUSTOMTYPES:
if current_module_name is None:
for version in listdir(MESSAGE_ROOT_PATH):
for module_name in listdir(join(MESSAGE_ROOT_PATH, version)):
load_customtypes(module_name)
else:
load_customtypes(current_module_name)
for message_name in messages:
message_def = get_message(message_name)
optiondescriptions_info[message_def.uri] = {'pattern': message_def.pattern}
message_def = get_message(message_name,
current_module_name)
optiondescriptions_info[message_def.uri] = {'pattern': message_def.pattern,
'default_roles': message_def.default_roles,
'version': message_name.split('.')[0]}
if message_def.pattern == 'rpc':
optiondescriptions_info[message_def.uri]['response'] = _parse_responses(message_def,
message_name)
elif message_def.response:
raise Exception(f'response not allowed for {message_def.uri}')
version = message_name.split('.')[0]
_getoptions_from_yml(message_def,
version,
optiondescriptions,
message_name,
needs,
@ -623,4 +659,6 @@ def get_messages(load_shortarg=False,
load_shortarg)
root = _get_root_option(select_option, optiondescriptions)
if current_module_name is None:
CUSTOMTYPES = {}
return optiondescriptions_info, root

26
src/risotto/register.py
View File

@ -35,13 +35,15 @@ class RegisterDispatcher:
self.injected_self = {}
# postgresql pool
self.pool = None
# list of uris with informations: {"v1": {"module_name.xxxxx": yyyyyy}}
self.messages = {}
# load tiramisu objects
messages, self.option = get_messages()
version = get_config()['global']['version']
self.messages[version] = {}
# list of uris with informations: {"v1": {"module_name.xxxxx": yyyyyy}}
version = 'v1'
self.messages = {}
for tiramisu_message, obj in messages.items():
version = obj['version']
if version not in self.messages:
self.messages[version] = {}
self.messages[version][tiramisu_message] = obj
self.risotto_module = get_config()['global']['module_name']
@ -245,15 +247,6 @@ class RegisterDispatcher:
info_msg)
await module.on_join(risotto_context)
async def insert_message(self,
connection,
uri):
sql = """INSERT INTO URI(URIName) VALUES ($1)
ON CONFLICT (URIName) DO NOTHING
"""
await connection.fetchval(sql,
uri)
async def load(self):
# valid function's arguments
db_conf = get_config()['database']['dsn']
@ -269,8 +262,8 @@ class RegisterDispatcher:
message,
function,
module_name)
else:
if 'functions' in message_infos:
elif 'functions' in message_infos:
# event with functions
for function_infos in message_infos['functions']:
module_name = function_infos['module']
function = function_infos['function']
@ -278,6 +271,3 @@ class RegisterDispatcher:
message,
function,
module_name)
await self.insert_message(connection,
f'{version}.{message}')

4
src/risotto/remote.py
View File

@ -30,7 +30,7 @@ class Remote:
self.submodules[submodule] = json
return Config(self.submodules[submodule])
async def call_or_publish(self,
async def remove_call(self,
submodule: str,
version: str,
message: str,
@ -41,12 +41,10 @@ class Remote:
config = await self._get_config(submodule,
remote_url)
print(config)
for key, value in payload.items():
path = message + '.' + key
config.option(path).value.set(value)
session = ClientSession()
print(message_url)
async with session.post(message_url, data=dumps(payload)) as resp:
response = await resp.json()
if 'error' in response: