Infra
/
risotto
9
0
Fork 1
Code Issues 1 Releases Wiki Activity

better config integration

This commit is contained in:
Emmanuel Garette 2019-12-02 10:29:40 +01:00
parent 847fbfc1e1
commit 8a551f85b2
23 changed files with 1064 additions and 754 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
messages/v1/messages/config.session.server.configure.yml
View File

@ -37,6 +37,11 @@ parameters:
shortarg: v
description: Valeur de la variable.
default: null
value_multi:
type: '[]Any'
shortarg: m
description: Valeur de la variable de type multi.
default: []
response:
type: ConfigStatus

2
messages/v1/messages/config.session.server.stop.yml
View File

@ -14,7 +14,7 @@ public: true
domain: config-domain
parameters:
sessionid:
session_id:
ref: Config.SessionId
type: String
shortarg: s

5
messages/v1/messages/config.session.servermodel.configure.yml
View File

@ -37,6 +37,11 @@ parameters:
shortarg: v
description: Valeur de la variable.
default: null
value_multi:
type: '[]Any'
shortarg: m
description: Valeur de la variable de type multi.
default: []
response:
type: ConfigStatus

2
messages/v1/messages/config.session.servermodel.stop.yml
View File

@ -13,7 +13,7 @@ public: true
domain: config-domain
parameters:
sessionid:
session_id:
ref: Config.SessionId
type: String
shortarg: s

0
messages/v1/messages/old/server.created.yml → messages/v1/messages/server.created.yml
View File

0
messages/v1/messages/old/server.deleted.yml → messages/v1/messages/server.deleted.yml
View File

0
messages/v1/messages/old/servermodel.created.yml → messages/v1/messages/servermodel.created.yml
View File

0
messages/v1/messages/old/servermodel.describe.yml → messages/v1/messages/servermodel.describe.yml
View File

0
messages/v1/messages/old/servermodel.list.yml → messages/v1/messages/servermodel.list.yml
View File

0
messages/v1/messages/old/servermodel.updated.yml → messages/v1/messages/servermodel.updated.yml
View File

2
messages/v1/types/config.configuration.status.yml
View File

@ -19,6 +19,6 @@ properties:
type: string
description: Liste des variables obligatoires non renseignées si la configuration a le statut incomplete.
required:
- sessionid
- session_id
- status

4
messages/v1/types/config.session.yml
View File

@ -3,7 +3,7 @@ title: ConfigSession
type: object
description: Description de la session.
properties:
sessionid:
session_id:
type: string
description: ID de la session.
ref: Config.SessionId
@ -30,7 +30,7 @@ properties:
type: file
description: Contenu de la configuration.
required:
- sessionid
- session_id
- id
- username
- timestamp

10
script/server.py
View File

@ -1,6 +1,12 @@
from aiohttp.web import run_app
from asyncio import get_event_loop
from risotto import get_app
if __name__ == '__main__':
run_app(get_app())
loop = get_event_loop()
loop.run_until_complete(get_app(loop))
try:
loop.run_forever()
except KeyboardInterrupt:
pass

4
src/risotto/__init__.py
View File

@ -1,8 +1,6 @@
from .utils import undefined
from .dispatcher import register, dispatcher
from .http import get_app
# just to register every route
from . import services as _services
__ALL__ = ('undefined', 'register', 'dispatcher', 'get_app')
__ALL__ = ('get_app',)

2
src/risotto/config.py
View File

@ -3,3 +3,5 @@ MESSAGE_ROOT_PATH = 'messages'
ROOT_CACHE_DIR = 'cache'
DEBUG = True
DATABASE_DIR = 'database'
INTERNAL_USER = 'internal'
ROUGAIL_DTD_PATH = '../rougail/data/creole.dtd'

4
src/risotto/controller.py
View File

@ -26,3 +26,7 @@ class Controller:
uri,
risotto_context,
**kwargs)
async def on_join(self,
risotto_context):
pass

455
src/risotto/dispatcher.py
View File

@ -1,284 +1,44 @@
from tiramisu import Config
from inspect import signature
from traceback import print_exc
from copy import copy
from typing import Dict, Callable
from .utils import undefined, _
from .error import RegistrationError, CallError, NotAllowedError
from .message import get_messages
from .utils import _
from .error import CallError, NotAllowedError
from .logger import log
from .config import DEBUG
from .context import Context
from . import register
def register(uri: str,
notification: str=undefined):
""" Decorator to register function to the dispatcher
"""
version, uri = uri.split('.', 1)
def decorator(function):
dispatcher.set_function(version,
uri,
notification,
function)
return decorator
class RegisterDispatcher:
def get_function_args(self, function):
# remove self
first_argument_index = 1
return [param.name for param in list(signature(function).parameters.values())[first_argument_index:]]
def _valid_rpc_params(self, version, uri, function, module_name):
""" parameters function must have strictly all arguments with the correct name
"""
def get_message_args():
# load config
config = Config(self.option)
config.property.read_write()
# set message to the uri name
config.option('message').value.set(uri)
# get message argument
subconfig = config.option(uri)
return set(config.option(uri).value.dict().keys())
def get_function_args():
function_args = self.get_function_args(function)
# risotto_context is a special argument, remove it
if function_args and function_args[0] == 'risotto_context':
function_args = function_args[1:]
return set(function_args)
# get message arguments
message_args = get_message_args()
# get function arguments
function_args = get_function_args()
# compare message arguments with function parameter
# it must not have more or less arguments
if message_args != function_args:
# raise if arguments are not equal
msg = []
missing_function_args = message_args - function_args
if missing_function_args:
msg.append(_(f'missing arguments: {missing_function_args}'))
extra_function_args = function_args - message_args
if extra_function_args:
msg.append(_(f'extra arguments: {extra_function_args}'))
function_name = function.__name__
msg = _(' and ').join(msg)
raise RegistrationError(_(f'error with {module_name}.{function_name} arguments: {msg}'))
def _valid_event_params(self, version, uri, function, module_name):
""" parameters function validation for event messages
"""
def get_message_args():
# load config
config = Config(self.option)
config.property.read_write()
# set message to the uri name
config.option('message').value.set(uri)
# get message argument
subconfig = config.option(uri)
return set(config.option(uri).value.dict().keys())
def get_function_args():
function_args = self.get_function_args(function)
# risotto_context is a special argument, remove it
if function_args[0] == 'risotto_context':
function_args = function_args[1:]
return set(function_args)
# get message arguments
message_args = get_message_args()
# get function arguments
function_args = get_function_args()
# compare message arguments with function parameter
# it can have less arguments but not more
extra_function_args = function_args - message_args
if extra_function_args:
# raise if too many arguments
function_name = function.__name__
msg = _(f'extra arguments: {extra_function_args}')
raise RegistrationError(_(f'error with {module_name}.{function_name} arguments: {msg}'))
def set_function(self,
version: str,
uri: str,
notification: str,
function: Callable):
""" register a function to an URI
URI is a message
"""
# xxx module can only be register with v1.xxxx..... message
module_name = function.__module__.split('.')[-2]
uri_namespace = uri.split('.', 1)[0]
if uri_namespace != module_name:
raise RegistrationError(_(f'cannot registered to {uri} message in module {module_name}'))
# check if message exists
try:
if not Config(self.option).option(uri).option.type() == 'message':
raise RegistrationError(_(f'{uri} is not a valid message'))
except AttributeError:
raise RegistrationError(_(f'the message {uri} not exists'))
# create an uris' version if needed
if version not in self.uris:
self.uris[version] = {}
self.function_names[version] = {}
# valid function is unique per module
if module_name not in self.function_names[version]:
self.function_names[version][module_name] = []
function_name = function.__name__
if function_name in self.function_names[version][module_name]:
raise RegistrationError(_(f'multiple registration of {module_name}.{function_name} function'))
self.function_names[version][module_name].append(function_name)
# True if first argument is the risotto_context
function_args = self.get_function_args(function)
if function_args and function_args[0] == 'risotto_context':
inject_risotto_context = True
function_args.pop(0)
else:
inject_risotto_context = False
if self.messages[uri]['pattern'] == 'rpc':
# check if a RPC function is already register for this uri
if uri in self.uris[version]:
raise RegistrationError(_(f'uri {uri} already registered'))
# valid function's arguments
self._valid_rpc_params(version, uri, function, module_name)
# register this function
dico = {'module': module_name,
'function': function,
'risotto_context': inject_risotto_context}
if notification is undefined:
raise RegistrationError(_('notification is mandatory when registered {uri} with {module_name}.{function_name} even if you set None'))
if notification:
dico['notification'] = notification
self.uris[version][uri] = dico
else:
# if event
# valid function's arguments
self._valid_event_params(version, uri, function, module_name)
# register this function
if uri not in self.uris[version]:
self.uris[version][uri] = []
dico = {'module': module_name,
'function': function,
'arguments': function_args,
'risotto_context': inject_risotto_context}
if notification and notification is not undefined:
dico['notification'] = notification
self.uris[version][uri].append(dico)
def set_module(self, module_name, module):
""" register and instanciate a new module
"""
try:
self.injected_self[module_name] = module.Risotto()
except AttributeError as err:
raise RegistrationError(_(f'unable to register the module {module_name}, this module must have Risotto class'))
def validate(self):
""" check if all messages have a function
"""
# FIXME only v1 supported
missing_messages = set(self.messages.keys()) - set(self.uris['v1'].keys())
if missing_messages:
raise RegistrationError(_(f'missing uri {missing_messages}'))
class Dispatcher(RegisterDispatcher):
""" Manage message (call or publish)
so launch a function when a message is called
"""
def __init__(self):
# reference to instanciate module (to inject self in method): {"module_name": instance_of_module}
self.injected_self = {}
# list of uris with informations: {"v1": {"module_name.xxxxx": yyyyyy}}
self.uris = {}
# all function for a module, to avoid conflict name {"v1": {"module_name": ["function_name"]}}
self.function_names = {}
self.messages, self.option = get_messages()
def new_context(self,
context: Context,
version: str,
uri: str):
new_context = Context()
new_context.paths = copy(context.paths)
new_context.paths.append(version + '.' + uri)
new_context.username = context.username
return new_context
def check_public_function(self,
version: str,
uri: str,
context: Context,
class CallDispatcher:
def valid_public_function(self,
risotto_context: Context,
kwargs: Dict,
public_only: bool):
if public_only and not self.messages[uri]['public']:
msg = _(f'the message {version}.{uri} is private')
log.error_msg(version, uri, context, kwargs, 'call', msg)
if public_only and not self.messages[risotto_context.version][risotto_context.message]['public']:
msg = _(f'the message {risotto_context.message} is private')
log.error_msg(risotto_context, kwargs, msg)
raise NotAllowedError(msg)
def check_pattern(self,
version: str,
uri: str,
type: str,
context: Context,
kwargs: Dict):
if self.messages[uri]['pattern'] != type:
msg = _(f'{version}.{uri} is not a {type} message')
log.error_msg(version, uri, context, kwargs, 'call', msg)
raise CallError(msg)
def set_config(self,
uri: str,
kwargs: Dict):
""" create a new Config et set values to it
"""
# create a new config
config = Config(self.option)
config.property.read_write()
# set message option
config.option('message').value.set(uri)
# store values
subconfig = config.option(uri)
for key, value in kwargs.items():
try:
subconfig.option(key).value.set(value)
except AttributeError:
raise AttributeError(_(f'unknown parameter "{key}"'))
# check mandatories options
config.property.read_only()
mandatories = list(config.value.mandatory())
if mandatories:
mand = [mand.split('.')[-1] for mand in mandatories]
raise ValueError(_(f'missing parameters: {mand}'))
# return the config
return config
def valid_call_returns(self,
function: Callable,
risotto_context: Context,
returns: Dict,
version: str,
uri:str,
context: Context,
kwargs: Dict):
if isinstance(returns, dict):
response = self.messages[risotto_context.version][risotto_context.message]['response']
module_name = risotto_context.function.__module__.split('.')[-2]
function_name = risotto_context.function.__name__
if response.impl_get_information('multi'):
if not isinstance(returns, list):
err = _(f'function {module_name}.{function_name} has to return a list')
log.error_msg(risotto_context, kwargs, err)
raise CallError(str(err))
else:
if not isinstance(returns, dict):
err = _(f'function {module_name}.{function_name} has to return a dict')
log.error_msg(risotto_context, kwargs, err)
raise CallError(str(err))
returns = [returns]
if not isinstance(returns, list):
module_name = function.__module__.split('.')[-2]
function_name = function.__name__
err = _(f'function {module_name}.{function_name} has to return a dict or a list')
log.error_msg(version, uri, context, kwargs, 'call', err)
raise CallError(str(err))
response = self.messages[uri]['response']
if response is None:
raise Exception('hu?')
else:
@ -289,69 +49,69 @@ class Dispatcher(RegisterDispatcher):
for key, value in ret.items():
config.option(key).value.set(value)
except AttributeError:
module_name = function.__module__.split('.')[-2]
function_name = function.__name__
err = _(f'function {module_name}.{function_name} return the unknown parameter "{key}"')
log.error_msg(version, uri, context, kwargs, 'call', err)
log.error_msg(risotto_context, kwargs, err)
raise CallError(str(err))
except ValueError:
module_name = function.__module__.split('.')[-2]
function_name = function.__name__
err = _(f'function {module_name}.{function_name} return the parameter "{key}" with an unvalid value "{value}"')
log.error_msg(version, uri, context, kwargs, 'call', err)
log.error_msg(risotto_context, kwargs, err)
raise CallError(str(err))
config.property.read_only()
mandatories = list(config.value.mandatory())
if mandatories:
mand = [mand.split('.')[-1] for mand in mandatories]
raise ValueError(_(f'missing parameters in response: {mand}'))
try:
config.value.dict()
except Exception as err:
module_name = function.__module__.split('.')[-2]
function_name = function.__name__
err = _(f'function {module_name}.{function_name} return an invalid response {err}')
log.error_msg(version, uri, context, kwargs, 'call', err)
log.error_msg(risotto_context, kwargs, err)
raise CallError(str(err))
async def call(self, version, uri, risotto_context, public_only=False, **kwargs):
async def call(self,
version: str,
message: str,
old_risotto_context: Context,
public_only: bool=False,
**kwargs):
""" execute the function associate with specified uri
arguments are validate before
"""
new_context = self.new_context(risotto_context,
version,
uri)
self.check_public_function(version,
uri,
new_context,
risotto_context = self.build_new_context(old_risotto_context,
version,
message,
'rpc')
self.valid_public_function(risotto_context,
kwargs,
public_only)
self.check_pattern(version,
uri,
'rpc',
new_context,
kwargs)
self.check_message_type(risotto_context,
kwargs)
try:
config = self.set_config(uri,
kwargs)
obj = self.uris[version][uri]
kw = config.option(uri).value.dict()
tiramisu_config = self.load_kwargs_to_config(risotto_context,
kwargs)
obj = self.messages[version][message]
kw = tiramisu_config.option(message).value.dict()
risotto_context.function = obj['function']
if obj['risotto_context']:
kw['risotto_context'] = new_context
returns = await obj['function'](self.injected_self[obj['module']], **kw)
kw['risotto_context'] = risotto_context
returns = await risotto_context.function(self.injected_self[obj['module']], **kw)
except CallError as err:
raise err
except Exception as err:
if DEBUG:
print_exc()
log.error_msg(version, uri, new_context, kwargs, 'call', err)
log.error_msg(risotto_context,
kwargs,
err)
raise CallError(str(err))
# valid returns
self.valid_call_returns(obj['function'],
self.valid_call_returns(risotto_context,
returns,
version,
uri,
new_context,
kwargs)
# log the success
log.info_msg(version, uri, new_context, kwargs, 'call', _(f'returns {returns}'))
log.info_msg(risotto_context,
kwargs,
_(f'returns {returns}'))
# notification
if obj.get('notification'):
notif_version, notif_message = obj['notification'].split('.', 1)
@ -362,34 +122,34 @@ class Dispatcher(RegisterDispatcher):
for ret in send_returns:
await self.publish(notif_version,
notif_message,
new_context,
risotto_context,
**ret)
return returns
async def publish(self, version, uri, risotto_context, public_only=False, **kwargs):
new_context = self.new_context(risotto_context,
version,
uri)
self.check_pattern(version,
uri,
'event',
new_context,
kwargs)
class PublishDispatcher:
async def publish(self, version, message, old_risotto_context, public_only=False, **kwargs):
risotto_context = self.build_new_context(old_risotto_context,
version,
message,
'event')
self.check_message_type(risotto_context,
kwargs)
try:
config = self.set_config(uri,
kwargs)
config_arguments = config.option(uri).value.dict()
config = self.load_kwargs_to_config(risotto_context,
kwargs)
config_arguments = config.option(message).value.dict()
except CallError as err:
return
except Exception as err:
# if there is a problem with arguments, just send an error et do nothing
if DEBUG:
print_exc()
log.error_msg(version, uri, new_context, kwargs, 'publish', err)
log.error_msg(risotto_context, kwargs, err)
return
# config is ok, so publish the message
for function_obj in self.uris[version][uri]:
for function_obj in self.messages[version][message]['functions']:
function = function_obj['function']
module_name = function.__module__.split('.')[-2]
function_name = function.__name__
@ -401,26 +161,83 @@ class Dispatcher(RegisterDispatcher):
if key in function_obj['arguments']:
kw[key] = value
if function_obj['risotto_context']:
kw['risotto_context'] = new_context
kw['risotto_context'] = risotto_context
# send event
await function(self.injected_self[function_obj['module']], **kw)
except Exception as err:
if DEBUG:
print_exc()
log.error_msg(version, uri, new_context, kwargs, 'publish', err, info_msg)
log.error_msg(risotto_context, kwargs, err, info_msg)
else:
module_name = function.__module__.split('.')[-2]
function_name = function.__name__
log.info_msg(version, uri, new_context, kwargs,'publish', info_msg)
log.info_msg(risotto_context, kwargs, info_msg)
# notification
if obj.get('notification'):
notif_version, notif_message = obj['notification'].split('.', 1)
await self.publish(notif_version,
notif_message,
new_context,
risotto_context,
**returns)
dispatcher = Dispatcher()
class Dispatcher(register.RegisterDispatcher, CallDispatcher, PublishDispatcher):
""" Manage message (call or publish)
so launch a function when a message is called
"""
def build_new_context(self,
old_risotto_context: Context,
version: str,
message: str,
type: str):
""" This is a new call or a new publish, so create a new context
"""
uri = version + '.' + message
risotto_context = Context()
risotto_context.username = old_risotto_context.username
risotto_context.paths = copy(old_risotto_context.paths)
risotto_context.paths.append(uri)
risotto_context.uri = uri
risotto_context.type = type
risotto_context.message = message
risotto_context.version = version
return risotto_context
def check_message_type(self,
risotto_context: Context,
kwargs: Dict):
if self.messages[risotto_context.version][risotto_context.message]['pattern'] != risotto_context.type:
msg = _(f'{risotto_context.uri} is not a {risotto_context.type} message')
log.error_msg(risotto_context, kwargs, msg)
raise CallError(msg)
def load_kwargs_to_config(self,
risotto_context: Context,
kwargs: Dict):
""" create a new Config et set values to it
"""
# create a new config
config = Config(self.option)
config.property.read_write()
# set message's option
config.option('message').value.set(risotto_context.message)
# store values
subconfig = config.option(risotto_context.message)
for key, value in kwargs.items():
try:
subconfig.option(key).value.set(value)
except AttributeError:
if DEBUG:
print_exc()
raise AttributeError(_(f'unknown parameter "{key}"'))
# check mandatories options
config.property.read_only()
mandatories = list(config.value.mandatory())
if mandatories:
mand = [mand.split('.')[-1] for mand in mandatories]
raise ValueError(_(f'missing parameters: {mand}'))
# return the config
return config
dispatcher = Dispatcher()
register.dispatcher = dispatcher

108
src/risotto/http.py
View File

@ -4,22 +4,67 @@ from json import dumps
from .dispatcher import dispatcher
from .utils import _
from .context import Context
from .error import CallError, NotAllowedError
from .error import CallError, NotAllowedError, RegistrationError
from .message import get_messages
from .logger import log
from .config import DEBUG
from .config import DEBUG, HTTP_PORT
from traceback import print_exc
def create_context(request):
risotto_context = Context()
risotto_context.username = request.match_info.get('username', "Anonymous")
return risotto_context
def register(version: str,
path: str):
""" Decorator to register function to the http route
"""
def decorator(function):
if path in extra_routes:
raise RegistrationError(f'the route {path} is already registered')
extra_routes[path] = {'function': function,
'version': version}
return decorator
class extra_route_handler:
async def __new__(cls, request):
kwargs = dict(request.match_info)
kwargs['request'] = request
kwargs['risotto_context'] = create_context(request)
kwargs['risotto_context'].version = cls.version
kwargs['risotto_context'].paths.append(cls.path)
kwargs['risotto_context'].type = 'http_get'
function_name = cls.function.__module__
# if not 'api' function
if function_name != 'risotto.http':
module_name = function_name.split('.')[-2]
kwargs['self'] = dispatcher.injected_self[module_name]
try:
returns = await cls.function(**kwargs)
except NotAllowedError as err:
raise HTTPNotFound(reason=str(err))
except CallError as err:
raise HTTPBadRequest(reason=str(err))
except Exception as err:
if DEBUG:
print_exc()
raise HTTPInternalServerError(reason=str(err))
log.info_msg(kwargs['risotto_context'],
dict(request.match_info))
return Response(text=dumps(returns))
async def handle(request):
version, uri = request.match_info.get_info()['path'].rsplit('/', 2)[-2:]
context = Context()
context.username = request.match_info.get('username', "Anonymous")
risotto_context = create_context(request)
kwargs = await request.json()
try:
text = await dispatcher.call(version,
uri,
context,
risotto_context,
public_only=True,
**kwargs)
except NotAllowedError as err:
@ -33,45 +78,52 @@ async def handle(request):
return Response(text=dumps({'response': text}))
async def api(request):
context = Context()
context.username = request.match_info.get('username', "Anonymous")
path = request.match_info.get_info()['path']
if path.endswith('/'):
path = path[:-1]
version = path.rsplit('/', 1)[-1]
log.info_msg(version, None, context, {}, None, _(f'get {version} API'))
async def api(request, risotto_context):
global tiramisu
if not tiramisu:
config = Config(get_messages(load_shortarg=True,
only_public=True)[1])
config.property.read_write()
tiramisu = config.option.dict(remotable='none')
return Response(text=dumps(tiramisu))
return tiramisu
def get_app():
extra_routes = {'': {'function': api,
'version': 'v1'}}
async def get_app(loop):
""" build all routes
"""
app = Application()
global extra_routes
app = Application(loop=loop)
routes = []
uris = list(dispatcher.uris.items())
uris.sort()
for version, uris in dispatcher.uris.items():
for version, messages in dispatcher.messages.items():
print()
print(_('======== Registered messages ========'))
for uri in uris:
web_uri = f'/api/{version}/{uri}'
if dispatcher.messages[uri]['public']:
print(f' - {web_uri}')
for message in messages:
web_message = f'/api/{version}/{message}'
if dispatcher.messages[version][message]['public']:
print(f' - {web_message}')
else:
pattern = dispatcher.messages[uri]['pattern']
print(f' - {web_uri} (private {pattern})')
routes.append(post(web_uri, handle))
routes.append(get(f'/api/{version}', api))
pattern = dispatcher.messages[version][message]['pattern']
print(f' - {web_message} (private {pattern})')
routes.append(post(web_message, handle))
print()
print(_('======== Registered extra routes ========'))
for path, extra in extra_routes.items():
version = extra['version']
path = f'/api/{version}{path}'
extra['path'] = path
extra_handler = type(path, (extra_route_handler,), extra)
routes.append(get(path, extra_handler))
print(f' - {path} (http_get)')
# routes.append(get(f'/api/{version}', api))
print()
del extra_routes
app.add_routes(routes)
return app
await dispatcher.on_join()
return await loop.create_server(app.make_handler(), '*', HTTP_PORT)
tiramisu = None

26
src/risotto/logger.py
View File

@ -8,40 +8,38 @@ class Logger:
FIXME should add event to a database
"""
def _get_message_paths(self,
risotto_context: Context,
type: str):
risotto_context: Context):
paths = risotto_context.paths
if len(paths) == 1:
paths_msg = f' messages {type}ed: {paths[0]}'
if risotto_context.type:
paths_msg = f' {risotto_context.type} '
else:
paths_msg = f' sub-messages {type}ed: '
paths_msg = ' '
if len(paths) == 1:
paths_msg += f'message: {paths[0]}'
else:
paths_msg += f'sub-messages: '
paths_msg += ' > '.join(paths)
paths_msg += ':'
return paths_msg
def error_msg(self,
version: 'str',
message: 'str',
risotto_context: Context,
arguments,
type: str,
error: str,
msg: str=''):
""" send message when an error append
"""
paths_msg = self._get_message_paths(risotto_context, type)
print(_(f'{risotto_context.username}: ERROR: {error} ({paths_msg} with arguments "{arguments}" {msg})'))
paths_msg = self._get_message_paths(risotto_context)
print(_(f'{risotto_context.username}: ERROR: {error} ({paths_msg} with arguments "{arguments}": {msg})'))
def info_msg(self,
version: 'str',
message: 'str',
risotto_context: Context,
arguments: Dict,
type: str,
msg: str=''):
""" send message with common information
"""
if risotto_context.paths:
paths_msg = self._get_message_paths(risotto_context, type)
paths_msg = self._get_message_paths(risotto_context)
else:
paths_msg = ''
tmsg = _(f'{risotto_context.username}: INFO:{paths_msg}')

46
src/risotto/message/message.py
View File

@ -4,7 +4,7 @@ from glob import glob
from tiramisu import StrOption, IntOption, BoolOption, ChoiceOption, OptionDescription, SymLinkOption, \
Config, Calculation, Params, ParamOption, ParamValue, calc_value, calc_value_property_help, \
groups
groups, Option
from yaml import load, SafeLoader
from os import listdir
@ -16,6 +16,25 @@ from ..utils import _
groups.addgroup('message')
class DictOption(Option):
__slots__ = tuple()
_type = 'dict'
_display_name = _('dict')
def validate(self, value):
if not isinstance(value, dict):
raise ValueError()
class AnyOption(Option):
__slots__ = tuple()
_type = 'any value'
_display_name = _('any')
def validate(self, value):
pass
class MessageDefinition:
"""
A MessageDefinition is a representation of a message in the Zephir application messaging context
@ -130,11 +149,13 @@ class ResponseDefinition:
'type',
'ref',
'parameters',
'required')
'required',
'multi')
def __init__(self, responses):
self.ref = None
self.parameters = None
self.multi = False
self.required = []
for key, value in responses.items():
if key in ['parameters', 'required']:
@ -142,6 +163,7 @@ class ResponseDefinition:
elif key == 'type':
if value.startswith('[]'):
tvalue = value[2:]
self.multi = True
else:
tvalue = value
if tvalue in customtypes:
@ -414,8 +436,15 @@ def _get_option(name,
if hasattr(arg, 'default'):
kwargs['default'] = arg.default
type_ = arg.type
if type_ == 'Dict' or 'String' in type_ or 'Any' in type_:
if type_.startswith('[]'):
kwargs['multi'] = True
type_ = type_[2:]
if type_ == 'Dict':
return DictOption(**kwargs)
elif type_ == 'String':
return StrOption(**kwargs)
elif type_ == 'Any':
return AnyOption(**kwargs)
elif 'Number' in type_ or type_ == 'ID' or type_ == 'Integer':
return IntOption(**kwargs)
elif type_ == 'Boolean':
@ -440,7 +469,7 @@ def _parse_args(message_def,
for name, arg in new_options.items():
current_opt = _get_option(name, arg, file_path, select_option, optiondescription)
options.append(current_opt)
if arg.shortarg and load_shortarg:
if hasattr(arg, 'shortarg') and arg.shortarg and load_shortarg:
options.append(SymLinkOption(arg.shortarg, current_opt))
@ -473,6 +502,7 @@ def _parse_responses(message_def,
option = {'String': StrOption,
'Number': IntOption,
'Boolean': BoolOption,
'Dict': DictOption,
# FIXME
'File': StrOption}.get(type_)
if not option:
@ -482,9 +512,11 @@ def _parse_responses(message_def,
else:
kwargs['properties'] = ('mandatory',)
options.append(option(**kwargs))
return OptionDescription(message_def.uri,
message_def.response.description,
options)
od = OptionDescription(message_def.uri,
message_def.response.description,
options)
od.impl_set_information('multi', message_def.response.multi)
return od
def _getoptions_from_yml(message_def,

243
src/risotto/register.py Normal file
View File

@ -0,0 +1,243 @@
from tiramisu import Config
from inspect import signature
from typing import Callable, Optional
from .utils import undefined, _
from .error import RegistrationError
from .message import get_messages
from .context import Context
from .config import INTERNAL_USER
def register(uris: str,
notification: str=undefined):
""" Decorator to register function to the dispatcher
"""
if not isinstance(uris, list):
uris = [uris]
def decorator(function):
for uri in uris:
version, message = uri.split('.', 1)
dispatcher.set_function(version,
message,
notification,
function)
return decorator
class RegisterDispatcher:
def __init__(self):
# reference to instanciate module (to inject self in method): {"module_name": instance_of_module}
self.injected_self = {}
# list of uris with informations: {"v1": {"module_name.xxxxx": yyyyyy}}
self.messages = {}
# load tiramisu objects
messages, self.option = get_messages()
#FIXME
version = 'v1'
self.messages[version] = {}
for tiramisu_message, obj in messages.items():
self.messages[version][tiramisu_message] = obj
def get_function_args(self,
function: Callable):
# remove self
first_argument_index = 1
return [param.name for param in list(signature(function).parameters.values())[first_argument_index:]]
def valid_rpc_params(self,
version: str,
message: str,
function: Callable,
module_name: str):
""" parameters function must have strictly all arguments with the correct name
"""
def get_message_args():
# load config
config = Config(self.option)
config.property.read_write()
# set message to the uri name
config.option('message').value.set(message)
# get message argument
subconfig = config.option(message)
return set(config.option(message).value.dict().keys())
def get_function_args():
function_args = self.get_function_args(function)
# risotto_context is a special argument, remove it
if function_args and function_args[0] == 'risotto_context':
function_args = function_args[1:]
return set(function_args)
# get message arguments
message_args = get_message_args()
# get function arguments
function_args = get_function_args()
# compare message arguments with function parameter
# it must not have more or less arguments
if message_args != function_args:
# raise if arguments are not equal
msg = []
missing_function_args = message_args - function_args
if missing_function_args:
msg.append(_(f'missing arguments: {missing_function_args}'))
extra_function_args = function_args - message_args
if extra_function_args:
msg.append(_(f'extra arguments: {extra_function_args}'))
function_name = function.__name__
msg = _(' and ').join(msg)
raise RegistrationError(_(f'error with {module_name}.{function_name} arguments: {msg}'))
def valid_event_params(self,
version: str,
message: str,
function: Callable,
module_name: str):
""" parameters function validation for event messages
"""
def get_message_args():
# load config
config = Config(self.option)
config.property.read_write()
# set message to the message name
config.option('message').value.set(message)
# get message argument
subconfig = config.option(message)
return set(config.option(message).value.dict().keys())
def get_function_args():
function_args = self.get_function_args(function)
# risotto_context is a special argument, remove it
if function_args[0] == 'risotto_context':
function_args = function_args[1:]
return set(function_args)
# get message arguments
message_args = get_message_args()
# get function arguments
function_args = get_function_args()
# compare message arguments with function parameter
# it can have less arguments but not more
extra_function_args = function_args - message_args
if extra_function_args:
# raise if too many arguments
function_name = function.__name__
msg = _(f'extra arguments: {extra_function_args}')
raise RegistrationError(_(f'error with {module_name}.{function_name} arguments: {msg}'))
def set_function(self,
version: str,
message: str,
notification: str,
function: Callable):
""" register a function to an URI
URI is a message
"""
# check if message exists
if message not in self.messages[version]:
raise RegistrationError(_(f'the message {message} not exists'))
# xxx module can only be register with v1.xxxx..... message
module_name = function.__module__.split('.')[-2]
message_namespace = message.split('.', 1)[0]
if self.messages[version][message]['pattern'] == 'rpc' and message_namespace != module_name:
raise RegistrationError(_(f'cannot registered the "{message}" message in module "{module_name}"'))
# True if first argument is the risotto_context
function_args = self.get_function_args(function)
if function_args and function_args[0] == 'risotto_context':
inject_risotto_context = True
function_args.pop(0)
else:
inject_risotto_context = False
# check if already register
if 'function' in self.messages[version][message]:
raise RegistrationError(_(f'uri {version}.{message} already registered'))
# valid function's arguments
if self.messages[version][message]['pattern'] == 'rpc':
if notification is undefined:
raise RegistrationError(_('notification is mandatory when registered {message} with {module_name}.{function_name} even if you set None'))
valid_params = self.valid_rpc_params
else:
valid_params = self.valid_event_params
valid_params(version,
message,
function,
module_name)
# register
if self.messages[version][message]['pattern'] == 'rpc':
register = self.register_rpc
else:
register = self.register_event
register(version,
message,
module_name,
function,
function_args,
inject_risotto_context,
notification)
def register_rpc(self,
version: str,
message: str,
module_name: str,
function: Callable,
function_args: list,
inject_risotto_context: bool,
notification: Optional[str]):
self.messages[version][message]['module'] = module_name
self.messages[version][message]['function'] = function
self.messages[version][message]['arguments'] = function_args
self.messages[version][message]['risotto_context'] = inject_risotto_context
if notification:
self.messages[version][message]['notification'] = notification
def register_event(self,
version: str,
message: str,
module_name: str,
function: Callable,
function_args: list,
inject_risotto_context: bool,
notification: Optional[str]):
if 'functions' not in self.messages[version][message]:
self.messages[version][message]['functions'] = []
dico = {'module': module_name,
'functions': function,
'arguments': function_args,
'risotto_context': inject_risotto_context}
if notification:
dico['notification'] = notification
self.messages[version][message]['functions'].append(dico)
def set_module(self, module_name, module):
""" register and instanciate a new module
"""
try:
self.injected_self[module_name] = module.Risotto()
except AttributeError as err:
raise RegistrationError(_(f'unable to register the module {module_name}, this module must have Risotto class'))
def validate(self):
""" check if all messages have a function
"""
missing_messages = []
for version, messages in self.messages.items():
for message, message_obj in messages.items():
if not 'functions' in message_obj and not 'function' in message_obj:
missing_messages.append(message)
if missing_messages:
raise RegistrationError(_(f'missing uri {missing_messages}'))
async def on_join(self):
for module_name, module in self.injected_self.items():
risotto_context = Context()
risotto_context.username = INTERNAL_USER
risotto_context.paths.append(f'{module_name}.on_join')
risotto_context.type = None
await module.on_join(risotto_context)

849
src/risotto/services/config/config.py
View File

@ -1,300 +1,429 @@
#!/usr/bin/env python3
#import logging
#from lxml.etree import parse
#from io import StringIO
#from autobahn.wamp.exception import ApplicationError
#import asyncio
from tiramisu import Storage, MixConfig, delete_session
#from tiramisu.error import PropertiesOptionError
#
#from os import urandom, unlink
#from os.path import isfile, join
#from binascii import hexlify
#from json import dumps, loads
#from aiohttp.web import HTTPForbidden
from lxml.etree import parse
from io import BytesIO
from os import urandom # , unlink
from os.path import isdir, isfile, join
from binascii import hexlify
from traceback import print_exc
from json import dumps
from typing import Dict, List, Optional, Any
from tiramisu import Storage, list_sessions, delete_session, Config, MetaConfig, MixConfig
from rougail import load as rougail_load
#from creole.loader import PopulateTiramisuObjects
#from zephir.controller import ZephirCommonController, run
#from zephir.http import register as register_http
#from zephir.wamp import register as register_wamp
#from zephir.config import DEBUG
##from eolegenconfig import webapi
#from eolegenconfig.lib import storage
#from eolegenconfig import lib
#from zephir.i18n import _
from ...controller import Controller
from ...dispatcher import register
from ...config import ROOT_CACHE_DIR, DATABASE_DIR
from ...register import register
from ...http import register as register_http
from ...config import ROOT_CACHE_DIR, DATABASE_DIR, DEBUG, ROUGAIL_DTD_PATH
from ...context import Context
from ...utils import _
from ...error import CallError, NotAllowedError, RegistrationError
from ...logger import log
from .storage import storage_server, storage_servermodel
if not isdir(ROOT_CACHE_DIR):
raise RegistrationError(_(f'unable to find the cache dir "{ROOT_CACHE_DIR}"'))
class Risotto(Controller):
servermodel = {}
# FIXME : should be renamed to probe
server = {}
def __init__(self, *args, **kwargs):
# add root and statics
# FIXME
#default_storage.setting(engine='sqlite3', dir_database='/srv/database')
def __init__(self) -> None:
self.save_storage = Storage(engine='sqlite3', dir_database=DATABASE_DIR)
self.modify_storage = Storage(engine='dictionary')
super().__init__(*args, **kwargs)
super().__init__()
def valid_user(self, sessionid, risotto_context):
def valid_user(self,
session_id: str,
risotto_context: Context,
type: str) -> None:
""" check if current user is the session owner
"""
if type == 'server':
storage = storage_server
else:
storage = storage_servermodel
username = risotto_context.username
if username != storage.get_username(sessionid):
raise HTTPForbidden()
if username != storage.get_session(session_id)['username']:
raise NotAllowedError()
async def onJoin(self, *args, **kwargs):
await super().onJoin(*args, **kwargs)
await asyncio.sleep(1)
await self.load_servermodels()
await self.load_servers()
async def on_join(self,
risotto_context: Context) -> None:
""" pre-load servermodel and server
"""
await self.load_servermodels(risotto_context)
# FIXME await self.load_servers(risotto_context)
async def load_servermodels(self):
print('Load servermodels')
try:
servermodels = await self.call('v1.servermodel.list')
except ApplicationError as err:
print(_('cannot load servermodel list: {}').format(str(err)))
return
async def load_servermodels(self,
risotto_context: Context) -> None:
""" load all available servermodels
"""
log.info_msg(risotto_context,
None,
'Load servermodels')
servermodels = await self.call('v1.servermodel.list',
risotto_context)
# load each servermodels
for servermodel in servermodels:
try:
await self.load_servermodel(servermodel['servermodelid'], servermodel['servermodelname'])
except ApplicationError as err:
if DEBUG:
print('Error, cannot load servermodel {}: {}'.format(servermodel['servermodelname'], err))
await self.load_servermodel(risotto_context,
servermodel['servermodelid'],
servermodel['servermodelname'])
except CallError as err:
pass
# do link to this servermodel
for servermodel in servermodels:
if 'servermodelparentsid' in servermodel:
for servermodelparentid in servermodel['servermodelparentsid']:
self.servermodel_legacy(servermodel['servermodelname'], servermodel['servermodelid'], servermodelparentid)
self.servermodel_legacy(servermodel['servermodelname'],
servermodel['servermodelid'],
servermodelparentid)
async def load_servermodel(self, servermodelid, servermodelname):
logging.getLogger().setLevel(logging.INFO)
async def load_servermodel(self,
risotto_context: Context,
servermodelid: int,
servermodelname: str) -> None:
""" Loads a servermodel
"""
cache_file = join(ROOT_CACHE_DIR, str(servermodelid)+".xml")
creolefunc_file = join(ROOT_CACHE_DIR, str(servermodelid)+".creolefuncs")
print('Load servermodel {} ({})'.format(servermodelname, servermodelid))
funcs_file = join(ROOT_CACHE_DIR, str(servermodelid)+".creolefuncs")
log.info_msg(risotto_context,
None,
f'Load servermodel {servermodelname} ({servermodelid})')
# use file in cache if found, otherwise retrieve it in servermodel context
if isfile(cache_file):
fileio = open(cache_file)
else:
servermodel = await self.call('v1.servermodel.describe',
risotto_context,
servermodelid=servermodelid,
inheritance=False,
resolvdepends=False,
schema=True,
creolefuncs=True)
fileio = StringIO()
fileio.write(servermodel['schema'])
fileio = BytesIO()
fileio.write(servermodel['schema'].encode())
fileio.seek(0)
with open(cache_file, 'w') as cache:
cache.write(servermodel['schema'])
with open(creolefunc_file, 'w') as cache:
with open(funcs_file, 'w') as cache:
cache.write(servermodel['creolefuncs'])
del servermodel
# loads tiramisu config and store it
xmlroot = parse(fileio).getroot()
tiramisu_objects = PopulateTiramisuObjects()
tiramisu_objects.parse_dtd('/srv/src/creole/data/creole.dtd')
tiramisu_objects.make_tiramisu_objects(xmlroot, creolefunc_file)
config = tiramisu_objects.build(persistent=True,
session_id='v_{}'.format(servermodelid),
meta_config=True)
self.servermodel[servermodelid] = self.build_metaconfig(servermodelid,
servermodelname,
xmlroot,
funcs_file)
config.owner.set('v_{}'.format(servermodelname))
config.information.set('servermodel_id', servermodelid)
config.information.set('servermodel_name', servermodelname)
def build_metaconfig(self,
servermodelid: int,
servermodelname: str,
xmlroot: str,
funcs_file: str) -> MetaConfig:
""" Build metaconfig for a servermodel
"""
# build tiramisu's session ID
session_id = f'v_{servermodelid}'
optiondescription = rougail_load(xmlroot,
ROUGAIL_DTD_PATH,
funcs_file)
self.servermodel[servermodelid] = config
# build servermodel metaconfig (v_xxx.m_v_xxx)
metaconfig = MetaConfig([],
optiondescription=optiondescription,
persistent=True,
session_id=session_id,
storage=self.save_storage)
mixconfig = MixConfig(children=[],
optiondescription=optiondescription,
persistent=True,
session_id='m_' + session_id,
storage=self.save_storage)
metaconfig.config.add(mixconfig)
def servermodel_legacy(self, servermodel_name, servermodel_id, servermodel_parent_id):
# change default rights
ro_origin = metaconfig.property.getdefault('read_only', 'append')
ro_append = frozenset(ro_origin - {'force_store_value'})
rw_origin = metaconfig.property.getdefault('read_write', 'append')
rw_append = frozenset(rw_origin - {'force_store_value'})
metaconfig.property.setdefault(ro_append, 'read_only', 'append')
metaconfig.property.setdefault(rw_append, 'read_write', 'append')
metaconfig.property.read_only()
metaconfig.permissive.add('basic')
metaconfig.permissive.add('normal')
metaconfig.permissive.add('expert')
# set informtion and owner
metaconfig.owner.set('v_{}'.format(servermodelname))
metaconfig.information.set('servermodel_id', servermodelid)
metaconfig.information.set('servermodel_name', servermodelname)
# return configuration
return metaconfig
def servermodel_legacy(self,
servermodel_name: str,
servermodel_id: int,
servermodel_parent_id: int) -> None:
""" Make link between parent and children
"""
if servermodel_parent_id is None:
return
if not self.servermodel.get(servermodel_parent_id):
if DEBUG:
print(f'Servermodel with id {servermodel_parent_id} not loaded, skipping legacy for servermodel {servermodel_name} ({servermodel_id})')
msg = _(f'Servermodel with id {servermodel_parent_id} not loaded, skipping legacy for servermodel {servermodel_name} ({servermodel_id})')
log.error_msg(risotto_context,
None,
msg)
return
servermodel_parent = self.servermodel[servermodel_parent_id]
servermodel_parent_name = servermodel_parent.information.get('servermodel_name')
if DEBUG:
print(f'Create legacy of servermodel {servermodel_name} ({servermodel_id}) with parent {servermodel_parent_name} ({servermodel_parent_id})')
msg = _(f'Create legacy of servermodel {servermodel_name} ({servermodel_id}) with parent {servermodel_parent_name} ({servermodel_parent_id})')
log.info_msg(risotto_context,
None,
msg)
# do link
mix = servermodel_parent.config.get('m_v_' + str(servermodel_parent_id))
try:
mix.config.add(self.servermodel[servermodel_id])
except Exception as err:
if DEBUG:
print(str(err))
log.error_msg(risotto_context,
None,
str(err))
async def load_servers(self):
print('Load servers')
try:
risotto_context = Context()
risotto_context.username = 'root'
servers = await self.call('v1.server.list', risotto_context)
except ApplicationError as err:
print(_('cannot load server list: {}').format(str(err)))
return
async def load_servers(self,
risotto_context: Context) -> None:
""" load all available servers
"""
log.info_msg(risotto_context,
None,
f'Load servers')
# get all servers
servers = await self.call('v1.server.list',
risotto_context)
# loads servers
for server in servers:
try:
self.load_server(server['serverid'], server['servername'], server['servermodelid'])
await self._load_env(server['serverid'])
self.load_server(server['serverid'],
server['servername'],
server['servermodelid'])
except Exception as err:
print('Unable to load server {} ({}): {}'.format(server['servername'], server['serverid'], err))
servername = server['servername']
serverid = server['serverid']
msg = _(f'Unable to load server {servername} ({serverid}): {err}')
log.error_msg(risotto_context,
None,
msg)
def load_server(self, serverid, servername, servermodelid):
def load_server(self,
risotto_context: Context,
serverid: int,
servername: str,
servermodelid: int) -> None:
""" Loads a server
"""
if serverid in self.server:
return
print('Load server {} ({})'.format(servername, serverid))
log.info_msg(risotto_context,
None,
f'Load server {servername} ({serverid})')
if not servermodelid in self.servermodel:
raise ValueError(f'unable to find servermodel with id {servermodelid}')
metaconfig = self.servermodel[servermodelid].config.new('p_{}'.format(serverid),
persistent=True,
type='metaconfig')
metaconfig.information.set('server_id', serverid)
metaconfig.information.set('server_name', servername)
metaconfig.owner.set('probe')
config = metaconfig.config.new('s_{}'.format(serverid),
persistent=True)
config.owner.set(servername)
config = metaconfig.config.new('std_{}'.format(serverid),
msg = f'unable to find servermodel with id {servermodelid}'
log.error_msg(risotto_context,
None,
msg)
raise CallError(msg)
# check if server was already created
session_id = f's_{serverid}'
is_new_config = session_id not in list_sessions()
# get the servermodel's metaconfig
metaconfig = self.servermodel[servermodelid]
# create server configuration and server 'to deploy' configuration and store it
self.server[serverid] = {'server': self.build_config(session_id,
is_new_config),
'server_to_deploy': self.build_config(f'std_{serverid}',
is_new_config)}
def build_config(self,
session_id: str,
is_new_config: bool) -> None:
""" build server's config
"""
config = metaconfig.config.new(session_id,
persistent=True)
config.information.set('server_id', serverid)
config.information.set('server_name', servername)
config.owner.set(servername)
if 'disabled' not in config.property.get():
# has to be read_only
# if new config, remove force_store_value before switchint to read-only mode
# force_store_value is not allowed for new server (wait when configuration is deploy)
if is_new_config:
ro = list(config.property.getdefault('read_only', 'append'))
if 'force_store_value' in ro:
# force_store_value is not allowed for new server (wait when configuration is deploy)
ro.remove('force_store_value')
config.property.setdefault(frozenset(ro), 'read_only', 'append')
rw = list(config.property.getdefault('read_write', 'append'))
rw.remove('force_store_value')
config.property.setdefault(frozenset(rw), 'read_write', 'append')
config.property.read_only()
ro.remove('force_store_value')
config.property.setdefault(frozenset(ro), 'read_only', 'append')
rw = list(config.property.getdefault('read_write', 'append'))
rw.remove('force_store_value')
config.property.setdefault(frozenset(rw), 'read_write', 'append')
config.property.read_only()
self.server[serverid] = metaconfig
@register('v1.server.created')
async def server_created(self,
serverid: int,
servername: str,
servermodelid: int) -> None:
""" Loads server's configuration when a new server is created
"""
self.load_server(serverid,
servername,
servermodelid)
async def _load_env(self, server_id):
metaconfig = self.server[server_id]
old_informations = {}
for old_information in metaconfig.information.list():
old_informations[old_information] = metaconfig.information.get(old_information)
metaconfig.config.reset()
for old_information, old_value in old_informations.items():
metaconfig.information.set(old_information, old_value)
risotto_context = Context()
risotto_context.username = 'root'
server = await self.call('v1.server.describe', risotto_context=risotto_context, serverid=server_id, environment=True)
for key, value in server['serverenvironment'].items():
metaconfig.unrestraint.option(key).value.set(value)
if server['serverenvironment']:
metaconfig.unrestraint.option('creole.general.available_probes').value.set("oui")
else:
metaconfig.unrestraint.option('creole.general.available_probes').value.set("non")
@register('v1.server.deleted')
async def server_deleted(self,
serverid: int) -> None:
# delete config to it's parents
for config in self.server[serverid].values():
for parent in config.config.parents():
parent.config.pop(config.config.name())
delete_session(config.config.name())
# delete metaconfig
del self.server[serverid]
# @register('v1.server.created', None)
# async def server_created(self, serverid, servername, servermodelid):
# self.load_server(serverid, servername, servermodelid)
#
# @register('v1.server.deleted', None)
# async def server_deleted(self, serverid):
# metaconfig = self.server[serverid]
# # remove config inside metaconfig
# for config in metaconfig.config.list():
# metaconfig.config.pop(config.config.name())
# delete_session(config.config.name())
# del config
# # delete config to parents
# for parent in metaconfig.config.parents():
# parent.config.pop(metaconfig.config.name())
# # delete metaconfig
# delete_session(metaconfig.config.name())
# del self.server[serverid]
# del metaconfig
@register('v1.servermodel.created')
async def servermodel_created(self,
servermodels) -> None:
""" when servermodels are created, load it and do link
"""
for servermodel in servermodels:
await self.load_servermodel(servermodel['servermodelid'], servermodel['servermodelname'])
for servermodel in servermodels:
if 'servermodelparentsid' in servermodel:
for servermodelparentid in servermodel['servermodelparentsid']:
self.servermodel_legacy(servermodel['servermodelname'], servermodel['servermodelid'], servermodelparentid)
# @register('v1.server.environment.updated', "v1.config.configuration.server.updated")
# async def env_updated(self, server_id):
# await self._load_env(server_id)
# self.publish('v1.config.configuration.server.updated', server_id=server_id, deploy=False)
# return {'server_id': server_id, 'deploy': True}
@register('v1.servermodel.updated')
async def servermodel_updated(self,
risotto_context: Context,
servermodels) -> None:
for servermodel in servermodels:
servermodelid = servermodel['servermodelid']
servermodelname = servermodel['servermodelname']
servermodelparentsid = servermodel.get('servermodelparentsid')
log.info_msg(risotto_context,
None,
f'Reload servermodel {servermodelname} ({servermodelid})')
# unlink cache to force download new aggregated file
cache_file = join(ROOT_CACHE_DIR, str(servermodelid)+".xml")
if isfile(cache_file):
unlink(cache_file)
# @register('v1.servermodel.created', None)
# async def servermodel_created(self, servermodels):
# for servermodel in servermodels:
# await self.load_servermodel(servermodel['servermodelid'], servermodel['servermodelname'])
# for servermodel in servermodels:
# if 'servermodelparentsid' in servermodel:
# for servermodelparentid in servermodel['servermodelparentsid']:
# self.servermodel_legacy(servermodel['servermodelname'], servermodel['servermodelid'], servermodelparentid)
#
# @register('v1.servermodel.updated', None)
# async def servermodel_updated(self, servermodels):
# for servermodel in servermodels:
# servermodelid = servermodel['servermodelid']
# servermodelname = servermodel['servermodelname']
# servermodelparentsid = servermodel.get('servermodelparentsid')
# print('Reload servermodel {} ({})'.format(servermodelname, servermodelid))
# # unlink cache to force download new aggregated file
# cache_file = join(ROOT_CACHE_DIR, str(servermodelid)+".xml")
# if isfile(cache_file):
# unlink(cache_file)
# # get current servermodel
# old_servermodel = self.servermodel[servermodelid]
# # create new one
# await self.load_servermodel(servermodelid, servermodelname)
# # migrate all informations
# self.servermodel[servermodelid].value.importation(old_servermodel.value.exportation())
# self.servermodel[servermodelid].permissive.importation(old_servermodel.permissive.exportation())
# self.servermodel[servermodelid].property.importation(old_servermodel.property.exportation())
# # remove link to legacy
# if servermodelparentsid:
# for servermodelparentid in servermodelparentsid:
# mix = self.servermodel[servermodelparentid].config.get('m_v_' + str(servermodelparentid))
# try:
# mix.config.pop(old_servermodel.config.name())
# except:
# # if mix config is reloaded too
# pass
# # add new link
# self.servermodel_legacy(servermodelname, servermodelid, servermodelparentid)
# # load servers in servermodel
# for subconfig in old_servermodel.config.list():
# if not isinstance(subconfig, MixConfig):
# name = subconfig.config.name()
# try:
# old_servermodel.config.pop(name)
# except:
# pass
# server_id = subconfig.information.get('server_id')
# server_name = subconfig.information.get('server_name')
# del self.server[server_id]
# self.load_server(server_id, server_name, servermodelid)
# else:
# for subsubconfig in subconfig.config.list():
# name = subsubconfig.config.name()
# try:
# subconfig.config.pop(name)
# except:
# pass
# self.servermodel_legacy(subsubconfig.information.get('servermodel_name'), subsubconfig.information.get('servermodel_id'), servermodelid)
# get current servermodel
old_servermodel = self.servermodel[servermodelid]
# create new one
await self.load_servermodel(servermodelid, servermodelname)
# migrate all informations
self.servermodel[servermodelid].value.importation(old_servermodel.value.exportation())
self.servermodel[servermodelid].permissive.importation(old_servermodel.permissive.exportation())
self.servermodel[servermodelid].property.importation(old_servermodel.property.exportation())
# remove link to legacy
if servermodelparentsid:
for servermodelparentid in servermodelparentsid:
mix = self.servermodel[servermodelparentid].config.get('m_v_' + str(servermodelparentid))
try:
mix.config.pop(old_servermodel.config.name())
except:
# if mix config is reloaded too
pass
# add new link
self.servermodel_legacy(servermodelname, servermodelid, servermodelparentid)
# reload servers or servermodels in servermodel
for subconfig in old_servermodel.config.list():
if not isinstance(subconfig, MixConfig):
# a server
name = subconfig.config.name()
if name.startswith('str_'):
continue
server_id = subconfig.information.get('server_id')
server_name = subconfig.information.get('server_name')
try:
old_servermodel.config.pop(name)
old_servermodel.config.pop(f'std_{server_id}')
except:
pass
del self.server[server_id]
self.load_server(server_id,
server_name,
servermodelid)
else:
# a servermodel
for subsubconfig in subconfig.config.list():
name = subsubconfig.config.name()
try:
subconfig.config.pop(name)
except:
pass
self.servermodel_legacy(subsubconfig.information.get('servermodel_name'),
subsubconfig.information.get('servermodel_id'),
servermodelid)
@register('v1.config.configuration.server.get', None)
async def get_configuration(self, server_id, deploy):
return {'configuration': (server_id, deploy)}
async def get_configuration(self,
server_id: int,
deploy: bool) -> bytes:
if server_id not in self.server:
msg = _(f'cannot find server with id {server_id}')
log.error_msg(risotto_context,
None,
msg)
raise CallError(msg)
if deploy:
server = self.server[server_id]['server']
else:
server = self.server[server_id]['server_to_deploy']
server.property.read_only()
try:
dico = server.value.dict(fullpath=True)
except:
if deploy:
msg = _(f'No configuration available for server {server_id}')
else:
msg = _(f'No undeployed configuration available for server {server_id}')
log.error_msg(risotto_context,
None,
msg)
raise CallError(msg)
return dumps(dico).encode()
@register('v1.config.configuration.server.deploy', 'v1.config.configuration.server.updated')
async def deploy_configuration(self, server_id):
async def deploy_configuration(self,
server_id: int) -> Dict:
"""Copy values, permissions, permissives from config 'to deploy' to active config
"""
metaconfig = self.server[server_id]
config_std = metaconfig.config("std_{}".format(server_id))
config = self.server[server_id]['server']
config_std = self.server[server_id]['server_to_deploy']
# when deploy, calculate force_store_value
ro = config_std.property.getdefault('read_only', 'append')
if 'force_store_value' not in ro:
ro = frozenset(list(ro) + ['force_store_value'])
@ -304,25 +433,39 @@ class Risotto(Controller):
config_std.property.setdefault(rw, 'read_write', 'append')
config_std.property.add('force_store_value')
config = metaconfig.config("s_{}".format(server_id))
# copy informations from server 'to deploy' configuration to server configuration
config.value.importation(config_std.value.exportation())
config.permissive.importation(config_std.permissive.exportation())
config.property.importation(config_std.property.exportation())
return {'server_id': server_id, 'deploy': True}
# SESSION
#__________________________________________________________________
def get_session(self, session_id, type):
return {'server_id': server_id,
'deploy': True}
def get_session(self,
session_id: str,
type: str) -> Dict:
""" Get session information from storage
"""
if type == 'server':
return storage_server.get_session(session_id)
return storage_servermodel.get_session(session_id)
def get_session_informations(self, session_id, type):
session = self.get_session(session_id, type)
return self.format_session(session_id, session)
def get_session_informations(self,
session_id: str,
type: str) -> Dict:
""" format session with a session ID name
"""
session = self.get_session(session_id,
type)
return self.format_session(session_id,
session)
def format_session(self, session_name, session):
return {'sessionid': session_name,
def format_session(self,
session_name: str,
session: Dict) -> Dict:
""" format session
"""
return {'session_id': session_name,
'id': session['id'],
'username': session['username'],
'timestamp': session['timestamp'],
@ -330,71 +473,79 @@ class Risotto(Controller):
'mode': session['mode'],
'debug': session['debug']}
def list_sessions(self, type):
def list_sessions(self,
type: str) -> List:
ret = []
if type == 'server':
storage = storage_server
else:
storage = storage_servermodel
for session in storage.list_sessions():
ret.append(self.format_session(session['sessionid'], session))
ret.append(self.format_session(session['session_id'], session))
return ret
def load_dict(self, session):
def load_dict(self,
session: Dict) -> Dict:
if not session['option']:
session['option'] = session['config'].option(session['namespace'])
return session['option'].dict(remotable='all')
# start
async def start_session(self, risotto_context, id, type, server_list):
@register(['v1.config.session.server.start', 'v1.config.session.servermodel.start'], None)
async def start_session(self,
risotto_context: Context,
id: int) -> Dict:
""" start a new config session for a server or a servermodel
"""
type = risotto_context.message.rsplit('.', 2)[-2]
server_list = getattr(self, type)
if id not in server_list:
raise Exception(_(f'cannot find {type} with id {id}'))
session_id = ''
# check if a session already exists, in this case returns it
session_list = self.list_sessions(type)
for sess in session_list:
if sess['id'] == id and sess['username'] == risotto_context.username:
session_id = sess['sessionid']
session_id = sess['session_id']
session = self.get_session(session_id, type)
return self.format_session(session_id, session)
# create a new session
if type == 'server':
storage = storage_server
else:
storage = storage_servermodel
while True:
session_id = 'z' + hexlify(urandom(23)).decode()
if not storage.has_session(session_id):
break
else:
session_id = ''
print('session {} already exists'.format(session_id))
username = risotto_context.username
storage.add_session(session_id,
server_list[id],
id,
username,
self.modify_storage)
return self.get_session_informations(session_id,
type)
if session_id == '':
if type == 'server':
storage = storage_server
else:
storage = storage_servermodel
while True:
session_id = 'z' + hexlify(urandom(23)).decode()
if not storage.has_session(session_id):
break
else:
print('session {} already exists'.format(session_id))
username = risotto_context.username
storage.add_session(session_id, server_list[id], type, id, username, self.modify_storage)
return self.get_session_informations(session_id, type)
@register(['v1.config.session.server.list', 'v1.config.session.servermodel.list'], None)
async def list_session_server(self,
risotto_context: Context):
type = risotto_context.message.rsplit('.', 2)[-2]
return self.list_sessions(type)
@register('v1.config.session.server.start', None)
async def start_session_server(self, risotto_context, id):
return await self.start_session(risotto_context, id, 'server', self.server)
@register('v1.config.session.servermodel.start', None)
async def start_session_servermodel(self, risotto_context, id):
return await self.start_session(risotto_context, id, 'servermodel', self.servermodel)
# list
@register('v1.config.session.server.list', None)
async def list_session_server(self):
return self.list_sessions('server')
@register('v1.config.session.servermodel.list', None)
async def list_session_servermodel(self):
return self.list_sessions('servermodel')
# filter
async def filter_session(self, session_id, type, namespace, mode, debug):
session = self.get_session(session_id, type)
@register(['v1.config.session.server.filter', 'v1.config.session.servermodel.filter'], None)
async def filter_session(self,
risotto_context: Context,
session_id: str,
namespace: str,
mode: str,
debug: Optional[bool]):
type = risotto_context.message.rsplit('.', 2)[-2]
session = self.get_session(session_id,
type)
if namespace is not None:
session['option'] = None
session['namespace'] = namespace
@ -404,27 +555,34 @@ class Risotto(Controller):
storage = storage_servermodel
if mode is not None:
if mode not in ('basic', 'normal', 'expert'):
raise Exception(f'unknown mode {mode}')
storage.set_config_mode(session_id, mode)
raise CallError(f'unknown mode {mode}')
storage.set_config_mode(session_id,
mode)
if debug is not None:
storage.set_config_debug(session_id, debug)
return self.get_session_informations(session_id, type)
storage.set_config_debug(session_id,
debug)
return self.get_session_informations(session_id,
type)
@register('v1.config.session.server.filter', None)
async def filter_session_server(self, session_id, namespace, mode, debug):
return await self.filter_session(session_id, 'server', namespace, mode, debug)
@register('v1.config.session.servermodel.filter', None)
async def filter_session_servermodel(self, session_id, namespace, mode, debug):
return await self.filter_session(session_id, 'servermodel', namespace, mode, debug)
# configure
async def configure_session(self, session_id, type, action, name, index, value):
session = self.get_session(session_id, type)
@register(['v1.config.session.server.configure', 'v1.config.session.servermodel.configure'], None)
async def configure_session(self,
risotto_context: Context,
session_id: str,
action: str,
name: str,
index: int,
value: Any,
value_multi: Optional[List]) -> Dict:
type = risotto_context.message.rsplit('.', 2)[-2]
session = self.get_session(session_id,
type)
ret = {'session_id': session_id,
'name': name}
if index is not None:
ret['index'] = index
option = session['config'].option(name).option
if option.ismulti() and not option.isfollower():
value = value_multi
try:
update = {'name': name,
'action': action,
@ -438,22 +596,17 @@ class Risotto(Controller):
session['option'].updates(updates)
ret['status'] = 'ok'
except Exception as err:
import traceback
traceback.print_exc()
if DEBUG:
print_exc()
ret['message'] = str(err)
ret['status'] = 'error'
return ret
@register('v1.config.session.server.configure', None)
async def configure_session_server(self, session_id, action, name, index, value):
return await self.configure_session(session_id, 'server', action, name, index, value)
@register('v1.config.session.servermodel.configure', None)
async def configure_session_servermodel(self, session_id, action, name, index, value):
return await self.configure_session(session_id, 'servermodel', action, name, index, value)
# validate
async def validate_session(self, session_id, type):
@register(['v1.config.session.server.validate', 'v1.config.session.servermodel.validate'], None)
async def validate_session(self,
risotto_context: Context,
session_id: str) -> Dict:
type = risotto_context.message.rsplit('.', 2)[-2]
session = self.get_session(session_id, type)
ret = {}
try:
@ -473,64 +626,72 @@ class Risotto(Controller):
ret['status'] = 'ok'
return ret
@register('v1.config.session.server.validate', None)
async def validate_session_server(self, session_id):
return await self.validate_session(session_id, 'server')
@register('v1.config.session.servermodel.validate', None)
async def validate_session_servermodel(self, session_id):
return await self.validate_session(session_id, 'servermodel')
# get
async def get_session_(self, session_id, type):
info = self.get_session_informations(session_id, type)
@register(['v1.config.session.server.get', 'v1.config.session.servermodel.get'], None)
async def get_session_(self,
risotto_context: Context,
session_id: str) -> Dict:
type = risotto_context.message.rsplit('.', 2)[-2]
info = self.get_session_informations(session_id,
type)
info['content'] = session_id
session = self.get_session(session_id,
type)
if not session['option']:
session['option'] = session['config'].option(session['namespace'])
info['content'] = dumps(session['option'].value.dict(fullpath=True))
return info
@register('v1.config.session.server.get', None)
async def get_session_server(self, session_id):
return await self.get_session_(session_id, 'server')
@register('v1.config.session.servermodel.get', None)
async def get_session_servermodel(self, session_id):
return await self.get_session_(session_id, 'servermodel')
# stop
async def stop_session(self, risotto_context, session_id, type, save):
session = self.get_session(session_id, type)
if save:
await self._post_save_config(risotto_context, None, session_id)
@register(['v1.config.session.server.stop', 'v1.config.session.servermodel.stop'], None)
async def stop_session(self,
risotto_context: Context,
session_id: str,
save: bool) -> Dict:
type = risotto_context.message.rsplit('.', 2)[-2]
self.valid_user(session_id,
risotto_context,
type)
session = self.get_session(session_id,
type)
id_ = session['id']
if type == 'server':
storage = storage_server
if save:
storage.save_values(session_id)
if self.server[id_].option('creole.general.available_probes').value.get() == "oui":
self.publish('v1.config.configuration.server.updated', server_id=id_, deploy=False)
else:
storage = storage_servermodel
storage.del_session(session_id, type)
if save:
storage.save_values(session_id)
for probe in self.servermodel[id_].config.list():
# FIXME should use config.information.get('server_id')
name = probe.config.name()
if name.startswith('p_'):
server_id = int(name.rsplit('_', 1)[-1])
if self.server[server_id].option('creole.general.available_probes').value.get() == "oui":
self.publish('v1.config.configuration.server.updated', server_id=server_id)
storage.del_session(session_id)
return self.format_session(session_id, session)
@register('v1.config.session.server.stop', None)
async def stop_session_server(self, risotto_context, sessionid, save):
return await self.stop_session(sessionid, 'server', save)
@register_http('v1', '/config/server/{session_id}')
async def get_server_api(self,
request,
risotto_context: Context,
session_id: str) -> Dict:
self.valid_user(session_id,
risotto_context,
'server')
session = storage_server.get_session(session_id)
return self.load_dict(session)
@register('v1.config.session.servermodel.stop', None)
async def stop_session_servermodel(self, risotto_context, sessionid, save):
return await self.stop_session(risotto_context, sessionid, 'servermodel', save)
# GEN_CONFIG
#__________________________________________________________________
async def _post_save_config(self, risotto_context, request, sessionid):
self.valid_user(sessionid, risotto_context)
lib.save_values(sessionid, 'save')
id_ = storage.get_id(sessionid)
if storage.get_type(sessionid) == 'server':
if self.server[id_].option('creole.general.available_probes').value.get() == "oui":
self.publish('v1.config.configuration.server.updated', server_id=id_, deploy=False)
else:
for probe in self.servermodel[id_].config.list():
# FIXME should use config.information.get('server_id')
name = probe.config.name()
if name.startswith('p_'):
server_id = int(name.rsplit('_', 1)[-1])
if self.server[server_id].option('creole.general.available_probes').value.get() == "oui":
self.publish('v1.config.configuration.server.updated', server_id=server_id)
return {}
@register_http('v1', '/config/servermodel/{session_id}')
async def get_servermodel_api(self,
request,
risotto_context: Context,
session_id: str) -> Dict:
self.valid_user(session_id,
risotto_context,
'servermodel')
session = storage_servermodel.get_session(session_id)
return self.load_dict(session)

51
src/risotto/services/config/storage.py
View File

@ -1,3 +1,7 @@
import time
from rougail import modes
class StorageError(Exception):
pass
@ -14,12 +18,12 @@ class Storage(object):
def config_exists(self, id_):
return self.sessions[id_]['config_exists']
def add_session(self, sessionid, orig_config, server_id, username, storage):
def add_session(self, session_id, orig_config, server_id, username, storage):
for session in self.sessions.values():
if session['id'] == server_id:
raise Storage(_(f'{username} already edits this configuration'))
prefix_id = "{}_".format(sessionid)
config_server, orig_config = self.transform_orig_config(orig_config)
prefix_id = "{}_".format(session_id)
config_server, orig_config = self.transform_orig_config(orig_config, server_id)
config_id = "{}{}".format(prefix_id, config_server)
meta = orig_config.config.deepcopy(session_id=config_id, storage=storage, metaconfig_prefix=prefix_id)
config = meta
@ -33,7 +37,7 @@ class Storage(object):
else:
break
config.property.read_write()
self.set_owner(self, config)
self.set_owner(config, username)
orig_values = config.value.exportation()
config.information.set('orig_values', orig_values)
config_exists = False
@ -45,22 +49,22 @@ class Storage(object):
elif owner != 'forced':
config_exists = True
break
self.sessions[sessionid] = {'config': config,
self.sessions[session_id] = {'config': config,
# do not delete meta, so keep it!
'meta': meta,
'orig_config': orig_config,
'id': server_id,
'timestamp': time.time(),
'timestamp': int(time.time()),
'username': username,
'option': None,
'namespace': 'creole',
'config_exists': config_exists}
self.set_config_mode(sessionid, 'normal')
self.set_config_debug(sessionid, False)
self.set_config_mode(session_id, 'normal')
self.set_config_debug(session_id, False)
def list_sessions(self):
for sessionid, session in self.sessions.items():
yield {'sessionid': sessionid,
for session_id, session in self.sessions.items():
yield {'session_id': session_id,
'id': session['id'],
'timestamp': session['timestamp'],
'username': session['username'],
@ -73,7 +77,7 @@ class Storage(object):
def get_session(self, id_):
if id_ not in self.sessions:
raise GenConfigError('please start a session before')
raise Exception('please start a session before')
return self.sessions[id_]
def save_values(self, id_):
@ -89,17 +93,8 @@ class Storage(object):
def get_username(self, id_):
return self.get_session(id_)['username']
def get_id(self, id_):
return self.get_session(id_)['id']
def set_config_mode(self, id_, mode):
""" Define which edition mode to select
:param id_: session id
:type id_: `str`
:param mode: possible values = ['basic', 'normal', 'expert']
:type mode: `str`
:returns: session mode value
:type :`bool`
"""
config = self.get_session(id_)['config']
for mode_level in modes.values():
@ -107,17 +102,10 @@ class Storage(object):
config.property.add(mode_level.name)
else:
config.property.pop(mode_level.name)
# store mode in session in case config object gets reloader
self.sessions[id_]['mode'] = mode
def set_config_debug(self, id_, is_debug):
""" Enable/Disable debug mode
:param id_: session id
:type id_: `str`
:param is_debug: True to enable debug mode
:type is_debug: `bool`
:returns: session debug value
:type :`bool`
"""
config = self.get_session(id_)['config']
if is_debug:
@ -125,25 +113,24 @@ class Storage(object):
else:
config.property.add('hidden')
self.sessions[id_]['debug'] = is_debug
return is_debug
class StorageServer(Storage):
def transform_orig_config(self, orig_config):
def transform_orig_config(self, orig_config, server_id):
config_server = "std_{}".format(server_id)
orig_config = orig_config.config(config_server)
return config_server, orig_config
def set_owner(self, config):
def set_owner(self, config, username):
config.owner.set(username)
class StorageServermodel(Storage):
def transform_orig_config(self, orig_config):
def transform_orig_config(self, orig_config, server_id):
config_server = "v_{}".format(server_id)
return config_server, orig_config
def set_owner(self, config):
def set_owner(self, config, username):
config.owner.set('servermodel_' + username)