Infra
/
risotto
9
0
Fork 1
Code Issues 1 Releases Wiki Activity

publish use now postgresql

This commit is contained in:
Emmanuel Garette 2020-09-16 17:37:46 +02:00
parent c84b9435b0
commit 7afccab9b1
4 changed files with 165 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/risotto/controller.py
View File

@ -39,9 +39,9 @@ class Controller:
**kwargs, **kwargs,
): ):
""" a wrapper to dispatcher's publish""" """ a wrapper to dispatcher's publish"""
version, message = uri.split('.', 1)
if args: if args:
raise ValueError(_(f'the URI "{uri}" can only be published with keyword arguments')) raise ValueError(_(f'the URI "{uri}" can only be published with keyword arguments'))
version, message = uri.split('.', 1)
await dispatcher.publish(version, await dispatcher.publish(version,
message, message,
risotto_context, risotto_context,

289
src/risotto/dispatcher.py
View File

@ -4,6 +4,7 @@ try:
except: except:
from tiramisu import Config from tiramisu import Config
from tiramisu.error import ValueOptionError from tiramisu.error import ValueOptionError
from asyncio import get_event_loop, ensure_future
from traceback import print_exc from traceback import print_exc
from copy import copy from copy import copy
from typing import Dict, Callable, List, Optional from typing import Dict, Callable, List, Optional
@ -15,7 +16,6 @@ from .logger import log
from .config import get_config from .config import get_config
from .context import Context from .context import Context
from . import register from . import register
from .remote import Remote
class CallDispatcher: class CallDispatcher:
@ -79,7 +79,7 @@ class CallDispatcher:
""" execute the function associate with specified uri """ execute the function associate with specified uri
arguments are validate before arguments are validate before
""" """
risotto_context = self.build_new_context(old_risotto_context, risotto_context = self.build_new_context(old_risotto_context.__dict__,
version, version,
message, message,
'rpc', 'rpc',
@ -88,20 +88,35 @@ class CallDispatcher:
raise CallError(_(f'cannot find version of message "{version}"')) raise CallError(_(f'cannot find version of message "{version}"'))
if message not in self.messages[version]: if message not in self.messages[version]:
raise CallError(_(f'cannot find message "{version}.{message}"')) raise CallError(_(f'cannot find message "{version}.{message}"'))
function_objs = [self.messages[version][message]] function_obj = self.messages[version][message]
# do not start a new database connection # do not start a new database connection
if hasattr(old_risotto_context, 'connection'): if hasattr(old_risotto_context, 'connection'):
risotto_context.connection = old_risotto_context.connection risotto_context.connection = old_risotto_context.connection
return await self.launch(version, await self.check_message_type(risotto_context,
message, kwargs,
risotto_context, )
check_role, config_arguments = await self.load_kwargs_to_config(risotto_context,
f'{version}.{message}',
kwargs,
check_role,
internal,
)
return await self.launch(risotto_context,
kwargs, kwargs,
function_objs, config_arguments,
internal, function_obj,
) )
else: else:
try: try:
await self.check_message_type(risotto_context,
kwargs,
)
config_arguments = await self.load_kwargs_to_config(risotto_context,
f'{version}.{message}',
kwargs,
check_role,
internal,
)
async with self.pool.acquire() as connection: async with self.pool.acquire() as connection:
await connection.set_type_codec( await connection.set_type_codec(
'json', 'json',
@ -111,13 +126,10 @@ class CallDispatcher:
) )
risotto_context.connection = connection risotto_context.connection = connection
async with connection.transaction(): async with connection.transaction():
return await self.launch(version, return await self.launch(risotto_context,
message,
risotto_context,
check_role,
kwargs, kwargs,
function_objs, config_arguments,
internal, function_obj,
) )
except CallError as err: except CallError as err:
raise err raise err
@ -139,66 +151,81 @@ class CallDispatcher:
class PublishDispatcher: class PublishDispatcher:
async def register_remote(self) -> None:
print()
print(_('======== Registered remote event ========'))
self.listened_connection = await self.pool.acquire()
for version, messages in self.messages.items():
for message, message_infos in messages.items():
# event not emit locally
if message_infos['pattern'] == 'event' and 'functions' in message_infos and message_infos['functions']:
# module, submodule, submessage = message.split('.', 2)
# if f'{module}.{submodule}' not in self.injected_self:
uri = f'{version}.{message}'
print(f' - {uri}')
await self.listened_connection.add_listener(uri,
self.to_async_publish,
)
async def publish(self, async def publish(self,
version: str, version: str,
message: str, message: str,
old_risotto_context: Context, risotto_context: Context,
check_role: bool=False,
internal: bool=True,
**kwargs, **kwargs,
) -> None: ) -> None:
risotto_context = self.build_new_context(old_risotto_context, if version not in self.messages or message not in self.messages[version]:
raise ValueError(_(f'cannot find URI "{version}.{message}"'))
# publish to remote
remote_kw = dumps({'kwargs': kwargs,
'context': {'username': risotto_context.username,
'paths': risotto_context.paths,
}
})
# FIXME should be better :/
remote_kw = remote_kw.replace("'", "''")
await risotto_context.connection.execute(f'NOTIFY "{version}.{message}", \'{remote_kw}\'')
def to_async_publish(self,
con: 'asyncpg.connection.Connection',
pid: int,
uri: str,
payload: str,
) -> None:
version, message = uri.split('.', 1)
loop = get_event_loop()
remote_kw = loads(payload)
risotto_context = self.build_new_context(remote_kw['context'],
version, version,
message, message,
'event', 'event',
) )
try: callback = lambda: ensure_future(self._publish(version,
function_objs = self.messages[version][message].get('functions', []) message,
except KeyError: risotto_context,
raise ValueError(_(f'cannot find message {version}.{message}')) **remote_kw['kwargs'],
# do not start a new database connection ))
if hasattr(old_risotto_context, 'connection'): loop.call_soon(callback)
# publish to remove
remote_kw = dumps({'kwargs': kwargs, async def _publish(self,
'context': risotto_context.__dict__, version: str,
}) message: str,
risotto_context.connection = old_risotto_context.connection risotto_context: Context,
# FIXME should be better :/ **kwargs,
remote_kw = remote_kw.replace("'", "''") ) -> None:
await risotto_context.connection.execute(f'NOTIFY "{version}.{message}", \'{remote_kw}\'') config_arguments = await self.load_kwargs_to_config(risotto_context,
return await self.launch(version, f'{version}.{message}',
message, kwargs,
risotto_context, False,
check_role, False,
kwargs, )
function_objs, for function_obj in self.messages[version][message]['functions']:
internal, print('======', function_obj['function'].__name__)
) async with self.pool.acquire() as connection:
async with self.pool.acquire() as connection: try:
try: await self.check_message_type(risotto_context,
await connection.set_type_codec( kwargs,
'json', )
encoder=dumps,
decoder=loads,
schema='pg_catalog'
)
risotto_context.connection = connection
async with connection.transaction():
return await self.launch(version,
message,
risotto_context,
check_role,
kwargs,
function_objs,
internal,
)
except CallError as err:
pass
except Exception as err:
# if there is a problem with arguments, log and do nothing
if get_config()['global']['debug']:
print_exc()
async with self.pool.acquire() as connection:
await connection.set_type_codec( await connection.set_type_codec(
'json', 'json',
encoder=dumps, encoder=dumps,
@ -207,18 +234,37 @@ class PublishDispatcher:
) )
risotto_context.connection = connection risotto_context.connection = connection
async with connection.transaction(): async with connection.transaction():
await log.error_msg(risotto_context, kwargs, err) await self.launch(risotto_context,
kwargs,
config_arguments,
function_obj,
)
except CallError as err:
pass
except Exception as err:
# if there is a problem with arguments, log and do nothing
if get_config()['global']['debug']:
print_exc()
async with self.pool.acquire() as connection:
await connection.set_type_codec(
'json',
encoder=dumps,
decoder=loads,
schema='pg_catalog'
)
risotto_context.connection = connection
async with connection.transaction():
await log.error_msg(risotto_context, kwargs, err)
class Dispatcher(register.RegisterDispatcher, class Dispatcher(register.RegisterDispatcher,
Remote,
CallDispatcher, CallDispatcher,
PublishDispatcher): PublishDispatcher):
""" Manage message (call or publish) """ Manage message (call or publish)
so launch a function when a message is called so launch a function when a message is called
""" """
def build_new_context(self, def build_new_context(self,
old_risotto_context: Context, context: dict,
version: str, version: str,
message: str, message: str,
type: str, type: str,
@ -227,8 +273,8 @@ class Dispatcher(register.RegisterDispatcher,
""" """
uri = version + '.' + message uri = version + '.' + message
risotto_context = Context() risotto_context = Context()
risotto_context.username = old_risotto_context.username risotto_context.username = context['username']
risotto_context.paths = copy(old_risotto_context.paths) risotto_context.paths = copy(context['paths'])
risotto_context.paths.append(uri) risotto_context.paths.append(uri)
risotto_context.uri = uri risotto_context.uri = uri
risotto_context.type = type risotto_context.type = type
@ -342,65 +388,56 @@ class Dispatcher(register.RegisterDispatcher,
raise NotAllowedError(_(f'You ({user_login}) don\'t have any authorisation to access to "{uri}"')) raise NotAllowedError(_(f'You ({user_login}) don\'t have any authorisation to access to "{uri}"'))
async def launch(self, async def launch(self,
version: str,
message: str,
risotto_context: Context, risotto_context: Context,
check_role: bool,
kwargs: Dict, kwargs: Dict,
function_objs: List, config_arguments: dict,
internal: bool, function_obj: Callable,
) -> Optional[Dict]: ) -> Optional[Dict]:
await self.check_message_type(risotto_context, # so send the message
kwargs) function = function_obj['function']
config_arguments = await self.load_kwargs_to_config(risotto_context, submodule_name = function_obj['module']
f'{version}.{message}', function_name = function.__name__
kwargs, risotto_context.module = submodule_name.split('.', 1)[0]
check_role, info_msg = _(f'in module {submodule_name}.{function_name}')
internal, # build argument for this function
) if risotto_context.type == 'rpc':
# config is ok, so send the message kw = config_arguments
for function_obj in function_objs: else:
function = function_obj['function'] kw = {}
submodule_name = function_obj['module'] for key, value in config_arguments.items():
function_name = function.__name__ if key in function_obj['arguments']:
risotto_context.module = submodule_name.split('.', 1)[0] kw[key] = value
info_msg = _(f'in module {submodule_name}.{function_name}')
# build argument for this function
if risotto_context.type == 'rpc':
kw = config_arguments
else:
kw = {}
for key, value in config_arguments.items():
if key in function_obj['arguments']:
kw[key] = value
kw['risotto_context'] = risotto_context kw['risotto_context'] = risotto_context
returns = await function(self.injected_self[function_obj['module']], **kw) returns = await function(self.injected_self[function_obj['module']], **kw)
if risotto_context.type == 'rpc': if risotto_context.type == 'rpc':
# valid returns # valid returns
await self.valid_call_returns(risotto_context, await self.valid_call_returns(risotto_context,
function, function,
returns, returns,
kwargs) kwargs,
# log the success )
await log.info_msg(risotto_context, # log the success
{'arguments': kwargs, await log.info_msg(risotto_context,
'returns': returns}, {'arguments': kwargs,
info_msg) 'returns': returns},
# notification info_msg,
if function_obj.get('notification'): )
notif_version, notif_message = function_obj['notification'].split('.', 1) # notification
if not isinstance(returns, list): if function_obj.get('notification'):
send_returns = [returns] notif_version, notif_message = function_obj['notification'].split('.', 1)
else: if not isinstance(returns, list):
send_returns = returns send_returns = [returns]
for ret in send_returns: else:
await self.publish(notif_version, send_returns = returns
notif_message, for ret in send_returns:
risotto_context, await self.publish(notif_version,
**ret) notif_message,
if risotto_context.type == 'rpc': risotto_context,
return returns **ret,
)
if risotto_context.type == 'rpc':
return returns
dispatcher = Dispatcher() dispatcher = Dispatcher()

2
src/risotto/register.py
View File

@ -300,7 +300,7 @@ class RegisterDispatcher:
) )
if truncate: if truncate:
async with connection.transaction(): async with connection.transaction():
await connection.execute('TRUNCATE InfraServer, InfraSite, InfraZone, Log, ProviderDeployment, ProviderFactoryCluster, ProviderFactoryClusterNode, SettingApplicationservice, SettingApplicationServiceDependency, SettingRelease, SettingServer, SettingServermodel, SettingSource, UserRole, UserRoleURI, UserURI, UserUser, InfraServermodel, ProviderZone, ProviderServer, ProviderSource, ProviderApplicationservice ProviderServermodel') await connection.execute('TRUNCATE InfraServer, InfraSite, InfraZone, Log, ProviderDeployment, ProviderFactoryCluster, ProviderFactoryClusterNode, SettingApplicationservice, SettingApplicationServiceDependency, SettingRelease, SettingServer, SettingServermodel, SettingSource, UserRole, UserRoleURI, UserURI, UserUser, InfraServermodel, ProviderZone, ProviderServer, ProviderSource, ProviderApplicationservice, ProviderServermodel')
async with connection.transaction(): async with connection.transaction():
for submodule_name, module in self.injected_self.items(): for submodule_name, module in self.injected_self.items():
risotto_context = Context() risotto_context = Context()

42
src/risotto/remote.py
View File

@ -1,42 +0,0 @@
from asyncio import get_event_loop, ensure_future
from json import loads
from .context import Context
from .config import get_config
from .utils import _
class Remote:
async def register_remote(self) -> None:
print()
print(_('======== Registered remote event ========'))
self.listened_connection = await self.pool.acquire()
for version, messages in self.messages.items():
for message, message_infos in messages.items():
# event not emit locally
if message_infos['pattern'] == 'event':
module, submodule, submessage = message.split('.', 2)
if f'{module}.{submodule}' not in self.injected_self:
uri = f'{version}.{message}'
print(f' - {uri}')
await self.listened_connection.add_listener(uri, self.to_async_publish)
def to_async_publish(self,
con: 'asyncpg.connection.Connection',
pid: int,
uri: str,
payload: str,
) -> None:
version, message = uri.split('.', 1)
loop = get_event_loop()
remote_kw = loads(payload)
context = Context()
for key, value in remote_kw['context'].items():
setattr(context, key, value)
callback = lambda: ensure_future(self.publish(version,
message,
context,
**remote_kw['kwargs'],
))
loop.call_soon(callback)