Infra/risotto
9
0
Fork 1
Code Issues 1 Releases Wiki Activity

on connection to database to log only

Browse Source
This commit is contained in:
Emmanuel Garette 2021-04-25 20:32:02 +02:00
parent ed51bc483d
commit 1063d2e735
4 changed files with 191 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/risotto/context.py
View File

@ -3,3 +3,11 @@ class Context:
self.paths = [] self.paths = []
self.context_id = None self.context_id = None
self.start_id = None self.start_id = None
def copy(self):
context = Context()
for key, value in self.__dict__.items():
if key.startswith('__'):
continue
setattr(context, key, value)
return context

247
src/risotto/dispatcher.py
View File

@ -89,7 +89,6 @@ class CallDispatcher:
if hasattr(old_risotto_context, 'connection'): if hasattr(old_risotto_context, 'connection'):
# do not start a new database connection # do not start a new database connection
risotto_context.connection = old_risotto_context.connection risotto_context.connection = old_risotto_context.connection
risotto_context.log_connection = old_risotto_context.log_connection
await log.start(risotto_context, await log.start(risotto_context,
kwargs, kwargs,
info_msg, info_msg,
@ -119,66 +118,58 @@ class CallDispatcher:
raise CallError(err) from err raise CallError(err) from err
else: else:
error = None error = None
async with self.pool.acquire() as log_connection: try:
await log_connection.set_type_codec( async with self.pool.acquire() as connection:
'json', await connection.set_type_codec(
encoder=dumps, 'json',
decoder=loads, encoder=dumps,
schema='pg_catalog' decoder=loads,
) schema='pg_catalog'
risotto_context.log_connection = log_connection )
try: risotto_context.connection = connection
async with self.pool.acquire() as connection: async with connection.transaction():
await connection.set_type_codec( try:
'json', await log.start(risotto_context,
encoder=dumps, kwargs,
decoder=loads, info_msg,
schema='pg_catalog' )
) await self.check_message_type(risotto_context,
risotto_context.connection = connection kwargs,
async with connection.transaction(): )
try: config_arguments = await self.load_kwargs_to_config(risotto_context,
await log.start(risotto_context, f'{version}.{message}',
kwargs, kwargs,
info_msg, check_role,
) internal,
await self.check_message_type(risotto_context, )
kwargs, ret = await self.launch(risotto_context,
) kwargs,
config_arguments = await self.load_kwargs_to_config(risotto_context, config_arguments,
f'{version}.{message}', function_obj,
kwargs, )
check_role, # log the success
internal, await log.success(risotto_context,
) ret,
ret = await self.launch(risotto_context, )
kwargs, if not internal and isinstance(ret, dict):
config_arguments, ret['context_id'] = risotto_context.context_id
function_obj, except CallError as err:
) if get_config()['global']['debug']:
# log the success print_exc()
await log.success(risotto_context, await log.failed(risotto_context,
ret, str(err),
) )
if not internal and isinstance(ret, dict): raise err from err
ret['context_id'] = risotto_context.context_id except CallError as err:
except CallError as err: error = err
if get_config()['global']['debug']: except Exception as err:
print_exc() # if there is a problem with arguments, just send an error and do nothing
await log.failed(risotto_context, if get_config()['global']['debug']:
str(err), print_exc()
) await log.failed(risotto_context,
raise err from err str(err),
except CallError as err: )
error = err error = err
except Exception as err:
# if there is a problem with arguments, just send an error and do nothing
if get_config()['global']['debug']:
print_exc()
await log.failed(risotto_context,
str(err),
)
error = err
if error: if error:
if not internal: if not internal:
err = CallError(str(error)) err = CallError(str(error))
@ -198,8 +189,6 @@ class PublishDispatcher:
for message, message_infos in messages.items(): for message, message_infos in messages.items():
# event not emit locally # event not emit locally
if message_infos['pattern'] == 'event' and 'functions' in message_infos and message_infos['functions']: if message_infos['pattern'] == 'event' and 'functions' in message_infos and message_infos['functions']:
# module, submodule, submessage = message.split('.', 2)
# if f'{module}.{submodule}' not in self.injected_self:
uri = f'{version}.{message}' uri = f'{version}.{message}'
print(f' - {uri}') print(f' - {uri}')
await self.listened_connection.add_listener(uri, await self.listened_connection.add_listener(uri,
@ -235,21 +224,34 @@ class PublishDispatcher:
version, message = uri.split('.', 1) version, message = uri.split('.', 1)
loop = get_event_loop() loop = get_event_loop()
remote_kw = loads(payload) remote_kw = loads(payload)
risotto_context = self.build_new_context(remote_kw['context'], for function_obj in self.messages[version][message]['functions']:
version, risotto_context = self.build_new_context(remote_kw['context'],
message, version,
'event', message,
) 'event',
callback = lambda: ensure_future(self._publish(version, )
message, callback = self.get_callback(version, message, function_obj, risotto_context, remote_kw['kwargs'],)
risotto_context, loop.call_soon(callback)
**remote_kw['kwargs'],
)) def get_callback(self,
loop.call_soon(callback) version,
message,
function_obj,
risotto_context,
kwargs,
):
return lambda: ensure_future(self._publish(version,
message,
function_obj,
risotto_context,
**kwargs,
))
async def _publish(self, async def _publish(self,
version: str, version: str,
message: str, message: str,
function_obj,
risotto_context: Context, risotto_context: Context,
**kwargs, **kwargs,
) -> None: ) -> None:
@ -259,66 +261,48 @@ class PublishDispatcher:
False, False,
False, False,
) )
for function_obj in self.messages[version][message]['functions']: async with self.pool.acquire() as connection:
async with self.pool.acquire() as log_connection: await connection.set_type_codec(
await log_connection.set_type_codec( 'json',
'json', encoder=dumps,
encoder=dumps, decoder=loads,
decoder=loads, schema='pg_catalog'
schema='pg_catalog' )
) risotto_context.connection = connection
risotto_context.log_connection = log_connection function_name = function_obj['function'].__name__
async with self.pool.acquire() as connection: info_msg = _(f"call function {function_obj['full_module_name']}.{function_name}")
await connection.set_type_codec( try:
'json', async with connection.transaction():
encoder=dumps,
decoder=loads,
schema='pg_catalog'
)
risotto_context.connection = connection
function_name = function_obj['function'].__name__
info_msg = _(f"call function {function_obj['full_module_name']}.{function_name}")
try: try:
async with connection.transaction(): await log.start(risotto_context,
try: kwargs,
await log.start(risotto_context, info_msg,
kwargs, )
info_msg, await self.check_message_type(risotto_context,
) kwargs,
await self.check_message_type(risotto_context, )
kwargs, await self.launch(risotto_context,
) kwargs,
await self.launch(risotto_context, config_arguments,
kwargs, function_obj,
config_arguments, )
function_obj, # log the success
) await log.success(risotto_context)
# log the success except CallError as err:
await log.success(risotto_context)
except CallError as err:
if get_config()['global']['debug']:
print_exc()
await log.failed(risotto_context,
str(err),
)
except CallError:
pass
except Exception as err:
# if there is a problem with arguments, log and do nothing
if get_config()['global']['debug']: if get_config()['global']['debug']:
print_exc() print_exc()
async with self.pool.acquire() as connection: await log.failed(risotto_context,
await connection.set_type_codec( str(err),
'json', )
encoder=dumps, except CallError:
decoder=loads, pass
schema='pg_catalog' except Exception as err:
) # if there is a problem with arguments, log and do nothing
risotto_context.connection = connection if get_config()['global']['debug']:
async with connection.transaction(): print_exc()
await log.failed(risotto_context, await log.failed(risotto_context,
str(err), str(err),
) )
class Dispatcher(register.RegisterDispatcher, class Dispatcher(register.RegisterDispatcher,
@ -346,6 +330,7 @@ class Dispatcher(register.RegisterDispatcher,
risotto_context.type = type risotto_context.type = type
risotto_context.message = message risotto_context.message = message
risotto_context.version = version risotto_context.version = version
risotto_context.pool = self.pool
return risotto_context return risotto_context
async def check_message_type(self, async def check_message_type(self,

53
src/risotto/logger.py
View File

@ -2,15 +2,34 @@ from typing import Dict, Any, Optional
from json import dumps, loads from json import dumps, loads
from asyncpg.exceptions import UndefinedTableError from asyncpg.exceptions import UndefinedTableError
from datetime import datetime from datetime import datetime
from asyncio import Lock
from .context import Context from .context import Context
from .utils import _ from .utils import _
from .config import get_config from .config import get_config
database_lock = Lock()
class Logger: class Logger:
""" An object to manager log """ An object to manager log
""" """
def __init__(self) -> None:
self.log_connection = None
async def get_connection(self,
risotto_context: Context,
):
if not self.log_connection:
self.log_connection = await risotto_context.pool.acquire()
await self.log_connection.set_type_codec(
'json',
encoder=dumps,
decoder=loads,
schema='pg_catalog'
)
return self.log_connection
async def insert(self, async def insert(self,
msg: str, msg: str,
risotto_context: Context, risotto_context: Context,
@ -38,8 +57,9 @@ class Logger:
sql = insert + ') ' + values + ') RETURNING LogId' sql = insert + ') ' + values + ') RETURNING LogId'
try: try:
async with risotto_context.log_connection.transaction(): async with database_lock:
log_id = await risotto_context.log_connection.fetchval(sql, *args) connection = await self.get_connection(risotto_context)
log_id = await connection.fetchval(sql, *args)
if context_id is None and start: if context_id is None and start:
risotto_context.context_id = log_id risotto_context.context_id = log_id
if start: if start:
@ -61,8 +81,9 @@ class Logger:
sql += ' AND URI = $3' sql += ' AND URI = $3'
args.append(uri) args.append(uri)
ret = [] ret = []
async with risotto_context.log_connection.transaction(): async with database_lock:
for row in await risotto_context.log_connection.fetch(*args): connection = await self.get_connection(risotto_context)
for row in await connection.fetch(*args):
d = {} d = {}
for key, value in row.items(): for key, value in row.items():
if key in ['kwargs', 'returns']: if key in ['kwargs', 'returns']:
@ -173,11 +194,12 @@ class Logger:
args.append(dumps(returns)) args.append(dumps(returns))
sql += """WHERE LogId = $1 sql += """WHERE LogId = $1
""" """
async with risotto_context.log_connection.transaction(): async with database_lock:
await risotto_context.log_connection.execute(sql, connection = await self.get_connection(risotto_context)
risotto_context.start_id, await connection.execute(sql,
*args, risotto_context.start_id,
) *args,
)
async def failed(self, async def failed(self,
risotto_context: Context, risotto_context: Context,
@ -196,12 +218,13 @@ class Logger:
Msg = $3 Msg = $3
WHERE LogId = $1 WHERE LogId = $1
""" """
async with risotto_context.log_connection.transaction(): async with database_lock:
await risotto_context.log_connection.execute(sql, connection = await self.get_connection(risotto_context)
risotto_context.start_id, await connection.execute(sql,
datetime.now(), risotto_context.start_id,
err, datetime.now(),
) err,
)
async def info(self, async def info(self,
risotto_context, risotto_context,

59
src/risotto/register.py
View File

@ -297,37 +297,36 @@ class RegisterDispatcher:
truncate: bool=False, truncate: bool=False,
) -> None: ) -> None:
internal_user = get_config()['global']['internal_user'] internal_user = get_config()['global']['internal_user']
async with self.pool.acquire() as log_connection: async with self.pool.acquire() as connection:
async with self.pool.acquire() as connection: await connection.set_type_codec(
await connection.set_type_codec( 'json',
'json', encoder=dumps,
encoder=dumps, decoder=loads,
decoder=loads, schema='pg_catalog'
schema='pg_catalog' )
) if truncate:
if truncate:
async with connection.transaction():
await connection.execute('TRUNCATE InfraServer, InfraSite, InfraZone, Log, ProviderDeployment, ProviderFactoryCluster, ProviderFactoryClusterNode, SettingApplicationservice, SettingApplicationServiceDependency, SettingRelease, SettingServer, SettingServermodel, SettingSource, UserRole, UserRoleURI, UserURI, UserUser, InfraServermodel, ProviderZone, ProviderServer, ProviderSource, ProviderApplicationservice, ProviderServermodel')
async with connection.transaction(): async with connection.transaction():
for submodule_name, module in self.injected_self.items(): await connection.execute('TRUNCATE InfraServer, InfraSite, InfraZone, Log, ProviderDeployment, ProviderFactoryCluster, ProviderFactoryClusterNode, SettingApplicationservice, SettingApplicationServiceDependency, SettingRelease, SettingServer, SettingServermodel, SettingSource, UserRole, UserRoleURI, UserURI, UserUser, InfraServermodel, ProviderZone, ProviderServer, ProviderSource, ProviderApplicationservice, ProviderServermodel')
risotto_context = Context() async with connection.transaction():
risotto_context.username = internal_user for submodule_name, module in self.injected_self.items():
risotto_context.paths.append(f'internal.{submodule_name}.on_join') risotto_context = Context()
risotto_context.type = None risotto_context.username = internal_user
risotto_context.log_connection = log_connection risotto_context.paths.append(f'internal.{submodule_name}.on_join')
risotto_context.connection = connection risotto_context.type = None
risotto_context.module = submodule_name.split('.', 1)[0] risotto_context.pool = self.pool
info_msg = _(f'in function risotto_{submodule_name}.on_join') risotto_context.connection = connection
await log.info_msg(risotto_context, risotto_context.module = submodule_name.split('.', 1)[0]
None, info_msg = _(f'in function risotto_{submodule_name}.on_join')
info_msg) await log.info_msg(risotto_context,
try: None,
await module.on_join(risotto_context) info_msg)
except Exception as err: try:
if get_config()['global']['debug']: await module.on_join(risotto_context)
print_exc() except Exception as err:
msg = _(f'on_join returns an error in module {submodule_name}: {err}') if get_config()['global']['debug']:
await log.error_msg(risotto_context, {}, msg) print_exc()
msg = _(f'on_join returns an error in module {submodule_name}: {err}')
await log.error_msg(risotto_context, {}, msg)
async def load(self): async def load(self):
# valid function's arguments # valid function's arguments