424 lines
23 KiB
Python
424 lines
23 KiB
Python
from shutil import rmtree, copyfile
|
|
from os import listdir, makedirs
|
|
from os.path import join, isdir, isfile
|
|
from yaml import load, SafeLoader
|
|
from traceback import print_exc
|
|
from typing import Dict, List, Optional
|
|
from rougail import CreoleObjSpace
|
|
from ...controller import Controller
|
|
from ...register import register
|
|
from ...utils import _
|
|
from ...context import Context
|
|
from ...config import get_config
|
|
from ...error import ExecutionError
|
|
from ...logger import log
|
|
|
|
|
|
class Risotto(Controller):
|
|
def __init__(self,
|
|
test: bool) -> None:
|
|
self.source_root_path = get_config().get('source').get('root_path')
|
|
self.cache_root_path = join(get_config().get('cache').get('root_path'), 'servermodel')
|
|
if not isdir(self.cache_root_path):
|
|
makedirs(join(self.cache_root_path))
|
|
|
|
async def on_join(self,
|
|
risotto_context: Context) -> None:
|
|
internal_source = await self.call('v1.source.create',
|
|
risotto_context,
|
|
source_name='internal',
|
|
source_url='none')
|
|
internal_release = await self.call('v1.source.release.create',
|
|
risotto_context,
|
|
source_name='internal',
|
|
release_name='none',
|
|
release_distribution='last')
|
|
self.internal_release_id = internal_release['release_id']
|
|
|
|
async def servermodel_gen_funcs(self,
|
|
servermodel_name: str,
|
|
servermodel_id: int,
|
|
dependencies: Dict,
|
|
release_cache: Dict,
|
|
risotto_context: Context) -> None:
|
|
as_names = []
|
|
dest_file = self.get_servermodel_cache(servermodel_id, 'funcs.py')
|
|
with open(dest_file, 'wb') as funcs:
|
|
funcs.write(b'from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value\n\n')
|
|
for applicationservice_id, applicationservice_infos in dependencies.items():
|
|
applicationservice_name, as_release_id = applicationservice_infos
|
|
path = join(self.source_root_path,
|
|
release_cache[as_release_id]['source_name'],
|
|
release_cache[as_release_id]['release_name'],
|
|
'applicationservice',
|
|
applicationservice_name,
|
|
'funcs')
|
|
if isdir(path):
|
|
as_names.append(applicationservice_name)
|
|
for fil in listdir(path):
|
|
if not fil.endswith('.py'):
|
|
continue
|
|
fil_path = join(path, fil)
|
|
with open(fil_path, 'rb') as fh:
|
|
funcs.write(f'# {fil_path}\n'.encode())
|
|
funcs.write(fh.read())
|
|
funcs.write(b'\n')
|
|
|
|
as_names_str = '", "'.join(as_names)
|
|
await log.info(risotto_context,
|
|
_(f'gen funcs for "{servermodel_name}" with application services "{as_names_str}"'))
|
|
#eolobj = CreoleObjSpace(get_config()['global']['rougail_dtd_path'])
|
|
|
|
async def servermodel_gen_schema(self,
|
|
servermodel_name: str,
|
|
servermodel_id: int,
|
|
dependencies: Dict,
|
|
release_cache: Dict,
|
|
risotto_context: Context) -> None:
|
|
paths = []
|
|
extras = []
|
|
as_names = set()
|
|
for applicationservice_id, applicationservice_infos in dependencies.items():
|
|
applicationservice_name, as_release_id = applicationservice_infos
|
|
# load creole dictionaries
|
|
path = join(self.source_root_path,
|
|
release_cache[as_release_id]['source_name'],
|
|
release_cache[as_release_id]['release_name'],
|
|
'applicationservice',
|
|
applicationservice_name,
|
|
'dictionaries')
|
|
if isdir(path):
|
|
as_names.add(applicationservice_name)
|
|
paths.append(path)
|
|
|
|
# load extra dictionaries
|
|
path = join(self.source_root_path,
|
|
release_cache[as_release_id]['source_name'],
|
|
release_cache[as_release_id]['release_name'],
|
|
'applicationservice',
|
|
applicationservice_name,
|
|
'extras')
|
|
if isdir(path):
|
|
for namespace in listdir(path):
|
|
extra_dir = join(path, namespace)
|
|
if not isdir(extra_dir):
|
|
continue
|
|
as_names.add(applicationservice_name)
|
|
extras.append((namespace, [extra_dir]))
|
|
eolobj = CreoleObjSpace(get_config()['global']['rougail_dtd_path'])
|
|
as_names_str = '", "'.join(as_names)
|
|
await log.info(risotto_context,
|
|
_(f'gen schema for "{servermodel_name}" with application services "{as_names_str}"'))
|
|
eolobj.create_or_populate_from_xml('creole', paths)
|
|
for extra in extras:
|
|
eolobj.create_or_populate_from_xml(extra[0], extra[1])
|
|
# FIXME extra
|
|
funcs_file = self.get_servermodel_cache(servermodel_id, 'funcs.py')
|
|
eolobj.space_visitor(funcs_file)
|
|
dest_dir = self.get_servermodel_cache(servermodel_id, 'dictionaries.xml')
|
|
eolobj.save(dest_dir)
|
|
|
|
def get_servermodel_cache(self,
|
|
servermodel_id: int,
|
|
subdir: Optional[str]=None) -> str:
|
|
if subdir:
|
|
return join(self.cache_root_path, str(servermodel_id), subdir)
|
|
return join(self.cache_root_path, str(servermodel_id))
|
|
|
|
async def servermodel_copy_templates(self,
|
|
servermodel_name: str,
|
|
servermodel_id: int,
|
|
dependencies: Dict,
|
|
release_cache: Dict,
|
|
risotto_context: Context) -> None:
|
|
as_names = []
|
|
dest_dir = self.get_servermodel_cache(servermodel_id, 'templates')
|
|
makedirs(dest_dir)
|
|
for applicationservice_id, applicationservice_infos in dependencies.items():
|
|
applicationservice_name, as_release_id = applicationservice_infos
|
|
path = join(self.source_root_path,
|
|
release_cache[as_release_id]['source_name'],
|
|
release_cache[as_release_id]['release_name'],
|
|
'applicationservice',
|
|
applicationservice_name,
|
|
'templates')
|
|
if isdir(path):
|
|
for template in listdir(path):
|
|
template_path = join(dest_dir, template)
|
|
if isfile(template_path):
|
|
as_names_str = '", "'.join(as_names)
|
|
raise Exception(_(f'duplicate "{template}" when copying template from "{applicationservice_name}" to "{dest_dir}" for servermodel "{servermodel_name}" (previous application services was "{as_names_str}"'))
|
|
copyfile(join(path, template), template_path)
|
|
as_names.append(applicationservice_name)
|
|
as_names_str = '", "'.join(as_names)
|
|
await log.info(risotto_context,
|
|
_(f'copy templates for "{servermodel_name}" with application services "{as_names_str}"'))
|
|
|
|
async def _servermodel_create(self,
|
|
risotto_context: Context,
|
|
servermodel_name: str,
|
|
servermodel_description: str,
|
|
servermodel_parents_id: List[int],
|
|
dependencies: List[int],
|
|
release_id: int,
|
|
release_cache: Dict=None) -> Dict:
|
|
servermodel_update = """INSERT INTO Servermodel(ServermodelName, ServermodelDescription, ServermodelParentsId, ServermodelReleaseId, ServermodelApplicationServiceId)
|
|
VALUES ($1,$2,$3,$4,$5)
|
|
RETURNING ServermodelId
|
|
"""
|
|
as_name = f"local_{servermodel_name}"
|
|
as_description = f'local application service for {servermodel_name}'
|
|
applicationservice = await self.call('v1.applicationservice.create',
|
|
risotto_context,
|
|
applicationservice_name=as_name,
|
|
applicationservice_description=as_description,
|
|
applicationservice_dependencies=dependencies,
|
|
release_id=self.internal_release_id)
|
|
applicationservice_id = applicationservice['applicationservice_id']
|
|
servermodel_id = await risotto_context.connection.fetchval(servermodel_update,
|
|
servermodel_name,
|
|
servermodel_description,
|
|
servermodel_parents_id,
|
|
release_id,
|
|
applicationservice_id)
|
|
dest_dir = self.get_servermodel_cache(servermodel_id)
|
|
if isdir(dest_dir):
|
|
rmtree(dest_dir)
|
|
makedirs(dest_dir)
|
|
# get all dependencies for this application service
|
|
dependencies = await self.get_applicationservices(risotto_context,
|
|
applicationservice_id)
|
|
# build cache to have all release informations
|
|
if release_cache is None:
|
|
release_cache = {}
|
|
for applicationservice_id, applicationservice_infos in dependencies.items():
|
|
applicationservice_name, as_release_id = applicationservice_infos
|
|
if as_release_id not in release_cache:
|
|
release_cache[as_release_id] = await self.call('v1.source.release.get_by_id',
|
|
risotto_context,
|
|
release_id=as_release_id)
|
|
|
|
await self.servermodel_gen_funcs(servermodel_name,
|
|
servermodel_id,
|
|
dependencies,
|
|
release_cache,
|
|
risotto_context)
|
|
await self.servermodel_gen_schema(servermodel_name,
|
|
servermodel_id,
|
|
dependencies,
|
|
release_cache,
|
|
risotto_context)
|
|
await self.servermodel_copy_templates(servermodel_name,
|
|
servermodel_id,
|
|
dependencies,
|
|
release_cache,
|
|
risotto_context)
|
|
sm_dict = {'servermodel_name': servermodel_name,
|
|
'servermodel_description': servermodel_description,
|
|
'servermodel_parents_id': servermodel_parents_id,
|
|
'release_id': release_id,
|
|
'servermodel_id': servermodel_id}
|
|
await self.publish('v1.servermodel.created',
|
|
risotto_context,
|
|
**sm_dict)
|
|
return sm_dict
|
|
|
|
def parse_parents(self,
|
|
servermodels: Dict,
|
|
servermodel: Dict,
|
|
parents: List=None) -> List:
|
|
if parents is None:
|
|
parents = [servermodel['name']]
|
|
parent = servermodel['parent']
|
|
if parent in servermodels:
|
|
parents.append(parent)
|
|
self.parse_parents(servermodels, servermodels[parent], parents)
|
|
return parents
|
|
|
|
async def get_servermodel_id_by_name(self,
|
|
risotto_context: Context,
|
|
servermodel_name: str,
|
|
release_id: int):
|
|
sql = 'SELECT ServermodelId as servermodel_id FROM Servermodel WHERE ServermodelName = $1 AND ServermodelReleaseId = $2',
|
|
return await risotto_context.connection.fetchval(sql,
|
|
servermodel_name,
|
|
release_id)['servermodel_id']
|
|
|
|
@register('v1.servermodel.dataset.updated')
|
|
async def servermodel_update(self,
|
|
risotto_context: Context,
|
|
source_name: str,
|
|
release_distribution: int):
|
|
source = await self.call('v1.source.describe',
|
|
risotto_context,
|
|
source_name=source_name)
|
|
release = await self.call('v1.source.release.get_by_distribution',
|
|
risotto_context,
|
|
source_id=source['source_id'],
|
|
release_distribution=release_distribution)
|
|
release_id = release['release_id']
|
|
servermodel_path = join(self.source_root_path,
|
|
source_name,
|
|
release['release_name'],
|
|
'servermodel')
|
|
servermodels = {}
|
|
for servermodel in listdir(servermodel_path):
|
|
if not servermodel.endswith('.yml'):
|
|
continue
|
|
servermodel_description_path = join(servermodel_path, servermodel)
|
|
try:
|
|
with open(servermodel_description_path, 'r') as servermodel_yml:
|
|
servermodel_description = load(servermodel_yml,
|
|
Loader=SafeLoader)
|
|
except Exception as err:
|
|
if get_config().get('global').get('debug'):
|
|
print_exc()
|
|
raise ExecutionError(_(f'Error while reading {servermodel_description_path}: {err}'))
|
|
servermodels[servermodel_description['name']] = servermodel_description
|
|
servermodels[servermodel_description['name']]['done'] = False
|
|
|
|
release_cache = {release['release_id']: release}
|
|
for servermodel in servermodels.values():
|
|
if not servermodel['done']:
|
|
# parent needs to create before child, so retrieve all parents
|
|
parents = self.parse_parents(servermodels,
|
|
servermodel)
|
|
parents.reverse()
|
|
servermodelparent_id = []
|
|
for new_servermodel in parents:
|
|
if not servermodels[new_servermodel]['done']:
|
|
servermodel_description = servermodels[new_servermodel]
|
|
parent = servermodel_description['parent']
|
|
if not servermodelparent_id and parent is not None:
|
|
# parent is a str, so get ID
|
|
servermodelparent_id = [await self.get_servermodel_id_by_name(risotto_context,
|
|
parent,
|
|
release_id)]
|
|
# link application service with this servermodel
|
|
dependencies = []
|
|
for depend in servermodels[new_servermodel]['applicationservices']:
|
|
applicationservice = await self.call('v1.applicationservice.describe',
|
|
risotto_context,
|
|
applicationservice_name=depend,
|
|
release_id=release_id)
|
|
dependencies.append(applicationservice['applicationservice_id'])
|
|
sm_name = servermodel_description['name']
|
|
sm_description = servermodel_description['description']
|
|
try:
|
|
servermodel_ob = await self._servermodel_create(risotto_context,
|
|
sm_name,
|
|
sm_description,
|
|
servermodelparent_id,
|
|
dependencies,
|
|
release_id,
|
|
release_cache)
|
|
servermodel_id = servermodel_ob['servermodel_id']
|
|
except Exception as err:
|
|
if get_config().get('global').get('debug'):
|
|
print_exc()
|
|
raise ExecutionError(_(f"Error while injecting servermodel {sm_name} in database: {err}"))
|
|
servermodelparent_id = [servermodel_id]
|
|
servermodel_description['done'] = True
|
|
return {'retcode': 0, 'returns': _('Servermodels successfully loaded')}
|
|
|
|
@register('v1.servermodel.list')
|
|
async def servermodel_list(self,
|
|
risotto_context: Context,
|
|
source_id: int):
|
|
sql = '''
|
|
SELECT ServermodelId as servermodel_id, ServermodelName as servermodel_name, ServermodelDescription as servermodel_description, ServermodelParentsId as servermodel_parents_id, ServermodelReleaseId as release_id
|
|
FROM Servermodel
|
|
'''
|
|
servermodels = await risotto_context.connection.fetch(sql)
|
|
return [dict(r) for r in servermodels]
|
|
|
|
@register('v1.servermodel.describe')
|
|
async def servermodel_describe(self,
|
|
risotto_context: Context,
|
|
servermodel_name,
|
|
source_name,
|
|
release_distribution) -> Dict:
|
|
release = await self.call('v1.source.release.describe',
|
|
risotto_context,
|
|
source_name=source_name,
|
|
release_distribution=release_distribution)
|
|
sql = '''
|
|
SELECT ServermodelId as servermodel_id, ServermodelName as servermodel_name, ServermodelDescription as servermodel_description, ServermodelParentsId as servermodel_parents_id, ServermodelReleaseId as release_id
|
|
FROM Servermodel
|
|
WHERE ServermodelName=$1 AND ServermodelReleaseId=$2
|
|
'''
|
|
servermodel = await risotto_context.connection.fetchrow(sql,
|
|
servermodel_name,
|
|
release['release_id'])
|
|
if not servermodel:
|
|
raise Exception(_(f'{servermodel_id} is not a valid ID for a servermodel'))
|
|
return dict(servermodel)
|
|
|
|
@register('v1.servermodel.get_by_id')
|
|
async def servermodel_get_by_id(self,
|
|
risotto_context: Context,
|
|
servermodel_id: int) -> Dict:
|
|
sql = '''
|
|
SELECT ServermodelId as servermodel_id, ServermodelName as servermodel_name, ServermodelDescription as servermodel_description, ServermodelParentsId as servermodel_parents_id, ServermodelReleaseId as release_id
|
|
FROM Servermodel
|
|
WHERE ServermodelId=$1
|
|
'''
|
|
servermodel = await risotto_context.connection.fetchrow(sql,
|
|
servermodel_id)
|
|
if not servermodel:
|
|
raise Exception(_(f'{servermodel_id} is not a valid ID for a servermodel'))
|
|
return dict(servermodel)
|
|
|
|
async def _parse_depends(self,
|
|
risotto_context: Context,
|
|
applicationservice_id: int,
|
|
or_depends: list,
|
|
ids: list) -> None:
|
|
applicationservice = await self.call('v1.applicationservice.get_by_id',
|
|
risotto_context,
|
|
applicationservice_id=applicationservice_id)
|
|
ids[applicationservice_id] = (applicationservice['applicationservice_name'],
|
|
applicationservice['applicationservice_release_id'])
|
|
for depend in applicationservice['applicationservice_dependencies']:
|
|
if isinstance(depend, dict):
|
|
or_depends.append(depend['or'])
|
|
elif depend not in ids:
|
|
await self._parse_depends(risotto_context,
|
|
depend,
|
|
or_depends,
|
|
ids)
|
|
|
|
async def _parse_or_depends(self,
|
|
risotto_context: Context,
|
|
or_depends: list,
|
|
ids: list) -> None:
|
|
new_or_depends = []
|
|
set_ids = set(ids)
|
|
for or_depend in or_depends:
|
|
if not set(or_depend) & set_ids:
|
|
applicationservice_id= or_depend[0]
|
|
await self._parse_depends(risotto_context,
|
|
applicationservice_id,
|
|
new_or_depends,
|
|
ids)
|
|
if new_or_depends:
|
|
await self._parse_or_depends(risotto_context,
|
|
new_or_depends,
|
|
ids)
|
|
|
|
async def get_applicationservices(self,
|
|
risotto_context: Context,
|
|
applicationservice_id: int) -> list:
|
|
"""Return consolidated dependencies or raise.
|
|
"""
|
|
or_depends = []
|
|
ids = {}
|
|
await self._parse_depends(risotto_context,
|
|
applicationservice_id,
|
|
or_depends,
|
|
ids)
|
|
await self._parse_or_depends(risotto_context,
|
|
or_depends,
|
|
ids)
|
|
return ids
|