import servermodel

This commit is contained in:
Emmanuel Garette 2019-12-18 17:11:42 +01:00
parent b006eda133
commit e19e718e22
12 changed files with 346 additions and 232 deletions

View File

@ -0,0 +1,32 @@
---
uri: servermodel.describe
description: |
Retourne les attributs détaillés d'un modèle de serveur.
pattern: rpc
public: true
parameters:
servermodel_id:
type: Number
shortarg: s
description: Identifiant du modèle de serveur à récupérer.
ref: Servermodel.ServermodelId
response:
type: Servermodel
description: Description du modèle de serveur.
errors:
- uri: servermodel.describe.error.database_not_available
- uri: servermodel.describe.error.invalid_servermodel_id
- uri: servermodel.describe.error.unknown_servermodel_id
related:
- servermodel.list
- servermodel.create
- servermodel.update
- servermodel.delete
- servermodel.event

View File

@ -8,14 +8,14 @@ pattern: rpc
public: true
parameters:
release_path:
source_name:
type: String
shortarg: s
description: Nom de la source.
release_id:
type: Number
release_distribution:
type: String
shortarg: r
description: Nom de la version.
description: Distribution de la version.
response:
type: ReturnStatus

View File

@ -1,62 +0,0 @@
---
uri: servermodel.describe
description: |
Retourne les attributs détaillés d'un modèle de serveur.
pattern: rpc
public: true
parameters:
servermodel_id:
type: Number
shortarg: s
description: Identifiant du modèle de serveur à récupérer.
ref: Servermodel.ServermodelId
inheritance:
type: Boolean
shortarg: i
description: Inclure les données héritées des modèles de serveur parents.
default: true
resolvdepends:
type: Boolean
shortarg: r
description: Résoudre les dépendances de services.
default: true
schema:
type: Boolean
shortarg: c
description: Inclure le schema de configuration (réaggrège les données provenant du datasource).
default: false
probes:
type: Boolean
shortarg: p
description: Inclure les informations sur les sondes de la configuration.
default: false
creolefuncs:
type: Boolean
shortarg: o
description: Inclure les fonctions Creole.
default: false
conffiles:
type: Boolean
shortarg: f
description: Inclure les fichier creole au format tar encodé en base64
default: false
response:
type: Servermodel
description: Description du modèle de serveur.
errors:
- uri: servermodel.describe.error.database_not_available
- uri: servermodel.describe.error.invalid_servermodel_id
- uri: servermodel.describe.error.unknown_servermodel_id
related:
- servermodel.list
- servermodel.create
- servermodel.update
- servermodel.delete
- servermodel.event

View File

@ -1,5 +1,5 @@
---
uri: source.get
uri: source.describe
description: |
Retourne une source.

View File

@ -0,0 +1,24 @@
---
uri: source.release.get_by_distribution
description: |
Retourne version suivant le nom de la distribution.
pattern: rpc
public: true
parameters:
source_id:
type: Number
shortarg: s
description: ID de la source.
release_distribution:
type: String
shortarg: r
description: Distribution de la version.
response:
type: 'Release'
description: La version disponibles.

View File

@ -0,0 +1,19 @@
---
uri: source.release.get_by_id
description: |
Retourne version suivant l'identifiant.
pattern: rpc
public: true
parameters:
release_id:
type: Number
shortarg: r
description: ID de la version.
response:
type: 'Release'
description: La version disponibles.

View File

@ -17,6 +17,7 @@ CREATE TABLE Release (
ReleaseSourceId INTEGER NOT NULL,
ReleaseDistribution VARCHAR(20) CONSTRAINT releasedistribution_choice CHECK (ReleaseDistribution IN ('dev', 'stable', 'maintained', 'deprecation-warning', 'deprecated')),
UNIQUE (ReleaseName, ReleaseSourceId),
UNIQUE (ReleaseDistribution, ReleaseSourceId),
FOREIGN KEY (ReleaseSourceId) REFERENCES Source(SourceId)
);

View File

@ -23,5 +23,6 @@ def get_config():
'debug': DEBUG,
'internal_user': 'internal',
'rougail_dtd_path': '../rougail/data/creole.dtd'},
'source': {'root_path': '/srv/seed'}
'source': {'root_path': '/srv/seed'},
'cache': {'root_path': '/var/cache/risotto'}
}

View File

@ -82,28 +82,29 @@ class Risotto(Controller):
f'Load servermodel {servermodel_name} ({servermodel_id})')
# use file in cache if found, otherwise retrieve it in servermodel context
if isfile(cache_file):
fileio = open(cache_file)
else:
servermodel = await self.call('v1.servermodel.describe',
risotto_context,
servermodel_id=servermodel_id,
inheritance=False,
resolvdepends=False,
schema=True,
creolefuncs=True)
fileio = BytesIO()
fileio.write(servermodel['schema'].encode())
fileio.seek(0)
with open(cache_file, 'w') as cache:
cache.write(servermodel['schema'])
with open(funcs_file, 'w') as cache:
cache.write(servermodel['creolefuncs'])
del servermodel
# loads tiramisu config and store it
xmlroot = parse(fileio).getroot()
# if isfile(cache_file):
# fileio = open(cache_file)
# else:
# servermodel = await self.call('v1.servermodel.describe',
# risotto_context,
# servermodel_id=servermodel_id,
# inheritance=False,
# resolvdepends=False,
# schema=True,
# creolefuncs=True)
# fileio = BytesIO()
# fileio.write(servermodel['schema'].encode())
# fileio.seek(0)
#
# with open(cache_file, 'w') as cache:
# cache.write(servermodel['schema'])
# with open(funcs_file, 'w') as cache:
# cache.write(servermodel['creolefuncs'])
# del servermodel
#
# # loads tiramisu config and store it
with open(cache_file) as fileio:
xmlroot = parse(fileio).getroot()
self.servermodel[servermodel_id] = self.build_metaconfig(servermodel_id,
servermodel_name,
xmlroot,

View File

@ -1,8 +1,10 @@
from os import listdir
from os.path import join
from os import listdir, makedirs
from os.path import join, isdir
from yaml import load, SafeLoader
from traceback import print_exc
from typing import Dict, List
from rougail import CreoleObjSpace
from rougail.config import dtdfilename
from ...controller import Controller
from ...register import register
from ...utils import _
@ -12,24 +14,61 @@ from ...error import ExecutionError
class Risotto(Controller):
def __init__(self):
self.source_root_path = get_config().get('source').get('root_path')
self.cache_root_path = join(get_config().get('cache').get('root_path'), 'servermodel')
if not isdir(self.cache_root_path):
makedirs(join(self.cache_root_path))
async def on_join(self,
risotto_context: Context) -> None:
internal_source = await self.call('v1.source.create',
risotto_context,
source_name='internal',
source_url='none')
self.internal_release_id = await self.call('v1.source.release.create',
risotto_context,
source_id=internal_source['source_id'],
release_name='none')
internal_release = await self.call('v1.source.release.create',
risotto_context,
source_id=internal_source['source_id'],
release_name='none')
self.internal_release_id = internal_release['release_id']
async def servermodel_gen_schema(self,
risotto_context: Context,
servermodel_id: int,
applicationservice_id: int) -> None:
dependencies = await self.get_applicationservices(risotto_context,
applicationservice_id)
if release_cache is None:
release_cache = {}
dict_paths = []
for applicationservice_id, applicationservice_infos in dependencies.items():
applicationservice_name, as_release_id = applicationservice_infos
if as_release_id not in release_cache:
release_cache[as_release_id] = await self.call('v1.source.release.get_by_id',
release_id=as_release_id)
dict_paths.append(join(self.source_root_path,
release_cache[as_release_id]['source_name'],
release_cache[as_release_id]['release_distribution'],
'applicationservice',
applicationservice_name,
'dictionaries'))
eolobj = CreoleObjSpace(dtdfilename)
eolobj.create_or_populate_from_xml('creole', dict_paths)
# FIXME extra
# FIXME eosfunc
eosfunc = '/eosfunc.py'
eolobj.space_visitor(eosfunc)
eolobj.save(join(self.cache_root_path, f'{servermodel_id}.xml'))
async def _servermodel_create(self,
risotto_context: Context,
servermodel_name: str,
servermodel_description: str,
servermodel_parents_id,
release_id):
servermodel_update = """INSERT INTO Servermodel(ServermodelName, ServermodelDescription, ServermodelParentsId, ServermodelReleaseId, ServermodelApplicationServiceId) VALUES ($1,$2,$3,$4,$5)
servermodel_parents_id: List[int],
release_id: int,
release_cache: Dict=None) -> Dict:
servermodel_update = """INSERT INTO Servermodel(ServermodelName, ServermodelDescription, ServermodelParentsId, ServermodelReleaseId, ServermodelApplicationServiceId)
VALUES ($1,$2,$3,$4,$5)
RETURNING ServermodelId
"""
as_name = f"local_{servermodel_name}"
@ -39,28 +78,32 @@ class Risotto(Controller):
applicationservice_name=as_name,
applicationservice_description=as_description,
release_id=self.internal_release_id)
servermodel = await risotto_context.connection.fetchval(servermodel_update,
sm_name,
sm_description,
sm_parentsid,
release_id,
applicationservice['applicationservice_id'])
applicationservice_id = applicationservice['applicationservice_id']
servermodel_id = await risotto_context.connection.fetchval(servermodel_update,
servermodel_name,
servermodel_description,
servermodel_parents_id,
release_id,
applicationservice_id)
self.servermodel_gen_schema(risotto_context,
servermodel_id,
applicationservice_id)
return {'servermodel_name': servermodel_name,
'servermodel_description': servermodel_description,
'servermodel_parents_id': servermodel_parents_id,
'release_id': release_id,
'servermodel_id': servermodel['servermodel_id']}
'servermodel_id': servermodel_id}
def parse_parents(self,
servermodels: Dict,
servermodel: Dict,
parents: List=None) -> List:
if parents is None:
parents = [servermodel]
parent = servermodels[servermodel]['parent']
parents = [servermodel['name']]
parent = servermodel['parent']
if parent in servermodels:
parents.append(parent)
self.parse_parents(servermodels, parent, parents)
self.parse_parents(servermodels, servermodels[parent], parents)
return parents
async def get_servermodel_id_by_name(self,
@ -75,11 +118,22 @@ class Risotto(Controller):
@register('v1.servermodel.dataset.updated', None, database=True)
async def servermodel_update(self,
risotto_context: Context,
release_path: str,
release_id: int):
servermodel_path = join(release_path, 'servermodel')
source_name: str,
release_distribution: int):
servermodel_path = join(self.source_root_path,
source_name,
release_distribution,
'servermodel')
servermodels = {}
# for dependencies reason load all servermodels
source = await self.call('v1.source.describe',
risotto_context,
source_name=source_name)
release = await self.call('v1.source.release.get_by_distribution',
risotto_context,
source_id=source['source_id'],
release_distribution=release_distribution)
release_id = release['release_id']
for servermodel in listdir(servermodel_path):
if not servermodel.endswith('.yml'):
continue
@ -92,10 +146,11 @@ class Risotto(Controller):
if get_config().get('global').get('debug'):
print_exc()
raise ExecutionError(_(f'Error while reading {servermodel_description_path}: {err}'))
servermodels[servermodel_yml['name']] = servermodel_description
servermodels[servermodel_yml['name']]['done'] = False
servermodels[servermodel_description['name']] = servermodel_description
servermodels[servermodel_description['name']]['done'] = False
for servermodel in servermodels:
release_cache = {release['release_id']: release}
for servermodel in servermodels.values():
if not servermodel['done']:
# parent needs to create before child, so retrieve all parents
parents = self.parse_parents(servermodels,
@ -114,10 +169,13 @@ class Risotto(Controller):
sm_name = servermodel_description['name']
sm_description = servermodel_description['description']
try:
servermodel_id = await self._servermodel_create(sm_name,
servermodel_ob = await self._servermodel_create(risotto_context,
sm_name,
sm_description,
servermodelparent_id,
release_id)['servermodel_id']
release_id,
release_cache)
servermodel_id = servermodel_ob['servermodel_id']
except Exception as err:
if get_config().get('global').get('debug'):
print_exc()
@ -136,56 +194,59 @@ class Risotto(Controller):
servermodels = await risotto_context.connection.fetch(sql)
return [dict(r) for r in servermodels]
@register('v1.servermodel.describe', None)
async def servermodel_describe(self, inheritance, creolefuncs, servermodel_id, schema, conffiles, resolvdepends, probes):
schema = """<?xml version='1.0' encoding='UTF-8'?>
<creole>
<family name="containers">
<family name="container0" doc="test">
<family doc="files" name="files">
<family doc="file0" name="file0">
<variable doc="" multi="False" name="mkdir" type="boolean">
<value>False</value>
</variable>
<variable doc="" multi="False" name="name" type="string">
<value>/etc/mailname</value>
</variable>
<variable doc="" multi="False" name="rm" type="boolean">
<value>False</value>
</variable>
<variable doc="" multi="False" name="source" type="string">
<value>mailname</value>
</variable>
<variable doc="" multi="False" name="activate" type="boolean">
<value>True</value>
</variable>
</family>
</family>
<property>basic</property>
</family>
</family>
<family doc="" name="creole">
<family doc="general" name="general">
<property>normal</property>
<variable doc="No change" multi="False" name="mode_conteneur_actif" type="choice">
<choice type="string">oui</choice>
<choice type="string">non</choice>
<property>mandatory</property>
<property>normal</property>
<value type="string">non</value>
</variable>
<leader doc="master" name="master">
<property>normal</property>
<variable doc="master" multi="True" name="master" type="string"/>
<variable doc="slave1" multi="True" name="slave1" type="string">
<property>normal</property>
</variable>
<variable doc="slave2" multi="True" name="slave2" type="string">
<property>normal</property>
</variable>
</leader>
</family>
<separators/>
</family>
</creole>"""
return {'servermodel_id': 1, 'servermodel_name': 'name', 'servermodel_description': 'description', 'release_id': 1, 'schema': schema, 'creolefuncs': ''}
async def _parse_depends(self,
risotto_context,
applicationservice_id: int,
or_depends: list,
ids: list) -> None:
applicationservice_query = """
SELECT ApplicationServiceName, ApplicationServiceDependencies, ApplicationServiceReleaseId
FROM applicationservice
WHERE applicationserviceid=$1"""
applicationservice = await risotto_context.connection.fetchval(servermodel_update,
applicationservice_id)
ids[applicationservice_id] = (applicationserverice['ApplicationServiceName'],
applicationserverice['ApplicationServiceReleaseId'])
if applicationserverice['ApplicationServiceDependencies']:
for depend in pplicationserverice['ApplicationServiceDependencies']:
if isinstance(depend, dict):
or_depends.append(depend['or'])
elif depend not in ids:
await self._parse_depends(risotto_context,
depend,
or_depends,
ids)
async def _parse_or_depends(self,
risotto_context,
or_depends: list,
ids: list) -> None:
new_or_depends = []
set_ids = set(ids)
for or_depend in or_depends:
if not set(or_depend) & set_ids:
applicationservice_id= or_depend[0]
await self._parse_depends(risotto_context,
applicationservice_id,
new_or_depends,
ids)
if new_or_depends:
await self._parse_or_depends(risotto_context,
new_or_depends,
ids)
async def get_applicationservices(self,
risotto_context,
applicationservice_id: int) -> list:
"""Return consolidated dependencies or raise.
"""
or_depends = []
ids = []
await self._parse_depends(risotto_context,
applicationservice_id,
or_depends,
ids)
await self._parse_or_depends(risotto_context,
or_depends,
ids)
return ids

View File

@ -43,16 +43,19 @@ class Risotto(Controller):
'source_url': source_url,
'source_id': source_id}
@register('v1.source.get', None, database=True)
async def source_get(self,
risotto_context: Context,
source_name: str) -> Dict:
@register('v1.source.describe', None, database=True)
async def source_describe(self,
risotto_context: Context,
source_name: str) -> Dict:
source_get = """SELECT SourceId as source_id, SourceName as source_name, SourceURL as source_url
FROM Source
WHERE SourceName = $1
"""
return dict(await risotto_context.connection.fetchrow(source_get,
source_name))
source = await risotto_context.connection.fetchrow(source_get,
source_name)
if not source:
raise Exception(f'unknown source with name {source_name}')
return dict(source)
@register('v1.source.list', None, database=True)
async def source_list(self,
@ -81,7 +84,7 @@ class Risotto(Controller):
@register('v1.source.release.create', None, database=True)
async def source_release_create(self,
risotto_context: Context,
source_id: str,
source_id: int,
release_name: str) -> Dict:
source_get = """SELECT SourceId as source_id, SourceName as source_name, SourceURL as source_url
FROM Source
@ -104,6 +107,34 @@ class Risotto(Controller):
@register('v1.source.release.list', None, database=True)
async def release_list(self,
risotto_context):
release_query = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name FROM Release, Source WHERE Source.SourceId=Release.ReleaseSourceId"""
result = await risotto_context.connection.fetch(release_name)
release_query = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name
FROM Release, Source
WHERE Source.SourceId=Release.ReleaseSourceId"""
result = await risotto_context.connection.fetch(release_query)
return [dict(r) for r in result]
@register('v1.source.release.get_by_id', None, database=True)
async def release_get_by_id(self,
risotto_context: Context,
release_id: int) -> Dict:
release_query = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name
FROM Release, Source
WHERE Release.ReleaseId = $1"""
result = await risotto_context.connection.fetchrow(release_query,
release_id)
return dict(result)
@register('v1.source.release.get_by_distribution', None, database=True)
async def release_get_by_distribution(self,
risotto_context: Context,
source_id: int,
release_distribution: str) -> Dict:
release_query = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name
FROM Release, Source
WHERE Source.SourceId = $1 AND Release.ReleaseName = $2"""
result = await risotto_context.connection.fetchrow(release_query,
source_id,
release_distribution)
if not result:
raise Exception(f'unknown distribution {release_distribution} with source {source_id}')
return dict(result)

View File

@ -14,56 +14,62 @@ class Risotto(Controller):
'servermodel_description': 'description2',
'servermodel_parents_id': [1]}]
@register('v1.servermodel.describe', None)
async def servermodel_describe(self, inheritance, creolefuncs, servermodel_id, schema, conffiles, resolvdepends, probes):
schema = """<?xml version='1.0' encoding='UTF-8'?>
<creole>
<family name="containers">
<family name="container0" doc="test">
<family doc="files" name="files">
<family doc="file0" name="file0">
<variable doc="" multi="False" name="mkdir" type="boolean">
<value>False</value>
</variable>
<variable doc="" multi="False" name="name" type="string">
<value>/etc/mailname</value>
</variable>
<variable doc="" multi="False" name="rm" type="boolean">
<value>False</value>
</variable>
<variable doc="" multi="False" name="source" type="string">
<value>mailname</value>
</variable>
<variable doc="" multi="False" name="activate" type="boolean">
<value>True</value>
</variable>
</family>
</family>
<property>basic</property>
</family>
</family>
<family doc="" name="creole">
<family doc="general" name="general">
<property>normal</property>
<variable doc="No change" multi="False" name="mode_conteneur_actif" type="choice">
<choice type="string">oui</choice>
<choice type="string">non</choice>
<property>mandatory</property>
<property>normal</property>
<value type="string">non</value>
</variable>
<leader doc="master" name="master">
<property>normal</property>
<variable doc="master" multi="True" name="master" type="string"/>
<variable doc="slave1" multi="True" name="slave1" type="string">
<property>normal</property>
</variable>
<variable doc="slave2" multi="True" name="slave2" type="string">
<property>normal</property>
</variable>
</leader>
</family>
<separators/>
</family>
</creole>"""
return {'servermodel_id': 1, 'servermodel_name': 'name', 'servermodel_description': 'description', 'release_id': 1, 'schema': schema, 'creolefuncs': ''}
# @register('v1.servermodel.describe', None)
# async def servermodel_describe(self,
# servermodel_id,
# inheritance,
# resolv_depends):
# schema = """<?xml version='1.0' encoding='UTF-8'?>
#<creole>
# <family name="containers">
# <family name="container0" doc="test">
# <family doc="files" name="files">
# <family doc="file0" name="file0">
# <variable doc="" multi="False" name="mkdir" type="boolean">
# <value>False</value>
# </variable>
# <variable doc="" multi="False" name="name" type="string">
# <value>/etc/mailname</value>
# </variable>
# <variable doc="" multi="False" name="rm" type="boolean">
# <value>False</value>
# </variable>
# <variable doc="" multi="False" name="source" type="string">
# <value>mailname</value>
# </variable>
# <variable doc="" multi="False" name="activate" type="boolean">
# <value>True</value>
# </variable>
# </family>
# </family>
# <property>basic</property>
# </family>
# </family>
# <family doc="" name="creole">
# <family doc="general" name="general">
# <property>normal</property>
# <variable doc="No change" multi="False" name="mode_conteneur_actif" type="choice">
# <choice type="string">oui</choice>
# <choice type="string">non</choice>
# <property>mandatory</property>
# <property>normal</property>
# <value type="string">non</value>
# </variable>
# <leader doc="master" name="master">
# <property>normal</property>
# <variable doc="master" multi="True" name="master" type="string"/>
# <variable doc="slave1" multi="True" name="slave1" type="string">
# <property>normal</property>
# </variable>
# <variable doc="slave2" multi="True" name="slave2" type="string">
# <property>normal</property>
# </variable>
# </leader>
# </family>
# <separators/>
# </family>
#</creole>"""
# return {'servermodel_id': 1,
# 'servermodel_name': 'name',
# 'servermodel_description': 'description',
# 'release_id': 1}