import servermodel

This commit is contained in:
Emmanuel Garette 2019-12-18 17:11:42 +01:00
parent b006eda133
commit e19e718e22
12 changed files with 346 additions and 232 deletions

View File

@ -0,0 +1,32 @@
---
uri: servermodel.describe
description: |
Retourne les attributs détaillés d'un modèle de serveur.
pattern: rpc
public: true
parameters:
servermodel_id:
type: Number
shortarg: s
description: Identifiant du modèle de serveur à récupérer.
ref: Servermodel.ServermodelId
response:
type: Servermodel
description: Description du modèle de serveur.
errors:
- uri: servermodel.describe.error.database_not_available
- uri: servermodel.describe.error.invalid_servermodel_id
- uri: servermodel.describe.error.unknown_servermodel_id
related:
- servermodel.list
- servermodel.create
- servermodel.update
- servermodel.delete
- servermodel.event

View File

@ -8,14 +8,14 @@ pattern: rpc
public: true public: true
parameters: parameters:
release_path: source_name:
type: String type: String
shortarg: s shortarg: s
description: Nom de la source. description: Nom de la source.
release_id: release_distribution:
type: Number type: String
shortarg: r shortarg: r
description: Nom de la version. description: Distribution de la version.
response: response:
type: ReturnStatus type: ReturnStatus

View File

@ -1,62 +0,0 @@
---
uri: servermodel.describe
description: |
Retourne les attributs détaillés d'un modèle de serveur.
pattern: rpc
public: true
parameters:
servermodel_id:
type: Number
shortarg: s
description: Identifiant du modèle de serveur à récupérer.
ref: Servermodel.ServermodelId
inheritance:
type: Boolean
shortarg: i
description: Inclure les données héritées des modèles de serveur parents.
default: true
resolvdepends:
type: Boolean
shortarg: r
description: Résoudre les dépendances de services.
default: true
schema:
type: Boolean
shortarg: c
description: Inclure le schema de configuration (réaggrège les données provenant du datasource).
default: false
probes:
type: Boolean
shortarg: p
description: Inclure les informations sur les sondes de la configuration.
default: false
creolefuncs:
type: Boolean
shortarg: o
description: Inclure les fonctions Creole.
default: false
conffiles:
type: Boolean
shortarg: f
description: Inclure les fichier creole au format tar encodé en base64
default: false
response:
type: Servermodel
description: Description du modèle de serveur.
errors:
- uri: servermodel.describe.error.database_not_available
- uri: servermodel.describe.error.invalid_servermodel_id
- uri: servermodel.describe.error.unknown_servermodel_id
related:
- servermodel.list
- servermodel.create
- servermodel.update
- servermodel.delete
- servermodel.event

View File

@ -1,5 +1,5 @@
--- ---
uri: source.get uri: source.describe
description: | description: |
Retourne une source. Retourne une source.

View File

@ -0,0 +1,24 @@
---
uri: source.release.get_by_distribution
description: |
Retourne version suivant le nom de la distribution.
pattern: rpc
public: true
parameters:
source_id:
type: Number
shortarg: s
description: ID de la source.
release_distribution:
type: String
shortarg: r
description: Distribution de la version.
response:
type: 'Release'
description: La version disponibles.

View File

@ -0,0 +1,19 @@
---
uri: source.release.get_by_id
description: |
Retourne version suivant l'identifiant.
pattern: rpc
public: true
parameters:
release_id:
type: Number
shortarg: r
description: ID de la version.
response:
type: 'Release'
description: La version disponibles.

View File

@ -17,6 +17,7 @@ CREATE TABLE Release (
ReleaseSourceId INTEGER NOT NULL, ReleaseSourceId INTEGER NOT NULL,
ReleaseDistribution VARCHAR(20) CONSTRAINT releasedistribution_choice CHECK (ReleaseDistribution IN ('dev', 'stable', 'maintained', 'deprecation-warning', 'deprecated')), ReleaseDistribution VARCHAR(20) CONSTRAINT releasedistribution_choice CHECK (ReleaseDistribution IN ('dev', 'stable', 'maintained', 'deprecation-warning', 'deprecated')),
UNIQUE (ReleaseName, ReleaseSourceId), UNIQUE (ReleaseName, ReleaseSourceId),
UNIQUE (ReleaseDistribution, ReleaseSourceId),
FOREIGN KEY (ReleaseSourceId) REFERENCES Source(SourceId) FOREIGN KEY (ReleaseSourceId) REFERENCES Source(SourceId)
); );

View File

@ -23,5 +23,6 @@ def get_config():
'debug': DEBUG, 'debug': DEBUG,
'internal_user': 'internal', 'internal_user': 'internal',
'rougail_dtd_path': '../rougail/data/creole.dtd'}, 'rougail_dtd_path': '../rougail/data/creole.dtd'},
'source': {'root_path': '/srv/seed'} 'source': {'root_path': '/srv/seed'},
'cache': {'root_path': '/var/cache/risotto'}
} }

View File

@ -82,27 +82,28 @@ class Risotto(Controller):
f'Load servermodel {servermodel_name} ({servermodel_id})') f'Load servermodel {servermodel_name} ({servermodel_id})')
# use file in cache if found, otherwise retrieve it in servermodel context # use file in cache if found, otherwise retrieve it in servermodel context
if isfile(cache_file): # if isfile(cache_file):
fileio = open(cache_file) # fileio = open(cache_file)
else: # else:
servermodel = await self.call('v1.servermodel.describe', # servermodel = await self.call('v1.servermodel.describe',
risotto_context, # risotto_context,
servermodel_id=servermodel_id, # servermodel_id=servermodel_id,
inheritance=False, # inheritance=False,
resolvdepends=False, # resolvdepends=False,
schema=True, # schema=True,
creolefuncs=True) # creolefuncs=True)
fileio = BytesIO() # fileio = BytesIO()
fileio.write(servermodel['schema'].encode()) # fileio.write(servermodel['schema'].encode())
fileio.seek(0) # fileio.seek(0)
#
with open(cache_file, 'w') as cache: # with open(cache_file, 'w') as cache:
cache.write(servermodel['schema']) # cache.write(servermodel['schema'])
with open(funcs_file, 'w') as cache: # with open(funcs_file, 'w') as cache:
cache.write(servermodel['creolefuncs']) # cache.write(servermodel['creolefuncs'])
del servermodel # del servermodel
#
# loads tiramisu config and store it # # loads tiramisu config and store it
with open(cache_file) as fileio:
xmlroot = parse(fileio).getroot() xmlroot = parse(fileio).getroot()
self.servermodel[servermodel_id] = self.build_metaconfig(servermodel_id, self.servermodel[servermodel_id] = self.build_metaconfig(servermodel_id,
servermodel_name, servermodel_name,

View File

@ -1,8 +1,10 @@
from os import listdir from os import listdir, makedirs
from os.path import join from os.path import join, isdir
from yaml import load, SafeLoader from yaml import load, SafeLoader
from traceback import print_exc from traceback import print_exc
from typing import Dict, List from typing import Dict, List
from rougail import CreoleObjSpace
from rougail.config import dtdfilename
from ...controller import Controller from ...controller import Controller
from ...register import register from ...register import register
from ...utils import _ from ...utils import _
@ -12,24 +14,61 @@ from ...error import ExecutionError
class Risotto(Controller): class Risotto(Controller):
def __init__(self):
self.source_root_path = get_config().get('source').get('root_path')
self.cache_root_path = join(get_config().get('cache').get('root_path'), 'servermodel')
if not isdir(self.cache_root_path):
makedirs(join(self.cache_root_path))
async def on_join(self, async def on_join(self,
risotto_context: Context) -> None: risotto_context: Context) -> None:
internal_source = await self.call('v1.source.create', internal_source = await self.call('v1.source.create',
risotto_context, risotto_context,
source_name='internal', source_name='internal',
source_url='none') source_url='none')
self.internal_release_id = await self.call('v1.source.release.create', internal_release = await self.call('v1.source.release.create',
risotto_context, risotto_context,
source_id=internal_source['source_id'], source_id=internal_source['source_id'],
release_name='none') release_name='none')
self.internal_release_id = internal_release['release_id']
async def servermodel_gen_schema(self,
risotto_context: Context,
servermodel_id: int,
applicationservice_id: int) -> None:
dependencies = await self.get_applicationservices(risotto_context,
applicationservice_id)
if release_cache is None:
release_cache = {}
dict_paths = []
for applicationservice_id, applicationservice_infos in dependencies.items():
applicationservice_name, as_release_id = applicationservice_infos
if as_release_id not in release_cache:
release_cache[as_release_id] = await self.call('v1.source.release.get_by_id',
release_id=as_release_id)
dict_paths.append(join(self.source_root_path,
release_cache[as_release_id]['source_name'],
release_cache[as_release_id]['release_distribution'],
'applicationservice',
applicationservice_name,
'dictionaries'))
eolobj = CreoleObjSpace(dtdfilename)
eolobj.create_or_populate_from_xml('creole', dict_paths)
# FIXME extra
# FIXME eosfunc
eosfunc = '/eosfunc.py'
eolobj.space_visitor(eosfunc)
eolobj.save(join(self.cache_root_path, f'{servermodel_id}.xml'))
async def _servermodel_create(self, async def _servermodel_create(self,
risotto_context: Context, risotto_context: Context,
servermodel_name: str, servermodel_name: str,
servermodel_description: str, servermodel_description: str,
servermodel_parents_id, servermodel_parents_id: List[int],
release_id): release_id: int,
servermodel_update = """INSERT INTO Servermodel(ServermodelName, ServermodelDescription, ServermodelParentsId, ServermodelReleaseId, ServermodelApplicationServiceId) VALUES ($1,$2,$3,$4,$5) release_cache: Dict=None) -> Dict:
servermodel_update = """INSERT INTO Servermodel(ServermodelName, ServermodelDescription, ServermodelParentsId, ServermodelReleaseId, ServermodelApplicationServiceId)
VALUES ($1,$2,$3,$4,$5)
RETURNING ServermodelId RETURNING ServermodelId
""" """
as_name = f"local_{servermodel_name}" as_name = f"local_{servermodel_name}"
@ -39,28 +78,32 @@ class Risotto(Controller):
applicationservice_name=as_name, applicationservice_name=as_name,
applicationservice_description=as_description, applicationservice_description=as_description,
release_id=self.internal_release_id) release_id=self.internal_release_id)
servermodel = await risotto_context.connection.fetchval(servermodel_update, applicationservice_id = applicationservice['applicationservice_id']
sm_name, servermodel_id = await risotto_context.connection.fetchval(servermodel_update,
sm_description, servermodel_name,
sm_parentsid, servermodel_description,
servermodel_parents_id,
release_id, release_id,
applicationservice['applicationservice_id']) applicationservice_id)
self.servermodel_gen_schema(risotto_context,
servermodel_id,
applicationservice_id)
return {'servermodel_name': servermodel_name, return {'servermodel_name': servermodel_name,
'servermodel_description': servermodel_description, 'servermodel_description': servermodel_description,
'servermodel_parents_id': servermodel_parents_id, 'servermodel_parents_id': servermodel_parents_id,
'release_id': release_id, 'release_id': release_id,
'servermodel_id': servermodel['servermodel_id']} 'servermodel_id': servermodel_id}
def parse_parents(self, def parse_parents(self,
servermodels: Dict, servermodels: Dict,
servermodel: Dict, servermodel: Dict,
parents: List=None) -> List: parents: List=None) -> List:
if parents is None: if parents is None:
parents = [servermodel] parents = [servermodel['name']]
parent = servermodels[servermodel]['parent'] parent = servermodel['parent']
if parent in servermodels: if parent in servermodels:
parents.append(parent) parents.append(parent)
self.parse_parents(servermodels, parent, parents) self.parse_parents(servermodels, servermodels[parent], parents)
return parents return parents
async def get_servermodel_id_by_name(self, async def get_servermodel_id_by_name(self,
@ -75,11 +118,22 @@ class Risotto(Controller):
@register('v1.servermodel.dataset.updated', None, database=True) @register('v1.servermodel.dataset.updated', None, database=True)
async def servermodel_update(self, async def servermodel_update(self,
risotto_context: Context, risotto_context: Context,
release_path: str, source_name: str,
release_id: int): release_distribution: int):
servermodel_path = join(release_path, 'servermodel') servermodel_path = join(self.source_root_path,
source_name,
release_distribution,
'servermodel')
servermodels = {} servermodels = {}
# for dependencies reason load all servermodels # for dependencies reason load all servermodels
source = await self.call('v1.source.describe',
risotto_context,
source_name=source_name)
release = await self.call('v1.source.release.get_by_distribution',
risotto_context,
source_id=source['source_id'],
release_distribution=release_distribution)
release_id = release['release_id']
for servermodel in listdir(servermodel_path): for servermodel in listdir(servermodel_path):
if not servermodel.endswith('.yml'): if not servermodel.endswith('.yml'):
continue continue
@ -92,10 +146,11 @@ class Risotto(Controller):
if get_config().get('global').get('debug'): if get_config().get('global').get('debug'):
print_exc() print_exc()
raise ExecutionError(_(f'Error while reading {servermodel_description_path}: {err}')) raise ExecutionError(_(f'Error while reading {servermodel_description_path}: {err}'))
servermodels[servermodel_yml['name']] = servermodel_description servermodels[servermodel_description['name']] = servermodel_description
servermodels[servermodel_yml['name']]['done'] = False servermodels[servermodel_description['name']]['done'] = False
for servermodel in servermodels: release_cache = {release['release_id']: release}
for servermodel in servermodels.values():
if not servermodel['done']: if not servermodel['done']:
# parent needs to create before child, so retrieve all parents # parent needs to create before child, so retrieve all parents
parents = self.parse_parents(servermodels, parents = self.parse_parents(servermodels,
@ -114,10 +169,13 @@ class Risotto(Controller):
sm_name = servermodel_description['name'] sm_name = servermodel_description['name']
sm_description = servermodel_description['description'] sm_description = servermodel_description['description']
try: try:
servermodel_id = await self._servermodel_create(sm_name, servermodel_ob = await self._servermodel_create(risotto_context,
sm_name,
sm_description, sm_description,
servermodelparent_id, servermodelparent_id,
release_id)['servermodel_id'] release_id,
release_cache)
servermodel_id = servermodel_ob['servermodel_id']
except Exception as err: except Exception as err:
if get_config().get('global').get('debug'): if get_config().get('global').get('debug'):
print_exc() print_exc()
@ -136,56 +194,59 @@ class Risotto(Controller):
servermodels = await risotto_context.connection.fetch(sql) servermodels = await risotto_context.connection.fetch(sql)
return [dict(r) for r in servermodels] return [dict(r) for r in servermodels]
@register('v1.servermodel.describe', None) async def _parse_depends(self,
async def servermodel_describe(self, inheritance, creolefuncs, servermodel_id, schema, conffiles, resolvdepends, probes): risotto_context,
schema = """<?xml version='1.0' encoding='UTF-8'?> applicationservice_id: int,
<creole> or_depends: list,
<family name="containers"> ids: list) -> None:
<family name="container0" doc="test"> applicationservice_query = """
<family doc="files" name="files"> SELECT ApplicationServiceName, ApplicationServiceDependencies, ApplicationServiceReleaseId
<family doc="file0" name="file0"> FROM applicationservice
<variable doc="" multi="False" name="mkdir" type="boolean"> WHERE applicationserviceid=$1"""
<value>False</value> applicationservice = await risotto_context.connection.fetchval(servermodel_update,
</variable> applicationservice_id)
<variable doc="" multi="False" name="name" type="string"> ids[applicationservice_id] = (applicationserverice['ApplicationServiceName'],
<value>/etc/mailname</value> applicationserverice['ApplicationServiceReleaseId'])
</variable> if applicationserverice['ApplicationServiceDependencies']:
<variable doc="" multi="False" name="rm" type="boolean"> for depend in pplicationserverice['ApplicationServiceDependencies']:
<value>False</value> if isinstance(depend, dict):
</variable> or_depends.append(depend['or'])
<variable doc="" multi="False" name="source" type="string"> elif depend not in ids:
<value>mailname</value> await self._parse_depends(risotto_context,
</variable> depend,
<variable doc="" multi="False" name="activate" type="boolean"> or_depends,
<value>True</value> ids)
</variable>
</family> async def _parse_or_depends(self,
</family> risotto_context,
<property>basic</property> or_depends: list,
</family> ids: list) -> None:
</family> new_or_depends = []
<family doc="" name="creole"> set_ids = set(ids)
<family doc="general" name="general"> for or_depend in or_depends:
<property>normal</property> if not set(or_depend) & set_ids:
<variable doc="No change" multi="False" name="mode_conteneur_actif" type="choice"> applicationservice_id= or_depend[0]
<choice type="string">oui</choice> await self._parse_depends(risotto_context,
<choice type="string">non</choice> applicationservice_id,
<property>mandatory</property> new_or_depends,
<property>normal</property> ids)
<value type="string">non</value> if new_or_depends:
</variable> await self._parse_or_depends(risotto_context,
<leader doc="master" name="master"> new_or_depends,
<property>normal</property> ids)
<variable doc="master" multi="True" name="master" type="string"/>
<variable doc="slave1" multi="True" name="slave1" type="string"> async def get_applicationservices(self,
<property>normal</property> risotto_context,
</variable> applicationservice_id: int) -> list:
<variable doc="slave2" multi="True" name="slave2" type="string"> """Return consolidated dependencies or raise.
<property>normal</property> """
</variable> or_depends = []
</leader> ids = []
</family> await self._parse_depends(risotto_context,
<separators/> applicationservice_id,
</family> or_depends,
</creole>""" ids)
return {'servermodel_id': 1, 'servermodel_name': 'name', 'servermodel_description': 'description', 'release_id': 1, 'schema': schema, 'creolefuncs': ''} await self._parse_or_depends(risotto_context,
or_depends,
ids)
return ids

View File

@ -43,16 +43,19 @@ class Risotto(Controller):
'source_url': source_url, 'source_url': source_url,
'source_id': source_id} 'source_id': source_id}
@register('v1.source.get', None, database=True) @register('v1.source.describe', None, database=True)
async def source_get(self, async def source_describe(self,
risotto_context: Context, risotto_context: Context,
source_name: str) -> Dict: source_name: str) -> Dict:
source_get = """SELECT SourceId as source_id, SourceName as source_name, SourceURL as source_url source_get = """SELECT SourceId as source_id, SourceName as source_name, SourceURL as source_url
FROM Source FROM Source
WHERE SourceName = $1 WHERE SourceName = $1
""" """
return dict(await risotto_context.connection.fetchrow(source_get, source = await risotto_context.connection.fetchrow(source_get,
source_name)) source_name)
if not source:
raise Exception(f'unknown source with name {source_name}')
return dict(source)
@register('v1.source.list', None, database=True) @register('v1.source.list', None, database=True)
async def source_list(self, async def source_list(self,
@ -81,7 +84,7 @@ class Risotto(Controller):
@register('v1.source.release.create', None, database=True) @register('v1.source.release.create', None, database=True)
async def source_release_create(self, async def source_release_create(self,
risotto_context: Context, risotto_context: Context,
source_id: str, source_id: int,
release_name: str) -> Dict: release_name: str) -> Dict:
source_get = """SELECT SourceId as source_id, SourceName as source_name, SourceURL as source_url source_get = """SELECT SourceId as source_id, SourceName as source_name, SourceURL as source_url
FROM Source FROM Source
@ -104,6 +107,34 @@ class Risotto(Controller):
@register('v1.source.release.list', None, database=True) @register('v1.source.release.list', None, database=True)
async def release_list(self, async def release_list(self,
risotto_context): risotto_context):
release_query = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name FROM Release, Source WHERE Source.SourceId=Release.ReleaseSourceId""" release_query = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name
result = await risotto_context.connection.fetch(release_name) FROM Release, Source
WHERE Source.SourceId=Release.ReleaseSourceId"""
result = await risotto_context.connection.fetch(release_query)
return [dict(r) for r in result] return [dict(r) for r in result]
@register('v1.source.release.get_by_id', None, database=True)
async def release_get_by_id(self,
risotto_context: Context,
release_id: int) -> Dict:
release_query = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name
FROM Release, Source
WHERE Release.ReleaseId = $1"""
result = await risotto_context.connection.fetchrow(release_query,
release_id)
return dict(result)
@register('v1.source.release.get_by_distribution', None, database=True)
async def release_get_by_distribution(self,
risotto_context: Context,
source_id: int,
release_distribution: str) -> Dict:
release_query = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name
FROM Release, Source
WHERE Source.SourceId = $1 AND Release.ReleaseName = $2"""
result = await risotto_context.connection.fetchrow(release_query,
source_id,
release_distribution)
if not result:
raise Exception(f'unknown distribution {release_distribution} with source {source_id}')
return dict(result)

View File

@ -14,56 +14,62 @@ class Risotto(Controller):
'servermodel_description': 'description2', 'servermodel_description': 'description2',
'servermodel_parents_id': [1]}] 'servermodel_parents_id': [1]}]
@register('v1.servermodel.describe', None) # @register('v1.servermodel.describe', None)
async def servermodel_describe(self, inheritance, creolefuncs, servermodel_id, schema, conffiles, resolvdepends, probes): # async def servermodel_describe(self,
schema = """<?xml version='1.0' encoding='UTF-8'?> # servermodel_id,
<creole> # inheritance,
<family name="containers"> # resolv_depends):
<family name="container0" doc="test"> # schema = """<?xml version='1.0' encoding='UTF-8'?>
<family doc="files" name="files"> #<creole>
<family doc="file0" name="file0"> # <family name="containers">
<variable doc="" multi="False" name="mkdir" type="boolean"> # <family name="container0" doc="test">
<value>False</value> # <family doc="files" name="files">
</variable> # <family doc="file0" name="file0">
<variable doc="" multi="False" name="name" type="string"> # <variable doc="" multi="False" name="mkdir" type="boolean">
<value>/etc/mailname</value> # <value>False</value>
</variable> # </variable>
<variable doc="" multi="False" name="rm" type="boolean"> # <variable doc="" multi="False" name="name" type="string">
<value>False</value> # <value>/etc/mailname</value>
</variable> # </variable>
<variable doc="" multi="False" name="source" type="string"> # <variable doc="" multi="False" name="rm" type="boolean">
<value>mailname</value> # <value>False</value>
</variable> # </variable>
<variable doc="" multi="False" name="activate" type="boolean"> # <variable doc="" multi="False" name="source" type="string">
<value>True</value> # <value>mailname</value>
</variable> # </variable>
</family> # <variable doc="" multi="False" name="activate" type="boolean">
</family> # <value>True</value>
<property>basic</property> # </variable>
</family> # </family>
</family> # </family>
<family doc="" name="creole"> # <property>basic</property>
<family doc="general" name="general"> # </family>
<property>normal</property> # </family>
<variable doc="No change" multi="False" name="mode_conteneur_actif" type="choice"> # <family doc="" name="creole">
<choice type="string">oui</choice> # <family doc="general" name="general">
<choice type="string">non</choice> # <property>normal</property>
<property>mandatory</property> # <variable doc="No change" multi="False" name="mode_conteneur_actif" type="choice">
<property>normal</property> # <choice type="string">oui</choice>
<value type="string">non</value> # <choice type="string">non</choice>
</variable> # <property>mandatory</property>
<leader doc="master" name="master"> # <property>normal</property>
<property>normal</property> # <value type="string">non</value>
<variable doc="master" multi="True" name="master" type="string"/> # </variable>
<variable doc="slave1" multi="True" name="slave1" type="string"> # <leader doc="master" name="master">
<property>normal</property> # <property>normal</property>
</variable> # <variable doc="master" multi="True" name="master" type="string"/>
<variable doc="slave2" multi="True" name="slave2" type="string"> # <variable doc="slave1" multi="True" name="slave1" type="string">
<property>normal</property> # <property>normal</property>
</variable> # </variable>
</leader> # <variable doc="slave2" multi="True" name="slave2" type="string">
</family> # <property>normal</property>
<separators/> # </variable>
</family> # </leader>
</creole>""" # </family>
return {'servermodel_id': 1, 'servermodel_name': 'name', 'servermodel_description': 'description', 'release_id': 1, 'schema': schema, 'creolefuncs': ''} # <separators/>
# </family>
#</creole>"""
# return {'servermodel_id': 1,
# 'servermodel_name': 'name',
# 'servermodel_description': 'description',
# 'release_id': 1}