WIP

README.md

@@ -29,4 +29,9 @@ psql -U postgres -h localhost -c "GRANT ALL ON DATABASE risotto TO risotto;"
 psql -U postgres -h localhost -c "CREATE EXTENSION hstore;" risotto
 ```
 
+Gestion de la base de données avec Sqitch
 
+```
+cpanm --quiet --notest App::Sqitch
+sqitch init risotto --uri https://forge.cadoles.com/Infra/risotto --engine pg
+```

messages/v1/messages/applicationservices.create.yml

@@ -0,0 +1,46 @@
+---
+uri: applicationservice.create
+
+description: |
+  Créé un service applicatif.
+
+pattern: rpc
+
+public: true
+
+parameters:
+  applicationservice_name:
+    type: String
+    shortarg: n
+    description: |
+      Nom du service applicatif à créer.
+  applicationservice_description:
+    type: String
+    shortarg: d
+    description: |
+      Description du service applicatif à créer.
+  release_id:
+    type: Number
+    shortarg: r
+    description: |
+      Identifiant de la version associée au service applicatif.
+
+response:
+  type: ApplicationService
+  description: Informations sur le modèle de serveur créé.
+
+errors:
+  - uri: servermodel.create.error.database_not_available
+  - uri: servermodel.create.error.duplicate_servermodel
+  - uri: servermodel.create.error.invalid_parentservermodel_id
+  - uri: servermodel.create.error.invalid_source_id
+  - uri: servermodel.create.error.unknown_parentservermodel_id
+  - uri: servermodel.create.error.unknown_source_id
+  - uri: servermodel.create.error.servermodelname_not_provided
+
+related:
+  - servermodel.list
+  - servermodel.describe
+  - servermodel.update
+  - servermodel.delete
+  - servermodel.event

messages/v1/messages/applicationservices.dataset.updated.yml

@@ -0,0 +1,24 @@
+uri: applicationservices.dataset.updated
+
+description: |
+  Initialise la table pour les services applicatifs.
+
+pattern: rpc
+
+public: true
+
+domain: applicationservices-domain
+
+parameters:
+  release_path:
+    type: String
+    shortarg: s
+    description: Nom de la source.
+  release_id:
+    type: Number
+    shortarg: r
+    description: Nom de la version.
+
+response:
+  type: ReturnStatus
+  description: Code de retour sur l’injection des services applicatifs en base.

messages/v1/messages/servermodel.dataset.updated.yml

@@ -0,0 +1,24 @@
+uri: servermodel.dataset.updated
+
+description: |
+  Initialise la table pour les modèles de serveur.
+
+pattern: rpc
+
+public: true
+
+domain: servermodel-domain
+
+parameters:
+  release_path:
+    type: String
+    shortarg: s
+    description: Nom de la source.
+  release_id:
+    type: Number
+    shortarg: r
+    description: Nom de la version.
+
+response:
+  type: ReturnStatus
+  description: Code de retour sur l’injection des modèles de serveur en base.

messages/v1/messages/servermodel.init.yml

@@ -1,27 +0,0 @@
-uri: servermodel.init
-
-description: |
-  Initialise la table pour les modèles de serveur.
-
-sampleuse: |
-  zephir-client servermodel.init
-
-pattern: rpc
-
-public: true
-
-domain: servermodel-domain
-
-response:
-  type: ReturnStatus
-  description: Liste des modèles de serveur disponibles.
-
-errors:
-  - uri: servermodel.list.error.database_engine_not_available
-
-related:
-  - servermodel.describe
-  - servermodel.create
-  - servermodel.update
-  - servermodel.delete
-  - servermodel.event

messages/v1/messages/source.dataset.updated.yml

@@ -0,0 +1,28 @@
+uri: source.dataset.updated
+
+description: |
+  Initialise la table pour les versions.
+
+pattern: rpc
+
+public: true
+
+domain: source-domain
+
+parameters:
+  source_name:
+    type: String
+    shortarg: s
+    description: Nom de la source.
+  source_url:
+    type: String
+    shortarg: u
+    description: URL de la source.
+  release_name:
+    type: String
+    shortarg: r
+    description: Nom de la version.
+
+response:
+  type: Release
+  description: Informations sur la version injectée en base.

messages/v1/messages/source.release.list.yml

@@ -0,0 +1,15 @@
+---
+uri: source.release.list
+
+description: |
+  Retourne la liste des versions.
+
+pattern: rpc
+
+public: true
+
+domain: source-domain
+
+response:
+  type: '[]Release'
+  description: Liste des versions disponibles.

messages/v1/types/applicationservice.yml

@@ -0,0 +1,31 @@
+---
+title: ApplicationService
+type: object
+description: Description d'un modèle de serveur.
+properties:
+  applicationservice_id:
+    type: number
+    description: ID du service applicatif.
+  applicationservice_name:
+    type: string
+    description: Nom du service applicatif.
+  applicationservice_description:
+    type: string
+    description: Description du service applicatif.
+  release_id:
+    type: number
+    ref: Version.ReleaseId
+    description: Version du service applicatif.
+  applicationservice_dependencies:
+    type: array
+    items:
+      type: integer
+    description: Liste des services applicatifs déclarés en dépendance de ce service applicatif.
+
+required:
+  - servermodelid
+  - servermodelname
+  - servermodeldescription
+  - servermodelsubreleaseid
+  - sourceid
+  - subreleasename

messages/v1/types/global.returnstatus.yml

@@ -6,9 +6,9 @@ properties:
   retcode:
     type: number
     description: Code de retour de la commande.
-  return:
+  returns:
     type: string
     description: Retour de la commande.
 required:
   - retcode
-  - return
+  - returns

messages/v1/types/release.yml

@@ -0,0 +1,24 @@
+---
+title: Release
+type: object
+description: Description de la version.
+properties:
+  release_id:
+    type: number
+    description: Identifiant de la version.
+  release_name:
+    type: string
+    description: Le nom de la version.
+  source_url:
+    type: string
+    description: URL de la source.
+    ref: Source.ReleaseId
+  source_name:
+    type: string
+    description: Le nom de la source.
+required:
+  - release_id
+  - release_name
+  - source_name
+  - source_url
+

script/database_manager.py

@@ -0,0 +1,84 @@
+import asyncpg
+import asyncio
+from risotto.config import get_config
+
+VERSION_INIT = """
+-- Création de la table Source
+CREATE TABLE Source (
+  SourceId SERIAL PRIMARY KEY,
+  SourceName VARCHAR(255) NOT NULL UNIQUE,
+  SourceURL TEXT
+);
+
+-- Création de la table Release
+CREATE TABLE Release (
+  ReleaseId SERIAL PRIMARY KEY,
+  ReleaseName VARCHAR(255) NOT NULL,
+  ReleaseSourceId INTEGER NOT NULL,
+  UNIQUE (ReleaseName, ReleaseSourceId),
+  FOREIGN KEY (ReleaseSourceId) REFERENCES Source(SourceId)
+);
+
+-- Création de la table ServerModel
+CREATE TABLE ServerModel (
+  ServerModelId SERIAL PRIMARY KEY,
+  ServerModelName VARCHAR(255) NOT NULL,
+  ServerModelDescription VARCHAR(255) NOT NULL,
+  ServerModelParentsId INTEGER [] DEFAULT '{}',
+  ServerModelReleaseId INTEGER NOT NULL,
+  ServerModelApplicationServiceId INTEGER NOT NULL,
+  ServerModelUsers hstore,
+  UNIQUE (ServerModelName, ServerModelReleaseId)
+);
+
+-- Création de la table ApplicationService
+CREATE TABLE ApplicationService (
+  ApplicationServiceId SERIAL PRIMARY KEY,
+  ApplicationServiceName VARCHAR(255) NOT NULL,
+  ApplicationServiceDescription VARCHAR(255) NOT NULL,
+  ApplicationServiceReleaseId INTEGER NOT NULL,
+  ApplicationServiceDependencies JSON,
+  UNIQUE (ApplicationServiceName, ApplicationServiceReleaseId)
+);
+
+-- Création de la table de jointure ApplicationServiceProvides
+CREATE TABLE ApplicationServiceProvides (
+  ApplicationServiceId INTEGER NOT NULL,
+  VirtualApplicationServiceId INTEGER NOT NULL,
+  FOREIGN KEY (ApplicationServiceId) REFERENCES ApplicationService(ApplicationServiceId),
+  FOREIGN KEY (VirtualApplicationServiceId) REFERENCES ApplicationService(ApplicationServiceId),
+  PRIMARY KEY (ApplicationServiceId, VirtualApplicationServiceId)
+);
+
+-- Création de la table Package
+CREATE TABLE Package (
+  PackageId SERIAL PRIMARY KEY,
+  PackageApplicationServiceId INTEGER,
+  PackageName VARCHAR(255) NOT NULL,
+  FOREIGN KEY (PackageApplicationServiceId) REFERENCES ApplicationService(ApplicationServiceId)
+);
+
+-- Création de la table Document
+CREATE TABLE Document (
+  DocumentId SERIAL PRIMARY KEY,
+  DocumentServiceId INTEGER,
+  DocumentName VARCHAR(255) NOT NULL,
+  DocumentPath TEXT,
+  DocumentOwner VARCHAR(255) DEFAULT 'root',
+  DocumentGroup VARCHAR(255) DEFAULT 'root',
+  DocumentMode VARCHAR(10) DEFAULT '0644',
+  DocumentType VARCHAR(100) CHECK ( DocumentType IN ('probes', 'aggregated_dico', 'dico', 'template', 'pretemplate', 'posttemplate', 'preservice', 'postservice', 'creolefuncs', 'file') ),
+  DocumentSHASUM VARCHAR(255),
+  DocumentContent BYTEA,
+  FOREIGN KEY (DocumentServiceId) REFERENCES ApplicationService(ApplicationServiceId)
+);
+"""
+
+async def main():
+    db_conf = get_config().get('database')
+    pool = await asyncpg.create_pool(database=db_conf.get('dbname'), user=db_conf.get('user'))
+    async with pool.acquire() as connection:
+        async with connection.transaction():
+            returns = await connection.execute(VERSION_INIT)
+    
+asyncio.run(main())

src/risotto/config.py

@@ -1,7 +1,7 @@
 HTTP_PORT = 8080
 MESSAGE_ROOT_PATH = 'messages'
 ROOT_CACHE_DIR = 'cache'
-DEBUG = False
+DEBUG = True
 DATABASE_DIR = 'database'
 INTERNAL_USER = 'internal'
 CONFIGURATION_DIR = 'configurations'
@@ -20,7 +20,7 @@ def get_config():
                          },
             'http_server': {'port': 8080},
             'global': {'message_root_path': 'messages',
-                       'debug': False,
+                       'debug': DEBUG,
                        'internal_user': 'internal',
                        'rougail_dtd_path': '../rougail/data/creole.dtd'}
             }

src/risotto/dispatcher.py

@@ -92,20 +92,20 @@ class CallDispatcher:
         try:
             tiramisu_config = self.load_kwargs_to_config(risotto_context,
                                                          kwargs)
-            obj = self.messages[version][message]
+            function_obj = self.messages[version][message]
             kw = tiramisu_config.option(message).value.dict()
-            risotto_context.function = obj['function']
+            risotto_context.function = function_obj['function']
-            if obj['risotto_context']:
+            if function_obj['risotto_context']:
                 kw['risotto_context'] = risotto_context
-            if 'database' in obj and obj['database']:
+            if function_obj['database']:
                 db_conf = get_config().get('database')
                 pool = await asyncpg.create_pool(database=db_conf.get('dbname'), user=db_conf.get('user'))
                 async with pool.acquire() as connection:
                     risotto_context.connection = connection
                     async with connection.transaction():
-                        returns = await risotto_context.function(self.injected_self[obj['module']], **kw)
+                        returns = await risotto_context.function(self.injected_self[function_obj['module']], **kw)
             else:
-                returns = await risotto_context.function(self.injected_self[obj['module']], **kw)
+                returns = await risotto_context.function(self.injected_self[function_obj['module']], **kw)
         except CallError as err:
             raise err
         except Exception as err:
@@ -124,8 +124,8 @@ class CallDispatcher:
                      kwargs,
                      _(f'returns {returns}'))
         # notification
-        if obj.get('notification'):
+        if function_obj.get('notification'):
-            notif_version, notif_message = obj['notification'].split('.', 1)
+            notif_version, notif_message = function_obj['notification'].split('.', 1)
             if not isinstance(returns, list):
                 send_returns = [returns]
             else:
@@ -174,7 +174,15 @@ class PublishDispatcher:
                 if function_obj['risotto_context']:
                     kw['risotto_context'] = risotto_context
                 # send event
-                returns = await function(self.injected_self[function_obj['module']], **kw)
+                if function_obj['database']:
+                    db_conf = get_config().get('database')
+                    pool = await asyncpg.create_pool(database=db_conf.get('dbname'), user=db_conf.get('user'))
+                    async with pool.acquire() as connection:
+                        risotto_context.connection = connection
+                        async with connection.transaction():
+                            returns = await function(self.injected_self[function_obj['module']], **kw)
+                else:
+                    returns = await function(self.injected_self[function_obj['module']], **kw)
             except Exception as err:
                 if DEBUG:
                     print_exc()

src/risotto/error.py

@@ -8,3 +8,7 @@ class CallError(Exception):
 
 class NotAllowedError(Exception):
     pass
+
+
+class ExecutionError(Exception):
+    pass

src/risotto/http.py

@@ -61,19 +61,24 @@ class extra_route_handler:
 
 
 async def handle(request):
-    version, uri = request.match_info.get_info()['path'].rsplit('/', 2)[-2:]
+    version, message = request.match_info.get_info()['path'].rsplit('/', 2)[-2:]
     risotto_context = create_context(request)
     kwargs = await request.json()
     try:
-        text = await dispatcher.call(version,
+        pattern = dispatcher.messages[version][message]['pattern']
-                                     uri,
+        if pattern == 'rpc':
-                                     risotto_context,
+            method = dispatcher.call
-                                     public_only=True,
+        else:
-                                     **kwargs)
+            method = dispatcher.publish
+        text = await method(version,
+                            message,
+                            risotto_context,
+                            public_only=True,
+                            **kwargs)
     except NotAllowedError as err:
         raise HTTPNotFound(reason=str(err))
     except CallError as err:
-        raise HTTPBadRequest(reason=str(err))
+        raise HTTPBadRequest(reason=str(err).replace('\n', ' '))
     except Exception as err:
         if DEBUG:
             print_exc()

src/risotto/message/message.py

@@ -51,7 +51,7 @@ class MessageDefinition:
                  'related',
                  'response')
 
-    def __init__(self, raw_def):
+    def __init__(self, raw_def, message):
         # default value for non mandatory key
         self.version = u''
         self.parameters = OrderedDict()
@@ -91,6 +91,8 @@ class MessageDefinition:
         # message with pattern = error must be public
         if self.public is False and self.pattern == 'error':
             raise Exception(_('Error message must be public : {}').format(self.uri))
+        if self.uri != message:
+            raise Exception(_(f'yaml file name "{message}.yml" does not match uri "{self.uri}"'))
 
 
 class ParameterDefinition:
@@ -217,8 +219,8 @@ def _parse_parameters(raw_defs):
     return parameters
 
 
-def parse_definition(filename: str):
+def parse_definition(filecontent: bytes, message: str):
-    return MessageDefinition(load(filename, Loader=SafeLoader))
+    return MessageDefinition(load(filecontent, Loader=SafeLoader), message)
 
 def is_message_defined(uri):
     version, message = split_message_uri(uri)
@@ -231,7 +233,7 @@ def get_message(uri):
         version, message = split_message_uri(uri)
         path = get_message_file_path(version, message)
         with open(path, "r") as message_file:
-            message_content = parse_definition(message_file.read())
+            message_content = parse_definition(message_file.read(), message)
         message_content.version = version
         return message_content
     except Exception as err:
@@ -478,13 +480,7 @@ def _parse_responses(message_def,
     """build option with returns
     """
     if message_def.response.parameters is None:
-        raise Exception('not implemented yet')
+        raise Exception('uri "{}" did not returned any valid parameters.'.format(message_def.uri))
-        #name = 'response'
-        #keys['']['columns'][name] = {'description': message_def.response.description,
-        #                             'type': message_def.response.type}
-        #responses = {}
-        #responses['keys'] = keys
-        #return responses
 
     options = []
     names = []

src/risotto/register.py

@@ -218,6 +218,7 @@ class RegisterDispatcher:
         dico = {'module': module_name,
                 'function': function,
                 'arguments': function_args,
+                'database': database,
                 'risotto_context': inject_risotto_context}
         if notification and notification is not undefined:
             dico['notification'] = notification
@@ -238,7 +239,10 @@ class RegisterDispatcher:
         for version, messages in self.messages.items():
             for message, message_obj in messages.items():
                 if not 'functions' in message_obj and not 'function' in message_obj:
-                    missing_messages.append(message)
+                    if message_obj['pattern'] == 'event':
+                        print(f'{message} prêche dans le désert')
+                    else:
+                        missing_messages.append(message)
         if missing_messages:
             raise RegistrationError(_(f'missing uri {missing_messages}'))
 

src/risotto/services/applicationservices/__init__.py

@@ -0,0 +1 @@
+from .application_services import Risotto

src/risotto/services/applicationservices/applicationservices.py

@@ -0,0 +1,31 @@
+from ...controller import Controller
+from ...register import register
+
+class Risotto(Controller):
+    @register('v1.applicationservice.create', None, database=True)
+    async def applicationservice_create(self, risotto_context, applicationservice_name, applicationservice_description, release_id):
+        applicationservice_update_query = """INSERT INTO ApplicationService(ApplicationServiceName, ApplicationServiceDescription, ApplicationServiceReleaseId) VALUES ($1,$2,$3) 
+        RETURNING ApplicationServiceId
+        """
+        applicationservice_id = await risotto_context.connection.fetchval(applicationservice_update_query, applicationservice_description['name'], applicationservice_description['description'], release_id)
+        return {'applicationservice_name': applicationservice_name, 'applicationservice_description': applicationservice_description, 'applicationservice_id': applicationservice_id}
+
+    @register('v1.applicationservice.dataset.updated', None, database=True)
+    async def applicationservice_update(self, risotto_context, release_path, release_id):
+        applicationservice_path = os.path.join(release_path, 'applicationservice')
+        for service in os.listdir(applicationservice_path):
+            try:
+                applicationservice_description_path = os.path.join(applicationservice_path, service, 'applicationservice.yml')
+                with open(applicationservice_description_path, 'r') as applicationservice_yml:
+                    applicationservice_description = yaml.load(applicationservice_yml, Loader=yaml.SafeLoader)
+            except Exception as err:
+                if get_config().get('global').get('debug'):
+                    print_exc()
+                raise ExecutionError(_(f'Error while reading {applicationservice_description_path}: {err}'))
+            try:
+                await self.applicationservice_create(risotto_context, applicationservice_description['name'], applicationservice_description['description'], release_id)
+            except Exception as err:
+                if get_config().get('global').get('debug'):
+                    print_exc()
+                raise ExecutionError(_(f"Error while injecting application service {applicationservice_description['name']} in database: {err}"))
+        return {'retcode': 0, 'returns': _('Application Services successfully loaded')}

src/risotto/services/servermodel/servermodel.py

@@ -1,27 +1,35 @@
 from ...controller import Controller
 from ...register import register
-
+from ...utils import _
-sql_init = """
+import os
--- Création de la table ServerModel
+import yaml
-CREATE TABLE ServerModel (
+from traceback import print_exc
-  ServerModelId SERIAL PRIMARY KEY,
+from ...config import get_config
-  ServerModelName VARCHAR(255) NOT NULL,
+from ...error import ExecutionError
-  ServerModelDescription VARCHAR(255) NOT NULL,
-  ServerModelSourceId INTEGER NOT NULL,
-  ServerModelParentId INTEGER,
-  ServerModelSubReleaseId INTEGER NOT NULL,
-  ServerModelSubReleaseName VARCHAR(255) NOT NULL,
-  UNIQUE (ServerModelName, ServerModelSubReleaseId)
-);
-"""
 
 
 class Risotto(Controller):
-    @register('v1.servermodel.init', None, database=True)
+    @register('v1.servermodel.dataset.updated', None, database=True)
-    async def servermodel_init(self, risotto_context):
+    async def servermodel_update(self, risotto_context, release_path, release_id, applicationservice_įd):
-        result = await risotto_context.connection.execute(sql_init)
+        applicationservice_update = """INSERT INTO ApplicationService(ApplicationServiceName, ApplicationServiceDescription, ApplicationServiceReleaseId) VALUES ($1,$2,$3) 
-        return {'retcode': 0, 'return': result}
+        RETURNING ApplicationServiceId
-      
+        """
+        servermodel_path = os.path.join(release_path, 'servermodel')
+        for servermodel in os.listdir(servermodel_path):
+            try:
+                with open(os.path.join(servermodel_path, servermodel), 'r') as applicationservice_yml:
+                    applicationservice_description = yaml.load(applicationservice_yml, Loader=yaml.SafeLoader)
+            except Exception as err:
+                if get_config().get('global').get('debug'):
+                    print_exc()
+                raise ExecutionError(_(f'Error while reading {applicationservice_description_path}: {err}'))
+            try:
+                await risotto_context.connection.fetch(applicationservice_update, applicationservice_description['name'], applicationservice_description['description'], release_id)
+            except Exception as err:
+                if get_config().get('global').get('debug'):
+                    print_exc()
+                raise ExecutionError(_(f"Error while injecting application service {applicationservice_description['name']} in database: {err}"))
+        return {'retcode': 0, 'returns': _('Application Services successfully loaded')}
 
     @register('v1.servermodel.list', None, database=True)
     async def servermodel_list(self, risotto_context, sourceid):

src/risotto/services/source/__init__.py

@@ -0,0 +1 @@
+from .source import Risotto

src/risotto/services/source/source.py

@@ -0,0 +1,41 @@
+from ...controller import Controller
+from ...register import register
+
+VERSION_INIT = """
+-- Création de la table Source
+CREATE TABLE Source (
+  SourceId SERIAL PRIMARY KEY,
+  SourceName VARCHAR(255) NOT NULL UNIQUE,
+  SourceURL TEXT
+);
+
+-- Création de la table Release
+CREATE TABLE Release (
+  ReleaseId SERIAL PRIMARY KEY,
+  ReleaseName VARCHAR(255) NOT NULL,
+  ReleaseSourceId INTEGER NOT NULL,
+  UNIQUE (ReleaseName, ReleaseSourceId),
+  FOREIGN KEY (ReleaseSourceId) REFERENCES Source(SourceId)
+);
+"""
+RELEASE_QUERY = """SELECT ReleaseId as release_id, SourceName as source_name, SourceURL as source_url, ReleaseName as release_name FROM Release, Source WHERE Source.SourceId=Release.ReleaseSourceId"""
+
+class Risotto(Controller):
+
+    @register('v1.source.dataset.updated', None, database=True)
+    async def version_update(self, risotto_context, source_name, source_url, release_name):
+        source_upsert = """INSERT INTO Source(SourceName, SourceURL) VALUES ($1, $2)
+        ON CONFLICT (SourceName) DO UPDATE SET SourceURL = $2
+        RETURNING SourceId
+        """
+        release_insert = """INSERT INTO Release(ReleaseName, ReleaseSourceId) VALUES ($1, $2)
+        RETURNING ReleaseId
+        """
+        source_id = await risotto_context.connection.fetchval(source_upsert, source_name, source_url)
+        result = await risotto_context.connection.fetchval(release_insert, release_name, source_id)
+        return {'release_id': result, 'source_name': source_name, 'source_url': source_url, 'release_name': release_name}
+
+    @register('v1.source.release.list', None, database=True)
+    async def release_list(self, risotto_context):
+        result = await risotto_context.connection.fetch(RELEASE_QUERY)
+        return [dict(r) for r in result]