test if a source is loaded

src/risotto/config.py

@@ -113,12 +113,12 @@ if 'PASSWORD_ADMIN_EMAIL' in environ:
     PASSWORD_ADMIN_EMAIL = environ['PASSWORD_ADMIN_EMAIL']
 else:
     # this parameter is mandatory
-    PASSWORD_ADMIN_EMAIL = config['PASSWORD_ADMIN_EMAIL']
+    PASSWORD_ADMIN_EMAIL = config.get('PASSWORD_ADMIN_EMAIL', 'XXX')
 if 'PASSWORD_ADMIN_PASSWORD' in environ:
     PASSWORD_ADMIN_PASSWORD = environ['PASSWORD_ADMIN_PASSWORD']
 else:
     # this parameter is mandatory
-    PASSWORD_ADMIN_PASSWORD = config['PASSWORD_ADMIN_PASSWORD']
+    PASSWORD_ADMIN_PASSWORD = config.get('PASSWORD_ADMIN_PASSWORD', 'XXX')
 if 'PASSWORD_DEVICE_IDENTIFIER' in environ:
     PASSWORD_DEVICE_IDENTIFIER = environ['PASSWORD_DEVICE_IDENTIFIER']
 else:
@@ -135,11 +135,11 @@ else:
 if 'PKI_ADMIN_PASSWORD' in environ:
     PKI_ADMIN_PASSWORD = environ['PKI_ADMIN_PASSWORD']
 else:
-    PKI_ADMIN_PASSWORD = config['PKI_ADMIN_PASSWORD']
+    PKI_ADMIN_PASSWORD = config.get('PKI_ADMIN_PASSWORD', 'XXX')
 if 'PKI_ADMIN_EMAIL' in environ:
     PKI_ADMIN_EMAIL = environ['PKI_ADMIN_EMAIL']
 else:
-    PKI_ADMIN_EMAIL = config['PKI_ADMIN_EMAIL']
+    PKI_ADMIN_EMAIL = config.get('PKI_ADMIN_EMAIL', 'XXX')
 if 'PKI_URL' in environ:
     PKI_URL = environ['PKI_URL']
 else:

src/risotto/controller.py

@@ -12,7 +12,7 @@ except:
 from .config import get_config
 from .utils import _, tiramisu_display_name
 from .logger import log
-from .dispatcher import dispatcher
+from .dispatcher import get_dispatcher
 from .context import Context
 
 
@@ -25,7 +25,7 @@ class Controller:
     def __init__(self,
                  test: bool,
                  ) -> None:
-        pass
+        self.dispatcher = get_dispatcher()
 
     async def call(self,
                    uri: str,
@@ -42,11 +42,11 @@ class Controller:
         module = message.split('.', 1)[0]
         if current_module != module:
             raise ValueError(_(f'cannot call to external module ("{module}") to the URI "{uri}" from "{current_module}"'))
-        return await dispatcher.call(version,
-                                     message,
-                                     risotto_context,
-                                     **kwargs,
-                                     )
+        return await self.dispatcher.call(version,
+                                          message,
+                                          risotto_context,
+                                          **kwargs,
+                                          )
 
     async def publish(self,
                       uri: str,
@@ -58,19 +58,19 @@ class Controller:
         if args:
             raise ValueError(_(f'the URI "{uri}" can only be published with keyword arguments'))
         version, message = uri.split('.', 1)
-        await dispatcher.publish(version,
-                                 message,
-                                 risotto_context,
-                                 **kwargs,
-                                 )
+        await self.dispatcher.publish(version,
+                                      message,
+                                      risotto_context,
+                                      **kwargs,
+                                      )
 
-    @staticmethod
-    async def check_role(uri: str,
+    async def check_role(self,
+                         uri: str,
                          username: str,
                          **kwargs: dict,
                          ) -> None:
         # create a new config
-        async with await Config(dispatcher.option) as config:
+        async with await Config(self.dispatcher.option) as config:
             await config.property.read_write()
             await config.option('message').value.set(uri)
             subconfig = config.option(uri)
@@ -83,10 +83,10 @@ class Controller:
                     raise ValueError(_(f'unknown parameter in "{uri}": "{key}"'))
                 except ValueOptionError as err:
                     raise ValueError(_(f'invalid parameter in "{uri}": {err}'))
-            await dispatcher.check_role(subconfig,
-                                        username,
-                                        uri,
-                                        )
+            await self.dispatcher.check_role(subconfig,
+                                             username,
+                                             uri,
+                                             )
 
     async def on_join(self,
                       risotto_context,
@@ -98,20 +98,23 @@ class TiramisuController(Controller):
     def __init__(self,
                  test: bool,
                  ) -> None:
+        self.source_imported = None
         if not 'dataset_name' in vars(self):
             raise Exception(f'please specify "dataset_name" to "{self.__class__.__name__}"')
         self.tiramisu_cache_root_path = join(get_config()['cache']['root_path'], self.dataset_name)
+        super().__init__(test)
+        self.internal_source_name = get_config()['servermodel']['internal_source']
         if not test:
             db_conf = get_config()['database']['tiramisu_dsn']
             self.save_storage = Storage(engine='postgres')
             self.save_storage.setting(dsn=db_conf)
         if self.dataset_name != 'servermodel':
             self.optiondescription = None
-            dispatcher.set_function('v1.setting.dataset.updated',
-                                    None,
-                                    TiramisuController.dataset_updated,
-                                    self.__class__.__module__,
-                                    )
+            self.dispatcher.set_function('v1.setting.dataset.updated',
+                                         None,
+                                         TiramisuController.dataset_updated,
+                                         self.__class__.__module__,
+                                         )
 
     async def on_join(self,
                       risotto_context: Context,
@@ -129,6 +132,12 @@ class TiramisuController(Controller):
                                risotto_context: Context,
                                ) -> None:
         sources = await self.get_sources(risotto_context)
+        source_imported = sources != [self.internal_source_name]
+        if source_imported and self.source_imported is False:
+            await self.load_datas(risotto_context)
+        self.source_imported = source_imported
+        if not self.source_imported:
+            return
         self._aggregate_tiramisu_funcs(sources)
         self._convert_dictionaries_to_tiramisu(sources)
 
@@ -174,7 +183,9 @@ except:
                             funcs.write(fh.read())
                             funcs.write(b'\n')
 
-    def _convert_dictionaries_to_tiramisu(self, sources: list) -> None:
+    def _convert_dictionaries_to_tiramisu(self,
+                                          sources: list,
+                                          ) -> None:
         funcs_file = join(self.tiramisu_cache_root_path, 'funcs.py')
         tiramisu_file = join(self.tiramisu_cache_root_path, 'tiramisu.py')
         dictionaries_dir = join(self.tiramisu_cache_root_path, 'dictionaries')

src/risotto/dispatcher.py

@@ -18,6 +18,9 @@ from .context import Context
 from . import register
 
 
+DISPATCHER = None
+
+
 class CallDispatcher:
     async def valid_call_returns(self,
                                  risotto_context: Context,
@@ -492,5 +495,9 @@ class Dispatcher(register.RegisterDispatcher,
             return returns
 
 
-dispatcher = Dispatcher()
-register.dispatcher = dispatcher
+def get_dispatcher():
+    global DISPATCHER
+    if DISPATCHER is None:
+        DISPATCHER = Dispatcher()
+        register.dispatcher = DISPATCHER
+    return DISPATCHER

src/risotto/http.py

@@ -7,7 +7,7 @@ except:
     from tiramisu import Config, default_storage
 
 
-from .dispatcher import dispatcher
+from .dispatcher import get_dispatcher
 from .utils import _
 from .context import Context
 from .error import CallError, NotAllowedError, RegistrationError
@@ -70,6 +70,7 @@ class extra_route_handler:
         if function_name != 'risotto.http':
             risotto_module_name, submodule_name = function_name.split('.', 2)[:-1]
             module_name = risotto_module_name.split('_')[-1]
+            dispatcher = get_dispatcher()
             kwargs['self'] = dispatcher.injected_self[module_name + '.' + submodule_name]
         try:
             returns = await cls.function(**kwargs)
@@ -93,6 +94,7 @@ async def handle(request):
     risotto_context = create_context(request)
     kwargs = await request.json()
     try:
+        dispatcher = get_dispatcher()
         pattern = dispatcher.messages[version][message]['pattern']
         if pattern == 'rpc':
             method = dispatcher.call
@@ -142,6 +144,7 @@ async def api(request,
         # check all URI that have an associated role
         # all URI without role is concidered has a private URI
         uris = []
+        dispatcher = get_dispatcher()
         async with dispatcher.pool.acquire() as connection:
             async with connection.transaction():
                 # Check role with ACL
@@ -167,6 +170,7 @@ async def get_app(loop):
     """ build all routes
     """
     global extra_routes, extra_statics
+    dispatcher = get_dispatcher()
     services.link_to_dispatcher(dispatcher)
     app = Application(loop=loop)
     routes = []

src/risotto/image.py

@@ -0,0 +1,378 @@
+from os import listdir, walk, makedirs
+from os.path import isfile, isdir, join, dirname
+from yaml import load, SafeLoader
+from json import load as jload, dump as jdump
+from time import time
+from shutil import copy2, rmtree, move
+from hashlib import sha512
+from subprocess import Popen
+
+from rougail import RougailConvert, RougailConfig, RougailUpgrade
+try:
+    from tiramisu3 import Config
+except:
+    from tiramisu import Config
+
+from .utils import _
+
+
+DATASET_PATH = '/usr/share/risotto/'
+TMP_DIRECTORY = '/tmp'
+PACKER_TMP_DIRECTORY = join(TMP_DIRECTORY, 'packer')
+PACKER_FILE_NAME = 'recipe.json'
+IMAGES_DIRECTORY = join(TMP_DIRECTORY, 'images')
+
+
+FUNCTIONS = b"""try:
+    from tiramisu3 import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value
+except:
+    from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value
+# =============================================================
+# fork of risotto-setting/src/risotto_setting/config/config.py
+def get_password(**kwargs):
+    return 'password'
+
+
+def get_ip(**kwargs):
+    return '1.1.1.1'
+
+
+def get_chain(**kwargs):
+    return 'chain'
+
+
+def get_certificates(**kwargs):
+    return []
+
+
+def get_certificate(**kwargs):
+    return 'certificate'
+
+
+def get_private_key(**kwargs):
+    return 'private_key'
+
+
+def get_linked_configuration(**kwargs):
+    if 'test' in kwargs and kwargs['test']:
+        return kwargs['test'][0]
+    return 'configuration'
+
+
+def zone_information(**kwargs):
+    return 'zone'
+# =============================================================
+
+"""
+
+
+class Images:
+    def __init__(self,
+                 image_dir: str=None,
+                 tmp_dir: str=None,
+                 ):
+        if image_dir is None:
+            image_dir = IMAGES_DIRECTORY
+        self.image_dir = image_dir
+        if isdir(self.image_dir):
+            rmtree(self.image_dir)
+        if tmp_dir is None:
+            tmp_dir = PACKER_TMP_DIRECTORY
+        self.tmp_dir = tmp_dir
+        self.load_applications()
+
+    def load_applications(self) -> None:
+        self.build_images = []
+        self.applications = {}
+        for distrib in listdir(join(DATASET_PATH, 'seed')):
+            distrib_dir = join(DATASET_PATH, 'seed', distrib, 'applicationservice')
+            if not isdir(distrib_dir):
+                continue
+            for release in listdir(distrib_dir):
+                release_dir = join(distrib_dir, release)
+                if not isdir(release_dir):
+                    continue
+                for applicationservice in listdir(release_dir):
+                    applicationservice_dir = join(release_dir, applicationservice)
+                    if not isdir(applicationservice_dir):
+                        continue
+                    if applicationservice in self.applications:
+                        raise Exception('multi applicationservice')
+                    with open(join(applicationservice_dir, 'applicationservice.yml')) as yaml:
+                        app = load(yaml, Loader=SafeLoader)
+                    self.applications[applicationservice] = {'path': applicationservice_dir,
+                                                             'yml': app,
+                                                             }
+                    if 'service' in app and app['service']:
+                        self.build_images.append(applicationservice)
+
+    def calc_depends(self,
+                     dependencies: list,
+                     appname,
+                     key_is_name=False,
+                     ):
+        app = self.applications[appname]['yml']
+        if not 'depends' in app or not app['depends']:
+            return
+        for dependency in app['depends']:
+            if key_is_name:
+                key = appname
+            else:
+                key = self.applications[dependency]['path']
+            if key not in dependencies:
+                dependencies.insert(0, key)
+            self.calc_depends(dependencies, dependency, key_is_name)
+
+    def list_oses(self):
+        oses = set()
+        for build in self.build_images:
+            dependencies = [build]
+            self.calc_depends(dependencies, build, True)
+            for dependency in dependencies:
+                if isdir(join(self.applications[dependency]['path'], 'packer', 'os')):
+                    oses.add(dependency)
+                    break
+        for os in oses:
+            dependencies = [self.applications[os]['path']]
+            self.calc_depends(dependencies, os)
+            yield os, dependencies
+
+    def list_images(self):
+        for build in self.build_images:
+            dependencies = [self.applications[build]['path']]
+            self.calc_depends(dependencies, build)
+            yield build, dependencies
+
+    async def build(self) -> None:
+        if isdir(self.tmp_dir):
+            rmtree(self.tmp_dir)
+        image = Image(self.image_dir,
+                      self.tmp_dir,
+                      )
+        print(_('Build OSes'))
+        if not isdir(join(self.image_dir, 'os')):
+            makedirs(join(self.image_dir, 'os'))
+        for application, dependencies_path in self.list_oses():
+            print(_(f'Build OS {application}'))
+            await image.build_os(application,
+                                 dependencies_path,
+                                 )
+        print(_('Build images'))
+        for application, dependencies_path in self.list_images():
+            print(_(f'Build image {application}'))
+            await image.build_image(application,
+                                    dependencies_path,
+                                    )
+
+
+class Image:
+    def __init__(self,
+                 image_dir: str,
+                 tmp_dir: str,
+                 ):
+        self.image_dir = image_dir
+        self.tmp_dir = tmp_dir
+
+    @staticmethod
+    def copy_files(dependencies_path: list,
+                   dst_path: str,
+                   element: str,
+                   ) -> None:
+        for dependency_path in dependencies_path:
+            src_path = join(dependency_path,
+                            'packer',
+                            element,
+                            )
+            root_len = len(src_path) + 1
+            for dir_name, subdir_names, filenames in walk(src_path):
+                subdir = join(dst_path, dir_name[root_len:])
+                if not isdir(subdir):
+                    makedirs(subdir)
+                for filename in filenames:
+                    path = join(dir_name, filename)
+                    sub_dst_path = join(subdir, filename)
+                    if isfile(sub_dst_path):
+                        raise Exception(_(f'Try to copy {sub_dst_path} which is already exists'))
+                    copy2(path, sub_dst_path)
+
+    async def load_configuration(self,
+                                 dependencies_path: list,
+                                 packer_tmp_directory: str,
+                                 ) -> dict:
+        config = RougailConfig.copy()
+        dictionaries = [join(dependency_path, 'dictionaries') for dependency_path in dependencies_path if isdir(join(dependency_path, 'dictionaries'))]
+        upgrade = RougailUpgrade()
+        dest_dictionaries = join(packer_tmp_directory, 'dictionaries')
+        makedirs(dest_dictionaries)
+        dest_dictionaries_extras = join(packer_tmp_directory, 'dictionaries_extras')
+        makedirs(dest_dictionaries_extras)
+        for dependency_path in dependencies_path:
+            dictionaries_dir = join(dependency_path, 'dictionaries')
+            if isdir(dictionaries_dir):
+                upgrade.load_xml_from_folders(dictionaries_dir,
+                                              dest_dictionaries,
+                                              RougailConfig['variable_namespace'],
+                                              )
+            extra_dir = join(dependency_path, 'extras', 'packer')
+            if isdir(extra_dir):
+                upgrade.load_xml_from_folders(extra_dir,
+                                              dest_dictionaries_extras,
+                                              'packer',
+                                              )
+        config['dictionaries_dir'] = [dest_dictionaries]
+        config['extra_dictionaries'] = {'packer': [dest_dictionaries_extras]}
+        self.merge_funcs(config, dependencies_path, packer_tmp_directory)
+        packer_configuration = await self.get_packer_information(config, packer_tmp_directory)
+        return packer_configuration
+
+    @staticmethod
+    def merge_funcs(config: RougailConfig,
+                    dependencies_path: list,
+                    packer_tmp_directory: str,
+                    ):
+        functions = FUNCTIONS
+        for dependency_path in dependencies_path:
+            funcs_dir = join(dependency_path, 'funcs')
+            if not isdir(funcs_dir):
+                continue
+            for func in listdir(funcs_dir):
+                with open(join(funcs_dir, func), 'rb') as fh:
+                    functions += fh.read()
+        func_name = join(packer_tmp_directory, 'func.py')
+        with open(func_name, 'wb') as fh:
+            fh.write(functions)
+        config['functions_file'] = func_name
+
+    @staticmethod
+    async def get_packer_information(config: RougailConfig,
+                                     packer_tmp_directory: str,
+                                     ) -> dict:
+        eolobj = RougailConvert(config)
+        xml = eolobj.save(join(packer_tmp_directory, 'tiramisu.py'))
+        optiondescription = {}
+        exec(xml, None, optiondescription)
+        config = await Config(optiondescription['option_0'])
+        return await config.option('packer').value.dict(leader_to_list=True, flatten=True)
+
+    @staticmethod
+    def do_recipe_checksum(path: str,
+                           ) -> str:
+        files = []
+        root_len = len(path) + 1
+        for dir_name, subdir_names, filenames in walk(path):
+            subpath = dir_name[root_len:]
+            for filename in filenames:
+                with open(join(dir_name, filename), 'rb') as fh:
+                    ctl_sum = sha512(fh.read()).hexdigest()
+                abs_path = join(subpath, filename)
+                files.append(f'{abs_path}/{ctl_sum}')
+        files.sort()
+        print(files, sha512('\n'.join(files).encode()).hexdigest())
+        return sha512('\n'.join(files).encode()).hexdigest()
+
+    def get_tmp_directory(self,
+                          application: str,
+                          ) -> str:
+        return join(self.tmp_dir,
+                    application + '_' + str(time()),
+                    )
+
+    def get_os_filename(self,
+                        packer_configuration: dict,
+                        ) -> str:
+        return join(self.image_dir,
+                    'os',
+                    packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img',
+                    )
+
+    def get_image_filename(self,
+                           recipe_checksum: str,
+                           ) -> str:
+        return join(self.image_dir,
+                    f'{recipe_checksum}.img',
+                    )
+
+    async def build_os(self,
+                       application: str,
+                       dependencies_path: list,
+                       ) -> None:
+        packer_tmp_directory = self.get_tmp_directory(application)
+        packer_configuration = await self.load_configuration(dependencies_path, packer_tmp_directory)
+        packer_dst_os_filename = self.get_os_filename(packer_configuration)
+        self.copy_files(dependencies_path,
+                        packer_tmp_directory,
+                        'os',
+                        )
+        packer_configuration['tmp_directory'] = packer_tmp_directory
+        recipe = {'variables': packer_configuration}
+        self.build(packer_dst_os_filename,
+                   packer_tmp_directory,
+                   recipe,
+                   )
+
+    async def build_image(self,
+                          application: str,
+                          dependencies_path: list,
+                          ) -> None:
+        packer_tmp_directory = self.get_tmp_directory(application)
+        makedirs(packer_tmp_directory)
+        self.copy_files(dependencies_path,
+                        packer_tmp_directory,
+                        'image',
+                        )
+        recipe_checksum = self.do_recipe_checksum(packer_tmp_directory)
+        packer_dst_filename = self.get_image_filename(recipe_checksum)
+        packer_configuration = await self.load_configuration(dependencies_path, packer_tmp_directory)
+        packer_dst_os_filename = join(self.image_dir,
+                                      'os',
+                                      packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img',
+                                      )
+        packer_configuration['tmp_directory'] = packer_tmp_directory
+        recipe = {'variables': packer_configuration}
+        recipe['variables']['iso_url'] = packer_dst_os_filename
+        self.build(packer_dst_filename,
+                   packer_tmp_directory,
+                   recipe,
+                   f'{packer_dst_os_filename}.sha256',
+                   )
+
+    @staticmethod
+    def build(packer_dst_filename: str,
+              tmp_directory: str,
+              recipe: dict,
+              sha_file: str=None,
+              ) -> None:
+        packer_filename = join(tmp_directory, PACKER_FILE_NAME)
+        if sha_file is not None:
+            with open(sha_file, 'r') as fh:
+                sha256 = fh.read().split(' ', 1)[0]
+            recipe['variables']['iso_checksum'] = sha256
+        with open(packer_filename, 'r') as recipe_fd:
+            for key, value in jload(recipe_fd).items():
+                recipe[key] = value
+        with open(packer_filename, 'w') as recipe_fd:
+            jdump(recipe, recipe_fd, indent=2)
+        preprocessors = join(tmp_directory, 'preprocessors')
+        if isfile(preprocessors):
+            proc = Popen([preprocessors],
+                         #stdout=PIPE,
+                         #stderr=PIPE,
+                         cwd=tmp_directory,
+                         )
+            proc.wait()
+            if proc.returncode:
+                raise Exception(_(f'error when executing {preprocessors}'))
+        proc = Popen(['packer', 'build', packer_filename],
+                     #stdout=PIPE,
+                     #stderr=PIPE,
+                     cwd=tmp_directory,
+                     )
+        proc.wait()
+        if proc.returncode:
+            raise Exception(_(f'cannot build {packer_dst_filename} with {packer_filename}'))
+        if not isdir(dirname(packer_dst_filename)):
+            makedirs(dirname(packer_dst_filename))
+        move(join(tmp_directory, 'image.img'), packer_dst_filename)
+        move(join(tmp_directory, 'image.sha256'), f'{packer_dst_filename}.sha256')
+        print(_(f'Image {packer_dst_filename} created'))
+        rmtree(tmp_directory)

src/risotto/message.py

@@ -19,8 +19,8 @@ from .utils import _
 
 MESSAGE_ROOT_PATH = get_config()['global']['message_root_path']
 groups.addgroup('message')
-MESSAGE_TRANSLATION = translation('risotto-message', join(MESSAGE_ROOT_PATH, '..', 'locale')).gettext
-
+CUSTOMTYPES = None
+MESSAGE_TRANSLATION = None
 
 
 class DictOption(Option):
@@ -593,6 +593,11 @@ def get_messages(current_module_names,
                  ):
     """generate description from yml files
     """
+    global MESSAGE_TRANSLATION, CUSTOMTYPES
+    if MESSAGE_TRANSLATION is None:
+        MESSAGE_TRANSLATION = translation('risotto-message', join(MESSAGE_ROOT_PATH, '..', 'locale')).gettext
+    if CUSTOMTYPES is None:
+        CUSTOMTYPES = load_customtypes()
     optiondescriptions = {}
     optiondescriptions_info = {}
     messages = list(list_messages(uris,
@@ -632,6 +637,3 @@ def get_messages(current_module_names,
                             optiondescriptions,
                             )
     return optiondescriptions_info, root
-
-
-CUSTOMTYPES = load_customtypes()

src/risotto/register.py

@@ -83,12 +83,16 @@ def register(uris: str,
         uris = [uris]
 
     def decorator(function):
-        for uri in uris:
-            dispatcher.set_function(uri,
-                                    notification,
-                                    function,
-                                    function.__module__
-                                    )
+        try:
+            for uri in uris:
+                dispatcher.set_function(uri,
+                                        notification,
+                                        function,
+                                        function.__module__
+                                        )
+        except NameError:
+            # if you when register uri, please use get_dispatcher before registered uri
+            pass
     return decorator