Merge branch 'develop' into dist/risotto/risotto-2.8.0/develop

debian/changelog

@@ -1,5 +0,0 @@
-risotto (0.1) unstable; urgency=low
-
-  * first version
-
- -- Cadoles <contact@cadoles.com>  Fri, 20 Mar 2020 15:18:25 +0100

debian/control

@@ -2,13 +2,23 @@ Source: risotto
 Section: admin
 Priority: extra
 Maintainer: Cadoles <contact@cadoles.com>
-Build-depends: debhelper (>=11), python3-all, python3-setuptools
+Build-depends: debhelper (>=11), python3-all, python3-setuptools, dh-python
 Standards-Version: 3.9.4
 Homepage: https://forge.cadoles.com/Infra/risotto
 
+Package: python3-risotto
+Architecture: any
+Pre-Depends: dpkg, python3, ${misc:Pre-Depends}
+Depends: ${python:Depends}, ${misc:Depends},
+	python3-asyncpg,
+	python3-rougail,
+	python3-aiohttp,
+	python3-sdnotify
+Description: configuration manager libraries
+
 Package: risotto
 Architecture: any
 Pre-Depends: dpkg, python3, ${misc:Pre-Depends}
-Depends: ${python:Depends}, ${misc:Depends}
+Depends: ${python:Depends}, ${misc:Depends}, python3-risotto
 Description: configuration manager
 

debian/risotto.install

@@ -0,0 +1,2 @@
+script/risotto-server usr/bin/
+sql/risotto.sql usr/share/eole/db/eole-risotto/gen/

src/risotto/controller.py

@@ -64,8 +64,8 @@ class Controller:
                                       **kwargs,
                                       )
 
-    async def check_role(self,
-                         uri: str,
+    @staticmethod
+    async def check_role(uri: str,
                          username: str,
                          **kwargs: dict,
                          ) -> None:
@@ -98,12 +98,10 @@ class TiramisuController(Controller):
     def __init__(self,
                  test: bool,
                  ) -> None:
-        self.source_imported = None
         if not 'dataset_name' in vars(self):
             raise Exception(f'please specify "dataset_name" to "{self.__class__.__name__}"')
         self.tiramisu_cache_root_path = join(get_config()['cache']['root_path'], self.dataset_name)
         super().__init__(test)
-        self.internal_source_name = get_config()['servermodel']['internal_source']
         if not test:
             db_conf = get_config()['database']['tiramisu_dsn']
             self.save_storage = Storage(engine='postgres')
@@ -132,12 +130,6 @@ class TiramisuController(Controller):
                                risotto_context: Context,
                                ) -> None:
         sources = await self.get_sources(risotto_context)
-        source_imported = sources != [self.internal_source_name]
-        if source_imported and self.source_imported is False:
-            await self.load_datas(risotto_context)
-        self.source_imported = source_imported
-        if not self.source_imported:
-            return
         self._aggregate_tiramisu_funcs(sources)
         self._convert_dictionaries_to_tiramisu(sources)
 
@@ -183,9 +175,7 @@ except:
                             funcs.write(fh.read())
                             funcs.write(b'\n')
 
-    def _convert_dictionaries_to_tiramisu(self,
-                                          sources: list,
-                                          ) -> None:
+    def _convert_dictionaries_to_tiramisu(self, sources: list) -> None:
         funcs_file = join(self.tiramisu_cache_root_path, 'funcs.py')
         tiramisu_file = join(self.tiramisu_cache_root_path, 'tiramisu.py')
         dictionaries_dir = join(self.tiramisu_cache_root_path, 'dictionaries')

src/risotto/image.py

@@ -66,7 +66,7 @@ def zone_information(**kwargs):
 """
 
 
-class Images:
+class Image:
     def __init__(self,
                  image_dir: str=None,
                  tmp_dir: str=None,
@@ -79,10 +79,10 @@ class Images:
         if tmp_dir is None:
             tmp_dir = PACKER_TMP_DIRECTORY
         self.tmp_dir = tmp_dir
-        self.load_applications()
+        self.parse_applications()
 
-    def load_applications(self) -> None:
-        self.build_images = []
+    def parse_applications(self) -> None:
+        self.builds = []
         self.applications = {}
         for distrib in listdir(join(DATASET_PATH, 'seed')):
             distrib_dir = join(DATASET_PATH, 'seed', distrib, 'applicationservice')
@@ -104,101 +104,49 @@ class Images:
                                                              'yml': app,
                                                              }
                     if 'service' in app and app['service']:
-                        self.build_images.append(applicationservice)
+                        self.builds.append(applicationservice)
 
     def calc_depends(self,
                      dependencies: list,
                      appname,
-                     key_is_name=False,
                      ):
         app = self.applications[appname]['yml']
         if not 'depends' in app or not app['depends']:
             return
         for dependency in app['depends']:
-            if key_is_name:
-                key = appname
-            else:
-                key = self.applications[dependency]['path']
-            if key not in dependencies:
-                dependencies.insert(0, key)
-            self.calc_depends(dependencies, dependency, key_is_name)
+            dependency_path = self.applications[dependency]['path']
+            if dependency_path not in dependencies:
+                dependencies.insert(0, dependency_path)
+            self.calc_depends(dependencies, dependency)
 
-    def list_oses(self):
-        oses = set()
-        for build in self.build_images:
-            dependencies = [build]
-            self.calc_depends(dependencies, build, True)
-            for dependency in dependencies:
-                if isdir(join(self.applications[dependency]['path'], 'packer', 'os')):
-                    oses.add(dependency)
-                    break
-        for os in oses:
-            dependencies = [self.applications[os]['path']]
-            self.calc_depends(dependencies, os)
-            yield os, dependencies
 
     def list_images(self):
-        for build in self.build_images:
+        print(self.builds)
+        for build in self.builds:
             dependencies = [self.applications[build]['path']]
             self.calc_depends(dependencies, build)
             yield build, dependencies
 
-    async def build(self) -> None:
-        if isdir(self.tmp_dir):
-            rmtree(self.tmp_dir)
-        image = Image(self.image_dir,
-                      self.tmp_dir,
-                      )
-        print(_('Build OSes'))
-        if not isdir(join(self.image_dir, 'os')):
-            makedirs(join(self.image_dir, 'os'))
-        for application, dependencies_path in self.list_oses():
-            print(_(f'Build OS {application}'))
-            await image.build_os(application,
-                                 dependencies_path,
-                                 )
-        print(_('Build images'))
-        for application, dependencies_path in self.list_images():
-            print(_(f'Build image {application}'))
-            await image.build_image(application,
-                                    dependencies_path,
-                                    )
-
-
-class Image:
-    def __init__(self,
-                 image_dir: str,
-                 tmp_dir: str,
-                 ):
-        self.image_dir = image_dir
-        self.tmp_dir = tmp_dir
-
-    @staticmethod
-    def copy_files(dependencies_path: list,
+    def copy_files(self,
+                   src_path: str,
                    dst_path: str,
-                   element: str,
                    ) -> None:
-        for dependency_path in dependencies_path:
-            src_path = join(dependency_path,
-                            'packer',
-                            element,
-                            )
-            root_len = len(src_path) + 1
-            for dir_name, subdir_names, filenames in walk(src_path):
-                subdir = join(dst_path, dir_name[root_len:])
-                if not isdir(subdir):
-                    makedirs(subdir)
-                for filename in filenames:
-                    path = join(dir_name, filename)
-                    sub_dst_path = join(subdir, filename)
-                    if isfile(sub_dst_path):
-                        raise Exception(_(f'Try to copy {sub_dst_path} which is already exists'))
-                    copy2(path, sub_dst_path)
+        root_len = len(src_path) + 1
+        for dir_name, subdir_names, filenames in walk(src_path):
+            subdir = join(dst_path, dir_name[root_len:])
+            if not isdir(subdir):
+                makedirs(subdir)
+            for filename in filenames:
+                path = join(dir_name, filename)
+                sub_dst_path = join(subdir, filename)
+                if isfile(sub_dst_path):
+                    raise Exception(_(f'Try to copy {sub_dst_path} which is already exists'))
+                copy2(path, sub_dst_path)
 
-    async def load_configuration(self,
-                                 dependencies_path: list,
-                                 packer_tmp_directory: str,
-                                 ) -> dict:
+    def load_configuration(self,
+                           dependencies_path: list,
+                           packer_tmp_directory: str,
+                           ) -> dict:
         config = RougailConfig.copy()
         dictionaries = [join(dependency_path, 'dictionaries') for dependency_path in dependencies_path if isdir(join(dependency_path, 'dictionaries'))]
         upgrade = RougailUpgrade()
@@ -221,12 +169,11 @@ class Image:
                                               )
         config['dictionaries_dir'] = [dest_dictionaries]
         config['extra_dictionaries'] = {'packer': [dest_dictionaries_extras]}
-        self.merge_funcs(config, dependencies_path, packer_tmp_directory)
-        packer_configuration = await self.get_packer_information(config, packer_tmp_directory)
-        return packer_configuration
+        return config
 
-    @staticmethod
-    def merge_funcs(config: RougailConfig,
+
+    def merge_funcs(self,
+                    config: RougailConfig,
                     dependencies_path: list,
                     packer_tmp_directory: str,
                     ):
@@ -243,8 +190,8 @@ class Image:
             fh.write(functions)
         config['functions_file'] = func_name
 
-    @staticmethod
-    async def get_packer_information(config: RougailConfig,
+    async def get_packer_information(self,
+                                     config: RougailConfig,
                                      packer_tmp_directory: str,
                                      ) -> dict:
         eolobj = RougailConvert(config)
@@ -252,10 +199,10 @@ class Image:
         optiondescription = {}
         exec(xml, None, optiondescription)
         config = await Config(optiondescription['option_0'])
-        return await config.option('packer').value.dict(leader_to_list=True, flatten=True)
+        return await config.option('packer').value.dict(flatten=True)
 
-    @staticmethod
-    def do_recipe_checksum(path: str,
+    def do_recipe_checksum(self,
+                           path: str,
                            ) -> str:
         files = []
         root_len = len(path) + 1
@@ -264,104 +211,81 @@ class Image:
             for filename in filenames:
                 with open(join(dir_name, filename), 'rb') as fh:
                     ctl_sum = sha512(fh.read()).hexdigest()
-                abs_path = join(subpath, filename)
-                files.append(f'{abs_path}/{ctl_sum}')
-        files.sort()
-        print(files, sha512('\n'.join(files).encode()).hexdigest())
+                files.append(f'{subpath}/{filename}/ctl_sum')
         return sha512('\n'.join(files).encode()).hexdigest()
 
-    def get_tmp_directory(self,
-                          application: str,
-                          ) -> str:
-        return join(self.tmp_dir,
-                    application + '_' + str(time()),
-                    )
-
-    def get_os_filename(self,
-                        packer_configuration: dict,
-                        ) -> str:
-        return join(self.image_dir,
-                    'os',
-                    packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img',
-                    )
-
-    def get_image_filename(self,
-                           recipe_checksum: str,
-                           ) -> str:
-        return join(self.image_dir,
-                    f'{recipe_checksum}.img',
-                    )
-
-    async def build_os(self,
-                       application: str,
-                       dependencies_path: list,
-                       ) -> None:
-        packer_tmp_directory = self.get_tmp_directory(application)
-        packer_configuration = await self.load_configuration(dependencies_path, packer_tmp_directory)
-        packer_dst_os_filename = self.get_os_filename(packer_configuration)
-        self.copy_files(dependencies_path,
-                        packer_tmp_directory,
-                        'os',
-                        )
-        packer_configuration['tmp_directory'] = packer_tmp_directory
-        recipe = {'variables': packer_configuration}
-        self.build(packer_dst_os_filename,
-                   packer_tmp_directory,
-                   recipe,
-                   )
-
-    async def build_image(self,
-                          application: str,
-                          dependencies_path: list,
-                          ) -> None:
-        packer_tmp_directory = self.get_tmp_directory(application)
-        makedirs(packer_tmp_directory)
-        self.copy_files(dependencies_path,
-                        packer_tmp_directory,
-                        'image',
-                        )
-        recipe_checksum = self.do_recipe_checksum(packer_tmp_directory)
-        packer_dst_filename = self.get_image_filename(recipe_checksum)
-        packer_configuration = await self.load_configuration(dependencies_path, packer_tmp_directory)
-        packer_dst_os_filename = join(self.image_dir,
-                                      'os',
-                                      packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img',
-                                      )
-        packer_configuration['tmp_directory'] = packer_tmp_directory
-        recipe = {'variables': packer_configuration}
-        recipe['variables']['iso_url'] = packer_dst_os_filename
-        self.build(packer_dst_filename,
-                   packer_tmp_directory,
-                   recipe,
-                   f'{packer_dst_os_filename}.sha256',
-                   )
-
-    @staticmethod
-    def build(packer_dst_filename: str,
-              tmp_directory: str,
-              recipe: dict,
-              sha_file: str=None,
-              ) -> None:
-        packer_filename = join(tmp_directory, PACKER_FILE_NAME)
-        if sha_file is not None:
+    async def build(self) -> None:
+        if isdir(self.tmp_dir):
+            rmtree(self.tmp_dir)
+        for application, dependencies_path in self.list_images():
+            packer_tmp_directory = join(self.tmp_dir,
+                                        application + '_' + str(time()),
+                                        )
+            makedirs(packer_tmp_directory)
+            packer_tmp_os_directory = join(packer_tmp_directory, 'os')
+            makedirs(packer_tmp_os_directory)
+            packer_tmp_img_directory = join(packer_tmp_directory, 'image')
+            makedirs(packer_tmp_img_directory)
+            config = self.load_configuration(dependencies_path, packer_tmp_directory)
+            self.merge_funcs(config, dependencies_path, packer_tmp_directory)
+            packer_configuration = await self.get_packer_information(config, packer_tmp_directory)
+            packer_dst_os_filename = join(self.image_dir,
+                                          'os',
+                                          packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img',
+                                          )
+            for dependency_path in dependencies_path:
+                packer_directory = join(dependency_path,
+                                        'packer',
+                                        'os',
+                                        )
+                self.copy_files(packer_directory,
+                                packer_tmp_os_directory,
+                                )
+                packer_directory = join(dependency_path,
+                                        'packer',
+                                        'image',
+                                        )
+                self.copy_files(packer_directory,
+                                packer_tmp_img_directory,
+                                )
+            if not isdir(join(self.image_dir, 'os')):
+                makedirs(join(self.image_dir, 'os'))
+            packer_configuration['tmp_directory'] = packer_tmp_os_directory
+            recipe = {'variables': packer_configuration}
+            if not isfile(packer_dst_os_filename):
+                self.build_image(packer_dst_os_filename,
+                                 packer_tmp_os_directory,
+                                 recipe,
+                                 )
+            recipe_checksum = self.do_recipe_checksum(packer_tmp_img_directory)
+            packer_dst_filename = join(self.image_dir,
+                                       f'{recipe_checksum}.img',
+                                       )
+            sha_file = f'{packer_dst_os_filename}.sha256'
             with open(sha_file, 'r') as fh:
                 sha256 = fh.read().split(' ', 1)[0]
+            packer_configuration['tmp_directory'] = packer_tmp_img_directory
+            recipe = {'variables': packer_configuration}
             recipe['variables']['iso_checksum'] = sha256
+            recipe['variables']['iso_url'] = packer_dst_os_filename
+            self.build_image(packer_dst_filename,
+                             packer_tmp_img_directory,
+                             recipe,
+                             )
+            rmtree(packer_tmp_directory)
+
+    def build_image(self,
+                    packer_dst_filename: str,
+                    tmp_directory: str,
+                    recipe: dict,
+                    ) -> None:
+        packer_filename = join(tmp_directory, PACKER_FILE_NAME)
         with open(packer_filename, 'r') as recipe_fd:
             for key, value in jload(recipe_fd).items():
                 recipe[key] = value
         with open(packer_filename, 'w') as recipe_fd:
             jdump(recipe, recipe_fd, indent=2)
-        preprocessors = join(tmp_directory, 'preprocessors')
-        if isfile(preprocessors):
-            proc = Popen([preprocessors],
-                         #stdout=PIPE,
-                         #stderr=PIPE,
-                         cwd=tmp_directory,
-                         )
-            proc.wait()
-            if proc.returncode:
-                raise Exception(_(f'error when executing {preprocessors}'))
+        print(['packer', 'build', packer_filename])
         proc = Popen(['packer', 'build', packer_filename],
                      #stdout=PIPE,
                      #stderr=PIPE,
@@ -370,9 +294,6 @@ class Image:
         proc.wait()
         if proc.returncode:
             raise Exception(_(f'cannot build {packer_dst_filename} with {packer_filename}'))
-        if not isdir(dirname(packer_dst_filename)):
-            makedirs(dirname(packer_dst_filename))
         move(join(tmp_directory, 'image.img'), packer_dst_filename)
         move(join(tmp_directory, 'image.sha256'), f'{packer_dst_filename}.sha256')
         print(_(f'Image {packer_dst_filename} created'))
-        rmtree(tmp_directory)

src/risotto/register.py

@@ -83,16 +83,12 @@ def register(uris: str,
         uris = [uris]
 
     def decorator(function):
-        try:
-            for uri in uris:
-                dispatcher.set_function(uri,
-                                        notification,
-                                        function,
-                                        function.__module__
-                                        )
-        except NameError:
-            # if you when register uri, please use get_dispatcher before registered uri
-            pass
+        for uri in uris:
+            dispatcher.set_function(uri,
+                                    notification,
+                                    function,
+                                    function.__module__
+                                    )
     return decorator