add image file

src/risotto/image.py

@@ -0,0 +1,278 @@
+from os import listdir, walk, makedirs
+from os.path import isfile, isdir, join, dirname
+from yaml import load, SafeLoader
+from json import load as jload, dump as jdump
+from time import time
+from shutil import copy2, rmtree, move
+from hashlib import sha512
+from subprocess import Popen
+
+from rougail import RougailConvert, RougailConfig, RougailUpgrade
+try:
+    from tiramisu3 import Config
+except:
+    from tiramisu import Config
+
+from .utils import _
+
+
+DATASET_PATH = ''
+TMP_DIRECTORY = '/tmp'
+PACKER_TMP_DIRECTORY = join(TMP_DIRECTORY, 'packer')
+PACKER_FILE_NAME = 'recipe.json'
+IMAGES_DIRECTORY = join(TMP_DIRECTORY, 'images')
+
+
+FUNCTIONS = b"""from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value
+# =============================================================
+# fork of risotto-setting/src/risotto_setting/config/config.py
+def get_password(**kwargs):
+    return 'password'
+
+
+def get_ip(**kwargs):
+    return '1.1.1.1'
+
+
+def get_chain(**kwargs):
+    return 'chain'
+
+
+def get_certificates(**kwargs):
+    return []
+
+
+def get_certificate(**kwargs):
+    return 'certificate'
+
+
+def get_private_key(**kwargs):
+    return 'private_key'
+
+
+def get_linked_configuration(**kwargs):
+    if 'test' in kwargs and kwargs['test']:
+        return kwargs['test'][0]
+    return 'configuration'
+
+
+def zone_information(**kwargs):
+    return 'zone'
+# =============================================================
+
+"""
+
+
+class Image:
+    def __init__(self):
+        self.parse_applications()
+
+    def parse_applications(self) -> None:
+        self.builds = []
+        self.applications = {}
+        for distrib in listdir(join(DATASET_PATH, 'seed')):
+            distrib_dir = join(DATASET_PATH, 'seed', distrib, 'applicationservice')
+            if not isdir(distrib_dir):
+                continue
+            for release in listdir(distrib_dir):
+                release_dir = join(distrib_dir, release)
+                if not isdir(release_dir):
+                    continue
+                for applicationservice in listdir(release_dir):
+                    applicationservice_dir = join(release_dir, applicationservice)
+                    if not isdir(applicationservice_dir):
+                        continue
+                    if applicationservice in self.applications:
+                        raise Exception('multi applicationservice')
+                    with open(join(applicationservice_dir, 'applicationservice.yml')) as yaml:
+                        app = load(yaml, Loader=SafeLoader)
+                    self.applications[applicationservice] = {'path': applicationservice_dir,
+                                                             'yml': app,
+                                                             }
+                    if 'service' in app and app['service']:
+                        self.builds.append(applicationservice)
+
+    def calc_depends(self,
+                     dependencies: list,
+                     appname,
+                     ):
+        app = self.applications[appname]['yml']
+        if not 'depends' in app or not app['depends']:
+            return
+        for dependency in app['depends']:
+            dependency_path = self.applications[dependency]['path']
+            if dependency_path not in dependencies:
+                dependencies.insert(0, dependency_path)
+            self.calc_depends(dependencies, dependency)
+
+
+    def list_images(self):
+        for build in self.builds:
+            dependencies = [self.applications[build]['path']]
+            self.calc_depends(dependencies, build)
+            yield build, dependencies
+
+    def copy_files(self,
+                   src_path: str,
+                   dst_path: str,
+                   ) -> None:
+        root_len = len(src_path) + 1
+        for dir_name, subdir_names, filenames in walk(src_path):
+            subdir = join(dst_path, dir_name[root_len:])
+            if not isdir(subdir):
+                makedirs(subdir)
+            for filename in filenames:
+                path = join(dir_name, filename)   
+                sub_dst_path = join(subdir, filename)  
+                if isfile(sub_dst_path):
+                    raise Exception(_(f'Try to copy {sub_dst_path} which is already exists'))
+                copy2(path, sub_dst_path)
+
+    def load_configuration(self,
+                           dependencies_path: list,
+                           packer_tmp_directory: str,
+                           ) -> dict:
+        config = RougailConfig.copy()
+        dictionaries = [join(dependency_path, 'dictionaries') for dependency_path in dependencies_path if isdir(join(dependency_path, 'dictionaries'))]
+        upgrade = RougailUpgrade()
+        dest_dictionaries = join(packer_tmp_directory, 'dictionaries')
+        makedirs(dest_dictionaries)
+        dest_dictionaries_extras = join(packer_tmp_directory, 'dictionaries_extras')
+        makedirs(dest_dictionaries_extras)
+        for dependency_path in dependencies_path:
+            dictionaries_dir = join(dependency_path, 'dictionaries')
+            if isdir(dictionaries_dir):
+                upgrade.load_xml_from_folders(dictionaries_dir,
+                                              dest_dictionaries,
+                                              RougailConfig['variable_namespace'],
+                                              )
+            extra_dir = join(dependency_path, 'extras', 'packer')
+            if isdir(extra_dir):
+                upgrade.load_xml_from_folders(extra_dir,
+                                              dest_dictionaries_extras,
+                                              'packer',
+                                              )
+        config['dictionaries_dir'] = [dest_dictionaries]
+        config['extra_dictionaries'] = {'packer': [dest_dictionaries_extras]}
+        return config
+
+
+    def merge_funcs(self,
+                    config: RougailConfig,
+                    dependencies_path: list,
+                    packer_tmp_directory: str,
+                    ):
+        functions = FUNCTIONS
+        for dependency_path in dependencies_path:
+            funcs_dir = join(dependency_path, 'funcs')
+            if not isdir(funcs_dir):
+                continue
+            for func in listdir(funcs_dir):
+                print(join(funcs_dir, func))
+                with open(join(funcs_dir, func), 'rb') as fh:
+                    functions += fh.read()
+        func_name = join(packer_tmp_directory, 'func.py')
+        with open(func_name, 'wb') as fh:
+            fh.write(functions)
+        config['functions_file'] = func_name
+
+    async def get_packer_information(self,
+                                     config: RougailConfig,
+                                     packer_tmp_directory: str,
+                                     ) -> dict:
+        eolobj = RougailConvert(config)
+        xml = eolobj.save(join(packer_tmp_directory, 'tiramisu.py'))
+        optiondescription = {}
+        exec(xml, None, optiondescription)
+        config = await Config(optiondescription['option_0'])
+        return await config.option('packer').value.dict(flatten=True)
+
+    def do_recipe_checksum(self,
+                           path: str,
+                           ) -> str:
+        files = []
+        root_len = len(path) + 1
+        for dir_name, subdir_names, filenames in walk(path):
+            subpath = dir_name[root_len:]
+            for filename in filenames:
+                with open(join(dir_name, filename), 'rb') as fh: 
+                    ctl_sum = sha512(fh.read()).hexdigest()
+                files.append(f'{subpath}/{filename}/ctl_sum')
+        return sha512('\n'.join(files).encode()).hexdigest()
+
+    async def build(self) -> None:
+        if isdir(PACKER_TMP_DIRECTORY):
+            rmtree(PACKER_TMP_DIRECTORY)
+        for application, dependencies_path in self.list_images():
+            packer_tmp_directory = join(PACKER_TMP_DIRECTORY,
+                                        application + '_' + str(time()),
+                                        )
+            makedirs(packer_tmp_directory)
+            packer_tmp_os_directory = join(packer_tmp_directory, 'os')
+            makedirs(packer_tmp_os_directory)
+            packer_tmp_img_directory = join(packer_tmp_directory, 'image')
+            makedirs(packer_tmp_img_directory)
+            config = self.load_configuration(dependencies_path, packer_tmp_directory)
+            self.merge_funcs(config, dependencies_path, packer_tmp_directory)
+            packer_configuration = await self.get_packer_information(config, packer_tmp_directory)
+            # OS image needed ?
+            packer_dst_os_filename = join(IMAGES_DIRECTORY,
+                                          'os',
+                                          packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img',
+                                          )
+            for dependency_path in dependencies_path:
+                packer_directory = join(dependency_path,
+                                        'packer',
+                                        'os',
+                                        )
+                self.copy_files(packer_directory,
+                                packer_tmp_os_directory,
+                                )
+                packer_directory = join(dependency_path,
+                                        'packer',
+                                        'image',
+                                        )
+                self.copy_files(packer_directory,
+                                packer_tmp_img_directory,
+                                )
+            if not isfile(packer_dst_os_filename):
+                self.build_image(packer_dst_os_filename,
+                                 packer_tmp_os_directory,
+                                 packer_configuration,
+                                 )
+            recipe_checksum = self.do_recipe_checksum(packer_tmp_img_directory)
+            packer_dst_filename = join(IMAGES_DIRECTORY,
+                                       f'{recipe_checksum}.img',
+                                       )
+            self.build_image(packer_dst_filename,
+                             packer_tmp_img_directory,
+                             packer_configuration,
+                             )
+
+    def build_image(self,
+                    packer_dst_filename: str,
+                    packer_tmp_directory: str,
+                    packer_configuration: dict,
+                    ) -> None:
+        tmp_directory = join(packer_tmp_directory, 'tmp')
+        makedirs(tmp_directory)
+        packer_configuration['tmp_directory'] = tmp_directory
+        recipe = {'variables': packer_configuration}
+        packer_filename = join(packer_tmp_directory, PACKER_FILE_NAME)
+        with open(packer_filename, 'r') as recipe_fd:
+            for key, value in jload(recipe_fd).items():
+                recipe[key] = value
+        with open(packer_filename, 'w') as recipe_fd:
+            jdump(recipe, recipe_fd, indent=2)
+        proc = Popen(['packer', 'build', packer_filename],
+                     #stdout=PIPE,
+                     #stderr=PIPE,
+                     cwd=tmp_directory,
+                     )
+        proc.wait()
+        if proc.returncode:
+            raise Exception(_(f'cannot build {application} with {packer_filename}'))
+        move(join(tmp_directory, 'image.img'), packer_dst_filename)
+        move(join(tmp_directory, 'image.sha256'), f'{packer_dst_filename}.sha256')
+        rmtree(tmp_directory)
+        print(_(f'Image {packer_dst_filename} created'))