From c0244eac8c06ef464fa481a37fbb22ee0018d620 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Mon, 24 May 2021 16:22:39 +0200 Subject: [PATCH] add image file --- src/risotto/image.py | 278 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 src/risotto/image.py diff --git a/src/risotto/image.py b/src/risotto/image.py new file mode 100644 index 0000000..e85dda7 --- /dev/null +++ b/src/risotto/image.py @@ -0,0 +1,278 @@ +from os import listdir, walk, makedirs +from os.path import isfile, isdir, join, dirname +from yaml import load, SafeLoader +from json import load as jload, dump as jdump +from time import time +from shutil import copy2, rmtree, move +from hashlib import sha512 +from subprocess import Popen + +from rougail import RougailConvert, RougailConfig, RougailUpgrade +try: + from tiramisu3 import Config +except: + from tiramisu import Config + +from .utils import _ + + +DATASET_PATH = '' +TMP_DIRECTORY = '/tmp' +PACKER_TMP_DIRECTORY = join(TMP_DIRECTORY, 'packer') +PACKER_FILE_NAME = 'recipe.json' +IMAGES_DIRECTORY = join(TMP_DIRECTORY, 'images') + + +FUNCTIONS = b"""from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value +# ============================================================= +# fork of risotto-setting/src/risotto_setting/config/config.py +def get_password(**kwargs): + return 'password' + + +def get_ip(**kwargs): + return '1.1.1.1' + + +def get_chain(**kwargs): + return 'chain' + + +def get_certificates(**kwargs): + return [] + + +def get_certificate(**kwargs): + return 'certificate' + + +def get_private_key(**kwargs): + return 'private_key' + + +def get_linked_configuration(**kwargs): + if 'test' in kwargs and kwargs['test']: + return kwargs['test'][0] + return 'configuration' + + +def zone_information(**kwargs): + return 'zone' +# ============================================================= + +""" + + +class Image: + def __init__(self): + self.parse_applications() + + def parse_applications(self) -> None: + self.builds = [] + self.applications = {} + for distrib in listdir(join(DATASET_PATH, 'seed')): + distrib_dir = join(DATASET_PATH, 'seed', distrib, 'applicationservice') + if not isdir(distrib_dir): + continue + for release in listdir(distrib_dir): + release_dir = join(distrib_dir, release) + if not isdir(release_dir): + continue + for applicationservice in listdir(release_dir): + applicationservice_dir = join(release_dir, applicationservice) + if not isdir(applicationservice_dir): + continue + if applicationservice in self.applications: + raise Exception('multi applicationservice') + with open(join(applicationservice_dir, 'applicationservice.yml')) as yaml: + app = load(yaml, Loader=SafeLoader) + self.applications[applicationservice] = {'path': applicationservice_dir, + 'yml': app, + } + if 'service' in app and app['service']: + self.builds.append(applicationservice) + + def calc_depends(self, + dependencies: list, + appname, + ): + app = self.applications[appname]['yml'] + if not 'depends' in app or not app['depends']: + return + for dependency in app['depends']: + dependency_path = self.applications[dependency]['path'] + if dependency_path not in dependencies: + dependencies.insert(0, dependency_path) + self.calc_depends(dependencies, dependency) + + + def list_images(self): + for build in self.builds: + dependencies = [self.applications[build]['path']] + self.calc_depends(dependencies, build) + yield build, dependencies + + def copy_files(self, + src_path: str, + dst_path: str, + ) -> None: + root_len = len(src_path) + 1 + for dir_name, subdir_names, filenames in walk(src_path): + subdir = join(dst_path, dir_name[root_len:]) + if not isdir(subdir): + makedirs(subdir) + for filename in filenames: + path = join(dir_name, filename) + sub_dst_path = join(subdir, filename) + if isfile(sub_dst_path): + raise Exception(_(f'Try to copy {sub_dst_path} which is already exists')) + copy2(path, sub_dst_path) + + def load_configuration(self, + dependencies_path: list, + packer_tmp_directory: str, + ) -> dict: + config = RougailConfig.copy() + dictionaries = [join(dependency_path, 'dictionaries') for dependency_path in dependencies_path if isdir(join(dependency_path, 'dictionaries'))] + upgrade = RougailUpgrade() + dest_dictionaries = join(packer_tmp_directory, 'dictionaries') + makedirs(dest_dictionaries) + dest_dictionaries_extras = join(packer_tmp_directory, 'dictionaries_extras') + makedirs(dest_dictionaries_extras) + for dependency_path in dependencies_path: + dictionaries_dir = join(dependency_path, 'dictionaries') + if isdir(dictionaries_dir): + upgrade.load_xml_from_folders(dictionaries_dir, + dest_dictionaries, + RougailConfig['variable_namespace'], + ) + extra_dir = join(dependency_path, 'extras', 'packer') + if isdir(extra_dir): + upgrade.load_xml_from_folders(extra_dir, + dest_dictionaries_extras, + 'packer', + ) + config['dictionaries_dir'] = [dest_dictionaries] + config['extra_dictionaries'] = {'packer': [dest_dictionaries_extras]} + return config + + + def merge_funcs(self, + config: RougailConfig, + dependencies_path: list, + packer_tmp_directory: str, + ): + functions = FUNCTIONS + for dependency_path in dependencies_path: + funcs_dir = join(dependency_path, 'funcs') + if not isdir(funcs_dir): + continue + for func in listdir(funcs_dir): + print(join(funcs_dir, func)) + with open(join(funcs_dir, func), 'rb') as fh: + functions += fh.read() + func_name = join(packer_tmp_directory, 'func.py') + with open(func_name, 'wb') as fh: + fh.write(functions) + config['functions_file'] = func_name + + async def get_packer_information(self, + config: RougailConfig, + packer_tmp_directory: str, + ) -> dict: + eolobj = RougailConvert(config) + xml = eolobj.save(join(packer_tmp_directory, 'tiramisu.py')) + optiondescription = {} + exec(xml, None, optiondescription) + config = await Config(optiondescription['option_0']) + return await config.option('packer').value.dict(flatten=True) + + def do_recipe_checksum(self, + path: str, + ) -> str: + files = [] + root_len = len(path) + 1 + for dir_name, subdir_names, filenames in walk(path): + subpath = dir_name[root_len:] + for filename in filenames: + with open(join(dir_name, filename), 'rb') as fh: + ctl_sum = sha512(fh.read()).hexdigest() + files.append(f'{subpath}/{filename}/ctl_sum') + return sha512('\n'.join(files).encode()).hexdigest() + + async def build(self) -> None: + if isdir(PACKER_TMP_DIRECTORY): + rmtree(PACKER_TMP_DIRECTORY) + for application, dependencies_path in self.list_images(): + packer_tmp_directory = join(PACKER_TMP_DIRECTORY, + application + '_' + str(time()), + ) + makedirs(packer_tmp_directory) + packer_tmp_os_directory = join(packer_tmp_directory, 'os') + makedirs(packer_tmp_os_directory) + packer_tmp_img_directory = join(packer_tmp_directory, 'image') + makedirs(packer_tmp_img_directory) + config = self.load_configuration(dependencies_path, packer_tmp_directory) + self.merge_funcs(config, dependencies_path, packer_tmp_directory) + packer_configuration = await self.get_packer_information(config, packer_tmp_directory) + # OS image needed ? + packer_dst_os_filename = join(IMAGES_DIRECTORY, + 'os', + packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img', + ) + for dependency_path in dependencies_path: + packer_directory = join(dependency_path, + 'packer', + 'os', + ) + self.copy_files(packer_directory, + packer_tmp_os_directory, + ) + packer_directory = join(dependency_path, + 'packer', + 'image', + ) + self.copy_files(packer_directory, + packer_tmp_img_directory, + ) + if not isfile(packer_dst_os_filename): + self.build_image(packer_dst_os_filename, + packer_tmp_os_directory, + packer_configuration, + ) + recipe_checksum = self.do_recipe_checksum(packer_tmp_img_directory) + packer_dst_filename = join(IMAGES_DIRECTORY, + f'{recipe_checksum}.img', + ) + self.build_image(packer_dst_filename, + packer_tmp_img_directory, + packer_configuration, + ) + + def build_image(self, + packer_dst_filename: str, + packer_tmp_directory: str, + packer_configuration: dict, + ) -> None: + tmp_directory = join(packer_tmp_directory, 'tmp') + makedirs(tmp_directory) + packer_configuration['tmp_directory'] = tmp_directory + recipe = {'variables': packer_configuration} + packer_filename = join(packer_tmp_directory, PACKER_FILE_NAME) + with open(packer_filename, 'r') as recipe_fd: + for key, value in jload(recipe_fd).items(): + recipe[key] = value + with open(packer_filename, 'w') as recipe_fd: + jdump(recipe, recipe_fd, indent=2) + proc = Popen(['packer', 'build', packer_filename], + #stdout=PIPE, + #stderr=PIPE, + cwd=tmp_directory, + ) + proc.wait() + if proc.returncode: + raise Exception(_(f'cannot build {application} with {packer_filename}')) + move(join(tmp_directory, 'image.img'), packer_dst_filename) + move(join(tmp_directory, 'image.sha256'), f'{packer_dst_filename}.sha256') + rmtree(tmp_directory) + print(_(f'Image {packer_dst_filename} created'))