diff --git a/src/risotto/controller.py b/src/risotto/controller.py index 16dfa87..75f4e50 100644 --- a/src/risotto/controller.py +++ b/src/risotto/controller.py @@ -64,8 +64,8 @@ class Controller: **kwargs, ) - @staticmethod - async def check_role(uri: str, + async def check_role(self, + uri: str, username: str, **kwargs: dict, ) -> None: diff --git a/src/risotto/image.py b/src/risotto/image.py index 8e7e318..1edaa8e 100644 --- a/src/risotto/image.py +++ b/src/risotto/image.py @@ -66,7 +66,7 @@ def zone_information(**kwargs): """ -class Image: +class Images: def __init__(self, image_dir: str=None, tmp_dir: str=None, @@ -79,10 +79,10 @@ class Image: if tmp_dir is None: tmp_dir = PACKER_TMP_DIRECTORY self.tmp_dir = tmp_dir - self.parse_applications() + self.load_applications() - def parse_applications(self) -> None: - self.builds = [] + def load_applications(self) -> None: + self.build_images = [] self.applications = {} for distrib in listdir(join(DATASET_PATH, 'seed')): distrib_dir = join(DATASET_PATH, 'seed', distrib, 'applicationservice') @@ -104,48 +104,101 @@ class Image: 'yml': app, } if 'service' in app and app['service']: - self.builds.append(applicationservice) + self.build_images.append(applicationservice) def calc_depends(self, dependencies: list, appname, + key_is_name=False, ): app = self.applications[appname]['yml'] if not 'depends' in app or not app['depends']: return for dependency in app['depends']: - dependency_path = self.applications[dependency]['path'] - if dependency_path not in dependencies: - dependencies.insert(0, dependency_path) - self.calc_depends(dependencies, dependency) + if key_is_name: + key = appname + else: + key = self.applications[dependency]['path'] + if key not in dependencies: + dependencies.insert(0, key) + self.calc_depends(dependencies, dependency, key_is_name) + def list_oses(self): + oses = set() + for build in self.build_images: + dependencies = [build] + self.calc_depends(dependencies, build, True) + for dependency in dependencies: + if isdir(join(self.applications[dependency]['path'], 'packer', 'os')): + oses.add(dependency) + break + for os in oses: + dependencies = [self.applications[os]['path']] + self.calc_depends(dependencies, os) + yield os, dependencies def list_images(self): - for build in self.builds: + for build in self.build_images: dependencies = [self.applications[build]['path']] self.calc_depends(dependencies, build) yield build, dependencies - def copy_files(self, - src_path: str, - dst_path: str, - ) -> None: - root_len = len(src_path) + 1 - for dir_name, subdir_names, filenames in walk(src_path): - subdir = join(dst_path, dir_name[root_len:]) - if not isdir(subdir): - makedirs(subdir) - for filename in filenames: - path = join(dir_name, filename) - sub_dst_path = join(subdir, filename) - if isfile(sub_dst_path): - raise Exception(_(f'Try to copy {sub_dst_path} which is already exists')) - copy2(path, sub_dst_path) + async def build(self) -> None: + if isdir(self.tmp_dir): + rmtree(self.tmp_dir) + image = Image(self.image_dir, + self.tmp_dir, + ) + print(_('Build OSes')) + if not isdir(join(self.image_dir, 'os')): + makedirs(join(self.image_dir, 'os')) + for application, dependencies_path in self.list_oses(): + print(_(f'Build OS {application}')) + await image.build_os(application, + dependencies_path, + ) + print(_('Build images')) + for application, dependencies_path in self.list_images(): + print(_(f'Build image {application}')) + await image.build_image(application, + dependencies_path, + ) - def load_configuration(self, - dependencies_path: list, - packer_tmp_directory: str, - ) -> dict: + +class Image: + def __init__(self, + image_dir: str, + tmp_dir: str, + ): + self.image_dir = image_dir + self.tmp_dir = tmp_dir + + @staticmethod + def copy_files(dependencies_path: list, + dst_path: str, + element: str, + ) -> None: + for dependency_path in dependencies_path: + src_path = join(dependency_path, + 'packer', + element, + ) + root_len = len(src_path) + 1 + for dir_name, subdir_names, filenames in walk(src_path): + subdir = join(dst_path, dir_name[root_len:]) + if not isdir(subdir): + makedirs(subdir) + for filename in filenames: + path = join(dir_name, filename) + sub_dst_path = join(subdir, filename) + if isfile(sub_dst_path): + raise Exception(_(f'Try to copy {sub_dst_path} which is already exists')) + copy2(path, sub_dst_path) + + async def load_configuration(self, + dependencies_path: list, + packer_tmp_directory: str, + ) -> dict: config = RougailConfig.copy() dictionaries = [join(dependency_path, 'dictionaries') for dependency_path in dependencies_path if isdir(join(dependency_path, 'dictionaries'))] upgrade = RougailUpgrade() @@ -168,11 +221,12 @@ class Image: ) config['dictionaries_dir'] = [dest_dictionaries] config['extra_dictionaries'] = {'packer': [dest_dictionaries_extras]} - return config + self.merge_funcs(config, dependencies_path, packer_tmp_directory) + packer_configuration = await self.get_packer_information(config, packer_tmp_directory) + return packer_configuration - - def merge_funcs(self, - config: RougailConfig, + @staticmethod + def merge_funcs(config: RougailConfig, dependencies_path: list, packer_tmp_directory: str, ): @@ -189,8 +243,8 @@ class Image: fh.write(functions) config['functions_file'] = func_name - async def get_packer_information(self, - config: RougailConfig, + @staticmethod + async def get_packer_information(config: RougailConfig, packer_tmp_directory: str, ) -> dict: eolobj = RougailConvert(config) @@ -198,10 +252,10 @@ class Image: optiondescription = {} exec(xml, None, optiondescription) config = await Config(optiondescription['option_0']) - return await config.option('packer').value.dict(flatten=True) + return await config.option('packer').value.dict(leader_to_list=True, flatten=True) - def do_recipe_checksum(self, - path: str, + @staticmethod + def do_recipe_checksum(path: str, ) -> str: files = [] root_len = len(path) + 1 @@ -210,75 +264,89 @@ class Image: for filename in filenames: with open(join(dir_name, filename), 'rb') as fh: ctl_sum = sha512(fh.read()).hexdigest() - files.append(f'{subpath}/{filename}/ctl_sum') + abs_path = join(subpath, filename) + files.append(f'{abs_path}/{ctl_sum}') + files.sort() + print(files, sha512('\n'.join(files).encode()).hexdigest()) return sha512('\n'.join(files).encode()).hexdigest() - async def build(self) -> None: - if isdir(self.tmp_dir): - rmtree(self.tmp_dir) - for application, dependencies_path in self.list_images(): - packer_tmp_directory = join(self.tmp_dir, - application + '_' + str(time()), - ) - makedirs(packer_tmp_directory) - packer_tmp_os_directory = join(packer_tmp_directory, 'os') - makedirs(packer_tmp_os_directory) - packer_tmp_img_directory = join(packer_tmp_directory, 'image') - makedirs(packer_tmp_img_directory) - config = self.load_configuration(dependencies_path, packer_tmp_directory) - self.merge_funcs(config, dependencies_path, packer_tmp_directory) - packer_configuration = await self.get_packer_information(config, packer_tmp_directory) - packer_dst_os_filename = join(self.image_dir, - 'os', - packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img', - ) - for dependency_path in dependencies_path: - packer_directory = join(dependency_path, - 'packer', - 'os', - ) - self.copy_files(packer_directory, - packer_tmp_os_directory, - ) - packer_directory = join(dependency_path, - 'packer', - 'image', - ) - self.copy_files(packer_directory, - packer_tmp_img_directory, - ) - if not isdir(join(self.image_dir, 'os')): - makedirs(join(self.image_dir, 'os')) - packer_configuration['tmp_directory'] = packer_tmp_os_directory - recipe = {'variables': packer_configuration} - if not isfile(packer_dst_os_filename): - self.build_image(packer_dst_os_filename, - packer_tmp_os_directory, - recipe, - ) - recipe_checksum = self.do_recipe_checksum(packer_tmp_img_directory) - packer_dst_filename = join(self.image_dir, - f'{recipe_checksum}.img', - ) - sha_file = f'{packer_dst_os_filename}.sha256' + def get_tmp_directory(self, + application: str, + ) -> str: + return join(self.tmp_dir, + application + '_' + str(time()), + ) + + def get_os_filename(self, + packer_configuration: dict, + ) -> str: + return join(self.image_dir, + 'os', + packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img', + ) + + def get_image_filename(self, + recipe_checksum: str, + ) -> str: + return join(self.image_dir, + f'{recipe_checksum}.img', + ) + + async def build_os(self, + application: str, + dependencies_path: list, + ) -> None: + packer_tmp_directory = self.get_tmp_directory(application) + packer_configuration = await self.load_configuration(dependencies_path, packer_tmp_directory) + packer_dst_os_filename = self.get_os_filename(packer_configuration) + self.copy_files(dependencies_path, + packer_tmp_directory, + 'os', + ) + packer_configuration['tmp_directory'] = packer_tmp_directory + recipe = {'variables': packer_configuration} + self.build(packer_dst_os_filename, + packer_tmp_directory, + recipe, + ) + + async def build_image(self, + application: str, + dependencies_path: list, + ) -> None: + packer_tmp_directory = self.get_tmp_directory(application) + makedirs(packer_tmp_directory) + self.copy_files(dependencies_path, + packer_tmp_directory, + 'image', + ) + recipe_checksum = self.do_recipe_checksum(packer_tmp_directory) + packer_dst_filename = self.get_image_filename(recipe_checksum) + packer_configuration = await self.load_configuration(dependencies_path, packer_tmp_directory) + packer_dst_os_filename = join(self.image_dir, + 'os', + packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img', + ) + packer_configuration['tmp_directory'] = packer_tmp_directory + recipe = {'variables': packer_configuration} + recipe['variables']['iso_url'] = packer_dst_os_filename + self.build(packer_dst_filename, + packer_tmp_directory, + recipe, + f'{packer_dst_os_filename}.sha256', + ) + + @staticmethod + def build(packer_dst_filename: str, + tmp_directory: str, + recipe: dict, + sha_file: str=None, + ) -> None: + packer_filename = join(tmp_directory, PACKER_FILE_NAME) + if sha_file is not None: with open(sha_file, 'r') as fh: sha256 = fh.read().split(' ', 1)[0] - packer_configuration['tmp_directory'] = packer_tmp_img_directory - recipe = {'variables': packer_configuration} recipe['variables']['iso_checksum'] = sha256 - recipe['variables']['iso_url'] = packer_dst_os_filename - self.build_image(packer_dst_filename, - packer_tmp_img_directory, - recipe, - ) - rmtree(packer_tmp_directory) - - def build_image(self, - packer_dst_filename: str, - tmp_directory: str, - recipe: dict, - ) -> None: - packer_filename = join(tmp_directory, PACKER_FILE_NAME) with open(packer_filename, 'r') as recipe_fd: for key, value in jload(recipe_fd).items(): recipe[key] = value @@ -302,6 +370,9 @@ class Image: proc.wait() if proc.returncode: raise Exception(_(f'cannot build {packer_dst_filename} with {packer_filename}')) + if not isdir(dirname(packer_dst_filename)): + makedirs(dirname(packer_dst_filename)) move(join(tmp_directory, 'image.img'), packer_dst_filename) move(join(tmp_directory, 'image.sha256'), f'{packer_dst_filename}.sha256') print(_(f'Image {packer_dst_filename} created')) + rmtree(tmp_directory) diff --git a/src/risotto/register.py b/src/risotto/register.py index ed01119..2f88214 100644 --- a/src/risotto/register.py +++ b/src/risotto/register.py @@ -83,12 +83,16 @@ def register(uris: str, uris = [uris] def decorator(function): - for uri in uris: - dispatcher.set_function(uri, - notification, - function, - function.__module__ - ) + try: + for uri in uris: + dispatcher.set_function(uri, + notification, + function, + function.__module__ + ) + except NameError: + # if you when register uri, please use get_dispatcher before registered uri + pass return decorator