Infra
/
risotto
9
0
Fork 1
Code Issues 1 Releases Wiki Activity

build image is now in risotto

This commit is contained in:
Emmanuel Garette 2021-08-28 07:34:31 +02:00
parent 1def6e4e4c
commit 8dca850683
3 changed files with 185 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/risotto/controller.py
View File

@ -64,8 +64,8 @@ class Controller:
**kwargs, **kwargs,
) )
@staticmethod async def check_role(self,
async def check_role(uri: str, uri: str,
username: str, username: str,
**kwargs: dict, **kwargs: dict,
) -> None: ) -> None:

275
src/risotto/image.py
View File

@ -66,7 +66,7 @@ def zone_information(**kwargs):
""" """
class Image: class Images:
def __init__(self, def __init__(self,
image_dir: str=None, image_dir: str=None,
tmp_dir: str=None, tmp_dir: str=None,
@ -79,10 +79,10 @@ class Image:
if tmp_dir is None: if tmp_dir is None:
tmp_dir = PACKER_TMP_DIRECTORY tmp_dir = PACKER_TMP_DIRECTORY
self.tmp_dir = tmp_dir self.tmp_dir = tmp_dir
self.parse_applications() self.load_applications()
def parse_applications(self) -> None: def load_applications(self) -> None:
self.builds = [] self.build_images = []
self.applications = {} self.applications = {}
for distrib in listdir(join(DATASET_PATH, 'seed')): for distrib in listdir(join(DATASET_PATH, 'seed')):
distrib_dir = join(DATASET_PATH, 'seed', distrib, 'applicationservice') distrib_dir = join(DATASET_PATH, 'seed', distrib, 'applicationservice')
@ -104,48 +104,101 @@ class Image:
'yml': app, 'yml': app,
} }
if 'service' in app and app['service']: if 'service' in app and app['service']:
self.builds.append(applicationservice) self.build_images.append(applicationservice)
def calc_depends(self, def calc_depends(self,
dependencies: list, dependencies: list,
appname, appname,
key_is_name=False,
): ):
app = self.applications[appname]['yml'] app = self.applications[appname]['yml']
if not 'depends' in app or not app['depends']: if not 'depends' in app or not app['depends']:
return return
for dependency in app['depends']: for dependency in app['depends']:
dependency_path = self.applications[dependency]['path'] if key_is_name:
if dependency_path not in dependencies: key = appname
dependencies.insert(0, dependency_path) else:
self.calc_depends(dependencies, dependency) key = self.applications[dependency]['path']
if key not in dependencies:
dependencies.insert(0, key)
self.calc_depends(dependencies, dependency, key_is_name)
def list_oses(self):
oses = set()
for build in self.build_images:
dependencies = [build]
self.calc_depends(dependencies, build, True)
for dependency in dependencies:
if isdir(join(self.applications[dependency]['path'], 'packer', 'os')):
oses.add(dependency)
break
for os in oses:
dependencies = [self.applications[os]['path']]
self.calc_depends(dependencies, os)
yield os, dependencies
def list_images(self): def list_images(self):
for build in self.builds: for build in self.build_images:
dependencies = [self.applications[build]['path']] dependencies = [self.applications[build]['path']]
self.calc_depends(dependencies, build) self.calc_depends(dependencies, build)
yield build, dependencies yield build, dependencies
def copy_files(self, async def build(self) -> None:
src_path: str, if isdir(self.tmp_dir):
dst_path: str, rmtree(self.tmp_dir)
) -> None: image = Image(self.image_dir,
root_len = len(src_path) + 1 self.tmp_dir,
for dir_name, subdir_names, filenames in walk(src_path): )
subdir = join(dst_path, dir_name[root_len:]) print(_('Build OSes'))
if not isdir(subdir): if not isdir(join(self.image_dir, 'os')):
makedirs(subdir) makedirs(join(self.image_dir, 'os'))
for filename in filenames: for application, dependencies_path in self.list_oses():
path = join(dir_name, filename) print(_(f'Build OS {application}'))
sub_dst_path = join(subdir, filename) await image.build_os(application,
if isfile(sub_dst_path): dependencies_path,
raise Exception(_(f'Try to copy {sub_dst_path} which is already exists')) )
copy2(path, sub_dst_path) print(_('Build images'))
for application, dependencies_path in self.list_images():
print(_(f'Build image {application}'))
await image.build_image(application,
dependencies_path,
)
def load_configuration(self,
dependencies_path: list, class Image:
packer_tmp_directory: str, def __init__(self,
) -> dict: image_dir: str,
tmp_dir: str,
):
self.image_dir = image_dir
self.tmp_dir = tmp_dir
@staticmethod
def copy_files(dependencies_path: list,
dst_path: str,
element: str,
) -> None:
for dependency_path in dependencies_path:
src_path = join(dependency_path,
'packer',
element,
)
root_len = len(src_path) + 1
for dir_name, subdir_names, filenames in walk(src_path):
subdir = join(dst_path, dir_name[root_len:])
if not isdir(subdir):
makedirs(subdir)
for filename in filenames:
path = join(dir_name, filename)
sub_dst_path = join(subdir, filename)
if isfile(sub_dst_path):
raise Exception(_(f'Try to copy {sub_dst_path} which is already exists'))
copy2(path, sub_dst_path)
async def load_configuration(self,
dependencies_path: list,
packer_tmp_directory: str,
) -> dict:
config = RougailConfig.copy() config = RougailConfig.copy()
dictionaries = [join(dependency_path, 'dictionaries') for dependency_path in dependencies_path if isdir(join(dependency_path, 'dictionaries'))] dictionaries = [join(dependency_path, 'dictionaries') for dependency_path in dependencies_path if isdir(join(dependency_path, 'dictionaries'))]
upgrade = RougailUpgrade() upgrade = RougailUpgrade()
@ -168,11 +221,12 @@ class Image:
) )
config['dictionaries_dir'] = [dest_dictionaries] config['dictionaries_dir'] = [dest_dictionaries]
config['extra_dictionaries'] = {'packer': [dest_dictionaries_extras]} config['extra_dictionaries'] = {'packer': [dest_dictionaries_extras]}
return config self.merge_funcs(config, dependencies_path, packer_tmp_directory)
packer_configuration = await self.get_packer_information(config, packer_tmp_directory)
return packer_configuration
@staticmethod
def merge_funcs(self, def merge_funcs(config: RougailConfig,
config: RougailConfig,
dependencies_path: list, dependencies_path: list,
packer_tmp_directory: str, packer_tmp_directory: str,
): ):
@ -189,8 +243,8 @@ class Image:
fh.write(functions) fh.write(functions)
config['functions_file'] = func_name config['functions_file'] = func_name
async def get_packer_information(self, @staticmethod
config: RougailConfig, async def get_packer_information(config: RougailConfig,
packer_tmp_directory: str, packer_tmp_directory: str,
) -> dict: ) -> dict:
eolobj = RougailConvert(config) eolobj = RougailConvert(config)
@ -198,10 +252,10 @@ class Image:
optiondescription = {} optiondescription = {}
exec(xml, None, optiondescription) exec(xml, None, optiondescription)
config = await Config(optiondescription['option_0']) config = await Config(optiondescription['option_0'])
return await config.option('packer').value.dict(flatten=True) return await config.option('packer').value.dict(leader_to_list=True, flatten=True)
def do_recipe_checksum(self, @staticmethod
path: str, def do_recipe_checksum(path: str,
) -> str: ) -> str:
files = [] files = []
root_len = len(path) + 1 root_len = len(path) + 1
@ -210,75 +264,89 @@ class Image:
for filename in filenames: for filename in filenames:
with open(join(dir_name, filename), 'rb') as fh: with open(join(dir_name, filename), 'rb') as fh:
ctl_sum = sha512(fh.read()).hexdigest() ctl_sum = sha512(fh.read()).hexdigest()
files.append(f'{subpath}/{filename}/ctl_sum') abs_path = join(subpath, filename)
files.append(f'{abs_path}/{ctl_sum}')
files.sort()
print(files, sha512('\n'.join(files).encode()).hexdigest())
return sha512('\n'.join(files).encode()).hexdigest() return sha512('\n'.join(files).encode()).hexdigest()
async def build(self) -> None: def get_tmp_directory(self,
if isdir(self.tmp_dir): application: str,
rmtree(self.tmp_dir) ) -> str:
for application, dependencies_path in self.list_images(): return join(self.tmp_dir,
packer_tmp_directory = join(self.tmp_dir, application + '_' + str(time()),
application + '_' + str(time()), )
)
makedirs(packer_tmp_directory) def get_os_filename(self,
packer_tmp_os_directory = join(packer_tmp_directory, 'os') packer_configuration: dict,
makedirs(packer_tmp_os_directory) ) -> str:
packer_tmp_img_directory = join(packer_tmp_directory, 'image') return join(self.image_dir,
makedirs(packer_tmp_img_directory) 'os',
config = self.load_configuration(dependencies_path, packer_tmp_directory) packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img',
self.merge_funcs(config, dependencies_path, packer_tmp_directory) )
packer_configuration = await self.get_packer_information(config, packer_tmp_directory)
packer_dst_os_filename = join(self.image_dir, def get_image_filename(self,
'os', recipe_checksum: str,
packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img', ) -> str:
) return join(self.image_dir,
for dependency_path in dependencies_path: f'{recipe_checksum}.img',
packer_directory = join(dependency_path, )
'packer',
'os', async def build_os(self,
) application: str,
self.copy_files(packer_directory, dependencies_path: list,
packer_tmp_os_directory, ) -> None:
) packer_tmp_directory = self.get_tmp_directory(application)
packer_directory = join(dependency_path, packer_configuration = await self.load_configuration(dependencies_path, packer_tmp_directory)
'packer', packer_dst_os_filename = self.get_os_filename(packer_configuration)
'image', self.copy_files(dependencies_path,
) packer_tmp_directory,
self.copy_files(packer_directory, 'os',
packer_tmp_img_directory, )
) packer_configuration['tmp_directory'] = packer_tmp_directory
if not isdir(join(self.image_dir, 'os')): recipe = {'variables': packer_configuration}
makedirs(join(self.image_dir, 'os')) self.build(packer_dst_os_filename,
packer_configuration['tmp_directory'] = packer_tmp_os_directory packer_tmp_directory,
recipe = {'variables': packer_configuration} recipe,
if not isfile(packer_dst_os_filename): )
self.build_image(packer_dst_os_filename,
packer_tmp_os_directory, async def build_image(self,
recipe, application: str,
) dependencies_path: list,
recipe_checksum = self.do_recipe_checksum(packer_tmp_img_directory) ) -> None:
packer_dst_filename = join(self.image_dir, packer_tmp_directory = self.get_tmp_directory(application)
f'{recipe_checksum}.img', makedirs(packer_tmp_directory)
) self.copy_files(dependencies_path,
sha_file = f'{packer_dst_os_filename}.sha256' packer_tmp_directory,
'image',
)
recipe_checksum = self.do_recipe_checksum(packer_tmp_directory)
packer_dst_filename = self.get_image_filename(recipe_checksum)
packer_configuration = await self.load_configuration(dependencies_path, packer_tmp_directory)
packer_dst_os_filename = join(self.image_dir,
'os',
packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img',
)
packer_configuration['tmp_directory'] = packer_tmp_directory
recipe = {'variables': packer_configuration}
recipe['variables']['iso_url'] = packer_dst_os_filename
self.build(packer_dst_filename,
packer_tmp_directory,
recipe,
f'{packer_dst_os_filename}.sha256',
)
@staticmethod
def build(packer_dst_filename: str,
tmp_directory: str,
recipe: dict,
sha_file: str=None,
) -> None:
packer_filename = join(tmp_directory, PACKER_FILE_NAME)
if sha_file is not None:
with open(sha_file, 'r') as fh: with open(sha_file, 'r') as fh:
sha256 = fh.read().split(' ', 1)[0] sha256 = fh.read().split(' ', 1)[0]
packer_configuration['tmp_directory'] = packer_tmp_img_directory
recipe = {'variables': packer_configuration}
recipe['variables']['iso_checksum'] = sha256 recipe['variables']['iso_checksum'] = sha256
recipe['variables']['iso_url'] = packer_dst_os_filename
self.build_image(packer_dst_filename,
packer_tmp_img_directory,
recipe,
)
rmtree(packer_tmp_directory)
def build_image(self,
packer_dst_filename: str,
tmp_directory: str,
recipe: dict,
) -> None:
packer_filename = join(tmp_directory, PACKER_FILE_NAME)
with open(packer_filename, 'r') as recipe_fd: with open(packer_filename, 'r') as recipe_fd:
for key, value in jload(recipe_fd).items(): for key, value in jload(recipe_fd).items():
recipe[key] = value recipe[key] = value
@ -302,6 +370,9 @@ class Image:
proc.wait() proc.wait()
if proc.returncode: if proc.returncode:
raise Exception(_(f'cannot build {packer_dst_filename} with {packer_filename}')) raise Exception(_(f'cannot build {packer_dst_filename} with {packer_filename}'))
if not isdir(dirname(packer_dst_filename)):
makedirs(dirname(packer_dst_filename))
move(join(tmp_directory, 'image.img'), packer_dst_filename) move(join(tmp_directory, 'image.img'), packer_dst_filename)
move(join(tmp_directory, 'image.sha256'), f'{packer_dst_filename}.sha256') move(join(tmp_directory, 'image.sha256'), f'{packer_dst_filename}.sha256')
print(_(f'Image {packer_dst_filename} created')) print(_(f'Image {packer_dst_filename} created'))
rmtree(tmp_directory)

16
src/risotto/register.py
View File

@ -83,12 +83,16 @@ def register(uris: str,
uris = [uris] uris = [uris]
def decorator(function): def decorator(function):
for uri in uris: try:
dispatcher.set_function(uri, for uri in uris:
notification, dispatcher.set_function(uri,
function, notification,
function.__module__ function,
) function.__module__
)
except NameError:
# if you when register uri, please use get_dispatcher before registered uri
pass
return decorator return decorator