Merge branch 'develop' into dist/risotto/risotto-2.8.0/develop
This commit is contained in:
commit
4d2439c250
|
|
@ -0,0 +1,278 @@
|
|||
from os import listdir, walk, makedirs
|
||||
from os.path import isfile, isdir, join, dirname
|
||||
from yaml import load, SafeLoader
|
||||
from json import load as jload, dump as jdump
|
||||
from time import time
|
||||
from shutil import copy2, rmtree, move
|
||||
from hashlib import sha512
|
||||
from subprocess import Popen
|
||||
|
||||
from rougail import RougailConvert, RougailConfig, RougailUpgrade
|
||||
try:
|
||||
from tiramisu3 import Config
|
||||
except:
|
||||
from tiramisu import Config
|
||||
|
||||
from .utils import _
|
||||
|
||||
|
||||
DATASET_PATH = ''
|
||||
TMP_DIRECTORY = '/tmp'
|
||||
PACKER_TMP_DIRECTORY = join(TMP_DIRECTORY, 'packer')
|
||||
PACKER_FILE_NAME = 'recipe.json'
|
||||
IMAGES_DIRECTORY = join(TMP_DIRECTORY, 'images')
|
||||
|
||||
|
||||
FUNCTIONS = b"""from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value
|
||||
# =============================================================
|
||||
# fork of risotto-setting/src/risotto_setting/config/config.py
|
||||
def get_password(**kwargs):
|
||||
return 'password'
|
||||
|
||||
|
||||
def get_ip(**kwargs):
|
||||
return '1.1.1.1'
|
||||
|
||||
|
||||
def get_chain(**kwargs):
|
||||
return 'chain'
|
||||
|
||||
|
||||
def get_certificates(**kwargs):
|
||||
return []
|
||||
|
||||
|
||||
def get_certificate(**kwargs):
|
||||
return 'certificate'
|
||||
|
||||
|
||||
def get_private_key(**kwargs):
|
||||
return 'private_key'
|
||||
|
||||
|
||||
def get_linked_configuration(**kwargs):
|
||||
if 'test' in kwargs and kwargs['test']:
|
||||
return kwargs['test'][0]
|
||||
return 'configuration'
|
||||
|
||||
|
||||
def zone_information(**kwargs):
|
||||
return 'zone'
|
||||
# =============================================================
|
||||
|
||||
"""
|
||||
|
||||
|
||||
class Image:
|
||||
def __init__(self):
|
||||
self.parse_applications()
|
||||
|
||||
def parse_applications(self) -> None:
|
||||
self.builds = []
|
||||
self.applications = {}
|
||||
for distrib in listdir(join(DATASET_PATH, 'seed')):
|
||||
distrib_dir = join(DATASET_PATH, 'seed', distrib, 'applicationservice')
|
||||
if not isdir(distrib_dir):
|
||||
continue
|
||||
for release in listdir(distrib_dir):
|
||||
release_dir = join(distrib_dir, release)
|
||||
if not isdir(release_dir):
|
||||
continue
|
||||
for applicationservice in listdir(release_dir):
|
||||
applicationservice_dir = join(release_dir, applicationservice)
|
||||
if not isdir(applicationservice_dir):
|
||||
continue
|
||||
if applicationservice in self.applications:
|
||||
raise Exception('multi applicationservice')
|
||||
with open(join(applicationservice_dir, 'applicationservice.yml')) as yaml:
|
||||
app = load(yaml, Loader=SafeLoader)
|
||||
self.applications[applicationservice] = {'path': applicationservice_dir,
|
||||
'yml': app,
|
||||
}
|
||||
if 'service' in app and app['service']:
|
||||
self.builds.append(applicationservice)
|
||||
|
||||
def calc_depends(self,
|
||||
dependencies: list,
|
||||
appname,
|
||||
):
|
||||
app = self.applications[appname]['yml']
|
||||
if not 'depends' in app or not app['depends']:
|
||||
return
|
||||
for dependency in app['depends']:
|
||||
dependency_path = self.applications[dependency]['path']
|
||||
if dependency_path not in dependencies:
|
||||
dependencies.insert(0, dependency_path)
|
||||
self.calc_depends(dependencies, dependency)
|
||||
|
||||
|
||||
def list_images(self):
|
||||
for build in self.builds:
|
||||
dependencies = [self.applications[build]['path']]
|
||||
self.calc_depends(dependencies, build)
|
||||
yield build, dependencies
|
||||
|
||||
def copy_files(self,
|
||||
src_path: str,
|
||||
dst_path: str,
|
||||
) -> None:
|
||||
root_len = len(src_path) + 1
|
||||
for dir_name, subdir_names, filenames in walk(src_path):
|
||||
subdir = join(dst_path, dir_name[root_len:])
|
||||
if not isdir(subdir):
|
||||
makedirs(subdir)
|
||||
for filename in filenames:
|
||||
path = join(dir_name, filename)
|
||||
sub_dst_path = join(subdir, filename)
|
||||
if isfile(sub_dst_path):
|
||||
raise Exception(_(f'Try to copy {sub_dst_path} which is already exists'))
|
||||
copy2(path, sub_dst_path)
|
||||
|
||||
def load_configuration(self,
|
||||
dependencies_path: list,
|
||||
packer_tmp_directory: str,
|
||||
) -> dict:
|
||||
config = RougailConfig.copy()
|
||||
dictionaries = [join(dependency_path, 'dictionaries') for dependency_path in dependencies_path if isdir(join(dependency_path, 'dictionaries'))]
|
||||
upgrade = RougailUpgrade()
|
||||
dest_dictionaries = join(packer_tmp_directory, 'dictionaries')
|
||||
makedirs(dest_dictionaries)
|
||||
dest_dictionaries_extras = join(packer_tmp_directory, 'dictionaries_extras')
|
||||
makedirs(dest_dictionaries_extras)
|
||||
for dependency_path in dependencies_path:
|
||||
dictionaries_dir = join(dependency_path, 'dictionaries')
|
||||
if isdir(dictionaries_dir):
|
||||
upgrade.load_xml_from_folders(dictionaries_dir,
|
||||
dest_dictionaries,
|
||||
RougailConfig['variable_namespace'],
|
||||
)
|
||||
extra_dir = join(dependency_path, 'extras', 'packer')
|
||||
if isdir(extra_dir):
|
||||
upgrade.load_xml_from_folders(extra_dir,
|
||||
dest_dictionaries_extras,
|
||||
'packer',
|
||||
)
|
||||
config['dictionaries_dir'] = [dest_dictionaries]
|
||||
config['extra_dictionaries'] = {'packer': [dest_dictionaries_extras]}
|
||||
return config
|
||||
|
||||
|
||||
def merge_funcs(self,
|
||||
config: RougailConfig,
|
||||
dependencies_path: list,
|
||||
packer_tmp_directory: str,
|
||||
):
|
||||
functions = FUNCTIONS
|
||||
for dependency_path in dependencies_path:
|
||||
funcs_dir = join(dependency_path, 'funcs')
|
||||
if not isdir(funcs_dir):
|
||||
continue
|
||||
for func in listdir(funcs_dir):
|
||||
print(join(funcs_dir, func))
|
||||
with open(join(funcs_dir, func), 'rb') as fh:
|
||||
functions += fh.read()
|
||||
func_name = join(packer_tmp_directory, 'func.py')
|
||||
with open(func_name, 'wb') as fh:
|
||||
fh.write(functions)
|
||||
config['functions_file'] = func_name
|
||||
|
||||
async def get_packer_information(self,
|
||||
config: RougailConfig,
|
||||
packer_tmp_directory: str,
|
||||
) -> dict:
|
||||
eolobj = RougailConvert(config)
|
||||
xml = eolobj.save(join(packer_tmp_directory, 'tiramisu.py'))
|
||||
optiondescription = {}
|
||||
exec(xml, None, optiondescription)
|
||||
config = await Config(optiondescription['option_0'])
|
||||
return await config.option('packer').value.dict(flatten=True)
|
||||
|
||||
def do_recipe_checksum(self,
|
||||
path: str,
|
||||
) -> str:
|
||||
files = []
|
||||
root_len = len(path) + 1
|
||||
for dir_name, subdir_names, filenames in walk(path):
|
||||
subpath = dir_name[root_len:]
|
||||
for filename in filenames:
|
||||
with open(join(dir_name, filename), 'rb') as fh:
|
||||
ctl_sum = sha512(fh.read()).hexdigest()
|
||||
files.append(f'{subpath}/{filename}/ctl_sum')
|
||||
return sha512('\n'.join(files).encode()).hexdigest()
|
||||
|
||||
async def build(self) -> None:
|
||||
if isdir(PACKER_TMP_DIRECTORY):
|
||||
rmtree(PACKER_TMP_DIRECTORY)
|
||||
for application, dependencies_path in self.list_images():
|
||||
packer_tmp_directory = join(PACKER_TMP_DIRECTORY,
|
||||
application + '_' + str(time()),
|
||||
)
|
||||
makedirs(packer_tmp_directory)
|
||||
packer_tmp_os_directory = join(packer_tmp_directory, 'os')
|
||||
makedirs(packer_tmp_os_directory)
|
||||
packer_tmp_img_directory = join(packer_tmp_directory, 'image')
|
||||
makedirs(packer_tmp_img_directory)
|
||||
config = self.load_configuration(dependencies_path, packer_tmp_directory)
|
||||
self.merge_funcs(config, dependencies_path, packer_tmp_directory)
|
||||
packer_configuration = await self.get_packer_information(config, packer_tmp_directory)
|
||||
# OS image needed ?
|
||||
packer_dst_os_filename = join(IMAGES_DIRECTORY,
|
||||
'os',
|
||||
packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img',
|
||||
)
|
||||
for dependency_path in dependencies_path:
|
||||
packer_directory = join(dependency_path,
|
||||
'packer',
|
||||
'os',
|
||||
)
|
||||
self.copy_files(packer_directory,
|
||||
packer_tmp_os_directory,
|
||||
)
|
||||
packer_directory = join(dependency_path,
|
||||
'packer',
|
||||
'image',
|
||||
)
|
||||
self.copy_files(packer_directory,
|
||||
packer_tmp_img_directory,
|
||||
)
|
||||
if not isfile(packer_dst_os_filename):
|
||||
self.build_image(packer_dst_os_filename,
|
||||
packer_tmp_os_directory,
|
||||
packer_configuration,
|
||||
)
|
||||
recipe_checksum = self.do_recipe_checksum(packer_tmp_img_directory)
|
||||
packer_dst_filename = join(IMAGES_DIRECTORY,
|
||||
f'{recipe_checksum}.img',
|
||||
)
|
||||
self.build_image(packer_dst_filename,
|
||||
packer_tmp_img_directory,
|
||||
packer_configuration,
|
||||
)
|
||||
|
||||
def build_image(self,
|
||||
packer_dst_filename: str,
|
||||
packer_tmp_directory: str,
|
||||
packer_configuration: dict,
|
||||
) -> None:
|
||||
tmp_directory = join(packer_tmp_directory, 'tmp')
|
||||
makedirs(tmp_directory)
|
||||
packer_configuration['tmp_directory'] = tmp_directory
|
||||
recipe = {'variables': packer_configuration}
|
||||
packer_filename = join(packer_tmp_directory, PACKER_FILE_NAME)
|
||||
with open(packer_filename, 'r') as recipe_fd:
|
||||
for key, value in jload(recipe_fd).items():
|
||||
recipe[key] = value
|
||||
with open(packer_filename, 'w') as recipe_fd:
|
||||
jdump(recipe, recipe_fd, indent=2)
|
||||
proc = Popen(['packer', 'build', packer_filename],
|
||||
#stdout=PIPE,
|
||||
#stderr=PIPE,
|
||||
cwd=tmp_directory,
|
||||
)
|
||||
proc.wait()
|
||||
if proc.returncode:
|
||||
raise Exception(_(f'cannot build {application} with {packer_filename}'))
|
||||
move(join(tmp_directory, 'image.img'), packer_dst_filename)
|
||||
move(join(tmp_directory, 'image.sha256'), f'{packer_dst_filename}.sha256')
|
||||
rmtree(tmp_directory)
|
||||
print(_(f'Image {packer_dst_filename} created'))
|
||||
Loading…
Reference in New Issue