379 lines
15 KiB
Python
379 lines
15 KiB
Python
from os import listdir, walk, makedirs
|
|
from os.path import isfile, isdir, join, dirname
|
|
from yaml import load, SafeLoader
|
|
from json import load as jload, dump as jdump
|
|
from time import time
|
|
from shutil import copy2, rmtree, move
|
|
from hashlib import sha512
|
|
from subprocess import Popen
|
|
|
|
from rougail import RougailConvert, RougailConfig, RougailUpgrade
|
|
try:
|
|
from tiramisu3 import Config
|
|
except:
|
|
from tiramisu import Config
|
|
|
|
from .utils import _
|
|
|
|
|
|
DATASET_PATH = '/usr/share/risotto/'
|
|
TMP_DIRECTORY = '/tmp'
|
|
PACKER_TMP_DIRECTORY = join(TMP_DIRECTORY, 'packer')
|
|
PACKER_FILE_NAME = 'recipe.json'
|
|
IMAGES_DIRECTORY = join(TMP_DIRECTORY, 'images')
|
|
|
|
|
|
FUNCTIONS = b"""try:
|
|
from tiramisu3 import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value
|
|
except:
|
|
from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value
|
|
# =============================================================
|
|
# fork of risotto-setting/src/risotto_setting/config/config.py
|
|
def get_password(**kwargs):
|
|
return 'password'
|
|
|
|
|
|
def get_ip(**kwargs):
|
|
return '1.1.1.1'
|
|
|
|
|
|
def get_chain(**kwargs):
|
|
return 'chain'
|
|
|
|
|
|
def get_certificates(**kwargs):
|
|
return []
|
|
|
|
|
|
def get_certificate(**kwargs):
|
|
return 'certificate'
|
|
|
|
|
|
def get_private_key(**kwargs):
|
|
return 'private_key'
|
|
|
|
|
|
def get_linked_configuration(**kwargs):
|
|
if 'test' in kwargs and kwargs['test']:
|
|
return kwargs['test'][0]
|
|
return 'configuration'
|
|
|
|
|
|
def zone_information(**kwargs):
|
|
return 'zone'
|
|
# =============================================================
|
|
|
|
"""
|
|
|
|
|
|
class Images:
|
|
def __init__(self,
|
|
image_dir: str=None,
|
|
tmp_dir: str=None,
|
|
):
|
|
if image_dir is None:
|
|
image_dir = IMAGES_DIRECTORY
|
|
self.image_dir = image_dir
|
|
if isdir(self.image_dir):
|
|
rmtree(self.image_dir)
|
|
if tmp_dir is None:
|
|
tmp_dir = PACKER_TMP_DIRECTORY
|
|
self.tmp_dir = tmp_dir
|
|
self.load_applications()
|
|
|
|
def load_applications(self) -> None:
|
|
self.build_images = []
|
|
self.applications = {}
|
|
for distrib in listdir(join(DATASET_PATH, 'seed')):
|
|
distrib_dir = join(DATASET_PATH, 'seed', distrib, 'applicationservice')
|
|
if not isdir(distrib_dir):
|
|
continue
|
|
for release in listdir(distrib_dir):
|
|
release_dir = join(distrib_dir, release)
|
|
if not isdir(release_dir):
|
|
continue
|
|
for applicationservice in listdir(release_dir):
|
|
applicationservice_dir = join(release_dir, applicationservice)
|
|
if not isdir(applicationservice_dir):
|
|
continue
|
|
if applicationservice in self.applications:
|
|
raise Exception('multi applicationservice')
|
|
with open(join(applicationservice_dir, 'applicationservice.yml')) as yaml:
|
|
app = load(yaml, Loader=SafeLoader)
|
|
self.applications[applicationservice] = {'path': applicationservice_dir,
|
|
'yml': app,
|
|
}
|
|
if 'service' in app and app['service']:
|
|
self.build_images.append(applicationservice)
|
|
|
|
def calc_depends(self,
|
|
dependencies: list,
|
|
appname,
|
|
key_is_name=False,
|
|
):
|
|
app = self.applications[appname]['yml']
|
|
if not 'depends' in app or not app['depends']:
|
|
return
|
|
for dependency in app['depends']:
|
|
if key_is_name:
|
|
key = appname
|
|
else:
|
|
key = self.applications[dependency]['path']
|
|
if key not in dependencies:
|
|
dependencies.insert(0, key)
|
|
self.calc_depends(dependencies, dependency, key_is_name)
|
|
|
|
def list_oses(self):
|
|
oses = set()
|
|
for build in self.build_images:
|
|
dependencies = [build]
|
|
self.calc_depends(dependencies, build, True)
|
|
for dependency in dependencies:
|
|
if isdir(join(self.applications[dependency]['path'], 'packer', 'os')):
|
|
oses.add(dependency)
|
|
break
|
|
for os in oses:
|
|
dependencies = [self.applications[os]['path']]
|
|
self.calc_depends(dependencies, os)
|
|
yield os, dependencies
|
|
|
|
def list_images(self):
|
|
for build in self.build_images:
|
|
dependencies = [self.applications[build]['path']]
|
|
self.calc_depends(dependencies, build)
|
|
yield build, dependencies
|
|
|
|
async def build(self) -> None:
|
|
if isdir(self.tmp_dir):
|
|
rmtree(self.tmp_dir)
|
|
image = Image(self.image_dir,
|
|
self.tmp_dir,
|
|
)
|
|
print(_('Build OSes'))
|
|
if not isdir(join(self.image_dir, 'os')):
|
|
makedirs(join(self.image_dir, 'os'))
|
|
for application, dependencies_path in self.list_oses():
|
|
print(_(f'Build OS {application}'))
|
|
await image.build_os(application,
|
|
dependencies_path,
|
|
)
|
|
print(_('Build images'))
|
|
for application, dependencies_path in self.list_images():
|
|
print(_(f'Build image {application}'))
|
|
await image.build_image(application,
|
|
dependencies_path,
|
|
)
|
|
|
|
|
|
class Image:
|
|
def __init__(self,
|
|
image_dir: str,
|
|
tmp_dir: str,
|
|
):
|
|
self.image_dir = image_dir
|
|
self.tmp_dir = tmp_dir
|
|
|
|
@staticmethod
|
|
def copy_files(dependencies_path: list,
|
|
dst_path: str,
|
|
element: str,
|
|
) -> None:
|
|
for dependency_path in dependencies_path:
|
|
src_path = join(dependency_path,
|
|
'packer',
|
|
element,
|
|
)
|
|
root_len = len(src_path) + 1
|
|
for dir_name, subdir_names, filenames in walk(src_path):
|
|
subdir = join(dst_path, dir_name[root_len:])
|
|
if not isdir(subdir):
|
|
makedirs(subdir)
|
|
for filename in filenames:
|
|
path = join(dir_name, filename)
|
|
sub_dst_path = join(subdir, filename)
|
|
if isfile(sub_dst_path):
|
|
raise Exception(_(f'Try to copy {sub_dst_path} which is already exists'))
|
|
copy2(path, sub_dst_path)
|
|
|
|
async def load_configuration(self,
|
|
dependencies_path: list,
|
|
packer_tmp_directory: str,
|
|
) -> dict:
|
|
config = RougailConfig.copy()
|
|
dictionaries = [join(dependency_path, 'dictionaries') for dependency_path in dependencies_path if isdir(join(dependency_path, 'dictionaries'))]
|
|
upgrade = RougailUpgrade()
|
|
dest_dictionaries = join(packer_tmp_directory, 'dictionaries')
|
|
makedirs(dest_dictionaries)
|
|
dest_dictionaries_extras = join(packer_tmp_directory, 'dictionaries_extras')
|
|
makedirs(dest_dictionaries_extras)
|
|
for dependency_path in dependencies_path:
|
|
dictionaries_dir = join(dependency_path, 'dictionaries')
|
|
if isdir(dictionaries_dir):
|
|
upgrade.load_xml_from_folders(dictionaries_dir,
|
|
dest_dictionaries,
|
|
RougailConfig['variable_namespace'],
|
|
)
|
|
extra_dir = join(dependency_path, 'extras', 'packer')
|
|
if isdir(extra_dir):
|
|
upgrade.load_xml_from_folders(extra_dir,
|
|
dest_dictionaries_extras,
|
|
'packer',
|
|
)
|
|
config['dictionaries_dir'] = [dest_dictionaries]
|
|
config['extra_dictionaries'] = {'packer': [dest_dictionaries_extras]}
|
|
self.merge_funcs(config, dependencies_path, packer_tmp_directory)
|
|
packer_configuration = await self.get_packer_information(config, packer_tmp_directory)
|
|
return packer_configuration
|
|
|
|
@staticmethod
|
|
def merge_funcs(config: RougailConfig,
|
|
dependencies_path: list,
|
|
packer_tmp_directory: str,
|
|
):
|
|
functions = FUNCTIONS
|
|
for dependency_path in dependencies_path:
|
|
funcs_dir = join(dependency_path, 'funcs')
|
|
if not isdir(funcs_dir):
|
|
continue
|
|
for func in listdir(funcs_dir):
|
|
with open(join(funcs_dir, func), 'rb') as fh:
|
|
functions += fh.read()
|
|
func_name = join(packer_tmp_directory, 'func.py')
|
|
with open(func_name, 'wb') as fh:
|
|
fh.write(functions)
|
|
config['functions_file'] = func_name
|
|
|
|
@staticmethod
|
|
async def get_packer_information(config: RougailConfig,
|
|
packer_tmp_directory: str,
|
|
) -> dict:
|
|
eolobj = RougailConvert(config)
|
|
xml = eolobj.save(join(packer_tmp_directory, 'tiramisu.py'))
|
|
optiondescription = {}
|
|
exec(xml, None, optiondescription)
|
|
config = await Config(optiondescription['option_0'])
|
|
return await config.option('packer').value.dict(leader_to_list=True, flatten=True)
|
|
|
|
@staticmethod
|
|
def do_recipe_checksum(path: str,
|
|
) -> str:
|
|
files = []
|
|
root_len = len(path) + 1
|
|
for dir_name, subdir_names, filenames in walk(path):
|
|
subpath = dir_name[root_len:]
|
|
for filename in filenames:
|
|
with open(join(dir_name, filename), 'rb') as fh:
|
|
ctl_sum = sha512(fh.read()).hexdigest()
|
|
abs_path = join(subpath, filename)
|
|
files.append(f'{abs_path}/{ctl_sum}')
|
|
files.sort()
|
|
print(files, sha512('\n'.join(files).encode()).hexdigest())
|
|
return sha512('\n'.join(files).encode()).hexdigest()
|
|
|
|
def get_tmp_directory(self,
|
|
application: str,
|
|
) -> str:
|
|
return join(self.tmp_dir,
|
|
application + '_' + str(time()),
|
|
)
|
|
|
|
def get_os_filename(self,
|
|
packer_configuration: dict,
|
|
) -> str:
|
|
return join(self.image_dir,
|
|
'os',
|
|
packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img',
|
|
)
|
|
|
|
def get_image_filename(self,
|
|
recipe_checksum: str,
|
|
) -> str:
|
|
return join(self.image_dir,
|
|
f'{recipe_checksum}.img',
|
|
)
|
|
|
|
async def build_os(self,
|
|
application: str,
|
|
dependencies_path: list,
|
|
) -> None:
|
|
packer_tmp_directory = self.get_tmp_directory(application)
|
|
packer_configuration = await self.load_configuration(dependencies_path, packer_tmp_directory)
|
|
packer_dst_os_filename = self.get_os_filename(packer_configuration)
|
|
self.copy_files(dependencies_path,
|
|
packer_tmp_directory,
|
|
'os',
|
|
)
|
|
packer_configuration['tmp_directory'] = packer_tmp_directory
|
|
recipe = {'variables': packer_configuration}
|
|
self.build(packer_dst_os_filename,
|
|
packer_tmp_directory,
|
|
recipe,
|
|
)
|
|
|
|
async def build_image(self,
|
|
application: str,
|
|
dependencies_path: list,
|
|
) -> None:
|
|
packer_tmp_directory = self.get_tmp_directory(application)
|
|
makedirs(packer_tmp_directory)
|
|
self.copy_files(dependencies_path,
|
|
packer_tmp_directory,
|
|
'image',
|
|
)
|
|
recipe_checksum = self.do_recipe_checksum(packer_tmp_directory)
|
|
packer_dst_filename = self.get_image_filename(recipe_checksum)
|
|
packer_configuration = await self.load_configuration(dependencies_path, packer_tmp_directory)
|
|
packer_dst_os_filename = join(self.image_dir,
|
|
'os',
|
|
packer_configuration['os_name'] + '_' + packer_configuration['os_version'] + '.img',
|
|
)
|
|
packer_configuration['tmp_directory'] = packer_tmp_directory
|
|
recipe = {'variables': packer_configuration}
|
|
recipe['variables']['iso_url'] = packer_dst_os_filename
|
|
self.build(packer_dst_filename,
|
|
packer_tmp_directory,
|
|
recipe,
|
|
f'{packer_dst_os_filename}.sha256',
|
|
)
|
|
|
|
@staticmethod
|
|
def build(packer_dst_filename: str,
|
|
tmp_directory: str,
|
|
recipe: dict,
|
|
sha_file: str=None,
|
|
) -> None:
|
|
packer_filename = join(tmp_directory, PACKER_FILE_NAME)
|
|
if sha_file is not None:
|
|
with open(sha_file, 'r') as fh:
|
|
sha256 = fh.read().split(' ', 1)[0]
|
|
recipe['variables']['iso_checksum'] = sha256
|
|
with open(packer_filename, 'r') as recipe_fd:
|
|
for key, value in jload(recipe_fd).items():
|
|
recipe[key] = value
|
|
with open(packer_filename, 'w') as recipe_fd:
|
|
jdump(recipe, recipe_fd, indent=2)
|
|
preprocessors = join(tmp_directory, 'preprocessors')
|
|
if isfile(preprocessors):
|
|
proc = Popen([preprocessors],
|
|
#stdout=PIPE,
|
|
#stderr=PIPE,
|
|
cwd=tmp_directory,
|
|
)
|
|
proc.wait()
|
|
if proc.returncode:
|
|
raise Exception(_(f'error when executing {preprocessors}'))
|
|
proc = Popen(['packer', 'build', packer_filename],
|
|
#stdout=PIPE,
|
|
#stderr=PIPE,
|
|
cwd=tmp_directory,
|
|
)
|
|
proc.wait()
|
|
if proc.returncode:
|
|
raise Exception(_(f'cannot build {packer_dst_filename} with {packer_filename}'))
|
|
if not isdir(dirname(packer_dst_filename)):
|
|
makedirs(dirname(packer_dst_filename))
|
|
move(join(tmp_directory, 'image.img'), packer_dst_filename)
|
|
move(join(tmp_directory, 'image.sha256'), f'{packer_dst_filename}.sha256')
|
|
print(_(f'Image {packer_dst_filename} created'))
|
|
rmtree(tmp_directory)
|