diff --git a/README.md b/README.md index b823ac8..b27257c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # Tamarin -Usine à paquets expérimentale basée sur rkt/acbuild. +Usine à paquets GNU/Linux + +## Statut + +Expérimental ## Formats de paquets/distributions supportés @@ -9,11 +13,7 @@ Usine à paquets expérimentale basée sur rkt/acbuild. ## Dépendances - [Python 3](https://www.python.org/downloads/) -- Un noyau Linux > 2.6.24 (avec support des cgroups) - -**Optionnel mais conseillé** - -- [systemd](https://freedesktop.org/wiki/Software/systemd/) +- [Docker](>= 17.03) ## Usage @@ -30,10 +30,6 @@ TODO TODO -### Répertoire de travail et mise en cache des images - -TODO - ## Licence GPLv3 diff --git a/hooks/containerbuild/debian/install-build-essential b/hooks/containerbuild/debian/install-build-essential index 9dcc1de..1db0c28 100755 --- a/hooks/containerbuild/debian/install-build-essential +++ b/hooks/containerbuild/debian/install-build-essential @@ -2,26 +2,6 @@ set -e -${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive - -if [ "${TAMARIN_ACBUILD_ENGINE}" == 'chroot' ]; then - # Ugly fix for Python installation in chrooted environment (require /dev/urandom) - head -c 65536 /dev/urandom > ./urandom - ${TAMARIN_ACBUILD} copy ./urandom /dev/urandom -fi - -sudo -E /usr/bin/env bash - <> Dockerfile +echo 'RUN apt-get update && apt-get install --yes --no-install-recommends build-essential devscripts equivs python3' >> Dockerfile +echo 'ENV DEBIAN_FRONTEND=' >> Dockerfile diff --git a/hooks/containerbuild/debian/install-git b/hooks/containerbuild/debian/install-git index a2a2733..12a9d17 100755 --- a/hooks/containerbuild/debian/install-git +++ b/hooks/containerbuild/debian/install-git @@ -2,10 +2,6 @@ set -e -${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive -sudo -E /usr/bin/env bash - <> Dockerfile +echo 'RUN apt-get update && apt-get install --yes --no-install-recommends git-core' >> Dockerfile +echo 'ENV DEBIAN_FRONTEND=' >> Dockerfile diff --git a/lib/build.py b/lib/build.py index 1f2d7de..025012c 100644 --- a/lib/build.py +++ b/lib/build.py @@ -1,6 +1,6 @@ import sys, os, argparse, tempfile sys.path.append(os.path.dirname(__file__) + '/lib') -import tamarin, system, rkt +import tamarin def get_args_parser(): parser = argparse.ArgumentParser(description="Tamarin's container entrypoint") @@ -12,7 +12,6 @@ def get_args_parser(): def get_buildtools_dir(): return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/buildtools") - if __name__ == '__main__': parser = get_args_parser() diff --git a/lib/rkt.py b/lib/rkt.py deleted file mode 100644 index 4e8254b..0000000 --- a/lib/rkt.py +++ /dev/null @@ -1,38 +0,0 @@ -import system, subprocess, os, tamarin, json, re - -def run(args, as_root = False, capture_output=False, debug=False): - """Run rkt with the specified args (use the local copy if rkt is not found in the $PATH)""" - rkt_bin = system.which('rkt', tamarin.get_workspace_subdir('rkt')) - cmd = ( ["sudo", "-E", rkt_bin] if os.geteuid() != 0 and as_root == True else [rkt_bin] ) + args - if debug: - print(" ".join(cmd)) - if capture_output: - return subprocess.check_output(cmd, stdin=subprocess.PIPE) - else: - return subprocess.check_call(cmd, stdin=subprocess.PIPE) - -def get_images_list(rkt_flags = [], debug=False): - output = run([ - "image", - "list", - "--format=json" - ] + rkt_flags, capture_output=True, debug=debug) - # Fetch the list of installed images - return json.loads(output.decode('utf-8')) - -def find_image_by_name(name_pattern, rkt_flags = []): - if type(name_pattern) is str: - name_pattern = re.compile(name_pattern) - images_list = get_images_list(rkt_flags = rkt_flags) - for image in images_list: - if name_pattern.search(image['name']): - return image - return None - -def export_image(image_id, dest_file, rkt_flags = [], debug=False): - run([ - "image", - "export", - image_id, - dest_file, - ] + rkt_flags, debug=debug) diff --git a/lib/system.py b/lib/system.py deleted file mode 100644 index 2c0ab72..0000000 --- a/lib/system.py +++ /dev/null @@ -1,29 +0,0 @@ -import tarfile, os - -def extract_tar(file_path, dest_dir = ".", debug=False): - if debug: - print('Extracting "{:s}" to "{:s}"'.format(file_path, dest_dir)) - with tarfile.open(file_path) as tar: - tar.extractall(dest_dir) - tar.close() - -def which(program, additional_paths = None): - - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - fpath, fname = os.path.split(program) - if fpath: - if is_exe(program): - return program - else: - paths = os.environ["PATH"].split(os.pathsep); - if additional_paths != None: - paths.append(additional_paths) - for path in paths: - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - - return None diff --git a/lib/tamarin.py b/lib/tamarin.py index fcb27cd..de1bfb8 100644 --- a/lib/tamarin.py +++ b/lib/tamarin.py @@ -1,8 +1,6 @@ -import os, glob, subprocess, configparser -import web, system -import codecs +import os, glob, subprocess, configparser, codecs, sys -def run_profile_hooks(profile, step, cwd=None, env=None, debug=False): +def run_profile_hooks(profile, step, **kwargs): hooks_dir = get_hooks_dir() step_hooks = profile[step]["hooks"] if not step_hooks: @@ -12,7 +10,7 @@ def run_profile_hooks(profile, step, cwd=None, env=None, debug=False): if not trimmed_hook_name: continue hook_path = os.path.join(hooks_dir, trimmed_hook_name) - code = subprocess.check_call(hook_path, cwd=cwd, stdin=subprocess.PIPE, env=env) + run([hook_path], **kwargs) def get_hooks_dir(): return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../hooks") @@ -51,37 +49,17 @@ def get_workspace_subdir(subdir): os.makedirs(dir_path, exist_ok=True) return dir_path -def get_acbuild_achive_dest_dir(): - """Return the first path matching the acbuild archive extraction destination in tamarin workspace""" - workspace_tmp = get_workspace_subdir('tmp') - return glob.glob(os.path.join(os.sep, workspace_tmp, 'acbuild-v*'))[0] - -def get_rkt_achive_dest_dir(): - """Return the first path matching the rkt archive extraction destination in tamarin workspace""" - workspace_tmp = get_workspace_subdir('tmp') - return glob.glob(os.path.join(os.sep, workspace_tmp, 'rkt-v*'))[0] - -def download_rkt(debug=False): - """Download a local copy of rkt in the tamarin workspace and return the absolute path to the archive""" - url = "https://github.com/coreos/rkt/releases/download/v1.25.0/rkt-v1.25.0.tar.gz" - file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "rkt.tar.gz") - web.download_file(file_url=url, dest_path=file_path) - return file_path - -def download_acbuild(debug=False): - """Download a local copy of acbuild in the tamarin workspace and return the absolute path to the archive""" - url = "https://github.com/containers/build/releases/download/v0.4.0/acbuild-v0.4.0.tar.gz" - file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "acbuild.tar.gz") - web.download_file(file_url=url, dest_path=file_path) - return file_path - -def run_acbuild(args, captureOutput=False, as_root=False, debug=False): - """Run acbuild with the specified args (use the local copy if acbuild is not found in the $PATH)""" - acbuild_bin = system.which('acbuild', get_workspace_subdir('acbuild')) - cmd = ( ["sudo", "-E", acbuild_bin] if os.geteuid() != 0 and as_root == True else [acbuild_bin] ) + args +def run(cmd, captureOutput=False, pty=False, debug=False, **kwargs): + """Execute an arbitrary command on the system""" if debug: print(" ".join(cmd)) + stdin=subprocess.PIPE + if pty: + kwargs['stdin'] = sys.stdin if captureOutput: - return subprocess.check_output(cmd, stdin=subprocess.PIPE) + return subprocess.check_output(cmd, **kwargs) else: - return subprocess.check_call(cmd, stdin=subprocess.PIPE) + return subprocess.check_call(cmd, **kwargs) + +def run_docker(args, captureOutput=False, **kwargs): + return run(["docker"] + args, captureOutput=captureOutput, **kwargs) diff --git a/lib/web.py b/lib/web.py deleted file mode 100644 index d6d41a8..0000000 --- a/lib/web.py +++ /dev/null @@ -1,33 +0,0 @@ -from urllib import request -import math, sys - -def print_progress_bar(percent_progress=0, char_size=50, clear_line=True): - - bar_progress = math.floor(char_size*(percent_progress/100)) - bar = "=" * bar_progress + " " * (char_size - bar_progress) - - if clear_line: - sys.stdout.write(u"\u001b[1000D") - sys.stdout.write("[{:s}] {:d}%".format(bar, int(percent_progress))) - sys.stdout.flush() - -def download_file(file_url, dest_path, bulk_size = 8192): - - req = request.urlopen(file_url) - meta = req.info() - file_size = int(meta.get('Content-Length')) - - with open(dest_path, 'wb') as dest_file: - print('Downloading "{:s}". Size: {:d}b'.format(file_url, file_size)) - downloaded_size = 0 - while True: - buff = req.read(bulk_size) - if not buff: - break - downloaded_size += len(buff) - dest_file.write(buff) - progress = downloaded_size/file_size*100 - print_progress_bar(progress) - dest_file.close() - # Add linebreak - print("") diff --git a/package b/package index 742ff27..4d95fe7 100755 --- a/package +++ b/package @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -import argparse, sys, shutil, os, hashlib +import argparse, sys, shutil, os, subprocess sys.path.append(os.path.dirname(__file__) + '/lib') -import tamarin, system, rkt +import tamarin def create_args_parser(): '''Return a new configured ArgumentParser''' @@ -20,80 +20,37 @@ def create_args_parser(): parser.add_argument("-b", "--base", help="Use the specified image instead of the profile's one", default='') parser.add_argument("--rebuild", help="Ignore cache and rebuild container's image", action="store_true", default=False) parser.add_argument("--debug", help="Will add extra output and start the container in interactive mode", action="store_true", default=False) + parser.add_argument("--cleanup", help="Clear the workspace and remove obsolete Docker images before build", action="store_true", default=False) return parser -def download_and_extract_rkt(dest_dir, debug=False): - '''Download and extract rkt to the given destination directory''' - rkt_archive_path = tamarin.download_rkt(debug=debug) - system.extract_tar(rkt_archive_path, workspace_tmp, debug=debug) - rkt_archive_dir = tamarin.get_rkt_achive_dest_dir() - shutil.rmtree(local_rkt_dir, ignore_errors=True) - os.rename(rkt_archive_dir, dest_dir) +def build_image(build_workspace, base_image, profile_name, profile, debug=False, rebuild=False): -def download_and_extract_acbuild(dest_dir, debug=False): - '''Download and extract acbuild to the given destination directory''' - acbuild_archive_path = tamarin.download_acbuild(debug=debug) - system.extract_tar(acbuild_archive_path, workspace_tmp, debug=debug) - acbuild_archive_dir = tamarin.get_acbuild_achive_dest_dir() - shutil.rmtree(local_acbuild_dir, ignore_errors=True) - os.rename(acbuild_archive_dir, dest_dir) - -def get_cached_image_path(profile, debug=False): - '''Compute and return the path for an hypothetic cached image for the given profile''' - containerbuild_hooks = profile['containerbuild']['hooks'] - hasher = hashlib.sha1() - hasher.update(base_image.encode()) - hasher.update(containerbuild_hooks.encode()) - image_hash = hasher.hexdigest() - cache_dir = tamarin.get_workspace_subdir('cache') - return os.path.join(os.sep, cache_dir, '{:s}.aci'.format(image_hash[:12])); - -def build_image(build_workspace, aci_file, base_image, profile, debug=False): - - acbuild_flags = ["--work-path", build_workspace] - - # Find and export base image from rkt' store - name_pattern = base_image.split('/')[-1] + '$' - image = rkt.find_image_by_name(name_pattern, rkt_flags=rkt_flags) - rkt.export_image(image['id'], aci_file, rkt_flags=rkt_flags, debug=debug); - - # Build image - tamarin.run_acbuild(acbuild_flags+["begin", aci_file], debug=debug) - tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)], debug=debug) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "src", "/src", "--read-only"], debug=debug) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "dist", "/dist"], debug=debug) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-hooks", "/tamarin/hooks", "--read-only"], debug=debug) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-lib", "/tamarin/lib", "--read-only"], debug=debug) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-profiles", "/tamarin/profiles", "--read-only"], debug=debug) + with open("{:s}/Dockerfile".format(build_workspace), 'w') as dockerfile: + dockerfile.write("FROM {:s}\n".format(base_image)) # Configure "containerbuild" hooks environment hooks_env = os.environ.copy() - hooks_env["PATH"] = os.environ['PATH'] + ':' + tamarin.get_workspace_subdir('acbuild') - hooks_env["TAMARIN_ACBUILD"] = " ".join([system.which('acbuild', local_acbuild_dir)]+acbuild_flags) - hooks_env["TAMARIN_ACBUILD_ENGINE"] = "chroot" if not system.which('systemctl') else "systemd-nspawn" + hooks_env["PATH"] = os.environ['PATH'] + ':' + tamarin.get_lib_dir() # Run hooks tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=hooks_env, debug=debug) - tamarin.run_acbuild(acbuild_flags+["write", "--overwrite", aci_file], as_root=True, debug=debug) - tamarin.run_acbuild(acbuild_flags+["end"], as_root=True, debug=debug) + image_tag = "tamarin:{:s}_{:s}_{:d}".format(profile_name, base_image.replace(':', '_'), os.getpid()) - return aci_file + build_args = [ "build", "-t", image_tag ] -def cleanup(build_workspace, rkt_flags, debug=False): + if rebuild: + build_args += [ "--no-cache" ] - # Nettoyage des conteneurs - rkt.run([ - "gc", - "--grace-period=0" - ] + rkt_flags, as_root=True, debug=debug) + tamarin.run_docker(build_args + [build_workspace], debug=debug) - # Nettoyage des images obsolètes du store - rkt.run([ - "image", - "gc" - ] + rkt_flags, as_root=True, debug=debug) + return image_tag + +def cleanup(build_workspace=None, debug=False): + + if build_workspace == None: + build_workspace = tamarin.get_workspace_subdir('tmp') # Suppression de l'espace de travail de build shutil.rmtree(build_workspace, ignore_errors=True) @@ -108,6 +65,9 @@ if __name__ == "__main__": validate_args(args) + if args.cleanup: + cleanup(debug=args.debug) + # Verify project directory project_dir = os.path.abspath(args.project_directory) output_dir = os.path.abspath(args.output) @@ -116,69 +76,42 @@ if __name__ == "__main__": profile = tamarin.load_profile(args.profile, debug=args.debug) workspace = tamarin.get_workspace_dir() - workspace_tmp = tamarin.get_workspace_subdir('tmp') - - local_rkt_dir = tamarin.get_workspace_subdir('rkt') - if not system.which('rkt', local_rkt_dir): - download_and_extract_rkt(local_rkt_dir) - - local_acbuild_dir = tamarin.get_workspace_subdir('acbuild') - if not system.which('acbuild', local_acbuild_dir): - download_and_extract_acbuild(local_acbuild_dir) pid = os.getpid() build_workspace = tamarin.get_workspace_subdir('tmp/build_{:d}'.format(pid)) - rkt_store = tamarin.get_workspace_subdir('store') - rkt_flags = ["--dir={:s}".format(rkt_store)] - base_image = args.base if args.base != '' else profile['profile']['default_image'] - # If the base image is Docker-based, download it - if base_image.startswith('docker://'): - rkt.run([ - "fetch", - "--insecure-options=image", - base_image - ] + rkt_flags, debug=args.debug) - - aci_file = os.path.join(os.sep, build_workspace, 'image.aci') - cached_image_file = get_cached_image_path(profile, debug=args.debug) - - if not args.rebuild and os.path.exists(cached_image_file): - # Copy cached image - shutil.copyfile(cached_image_file, aci_file) - else: - build_image(build_workspace, aci_file, base_image, profile, debug=args.debug) - # Cache image - shutil.copyfile(aci_file, cached_image_file) + image_tag = build_image(build_workspace, base_image, args.profile, profile, debug=args.debug, rebuild=args.rebuild) # rkt run arguments - rkt_args = [ - "run", - "--insecure-options=image", - aci_file, "--net=host", - "--volume=src,kind=host,source={:s}".format(project_dir), - "--volume=dist,kind=host,source={:s}".format(output_dir), - "--volume=tamarin-hooks,kind=host,source={:s}".format(tamarin.get_hooks_dir()), - "--volume=tamarin-lib,kind=host,source={:s}".format(tamarin.get_lib_dir()), - "--volume=tamarin-profiles,kind=host,source={:s}".format(tamarin.get_profiles_dir()) + docker_args = [ + "run", + "--rm", + "-v", "{:s}:/src:ro".format(project_dir), + "-v", "{:s}:/dist".format(output_dir), + "-v", "{:s}:/tamarin/hooks:ro".format(tamarin.get_hooks_dir()), + "-v", "{:s}:/tamarin/lib:ro".format(tamarin.get_lib_dir()), + "-v", "{:s}:/tamarin/profiles:ro".format(tamarin.get_profiles_dir()) ] # Use environment proxy if defined for proxy_var in ['HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy']: - if proxy_var in os.environ: - rkt_args += ["--set-env={:s}={:s}".format(proxy_var, os.environ[proxy_var])] + if proxy_var in os.environ: + docker_args += ["-e", "{:s}={:s}".format(proxy_var, os.environ[proxy_var])] + + kwargs = dict() + kwargs['debug'] = args.debug if args.debug: - rkt_args += ["--interactive", "--exec", "/bin/bash"] - helper_cmd = " ".join(["/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture]) - print("Executer '{:s}' pour lancer la construction du paquet.".format(helper_cmd)) + kwargs['pty'] = True + docker_args += ["-it", image_tag, "/bin/sh"] + helper_cmd = " ".join(["/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture]) + print("Executer '{:s}' pour lancer la construction du paquet.".format(helper_cmd)) else: - rkt_args += ["--exec", "/usr/bin/python3", "--", "/tamarin/lib/build.py", args.profile, args.architecture] + docker_args += [image_tag, "/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture] # Start container - rkt.run(rkt_flags+rkt_args, as_root=True, debug=args.debug) + tamarin.run_docker(docker_args, **kwargs) - # Cleanup - cleanup(build_workspace, rkt_flags, debug=args.debug) + cleanup(build_workspace, debug=args.debug) diff --git a/profiles/debian.conf b/profiles/debian.conf index f9afe98..8fb00a0 100644 --- a/profiles/debian.conf +++ b/profiles/debian.conf @@ -1,7 +1,7 @@ # Configuration générale du profil [profile] # Image Docker par défaut -default_image=docker://debian:jessie +default_image=debian:jessie # Configuration de l'étape de pré-construction du conteneur [containerbuild]