Python wrapper with Docker backend with profiles

README.md

@@ -1,6 +1,10 @@
 # Tamarin
 
-Usine à paquets expérimentale basée sur rkt/acbuild.
+Usine à paquets GNU/Linux
+
+## Statut
+
+Expérimental
 
 ## Formats de paquets/distributions supportés
 
@@ -9,11 +13,7 @@ Usine à paquets expérimentale basée sur rkt/acbuild.
 ## Dépendances
 
 - [Python 3](https://www.python.org/downloads/)
-- Un noyau Linux > 2.6.24 (avec support des cgroups)
+- [Docker](>= 17.03)
-
-**Optionnel mais conseillé**
-
-- [systemd](https://freedesktop.org/wiki/Software/systemd/)
 
 ## Usage
 
@@ -30,10 +30,6 @@ TODO
 
 TODO
 
-### Répertoire de travail et mise en cache des images
-
-TODO
-
 ## Licence
 
 GPLv3

hooks/containerbuild/debian/install-build-essential

@@ -2,26 +2,6 @@
 
 set -e
 
-${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive
+echo 'ENV DEBIAN_FRONTEND=noninteractive' >> Dockerfile
-
+echo 'RUN apt-get update && apt-get install --yes --no-install-recommends build-essential devscripts equivs python3' >> Dockerfile
-if [ "${TAMARIN_ACBUILD_ENGINE}" == 'chroot' ]; then
+echo 'ENV DEBIAN_FRONTEND=' >> Dockerfile
-  # Ugly fix for Python installation in chrooted environment (require /dev/urandom)
-  head -c 65536 /dev/urandom > ./urandom
-  ${TAMARIN_ACBUILD} copy ./urandom /dev/urandom
-fi
-
-sudo -E /usr/bin/env bash - <<EOF
-export PATH=${PATH}
-${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- apt-get update
-${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- apt-get install --yes --no-install-recommends build-essential devscripts equivs python3
-EOF
-
-if [ "${TAMARIN_ACBUILD_ENGINE}" == 'chroot' ]; then
-  # Clean up Python fix (see above)
-  sudo -E /usr/bin/env bash - <<EOF
-  export PATH=${PATH}
-  ${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- rm -f /dev/urandom
-EOF
-fi
-
-${TAMARIN_ACBUILD} environment remove DEBIAN_FRONTEND

hooks/containerbuild/debian/install-git

@@ -2,10 +2,6 @@
 
 set -e
 
-${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive
+echo 'ENV DEBIAN_FRONTEND=noninteractive' >> Dockerfile
-sudo -E /usr/bin/env bash - <<EOF
+echo 'RUN apt-get update && apt-get install --yes --no-install-recommends git-core' >> Dockerfile
-export PATH=${PATH}
+echo 'ENV DEBIAN_FRONTEND=' >> Dockerfile
-${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- apt-get update
-${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- apt-get install --yes --no-install-recommends git-core
-EOF
-${TAMARIN_ACBUILD} environment remove DEBIAN_FRONTEND

lib/build.py

@@ -1,6 +1,6 @@
 import sys, os, argparse, tempfile
 sys.path.append(os.path.dirname(__file__) + '/lib')
-import tamarin, system, rkt
+import tamarin
 
 def get_args_parser():
     parser = argparse.ArgumentParser(description="Tamarin's container entrypoint")
@@ -12,7 +12,6 @@ def get_args_parser():
 def get_buildtools_dir():
   return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/buildtools")
 
-
 if __name__ == '__main__':
 
   parser = get_args_parser()

lib/rkt.py

@@ -1,38 +0,0 @@
-import system, subprocess, os, tamarin, json, re
-
-def run(args, as_root = False, capture_output=False, debug=False):
-    """Run rkt with the specified args (use the local copy if rkt is not found in the $PATH)"""
-    rkt_bin = system.which('rkt', tamarin.get_workspace_subdir('rkt'))
-    cmd = ( ["sudo", "-E", rkt_bin] if os.geteuid() != 0 and as_root == True else [rkt_bin] ) + args
-    if debug:
-        print(" ".join(cmd))
-    if capture_output:
-        return subprocess.check_output(cmd, stdin=subprocess.PIPE)
-    else:
-        return subprocess.check_call(cmd, stdin=subprocess.PIPE)
-
-def get_images_list(rkt_flags = [], debug=False):
-    output = run([
-        "image",
-        "list",
-        "--format=json"
-    ] + rkt_flags, capture_output=True, debug=debug)
-    # Fetch the list of installed images
-    return json.loads(output.decode('utf-8'))
-
-def find_image_by_name(name_pattern, rkt_flags = []):
-    if type(name_pattern) is str:
-        name_pattern = re.compile(name_pattern)
-    images_list = get_images_list(rkt_flags = rkt_flags)
-    for image in images_list:
-        if name_pattern.search(image['name']):
-            return image
-    return None
-
-def export_image(image_id, dest_file, rkt_flags = [], debug=False):
-    run([
-        "image",
-        "export",
-        image_id,
-        dest_file,
-    ] + rkt_flags, debug=debug)

lib/system.py

@@ -1,29 +0,0 @@
-import tarfile, os
-
-def extract_tar(file_path, dest_dir = ".", debug=False):
-    if debug:
-        print('Extracting "{:s}" to "{:s}"'.format(file_path, dest_dir))
-    with tarfile.open(file_path) as tar:
-        tar.extractall(dest_dir)
-        tar.close()
-
-def which(program, additional_paths = None):
-
-    def is_exe(fpath):
-        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
-
-    fpath, fname = os.path.split(program)
-    if fpath:
-        if is_exe(program):
-            return program
-    else:
-        paths = os.environ["PATH"].split(os.pathsep);
-        if additional_paths != None:
-            paths.append(additional_paths)
-        for path in paths:
-            path = path.strip('"')
-            exe_file = os.path.join(path, program)
-            if is_exe(exe_file):
-                return exe_file
-
-    return None

lib/tamarin.py

@@ -1,8 +1,6 @@
-import os, glob, subprocess, configparser
+import os, glob, subprocess, configparser, codecs, sys
-import web, system
-import codecs
 
-def run_profile_hooks(profile, step, cwd=None, env=None, debug=False):
+def run_profile_hooks(profile, step, **kwargs):
   hooks_dir = get_hooks_dir()
   step_hooks = profile[step]["hooks"]
   if not step_hooks:
@@ -12,7 +10,7 @@ def run_profile_hooks(profile, step, cwd=None, env=None, debug=False):
     if not trimmed_hook_name:
       continue
     hook_path = os.path.join(hooks_dir, trimmed_hook_name)
-    code = subprocess.check_call(hook_path, cwd=cwd, stdin=subprocess.PIPE, env=env)
+    run([hook_path], **kwargs)
 
 def get_hooks_dir():
   return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../hooks")
@@ -51,37 +49,17 @@ def get_workspace_subdir(subdir):
     os.makedirs(dir_path, exist_ok=True)
     return dir_path
 
-def get_acbuild_achive_dest_dir():
+def run(cmd, captureOutput=False, pty=False, debug=False, **kwargs):
-    """Return the first path matching the acbuild archive extraction destination in tamarin workspace"""
+    """Execute an arbitrary command on the system"""
-    workspace_tmp = get_workspace_subdir('tmp')
-    return glob.glob(os.path.join(os.sep, workspace_tmp, 'acbuild-v*'))[0]
-
-def get_rkt_achive_dest_dir():
-    """Return the first path matching the rkt archive extraction destination in tamarin workspace"""
-    workspace_tmp = get_workspace_subdir('tmp')
-    return glob.glob(os.path.join(os.sep, workspace_tmp, 'rkt-v*'))[0]
-
-def download_rkt(debug=False):
-    """Download a local copy of rkt in the tamarin workspace and return the absolute path to the archive"""
-    url = "https://github.com/coreos/rkt/releases/download/v1.25.0/rkt-v1.25.0.tar.gz"
-    file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "rkt.tar.gz")
-    web.download_file(file_url=url, dest_path=file_path)
-    return file_path
-
-def download_acbuild(debug=False):
-    """Download a local copy of acbuild in the tamarin workspace and return the absolute path to the archive"""
-    url = "https://github.com/containers/build/releases/download/v0.4.0/acbuild-v0.4.0.tar.gz"
-    file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "acbuild.tar.gz")
-    web.download_file(file_url=url, dest_path=file_path)
-    return file_path
-
-def run_acbuild(args, captureOutput=False, as_root=False, debug=False):
-    """Run acbuild with the specified args (use the local copy if acbuild is not found in the $PATH)"""
-    acbuild_bin = system.which('acbuild', get_workspace_subdir('acbuild'))
-    cmd = ( ["sudo", "-E", acbuild_bin] if os.geteuid() != 0 and as_root == True else [acbuild_bin] ) + args
     if debug:
         print(" ".join(cmd))
+    stdin=subprocess.PIPE
+    if pty:
+        kwargs['stdin'] = sys.stdin
     if captureOutput:
-        return subprocess.check_output(cmd, stdin=subprocess.PIPE)
+        return subprocess.check_output(cmd, **kwargs)
     else:
-        return subprocess.check_call(cmd, stdin=subprocess.PIPE)
+        return subprocess.check_call(cmd, **kwargs)
+
+def run_docker(args, captureOutput=False, **kwargs):
+    return run(["docker"] + args, captureOutput=captureOutput, **kwargs)

lib/web.py

@@ -1,33 +0,0 @@
-from urllib import request
-import math, sys
-
-def print_progress_bar(percent_progress=0, char_size=50, clear_line=True):
-
-    bar_progress = math.floor(char_size*(percent_progress/100))
-    bar = "=" * bar_progress + " " * (char_size - bar_progress)
-
-    if clear_line:
-        sys.stdout.write(u"\u001b[1000D")
-    sys.stdout.write("[{:s}] {:d}%".format(bar, int(percent_progress)))
-    sys.stdout.flush()
-
-def download_file(file_url, dest_path, bulk_size = 8192):
-
-    req = request.urlopen(file_url)
-    meta = req.info()
-    file_size = int(meta.get('Content-Length'))
-
-    with open(dest_path, 'wb') as dest_file:
-        print('Downloading "{:s}". Size: {:d}b'.format(file_url, file_size))
-        downloaded_size = 0
-        while True:
-            buff = req.read(bulk_size)
-            if not buff:
-                break
-            downloaded_size += len(buff)
-            dest_file.write(buff)
-            progress = downloaded_size/file_size*100
-            print_progress_bar(progress)
-        dest_file.close()
-    # Add linebreak
-    print("")

package

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 
-import argparse, sys, shutil, os, hashlib
+import argparse, sys, shutil, os, subprocess
 
 sys.path.append(os.path.dirname(__file__) + '/lib')
 
-import tamarin, system, rkt
+import tamarin
 
 def create_args_parser():
     '''Return a new configured ArgumentParser'''
@@ -20,80 +20,37 @@ def create_args_parser():
     parser.add_argument("-b", "--base", help="Use the specified image instead of the profile's one", default='')
     parser.add_argument("--rebuild", help="Ignore cache and rebuild container's image", action="store_true", default=False)
     parser.add_argument("--debug", help="Will add extra output and start the container in interactive mode", action="store_true", default=False)
+    parser.add_argument("--cleanup", help="Clear the workspace and remove obsolete Docker images before build", action="store_true", default=False)
 
     return parser
 
-def download_and_extract_rkt(dest_dir, debug=False):
+def build_image(build_workspace, base_image, profile_name, profile, debug=False, rebuild=False):
-    '''Download and extract rkt to the given destination directory'''
-    rkt_archive_path = tamarin.download_rkt(debug=debug)
-    system.extract_tar(rkt_archive_path, workspace_tmp, debug=debug)
-    rkt_archive_dir = tamarin.get_rkt_achive_dest_dir()
-    shutil.rmtree(local_rkt_dir, ignore_errors=True)
-    os.rename(rkt_archive_dir, dest_dir)
 
-def download_and_extract_acbuild(dest_dir, debug=False):
+    with open("{:s}/Dockerfile".format(build_workspace), 'w') as dockerfile:
-    '''Download and extract acbuild to the given destination directory'''
+        dockerfile.write("FROM {:s}\n".format(base_image))
-    acbuild_archive_path = tamarin.download_acbuild(debug=debug)
-    system.extract_tar(acbuild_archive_path, workspace_tmp, debug=debug)
-    acbuild_archive_dir = tamarin.get_acbuild_achive_dest_dir()
-    shutil.rmtree(local_acbuild_dir, ignore_errors=True)
-    os.rename(acbuild_archive_dir, dest_dir)
-
-def get_cached_image_path(profile, debug=False):
-    '''Compute and return the path for an hypothetic cached image for the given profile'''
-    containerbuild_hooks = profile['containerbuild']['hooks']
-    hasher = hashlib.sha1()
-    hasher.update(base_image.encode())
-    hasher.update(containerbuild_hooks.encode())
-    image_hash = hasher.hexdigest()
-    cache_dir = tamarin.get_workspace_subdir('cache')
-    return os.path.join(os.sep, cache_dir, '{:s}.aci'.format(image_hash[:12]));
-
-def build_image(build_workspace, aci_file, base_image, profile, debug=False):
-
-    acbuild_flags = ["--work-path", build_workspace]
-
-    # Find and export base image from rkt' store
-    name_pattern = base_image.split('/')[-1] + '$'
-    image = rkt.find_image_by_name(name_pattern, rkt_flags=rkt_flags)
-    rkt.export_image(image['id'], aci_file, rkt_flags=rkt_flags, debug=debug);
-
-    # Build image
-    tamarin.run_acbuild(acbuild_flags+["begin", aci_file], debug=debug)
-    tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)], debug=debug)
-    tamarin.run_acbuild(acbuild_flags+["mount", "add", "src", "/src", "--read-only"], debug=debug)
-    tamarin.run_acbuild(acbuild_flags+["mount", "add", "dist", "/dist"], debug=debug)
-    tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-hooks", "/tamarin/hooks", "--read-only"], debug=debug)
-    tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-lib", "/tamarin/lib", "--read-only"], debug=debug)
-    tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-profiles", "/tamarin/profiles", "--read-only"], debug=debug)
 
     # Configure "containerbuild" hooks environment
     hooks_env = os.environ.copy()
-    hooks_env["PATH"] = os.environ['PATH'] + ':' + tamarin.get_workspace_subdir('acbuild')
+    hooks_env["PATH"] = os.environ['PATH'] + ':' + tamarin.get_lib_dir()
-    hooks_env["TAMARIN_ACBUILD"] = " ".join([system.which('acbuild', local_acbuild_dir)]+acbuild_flags)
-    hooks_env["TAMARIN_ACBUILD_ENGINE"] = "chroot" if not system.which('systemctl') else "systemd-nspawn"
 
     # Run hooks
     tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=hooks_env, debug=debug)
 
-    tamarin.run_acbuild(acbuild_flags+["write", "--overwrite", aci_file], as_root=True, debug=debug)
+    image_tag = "tamarin:{:s}_{:s}_{:d}".format(profile_name, base_image.replace(':', '_'), os.getpid())
-    tamarin.run_acbuild(acbuild_flags+["end"], as_root=True, debug=debug)
 
-    return aci_file
+    build_args = [ "build", "-t", image_tag ]
 
-def cleanup(build_workspace, rkt_flags, debug=False):
+    if rebuild:
+        build_args += [ "--no-cache" ]
 
-    # Nettoyage des conteneurs
+    tamarin.run_docker(build_args + [build_workspace], debug=debug)
-    rkt.run([
-        "gc",
-        "--grace-period=0"
-    ] + rkt_flags, as_root=True, debug=debug)
 
-    # Nettoyage des images obsolètes du store
+    return image_tag
-    rkt.run([
+
-        "image",
+def cleanup(build_workspace=None, debug=False):
-        "gc"
+
-    ] + rkt_flags, as_root=True, debug=debug)
+    if build_workspace == None:
+        build_workspace = tamarin.get_workspace_subdir('tmp')
 
     # Suppression de l'espace de travail de build
     shutil.rmtree(build_workspace, ignore_errors=True)
@@ -108,6 +65,9 @@ if __name__ == "__main__":
 
     validate_args(args)
 
+    if args.cleanup:
+        cleanup(debug=args.debug)
+
     # Verify project directory
     project_dir = os.path.abspath(args.project_directory)
     output_dir = os.path.abspath(args.output)
@@ -116,69 +76,42 @@ if __name__ == "__main__":
     profile = tamarin.load_profile(args.profile, debug=args.debug)
 
     workspace = tamarin.get_workspace_dir()
-    workspace_tmp = tamarin.get_workspace_subdir('tmp')
-
-    local_rkt_dir = tamarin.get_workspace_subdir('rkt')
-    if not system.which('rkt', local_rkt_dir):
-        download_and_extract_rkt(local_rkt_dir)
-
-    local_acbuild_dir = tamarin.get_workspace_subdir('acbuild')
-    if not system.which('acbuild', local_acbuild_dir):
-        download_and_extract_acbuild(local_acbuild_dir)
 
     pid = os.getpid()
     build_workspace = tamarin.get_workspace_subdir('tmp/build_{:d}'.format(pid))
 
-    rkt_store = tamarin.get_workspace_subdir('store')
-    rkt_flags = ["--dir={:s}".format(rkt_store)]
-
     base_image = args.base if args.base != '' else profile['profile']['default_image']
 
-    # If the base image is Docker-based, download it
+    image_tag = build_image(build_workspace, base_image, args.profile, profile,  debug=args.debug, rebuild=args.rebuild)
-    if base_image.startswith('docker://'):
-        rkt.run([
-            "fetch",
-            "--insecure-options=image",
-            base_image
-        ] + rkt_flags, debug=args.debug)
-
-    aci_file = os.path.join(os.sep, build_workspace, 'image.aci')
-    cached_image_file = get_cached_image_path(profile, debug=args.debug)
-
-    if not args.rebuild and os.path.exists(cached_image_file):
-        # Copy cached image
-        shutil.copyfile(cached_image_file, aci_file)
-    else:
-        build_image(build_workspace, aci_file, base_image, profile, debug=args.debug)
-        # Cache image
-        shutil.copyfile(aci_file, cached_image_file)
 
     # rkt run arguments
-    rkt_args = [
+    docker_args = [
-      "run",
+        "run",
-      "--insecure-options=image",
+        "--rm",
-      aci_file, "--net=host",
+        "-v", "{:s}:/src:ro".format(project_dir),
-      "--volume=src,kind=host,source={:s}".format(project_dir),
+        "-v", "{:s}:/dist".format(output_dir),
-      "--volume=dist,kind=host,source={:s}".format(output_dir),
+        "-v", "{:s}:/tamarin/hooks:ro".format(tamarin.get_hooks_dir()),
-      "--volume=tamarin-hooks,kind=host,source={:s}".format(tamarin.get_hooks_dir()),
+        "-v", "{:s}:/tamarin/lib:ro".format(tamarin.get_lib_dir()),
-      "--volume=tamarin-lib,kind=host,source={:s}".format(tamarin.get_lib_dir()),
+        "-v", "{:s}:/tamarin/profiles:ro".format(tamarin.get_profiles_dir())
-      "--volume=tamarin-profiles,kind=host,source={:s}".format(tamarin.get_profiles_dir())
     ]
 
     # Use environment proxy if defined
     for proxy_var in ['HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy']:
-      if proxy_var in os.environ:
+        if proxy_var in os.environ:
-        rkt_args += ["--set-env={:s}={:s}".format(proxy_var, os.environ[proxy_var])]
+            docker_args += ["-e", "{:s}={:s}".format(proxy_var, os.environ[proxy_var])]
+
+    kwargs = dict()
+    kwargs['debug'] = args.debug
 
     if args.debug:
-      rkt_args += ["--interactive", "--exec", "/bin/bash"]
+        kwargs['pty'] = True
-      helper_cmd = " ".join(["/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture])
+        docker_args += ["-it", image_tag, "/bin/sh"]
-      print("Executer '{:s}' pour lancer la construction du paquet.".format(helper_cmd))
+        helper_cmd = " ".join(["/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture])
+        print("Executer '{:s}' pour lancer la construction du paquet.".format(helper_cmd))
     else:
-      rkt_args += ["--exec", "/usr/bin/python3", "--", "/tamarin/lib/build.py", args.profile, args.architecture]
+        docker_args += [image_tag, "/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture]
 
     # Start container
-    rkt.run(rkt_flags+rkt_args, as_root=True, debug=args.debug)
+    tamarin.run_docker(docker_args, **kwargs)
 
-    # Cleanup
+    cleanup(build_workspace, debug=args.debug)
-    cleanup(build_workspace, rkt_flags, debug=args.debug)

profiles/debian.conf

@@ -1,7 +1,7 @@
 # Configuration générale du profil
 [profile]
 # Image Docker par défaut
-default_image=docker://debian:jessie
+default_image=debian:jessie
 
 # Configuration de l'étape de pré-construction du conteneur
 [containerbuild]