Cadoles
/
Tamarin
21
0
Fork 0
Code Issues 1 Pull Requests 1 Releases Wiki Activity

Python wrapper with Docker backend with profiles

This commit is contained in:
wpetit 2017-03-21 23:05:16 +01:00
parent d99305182e
commit 9dc46f5665
10 changed files with 69 additions and 287 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@ -1,6 +1,10 @@
# Tamarin
Usine à paquets expérimentale basée sur rkt/acbuild.
Usine à paquets GNU/Linux
## Statut
Expérimental
## Formats de paquets/distributions supportés
@ -9,11 +13,7 @@ Usine à paquets expérimentale basée sur rkt/acbuild.
## Dépendances
- [Python 3](https://www.python.org/downloads/)
- Un noyau Linux > 2.6.24 (avec support des cgroups)
**Optionnel mais conseillé**
- [systemd](https://freedesktop.org/wiki/Software/systemd/)
- [Docker](>= 17.03)
## Usage
@ -30,10 +30,6 @@ TODO
TODO
### Répertoire de travail et mise en cache des images
TODO
## Licence
GPLv3

26
hooks/containerbuild/debian/install-build-essential
View File

@ -2,26 +2,6 @@
set -e
${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive
if [ "${TAMARIN_ACBUILD_ENGINE}" == 'chroot' ]; then
# Ugly fix for Python installation in chrooted environment (require /dev/urandom)
head -c 65536 /dev/urandom > ./urandom
${TAMARIN_ACBUILD} copy ./urandom /dev/urandom
fi
sudo -E /usr/bin/env bash - <<EOF
export PATH=${PATH}
${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- apt-get update
${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- apt-get install --yes --no-install-recommends build-essential devscripts equivs python3
EOF
if [ "${TAMARIN_ACBUILD_ENGINE}" == 'chroot' ]; then
# Clean up Python fix (see above)
sudo -E /usr/bin/env bash - <<EOF
export PATH=${PATH}
${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- rm -f /dev/urandom
EOF
fi
${TAMARIN_ACBUILD} environment remove DEBIAN_FRONTEND
echo 'ENV DEBIAN_FRONTEND=noninteractive' >> Dockerfile
echo 'RUN apt-get update && apt-get install --yes --no-install-recommends build-essential devscripts equivs python3' >> Dockerfile
echo 'ENV DEBIAN_FRONTEND=' >> Dockerfile

10
hooks/containerbuild/debian/install-git
View File

@ -2,10 +2,6 @@
set -e
${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive
sudo -E /usr/bin/env bash - <<EOF
export PATH=${PATH}
${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- apt-get update
${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- apt-get install --yes --no-install-recommends git-core
EOF
${TAMARIN_ACBUILD} environment remove DEBIAN_FRONTEND
echo 'ENV DEBIAN_FRONTEND=noninteractive' >> Dockerfile
echo 'RUN apt-get update && apt-get install --yes --no-install-recommends git-core' >> Dockerfile
echo 'ENV DEBIAN_FRONTEND=' >> Dockerfile

3
lib/build.py
View File

@ -1,6 +1,6 @@
import sys, os, argparse, tempfile
sys.path.append(os.path.dirname(__file__) + '/lib')
import tamarin, system, rkt
import tamarin
def get_args_parser():
parser = argparse.ArgumentParser(description="Tamarin's container entrypoint")
@ -12,7 +12,6 @@ def get_args_parser():
def get_buildtools_dir():
return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/buildtools")
if __name__ == '__main__':
parser = get_args_parser()

38
lib/rkt.py
View File

@ -1,38 +0,0 @@
import system, subprocess, os, tamarin, json, re
def run(args, as_root = False, capture_output=False, debug=False):
"""Run rkt with the specified args (use the local copy if rkt is not found in the $PATH)"""
rkt_bin = system.which('rkt', tamarin.get_workspace_subdir('rkt'))
cmd = ( ["sudo", "-E", rkt_bin] if os.geteuid() != 0 and as_root == True else [rkt_bin] ) + args
if debug:
print(" ".join(cmd))
if capture_output:
return subprocess.check_output(cmd, stdin=subprocess.PIPE)
else:
return subprocess.check_call(cmd, stdin=subprocess.PIPE)
def get_images_list(rkt_flags = [], debug=False):
output = run([
"image",
"list",
"--format=json"
] + rkt_flags, capture_output=True, debug=debug)
# Fetch the list of installed images
return json.loads(output.decode('utf-8'))
def find_image_by_name(name_pattern, rkt_flags = []):
if type(name_pattern) is str:
name_pattern = re.compile(name_pattern)
images_list = get_images_list(rkt_flags = rkt_flags)
for image in images_list:
if name_pattern.search(image['name']):
return image
return None
def export_image(image_id, dest_file, rkt_flags = [], debug=False):
run([
"image",
"export",
image_id,
dest_file,
] + rkt_flags, debug=debug)

29
lib/system.py
View File

@ -1,29 +0,0 @@
import tarfile, os
def extract_tar(file_path, dest_dir = ".", debug=False):
if debug:
print('Extracting "{:s}" to "{:s}"'.format(file_path, dest_dir))
with tarfile.open(file_path) as tar:
tar.extractall(dest_dir)
tar.close()
def which(program, additional_paths = None):
def is_exe(fpath):
return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
fpath, fname = os.path.split(program)
if fpath:
if is_exe(program):
return program
else:
paths = os.environ["PATH"].split(os.pathsep);
if additional_paths != None:
paths.append(additional_paths)
for path in paths:
path = path.strip('"')
exe_file = os.path.join(path, program)
if is_exe(exe_file):
return exe_file
return None

48
lib/tamarin.py
View File

@ -1,8 +1,6 @@
import os, glob, subprocess, configparser
import web, system
import codecs
import os, glob, subprocess, configparser, codecs, sys
def run_profile_hooks(profile, step, cwd=None, env=None, debug=False):
def run_profile_hooks(profile, step, **kwargs):
hooks_dir = get_hooks_dir()
step_hooks = profile[step]["hooks"]
if not step_hooks:
@ -12,7 +10,7 @@ def run_profile_hooks(profile, step, cwd=None, env=None, debug=False):
if not trimmed_hook_name:
continue
hook_path = os.path.join(hooks_dir, trimmed_hook_name)
code = subprocess.check_call(hook_path, cwd=cwd, stdin=subprocess.PIPE, env=env)
run([hook_path], **kwargs)
def get_hooks_dir():
return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../hooks")
@ -51,37 +49,17 @@ def get_workspace_subdir(subdir):
os.makedirs(dir_path, exist_ok=True)
return dir_path
def get_acbuild_achive_dest_dir():
"""Return the first path matching the acbuild archive extraction destination in tamarin workspace"""
workspace_tmp = get_workspace_subdir('tmp')
return glob.glob(os.path.join(os.sep, workspace_tmp, 'acbuild-v*'))[0]
def get_rkt_achive_dest_dir():
"""Return the first path matching the rkt archive extraction destination in tamarin workspace"""
workspace_tmp = get_workspace_subdir('tmp')
return glob.glob(os.path.join(os.sep, workspace_tmp, 'rkt-v*'))[0]
def download_rkt(debug=False):
"""Download a local copy of rkt in the tamarin workspace and return the absolute path to the archive"""
url = "https://github.com/coreos/rkt/releases/download/v1.25.0/rkt-v1.25.0.tar.gz"
file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "rkt.tar.gz")
web.download_file(file_url=url, dest_path=file_path)
return file_path
def download_acbuild(debug=False):
"""Download a local copy of acbuild in the tamarin workspace and return the absolute path to the archive"""
url = "https://github.com/containers/build/releases/download/v0.4.0/acbuild-v0.4.0.tar.gz"
file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "acbuild.tar.gz")
web.download_file(file_url=url, dest_path=file_path)
return file_path
def run_acbuild(args, captureOutput=False, as_root=False, debug=False):
"""Run acbuild with the specified args (use the local copy if acbuild is not found in the $PATH)"""
acbuild_bin = system.which('acbuild', get_workspace_subdir('acbuild'))
cmd = ( ["sudo", "-E", acbuild_bin] if os.geteuid() != 0 and as_root == True else [acbuild_bin] ) + args
def run(cmd, captureOutput=False, pty=False, debug=False, **kwargs):
"""Execute an arbitrary command on the system"""
if debug:
print(" ".join(cmd))
stdin=subprocess.PIPE
if pty:
kwargs['stdin'] = sys.stdin
if captureOutput:
return subprocess.check_output(cmd, stdin=subprocess.PIPE)
return subprocess.check_output(cmd, **kwargs)
else:
return subprocess.check_call(cmd, stdin=subprocess.PIPE)
return subprocess.check_call(cmd, **kwargs)
def run_docker(args, captureOutput=False, **kwargs):
return run(["docker"] + args, captureOutput=captureOutput, **kwargs)

33
lib/web.py
View File

@ -1,33 +0,0 @@
from urllib import request
import math, sys
def print_progress_bar(percent_progress=0, char_size=50, clear_line=True):
bar_progress = math.floor(char_size*(percent_progress/100))
bar = "=" * bar_progress + " " * (char_size - bar_progress)
if clear_line:
sys.stdout.write(u"\u001b[1000D")
sys.stdout.write("[{:s}] {:d}%".format(bar, int(percent_progress)))
sys.stdout.flush()
def download_file(file_url, dest_path, bulk_size = 8192):
req = request.urlopen(file_url)
meta = req.info()
file_size = int(meta.get('Content-Length'))
with open(dest_path, 'wb') as dest_file:
print('Downloading "{:s}". Size: {:d}b'.format(file_url, file_size))
downloaded_size = 0
while True:
buff = req.read(bulk_size)
if not buff:
break
downloaded_size += len(buff)
dest_file.write(buff)
progress = downloaded_size/file_size*100
print_progress_bar(progress)
dest_file.close()
# Add linebreak
print("")

151
package
View File

@ -1,10 +1,10 @@
#!/usr/bin/env python3
import argparse, sys, shutil, os, hashlib
import argparse, sys, shutil, os, subprocess
sys.path.append(os.path.dirname(__file__) + '/lib')
import tamarin, system, rkt
import tamarin
def create_args_parser():
'''Return a new configured ArgumentParser'''
@ -20,80 +20,37 @@ def create_args_parser():
parser.add_argument("-b", "--base", help="Use the specified image instead of the profile's one", default='')
parser.add_argument("--rebuild", help="Ignore cache and rebuild container's image", action="store_true", default=False)
parser.add_argument("--debug", help="Will add extra output and start the container in interactive mode", action="store_true", default=False)
parser.add_argument("--cleanup", help="Clear the workspace and remove obsolete Docker images before build", action="store_true", default=False)
return parser
def download_and_extract_rkt(dest_dir, debug=False):
'''Download and extract rkt to the given destination directory'''
rkt_archive_path = tamarin.download_rkt(debug=debug)
system.extract_tar(rkt_archive_path, workspace_tmp, debug=debug)
rkt_archive_dir = tamarin.get_rkt_achive_dest_dir()
shutil.rmtree(local_rkt_dir, ignore_errors=True)
os.rename(rkt_archive_dir, dest_dir)
def build_image(build_workspace, base_image, profile_name, profile, debug=False, rebuild=False):
def download_and_extract_acbuild(dest_dir, debug=False):
'''Download and extract acbuild to the given destination directory'''
acbuild_archive_path = tamarin.download_acbuild(debug=debug)
system.extract_tar(acbuild_archive_path, workspace_tmp, debug=debug)
acbuild_archive_dir = tamarin.get_acbuild_achive_dest_dir()
shutil.rmtree(local_acbuild_dir, ignore_errors=True)
os.rename(acbuild_archive_dir, dest_dir)
def get_cached_image_path(profile, debug=False):
'''Compute and return the path for an hypothetic cached image for the given profile'''
containerbuild_hooks = profile['containerbuild']['hooks']
hasher = hashlib.sha1()
hasher.update(base_image.encode())
hasher.update(containerbuild_hooks.encode())
image_hash = hasher.hexdigest()
cache_dir = tamarin.get_workspace_subdir('cache')
return os.path.join(os.sep, cache_dir, '{:s}.aci'.format(image_hash[:12]));
def build_image(build_workspace, aci_file, base_image, profile, debug=False):
acbuild_flags = ["--work-path", build_workspace]
# Find and export base image from rkt' store
name_pattern = base_image.split('/')[-1] + '$'
image = rkt.find_image_by_name(name_pattern, rkt_flags=rkt_flags)
rkt.export_image(image['id'], aci_file, rkt_flags=rkt_flags, debug=debug);
# Build image
tamarin.run_acbuild(acbuild_flags+["begin", aci_file], debug=debug)
tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)], debug=debug)
tamarin.run_acbuild(acbuild_flags+["mount", "add", "src", "/src", "--read-only"], debug=debug)
tamarin.run_acbuild(acbuild_flags+["mount", "add", "dist", "/dist"], debug=debug)
tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-hooks", "/tamarin/hooks", "--read-only"], debug=debug)
tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-lib", "/tamarin/lib", "--read-only"], debug=debug)
tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-profiles", "/tamarin/profiles", "--read-only"], debug=debug)
with open("{:s}/Dockerfile".format(build_workspace), 'w') as dockerfile:
dockerfile.write("FROM {:s}\n".format(base_image))
# Configure "containerbuild" hooks environment
hooks_env = os.environ.copy()
hooks_env["PATH"] = os.environ['PATH'] + ':' + tamarin.get_workspace_subdir('acbuild')
hooks_env["TAMARIN_ACBUILD"] = " ".join([system.which('acbuild', local_acbuild_dir)]+acbuild_flags)
hooks_env["TAMARIN_ACBUILD_ENGINE"] = "chroot" if not system.which('systemctl') else "systemd-nspawn"
hooks_env["PATH"] = os.environ['PATH'] + ':' + tamarin.get_lib_dir()
# Run hooks
tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=hooks_env, debug=debug)
tamarin.run_acbuild(acbuild_flags+["write", "--overwrite", aci_file], as_root=True, debug=debug)
tamarin.run_acbuild(acbuild_flags+["end"], as_root=True, debug=debug)
image_tag = "tamarin:{:s}_{:s}_{:d}".format(profile_name, base_image.replace(':', '_'), os.getpid())
return aci_file
build_args = [ "build", "-t", image_tag ]
def cleanup(build_workspace, rkt_flags, debug=False):
if rebuild:
build_args += [ "--no-cache" ]
# Nettoyage des conteneurs
rkt.run([
"gc",
"--grace-period=0"
] + rkt_flags, as_root=True, debug=debug)
tamarin.run_docker(build_args + [build_workspace], debug=debug)
# Nettoyage des images obsolètes du store
rkt.run([
"image",
"gc"
] + rkt_flags, as_root=True, debug=debug)
return image_tag
def cleanup(build_workspace=None, debug=False):
if build_workspace == None:
build_workspace = tamarin.get_workspace_subdir('tmp')
# Suppression de l'espace de travail de build
shutil.rmtree(build_workspace, ignore_errors=True)
@ -108,6 +65,9 @@ if __name__ == "__main__":
validate_args(args)
if args.cleanup:
cleanup(debug=args.debug)
# Verify project directory
project_dir = os.path.abspath(args.project_directory)
output_dir = os.path.abspath(args.output)
@ -116,69 +76,42 @@ if __name__ == "__main__":
profile = tamarin.load_profile(args.profile, debug=args.debug)
workspace = tamarin.get_workspace_dir()
workspace_tmp = tamarin.get_workspace_subdir('tmp')
local_rkt_dir = tamarin.get_workspace_subdir('rkt')
if not system.which('rkt', local_rkt_dir):
download_and_extract_rkt(local_rkt_dir)
local_acbuild_dir = tamarin.get_workspace_subdir('acbuild')
if not system.which('acbuild', local_acbuild_dir):
download_and_extract_acbuild(local_acbuild_dir)
pid = os.getpid()
build_workspace = tamarin.get_workspace_subdir('tmp/build_{:d}'.format(pid))
rkt_store = tamarin.get_workspace_subdir('store')
rkt_flags = ["--dir={:s}".format(rkt_store)]
base_image = args.base if args.base != '' else profile['profile']['default_image']
# If the base image is Docker-based, download it
if base_image.startswith('docker://'):
rkt.run([
"fetch",
"--insecure-options=image",
base_image
] + rkt_flags, debug=args.debug)
aci_file = os.path.join(os.sep, build_workspace, 'image.aci')
cached_image_file = get_cached_image_path(profile, debug=args.debug)
if not args.rebuild and os.path.exists(cached_image_file):
# Copy cached image
shutil.copyfile(cached_image_file, aci_file)
else:
build_image(build_workspace, aci_file, base_image, profile, debug=args.debug)
# Cache image
shutil.copyfile(aci_file, cached_image_file)
image_tag = build_image(build_workspace, base_image, args.profile, profile, debug=args.debug, rebuild=args.rebuild)
# rkt run arguments
rkt_args = [
"run",
"--insecure-options=image",
aci_file, "--net=host",
"--volume=src,kind=host,source={:s}".format(project_dir),
"--volume=dist,kind=host,source={:s}".format(output_dir),
"--volume=tamarin-hooks,kind=host,source={:s}".format(tamarin.get_hooks_dir()),
"--volume=tamarin-lib,kind=host,source={:s}".format(tamarin.get_lib_dir()),
"--volume=tamarin-profiles,kind=host,source={:s}".format(tamarin.get_profiles_dir())
docker_args = [
"run",
"--rm",
"-v", "{:s}:/src:ro".format(project_dir),
"-v", "{:s}:/dist".format(output_dir),
"-v", "{:s}:/tamarin/hooks:ro".format(tamarin.get_hooks_dir()),
"-v", "{:s}:/tamarin/lib:ro".format(tamarin.get_lib_dir()),
"-v", "{:s}:/tamarin/profiles:ro".format(tamarin.get_profiles_dir())
]
# Use environment proxy if defined
for proxy_var in ['HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy']:
if proxy_var in os.environ:
rkt_args += ["--set-env={:s}={:s}".format(proxy_var, os.environ[proxy_var])]
if proxy_var in os.environ:
docker_args += ["-e", "{:s}={:s}".format(proxy_var, os.environ[proxy_var])]
kwargs = dict()
kwargs['debug'] = args.debug
if args.debug:
rkt_args += ["--interactive", "--exec", "/bin/bash"]
helper_cmd = " ".join(["/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture])
print("Executer '{:s}' pour lancer la construction du paquet.".format(helper_cmd))
kwargs['pty'] = True
docker_args += ["-it", image_tag, "/bin/sh"]
helper_cmd = " ".join(["/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture])
print("Executer '{:s}' pour lancer la construction du paquet.".format(helper_cmd))
else:
rkt_args += ["--exec", "/usr/bin/python3", "--", "/tamarin/lib/build.py", args.profile, args.architecture]
docker_args += [image_tag, "/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture]
# Start container
rkt.run(rkt_flags+rkt_args, as_root=True, debug=args.debug)
tamarin.run_docker(docker_args, **kwargs)
# Cleanup
cleanup(build_workspace, rkt_flags, debug=args.debug)
cleanup(build_workspace, debug=args.debug)

2
profiles/debian.conf
View File

@ -1,7 +1,7 @@
# Configuration générale du profil
[profile]
# Image Docker par défaut
default_image=docker://debian:jessie
default_image=debian:jessie
# Configuration de l'étape de pré-construction du conteneur
[containerbuild]