@@ -1,6 +1,10 @@ | |||
# Tamarin | |||
Usine à paquets expérimentale basée sur rkt/acbuild. | |||
Usine à paquets GNU/Linux | |||
## Statut | |||
Expérimental | |||
## Formats de paquets/distributions supportés | |||
@@ -9,11 +13,7 @@ Usine à paquets expérimentale basée sur rkt/acbuild. | |||
## Dépendances | |||
- [Python 3](https://www.python.org/downloads/) | |||
- Un noyau Linux > 2.6.24 (avec support des cgroups) | |||
**Optionnel mais conseillé** | |||
- [systemd](https://freedesktop.org/wiki/Software/systemd/) | |||
- [Docker](>= 17.03) | |||
## Usage | |||
@@ -30,10 +30,6 @@ TODO | |||
TODO | |||
### Répertoire de travail et mise en cache des images | |||
TODO | |||
## Licence | |||
GPLv3 |
@@ -2,26 +2,6 @@ | |||
set -e | |||
${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive | |||
if [ "${TAMARIN_ACBUILD_ENGINE}" == 'chroot' ]; then | |||
# Ugly fix for Python installation in chrooted environment (require /dev/urandom) | |||
head -c 65536 /dev/urandom > ./urandom | |||
${TAMARIN_ACBUILD} copy ./urandom /dev/urandom | |||
fi | |||
sudo -E /usr/bin/env bash - <<EOF | |||
export PATH=${PATH} | |||
${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- apt-get update | |||
${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- apt-get install --yes --no-install-recommends build-essential devscripts equivs python3 | |||
EOF | |||
if [ "${TAMARIN_ACBUILD_ENGINE}" == 'chroot' ]; then | |||
# Clean up Python fix (see above) | |||
sudo -E /usr/bin/env bash - <<EOF | |||
export PATH=${PATH} | |||
${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- rm -f /dev/urandom | |||
EOF | |||
fi | |||
${TAMARIN_ACBUILD} environment remove DEBIAN_FRONTEND | |||
echo 'ENV DEBIAN_FRONTEND=noninteractive' >> Dockerfile | |||
echo 'RUN apt-get update && apt-get install --yes --no-install-recommends build-essential devscripts equivs python3' >> Dockerfile | |||
echo 'ENV DEBIAN_FRONTEND=' >> Dockerfile |
@@ -2,10 +2,6 @@ | |||
set -e | |||
${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive | |||
sudo -E /usr/bin/env bash - <<EOF | |||
export PATH=${PATH} | |||
${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- apt-get update | |||
${TAMARIN_ACBUILD} run --engine "${TAMARIN_ACBUILD_ENGINE}" -- apt-get install --yes --no-install-recommends git-core | |||
EOF | |||
${TAMARIN_ACBUILD} environment remove DEBIAN_FRONTEND | |||
echo 'ENV DEBIAN_FRONTEND=noninteractive' >> Dockerfile | |||
echo 'RUN apt-get update && apt-get install --yes --no-install-recommends git-core' >> Dockerfile | |||
echo 'ENV DEBIAN_FRONTEND=' >> Dockerfile |
@@ -1,6 +1,6 @@ | |||
import sys, os, argparse, tempfile | |||
sys.path.append(os.path.dirname(__file__) + '/lib') | |||
import tamarin, system, rkt | |||
import tamarin | |||
def get_args_parser(): | |||
parser = argparse.ArgumentParser(description="Tamarin's container entrypoint") | |||
@@ -12,7 +12,6 @@ def get_args_parser(): | |||
def get_buildtools_dir(): | |||
return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/buildtools") | |||
if __name__ == '__main__': | |||
parser = get_args_parser() |
@@ -1,38 +0,0 @@ | |||
import system, subprocess, os, tamarin, json, re | |||
def run(args, as_root = False, capture_output=False, debug=False): | |||
"""Run rkt with the specified args (use the local copy if rkt is not found in the $PATH)""" | |||
rkt_bin = system.which('rkt', tamarin.get_workspace_subdir('rkt')) | |||
cmd = ( ["sudo", "-E", rkt_bin] if os.geteuid() != 0 and as_root == True else [rkt_bin] ) + args | |||
if debug: | |||
print(" ".join(cmd)) | |||
if capture_output: | |||
return subprocess.check_output(cmd, stdin=subprocess.PIPE) | |||
else: | |||
return subprocess.check_call(cmd, stdin=subprocess.PIPE) | |||
def get_images_list(rkt_flags = [], debug=False): | |||
output = run([ | |||
"image", | |||
"list", | |||
"--format=json" | |||
] + rkt_flags, capture_output=True, debug=debug) | |||
# Fetch the list of installed images | |||
return json.loads(output.decode('utf-8')) | |||
def find_image_by_name(name_pattern, rkt_flags = []): | |||
if type(name_pattern) is str: | |||
name_pattern = re.compile(name_pattern) | |||
images_list = get_images_list(rkt_flags = rkt_flags) | |||
for image in images_list: | |||
if name_pattern.search(image['name']): | |||
return image | |||
return None | |||
def export_image(image_id, dest_file, rkt_flags = [], debug=False): | |||
run([ | |||
"image", | |||
"export", | |||
image_id, | |||
dest_file, | |||
] + rkt_flags, debug=debug) |
@@ -1,29 +0,0 @@ | |||
import tarfile, os | |||
def extract_tar(file_path, dest_dir = ".", debug=False): | |||
if debug: | |||
print('Extracting "{:s}" to "{:s}"'.format(file_path, dest_dir)) | |||
with tarfile.open(file_path) as tar: | |||
tar.extractall(dest_dir) | |||
tar.close() | |||
def which(program, additional_paths = None): | |||
def is_exe(fpath): | |||
return os.path.isfile(fpath) and os.access(fpath, os.X_OK) | |||
fpath, fname = os.path.split(program) | |||
if fpath: | |||
if is_exe(program): | |||
return program | |||
else: | |||
paths = os.environ["PATH"].split(os.pathsep); | |||
if additional_paths != None: | |||
paths.append(additional_paths) | |||
for path in paths: | |||
path = path.strip('"') | |||
exe_file = os.path.join(path, program) | |||
if is_exe(exe_file): | |||
return exe_file | |||
return None |
@@ -1,8 +1,6 @@ | |||
import os, glob, subprocess, configparser | |||
import web, system | |||
import codecs | |||
import os, glob, subprocess, configparser, codecs, sys | |||
def run_profile_hooks(profile, step, cwd=None, env=None, debug=False): | |||
def run_profile_hooks(profile, step, **kwargs): | |||
hooks_dir = get_hooks_dir() | |||
step_hooks = profile[step]["hooks"] | |||
if not step_hooks: | |||
@@ -12,7 +10,7 @@ def run_profile_hooks(profile, step, cwd=None, env=None, debug=False): | |||
if not trimmed_hook_name: | |||
continue | |||
hook_path = os.path.join(hooks_dir, trimmed_hook_name) | |||
code = subprocess.check_call(hook_path, cwd=cwd, stdin=subprocess.PIPE, env=env) | |||
run([hook_path], **kwargs) | |||
def get_hooks_dir(): | |||
return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../hooks") | |||
@@ -51,37 +49,17 @@ def get_workspace_subdir(subdir): | |||
os.makedirs(dir_path, exist_ok=True) | |||
return dir_path | |||
def get_acbuild_achive_dest_dir(): | |||
"""Return the first path matching the acbuild archive extraction destination in tamarin workspace""" | |||
workspace_tmp = get_workspace_subdir('tmp') | |||
return glob.glob(os.path.join(os.sep, workspace_tmp, 'acbuild-v*'))[0] | |||
def get_rkt_achive_dest_dir(): | |||
"""Return the first path matching the rkt archive extraction destination in tamarin workspace""" | |||
workspace_tmp = get_workspace_subdir('tmp') | |||
return glob.glob(os.path.join(os.sep, workspace_tmp, 'rkt-v*'))[0] | |||
def download_rkt(debug=False): | |||
"""Download a local copy of rkt in the tamarin workspace and return the absolute path to the archive""" | |||
url = "https://github.com/coreos/rkt/releases/download/v1.25.0/rkt-v1.25.0.tar.gz" | |||
file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "rkt.tar.gz") | |||
web.download_file(file_url=url, dest_path=file_path) | |||
return file_path | |||
def download_acbuild(debug=False): | |||
"""Download a local copy of acbuild in the tamarin workspace and return the absolute path to the archive""" | |||
url = "https://github.com/containers/build/releases/download/v0.4.0/acbuild-v0.4.0.tar.gz" | |||
file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "acbuild.tar.gz") | |||
web.download_file(file_url=url, dest_path=file_path) | |||
return file_path | |||
def run_acbuild(args, captureOutput=False, as_root=False, debug=False): | |||
"""Run acbuild with the specified args (use the local copy if acbuild is not found in the $PATH)""" | |||
acbuild_bin = system.which('acbuild', get_workspace_subdir('acbuild')) | |||
cmd = ( ["sudo", "-E", acbuild_bin] if os.geteuid() != 0 and as_root == True else [acbuild_bin] ) + args | |||
def run(cmd, captureOutput=False, pty=False, debug=False, **kwargs): | |||
"""Execute an arbitrary command on the system""" | |||
if debug: | |||
print(" ".join(cmd)) | |||
stdin=subprocess.PIPE | |||
if pty: | |||
kwargs['stdin'] = sys.stdin | |||
if captureOutput: | |||
return subprocess.check_output(cmd, stdin=subprocess.PIPE) | |||
return subprocess.check_output(cmd, **kwargs) | |||
else: | |||
return subprocess.check_call(cmd, stdin=subprocess.PIPE) | |||
return subprocess.check_call(cmd, **kwargs) | |||
def run_docker(args, captureOutput=False, **kwargs): | |||
return run(["docker"] + args, captureOutput=captureOutput, **kwargs) |
@@ -1,33 +0,0 @@ | |||
from urllib import request | |||
import math, sys | |||
def print_progress_bar(percent_progress=0, char_size=50, clear_line=True): | |||
bar_progress = math.floor(char_size*(percent_progress/100)) | |||
bar = "=" * bar_progress + " " * (char_size - bar_progress) | |||
if clear_line: | |||
sys.stdout.write(u"\u001b[1000D") | |||
sys.stdout.write("[{:s}] {:d}%".format(bar, int(percent_progress))) | |||
sys.stdout.flush() | |||
def download_file(file_url, dest_path, bulk_size = 8192): | |||
req = request.urlopen(file_url) | |||
meta = req.info() | |||
file_size = int(meta.get('Content-Length')) | |||
with open(dest_path, 'wb') as dest_file: | |||
print('Downloading "{:s}". Size: {:d}b'.format(file_url, file_size)) | |||
downloaded_size = 0 | |||
while True: | |||
buff = req.read(bulk_size) | |||
if not buff: | |||
break | |||
downloaded_size += len(buff) | |||
dest_file.write(buff) | |||
progress = downloaded_size/file_size*100 | |||
print_progress_bar(progress) | |||
dest_file.close() | |||
# Add linebreak | |||
print("") |
@@ -1,10 +1,10 @@ | |||
#!/usr/bin/env python3 | |||
import argparse, sys, shutil, os, hashlib | |||
import argparse, sys, shutil, os, subprocess | |||
sys.path.append(os.path.dirname(__file__) + '/lib') | |||
import tamarin, system, rkt | |||
import tamarin | |||
def create_args_parser(): | |||
'''Return a new configured ArgumentParser''' | |||
@@ -20,80 +20,37 @@ def create_args_parser(): | |||
parser.add_argument("-b", "--base", help="Use the specified image instead of the profile's one", default='') | |||
parser.add_argument("--rebuild", help="Ignore cache and rebuild container's image", action="store_true", default=False) | |||
parser.add_argument("--debug", help="Will add extra output and start the container in interactive mode", action="store_true", default=False) | |||
parser.add_argument("--cleanup", help="Clear the workspace and remove obsolete Docker images before build", action="store_true", default=False) | |||
return parser | |||
def download_and_extract_rkt(dest_dir, debug=False): | |||
'''Download and extract rkt to the given destination directory''' | |||
rkt_archive_path = tamarin.download_rkt(debug=debug) | |||
system.extract_tar(rkt_archive_path, workspace_tmp, debug=debug) | |||
rkt_archive_dir = tamarin.get_rkt_achive_dest_dir() | |||
shutil.rmtree(local_rkt_dir, ignore_errors=True) | |||
os.rename(rkt_archive_dir, dest_dir) | |||
def download_and_extract_acbuild(dest_dir, debug=False): | |||
'''Download and extract acbuild to the given destination directory''' | |||
acbuild_archive_path = tamarin.download_acbuild(debug=debug) | |||
system.extract_tar(acbuild_archive_path, workspace_tmp, debug=debug) | |||
acbuild_archive_dir = tamarin.get_acbuild_achive_dest_dir() | |||
shutil.rmtree(local_acbuild_dir, ignore_errors=True) | |||
os.rename(acbuild_archive_dir, dest_dir) | |||
def get_cached_image_path(profile, debug=False): | |||
'''Compute and return the path for an hypothetic cached image for the given profile''' | |||
containerbuild_hooks = profile['containerbuild']['hooks'] | |||
hasher = hashlib.sha1() | |||
hasher.update(base_image.encode()) | |||
hasher.update(containerbuild_hooks.encode()) | |||
image_hash = hasher.hexdigest() | |||
cache_dir = tamarin.get_workspace_subdir('cache') | |||
return os.path.join(os.sep, cache_dir, '{:s}.aci'.format(image_hash[:12])); | |||
def build_image(build_workspace, aci_file, base_image, profile, debug=False): | |||
acbuild_flags = ["--work-path", build_workspace] | |||
# Find and export base image from rkt' store | |||
name_pattern = base_image.split('/')[-1] + '$' | |||
image = rkt.find_image_by_name(name_pattern, rkt_flags=rkt_flags) | |||
rkt.export_image(image['id'], aci_file, rkt_flags=rkt_flags, debug=debug); | |||
# Build image | |||
tamarin.run_acbuild(acbuild_flags+["begin", aci_file], debug=debug) | |||
tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)], debug=debug) | |||
tamarin.run_acbuild(acbuild_flags+["mount", "add", "src", "/src", "--read-only"], debug=debug) | |||
tamarin.run_acbuild(acbuild_flags+["mount", "add", "dist", "/dist"], debug=debug) | |||
tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-hooks", "/tamarin/hooks", "--read-only"], debug=debug) | |||
tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-lib", "/tamarin/lib", "--read-only"], debug=debug) | |||
tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-profiles", "/tamarin/profiles", "--read-only"], debug=debug) | |||
def build_image(build_workspace, base_image, profile_name, profile, debug=False, rebuild=False): | |||
with open("{:s}/Dockerfile".format(build_workspace), 'w') as dockerfile: | |||
dockerfile.write("FROM {:s}\n".format(base_image)) | |||
# Configure "containerbuild" hooks environment | |||
hooks_env = os.environ.copy() | |||
hooks_env["PATH"] = os.environ['PATH'] + ':' + tamarin.get_workspace_subdir('acbuild') | |||
hooks_env["TAMARIN_ACBUILD"] = " ".join([system.which('acbuild', local_acbuild_dir)]+acbuild_flags) | |||
hooks_env["TAMARIN_ACBUILD_ENGINE"] = "chroot" if not system.which('systemctl') else "systemd-nspawn" | |||
hooks_env["PATH"] = os.environ['PATH'] + ':' + tamarin.get_lib_dir() | |||
# Run hooks | |||
tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=hooks_env, debug=debug) | |||
tamarin.run_acbuild(acbuild_flags+["write", "--overwrite", aci_file], as_root=True, debug=debug) | |||
tamarin.run_acbuild(acbuild_flags+["end"], as_root=True, debug=debug) | |||
image_tag = "tamarin:{:s}_{:s}_{:d}".format(profile_name, base_image.replace(':', '_'), os.getpid()) | |||
build_args = [ "build", "-t", image_tag ] | |||
if rebuild: | |||
build_args += [ "--no-cache" ] | |||
return aci_file | |||
tamarin.run_docker(build_args + [build_workspace], debug=debug) | |||
def cleanup(build_workspace, rkt_flags, debug=False): | |||
return image_tag | |||
# Nettoyage des conteneurs | |||
rkt.run([ | |||
"gc", | |||
"--grace-period=0" | |||
] + rkt_flags, as_root=True, debug=debug) | |||
def cleanup(build_workspace=None, debug=False): | |||
# Nettoyage des images obsolètes du store | |||
rkt.run([ | |||
"image", | |||
"gc" | |||
] + rkt_flags, as_root=True, debug=debug) | |||
if build_workspace == None: | |||
build_workspace = tamarin.get_workspace_subdir('tmp') | |||
# Suppression de l'espace de travail de build | |||
shutil.rmtree(build_workspace, ignore_errors=True) | |||
@@ -108,6 +65,9 @@ if __name__ == "__main__": | |||
validate_args(args) | |||
if args.cleanup: | |||
cleanup(debug=args.debug) | |||
# Verify project directory | |||
project_dir = os.path.abspath(args.project_directory) | |||
output_dir = os.path.abspath(args.output) | |||
@@ -116,69 +76,42 @@ if __name__ == "__main__": | |||
profile = tamarin.load_profile(args.profile, debug=args.debug) | |||
workspace = tamarin.get_workspace_dir() | |||
workspace_tmp = tamarin.get_workspace_subdir('tmp') | |||
local_rkt_dir = tamarin.get_workspace_subdir('rkt') | |||
if not system.which('rkt', local_rkt_dir): | |||
download_and_extract_rkt(local_rkt_dir) | |||
local_acbuild_dir = tamarin.get_workspace_subdir('acbuild') | |||
if not system.which('acbuild', local_acbuild_dir): | |||
download_and_extract_acbuild(local_acbuild_dir) | |||
pid = os.getpid() | |||
build_workspace = tamarin.get_workspace_subdir('tmp/build_{:d}'.format(pid)) | |||
rkt_store = tamarin.get_workspace_subdir('store') | |||
rkt_flags = ["--dir={:s}".format(rkt_store)] | |||
base_image = args.base if args.base != '' else profile['profile']['default_image'] | |||
# If the base image is Docker-based, download it | |||
if base_image.startswith('docker://'): | |||
rkt.run([ | |||
"fetch", | |||
"--insecure-options=image", | |||
base_image | |||
] + rkt_flags, debug=args.debug) | |||
aci_file = os.path.join(os.sep, build_workspace, 'image.aci') | |||
cached_image_file = get_cached_image_path(profile, debug=args.debug) | |||
if not args.rebuild and os.path.exists(cached_image_file): | |||
# Copy cached image | |||
shutil.copyfile(cached_image_file, aci_file) | |||
else: | |||
build_image(build_workspace, aci_file, base_image, profile, debug=args.debug) | |||
# Cache image | |||
shutil.copyfile(aci_file, cached_image_file) | |||
image_tag = build_image(build_workspace, base_image, args.profile, profile, debug=args.debug, rebuild=args.rebuild) | |||
# rkt run arguments | |||
rkt_args = [ | |||
"run", | |||
"--insecure-options=image", | |||
aci_file, "--net=host", | |||
"--volume=src,kind=host,source={:s}".format(project_dir), | |||
"--volume=dist,kind=host,source={:s}".format(output_dir), | |||
"--volume=tamarin-hooks,kind=host,source={:s}".format(tamarin.get_hooks_dir()), | |||
"--volume=tamarin-lib,kind=host,source={:s}".format(tamarin.get_lib_dir()), | |||
"--volume=tamarin-profiles,kind=host,source={:s}".format(tamarin.get_profiles_dir()) | |||
docker_args = [ | |||
"run", | |||
"--rm", | |||
"-v", "{:s}:/src:ro".format(project_dir), | |||
"-v", "{:s}:/dist".format(output_dir), | |||
"-v", "{:s}:/tamarin/hooks:ro".format(tamarin.get_hooks_dir()), | |||
"-v", "{:s}:/tamarin/lib:ro".format(tamarin.get_lib_dir()), | |||
"-v", "{:s}:/tamarin/profiles:ro".format(tamarin.get_profiles_dir()) | |||
] | |||
# Use environment proxy if defined | |||
for proxy_var in ['HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy']: | |||
if proxy_var in os.environ: | |||
rkt_args += ["--set-env={:s}={:s}".format(proxy_var, os.environ[proxy_var])] | |||
if proxy_var in os.environ: | |||
docker_args += ["-e", "{:s}={:s}".format(proxy_var, os.environ[proxy_var])] | |||
kwargs = dict() | |||
kwargs['debug'] = args.debug | |||
if args.debug: | |||
rkt_args += ["--interactive", "--exec", "/bin/bash"] | |||
helper_cmd = " ".join(["/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture]) | |||
print("Executer '{:s}' pour lancer la construction du paquet.".format(helper_cmd)) | |||
kwargs['pty'] = True | |||
docker_args += ["-it", image_tag, "/bin/sh"] | |||
helper_cmd = " ".join(["/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture]) | |||
print("Executer '{:s}' pour lancer la construction du paquet.".format(helper_cmd)) | |||
else: | |||
rkt_args += ["--exec", "/usr/bin/python3", "--", "/tamarin/lib/build.py", args.profile, args.architecture] | |||
docker_args += [image_tag, "/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture] | |||
# Start container | |||
rkt.run(rkt_flags+rkt_args, as_root=True, debug=args.debug) | |||
tamarin.run_docker(docker_args, **kwargs) | |||
# Cleanup | |||
cleanup(build_workspace, rkt_flags, debug=args.debug) | |||
cleanup(build_workspace, debug=args.debug) |
@@ -1,7 +1,7 @@ | |||
# Configuration générale du profil | |||
[profile] | |||
# Image Docker par défaut | |||
default_image=docker://debian:jessie | |||
default_image=debian:jessie | |||
# Configuration de l'étape de pré-construction du conteneur | |||
[containerbuild] |