From cc5df7cfcd14275b6648f6b4a54ce62fe79dc259 Mon Sep 17 00:00:00 2001 From: William Petit Date: Wed, 11 Jan 2017 14:51:31 +0100 Subject: [PATCH 01/20] =?UTF-8?q?D=C3=A9but=20impl=C3=A9mentation=20wrappe?= =?UTF-8?q?r=20Python=20sur=20rkt/acbuild?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rkt.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 rkt.py diff --git a/rkt.py b/rkt.py new file mode 100644 index 0000000..9ca4530 --- /dev/null +++ b/rkt.py @@ -0,0 +1,58 @@ +from urllib import request +from os import environ +import sys +import math +import tarfile + +def print_progress_bar(percent_progress=0, char_size=50, clear_line=True): + + bar_progress = math.floor(char_size*(percent_progress/100)) + bar = "=" * bar_progress + " " * (char_size - bar_progress) + + if clear_line: + sys.stdout.write(u"\u001b[1000D") + sys.stdout.write("[{:s}] {:d}%".format(bar, int(percent_progress))) + sys.stdout.flush() + +def download_file(file_url, dest_path, bulk_size = 8192): + + req = request.urlopen(file_url) + meta = req.info() + file_size = int(meta.get('Content-Length')) + + with open(dest_path, 'wb') as dest_file: + print('Downloading "{:s}". Size: {:d}b'.format(file_url, file_size)) + downloaded_size = 0 + while True: + buff = req.read(bulk_size) + if not buff: + break + downloaded_size += len(buff) + dest_file.write(buff) + progress = downloaded_size/file_size*100 + print_progress_bar(progress) + dest_file.close() + # Add linebreak + print("") + +def extract_tar(file_path, dest_dir = "."): + print('Extracting "{:s}" to "{:s}"'.format(file_path, dest_dir)) + with tarfile.open(file_path) as tar: + tar.extractall(dest_dir) + tar.close() + +def download_rkt(): + url = "https://github.com/coreos/rkt/releases/download/v1.22.0/rkt-v1.22.0.tar.gz" + file_path="rkt.tar.gz" + download_file(file_url=url, dest_path=file_path) + extract_tar(file_path) + +def download_acbuild(): + url = "https://github.com/containers/build/releases/download/v0.4.0/acbuild-v0.4.0.tar.gz" + file_path="acbuild.tar.gz" + download_file(file_url=url, dest_path=file_path) + extract_tar(file_path) + +if __name__ == "__main__": + download_rkt() + download_acbuild() From 86216e639e0d6a8f17cc666a3c2d65a2c2fec30f Mon Sep 17 00:00:00 2001 From: William Petit Date: Wed, 18 Jan 2017 17:43:46 +0100 Subject: [PATCH 02/20] =?UTF-8?q?S=C3=A9paration=20m=C3=A9thodes=20en=20mo?= =?UTF-8?q?dules,=20d=C3=A9but=20docstring?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + check-install => check-install.sh | 0 lib/system.py | 28 ++++ lib/tamarin.py | 54 ++++++++ rkt.py => lib/web.py | 27 +--- package | 208 ++++++------------------------ package.sh | 182 ++++++++++++++++++++++++++ 7 files changed, 302 insertions(+), 198 deletions(-) rename check-install => check-install.sh (100%) create mode 100644 lib/system.py create mode 100644 lib/tamarin.py rename rkt.py => lib/web.py (57%) create mode 100755 package.sh diff --git a/.gitignore b/.gitignore index 4463bf0..bab4f0f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ packages *.log *~ +__pycache__ diff --git a/check-install b/check-install.sh similarity index 100% rename from check-install rename to check-install.sh diff --git a/lib/system.py b/lib/system.py new file mode 100644 index 0000000..bbf8357 --- /dev/null +++ b/lib/system.py @@ -0,0 +1,28 @@ +import tarfile, os + +def extract_tar(file_path, dest_dir = "."): + print('Extracting "{:s}" to "{:s}"'.format(file_path, dest_dir)) + with tarfile.open(file_path) as tar: + tar.extractall(dest_dir) + tar.close() + +def which(program, additional_paths = None): + + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + fpath, fname = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + paths = os.environ["PATH"].split(os.pathsep); + if additional_paths != None: + paths.append(additional_paths) + for path in paths: + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + + return None diff --git a/lib/tamarin.py b/lib/tamarin.py new file mode 100644 index 0000000..a66cbfa --- /dev/null +++ b/lib/tamarin.py @@ -0,0 +1,54 @@ +import os, glob, subprocess +import web, system + +def get_workspace_dir(): + """Return the absolute path to the tamarin workspace ($HOME/.tamarin)""" + home = os.environ["HOME"] + return os.path.join(os.sep, home, '.tamarin') + +def get_workspace_subdir(subdir): + """Return the absolute path to a subdirectory in tamarin workspace""" + dir_path = os.path.join(os.sep, get_workspace_dir(), subdir) + os.makedirs(dir_path, exist_ok=True) + return dir_path + +def get_acbuild_achive_dest_dir(): + """Return the first path matching the acbuild archive extraction destination in tamarin workspace""" + workspace_tmp = get_workspace_subdir('tmp') + return glob.glob(os.path.join(os.sep, workspace_tmp, 'acbuild-v*'))[0] + +def get_rkt_achive_dest_dir(): + """Return the first path matching the rkt archive extraction destination in tamarin workspace""" + workspace_tmp = get_workspace_subdir('tmp') + return glob.glob(os.path.join(os.sep, workspace_tmp, 'rkt-v*'))[0] + +def get_acbuild_workspace_dir(): + """Return the current path (linked to the process pid) to the acbuild workspace""" + return get_workspace_subdir("tmp/build_{:d}".format(os.getpid())) + +def download_rkt(): + """Download a local copy of rkt in the tamarin workspace and return the absolute path to the archive""" + url = "https://github.com/coreos/rkt/releases/download/v1.22.0/rkt-v1.22.0.tar.gz" + file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "rkt.tar.gz") + web.download_file(file_url=url, dest_path=file_path) + return file_path + +def download_acbuild(): + """Download a local copy of acbuild in the tamarin workspace and return the absolute path to the archive""" + url = "https://github.com/containers/build/releases/download/v0.4.0/acbuild-v0.4.0.tar.gz" + file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "acbuild.tar.gz") + web.download_file(file_url=url, dest_path=file_path) + return file_path + +def run_rkt(args): + """Run rkt with the specified args (use the local copy if rkt is not found in the $PATH)""" + rkt_bin = system.which('rkt', get_workspace_subdir('rkt')) + cmd = ( ["sudo", "-E", rkt_bin] if os.geteuid() != 0 else [rkt_bin] ) + args + print(" ".join(cmd)) + return subprocess.call(cmd, stdin=subprocess.PIPE) + +def run_acbuild(args): + """Run acbuild with the specified args (use the local copy if acbuild is not found in the $PATH)""" + acbuild_bin = system.which('acbuild', get_workspace_subdir('acbuild')) + print(" ".join([acbuild_bin] + args)) + return subprocess.call([acbuild_bin] + args, stdin=subprocess.PIPE) diff --git a/rkt.py b/lib/web.py similarity index 57% rename from rkt.py rename to lib/web.py index 9ca4530..d6d41a8 100644 --- a/rkt.py +++ b/lib/web.py @@ -1,8 +1,5 @@ from urllib import request -from os import environ -import sys -import math -import tarfile +import math, sys def print_progress_bar(percent_progress=0, char_size=50, clear_line=True): @@ -34,25 +31,3 @@ def download_file(file_url, dest_path, bulk_size = 8192): dest_file.close() # Add linebreak print("") - -def extract_tar(file_path, dest_dir = "."): - print('Extracting "{:s}" to "{:s}"'.format(file_path, dest_dir)) - with tarfile.open(file_path) as tar: - tar.extractall(dest_dir) - tar.close() - -def download_rkt(): - url = "https://github.com/coreos/rkt/releases/download/v1.22.0/rkt-v1.22.0.tar.gz" - file_path="rkt.tar.gz" - download_file(file_url=url, dest_path=file_path) - extract_tar(file_path) - -def download_acbuild(): - url = "https://github.com/containers/build/releases/download/v0.4.0/acbuild-v0.4.0.tar.gz" - file_path="acbuild.tar.gz" - download_file(file_url=url, dest_path=file_path) - extract_tar(file_path) - -if __name__ == "__main__": - download_rkt() - download_acbuild() diff --git a/package b/package index a898714..0001b8c 100755 --- a/package +++ b/package @@ -1,182 +1,46 @@ -#!/usr/bin/env bash +#!/usr/bin/env python3 -set -e +if __name__ == "__main__": -TAMARIN_VERSION=0.0.1 -TAMARIN_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + import sys, os, shutil -BASE_DIR="$TAMARIN_DIR" source "$TAMARIN_DIR/lib/util.sh" -DEFAULT_64_IMAGE="debian:jessie" -DEFAULT_32_IMAGE="32bit/debian:jessie" + sys.path.append(os.path.dirname(__file__) + '/lib') -function show_usage { - echo - echo "Usage: $0 -p project_path [-a arch] [-d destination] [-i image] [-k]" - echo - echo "Parameters: " - echo - echo " -p Path to the project to build" - echo " -v Optional : Show Tamarin version" - echo " -a Optional : Target architecture (default amd64)" - echo " -d Optional : Destination of the builed packages (default ./packages)" - echo " -i Optional : Name of the Docker image to use for build (default: debian:jessie)" - echo " -k Optional : Keep the Docker container after build " - echo " -B Optional : Build branch (for git projects only) (default dist/ubuntu/precise/master)" - echo - exit 2 -} + import tamarin, system -# Create a build container based on the $BASE_IMAGE argument -function create_container { + workspace = tamarin.get_workspace_dir() + workspace_tmp = tamarin.get_workspace_subdir('tmp') - # Escape image name - local escaped_basename=$(echo "$BASE_IMAGE" | sed 's/[^a-z0-9\-\_\.]/\_/gi') - # Generate container tag - container_tag="tamarin:${escaped_basename}_$(date +%s)" - # Create temporary dir for the Dockerfile - local temp_dir="$(mktemp -d)" + local_rkt_dir = tamarin.get_workspace_subdir('rkt') + if not system.which('rkt', local_rkt_dir): + # Download and extract rkt + rkt_archive_path = tamarin.download_rkt() + system.extract_tar(rkt_archive_path, workspace_tmp) + rkt_archive_dir = tamarin.get_rkt_achive_dest_dir() + shutil.rmtree(local_rkt_dir, ignore_errors=True) + os.rename(rkt_archive_dir, local_rkt_dir) - local projectName=${1} + local_acbuild_dir = tamarin.get_workspace_subdir('acbuild') + if not system.which('acbuild', local_acbuild_dir): + # Download and extract acbuild + acbuild_archive_path = tamarin.download_acbuild() + system.extract_tar(acbuild_archive_path, workspace_tmp) + acbuild_archive_dir = tamarin.get_acbuild_achive_dest_dir() + shutil.rmtree(local_acbuild_dir, ignore_errors=True) + os.rename(acbuild_archive_dir, local_acbuild_dir) - # Link lib & hooks folders - ln -s $(readlink -f "$TAMARIN_DIR/lib") "$temp_dir/lib" - ln -s $(readlink -f "$TAMARIN_DIR/hooks") "$temp_dir/hooks" + acbuild_workspace = tamarin.get_acbuild_workspace_dir() + image_name = "image_{:d}.aci".format(os.getpid()) + image_path = os.path.join(os.sep, acbuild_workspace, 'image.aci') + acbuild_flags = ["--work-path", acbuild_workspace] - # Create Dockerfile - cat << EOF > "$temp_dir/Dockerfile" - FROM ${BASE_IMAGE} + # Start building image + tamarin.run_acbuild(acbuild_flags+["begin"]) + tamarin.run_acbuild(acbuild_flags+["set-name", "test"]) + tamarin.run_acbuild(acbuild_flags+["set-exec", "--", "/bin/sh", "-c", "ping 8.8.8.8"]) + tamarin.run_acbuild(acbuild_flags+["dependency", "add", "quay.io/coreos/alpine-sh"]) + tamarin.run_acbuild(acbuild_flags+["write", image_path]) + tamarin.run_acbuild(acbuild_flags+["end"]) - ENV DEBIAN_FRONTEND noninteractive - - RUN apt-get update &&\ - apt-get install --yes --no-install-recommends build-essential devscripts equivs - - RUN mkdir /root/.tamarin - RUN mkdir /project - - ADD ./lib /root/.tamarin/lib - ADD ./hooks /hooks - RUN chmod +x /root/.tamarin/lib/build.sh - - VOLUME /src - VOLUME /dist - -EOF - - exec_hooks "containerbuild" "$temp_dir" - - echo "CMD /root/.tamarin/lib/build.sh ${projectName} ${BUILD_BRANCH} /tmp ${BUILD_TAG}" >> "$temp_dir/Dockerfile" - - # Build image - tar -C "$temp_dir" -czh . | docker build -t "$container_tag" - 2> >(stderr) 1> >(stdout) - - # Delete temporary folder - rm -rf "$temp_dir" - -} - -# Main function -function main { - - info "Building container from $BASE_IMAGE..." - - local project_name=$(basename ${PROJECT_PATH}) - - # Create container & "$container_tag" variable - create_container ${project_name} - - local docker_opt="run -e \"DISTRIB=$BASE_IMAGE\" -e \"PROJECT_NAME=$project_name\"" - - # Expose host uid & gid - docker_opt="${docker_opt} -e HOST_UID=$(id -u) -e HOST_GID=$(id -g)" - - # Expose host proxy variables - docker_opt="${docker_opt} -e HTTP_PROXY=${HTTP_PROXY} -e HTTPS_PROXY=${HTTPS_PROXY}" - docker_opt="${docker_opt} -e http_proxy=${http_proxy} -e https_proxy=${https_proxy}" - - # Target architecture - docker_opt="${docker_opt} -e TARGET_ARCH=${TARGET_ARCH}" - - # If running in terminal, set docker to interactive - if [[ -t 1 ]]; then - docker_opt="${docker_opt} -it" - fi - - if [[ ${PERSIST_CONTAINER} -eq 0 ]] - then - docker_opt="${docker_opt} --rm " - else - docker_opt="${docker_opt}" - fi - - docker_opt="${docker_opt} -v $PROJECT_PATH:/src:ro -v $PROJECT_DEST:/dist:rw $container_tag" - - info "Switching to container..." - debug "docker ${docker_opt}" - docker ${docker_opt} - res=${?} - - success "Done" - return ${res} -} - -# -# Parsing options -# -while getopts "vkp:d:i:B:t:a:o:" option -do - case $option in - k) - PERSIST_CONTAINER=1 - ;; - p) - PROJECT_PATH=$(readlink -f ${OPTARG}) - ;; - d) - PROJECT_DEST=$(readlink -f ${OPTARG}) - ;; - i) - BASE_IMAGE="${OPTARG}" - ;; - B) - BUILD_BRANCH=${OPTARG} - ;; - t) - BUILD_TAG=${OPTARG} - ;; - a) - TARGET_ARCH=${OPTARG} - ;; - v) - echo "Tamarin v${TAMARIN_VERSION}" - exit - ;; - *) - show_usage - ;; - esac -done - -[[ -z ${PROJECT_PATH} ]] && show_usage -[[ -z ${PROJECT_DEST} ]] && PROJECT_DEST=$(readlink -f "./packages") -[[ -z ${BUILD_BRANCH} ]] && BUILD_BRANCH="dist/ubuntu/precise/master" -[[ -z ${PERSIST_CONTAINER} ]] && PERSIST_CONTAINER=0 -[[ -z ${TARGET_ARCH} ]] && TARGET_ARCH=amd64 - -if [[ "${TARGET_ARCH}" =~ ^i[0-9]86$ ]] && [ -z "${BASE_IMAGE}" ]; then - info "32bit architecture specified and no specific image given, switching to default 32bit image..." - BASE_IMAGE=${DEFAULT_32_IMAGE} -fi - -[[ -z ${BASE_IMAGE} ]] && BASE_IMAGE=${DEFAULT_64_IMAGE} - -# -# Warn user about "proxy" -# - -if [[ -n ${http_proxy} ]] -then - warn "You have a proxy defined please make sure docker deamon is configured to use this proxy" -fi - -main + # rkt run --insecure-options=image ./nginx.aci --volume html,kind=host,source=/path/to/test --net=host + tamarin.run_rkt(["run", "--insecure-options=image", image_path, "--net=host"]) diff --git a/package.sh b/package.sh new file mode 100755 index 0000000..a898714 --- /dev/null +++ b/package.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +set -e + +TAMARIN_VERSION=0.0.1 +TAMARIN_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + +BASE_DIR="$TAMARIN_DIR" source "$TAMARIN_DIR/lib/util.sh" +DEFAULT_64_IMAGE="debian:jessie" +DEFAULT_32_IMAGE="32bit/debian:jessie" + +function show_usage { + echo + echo "Usage: $0 -p project_path [-a arch] [-d destination] [-i image] [-k]" + echo + echo "Parameters: " + echo + echo " -p Path to the project to build" + echo " -v Optional : Show Tamarin version" + echo " -a Optional : Target architecture (default amd64)" + echo " -d Optional : Destination of the builed packages (default ./packages)" + echo " -i Optional : Name of the Docker image to use for build (default: debian:jessie)" + echo " -k Optional : Keep the Docker container after build " + echo " -B Optional : Build branch (for git projects only) (default dist/ubuntu/precise/master)" + echo + exit 2 +} + +# Create a build container based on the $BASE_IMAGE argument +function create_container { + + # Escape image name + local escaped_basename=$(echo "$BASE_IMAGE" | sed 's/[^a-z0-9\-\_\.]/\_/gi') + # Generate container tag + container_tag="tamarin:${escaped_basename}_$(date +%s)" + # Create temporary dir for the Dockerfile + local temp_dir="$(mktemp -d)" + + local projectName=${1} + + # Link lib & hooks folders + ln -s $(readlink -f "$TAMARIN_DIR/lib") "$temp_dir/lib" + ln -s $(readlink -f "$TAMARIN_DIR/hooks") "$temp_dir/hooks" + + # Create Dockerfile + cat << EOF > "$temp_dir/Dockerfile" + FROM ${BASE_IMAGE} + + ENV DEBIAN_FRONTEND noninteractive + + RUN apt-get update &&\ + apt-get install --yes --no-install-recommends build-essential devscripts equivs + + RUN mkdir /root/.tamarin + RUN mkdir /project + + ADD ./lib /root/.tamarin/lib + ADD ./hooks /hooks + RUN chmod +x /root/.tamarin/lib/build.sh + + VOLUME /src + VOLUME /dist + +EOF + + exec_hooks "containerbuild" "$temp_dir" + + echo "CMD /root/.tamarin/lib/build.sh ${projectName} ${BUILD_BRANCH} /tmp ${BUILD_TAG}" >> "$temp_dir/Dockerfile" + + # Build image + tar -C "$temp_dir" -czh . | docker build -t "$container_tag" - 2> >(stderr) 1> >(stdout) + + # Delete temporary folder + rm -rf "$temp_dir" + +} + +# Main function +function main { + + info "Building container from $BASE_IMAGE..." + + local project_name=$(basename ${PROJECT_PATH}) + + # Create container & "$container_tag" variable + create_container ${project_name} + + local docker_opt="run -e \"DISTRIB=$BASE_IMAGE\" -e \"PROJECT_NAME=$project_name\"" + + # Expose host uid & gid + docker_opt="${docker_opt} -e HOST_UID=$(id -u) -e HOST_GID=$(id -g)" + + # Expose host proxy variables + docker_opt="${docker_opt} -e HTTP_PROXY=${HTTP_PROXY} -e HTTPS_PROXY=${HTTPS_PROXY}" + docker_opt="${docker_opt} -e http_proxy=${http_proxy} -e https_proxy=${https_proxy}" + + # Target architecture + docker_opt="${docker_opt} -e TARGET_ARCH=${TARGET_ARCH}" + + # If running in terminal, set docker to interactive + if [[ -t 1 ]]; then + docker_opt="${docker_opt} -it" + fi + + if [[ ${PERSIST_CONTAINER} -eq 0 ]] + then + docker_opt="${docker_opt} --rm " + else + docker_opt="${docker_opt}" + fi + + docker_opt="${docker_opt} -v $PROJECT_PATH:/src:ro -v $PROJECT_DEST:/dist:rw $container_tag" + + info "Switching to container..." + debug "docker ${docker_opt}" + docker ${docker_opt} + res=${?} + + success "Done" + return ${res} +} + +# +# Parsing options +# +while getopts "vkp:d:i:B:t:a:o:" option +do + case $option in + k) + PERSIST_CONTAINER=1 + ;; + p) + PROJECT_PATH=$(readlink -f ${OPTARG}) + ;; + d) + PROJECT_DEST=$(readlink -f ${OPTARG}) + ;; + i) + BASE_IMAGE="${OPTARG}" + ;; + B) + BUILD_BRANCH=${OPTARG} + ;; + t) + BUILD_TAG=${OPTARG} + ;; + a) + TARGET_ARCH=${OPTARG} + ;; + v) + echo "Tamarin v${TAMARIN_VERSION}" + exit + ;; + *) + show_usage + ;; + esac +done + +[[ -z ${PROJECT_PATH} ]] && show_usage +[[ -z ${PROJECT_DEST} ]] && PROJECT_DEST=$(readlink -f "./packages") +[[ -z ${BUILD_BRANCH} ]] && BUILD_BRANCH="dist/ubuntu/precise/master" +[[ -z ${PERSIST_CONTAINER} ]] && PERSIST_CONTAINER=0 +[[ -z ${TARGET_ARCH} ]] && TARGET_ARCH=amd64 + +if [[ "${TARGET_ARCH}" =~ ^i[0-9]86$ ]] && [ -z "${BASE_IMAGE}" ]; then + info "32bit architecture specified and no specific image given, switching to default 32bit image..." + BASE_IMAGE=${DEFAULT_32_IMAGE} +fi + +[[ -z ${BASE_IMAGE} ]] && BASE_IMAGE=${DEFAULT_64_IMAGE} + +# +# Warn user about "proxy" +# + +if [[ -n ${http_proxy} ]] +then + warn "You have a proxy defined please make sure docker deamon is configured to use this proxy" +fi + +main From 8469bf2c0801ecf289c35a8e2a384c78900a4b64 Mon Sep 17 00:00:00 2001 From: William Petit Date: Thu, 19 Jan 2017 17:39:55 +0100 Subject: [PATCH 03/20] =?UTF-8?q?Base=20r=C3=A9=C3=A9criture=20script=20pa?= =?UTF-8?q?ckage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/tamarin.py | 5 +++ package | 102 ++++++++++++++++++++++++++----------------- profiles/debian.conf | 0 3 files changed, 66 insertions(+), 41 deletions(-) create mode 100644 profiles/debian.conf diff --git a/lib/tamarin.py b/lib/tamarin.py index a66cbfa..5d31b16 100644 --- a/lib/tamarin.py +++ b/lib/tamarin.py @@ -1,6 +1,11 @@ import os, glob, subprocess import web, system +def get_available_profiles(): + profiles_dir = os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../profiles") + profile_files = glob.glob(profiles_dir + '/*.conf') + return [os.path.splitext(os.path.basename(f))[0] for f in profile_files] + def get_workspace_dir(): """Return the absolute path to the tamarin workspace ($HOME/.tamarin)""" home = os.environ["HOME"] diff --git a/package b/package index 0001b8c..effee82 100755 --- a/package +++ b/package @@ -1,46 +1,66 @@ #!/usr/bin/env python3 +import argparse, sys, shutil +from os import path + +sys.path.append(path.dirname(__file__) + '/lib') + +import tamarin, system + +def configure_args_parser(): + + profiles = tamarin.get_available_profiles() + + parser = argparse.ArgumentParser(description="Generate packages for various GNU/Linux distribution") + + # Define available/required arguments and flags + parser.add_argument("project_path", help="The path to your project to package") + parser.add_argument("-p", "--profile", help="The profile to use to package this project", choices=profiles) + + return parser + if __name__ == "__main__": - import sys, os, shutil + parser = configure_args_parser() + parser.parse_args() - sys.path.append(os.path.dirname(__file__) + '/lib') - - import tamarin, system - - workspace = tamarin.get_workspace_dir() - workspace_tmp = tamarin.get_workspace_subdir('tmp') - - local_rkt_dir = tamarin.get_workspace_subdir('rkt') - if not system.which('rkt', local_rkt_dir): - # Download and extract rkt - rkt_archive_path = tamarin.download_rkt() - system.extract_tar(rkt_archive_path, workspace_tmp) - rkt_archive_dir = tamarin.get_rkt_achive_dest_dir() - shutil.rmtree(local_rkt_dir, ignore_errors=True) - os.rename(rkt_archive_dir, local_rkt_dir) - - local_acbuild_dir = tamarin.get_workspace_subdir('acbuild') - if not system.which('acbuild', local_acbuild_dir): - # Download and extract acbuild - acbuild_archive_path = tamarin.download_acbuild() - system.extract_tar(acbuild_archive_path, workspace_tmp) - acbuild_archive_dir = tamarin.get_acbuild_achive_dest_dir() - shutil.rmtree(local_acbuild_dir, ignore_errors=True) - os.rename(acbuild_archive_dir, local_acbuild_dir) - - acbuild_workspace = tamarin.get_acbuild_workspace_dir() - image_name = "image_{:d}.aci".format(os.getpid()) - image_path = os.path.join(os.sep, acbuild_workspace, 'image.aci') - acbuild_flags = ["--work-path", acbuild_workspace] - - # Start building image - tamarin.run_acbuild(acbuild_flags+["begin"]) - tamarin.run_acbuild(acbuild_flags+["set-name", "test"]) - tamarin.run_acbuild(acbuild_flags+["set-exec", "--", "/bin/sh", "-c", "ping 8.8.8.8"]) - tamarin.run_acbuild(acbuild_flags+["dependency", "add", "quay.io/coreos/alpine-sh"]) - tamarin.run_acbuild(acbuild_flags+["write", image_path]) - tamarin.run_acbuild(acbuild_flags+["end"]) - - # rkt run --insecure-options=image ./nginx.aci --volume html,kind=host,source=/path/to/test --net=host - tamarin.run_rkt(["run", "--insecure-options=image", image_path, "--net=host"]) + # sys.path.append(os.path.dirname(__file__) + '/lib') + # + # import tamarin, system + # + # workspace = tamarin.get_workspace_dir() + # workspace_tmp = tamarin.get_workspace_subdir('tmp') + # + # local_rkt_dir = tamarin.get_workspace_subdir('rkt') + # if not system.which('rkt', local_rkt_dir): + # # Download and extract rkt + # rkt_archive_path = tamarin.download_rkt() + # system.extract_tar(rkt_archive_path, workspace_tmp) + # rkt_archive_dir = tamarin.get_rkt_achive_dest_dir() + # shutil.rmtree(local_rkt_dir, ignore_errors=True) + # os.rename(rkt_archive_dir, local_rkt_dir) + # + # local_acbuild_dir = tamarin.get_workspace_subdir('acbuild') + # if not system.which('acbuild', local_acbuild_dir): + # # Download and extract acbuild + # acbuild_archive_path = tamarin.download_acbuild() + # system.extract_tar(acbuild_archive_path, workspace_tmp) + # acbuild_archive_dir = tamarin.get_acbuild_achive_dest_dir() + # shutil.rmtree(local_acbuild_dir, ignore_errors=True) + # os.rename(acbuild_archive_dir, local_acbuild_dir) + # + # acbuild_workspace = tamarin.get_acbuild_workspace_dir() + # image_name = "image_{:d}.aci".format(os.getpid()) + # image_path = os.path.join(os.sep, acbuild_workspace, 'image.aci') + # acbuild_flags = ["--work-path", acbuild_workspace] + # + # # Start building image + # tamarin.run_acbuild(acbuild_flags+["begin"]) + # tamarin.run_acbuild(acbuild_flags+["set-name", "test"]) + # tamarin.run_acbuild(acbuild_flags+["set-exec", "--", "/bin/sh", "-c", "ping 8.8.8.8"]) + # tamarin.run_acbuild(acbuild_flags+["dependency", "add", "quay.io/coreos/alpine-sh"]) + # tamarin.run_acbuild(acbuild_flags+["write", image_path]) + # tamarin.run_acbuild(acbuild_flags+["end"]) + # + # # rkt run --insecure-options=image ./nginx.aci --volume html,kind=host,source=/path/to/test --net=host + # tamarin.run_rkt(["run", "--insecure-options=image", image_path, "--net=host"]) diff --git a/profiles/debian.conf b/profiles/debian.conf new file mode 100644 index 0000000..e69de29 From b88493523f8adce869e92bcbb79439c31b185234 Mon Sep 17 00:00:00 2001 From: William Petit Date: Thu, 19 Jan 2017 22:56:17 +0100 Subject: [PATCH 04/20] =?UTF-8?q?D=C3=A9but=20g=C3=A9n=C3=A9ration=20d'ima?= =?UTF-8?q?ges=20pour=20construction=20des=20paquets=20via=20acbuild?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/tamarin.py | 20 ++++++++-- package | 90 ++++++++++++++++++++++---------------------- profiles/debian.conf | 22 +++++++++++ 3 files changed, 83 insertions(+), 49 deletions(-) diff --git a/lib/tamarin.py b/lib/tamarin.py index 5d31b16..8123dd5 100644 --- a/lib/tamarin.py +++ b/lib/tamarin.py @@ -1,9 +1,23 @@ -import os, glob, subprocess +import os, glob, subprocess, configparser import web, system +def load_profile(profile_name): + profile_filename = profile_name+".conf" + for profile_file in get_available_profiles(): + if profile_filename == os.path.basename(profile_file): + config = configparser.ConfigParser() + config.read(profile_file) + return config + return None + +def get_profiles_dir(): + return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../profiles") + def get_available_profiles(): - profiles_dir = os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../profiles") - profile_files = glob.glob(profiles_dir + '/*.conf') + return glob.glob(get_profiles_dir() + '/*.conf') + +def get_available_profile_names(): + profile_files = get_available_profiles() return [os.path.splitext(os.path.basename(f))[0] for f in profile_files] def get_workspace_dir(): diff --git a/package b/package index effee82..6946135 100755 --- a/package +++ b/package @@ -1,66 +1,64 @@ #!/usr/bin/env python3 -import argparse, sys, shutil -from os import path +import argparse, sys, shutil, os -sys.path.append(path.dirname(__file__) + '/lib') +sys.path.append(os.path.dirname(__file__) + '/lib') import tamarin, system def configure_args_parser(): - profiles = tamarin.get_available_profiles() + profile_names = tamarin.get_available_profile_names() parser = argparse.ArgumentParser(description="Generate packages for various GNU/Linux distribution") # Define available/required arguments and flags parser.add_argument("project_path", help="The path to your project to package") - parser.add_argument("-p", "--profile", help="The profile to use to package this project", choices=profiles) + parser.add_argument("-p", "--profile", help="The profile to use to package this project (default: debian)", choices=profile_names, default='debian') return parser if __name__ == "__main__": parser = configure_args_parser() - parser.parse_args() + args = parser.parse_args() - # sys.path.append(os.path.dirname(__file__) + '/lib') - # - # import tamarin, system - # - # workspace = tamarin.get_workspace_dir() - # workspace_tmp = tamarin.get_workspace_subdir('tmp') - # - # local_rkt_dir = tamarin.get_workspace_subdir('rkt') - # if not system.which('rkt', local_rkt_dir): - # # Download and extract rkt - # rkt_archive_path = tamarin.download_rkt() - # system.extract_tar(rkt_archive_path, workspace_tmp) - # rkt_archive_dir = tamarin.get_rkt_achive_dest_dir() - # shutil.rmtree(local_rkt_dir, ignore_errors=True) - # os.rename(rkt_archive_dir, local_rkt_dir) - # - # local_acbuild_dir = tamarin.get_workspace_subdir('acbuild') - # if not system.which('acbuild', local_acbuild_dir): - # # Download and extract acbuild - # acbuild_archive_path = tamarin.download_acbuild() - # system.extract_tar(acbuild_archive_path, workspace_tmp) - # acbuild_archive_dir = tamarin.get_acbuild_achive_dest_dir() - # shutil.rmtree(local_acbuild_dir, ignore_errors=True) - # os.rename(acbuild_archive_dir, local_acbuild_dir) - # - # acbuild_workspace = tamarin.get_acbuild_workspace_dir() - # image_name = "image_{:d}.aci".format(os.getpid()) - # image_path = os.path.join(os.sep, acbuild_workspace, 'image.aci') - # acbuild_flags = ["--work-path", acbuild_workspace] - # - # # Start building image - # tamarin.run_acbuild(acbuild_flags+["begin"]) - # tamarin.run_acbuild(acbuild_flags+["set-name", "test"]) - # tamarin.run_acbuild(acbuild_flags+["set-exec", "--", "/bin/sh", "-c", "ping 8.8.8.8"]) - # tamarin.run_acbuild(acbuild_flags+["dependency", "add", "quay.io/coreos/alpine-sh"]) - # tamarin.run_acbuild(acbuild_flags+["write", image_path]) - # tamarin.run_acbuild(acbuild_flags+["end"]) - # - # # rkt run --insecure-options=image ./nginx.aci --volume html,kind=host,source=/path/to/test --net=host - # tamarin.run_rkt(["run", "--insecure-options=image", image_path, "--net=host"]) + profile = tamarin.load_profile(args.profile) + + workspace = tamarin.get_workspace_dir() + workspace_tmp = tamarin.get_workspace_subdir('tmp') + + local_rkt_dir = tamarin.get_workspace_subdir('rkt') + if not system.which('rkt', local_rkt_dir): + # Download and extract rkt + rkt_archive_path = tamarin.download_rkt() + system.extract_tar(rkt_archive_path, workspace_tmp) + rkt_archive_dir = tamarin.get_rkt_achive_dest_dir() + shutil.rmtree(local_rkt_dir, ignore_errors=True) + os.rename(rkt_archive_dir, local_rkt_dir) + + local_acbuild_dir = tamarin.get_workspace_subdir('acbuild') + if not system.which('acbuild', local_acbuild_dir): + # Download and extract acbuild + acbuild_archive_path = tamarin.download_acbuild() + system.extract_tar(acbuild_archive_path, workspace_tmp) + acbuild_archive_dir = tamarin.get_acbuild_achive_dest_dir() + shutil.rmtree(local_acbuild_dir, ignore_errors=True) + os.rename(acbuild_archive_dir, local_acbuild_dir) + + acbuild_workspace = tamarin.get_acbuild_workspace_dir() + pid = os.getpid() + image_name = "image_{:d}.aci".format(pid) + image_path = os.path.join(os.sep, acbuild_workspace, image_name) + acbuild_flags = ["--work-path", acbuild_workspace] + + # Start building image + tamarin.run_acbuild(acbuild_flags+["begin"]) + tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)]) + # tamarin.run_acbuild(acbuild_flags+["dependency", "add", profile['profile']['default_image']]) + tamarin.run_acbuild(acbuild_flags+["set-exec", "--", "/bin/sh", "-c", "echo Hello World"]) + tamarin.run_acbuild(acbuild_flags+["write", image_path]) + tamarin.run_acbuild(acbuild_flags+["end"]) + + # rkt run --insecure-options=image ./nginx.aci --volume html,kind=host,source=/path/to/test --net=host + tamarin.run_rkt(["run", "--insecure-options=image", "--dir={:s}".format(workspace_tmp), "--stage1-name={:s}".format(profile['profile']['default_image']), image_path, "--net=host"]) diff --git a/profiles/debian.conf b/profiles/debian.conf index e69de29..ccd8216 100644 --- a/profiles/debian.conf +++ b/profiles/debian.conf @@ -0,0 +1,22 @@ +# Configuration générale du profil +[profile] +# Image Docker par défaut +default_image=docker://debian:jessie + +# Configuration de l'étape de pré-construction du conteneur +[containerprebuild] +hooks=install-git-containerbuild + +# Configuration de l'étape de pré-construction du paquet +[prebuild] +hooks=05-create-changelog-prebuild,06-create-dummy-changelog-prebuild,07-add-version-suffix-prebuild,10-install-build-depends-prebuild + +# Configuration de l'étape de post-construction du paquet +[postbuild] +hooks=99-export-dist-postbuild + +[preinstall] +hooks= + +[postinstall] +hooks= From 487c97e88cf09c9f2cfdb555451319b7fffc86af Mon Sep 17 00:00:00 2001 From: William Petit Date: Fri, 20 Jan 2017 17:22:54 +0100 Subject: [PATCH 05/20] =?UTF-8?q?Construction/execution=20d'un=20conteneur?= =?UTF-8?q?=20=C3=A0=20partir=20de=20l'image=20de=20base=20sp=C3=A9cifi?= =?UTF-8?q?=C3=A9e=20dans=20le=20profil?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/tamarin.py | 16 ++++++++++----- package | 43 ++++++++++++++++++++++++++++++++++++++--- src-example/hello-world | 3 --- 3 files changed, 51 insertions(+), 11 deletions(-) delete mode 100755 src-example/hello-world diff --git a/lib/tamarin.py b/lib/tamarin.py index 8123dd5..88e96b2 100644 --- a/lib/tamarin.py +++ b/lib/tamarin.py @@ -59,15 +59,21 @@ def download_acbuild(): web.download_file(file_url=url, dest_path=file_path) return file_path -def run_rkt(args): +def run_rkt(args, asRoot = False, captureOutput=False): """Run rkt with the specified args (use the local copy if rkt is not found in the $PATH)""" rkt_bin = system.which('rkt', get_workspace_subdir('rkt')) - cmd = ( ["sudo", "-E", rkt_bin] if os.geteuid() != 0 else [rkt_bin] ) + args + cmd = ( ["sudo", "-E", rkt_bin] if os.geteuid() != 0 and asRoot == True else [rkt_bin] ) + args print(" ".join(cmd)) - return subprocess.call(cmd, stdin=subprocess.PIPE) + if captureOutput: + return subprocess.check_output(cmd, stdin=subprocess.PIPE) + else: + return subprocess.call(cmd, stdin=subprocess.PIPE) -def run_acbuild(args): +def run_acbuild(args, captureOutput=False): """Run acbuild with the specified args (use the local copy if acbuild is not found in the $PATH)""" acbuild_bin = system.which('acbuild', get_workspace_subdir('acbuild')) print(" ".join([acbuild_bin] + args)) - return subprocess.call([acbuild_bin] + args, stdin=subprocess.PIPE) + if captureOutput: + return subprocess.check_output([acbuild_bin] + args, stdin=subprocess.PIPE) + else: + return subprocess.call([acbuild_bin] + args, stdin=subprocess.PIPE) diff --git a/package b/package index 6946135..befa500 100755 --- a/package +++ b/package @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import argparse, sys, shutil, os +import argparse, sys, shutil, os, json sys.path.append(os.path.dirname(__file__) + '/lib') @@ -52,13 +52,50 @@ if __name__ == "__main__": image_path = os.path.join(os.sep, acbuild_workspace, image_name) acbuild_flags = ["--work-path", acbuild_workspace] + rkt_flags = ["--dir={:s}".format(workspace_tmp)] + + base_image = profile['profile']['default_image'] + # If the base image is Docker-based, preload it and get its name from the store + if base_image.startswith('docker://'): + tamarin.run_rkt([ + "fetch", + "--insecure-options=image", + base_image + ] + rkt_flags) + output = tamarin.run_rkt([ + "image", + "list", + "--format=json" + ] + rkt_flags, captureOutput=True) + # Fetch the list of installed images + images_list = json.loads(output.decode('utf-8')) + base_image = images_list[0]['name'] + # Start building image tamarin.run_acbuild(acbuild_flags+["begin"]) tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)]) - # tamarin.run_acbuild(acbuild_flags+["dependency", "add", profile['profile']['default_image']]) + tamarin.run_acbuild(acbuild_flags+["dependency", "add", base_image]) tamarin.run_acbuild(acbuild_flags+["set-exec", "--", "/bin/sh", "-c", "echo Hello World"]) tamarin.run_acbuild(acbuild_flags+["write", image_path]) tamarin.run_acbuild(acbuild_flags+["end"]) # rkt run --insecure-options=image ./nginx.aci --volume html,kind=host,source=/path/to/test --net=host - tamarin.run_rkt(["run", "--insecure-options=image", "--dir={:s}".format(workspace_tmp), "--stage1-name={:s}".format(profile['profile']['default_image']), image_path, "--net=host"]) + tamarin.run_rkt([ + "run", + "--insecure-options=image", + image_path, "--net=host" + ] + rkt_flags, asRoot=True) + + # Cleanup + + tamarin.run_rkt([ + "gc", + "--grace-period=0" + ] + rkt_flags, asRoot=True) + + tamarin.run_rkt([ + "image", + "gc" + ] + rkt_flags, asRoot=True) + + shutil.rmtree(acbuild_workspace, ignore_errors=True) diff --git a/src-example/hello-world b/src-example/hello-world deleted file mode 100755 index 124f6f6..0000000 --- a/src-example/hello-world +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -echo $1 From 82fb9d31e1e398a7b48db713f2c3bf4e7f485ebf Mon Sep 17 00:00:00 2001 From: William Petit Date: Tue, 24 Jan 2017 17:32:19 +0100 Subject: [PATCH 06/20] Gestion de l'image de base --- lib/rkt.py | 37 +++++++++++++++++++++++++++++++++++++ lib/tamarin.py | 10 ---------- package | 41 +++++++++++++++++++++-------------------- 3 files changed, 58 insertions(+), 30 deletions(-) create mode 100644 lib/rkt.py diff --git a/lib/rkt.py b/lib/rkt.py new file mode 100644 index 0000000..c34e183 --- /dev/null +++ b/lib/rkt.py @@ -0,0 +1,37 @@ +import system, subprocess, os, tamarin, json, re + +def run(args, as_root = False, capture_output=False): + """Run rkt with the specified args (use the local copy if rkt is not found in the $PATH)""" + rkt_bin = system.which('rkt', tamarin.get_workspace_subdir('rkt')) + cmd = ( ["sudo", "-E", rkt_bin] if os.geteuid() != 0 and as_root == True else [rkt_bin] ) + args + print(" ".join(cmd)) + if capture_output: + return subprocess.check_output(cmd, stdin=subprocess.PIPE) + else: + return subprocess.call(cmd, stdin=subprocess.PIPE) + +def get_images_list(rkt_flags = []): + output = run([ + "image", + "list", + "--format=json" + ] + rkt_flags, capture_output=True) + # Fetch the list of installed images + return json.loads(output.decode('utf-8')) + +def find_image_by_name(name_pattern, rkt_flags = []): + if type(name_pattern) is str: + name_pattern = re.compile(name_pattern) + images_list = get_images_list(rkt_flags = rkt_flags) + for image in images_list: + if name_pattern.search(image['name']): + return image + return None + +def export_image(image_id, dest_file, rkt_flags = []): + output = run([ + "image", + "export", + image_id, + dest_file, + ] + rkt_flags) diff --git a/lib/tamarin.py b/lib/tamarin.py index 88e96b2..22f29a7 100644 --- a/lib/tamarin.py +++ b/lib/tamarin.py @@ -59,16 +59,6 @@ def download_acbuild(): web.download_file(file_url=url, dest_path=file_path) return file_path -def run_rkt(args, asRoot = False, captureOutput=False): - """Run rkt with the specified args (use the local copy if rkt is not found in the $PATH)""" - rkt_bin = system.which('rkt', get_workspace_subdir('rkt')) - cmd = ( ["sudo", "-E", rkt_bin] if os.geteuid() != 0 and asRoot == True else [rkt_bin] ) + args - print(" ".join(cmd)) - if captureOutput: - return subprocess.check_output(cmd, stdin=subprocess.PIPE) - else: - return subprocess.call(cmd, stdin=subprocess.PIPE) - def run_acbuild(args, captureOutput=False): """Run acbuild with the specified args (use the local copy if acbuild is not found in the $PATH)""" acbuild_bin = system.which('acbuild', get_workspace_subdir('acbuild')) diff --git a/package b/package index befa500..89f65df 100755 --- a/package +++ b/package @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -import argparse, sys, shutil, os, json +import argparse, sys, shutil, os sys.path.append(os.path.dirname(__file__) + '/lib') -import tamarin, system +import tamarin, system, rkt def configure_args_parser(): @@ -57,45 +57,46 @@ if __name__ == "__main__": base_image = profile['profile']['default_image'] # If the base image is Docker-based, preload it and get its name from the store if base_image.startswith('docker://'): - tamarin.run_rkt([ + rkt.run([ "fetch", "--insecure-options=image", base_image ] + rkt_flags) - output = tamarin.run_rkt([ - "image", - "list", - "--format=json" - ] + rkt_flags, captureOutput=True) - # Fetch the list of installed images - images_list = json.loads(output.decode('utf-8')) - base_image = images_list[0]['name'] + + name_pattern = base_image.split('/')[-1] + '$' + image = rkt.find_image_by_name(name_pattern, rkt_flags=rkt_flags) + + + base_aci_name = image['id'][:21]+'.aci' + base_aci_file = workspace_tmp+'/'+base_aci_name + + if not os.path.isfile(base_aci_file): + rkt.export_image(image['id'], workspace_tmp+'/'+base_aci_name, rkt_flags=rkt_flags); # Start building image - tamarin.run_acbuild(acbuild_flags+["begin"]) + tamarin.run_acbuild(acbuild_flags+["begin", base_aci_file]) tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)]) - tamarin.run_acbuild(acbuild_flags+["dependency", "add", base_image]) - tamarin.run_acbuild(acbuild_flags+["set-exec", "--", "/bin/sh", "-c", "echo Hello World"]) + tamarin.run_acbuild(acbuild_flags+["set-exec", "--", "/bin/sh", "-c", "uname -a"]) tamarin.run_acbuild(acbuild_flags+["write", image_path]) tamarin.run_acbuild(acbuild_flags+["end"]) # rkt run --insecure-options=image ./nginx.aci --volume html,kind=host,source=/path/to/test --net=host - tamarin.run_rkt([ + rkt.run([ "run", "--insecure-options=image", image_path, "--net=host" - ] + rkt_flags, asRoot=True) + ] + rkt_flags, as_root=True) # Cleanup - tamarin.run_rkt([ + rkt.run([ "gc", "--grace-period=0" - ] + rkt_flags, asRoot=True) + ] + rkt_flags, as_root=True) - tamarin.run_rkt([ + rkt.run([ "image", "gc" - ] + rkt_flags, asRoot=True) + ] + rkt_flags, as_root=True) shutil.rmtree(acbuild_workspace, ignore_errors=True) From ce05d9a3397f3b73bc341a36db69deb4e57ad36d Mon Sep 17 00:00:00 2001 From: William Petit Date: Tue, 24 Jan 2017 23:15:13 +0100 Subject: [PATCH 07/20] =?UTF-8?q?G=C3=A9n=C3=A9ration=20de=20l'image=20ave?= =?UTF-8?q?c=20mise=20en=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- check-install.sh | 82 ------------------- .../debian/install-build-essential | 6 ++ hooks/containerbuild/debian/install-git | 6 ++ hooks/install-git-containerbuild | 3 - lib/tamarin.py | 14 +++- package | 59 ++++++++----- profiles/debian.conf | 4 +- 7 files changed, 62 insertions(+), 112 deletions(-) delete mode 100755 check-install.sh create mode 100755 hooks/containerbuild/debian/install-build-essential create mode 100755 hooks/containerbuild/debian/install-git delete mode 100755 hooks/install-git-containerbuild diff --git a/check-install.sh b/check-install.sh deleted file mode 100755 index 6364a11..0000000 --- a/check-install.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -source "$DIR/lib/util.sh" - -function show_usage { - echo - echo "Usage: $0 deb_file [image]" - echo - echo "Paramètres: " - echo - echo " - deb_file Chemin vers le paquet Debian dont on doit vérifier l'installation" - echo " - image Optionnel - Nom de l'image Docker à utiliser comme environnement pour tester l'installation. Défaut: debian:jessie" - echo -} - -function create_container { - - # Escape image name - local escaped_basename=$(echo "$BASE_IMAGE" | sed 's/[^a-z0-9\-\_\.]/\_/gi') - # Generate container tag - container_tag="tamarin:${escaped_basename}_$(date +%s)" - # Create temporary dir for the Dockerfile - local temp_dir="$(mktemp -d)" - - # Link lib folder - ln -s $(readlink -f "$DIR/lib") "$temp_dir/lib" - - # Create Dockerfile - cat << EOF > "$temp_dir/Dockerfile" - FROM $BASE_IMAGE - - ENV DEBIAN_FRONTEND noninteractive - - RUN apt-get update && apt-get install --yes gdebi-core - - ADD ./lib /root/.tamarin - RUN chmod +x /root/.tamarin/install.sh - - VOLUME /deb - - ENTRYPOINT ["/root/.tamarin/install.sh"] - -EOF - - # Build image - tar -C "$temp_dir" -czh . | docker build -t "$container_tag" - 2> >(error) 1> >(info) - - # Delete temporary folder - rm -rf "$temp_dir" - -} - -function main { - - # Create container image - create_container - - # Run container and install package - docker run -e "DISTRIB=$BASE_IMAGE" --rm -v="$DEB_DIR:/deb" "$container_tag" "/deb/$DEB_NAME" - - # Check for return - if [ $? != 0 ]; then - fatal "Installation did not complete correctly !" - fi - - info "Installation complete." - -} - -# Test for arguments -if [ -z "$1" ]; then - show_usage - exit 1 -fi - -DEB_PATH=$(readlink -f "$1") -DEB_NAME=$(basename "$DEB_PATH") -DEB_DIR=$(dirname "$DEB_PATH") -BASE_IMAGE="${2:-debian:jessie}" - -main diff --git a/hooks/containerbuild/debian/install-build-essential b/hooks/containerbuild/debian/install-build-essential new file mode 100755 index 0000000..766557a --- /dev/null +++ b/hooks/containerbuild/debian/install-build-essential @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive +sudo -E ${TAMARIN_ACBUILD} run -- apt-get update +sudo -E ${TAMARIN_ACBUILD} run -- apt-get install --yes --no-install-recommends build-essential devscripts equivs +${TAMARIN_ACBUILD} environment remove DEBIAN_FRONTEND diff --git a/hooks/containerbuild/debian/install-git b/hooks/containerbuild/debian/install-git new file mode 100755 index 0000000..890305a --- /dev/null +++ b/hooks/containerbuild/debian/install-git @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive +sudo -E ${TAMARIN_ACBUILD} run -- apt-get update +sudo -E ${TAMARIN_ACBUILD} run -- apt-get install --yes --no-install-recommends git-core +${TAMARIN_ACBUILD} environment remove DEBIAN_FRONTEND diff --git a/hooks/install-git-containerbuild b/hooks/install-git-containerbuild deleted file mode 100755 index b1f91ce..0000000 --- a/hooks/install-git-containerbuild +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -echo "RUN apt-get update && apt-get install --yes --no-install-recommends git-core" >> Dockerfile diff --git a/lib/tamarin.py b/lib/tamarin.py index 22f29a7..f38f53b 100644 --- a/lib/tamarin.py +++ b/lib/tamarin.py @@ -1,6 +1,16 @@ import os, glob, subprocess, configparser import web, system +def run_profile_hooks(profile, step, cwd=None, env=None): + hooks_dir = get_hooks_dir() + step_hooks = profile[step]["hooks"].split(",") + for hook_name in step_hooks: + hook_path = os.path.join(hooks_dir, hook_name) + subprocess.call(hook_path, cwd=cwd, stdin=subprocess.PIPE, env=env) + +def get_hooks_dir(): + return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../hooks") + def load_profile(profile_name): profile_filename = profile_name+".conf" for profile_file in get_available_profiles(): @@ -41,10 +51,6 @@ def get_rkt_achive_dest_dir(): workspace_tmp = get_workspace_subdir('tmp') return glob.glob(os.path.join(os.sep, workspace_tmp, 'rkt-v*'))[0] -def get_acbuild_workspace_dir(): - """Return the current path (linked to the process pid) to the acbuild workspace""" - return get_workspace_subdir("tmp/build_{:d}".format(os.getpid())) - def download_rkt(): """Download a local copy of rkt in the tamarin workspace and return the absolute path to the archive""" url = "https://github.com/coreos/rkt/releases/download/v1.22.0/rkt-v1.22.0.tar.gz" diff --git a/package b/package index 89f65df..f040d8d 100755 --- a/package +++ b/package @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import argparse, sys, shutil, os +import argparse, sys, shutil, os, hashlib sys.path.append(os.path.dirname(__file__) + '/lib') @@ -15,6 +15,7 @@ def configure_args_parser(): # Define available/required arguments and flags parser.add_argument("project_path", help="The path to your project to package") parser.add_argument("-p", "--profile", help="The profile to use to package this project (default: debian)", choices=profile_names, default='debian') + parser.add_argument("--rebuild", help="Ignore cache and rebuild container's image", action="store_true", default=False) return parser @@ -23,6 +24,7 @@ if __name__ == "__main__": parser = configure_args_parser() args = parser.parse_args() + # Load build profile profile = tamarin.load_profile(args.profile) workspace = tamarin.get_workspace_dir() @@ -46,13 +48,11 @@ if __name__ == "__main__": shutil.rmtree(local_acbuild_dir, ignore_errors=True) os.rename(acbuild_archive_dir, local_acbuild_dir) - acbuild_workspace = tamarin.get_acbuild_workspace_dir() pid = os.getpid() - image_name = "image_{:d}.aci".format(pid) - image_path = os.path.join(os.sep, acbuild_workspace, image_name) - acbuild_flags = ["--work-path", acbuild_workspace] + build_workspace = tamarin.get_workspace_subdir('tmp/build_{:d}'.format(pid)) - rkt_flags = ["--dir={:s}".format(workspace_tmp)] + rkt_store = tamarin.get_workspace_subdir('store') + rkt_flags = ["--dir={:s}".format(rkt_store)] base_image = profile['profile']['default_image'] # If the base image is Docker-based, preload it and get its name from the store @@ -63,28 +63,45 @@ if __name__ == "__main__": base_image ] + rkt_flags) - name_pattern = base_image.split('/')[-1] + '$' - image = rkt.find_image_by_name(name_pattern, rkt_flags=rkt_flags) + aci_file = os.path.join(os.sep, build_workspace, 'image.aci') + acbuild_flags = ["--modify", aci_file, "--work-path", build_workspace] + # Use cached image base on base_image and containerbuild hooks + containerbuild_hooks = profile['containerbuild']['hooks'] + hasher = hashlib.sha1() + hasher.update(base_image.encode()) + hasher.update(containerbuild_hooks.encode()) + image_hash = hasher.hexdigest() + cache_dir = tamarin.get_workspace_subdir('cache') + cached_image_file = os.path.join(os.sep, cache_dir, '{:s}.aci'.format(image_hash[:12])); - base_aci_name = image['id'][:21]+'.aci' - base_aci_file = workspace_tmp+'/'+base_aci_name + if not args.rebuild and os.path.exists(cached_image_file): + # Copy cached image + shutil.copyfile(cached_image_file, aci_file) + else: + # Find and export base image from rkt' store + name_pattern = base_image.split('/')[-1] + '$' + image = rkt.find_image_by_name(name_pattern, rkt_flags=rkt_flags) + rkt.export_image(image['id'], aci_file, rkt_flags=rkt_flags); - if not os.path.isfile(base_aci_file): - rkt.export_image(image['id'], workspace_tmp+'/'+base_aci_name, rkt_flags=rkt_flags); + # Build image + tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)]) + tamarin.run_acbuild(acbuild_flags+["set-exec", "--", "/bin/sh", "-c", "echo Hello World"]) - # Start building image - tamarin.run_acbuild(acbuild_flags+["begin", base_aci_file]) - tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)]) - tamarin.run_acbuild(acbuild_flags+["set-exec", "--", "/bin/sh", "-c", "uname -a"]) - tamarin.run_acbuild(acbuild_flags+["write", image_path]) - tamarin.run_acbuild(acbuild_flags+["end"]) + # Execute containerbuild hooks + cpb_hooks_env = os.environ.copy() + cpb_hooks_env["PATH"] = os.environ['PATH'] + ':' + local_acbuild_dir + cpb_hooks_env["TAMARIN_ACBUILD"] = " ".join([system.which('acbuild', local_acbuild_dir)]+acbuild_flags) + tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=cpb_hooks_env) - # rkt run --insecure-options=image ./nginx.aci --volume html,kind=host,source=/path/to/test --net=host + # Cache image + shutil.copyfile(aci_file, cached_image_file) + + # Start container rkt.run([ "run", "--insecure-options=image", - image_path, "--net=host" + aci_file, "--net=host" ] + rkt_flags, as_root=True) # Cleanup @@ -99,4 +116,4 @@ if __name__ == "__main__": "gc" ] + rkt_flags, as_root=True) - shutil.rmtree(acbuild_workspace, ignore_errors=True) + shutil.rmtree(build_workspace, ignore_errors=True) diff --git a/profiles/debian.conf b/profiles/debian.conf index ccd8216..3a8a572 100644 --- a/profiles/debian.conf +++ b/profiles/debian.conf @@ -4,8 +4,8 @@ default_image=docker://debian:jessie # Configuration de l'étape de pré-construction du conteneur -[containerprebuild] -hooks=install-git-containerbuild +[containerbuild] +hooks=containerbuild/debian/install-build-essential,containerbuild/debian/install-git # Configuration de l'étape de pré-construction du paquet [prebuild] From 38d236e09bfe99525eb0bb5f2fc031b353a963b1 Mon Sep 17 00:00:00 2001 From: William Petit Date: Wed, 25 Jan 2017 14:49:30 +0100 Subject: [PATCH 08/20] Build de l'image de base + execution build.py dans le conteneur --- .../debian/install-build-essential | 29 +++++++++++++++++-- hooks/containerbuild/debian/install-git | 13 +++++++-- lib/build.py | 3 ++ lib/rkt.py | 2 +- lib/tamarin.py | 4 +++ package | 10 ++++--- 6 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 lib/build.py diff --git a/hooks/containerbuild/debian/install-build-essential b/hooks/containerbuild/debian/install-build-essential index 766557a..0ac1731 100755 --- a/hooks/containerbuild/debian/install-build-essential +++ b/hooks/containerbuild/debian/install-build-essential @@ -1,6 +1,31 @@ #!/usr/bin/env bash +HAS_SYSTEMD=$(which systemctl) +ENGINE=chroot +if [ ! -z "${HAS_SYSTEMD}" ]; then + ENGINE=systemd-nspawn +fi + ${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive -sudo -E ${TAMARIN_ACBUILD} run -- apt-get update -sudo -E ${TAMARIN_ACBUILD} run -- apt-get install --yes --no-install-recommends build-essential devscripts equivs + +if [ "${ENGINE}" == 'chroot' ]; then + # Ugly fix for Python installation in chrooted environment (require /dev/urandom) + head -c 65536 /dev/urandom > ./urandom + ${TAMARIN_ACBUILD} copy ./urandom /dev/urandom +fi + +sudo -E /usr/bin/env bash - < Date: Thu, 9 Feb 2017 21:53:24 +0100 Subject: [PATCH 09/20] POC fonctionnel --- README.md | 38 ++-- hooks/build/debian/build | 4 + .../debian/install-build-essential | 16 +- hooks/containerbuild/debian/install-git | 10 +- .../debian/export-dist-postbuild} | 7 +- .../debian/add-package-version-suffix} | 13 +- .../prebuild/debian/copy-sources-to-workspace | 7 + .../debian/create-changelog} | 27 +-- .../debian/create-dummy-changelog} | 9 +- .../debian/install-build-depends} | 2 + hooks/prebuild/debian/load-project-db | 3 + lib/build.py | 31 ++- lib/build.sh | 47 ----- lib/buildtools/tamarin_db | 53 +++++ lib/buildtools/tamarin_debug | 9 + lib/buildtools/tamarin_error | 9 + lib/buildtools/tamarin_info | 9 + lib/buildtools/tamarin_log | 23 +++ lib/buildtools/tamarin_success | 9 + lib/buildtools/tamarin_warn | 9 + lib/tamarin.py | 22 ++- lib/util.sh | 154 --------------- package | 34 +++- package.sh | 182 ------------------ profiles/debian.conf | 22 ++- 25 files changed, 279 insertions(+), 470 deletions(-) create mode 100755 hooks/build/debian/build rename hooks/{99-export-dist-postbuild => postbuild/debian/export-dist-postbuild} (58%) rename hooks/{07-add-package-version-suffix-prebuild => prebuild/debian/add-package-version-suffix} (52%) create mode 100755 hooks/prebuild/debian/copy-sources-to-workspace rename hooks/{05-create-changelog-prebuild => prebuild/debian/create-changelog} (64%) rename hooks/{06-create-dummy-changelog-prebuild => prebuild/debian/create-dummy-changelog} (74%) rename hooks/{10-install-build-depends-prebuild => prebuild/debian/install-build-depends} (96%) create mode 100755 hooks/prebuild/debian/load-project-db delete mode 100755 lib/build.sh create mode 100755 lib/buildtools/tamarin_db create mode 100755 lib/buildtools/tamarin_debug create mode 100755 lib/buildtools/tamarin_error create mode 100755 lib/buildtools/tamarin_info create mode 100755 lib/buildtools/tamarin_log create mode 100755 lib/buildtools/tamarin_success create mode 100755 lib/buildtools/tamarin_warn delete mode 100644 lib/util.sh delete mode 100755 package.sh diff --git a/README.md b/README.md index fd502a3..b823ac8 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,38 @@ # Tamarin -Usine à paquets binaires Debian basée sur Docker. +Usine à paquets expérimentale basée sur rkt/acbuild. + +## Formats de paquets/distributions supportés + +- `.deb` (Debian, Ubuntu) ## Dépendances -Vous devez avoir [docker](https://docs.docker.com/installation/) installé sur votre machine. +- [Python 3](https://www.python.org/downloads/) +- Un noyau Linux > 2.6.24 (avec support des cgroups) -## Utilisation +**Optionnel mais conseillé** +- [systemd](https://freedesktop.org/wiki/Software/systemd/) + +## Usage + +```bash +./package --help ``` -Usage: ./package.sh -p project_path [-d destination] [-i image] [-k] +## Fonctionnement -Parameters: +### Les profils de construction - -p Path to the project to build - -a Optional : Architecture choice (-a i386 for example) - -d Optional : Destination of the builed packages (default ./packages) - -i Optional : Name of the Docker image to use for build (default: debian:jessie) - -k Optional : Keep the Docker container after build - -b Optional : Build directory (default /tmp) - -B Optional : Build branch (for git projects only) (default dist/ubuntu/precise/master) -``` +TODO + +### Les attributs de construction + +TODO + +### Répertoire de travail et mise en cache des images + +TODO ## Licence diff --git a/hooks/build/debian/build b/hooks/build/debian/build new file mode 100755 index 0000000..1f47226 --- /dev/null +++ b/hooks/build/debian/build @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +cd src +dpkg-buildpackage -b -a"${TAMARIN_TARGET_ARCH}" diff --git a/hooks/containerbuild/debian/install-build-essential b/hooks/containerbuild/debian/install-build-essential index 0ac1731..ed0cbfb 100755 --- a/hooks/containerbuild/debian/install-build-essential +++ b/hooks/containerbuild/debian/install-build-essential @@ -1,14 +1,8 @@ #!/usr/bin/env bash -HAS_SYSTEMD=$(which systemctl) -ENGINE=chroot -if [ ! -z "${HAS_SYSTEMD}" ]; then - ENGINE=systemd-nspawn -fi - ${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive -if [ "${ENGINE}" == 'chroot' ]; then +if [ "${TAMARIN_ACBUILD_ENGINE}" == 'chroot' ]; then # Ugly fix for Python installation in chrooted environment (require /dev/urandom) head -c 65536 /dev/urandom > ./urandom ${TAMARIN_ACBUILD} copy ./urandom /dev/urandom @@ -16,15 +10,15 @@ fi sudo -E /usr/bin/env bash - <> debian/changelog diff --git a/hooks/06-create-dummy-changelog-prebuild b/hooks/prebuild/debian/create-dummy-changelog similarity index 74% rename from hooks/06-create-dummy-changelog-prebuild rename to hooks/prebuild/debian/create-dummy-changelog index 288e586..9b97488 100755 --- a/hooks/06-create-dummy-changelog-prebuild +++ b/hooks/prebuild/debian/create-dummy-changelog @@ -1,14 +1,15 @@ #!/usr/bin/env bash -source "${TAMARIN_UTIL}" + +cd src if [ -f debian/changelog ] || [ ! -d .git ]; then - info "Not a Git repository or Debian changelog already exists !" + tamarin_info "Not a Git repository or Debian changelog already exists !" exit fi changelog="debian/changelog" -project_name=$(get_opt project_name) -project_version=$(get_opt project_version 0.0.0) +project_name=$(tamarin_db get project_name) +project_version=$(tamarin_db get project_version 0.0.0) date=$(date -R) top_contributor=$(git log --pretty=short | git shortlog -s -n -e | sed 's/^\s*[0-9]*\s*//g' | head -n 1) current_commit=$(git log -n 1 --pretty=format:%h) diff --git a/hooks/10-install-build-depends-prebuild b/hooks/prebuild/debian/install-build-depends similarity index 96% rename from hooks/10-install-build-depends-prebuild rename to hooks/prebuild/debian/install-build-depends index e63e432..0ff2014 100755 --- a/hooks/10-install-build-depends-prebuild +++ b/hooks/prebuild/debian/install-build-depends @@ -1,5 +1,7 @@ #!/bin/bash +cd src + if [ -f debian/control ]; then echo "Installing build dependencies..." apt-get update diff --git a/hooks/prebuild/debian/load-project-db b/hooks/prebuild/debian/load-project-db new file mode 100755 index 0000000..0f7d99b --- /dev/null +++ b/hooks/prebuild/debian/load-project-db @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +tamarin_db load diff --git a/lib/build.py b/lib/build.py index e067568..1f2d7de 100644 --- a/lib/build.py +++ b/lib/build.py @@ -1,3 +1,32 @@ +import sys, os, argparse, tempfile +sys.path.append(os.path.dirname(__file__) + '/lib') +import tamarin, system, rkt + +def get_args_parser(): + parser = argparse.ArgumentParser(description="Tamarin's container entrypoint") + # Define available/required arguments and flags + parser.add_argument("profile", help="The selected profile to use for the build") + parser.add_argument("arch", help="The selected profile to use for the build") + return parser + +def get_buildtools_dir(): + return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/buildtools") + if __name__ == '__main__': - print('Test') + + parser = get_args_parser() + args = parser.parse_args() + + profile = tamarin.load_profile(args.profile) + + workspace = tempfile.mkdtemp(prefix="tamarin_") + + env = os.environ.copy() + env['TAMARIN_TARGET_ARCH'] = args.arch + env['TAMARIN_WORKSPACE'] = workspace + env['PATH'] = env['PATH'] + ':' + get_buildtools_dir() + + tamarin.run_profile_hooks(profile, 'prebuild', cwd=workspace, env=env) + tamarin.run_profile_hooks(profile, 'build', cwd=workspace, env=env) + tamarin.run_profile_hooks(profile, 'postbuild', cwd=workspace, env=env) diff --git a/lib/build.sh b/lib/build.sh deleted file mode 100755 index b7bb8a2..0000000 --- a/lib/build.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -LIB_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -export TAMARIN_UTIL="${LIB_DIR}/util.sh" - -source "${TAMARIN_UTIL}" - -DIST_DIR="${BASE_DIR}/dist" -SRC_DIR="${BASE_DIR}/src" -PROJECT_NAME=${1} -BUILD_BRANCH=${2} -BUILD_TAG=${4} - -function build_project() -{ - - info "Building project '${PROJECT_NAME}' for ${TARGET_ARCH} architecture..." - - # Initalize opts - set_opt project_name "${PROJECT_NAME}" - set_opt build_branch "${BUILD_BRANCH}" - set_opt build_tag "${BUILD_TAG}" - - local workspace=$(mktemp -d)/${PROJECT_NAME} - info "Build dir is ${workspace}" - mkdir -p "${workspace}" - - # Copy sources to workspace - cd ${SRC_DIR} - cp -r ${SRC_DIR}/. "${workspace}" - - cd "$workspace" - - load_local_opts - exec_hooks "prebuild" "${workspace}" - - dpkg-buildpackage -b -a"${TARGET_ARCH}" 2> >(stderr) 1> >(stdout) - - if [ $? != 0 ]; then - fatal "The build process has not completed successfuly !" - fi - - exec_hooks "postbuild" "${workspace}" - -} - -build_project diff --git a/lib/buildtools/tamarin_db b/lib/buildtools/tamarin_db new file mode 100755 index 0000000..3446635 --- /dev/null +++ b/lib/buildtools/tamarin_db @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +PROJECT_DB_FILE="/src/.tamarinrc" +GLOBAL_DB_FILE="${TAMARIN_WORKSPACE}/.tamarinrc" +KEY_PREFIX="tamarin_db_" + +function tamarin_load_project_db { + if [ -e "${PROJECT_DB_FILE}" ]; then + tamarin_info "Loading project database..." + while read line; do + if [[ ! "${line}" =~ ^\s*# ]]; then + set -- $(echo $line | tr '=' ' ') + local key=$1 + local value=$2 + tamarin_debug "Load $key=$value" + tamarin_db_set $key $value + fi + done < "${PROJECT_DB_FILE}" + fi +} + +function tamarin_db_get { + local opt_name=${KEY_PREFIX}${1} + local default_value=${2} + touch "${GLOBAL_DB_FILE}" + source "${GLOBAL_DB_FILE}" + echo ${!opt_name:-${default_value}} +} + +function tamarin_db_set { + local opt_name=${1} + local opt_value=${2} + mkdir -p "$(dirname ${GLOBAL_DB_FILE})" + touch "${GLOBAL_DB_FILE}" + sed -i "s/^${KEY_PREFIX}${opt_name}*$//" "${GLOBAL_DB_FILE}" + echo "local ${KEY_PREFIX}${opt_name}=\"${opt_value}\"" >> "${GLOBAL_DB_FILE}" +} + +case $1 in + set) + tamarin_db_set ${@:2} + ;; + get) + tamarin_db_get ${@:2} + ;; + load) + tamarin_load_project_db + ;; + *) + tamarin_error "Invalid action '$1'. Must be get, set or load !" + exit 1 + ;; +esac diff --git a/lib/buildtools/tamarin_debug b/lib/buildtools/tamarin_debug new file mode 100755 index 0000000..4266a19 --- /dev/null +++ b/lib/buildtools/tamarin_debug @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +if [ -z "$@" ]; then + while read str; do + tamarin_log DEBUG "${str}" + done +else + tamarin_log DEBUG "$@" +fi diff --git a/lib/buildtools/tamarin_error b/lib/buildtools/tamarin_error new file mode 100755 index 0000000..9be8c07 --- /dev/null +++ b/lib/buildtools/tamarin_error @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +if [ -z "$@" ]; then + while read str; do + tamarin_log ERROR "${str}" + done +else + tamarin_log ERROR "$@" +fi diff --git a/lib/buildtools/tamarin_info b/lib/buildtools/tamarin_info new file mode 100755 index 0000000..518aa98 --- /dev/null +++ b/lib/buildtools/tamarin_info @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +if [ -z "$@" ]; then + while read str; do + tamarin_log INFO "${str}" + done +else + tamarin_log INFO "$@" +fi diff --git a/lib/buildtools/tamarin_log b/lib/buildtools/tamarin_log new file mode 100755 index 0000000..b8c6072 --- /dev/null +++ b/lib/buildtools/tamarin_log @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Colors + +COLOR_INFO='\e[0;36m' +COLOR_ERROR='\e[0;31m' +COLOR_WARN='\e[0;33m' +COLOR_SUCCESS='\e[0;32m' +COLOR_DEBUG='\e[0;35m' + +function log { + local args=( $@ ) + local color=COLOR_${args[0]} + echo -e "${!color}[${args[0]}] $(remove_ansi ${args[@]:1})\e[0m" +} + +function remove_ansi { + echo "$@" | sed 's,\x1B\[[0-9;]*[a-zA-Z],,g' +} + +if [ $# -ne 0 ]; then + log $@ +fi diff --git a/lib/buildtools/tamarin_success b/lib/buildtools/tamarin_success new file mode 100755 index 0000000..4484f97 --- /dev/null +++ b/lib/buildtools/tamarin_success @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +if [ -z "$@" ]; then + while read str; do + tamarin_log SUCCESS "${str}" + done +else + tamarin_log SUCCESS "$@" +fi diff --git a/lib/buildtools/tamarin_warn b/lib/buildtools/tamarin_warn new file mode 100755 index 0000000..4935b7e --- /dev/null +++ b/lib/buildtools/tamarin_warn @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +if [ -z "$@" ]; then + while read str; do + tamarin_log WARN "${str}" + done +else + tamarin_log WARN "$@" +fi diff --git a/lib/tamarin.py b/lib/tamarin.py index ed8c708..164866a 100644 --- a/lib/tamarin.py +++ b/lib/tamarin.py @@ -1,13 +1,20 @@ import os, glob, subprocess, configparser import web, system +import codecs def run_profile_hooks(profile, step, cwd=None, env=None): hooks_dir = get_hooks_dir() - step_hooks = profile[step]["hooks"].split(",") - for hook_name in step_hooks: - hook_path = os.path.join(hooks_dir, hook_name) - print(hook_path) - subprocess.call(hook_path, cwd=cwd, stdin=subprocess.PIPE, env=env) + step_hooks = profile[step]["hooks"] + if not step_hooks: + return + for hook_name in step_hooks.split(","): + trimmed_hook_name = hook_name.strip(' \t\n\r') + if not trimmed_hook_name: + continue + hook_path = os.path.join(hooks_dir, trimmed_hook_name) + code = subprocess.call(hook_path, cwd=cwd, stdin=subprocess.PIPE, env=env) + if code != 0: + raise Exception("Hook '{:s}' exited with a non zero code ({:d}) !".format(trimmed_hook_name, code)) def get_hooks_dir(): return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../hooks") @@ -20,8 +27,9 @@ def load_profile(profile_name): for profile_file in get_available_profiles(): if profile_filename == os.path.basename(profile_file): config = configparser.ConfigParser() - config.read(profile_file) - return config + with codecs.open(profile_file, encoding = 'utf-8', mode = 'r') as handle: + config.read_file(handle) + return config return None def get_profiles_dir(): diff --git a/lib/util.sh b/lib/util.sh deleted file mode 100644 index e0f0857..0000000 --- a/lib/util.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env bash - -HOOKS_DIR="${BASE_DIR}/hooks" -DEFAULT_OPTS_FILE="${BASE_DIR}/tmp/default_opts" -OPT_FILE="${BASE_DIR}/tmp/tamarin/opts" -OPT_PREFIX="tamarin_opt_" -LOCAL_OPTS_FILE=".tamarinrc" - -# Colors - -COLOR_INFO='\e[0;36m' -COLOR_FATAL='\e[0;31m' -COLOR_WARN='\e[0;33m' -COLOR_SUCCESS='\e[0;32m' -COLOR_ERR='\e[0;37m' -COLOR_OUT='\e[0;37m' -COLOR_DEBUG='\e[0;35m' - -function stderr { - if [ -z "$@" ]; then - while read str; do - log ERR "${str}" - done - else - log stderr "$@" - fi -} - -function stdout { - if [ -z "$@" ]; then - while read str; do - log OUT "${str}" - done - else - log OUT "$@" - fi -} - -function info { - if [ -z "$@" ]; then - while read str; do - log INFO "${str}" - done - else - log INFO "$@" - fi -} - -function warn { - if [ -z "$@" ]; then - while read str; do - log WARN "${str}" - done - else - log WARN "$@" - fi -} - -function debug { - if [ -z "$@" ]; then - while read str; do - log DEBUG "${str}" - done - else - log DEBUG "$@" - fi -} - -function fatal { - if [ -z "$@" ]; then - while read str; do - log FATAL "${str}" >&2 - done - else - log FATAL "$@" >&2 - fi - exit 1 -} - -function success { - if [ -z "$@" ]; then - while read str; do - log SUCCESS "${str}" - done - else - log SUCCESS "$@" - fi -} - -function log { - local args=( $@ ) - local color=COLOR_${args[0]} - echo -e "${!color}[${args[0]}] $(remove_ansi ${args[@]:1})\e[0m" -} - -function remove_ansi { - echo "$@" | sed 's,\x1B\[[0-9;]*[a-zA-Z],,g' -} - -function load_local_opts { - if [ -e "${LOCAL_OPTS_FILE}" ]; then - info "Loading local opts..." - while read line; do - if [[ ! "${line}" =~ ^\s*# ]]; then - set -- $(echo $line | tr '=' ' ') - local key=$1 - local value=$2 - debug "Load opt $key=$value" - set_opt $key $value - fi - done < "${LOCAL_OPTS_FILE}" - fi -} - -function get_opt { - local opt_name=${OPT_PREFIX}${1} - local default_value=${2} - touch "${OPT_FILE}" - source "${OPT_FILE}" - echo ${!opt_name:-${default_value}} -} - -function set_opt { - local opt_name=${1} - local opt_value=${2} - mkdir -p "$(dirname ${OPT_FILE})" - touch "${OPT_FILE}" - sed -i "s/^${OPT_PREFIX}${opt_name}*$//" "${OPT_FILE}" - echo "local ${OPT_PREFIX}${opt_name}=\"${opt_value}\"" >> "${OPT_FILE}" -} - -function exec_hooks { - - local hook=${1} - local workspace=${2} - - local hook_scripts=$( find "${HOOKS_DIR}" -type f -name "*${hook}" -executable | sort ) - - for hook_script in ${hook_scripts}; do - - info "[>> ${hook}] ${hook_script}" - - ( cd "${workspace}" && "${hook_script}" ) 2> >(stderr) 1> >(stdout) - - # If the script did not execute properly, we stop here - if [ $? != 0 ]; then - fatal "The '${hook_script}' hook script did not finished properly !" - fi - - info "[<< ${hook}] ${hook_script}" - - done - -} diff --git a/package b/package index 9214d3d..cec8153 100755 --- a/package +++ b/package @@ -10,11 +10,13 @@ def configure_args_parser(): profile_names = tamarin.get_available_profile_names() - parser = argparse.ArgumentParser(description="Generate packages for various GNU/Linux distribution") + parser = argparse.ArgumentParser(description="Generate packages for various GNU/Linux distributions") # Define available/required arguments and flags - parser.add_argument("project_path", help="The path to your project to package") + parser.add_argument("project_directory", help="The path to your project to package") + parser.add_argument("-o", "--output", help="The path to the generated packages destination directory", default=".") parser.add_argument("-p", "--profile", help="The profile to use to package this project (default: debian)", choices=profile_names, default='debian') + parser.add_argument("-a", "--architecture", help="The target architecture for the package (default: amd64)", default='amd64') parser.add_argument("--rebuild", help="Ignore cache and rebuild container's image", action="store_true", default=False) return parser @@ -24,6 +26,10 @@ if __name__ == "__main__": parser = configure_args_parser() args = parser.parse_args() + # Verify project directory + project_dir = os.path.abspath(args.project_directory) + output_dir = os.path.abspath(args.output) + # Load build profile profile = tamarin.load_profile(args.profile) @@ -86,14 +92,18 @@ if __name__ == "__main__": # Build image tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)]) - tamarin.run_acbuild(acbuild_flags+["copy-to-dir", tamarin.get_hooks_dir(), '/tamarin']) - tamarin.run_acbuild(acbuild_flags+["copy-to-dir", tamarin.get_lib_dir(), '/tamarin']) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "src", "/src", "--read-only"]) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "dist", "/dist"]) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-hooks", "/tamarin/hooks", "--read-only"]) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-lib", "/tamarin/lib", "--read-only"]) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-profiles", "/tamarin/profiles", "--read-only"]) # Execute containerbuild hooks - cpb_hooks_env = os.environ.copy() - cpb_hooks_env["PATH"] = os.environ['PATH'] + ':' + local_acbuild_dir - cpb_hooks_env["TAMARIN_ACBUILD"] = " ".join([system.which('acbuild', local_acbuild_dir)]+acbuild_flags) - tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=cpb_hooks_env) + cb_hooks_env = os.environ.copy() + cb_hooks_env["PATH"] = os.environ['PATH'] + ':' + local_acbuild_dir + cb_hooks_env["TAMARIN_ACBUILD"] = " ".join([system.which('acbuild', local_acbuild_dir)]+acbuild_flags) + cb_hooks_env["TAMARIN_ACBUILD_ENGINE"] = "chroot" if system.which('systemd-nspawn') == "" else "systemd-nspawn" + tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=cb_hooks_env) # Cache image shutil.copyfile(aci_file, cached_image_file) @@ -103,7 +113,13 @@ if __name__ == "__main__": "run", "--insecure-options=image", aci_file, "--net=host", - "--exec", "/usr/bin/python3", "--", "/tamarin/lib/build.py" + "--volume=src,kind=host,source={:s}".format(project_dir), + "--volume=dist,kind=host,source={:s}".format(output_dir), + "--volume=tamarin-hooks,kind=host,source={:s}".format(tamarin.get_hooks_dir()), + "--volume=tamarin-lib,kind=host,source={:s}".format(tamarin.get_lib_dir()), + "--volume=tamarin-profiles,kind=host,source={:s}".format(tamarin.get_profiles_dir()), + "--interactive", "--exec", "/bin/bash" + #"--exec", "/usr/bin/python3", "--", "/tamarin/lib/build.py", args.profile, args.architecture ], as_root=True) # Cleanup diff --git a/package.sh b/package.sh deleted file mode 100755 index a898714..0000000 --- a/package.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env bash - -set -e - -TAMARIN_VERSION=0.0.1 -TAMARIN_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) - -BASE_DIR="$TAMARIN_DIR" source "$TAMARIN_DIR/lib/util.sh" -DEFAULT_64_IMAGE="debian:jessie" -DEFAULT_32_IMAGE="32bit/debian:jessie" - -function show_usage { - echo - echo "Usage: $0 -p project_path [-a arch] [-d destination] [-i image] [-k]" - echo - echo "Parameters: " - echo - echo " -p Path to the project to build" - echo " -v Optional : Show Tamarin version" - echo " -a Optional : Target architecture (default amd64)" - echo " -d Optional : Destination of the builed packages (default ./packages)" - echo " -i Optional : Name of the Docker image to use for build (default: debian:jessie)" - echo " -k Optional : Keep the Docker container after build " - echo " -B Optional : Build branch (for git projects only) (default dist/ubuntu/precise/master)" - echo - exit 2 -} - -# Create a build container based on the $BASE_IMAGE argument -function create_container { - - # Escape image name - local escaped_basename=$(echo "$BASE_IMAGE" | sed 's/[^a-z0-9\-\_\.]/\_/gi') - # Generate container tag - container_tag="tamarin:${escaped_basename}_$(date +%s)" - # Create temporary dir for the Dockerfile - local temp_dir="$(mktemp -d)" - - local projectName=${1} - - # Link lib & hooks folders - ln -s $(readlink -f "$TAMARIN_DIR/lib") "$temp_dir/lib" - ln -s $(readlink -f "$TAMARIN_DIR/hooks") "$temp_dir/hooks" - - # Create Dockerfile - cat << EOF > "$temp_dir/Dockerfile" - FROM ${BASE_IMAGE} - - ENV DEBIAN_FRONTEND noninteractive - - RUN apt-get update &&\ - apt-get install --yes --no-install-recommends build-essential devscripts equivs - - RUN mkdir /root/.tamarin - RUN mkdir /project - - ADD ./lib /root/.tamarin/lib - ADD ./hooks /hooks - RUN chmod +x /root/.tamarin/lib/build.sh - - VOLUME /src - VOLUME /dist - -EOF - - exec_hooks "containerbuild" "$temp_dir" - - echo "CMD /root/.tamarin/lib/build.sh ${projectName} ${BUILD_BRANCH} /tmp ${BUILD_TAG}" >> "$temp_dir/Dockerfile" - - # Build image - tar -C "$temp_dir" -czh . | docker build -t "$container_tag" - 2> >(stderr) 1> >(stdout) - - # Delete temporary folder - rm -rf "$temp_dir" - -} - -# Main function -function main { - - info "Building container from $BASE_IMAGE..." - - local project_name=$(basename ${PROJECT_PATH}) - - # Create container & "$container_tag" variable - create_container ${project_name} - - local docker_opt="run -e \"DISTRIB=$BASE_IMAGE\" -e \"PROJECT_NAME=$project_name\"" - - # Expose host uid & gid - docker_opt="${docker_opt} -e HOST_UID=$(id -u) -e HOST_GID=$(id -g)" - - # Expose host proxy variables - docker_opt="${docker_opt} -e HTTP_PROXY=${HTTP_PROXY} -e HTTPS_PROXY=${HTTPS_PROXY}" - docker_opt="${docker_opt} -e http_proxy=${http_proxy} -e https_proxy=${https_proxy}" - - # Target architecture - docker_opt="${docker_opt} -e TARGET_ARCH=${TARGET_ARCH}" - - # If running in terminal, set docker to interactive - if [[ -t 1 ]]; then - docker_opt="${docker_opt} -it" - fi - - if [[ ${PERSIST_CONTAINER} -eq 0 ]] - then - docker_opt="${docker_opt} --rm " - else - docker_opt="${docker_opt}" - fi - - docker_opt="${docker_opt} -v $PROJECT_PATH:/src:ro -v $PROJECT_DEST:/dist:rw $container_tag" - - info "Switching to container..." - debug "docker ${docker_opt}" - docker ${docker_opt} - res=${?} - - success "Done" - return ${res} -} - -# -# Parsing options -# -while getopts "vkp:d:i:B:t:a:o:" option -do - case $option in - k) - PERSIST_CONTAINER=1 - ;; - p) - PROJECT_PATH=$(readlink -f ${OPTARG}) - ;; - d) - PROJECT_DEST=$(readlink -f ${OPTARG}) - ;; - i) - BASE_IMAGE="${OPTARG}" - ;; - B) - BUILD_BRANCH=${OPTARG} - ;; - t) - BUILD_TAG=${OPTARG} - ;; - a) - TARGET_ARCH=${OPTARG} - ;; - v) - echo "Tamarin v${TAMARIN_VERSION}" - exit - ;; - *) - show_usage - ;; - esac -done - -[[ -z ${PROJECT_PATH} ]] && show_usage -[[ -z ${PROJECT_DEST} ]] && PROJECT_DEST=$(readlink -f "./packages") -[[ -z ${BUILD_BRANCH} ]] && BUILD_BRANCH="dist/ubuntu/precise/master" -[[ -z ${PERSIST_CONTAINER} ]] && PERSIST_CONTAINER=0 -[[ -z ${TARGET_ARCH} ]] && TARGET_ARCH=amd64 - -if [[ "${TARGET_ARCH}" =~ ^i[0-9]86$ ]] && [ -z "${BASE_IMAGE}" ]; then - info "32bit architecture specified and no specific image given, switching to default 32bit image..." - BASE_IMAGE=${DEFAULT_32_IMAGE} -fi - -[[ -z ${BASE_IMAGE} ]] && BASE_IMAGE=${DEFAULT_64_IMAGE} - -# -# Warn user about "proxy" -# - -if [[ -n ${http_proxy} ]] -then - warn "You have a proxy defined please make sure docker deamon is configured to use this proxy" -fi - -main diff --git a/profiles/debian.conf b/profiles/debian.conf index 3a8a572..cc33185 100644 --- a/profiles/debian.conf +++ b/profiles/debian.conf @@ -5,18 +5,22 @@ default_image=docker://debian:jessie # Configuration de l'étape de pré-construction du conteneur [containerbuild] -hooks=containerbuild/debian/install-build-essential,containerbuild/debian/install-git +hooks=containerbuild/debian/install-build-essential, + containerbuild/debian/install-git # Configuration de l'étape de pré-construction du paquet [prebuild] -hooks=05-create-changelog-prebuild,06-create-dummy-changelog-prebuild,07-add-version-suffix-prebuild,10-install-build-depends-prebuild +hooks=prebuild/debian/load-project-db, + prebuild/debian/copy-sources-to-workspace, + prebuild/debian/create-changelog, + prebuild/debian/create-dummy-changelog, + prebuild/debian/add-package-version-suffix, + prebuild/debian/install-build-depends + +# Configuration de l'étape de construction du paquet +[build] +hooks=build/debian/build # Configuration de l'étape de post-construction du paquet [postbuild] -hooks=99-export-dist-postbuild - -[preinstall] -hooks= - -[postinstall] -hooks= +hooks=postbuild/debian/export-dist-postbuild From 15643cf14530ac3da4a2b9d5a6627b94e510b710 Mon Sep 17 00:00:00 2001 From: William Petit Date: Thu, 9 Feb 2017 22:38:04 +0100 Subject: [PATCH 10/20] =?UTF-8?q?Correction=20point=20d'entr=C3=A9e=20cont?= =?UTF-8?q?eneur?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package b/package index cec8153..ff5df1b 100755 --- a/package +++ b/package @@ -118,8 +118,8 @@ if __name__ == "__main__": "--volume=tamarin-hooks,kind=host,source={:s}".format(tamarin.get_hooks_dir()), "--volume=tamarin-lib,kind=host,source={:s}".format(tamarin.get_lib_dir()), "--volume=tamarin-profiles,kind=host,source={:s}".format(tamarin.get_profiles_dir()), - "--interactive", "--exec", "/bin/bash" - #"--exec", "/usr/bin/python3", "--", "/tamarin/lib/build.py", args.profile, args.architecture + #"--interactive", "--exec", "/bin/bash" + "--exec", "/usr/bin/python3", "--", "/tamarin/lib/build.py", args.profile, args.architecture ], as_root=True) # Cleanup From 97df663236cf9e881501d451e78a6b9208f57ee9 Mon Sep 17 00:00:00 2001 From: William Petit Date: Fri, 10 Feb 2017 09:23:10 +0100 Subject: [PATCH 11/20] =?UTF-8?q?Correction=20d=C3=A9tection=20pr=C3=A9sen?= =?UTF-8?q?ce=20de=20systemd?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package b/package index ff5df1b..72472e1 100755 --- a/package +++ b/package @@ -102,7 +102,8 @@ if __name__ == "__main__": cb_hooks_env = os.environ.copy() cb_hooks_env["PATH"] = os.environ['PATH'] + ':' + local_acbuild_dir cb_hooks_env["TAMARIN_ACBUILD"] = " ".join([system.which('acbuild', local_acbuild_dir)]+acbuild_flags) - cb_hooks_env["TAMARIN_ACBUILD_ENGINE"] = "chroot" if system.which('systemd-nspawn') == "" else "systemd-nspawn" + cb_hooks_env["TAMARIN_ACBUILD_ENGINE"] = "chroot" if not system.which('systemctl') else "systemd-nspawn" + tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=cb_hooks_env) # Cache image From 24dbf2fe47e7c5d15cb9adbfa2a94105b8c42c85 Mon Sep 17 00:00:00 2001 From: William Petit Date: Fri, 10 Feb 2017 15:31:42 +0100 Subject: [PATCH 12/20] Exposition des variables http_proxy* dans le conteneur --- package | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/package b/package index 72472e1..b036bee 100755 --- a/package +++ b/package @@ -119,6 +119,10 @@ if __name__ == "__main__": "--volume=tamarin-hooks,kind=host,source={:s}".format(tamarin.get_hooks_dir()), "--volume=tamarin-lib,kind=host,source={:s}".format(tamarin.get_lib_dir()), "--volume=tamarin-profiles,kind=host,source={:s}".format(tamarin.get_profiles_dir()), + "--set-env=HTTP_PROXY={:s}".format(os.environ['HTTP_PROXY']), + "--set-env=HTTPS_PROXY={:s}".format(os.environ['HTTP_PROXY']), + "--set-env=http_proxy={:s}".format(os.environ['http_proxy']), + "--set-env=https_proxy={:s}".format(os.environ['https_proxy']), #"--interactive", "--exec", "/bin/bash" "--exec", "/usr/bin/python3", "--", "/tamarin/lib/build.py", args.profile, args.architecture ], as_root=True) From e58ab60bd82ff1fb0f1af2ef23e1b78fe836e52d Mon Sep 17 00:00:00 2001 From: William Petit Date: Sat, 11 Feb 2017 11:51:01 +0100 Subject: [PATCH 13/20] Refactoring et mise en place des tests --- .gitignore | 1 + Makefile | 2 + package | 153 ++++++++++-------- .../debian/dummy-project/debian/changelog | 5 + test/data/debian/dummy-project/debian/compat | 1 + test/data/debian/dummy-project/debian/control | 13 ++ test/data/debian/dummy-project/debian/rules | 8 + .../debian/dummy-project/debian/source/format | 1 + test/package_test.py | 16 ++ 9 files changed, 137 insertions(+), 63 deletions(-) create mode 100644 Makefile create mode 100644 test/data/debian/dummy-project/debian/changelog create mode 100644 test/data/debian/dummy-project/debian/compat create mode 100644 test/data/debian/dummy-project/debian/control create mode 100644 test/data/debian/dummy-project/debian/rules create mode 100644 test/data/debian/dummy-project/debian/source/format create mode 100644 test/package_test.py diff --git a/.gitignore b/.gitignore index bab4f0f..0cc9317 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ packages *.log *~ __pycache__ +*.pyc diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6f08439 --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +test: + python3 -m unittest discover test "*_test.py" diff --git a/package b/package index b036bee..de6ab43 100755 --- a/package +++ b/package @@ -6,8 +6,8 @@ sys.path.append(os.path.dirname(__file__) + '/lib') import tamarin, system, rkt -def configure_args_parser(): - +def create_args_parser(): + '''Return a new configured ArgumentParser''' profile_names = tamarin.get_available_profile_names() parser = argparse.ArgumentParser(description="Generate packages for various GNU/Linux distributions") @@ -21,11 +21,87 @@ def configure_args_parser(): return parser +def download_and_extract_rkt(dest_dir, verbose=True): + '''Download and extract rkt to the given destination directory''' + rkt_archive_path = tamarin.download_rkt() + system.extract_tar(rkt_archive_path, workspace_tmp) + rkt_archive_dir = tamarin.get_rkt_achive_dest_dir() + shutil.rmtree(local_rkt_dir, ignore_errors=True) + os.rename(rkt_archive_dir, dest_dir) + +def download_and_extract_acbuild(dest_dir, verbose=True): + '''Download and extract acbuild to the given destination directory''' + acbuild_archive_path = tamarin.download_acbuild() + system.extract_tar(acbuild_archive_path, workspace_tmp) + acbuild_archive_dir = tamarin.get_acbuild_achive_dest_dir() + shutil.rmtree(local_acbuild_dir, ignore_errors=True) + os.rename(acbuild_archive_dir, dest_dir) + +def get_cached_image_path(profile): + '''Compute and return the path for an hypothetic cached image for the given profile''' + containerbuild_hooks = profile['containerbuild']['hooks'] + hasher = hashlib.sha1() + hasher.update(base_image.encode()) + hasher.update(containerbuild_hooks.encode()) + image_hash = hasher.hexdigest() + cache_dir = tamarin.get_workspace_subdir('cache') + return os.path.join(os.sep, cache_dir, '{:s}.aci'.format(image_hash[:12])); + +def build_image(build_workspace, aci_file, base_image, profile): + + acbuild_flags = ["--modify", aci_file, "--work-path", build_workspace] + + # Find and export base image from rkt' store + name_pattern = base_image.split('/')[-1] + '$' + image = rkt.find_image_by_name(name_pattern, rkt_flags=rkt_flags) + rkt.export_image(image['id'], aci_file, rkt_flags=rkt_flags); + + # Build image + tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)]) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "src", "/src", "--read-only"]) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "dist", "/dist"]) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-hooks", "/tamarin/hooks", "--read-only"]) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-lib", "/tamarin/lib", "--read-only"]) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-profiles", "/tamarin/profiles", "--read-only"]) + + # Configure "containerbuild" hooks environment + hooks_env = os.environ.copy() + hooks_env["PATH"] = os.environ['PATH'] + ':' + tamarin.get_workspace_subdir('acbuild') + hooks_env["TAMARIN_ACBUILD"] = " ".join([system.which('acbuild', local_acbuild_dir)]+acbuild_flags) + hooks_env["TAMARIN_ACBUILD_ENGINE"] = "chroot" if not system.which('systemctl') else "systemd-nspawn" + + # Run hooks + tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=hooks_env) + + return aci_file + +def cleanup(build_workspace, rkt_flags): + + # Nettoyage des conteneurs + rkt.run([ + "gc", + "--grace-period=0" + ] + rkt_flags, as_root=True) + + # Nettoyage des images obsolètes du store + rkt.run([ + "image", + "gc" + ] + rkt_flags, as_root=True) + + # Suppression de l'espace de travail de build + shutil.rmtree(build_workspace, ignore_errors=True) + +def validate_args(args): + '''TODO''' + if __name__ == "__main__": - parser = configure_args_parser() + parser = create_args_parser() args = parser.parse_args() + validate_args(args) + # Verify project directory project_dir = os.path.abspath(args.project_directory) output_dir = os.path.abspath(args.output) @@ -38,21 +114,11 @@ if __name__ == "__main__": local_rkt_dir = tamarin.get_workspace_subdir('rkt') if not system.which('rkt', local_rkt_dir): - # Download and extract rkt - rkt_archive_path = tamarin.download_rkt() - system.extract_tar(rkt_archive_path, workspace_tmp) - rkt_archive_dir = tamarin.get_rkt_achive_dest_dir() - shutil.rmtree(local_rkt_dir, ignore_errors=True) - os.rename(rkt_archive_dir, local_rkt_dir) + download_and_extract_rkt(local_rkt_dir) local_acbuild_dir = tamarin.get_workspace_subdir('acbuild') if not system.which('acbuild', local_acbuild_dir): - # Download and extract acbuild - acbuild_archive_path = tamarin.download_acbuild() - system.extract_tar(acbuild_archive_path, workspace_tmp) - acbuild_archive_dir = tamarin.get_acbuild_achive_dest_dir() - shutil.rmtree(local_acbuild_dir, ignore_errors=True) - os.rename(acbuild_archive_dir, local_acbuild_dir) + download_and_extract_acbuild(local_acbuild_dir) pid = os.getpid() build_workspace = tamarin.get_workspace_subdir('tmp/build_{:d}'.format(pid)) @@ -61,7 +127,8 @@ if __name__ == "__main__": rkt_flags = ["--dir={:s}".format(rkt_store)] base_image = profile['profile']['default_image'] - # If the base image is Docker-based, preload it and get its name from the store + + # If the base image is Docker-based, download it if base_image.startswith('docker://'): rkt.run([ "fetch", @@ -70,44 +137,15 @@ if __name__ == "__main__": ] + rkt_flags) aci_file = os.path.join(os.sep, build_workspace, 'image.aci') - acbuild_flags = ["--modify", aci_file, "--work-path", build_workspace] - - # Use cached image base on base_image and containerbuild hooks - containerbuild_hooks = profile['containerbuild']['hooks'] - hasher = hashlib.sha1() - hasher.update(base_image.encode()) - hasher.update(containerbuild_hooks.encode()) - image_hash = hasher.hexdigest() - cache_dir = tamarin.get_workspace_subdir('cache') - cached_image_file = os.path.join(os.sep, cache_dir, '{:s}.aci'.format(image_hash[:12])); + cached_image_file = get_cached_image_path(profile) if not args.rebuild and os.path.exists(cached_image_file): - # Copy cached image - shutil.copyfile(cached_image_file, aci_file) + # Copy cached image + shutil.copyfile(cached_image_file, aci_file) else: - # Find and export base image from rkt' store - name_pattern = base_image.split('/')[-1] + '$' - image = rkt.find_image_by_name(name_pattern, rkt_flags=rkt_flags) - rkt.export_image(image['id'], aci_file, rkt_flags=rkt_flags); - - # Build image - tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)]) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "src", "/src", "--read-only"]) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "dist", "/dist"]) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-hooks", "/tamarin/hooks", "--read-only"]) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-lib", "/tamarin/lib", "--read-only"]) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-profiles", "/tamarin/profiles", "--read-only"]) - - # Execute containerbuild hooks - cb_hooks_env = os.environ.copy() - cb_hooks_env["PATH"] = os.environ['PATH'] + ':' + local_acbuild_dir - cb_hooks_env["TAMARIN_ACBUILD"] = " ".join([system.which('acbuild', local_acbuild_dir)]+acbuild_flags) - cb_hooks_env["TAMARIN_ACBUILD_ENGINE"] = "chroot" if not system.which('systemctl') else "systemd-nspawn" - - tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=cb_hooks_env) - - # Cache image - shutil.copyfile(aci_file, cached_image_file) + build_image(build_workspace, aci_file, base_image, profile) + # Cache image + shutil.copyfile(aci_file, cached_image_file) # Start container rkt.run(rkt_flags+[ @@ -128,15 +166,4 @@ if __name__ == "__main__": ], as_root=True) # Cleanup - - rkt.run([ - "gc", - "--grace-period=0" - ] + rkt_flags, as_root=True) - - rkt.run([ - "image", - "gc" - ] + rkt_flags, as_root=True) - - shutil.rmtree(build_workspace, ignore_errors=True) + cleanup(build_workspace, rkt_flags) diff --git a/test/data/debian/dummy-project/debian/changelog b/test/data/debian/dummy-project/debian/changelog new file mode 100644 index 0000000..5c78aa0 --- /dev/null +++ b/test/data/debian/dummy-project/debian/changelog @@ -0,0 +1,5 @@ +dummy-project (0.0.1) unstable; urgency=low + + * Dev release + + -- William Petit Fri, 16 Oct 2015 15:57:03 +0200 diff --git a/test/data/debian/dummy-project/debian/compat b/test/data/debian/dummy-project/debian/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/test/data/debian/dummy-project/debian/compat @@ -0,0 +1 @@ +9 diff --git a/test/data/debian/dummy-project/debian/control b/test/data/debian/dummy-project/debian/control new file mode 100644 index 0000000..a628610 --- /dev/null +++ b/test/data/debian/dummy-project/debian/control @@ -0,0 +1,13 @@ +Source: dummy-project +Section: unknown +Priority: optional +Maintainer: William Petit +Build-Depends: debhelper (>= 8.0.0) +Standards-Version: 3.9.4 +Homepage: +Vcs-Git: https://forge.cadoles.com/wpetit/tamarin.git +Vcs-Browser: https://forge.cadoles.com/wpetit/tamarin + +Package: dummy-project +Architecture: any +Description: Projet test pour la construction de paquets via Tamarin diff --git a/test/data/debian/dummy-project/debian/rules b/test/data/debian/dummy-project/debian/rules new file mode 100644 index 0000000..ecf4506 --- /dev/null +++ b/test/data/debian/dummy-project/debian/rules @@ -0,0 +1,8 @@ +#!/usr/bin/make -f +# -*- makefile -*- + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +%: + dh $@ diff --git a/test/data/debian/dummy-project/debian/source/format b/test/data/debian/dummy-project/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/test/data/debian/dummy-project/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/test/package_test.py b/test/package_test.py new file mode 100644 index 0000000..034c925 --- /dev/null +++ b/test/package_test.py @@ -0,0 +1,16 @@ +import unittest, os +from importlib.machinery import SourceFileLoader + +package = SourceFileLoader('package', os.path.dirname(__file__) + '/../package').load_module() + +class TestPackage(unittest.TestCase): + + def test_args_parser(self): + parser = package.create_args_parser() + + def test_download_and_extract_rkt(self): + + + +if __name__ == '__main__': + unittest.main() From 0bf686d5f3d9a8af121bfea5e0b482d5d2a45dc6 Mon Sep 17 00:00:00 2001 From: William Petit Date: Thu, 16 Feb 2017 10:03:38 +0100 Subject: [PATCH 14/20] Correction execution des tests --- Makefile | 3 ++- test/package_test.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6f08439..3a4edb2 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,3 @@ +.PHONY: test test: - python3 -m unittest discover test "*_test.py" + python3 -m unittest discover "./test" "*_test.py" diff --git a/test/package_test.py b/test/package_test.py index 034c925..d600484 100644 --- a/test/package_test.py +++ b/test/package_test.py @@ -9,7 +9,7 @@ class TestPackage(unittest.TestCase): parser = package.create_args_parser() def test_download_and_extract_rkt(self): - + """TODO""" if __name__ == '__main__': From 9780cafffdfedfdc7b62c0722e36587088fb463a Mon Sep 17 00:00:00 2001 From: William Petit Date: Sat, 18 Feb 2017 11:26:25 +0100 Subject: [PATCH 15/20] Ajout argument debug et base, acbuild sans modification de l'image --- package | 49 +++++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/package b/package index de6ab43..6bc6969 100755 --- a/package +++ b/package @@ -17,7 +17,9 @@ def create_args_parser(): parser.add_argument("-o", "--output", help="The path to the generated packages destination directory", default=".") parser.add_argument("-p", "--profile", help="The profile to use to package this project (default: debian)", choices=profile_names, default='debian') parser.add_argument("-a", "--architecture", help="The target architecture for the package (default: amd64)", default='amd64') + parser.add_argument("-b", "--base", help="Use the specified image instead of the profile's one", default='') parser.add_argument("--rebuild", help="Ignore cache and rebuild container's image", action="store_true", default=False) + parser.add_argument("--debug", help="Will add extra output and start the container in interactive mode", action="store_true", default=False) return parser @@ -49,7 +51,7 @@ def get_cached_image_path(profile): def build_image(build_workspace, aci_file, base_image, profile): - acbuild_flags = ["--modify", aci_file, "--work-path", build_workspace] + acbuild_flags = ["--work-path", build_workspace] # Find and export base image from rkt' store name_pattern = base_image.split('/')[-1] + '$' @@ -57,6 +59,7 @@ def build_image(build_workspace, aci_file, base_image, profile): rkt.export_image(image['id'], aci_file, rkt_flags=rkt_flags); # Build image + tamarin.run_acbuild(acbuild_flags+["begin", aci_file]) tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)]) tamarin.run_acbuild(acbuild_flags+["mount", "add", "src", "/src", "--read-only"]) tamarin.run_acbuild(acbuild_flags+["mount", "add", "dist", "/dist"]) @@ -73,6 +76,9 @@ def build_image(build_workspace, aci_file, base_image, profile): # Run hooks tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=hooks_env) + tamarin.run_acbuild(acbuild_flags+["write", "--overwrite", aci_file]) + tamarin.run_acbuild(acbuild_flags+["end"]) + return aci_file def cleanup(build_workspace, rkt_flags): @@ -126,7 +132,7 @@ if __name__ == "__main__": rkt_store = tamarin.get_workspace_subdir('store') rkt_flags = ["--dir={:s}".format(rkt_store)] - base_image = profile['profile']['default_image'] + base_image = args.base if args.base != '' else profile['profile']['default_image'] # If the base image is Docker-based, download it if base_image.startswith('docker://'): @@ -147,23 +153,30 @@ if __name__ == "__main__": # Cache image shutil.copyfile(aci_file, cached_image_file) + # rkt run arguments + rkt_args = [ + "run", + "--insecure-options=image", + aci_file, "--net=host", + "--volume=src,kind=host,source={:s}".format(project_dir), + "--volume=dist,kind=host,source={:s}".format(output_dir), + "--volume=tamarin-hooks,kind=host,source={:s}".format(tamarin.get_hooks_dir()), + "--volume=tamarin-lib,kind=host,source={:s}".format(tamarin.get_lib_dir()), + "--volume=tamarin-profiles,kind=host,source={:s}".format(tamarin.get_profiles_dir()) + ] + + # Use environment proxy if defined + for proxy_var in ['HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy']: + if proxy_var in os.environ: + rkt_args += ["--set-env={:s}={:s}".format(proxy_var, os.environ[proxy_var])] + + if args.debug: + rkt_args += ["--interactive", "--exec", "/bin/bash"] + else: + rkt_args += ["--exec", "/usr/bin/python3", "--", "/tamarin/lib/build.py", args.profile, args.architecture] + # Start container - rkt.run(rkt_flags+[ - "run", - "--insecure-options=image", - aci_file, "--net=host", - "--volume=src,kind=host,source={:s}".format(project_dir), - "--volume=dist,kind=host,source={:s}".format(output_dir), - "--volume=tamarin-hooks,kind=host,source={:s}".format(tamarin.get_hooks_dir()), - "--volume=tamarin-lib,kind=host,source={:s}".format(tamarin.get_lib_dir()), - "--volume=tamarin-profiles,kind=host,source={:s}".format(tamarin.get_profiles_dir()), - "--set-env=HTTP_PROXY={:s}".format(os.environ['HTTP_PROXY']), - "--set-env=HTTPS_PROXY={:s}".format(os.environ['HTTP_PROXY']), - "--set-env=http_proxy={:s}".format(os.environ['http_proxy']), - "--set-env=https_proxy={:s}".format(os.environ['https_proxy']), - #"--interactive", "--exec", "/bin/bash" - "--exec", "/usr/bin/python3", "--", "/tamarin/lib/build.py", args.profile, args.architecture - ], as_root=True) + rkt.run(rkt_flags+rkt_args, as_root=True) # Cleanup cleanup(build_workspace, rkt_flags) From 448a16f659286736345938bd8b1923fb6336077e Mon Sep 17 00:00:00 2001 From: William Petit Date: Fri, 10 Mar 2017 21:51:57 +0100 Subject: [PATCH 16/20] Base gestion mode debug --- .gitignore | 2 ++ lib/rkt.py | 13 +++++------ lib/system.py | 5 +++-- lib/tamarin.py | 18 +++++++++------- package | 58 ++++++++++++++++++++++++++------------------------ 5 files changed, 52 insertions(+), 44 deletions(-) diff --git a/.gitignore b/.gitignore index 0cc9317..6d68533 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ packages *~ __pycache__ *.pyc +*.changes +*.deb diff --git a/lib/rkt.py b/lib/rkt.py index 60d66fc..37d9fa1 100644 --- a/lib/rkt.py +++ b/lib/rkt.py @@ -1,21 +1,22 @@ import system, subprocess, os, tamarin, json, re -def run(args, as_root = False, capture_output=False): +def run(args, as_root = False, capture_output=False, debug=False): """Run rkt with the specified args (use the local copy if rkt is not found in the $PATH)""" rkt_bin = system.which('rkt', tamarin.get_workspace_subdir('rkt')) cmd = ( ["sudo", "-E", rkt_bin] if os.geteuid() != 0 and as_root == True else [rkt_bin] ) + args - print(" ".join(cmd)) + if debug: + print(" ".join(cmd)) if capture_output: return subprocess.check_output(cmd, stdin=subprocess.PIPE) else: return subprocess.call(cmd, stdin=subprocess.PIPE) -def get_images_list(rkt_flags = []): +def get_images_list(rkt_flags = [], debug=False): output = run([ "image", "list", "--format=json" - ] + rkt_flags, capture_output=True) + ] + rkt_flags, capture_output=True, debug=debug) # Fetch the list of installed images return json.loads(output.decode('utf-8')) @@ -28,10 +29,10 @@ def find_image_by_name(name_pattern, rkt_flags = []): return image return None -def export_image(image_id, dest_file, rkt_flags = []): +def export_image(image_id, dest_file, rkt_flags = [], debug=False): run([ "image", "export", image_id, dest_file, - ] + rkt_flags) + ] + rkt_flags, debug=debug) diff --git a/lib/system.py b/lib/system.py index bbf8357..2c0ab72 100644 --- a/lib/system.py +++ b/lib/system.py @@ -1,7 +1,8 @@ import tarfile, os -def extract_tar(file_path, dest_dir = "."): - print('Extracting "{:s}" to "{:s}"'.format(file_path, dest_dir)) +def extract_tar(file_path, dest_dir = ".", debug=False): + if debug: + print('Extracting "{:s}" to "{:s}"'.format(file_path, dest_dir)) with tarfile.open(file_path) as tar: tar.extractall(dest_dir) tar.close() diff --git a/lib/tamarin.py b/lib/tamarin.py index 164866a..eb66e19 100644 --- a/lib/tamarin.py +++ b/lib/tamarin.py @@ -2,7 +2,7 @@ import os, glob, subprocess, configparser import web, system import codecs -def run_profile_hooks(profile, step, cwd=None, env=None): +def run_profile_hooks(profile, step, cwd=None, env=None, debug=False): hooks_dir = get_hooks_dir() step_hooks = profile[step]["hooks"] if not step_hooks: @@ -22,7 +22,7 @@ def get_hooks_dir(): def get_lib_dir(): return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../lib") -def load_profile(profile_name): +def load_profile(profile_name, debug=False): profile_filename = profile_name+".conf" for profile_file in get_available_profiles(): if profile_filename == os.path.basename(profile_file): @@ -63,25 +63,27 @@ def get_rkt_achive_dest_dir(): workspace_tmp = get_workspace_subdir('tmp') return glob.glob(os.path.join(os.sep, workspace_tmp, 'rkt-v*'))[0] -def download_rkt(): +def download_rkt(debug=False): """Download a local copy of rkt in the tamarin workspace and return the absolute path to the archive""" url = "https://github.com/coreos/rkt/releases/download/v1.22.0/rkt-v1.22.0.tar.gz" file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "rkt.tar.gz") web.download_file(file_url=url, dest_path=file_path) return file_path -def download_acbuild(): +def download_acbuild(debug=False): """Download a local copy of acbuild in the tamarin workspace and return the absolute path to the archive""" url = "https://github.com/containers/build/releases/download/v0.4.0/acbuild-v0.4.0.tar.gz" file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "acbuild.tar.gz") web.download_file(file_url=url, dest_path=file_path) return file_path -def run_acbuild(args, captureOutput=False): +def run_acbuild(args, captureOutput=False, as_root=False, debug=False): """Run acbuild with the specified args (use the local copy if acbuild is not found in the $PATH)""" acbuild_bin = system.which('acbuild', get_workspace_subdir('acbuild')) - print(" ".join([acbuild_bin] + args)) + cmd = ( ["sudo", "-E", acbuild_bin] if os.geteuid() != 0 and as_root == True else [acbuild_bin] ) + args + if debug: + print(" ".join(cmd)) if captureOutput: - return subprocess.check_output([acbuild_bin] + args, stdin=subprocess.PIPE) + return subprocess.check_output(cmd, stdin=subprocess.PIPE) else: - return subprocess.call([acbuild_bin] + args, stdin=subprocess.PIPE) + return subprocess.call(cmd, stdin=subprocess.PIPE) diff --git a/package b/package index 6bc6969..742ff27 100755 --- a/package +++ b/package @@ -23,23 +23,23 @@ def create_args_parser(): return parser -def download_and_extract_rkt(dest_dir, verbose=True): +def download_and_extract_rkt(dest_dir, debug=False): '''Download and extract rkt to the given destination directory''' - rkt_archive_path = tamarin.download_rkt() - system.extract_tar(rkt_archive_path, workspace_tmp) + rkt_archive_path = tamarin.download_rkt(debug=debug) + system.extract_tar(rkt_archive_path, workspace_tmp, debug=debug) rkt_archive_dir = tamarin.get_rkt_achive_dest_dir() shutil.rmtree(local_rkt_dir, ignore_errors=True) os.rename(rkt_archive_dir, dest_dir) -def download_and_extract_acbuild(dest_dir, verbose=True): +def download_and_extract_acbuild(dest_dir, debug=False): '''Download and extract acbuild to the given destination directory''' - acbuild_archive_path = tamarin.download_acbuild() - system.extract_tar(acbuild_archive_path, workspace_tmp) + acbuild_archive_path = tamarin.download_acbuild(debug=debug) + system.extract_tar(acbuild_archive_path, workspace_tmp, debug=debug) acbuild_archive_dir = tamarin.get_acbuild_achive_dest_dir() shutil.rmtree(local_acbuild_dir, ignore_errors=True) os.rename(acbuild_archive_dir, dest_dir) -def get_cached_image_path(profile): +def get_cached_image_path(profile, debug=False): '''Compute and return the path for an hypothetic cached image for the given profile''' containerbuild_hooks = profile['containerbuild']['hooks'] hasher = hashlib.sha1() @@ -49,23 +49,23 @@ def get_cached_image_path(profile): cache_dir = tamarin.get_workspace_subdir('cache') return os.path.join(os.sep, cache_dir, '{:s}.aci'.format(image_hash[:12])); -def build_image(build_workspace, aci_file, base_image, profile): +def build_image(build_workspace, aci_file, base_image, profile, debug=False): acbuild_flags = ["--work-path", build_workspace] # Find and export base image from rkt' store name_pattern = base_image.split('/')[-1] + '$' image = rkt.find_image_by_name(name_pattern, rkt_flags=rkt_flags) - rkt.export_image(image['id'], aci_file, rkt_flags=rkt_flags); + rkt.export_image(image['id'], aci_file, rkt_flags=rkt_flags, debug=debug); # Build image - tamarin.run_acbuild(acbuild_flags+["begin", aci_file]) - tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)]) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "src", "/src", "--read-only"]) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "dist", "/dist"]) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-hooks", "/tamarin/hooks", "--read-only"]) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-lib", "/tamarin/lib", "--read-only"]) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-profiles", "/tamarin/profiles", "--read-only"]) + tamarin.run_acbuild(acbuild_flags+["begin", aci_file], debug=debug) + tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)], debug=debug) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "src", "/src", "--read-only"], debug=debug) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "dist", "/dist"], debug=debug) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-hooks", "/tamarin/hooks", "--read-only"], debug=debug) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-lib", "/tamarin/lib", "--read-only"], debug=debug) + tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-profiles", "/tamarin/profiles", "--read-only"], debug=debug) # Configure "containerbuild" hooks environment hooks_env = os.environ.copy() @@ -74,26 +74,26 @@ def build_image(build_workspace, aci_file, base_image, profile): hooks_env["TAMARIN_ACBUILD_ENGINE"] = "chroot" if not system.which('systemctl') else "systemd-nspawn" # Run hooks - tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=hooks_env) + tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=hooks_env, debug=debug) - tamarin.run_acbuild(acbuild_flags+["write", "--overwrite", aci_file]) - tamarin.run_acbuild(acbuild_flags+["end"]) + tamarin.run_acbuild(acbuild_flags+["write", "--overwrite", aci_file], as_root=True, debug=debug) + tamarin.run_acbuild(acbuild_flags+["end"], as_root=True, debug=debug) return aci_file -def cleanup(build_workspace, rkt_flags): +def cleanup(build_workspace, rkt_flags, debug=False): # Nettoyage des conteneurs rkt.run([ "gc", "--grace-period=0" - ] + rkt_flags, as_root=True) + ] + rkt_flags, as_root=True, debug=debug) # Nettoyage des images obsolètes du store rkt.run([ "image", "gc" - ] + rkt_flags, as_root=True) + ] + rkt_flags, as_root=True, debug=debug) # Suppression de l'espace de travail de build shutil.rmtree(build_workspace, ignore_errors=True) @@ -113,7 +113,7 @@ if __name__ == "__main__": output_dir = os.path.abspath(args.output) # Load build profile - profile = tamarin.load_profile(args.profile) + profile = tamarin.load_profile(args.profile, debug=args.debug) workspace = tamarin.get_workspace_dir() workspace_tmp = tamarin.get_workspace_subdir('tmp') @@ -140,16 +140,16 @@ if __name__ == "__main__": "fetch", "--insecure-options=image", base_image - ] + rkt_flags) + ] + rkt_flags, debug=args.debug) aci_file = os.path.join(os.sep, build_workspace, 'image.aci') - cached_image_file = get_cached_image_path(profile) + cached_image_file = get_cached_image_path(profile, debug=args.debug) if not args.rebuild and os.path.exists(cached_image_file): # Copy cached image shutil.copyfile(cached_image_file, aci_file) else: - build_image(build_workspace, aci_file, base_image, profile) + build_image(build_workspace, aci_file, base_image, profile, debug=args.debug) # Cache image shutil.copyfile(aci_file, cached_image_file) @@ -172,11 +172,13 @@ if __name__ == "__main__": if args.debug: rkt_args += ["--interactive", "--exec", "/bin/bash"] + helper_cmd = " ".join(["/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture]) + print("Executer '{:s}' pour lancer la construction du paquet.".format(helper_cmd)) else: rkt_args += ["--exec", "/usr/bin/python3", "--", "/tamarin/lib/build.py", args.profile, args.architecture] # Start container - rkt.run(rkt_flags+rkt_args, as_root=True) + rkt.run(rkt_flags+rkt_args, as_root=True, debug=args.debug) # Cleanup - cleanup(build_workspace, rkt_flags) + cleanup(build_workspace, rkt_flags, debug=args.debug) From 1d566f8de85808d39f5aae72f5ef9ad382f579ab Mon Sep 17 00:00:00 2001 From: William Petit Date: Fri, 10 Mar 2017 22:01:05 +0100 Subject: [PATCH 17/20] Extraction automatique du nom de projet avant le packaging --- hooks/prebuild/debian/complete-project-db | 13 +++++++++++++ lib/install.sh | 12 ------------ profiles/debian.conf | 1 + 3 files changed, 14 insertions(+), 12 deletions(-) create mode 100755 hooks/prebuild/debian/complete-project-db delete mode 100644 lib/install.sh diff --git a/hooks/prebuild/debian/complete-project-db b/hooks/prebuild/debian/complete-project-db new file mode 100755 index 0000000..c86ef37 --- /dev/null +++ b/hooks/prebuild/debian/complete-project-db @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +cd src + +if [ -z "$(tamarin_db get project_name)" ]; then + if [ ! -d ".git" ]; then + tamarin_error "This project is not managed with Git ! Cannot extract the project's name without it !" + exit 1 + fi + project_name=$(basename $(git config --get remote.origin.url) | sed 's/.git$//' | tr '[:upper:]' '[:lower:]') + tamarin_info "Extracted project name from Git metadata: \"${project_name}\"" + tamarin_db set project_name "${project_name}" +fi diff --git a/lib/install.sh b/lib/install.sh deleted file mode 100644 index 78a5112..0000000 --- a/lib/install.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -source "${DIR}/util.sh" - -info "Updating packages definition..." -apt-get update 2> >(stderr) 1> >(stdout) - -info "Installing package $1..." -gdebi --n "$1" 2> >(stderr) 1> >(stdout) diff --git a/profiles/debian.conf b/profiles/debian.conf index cc33185..f9afe98 100644 --- a/profiles/debian.conf +++ b/profiles/debian.conf @@ -12,6 +12,7 @@ hooks=containerbuild/debian/install-build-essential, [prebuild] hooks=prebuild/debian/load-project-db, prebuild/debian/copy-sources-to-workspace, + prebuild/debian/complete-project-db, prebuild/debian/create-changelog, prebuild/debian/create-dummy-changelog, prebuild/debian/add-package-version-suffix, From 6d5e898cb8184803ca5845e62a9dee971cd6d204 Mon Sep 17 00:00:00 2001 From: William Petit Date: Wed, 15 Mar 2017 18:45:23 +0100 Subject: [PATCH 18/20] Bump default rkt version --- lib/tamarin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tamarin.py b/lib/tamarin.py index eb66e19..95891c1 100644 --- a/lib/tamarin.py +++ b/lib/tamarin.py @@ -65,7 +65,7 @@ def get_rkt_achive_dest_dir(): def download_rkt(debug=False): """Download a local copy of rkt in the tamarin workspace and return the absolute path to the archive""" - url = "https://github.com/coreos/rkt/releases/download/v1.22.0/rkt-v1.22.0.tar.gz" + url = "https://github.com/coreos/rkt/releases/download/v1.25.0/rkt-v1.25.0.tar.gz" file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "rkt.tar.gz") web.download_file(file_url=url, dest_path=file_path) return file_path From d99305182e6c08a7da90bc26c3a1ef69ac44cac5 Mon Sep 17 00:00:00 2001 From: William Petit Date: Mon, 20 Mar 2017 18:35:59 +0100 Subject: [PATCH 19/20] =?UTF-8?q?Arr=C3=AAt=20du=20processus=20en=20cas=20?= =?UTF-8?q?d'erreur?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hooks/containerbuild/debian/install-build-essential | 2 ++ hooks/containerbuild/debian/install-git | 2 ++ lib/rkt.py | 2 +- lib/tamarin.py | 6 ++---- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/hooks/containerbuild/debian/install-build-essential b/hooks/containerbuild/debian/install-build-essential index ed0cbfb..9dcc1de 100755 --- a/hooks/containerbuild/debian/install-build-essential +++ b/hooks/containerbuild/debian/install-build-essential @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -e + ${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive if [ "${TAMARIN_ACBUILD_ENGINE}" == 'chroot' ]; then diff --git a/hooks/containerbuild/debian/install-git b/hooks/containerbuild/debian/install-git index 4e5c7fe..a2a2733 100755 --- a/hooks/containerbuild/debian/install-git +++ b/hooks/containerbuild/debian/install-git @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -e + ${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive sudo -E /usr/bin/env bash - < Date: Tue, 21 Mar 2017 23:05:16 +0100 Subject: [PATCH 20/20] Python wrapper with Docker backend with profiles --- README.md | 16 +- .../debian/install-build-essential | 26 +-- hooks/containerbuild/debian/install-git | 10 +- lib/build.py | 3 +- lib/rkt.py | 38 ----- lib/system.py | 29 ---- lib/tamarin.py | 48 ++---- lib/web.py | 33 ---- package | 151 +++++------------- profiles/debian.conf | 2 +- 10 files changed, 69 insertions(+), 287 deletions(-) delete mode 100644 lib/rkt.py delete mode 100644 lib/system.py delete mode 100644 lib/web.py diff --git a/README.md b/README.md index b823ac8..b27257c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # Tamarin -Usine à paquets expérimentale basée sur rkt/acbuild. +Usine à paquets GNU/Linux + +## Statut + +Expérimental ## Formats de paquets/distributions supportés @@ -9,11 +13,7 @@ Usine à paquets expérimentale basée sur rkt/acbuild. ## Dépendances - [Python 3](https://www.python.org/downloads/) -- Un noyau Linux > 2.6.24 (avec support des cgroups) - -**Optionnel mais conseillé** - -- [systemd](https://freedesktop.org/wiki/Software/systemd/) +- [Docker](>= 17.03) ## Usage @@ -30,10 +30,6 @@ TODO TODO -### Répertoire de travail et mise en cache des images - -TODO - ## Licence GPLv3 diff --git a/hooks/containerbuild/debian/install-build-essential b/hooks/containerbuild/debian/install-build-essential index 9dcc1de..1db0c28 100755 --- a/hooks/containerbuild/debian/install-build-essential +++ b/hooks/containerbuild/debian/install-build-essential @@ -2,26 +2,6 @@ set -e -${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive - -if [ "${TAMARIN_ACBUILD_ENGINE}" == 'chroot' ]; then - # Ugly fix for Python installation in chrooted environment (require /dev/urandom) - head -c 65536 /dev/urandom > ./urandom - ${TAMARIN_ACBUILD} copy ./urandom /dev/urandom -fi - -sudo -E /usr/bin/env bash - <> Dockerfile +echo 'RUN apt-get update && apt-get install --yes --no-install-recommends build-essential devscripts equivs python3' >> Dockerfile +echo 'ENV DEBIAN_FRONTEND=' >> Dockerfile diff --git a/hooks/containerbuild/debian/install-git b/hooks/containerbuild/debian/install-git index a2a2733..12a9d17 100755 --- a/hooks/containerbuild/debian/install-git +++ b/hooks/containerbuild/debian/install-git @@ -2,10 +2,6 @@ set -e -${TAMARIN_ACBUILD} environment add DEBIAN_FRONTEND noninteractive -sudo -E /usr/bin/env bash - <> Dockerfile +echo 'RUN apt-get update && apt-get install --yes --no-install-recommends git-core' >> Dockerfile +echo 'ENV DEBIAN_FRONTEND=' >> Dockerfile diff --git a/lib/build.py b/lib/build.py index 1f2d7de..025012c 100644 --- a/lib/build.py +++ b/lib/build.py @@ -1,6 +1,6 @@ import sys, os, argparse, tempfile sys.path.append(os.path.dirname(__file__) + '/lib') -import tamarin, system, rkt +import tamarin def get_args_parser(): parser = argparse.ArgumentParser(description="Tamarin's container entrypoint") @@ -12,7 +12,6 @@ def get_args_parser(): def get_buildtools_dir(): return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/buildtools") - if __name__ == '__main__': parser = get_args_parser() diff --git a/lib/rkt.py b/lib/rkt.py deleted file mode 100644 index 4e8254b..0000000 --- a/lib/rkt.py +++ /dev/null @@ -1,38 +0,0 @@ -import system, subprocess, os, tamarin, json, re - -def run(args, as_root = False, capture_output=False, debug=False): - """Run rkt with the specified args (use the local copy if rkt is not found in the $PATH)""" - rkt_bin = system.which('rkt', tamarin.get_workspace_subdir('rkt')) - cmd = ( ["sudo", "-E", rkt_bin] if os.geteuid() != 0 and as_root == True else [rkt_bin] ) + args - if debug: - print(" ".join(cmd)) - if capture_output: - return subprocess.check_output(cmd, stdin=subprocess.PIPE) - else: - return subprocess.check_call(cmd, stdin=subprocess.PIPE) - -def get_images_list(rkt_flags = [], debug=False): - output = run([ - "image", - "list", - "--format=json" - ] + rkt_flags, capture_output=True, debug=debug) - # Fetch the list of installed images - return json.loads(output.decode('utf-8')) - -def find_image_by_name(name_pattern, rkt_flags = []): - if type(name_pattern) is str: - name_pattern = re.compile(name_pattern) - images_list = get_images_list(rkt_flags = rkt_flags) - for image in images_list: - if name_pattern.search(image['name']): - return image - return None - -def export_image(image_id, dest_file, rkt_flags = [], debug=False): - run([ - "image", - "export", - image_id, - dest_file, - ] + rkt_flags, debug=debug) diff --git a/lib/system.py b/lib/system.py deleted file mode 100644 index 2c0ab72..0000000 --- a/lib/system.py +++ /dev/null @@ -1,29 +0,0 @@ -import tarfile, os - -def extract_tar(file_path, dest_dir = ".", debug=False): - if debug: - print('Extracting "{:s}" to "{:s}"'.format(file_path, dest_dir)) - with tarfile.open(file_path) as tar: - tar.extractall(dest_dir) - tar.close() - -def which(program, additional_paths = None): - - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - fpath, fname = os.path.split(program) - if fpath: - if is_exe(program): - return program - else: - paths = os.environ["PATH"].split(os.pathsep); - if additional_paths != None: - paths.append(additional_paths) - for path in paths: - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - - return None diff --git a/lib/tamarin.py b/lib/tamarin.py index fcb27cd..de1bfb8 100644 --- a/lib/tamarin.py +++ b/lib/tamarin.py @@ -1,8 +1,6 @@ -import os, glob, subprocess, configparser -import web, system -import codecs +import os, glob, subprocess, configparser, codecs, sys -def run_profile_hooks(profile, step, cwd=None, env=None, debug=False): +def run_profile_hooks(profile, step, **kwargs): hooks_dir = get_hooks_dir() step_hooks = profile[step]["hooks"] if not step_hooks: @@ -12,7 +10,7 @@ def run_profile_hooks(profile, step, cwd=None, env=None, debug=False): if not trimmed_hook_name: continue hook_path = os.path.join(hooks_dir, trimmed_hook_name) - code = subprocess.check_call(hook_path, cwd=cwd, stdin=subprocess.PIPE, env=env) + run([hook_path], **kwargs) def get_hooks_dir(): return os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../hooks") @@ -51,37 +49,17 @@ def get_workspace_subdir(subdir): os.makedirs(dir_path, exist_ok=True) return dir_path -def get_acbuild_achive_dest_dir(): - """Return the first path matching the acbuild archive extraction destination in tamarin workspace""" - workspace_tmp = get_workspace_subdir('tmp') - return glob.glob(os.path.join(os.sep, workspace_tmp, 'acbuild-v*'))[0] - -def get_rkt_achive_dest_dir(): - """Return the first path matching the rkt archive extraction destination in tamarin workspace""" - workspace_tmp = get_workspace_subdir('tmp') - return glob.glob(os.path.join(os.sep, workspace_tmp, 'rkt-v*'))[0] - -def download_rkt(debug=False): - """Download a local copy of rkt in the tamarin workspace and return the absolute path to the archive""" - url = "https://github.com/coreos/rkt/releases/download/v1.25.0/rkt-v1.25.0.tar.gz" - file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "rkt.tar.gz") - web.download_file(file_url=url, dest_path=file_path) - return file_path - -def download_acbuild(debug=False): - """Download a local copy of acbuild in the tamarin workspace and return the absolute path to the archive""" - url = "https://github.com/containers/build/releases/download/v0.4.0/acbuild-v0.4.0.tar.gz" - file_path=os.path.join(os.sep, get_workspace_subdir('tmp'), "acbuild.tar.gz") - web.download_file(file_url=url, dest_path=file_path) - return file_path - -def run_acbuild(args, captureOutput=False, as_root=False, debug=False): - """Run acbuild with the specified args (use the local copy if acbuild is not found in the $PATH)""" - acbuild_bin = system.which('acbuild', get_workspace_subdir('acbuild')) - cmd = ( ["sudo", "-E", acbuild_bin] if os.geteuid() != 0 and as_root == True else [acbuild_bin] ) + args +def run(cmd, captureOutput=False, pty=False, debug=False, **kwargs): + """Execute an arbitrary command on the system""" if debug: print(" ".join(cmd)) + stdin=subprocess.PIPE + if pty: + kwargs['stdin'] = sys.stdin if captureOutput: - return subprocess.check_output(cmd, stdin=subprocess.PIPE) + return subprocess.check_output(cmd, **kwargs) else: - return subprocess.check_call(cmd, stdin=subprocess.PIPE) + return subprocess.check_call(cmd, **kwargs) + +def run_docker(args, captureOutput=False, **kwargs): + return run(["docker"] + args, captureOutput=captureOutput, **kwargs) diff --git a/lib/web.py b/lib/web.py deleted file mode 100644 index d6d41a8..0000000 --- a/lib/web.py +++ /dev/null @@ -1,33 +0,0 @@ -from urllib import request -import math, sys - -def print_progress_bar(percent_progress=0, char_size=50, clear_line=True): - - bar_progress = math.floor(char_size*(percent_progress/100)) - bar = "=" * bar_progress + " " * (char_size - bar_progress) - - if clear_line: - sys.stdout.write(u"\u001b[1000D") - sys.stdout.write("[{:s}] {:d}%".format(bar, int(percent_progress))) - sys.stdout.flush() - -def download_file(file_url, dest_path, bulk_size = 8192): - - req = request.urlopen(file_url) - meta = req.info() - file_size = int(meta.get('Content-Length')) - - with open(dest_path, 'wb') as dest_file: - print('Downloading "{:s}". Size: {:d}b'.format(file_url, file_size)) - downloaded_size = 0 - while True: - buff = req.read(bulk_size) - if not buff: - break - downloaded_size += len(buff) - dest_file.write(buff) - progress = downloaded_size/file_size*100 - print_progress_bar(progress) - dest_file.close() - # Add linebreak - print("") diff --git a/package b/package index 742ff27..4d95fe7 100755 --- a/package +++ b/package @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -import argparse, sys, shutil, os, hashlib +import argparse, sys, shutil, os, subprocess sys.path.append(os.path.dirname(__file__) + '/lib') -import tamarin, system, rkt +import tamarin def create_args_parser(): '''Return a new configured ArgumentParser''' @@ -20,80 +20,37 @@ def create_args_parser(): parser.add_argument("-b", "--base", help="Use the specified image instead of the profile's one", default='') parser.add_argument("--rebuild", help="Ignore cache and rebuild container's image", action="store_true", default=False) parser.add_argument("--debug", help="Will add extra output and start the container in interactive mode", action="store_true", default=False) + parser.add_argument("--cleanup", help="Clear the workspace and remove obsolete Docker images before build", action="store_true", default=False) return parser -def download_and_extract_rkt(dest_dir, debug=False): - '''Download and extract rkt to the given destination directory''' - rkt_archive_path = tamarin.download_rkt(debug=debug) - system.extract_tar(rkt_archive_path, workspace_tmp, debug=debug) - rkt_archive_dir = tamarin.get_rkt_achive_dest_dir() - shutil.rmtree(local_rkt_dir, ignore_errors=True) - os.rename(rkt_archive_dir, dest_dir) +def build_image(build_workspace, base_image, profile_name, profile, debug=False, rebuild=False): -def download_and_extract_acbuild(dest_dir, debug=False): - '''Download and extract acbuild to the given destination directory''' - acbuild_archive_path = tamarin.download_acbuild(debug=debug) - system.extract_tar(acbuild_archive_path, workspace_tmp, debug=debug) - acbuild_archive_dir = tamarin.get_acbuild_achive_dest_dir() - shutil.rmtree(local_acbuild_dir, ignore_errors=True) - os.rename(acbuild_archive_dir, dest_dir) - -def get_cached_image_path(profile, debug=False): - '''Compute and return the path for an hypothetic cached image for the given profile''' - containerbuild_hooks = profile['containerbuild']['hooks'] - hasher = hashlib.sha1() - hasher.update(base_image.encode()) - hasher.update(containerbuild_hooks.encode()) - image_hash = hasher.hexdigest() - cache_dir = tamarin.get_workspace_subdir('cache') - return os.path.join(os.sep, cache_dir, '{:s}.aci'.format(image_hash[:12])); - -def build_image(build_workspace, aci_file, base_image, profile, debug=False): - - acbuild_flags = ["--work-path", build_workspace] - - # Find and export base image from rkt' store - name_pattern = base_image.split('/')[-1] + '$' - image = rkt.find_image_by_name(name_pattern, rkt_flags=rkt_flags) - rkt.export_image(image['id'], aci_file, rkt_flags=rkt_flags, debug=debug); - - # Build image - tamarin.run_acbuild(acbuild_flags+["begin", aci_file], debug=debug) - tamarin.run_acbuild(acbuild_flags+["set-name", "image_{:d}".format(pid)], debug=debug) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "src", "/src", "--read-only"], debug=debug) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "dist", "/dist"], debug=debug) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-hooks", "/tamarin/hooks", "--read-only"], debug=debug) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-lib", "/tamarin/lib", "--read-only"], debug=debug) - tamarin.run_acbuild(acbuild_flags+["mount", "add", "tamarin-profiles", "/tamarin/profiles", "--read-only"], debug=debug) + with open("{:s}/Dockerfile".format(build_workspace), 'w') as dockerfile: + dockerfile.write("FROM {:s}\n".format(base_image)) # Configure "containerbuild" hooks environment hooks_env = os.environ.copy() - hooks_env["PATH"] = os.environ['PATH'] + ':' + tamarin.get_workspace_subdir('acbuild') - hooks_env["TAMARIN_ACBUILD"] = " ".join([system.which('acbuild', local_acbuild_dir)]+acbuild_flags) - hooks_env["TAMARIN_ACBUILD_ENGINE"] = "chroot" if not system.which('systemctl') else "systemd-nspawn" + hooks_env["PATH"] = os.environ['PATH'] + ':' + tamarin.get_lib_dir() # Run hooks tamarin.run_profile_hooks(profile, 'containerbuild', cwd=build_workspace, env=hooks_env, debug=debug) - tamarin.run_acbuild(acbuild_flags+["write", "--overwrite", aci_file], as_root=True, debug=debug) - tamarin.run_acbuild(acbuild_flags+["end"], as_root=True, debug=debug) + image_tag = "tamarin:{:s}_{:s}_{:d}".format(profile_name, base_image.replace(':', '_'), os.getpid()) - return aci_file + build_args = [ "build", "-t", image_tag ] -def cleanup(build_workspace, rkt_flags, debug=False): + if rebuild: + build_args += [ "--no-cache" ] - # Nettoyage des conteneurs - rkt.run([ - "gc", - "--grace-period=0" - ] + rkt_flags, as_root=True, debug=debug) + tamarin.run_docker(build_args + [build_workspace], debug=debug) - # Nettoyage des images obsolètes du store - rkt.run([ - "image", - "gc" - ] + rkt_flags, as_root=True, debug=debug) + return image_tag + +def cleanup(build_workspace=None, debug=False): + + if build_workspace == None: + build_workspace = tamarin.get_workspace_subdir('tmp') # Suppression de l'espace de travail de build shutil.rmtree(build_workspace, ignore_errors=True) @@ -108,6 +65,9 @@ if __name__ == "__main__": validate_args(args) + if args.cleanup: + cleanup(debug=args.debug) + # Verify project directory project_dir = os.path.abspath(args.project_directory) output_dir = os.path.abspath(args.output) @@ -116,69 +76,42 @@ if __name__ == "__main__": profile = tamarin.load_profile(args.profile, debug=args.debug) workspace = tamarin.get_workspace_dir() - workspace_tmp = tamarin.get_workspace_subdir('tmp') - - local_rkt_dir = tamarin.get_workspace_subdir('rkt') - if not system.which('rkt', local_rkt_dir): - download_and_extract_rkt(local_rkt_dir) - - local_acbuild_dir = tamarin.get_workspace_subdir('acbuild') - if not system.which('acbuild', local_acbuild_dir): - download_and_extract_acbuild(local_acbuild_dir) pid = os.getpid() build_workspace = tamarin.get_workspace_subdir('tmp/build_{:d}'.format(pid)) - rkt_store = tamarin.get_workspace_subdir('store') - rkt_flags = ["--dir={:s}".format(rkt_store)] - base_image = args.base if args.base != '' else profile['profile']['default_image'] - # If the base image is Docker-based, download it - if base_image.startswith('docker://'): - rkt.run([ - "fetch", - "--insecure-options=image", - base_image - ] + rkt_flags, debug=args.debug) - - aci_file = os.path.join(os.sep, build_workspace, 'image.aci') - cached_image_file = get_cached_image_path(profile, debug=args.debug) - - if not args.rebuild and os.path.exists(cached_image_file): - # Copy cached image - shutil.copyfile(cached_image_file, aci_file) - else: - build_image(build_workspace, aci_file, base_image, profile, debug=args.debug) - # Cache image - shutil.copyfile(aci_file, cached_image_file) + image_tag = build_image(build_workspace, base_image, args.profile, profile, debug=args.debug, rebuild=args.rebuild) # rkt run arguments - rkt_args = [ - "run", - "--insecure-options=image", - aci_file, "--net=host", - "--volume=src,kind=host,source={:s}".format(project_dir), - "--volume=dist,kind=host,source={:s}".format(output_dir), - "--volume=tamarin-hooks,kind=host,source={:s}".format(tamarin.get_hooks_dir()), - "--volume=tamarin-lib,kind=host,source={:s}".format(tamarin.get_lib_dir()), - "--volume=tamarin-profiles,kind=host,source={:s}".format(tamarin.get_profiles_dir()) + docker_args = [ + "run", + "--rm", + "-v", "{:s}:/src:ro".format(project_dir), + "-v", "{:s}:/dist".format(output_dir), + "-v", "{:s}:/tamarin/hooks:ro".format(tamarin.get_hooks_dir()), + "-v", "{:s}:/tamarin/lib:ro".format(tamarin.get_lib_dir()), + "-v", "{:s}:/tamarin/profiles:ro".format(tamarin.get_profiles_dir()) ] # Use environment proxy if defined for proxy_var in ['HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy']: - if proxy_var in os.environ: - rkt_args += ["--set-env={:s}={:s}".format(proxy_var, os.environ[proxy_var])] + if proxy_var in os.environ: + docker_args += ["-e", "{:s}={:s}".format(proxy_var, os.environ[proxy_var])] + + kwargs = dict() + kwargs['debug'] = args.debug if args.debug: - rkt_args += ["--interactive", "--exec", "/bin/bash"] - helper_cmd = " ".join(["/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture]) - print("Executer '{:s}' pour lancer la construction du paquet.".format(helper_cmd)) + kwargs['pty'] = True + docker_args += ["-it", image_tag, "/bin/sh"] + helper_cmd = " ".join(["/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture]) + print("Executer '{:s}' pour lancer la construction du paquet.".format(helper_cmd)) else: - rkt_args += ["--exec", "/usr/bin/python3", "--", "/tamarin/lib/build.py", args.profile, args.architecture] + docker_args += [image_tag, "/usr/bin/python3", "/tamarin/lib/build.py", args.profile, args.architecture] # Start container - rkt.run(rkt_flags+rkt_args, as_root=True, debug=args.debug) + tamarin.run_docker(docker_args, **kwargs) - # Cleanup - cleanup(build_workspace, rkt_flags, debug=args.debug) + cleanup(build_workspace, debug=args.debug) diff --git a/profiles/debian.conf b/profiles/debian.conf index f9afe98..8fb00a0 100644 --- a/profiles/debian.conf +++ b/profiles/debian.conf @@ -1,7 +1,7 @@ # Configuration générale du profil [profile] # Image Docker par défaut -default_image=docker://debian:jessie +default_image=debian:jessie # Configuration de l'étape de pré-construction du conteneur [containerbuild]