From 0c9d86b8504408969e2c697c9c8cfec06df75646 Mon Sep 17 00:00:00 2001 From: William Petit Date: Mon, 30 Oct 2023 20:05:00 +0100 Subject: [PATCH] feat: add docker recipe + environment with default interpolation in config --- .dockerignore | 14 +++++++ Dockerfile | 58 ++++++++++++++++++++++---- Makefile | 23 ++++++----- doc/README.md | 1 + doc/tutorials/fr/docker-agent.md | 10 +++++ internal/config/environment.go | 62 ++++++++++++++++++---------- internal/config/environment_test.go | 63 +++++++++++++++++++++++++++++ misc/docker/agent.yml | 32 +++++++++++++++ misc/docker/docker-agent-wrapper.sh | 10 +++++ misc/docker/server.yml | 35 ++++++++++++++++ 10 files changed, 271 insertions(+), 37 deletions(-) create mode 100644 .dockerignore create mode 100644 doc/tutorials/fr/docker-agent.md create mode 100644 internal/config/environment_test.go create mode 100644 misc/docker/agent.yml create mode 100644 misc/docker/docker-agent-wrapper.sh create mode 100644 misc/docker/server.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c003a7b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +/apps +/bin +/dist +/out +/tmp +/tools +/.emissary-token +/.env +/agent-key.json +/server-key.json +/CHANGELOG.md +/state.json +/*.sqlite* +/.mktools \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 242d825..780d004 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,12 @@ -FROM golang:1.19 AS BUILD +FROM alpine as certs +RUN apk update && apk add ca-certificates curl openssl bash +RUN curl -k https://forge.cadoles.com/Cadoles/Jenkins/raw/branch/master/resources/com/cadoles/common/add-letsencrypt-ca.sh | bash + +##################################### +# Emissary Server # +##################################### + +FROM golang:1.21 AS build-emissary-server RUN apt-get update \ && apt-get install -y make @@ -7,9 +15,9 @@ COPY . /src WORKDIR /src -RUN make GORELEASER_ARGS='build --rm-dist --single-target --snapshot' release +RUN make mktools && make GORELEASER_ARGS="build --snapshot --clean --single-target --id emissary-server" goreleaser -FROM busybox:latest AS RUNTIME +FROM busybox:latest AS emissary-server ARG DUMB_INIT_VERSION=1.2.5 @@ -19,11 +27,47 @@ RUN mkdir -p /usr/local/bin \ ENTRYPOINT ["/usr/local/bin/dumb-init", "--"] -COPY --from=BUILD /src/dist/emissary_linux_amd64_v1 /app -COPY --from=BUILD /src/tmp/config.yml /etc/emissary/config.yml +COPY --from=build-emissary-server /src/dist/emissary-server_linux_amd64_v1 /app +COPY misc/docker/server.yml /etc/emissary/server.yml +COPY --from=certs /etc/ssl/certs /etc/ssl/certs EXPOSE 3000 -ENTRYPOINT ["/app/emissary"] +RUN mkdir -p /data -CMD ["server", "run", "-c", "/etc/emissary/config.yml"] \ No newline at end of file +CMD [ "/app/emissary", "-c", "/etc/emissary/config.yml", "server", "run"] + +##################################### +# Emissary Agent # +##################################### + +FROM golang:1.21 AS build-emissary-agent + +RUN apt-get update \ + && apt-get install -y make + +COPY . /src + +WORKDIR /src + +RUN make mktools && make GORELEASER_ARGS="build --snapshot --clean --single-target --id emissary-agent" goreleaser + +FROM busybox:latest AS emissary-agent + +ARG DUMB_INIT_VERSION=1.2.5 + +RUN mkdir -p /usr/local/bin \ + && wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMB_INIT_VERSION}/dumb-init_${DUMB_INIT_VERSION}_x86_64 \ + && chmod +x /usr/local/bin/dumb-init + +ENTRYPOINT ["/usr/local/bin/dumb-init", "--"] + +COPY --from=build-emissary-agent /src/dist/emissary-agent_linux_amd64_v1 /app + +COPY --chmod=777 misc/docker/docker-agent-wrapper.sh /usr/local/bin/docker-agent-wrapper +COPY misc/docker/agent.yml /etc/emissary/agent.yml +COPY --from=certs /etc/ssl/certs /etc/ssl/certs + +RUN mkdir -p /data + +CMD [ "/usr/local/bin/docker-agent-wrapper", "/app/emissary", "-c", "/etc/emissary/agent.yml", "agent", "run" ] \ No newline at end of file diff --git a/Makefile b/Makefile index aa7f8e1..5976ce1 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,11 @@ LINT_ARGS ?= --timeout 5m -GORELEASER_VERSION ?= v1.13.1 -GORELEASER_ARGS ?= release --snapshot --rm-dist +GORELEASER_ARGS ?= release --snapshot --rm-dist --help GITCHLOG_ARGS ?= SHELL := /bin/bash EMISSARY_VERSION ?= -DOCKER_IMAGE_NAME ?= bornholm/emissary +DOCKER_IMAGE_NAME ?= reg.cadoles.com/cadoles/emissary DOCKER_IMAGE_TAG ?= $(MKT_PROJECT_VERSION) GOTEST_ARGS ?= -short @@ -71,17 +70,23 @@ dump-config: build-emissary ./bin/emissary config dump > tmp/config.yml .PHONY: goreleaser -goreleaser: .mktools - ( set -o allexport && source .env && set +o allexport && VERSION=$(GORELEASER_VERSION) curl -sfL https://goreleaser.com/static/run | GORELEASER_CURRENT_TAG="$(MKT_PROJECT_VERSION)" bash /dev/stdin $(GORELEASER_ARGS) ) +goreleaser: .env .mktools + ( set -o allexport && source .env && set +o allexport && curl -sfL https://goreleaser.com/static/run | GORELEASER_CURRENT_TAG="$(MKT_PROJECT_VERSION)" bash /dev/stdin $(GORELEASER_ARGS) ) install-git-hooks: git config core.hooksPath .githooks -docker-build: - docker build -t $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) . +docker-build: docker-build-agent docker-build-server -docker-release: - docker push $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) +docker-build-%: + docker build --target emissary-$* -t $(DOCKER_IMAGE_NAME)-$*:latest . + +docker-release: docker-release-agent docker-release-server + +docker-release-%: + docker tag $(DOCKER_IMAGE_NAME)-$*:latest $(DOCKER_IMAGE_NAME)-$*:$(DOCKER_IMAGE_TAG) + docker push $(DOCKER_IMAGE_NAME)-$*:latest + docker push $(DOCKER_IMAGE_NAME)-$*:$(DOCKER_IMAGE_TAG) deploy-openwrt-agent: $(MAKE) GOARCH="arm" GORELEASER_ARGS='build --single-target --snapshot --clean' goreleaser diff --git a/doc/README.md b/doc/README.md index 74c545d..7719b81 100644 --- a/doc/README.md +++ b/doc/README.md @@ -7,6 +7,7 @@ - (FR) - [Premiers pas](./tutorials/fr/first-steps.md) - (FR) - [Déployer un serveur mandataire inverse sur un agent](./tutorials/fr/deploy-reverse-proxy.md) - (FR) - [Déployer une configuration UCI personnalisée sur un agent](./tutorials/fr/deploy-uci-configuration.md) +- (FR) - [Démarrer un agent avec Docker](./tutorials/fr/docker-agent.md) ## References diff --git a/doc/tutorials/fr/docker-agent.md b/doc/tutorials/fr/docker-agent.md new file mode 100644 index 0000000..3bbbafc --- /dev/null +++ b/doc/tutorials/fr/docker-agent.md @@ -0,0 +1,10 @@ +# Lancer un agent avec Docker + +```shell +docker run \ + --rm -it \ + --network bridge \ + -v emissary-agent-data:/data \ + -e EMISSARY_AGENT_SERVER_URL= \ + reg.cadoles.com/cadoles/emissary-agent:latest +``` \ No newline at end of file diff --git a/internal/config/environment.go b/internal/config/environment.go index 82470d8..f4199a0 100644 --- a/internal/config/environment.go +++ b/internal/config/environment.go @@ -10,7 +10,41 @@ import ( "gopkg.in/yaml.v3" ) -var reVar = regexp.MustCompile(`^\${(\w+)}$`) +var ( + interpolationRegExp = regexp.MustCompile(`^\${((?P\w+)|((?P\w+):-(?P[^}]+)))}$`) + varNameGroupIndex = interpolationRegExp.SubexpIndex("varName") + varNameWithDefaultGroupIndex = interpolationRegExp.SubexpIndex("varNameWithDefault") + defaultValueGroupIndex = interpolationRegExp.SubexpIndex("defaultValue") +) + +func interpolate(str string, getValueFunc func(name string) string) string { + for _, match := range interpolationRegExp.FindAllStringSubmatch(str, -1) { + varName := match[varNameWithDefaultGroupIndex] + if varName == "" { + varName = match[varNameGroupIndex] + } + + if varName == "" { + continue + } + + defaultValue := "" + if defaultValueGroupIndex < len(match) { + defaultValue = match[defaultValueGroupIndex] + } + + str = getValueFunc(varName) + if str == "" { + str = defaultValue + } + } + + return str +} + +func interpolateEnv(str string) string { + return interpolate(str, os.Getenv) +} type InterpolatedString string @@ -21,11 +55,7 @@ func (is *InterpolatedString) UnmarshalYAML(value *yaml.Node) error { return errors.WithStack(err) } - if match := reVar.FindStringSubmatch(str); len(match) > 0 { - *is = InterpolatedString(os.Getenv(match[1])) - } else { - *is = InterpolatedString(str) - } + *is = InterpolatedString(interpolateEnv(str)) return nil } @@ -39,9 +69,7 @@ func (ii *InterpolatedInt) UnmarshalYAML(value *yaml.Node) error { return errors.Wrapf(err, "could not decode value '%v' (line '%d') into string", value.Value, value.Line) } - if match := reVar.FindStringSubmatch(str); len(match) > 0 { - str = os.Getenv(match[1]) - } + str = interpolateEnv(str) intVal, err := strconv.ParseInt(str, 10, 32) if err != nil { @@ -62,9 +90,7 @@ func (ib *InterpolatedBool) UnmarshalYAML(value *yaml.Node) error { return errors.Wrapf(err, "could not decode value '%v' (line '%d') into string", value.Value, value.Line) } - if match := reVar.FindStringSubmatch(str); len(match) > 0 { - str = os.Getenv(match[1]) - } + str = interpolateEnv(str) boolVal, err := strconv.ParseBool(str) if err != nil { @@ -91,9 +117,7 @@ func (im *InterpolatedMap) UnmarshalYAML(value *yaml.Node) error { continue } - if match := reVar.FindStringSubmatch(strVal); len(match) > 0 { - strVal = os.Getenv(match[1]) - } + strVal = interpolateEnv(strVal) data[key] = strVal } @@ -113,9 +137,7 @@ func (iss *InterpolatedStringSlice) UnmarshalYAML(value *yaml.Node) error { } for index, value := range data { - if match := reVar.FindStringSubmatch(value); len(match) > 0 { - value = os.Getenv(match[1]) - } + value = interpolateEnv(value) data[index] = value } @@ -134,9 +156,7 @@ func (id *InterpolatedDuration) UnmarshalYAML(value *yaml.Node) error { return errors.Wrapf(err, "could not decode value '%v' (line '%d') into string", value.Value, value.Line) } - if match := reVar.FindStringSubmatch(str); len(match) > 0 { - str = os.Getenv(match[1]) - } + str = interpolateEnv(str) duration, err := time.ParseDuration(str) if err != nil { diff --git a/internal/config/environment_test.go b/internal/config/environment_test.go new file mode 100644 index 0000000..e46bb3f --- /dev/null +++ b/internal/config/environment_test.go @@ -0,0 +1,63 @@ +package config + +import ( + "fmt" + "testing" +) + +type interpolateTestCase struct { + String string + Data map[string]string + Expected string +} + +var interpolateTestCases = []interpolateTestCase{ + { + String: "${foo}", + Data: map[string]string{ + "foo": "bar", + }, + Expected: "bar", + }, + { + String: "${hello:-world}", + Data: map[string]string{}, + Expected: "world", + }, + { + String: "${hello:-}", + Data: map[string]string{}, + Expected: "${hello:-}", + }, + { + String: "foo", + Data: map[string]string{}, + Expected: "foo", + }, + { + String: "", + Data: map[string]string{}, + Expected: "", + }, +} + +func TestInterpolate(t *testing.T) { + for idx, tc := range interpolateTestCases { + func(idx int, tc interpolateTestCase) { + t.Run(fmt.Sprintf("Case_%d", idx), func(t *testing.T) { + result := interpolate(tc.String, func(name string) string { + value, exists := tc.Data[name] + if !exists { + return "" + } + + return value + }) + + if e, g := tc.Expected, result; e != g { + t.Errorf("result: expected '%v', got '%v'", tc.Expected, result) + } + }) + }(idx, tc) + } +} diff --git a/misc/docker/agent.yml b/misc/docker/agent.yml new file mode 100644 index 0000000..1b1cad0 --- /dev/null +++ b/misc/docker/agent.yml @@ -0,0 +1,32 @@ +logger: + level: ${EMISSARY_AGENT_LOGGER_LEVEL:-1} + format: ${EMISSARY_AGENT_LOGGER_FORMAT:-human} +sentry: + dsn: ${EMISSARY_AGENT_SENTRY_DSN} +agent: + serverUrl: ${EMISSARY_AGENT_SERVER_URL:-http://127.0.0.1:3000} + privateKeyPath: ${EMISSARY_AGENT_PRIVATE_KEY_PATH:-/data/agent-key.json} + reconciliationInterval: ${EMISSARY_AGENT_RECONCILIATION_INTERVAL:-30} + controllers: + persistence: + enabled: true + stateFile: ${EMISSARY_AGENT_CONTROLLERS_PERSISTENCE_STATE_FILE:-/data/state.json} + spec: + enabled: true + proxy: + enabled: true + uci: + enabled: false + app: + enabled: true + dataDir: ${EMISSARY_AGENT_CONTROLLERS_APP_DATA_DIR:-/data/apps/data} + downloadDir: ${EMISSARY_AGENT_CONTROLLERS_APP_DOWNLOAD_DIR:-/data/apps/bundles} + sysupgrade: + enabled: false + mdns: + enabled: true + collectors: + - name: uname + command: uname + args: + - -a diff --git a/misc/docker/docker-agent-wrapper.sh b/misc/docker/docker-agent-wrapper.sh new file mode 100644 index 0000000..fc37ab4 --- /dev/null +++ b/misc/docker/docker-agent-wrapper.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -e + +# Generate machine id if not exists +if [ ! -f /etc/machine-id ]; then + cat /proc/sys/kernel/random/uuid > /etc/machine-id +fi + +exec $@ \ No newline at end of file diff --git a/misc/docker/server.yml b/misc/docker/server.yml new file mode 100644 index 0000000..48057f7 --- /dev/null +++ b/misc/docker/server.yml @@ -0,0 +1,35 @@ +logger: + level: ${EMISSARY_SERVER_LOGGER_LEVEL:-1} + format: ${EMISSARY_SERVER_LOGGER_FORMAT:-human} +sentry: + dsn: ${EMISSARY_SERVER_SENTRY_DSN} +server: + http: + host: ${EMISSARY_SERVER_HTTP_HOST:-0.0.0.0} + port: ${EMISSARY_SERVER_HTTP_HOST:-3000} + database: + driver: ${EMISSARY_SERVER_DATABASE_DRIVER:-sqlite} + dsn: ${EMISSARY_SERVER_DATABASE_DSN:-sqlite:///data/emissary.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=150000&_pragma=journal_mode=WAL} + cors: + allowedOrigins: + - ${EMISSARY_SERVER_CORS_ALLOWED_ORIGINS:-http://localhost:3001} + allowCredentials: ${EMISSARY_SERVER_CORS_ALLOW_CREDENTIALS:-true} + allowMethods: + - POST + - GET + - PUT + - DELETE + allowedHeaders: + - Origin + - Accept + - Content-Type + - Authorization + - Sentry-Trace + debug: ${EMISSARY_SERVER_CORS_DEBUG:-false} + auth: + local: + privateKeyPath: ${EMISSARY_SERVER_AUTH_LOCAL_PRIVATE_KEY_PATH:-/data/server-key.json} + remote: + jwksUrl: "${EMISSARY_SERVER_AUTH_REMOTE_JWKS_URL}" + roleExtractionRules: + - "${EMISSARY_SERVER_AUTH_ROLE_EXTRACTION_RULES_0:-jwt.role != nil ? str(jwt.role) : ''}" \ No newline at end of file