feat: update arcad/edge dependency

.dockerignore

@@ -0,0 +1,14 @@
+/apps
+/bin
+/dist
+/out
+/tmp
+/tools
+/.emissary-token
+/.env
+/agent-key.json
+/server-key.json
+/CHANGELOG.md
+/state.json
+/*.sqlite*
+/.mktools

Dockerfile

@@ -1,4 +1,12 @@
-FROM golang:1.19 AS BUILD
+FROM alpine as certs
+RUN apk update && apk add ca-certificates curl openssl bash
+RUN curl -k https://forge.cadoles.com/Cadoles/Jenkins/raw/branch/master/resources/com/cadoles/common/add-letsencrypt-ca.sh | bash
+
+#####################################
+#          Emissary Server          #
+#####################################
+
+FROM golang:1.21 AS build-emissary-server
 
 RUN apt-get update \
     && apt-get install -y make
@@ -7,9 +15,9 @@ COPY . /src
 
 WORKDIR /src
 
-RUN make GORELEASER_ARGS='build --rm-dist --single-target --snapshot' release
+RUN make mktools && make GORELEASER_ARGS="build --snapshot --clean --single-target --id emissary-server" goreleaser
 
-FROM busybox:latest AS RUNTIME
+FROM busybox:latest AS emissary-server
 
 ARG DUMB_INIT_VERSION=1.2.5
 
@@ -19,11 +27,47 @@ RUN mkdir -p /usr/local/bin \
 
 ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
 
-COPY --from=BUILD /src/dist/emissary_linux_amd64_v1 /app
+COPY --from=build-emissary-server /src/dist/emissary-server_linux_amd64_v1 /app
-COPY --from=BUILD /src/tmp/config.yml /etc/emissary/config.yml
+COPY misc/docker/server.yml /etc/emissary/server.yml
+COPY --from=certs /etc/ssl/certs /etc/ssl/certs
 
 EXPOSE 3000
 
-ENTRYPOINT ["/app/emissary"]
+RUN mkdir -p /data
 
-CMD ["server", "run", "-c", "/etc/emissary/config.yml"]
+CMD [ "/app/emissary", "-c", "/etc/emissary/config.yml", "server", "run"]
+
+#####################################
+#           Emissary Agent          #
+#####################################
+
+FROM golang:1.21 AS build-emissary-agent
+
+RUN apt-get update \
+    && apt-get install -y make
+
+COPY . /src
+
+WORKDIR /src
+
+RUN make mktools && make GORELEASER_ARGS="build --snapshot --clean --single-target --id emissary-agent" goreleaser
+
+FROM busybox:latest AS emissary-agent
+
+ARG DUMB_INIT_VERSION=1.2.5
+
+RUN mkdir -p /usr/local/bin \
+    && wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMB_INIT_VERSION}/dumb-init_${DUMB_INIT_VERSION}_x86_64 \
+    && chmod +x /usr/local/bin/dumb-init
+
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
+
+COPY --from=build-emissary-agent /src/dist/emissary-agent_linux_amd64_v1 /app
+
+COPY --chmod=777 misc/docker/docker-agent-wrapper.sh /usr/local/bin/docker-agent-wrapper
+COPY misc/docker/agent.yml /etc/emissary/agent.yml
+COPY --from=certs /etc/ssl/certs /etc/ssl/certs
+
+RUN mkdir -p /data
+
+CMD [ "/usr/local/bin/docker-agent-wrapper", "/app/emissary", "-c", "/etc/emissary/agent.yml", "agent", "run" ]

Makefile

@@ -1,12 +1,11 @@
 LINT_ARGS ?= --timeout 5m
-GORELEASER_VERSION ?= v1.13.1
+GORELEASER_ARGS ?= release --snapshot --clean
-GORELEASER_ARGS ?= release --snapshot --rm-dist
 GITCHLOG_ARGS ?=
 SHELL := /bin/bash
 
 EMISSARY_VERSION ?= 
 
-DOCKER_IMAGE_NAME ?= bornholm/emissary
+DOCKER_IMAGE_NAME ?= reg.cadoles.com/cadoles/emissary
 DOCKER_IMAGE_TAG ?= $(MKT_PROJECT_VERSION)
 
 GOTEST_ARGS ?= -short
@@ -71,17 +70,23 @@ dump-config: build-emissary
 	./bin/emissary config dump > tmp/config.yml
 
 .PHONY: goreleaser
-goreleaser: .mktools
+goreleaser: .env .mktools
-	( set -o allexport && source .env && set +o allexport && VERSION=$(GORELEASER_VERSION) curl -sfL https://goreleaser.com/static/run | GORELEASER_CURRENT_TAG="$(MKT_PROJECT_VERSION)" bash /dev/stdin $(GORELEASER_ARGS) )
+	( set -o allexport && source .env && set +o allexport && curl -sfL https://goreleaser.com/static/run | GORELEASER_CURRENT_TAG="$(MKT_PROJECT_VERSION)" bash /dev/stdin $(GORELEASER_ARGS) )
 
 install-git-hooks:
 	git config core.hooksPath .githooks
 
-docker-build:
+docker-build: docker-build-agent docker-build-server
-	docker build -t $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) .
 
-docker-release:
+docker-build-%:
-	docker push $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)
+	docker build --target emissary-$* -t $(DOCKER_IMAGE_NAME)-$*:latest .
+
+docker-release: docker-release-agent docker-release-server
+
+docker-release-%:
+	docker tag $(DOCKER_IMAGE_NAME)-$*:latest $(DOCKER_IMAGE_NAME)-$*:$(DOCKER_IMAGE_TAG)
+	docker push $(DOCKER_IMAGE_NAME)-$*:latest
+	docker push $(DOCKER_IMAGE_NAME)-$*:$(DOCKER_IMAGE_TAG)
 
 deploy-openwrt-agent:
 	$(MAKE) GOARCH="arm" GORELEASER_ARGS='build --single-target --snapshot --clean' goreleaser
@@ -133,7 +138,8 @@ version: .mktools
 
 update-edge-lib:
 	git pull --rebase
-	GOPROXY=direct GOPRIVATE=forge.cadoles.com/arcad/edge go get -u forge.cadoles.com/arcad/edge
+	go clean -modcache
+	GONOPROXY=forge.cadoles.com/arcad/edge GOPRIVATE=forge.cadoles.com/arcad/edge go get -v -u forge.cadoles.com/arcad/edge
 	go mod tidy
 	$(MAKE) test
 	git add go.mod go.sum

doc/README.md

@@ -7,6 +7,7 @@
 - (FR) - [Premiers pas](./tutorials/fr/first-steps.md)
 - (FR) - [Déployer un serveur mandataire inverse sur un agent](./tutorials/fr/deploy-reverse-proxy.md)
 - (FR) - [Déployer une configuration UCI personnalisée sur un agent](./tutorials/fr/deploy-uci-configuration.md)
+- (FR) - [Démarrer un agent avec Docker](./tutorials/fr/docker-agent.md)
 
 ## References
 

doc/tutorials/fr/docker-agent.md

@@ -0,0 +1,10 @@
+# Lancer un agent avec Docker
+
+```shell
+docker run \
+    --rm -it \
+    --network bridge \
+    -v emissary-agent-data:/data \
+    -e EMISSARY_AGENT_SERVER_URL=<server_url> \
+    reg.cadoles.com/cadoles/emissary-agent:latest
+```

go.mod

@@ -5,7 +5,7 @@ go 1.21
 toolchain go1.21.2
 
 require (
-	forge.cadoles.com/arcad/edge v0.0.0-20231021193834-de4ab0d02cc1
+	forge.cadoles.com/arcad/edge v0.0.0-20231201112253-83a1e8966559
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/alecthomas/participle/v2 v2.0.0-beta.5
 	github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883
@@ -17,7 +17,6 @@ require (
 	github.com/dop251/goja v0.0.0-20230304130813-e2f543bf4b4c
 	github.com/evanphx/json-patch/v5 v5.6.0
 	github.com/getsentry/sentry-go v0.25.0
-	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/go-chi/cors v1.2.1
 	github.com/golang-migrate/migrate/v4 v4.15.2
 	github.com/jackc/pgx/v5 v5.3.1
@@ -37,6 +36,7 @@ require (
 require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
+	github.com/allegro/bigcache/v3 v3.1.0 // indirect
 	github.com/barnybug/go-cast v0.0.0-20201201064555-a87ccbc26692 // indirect
 	github.com/dop251/goja_nodejs v0.0.0-20230320130059-dcf93ba651dd // indirect
 	github.com/gabriel-vasile/mimetype v1.4.1 // indirect
@@ -82,7 +82,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
-	github.com/jackc/puddle/v2 v2.2.0 // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/leodido/go-urn v1.2.2 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.1 // indirect

go.sum

@@ -54,8 +54,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-forge.cadoles.com/arcad/edge v0.0.0-20231021193834-de4ab0d02cc1 h1:aV+xJtYz9bfXMuX09rVyyl9mUeESLZkSyr59vPrY7vA=
+forge.cadoles.com/arcad/edge v0.0.0-20231201112253-83a1e8966559 h1:w44upL8eAJdCzS2+Sx0WIRT2QaAq8gotRwwL9j9z3yQ=
-forge.cadoles.com/arcad/edge v0.0.0-20231021193834-de4ab0d02cc1/go.mod h1:8AYyWhcvG1to3Ig+WcG3TGSs1pp7qZwsXK7tG3Py3Es=
+forge.cadoles.com/arcad/edge v0.0.0-20231201112253-83a1e8966559/go.mod h1:tMysrBPkzPFZAi89qD4TioDjcP3L2hadBJ5Eoij8GcA=
 gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg=
 github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
@@ -148,6 +148,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
 github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk=
+github.com/allegro/bigcache/v3 v3.1.0 h1:H2Vp8VOvxcrB91o86fUSVJFqeuz8kpyyB02eH3bSzwk=
+github.com/allegro/bigcache/v3 v3.1.0/go.mod h1:aPyh7jEvrog9zAwx5N7+JUQX5dZTSGpxF1LAR4dr35I=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -511,8 +513,6 @@ github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
-github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
-github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
 github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0=
 github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
@@ -847,8 +847,8 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f
 github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
-github.com/jackc/puddle/v2 v2.2.0 h1:RdcDk92EJBuBS55nQMMYFXTxwstHug4jkhT5pq8VxPk=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
-github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jedib0t/go-pretty/v6 v6.4.4 h1:N+gz6UngBPF4M288kiMURPHELDMIhF/Em35aYuKrsSc=
 github.com/jedib0t/go-pretty/v6 v6.4.4/go.mod h1:MgmISkTWDSFu0xOqiZ0mKNntMQ2mDgOcwOkwBEkMDJI=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
@@ -1389,6 +1389,8 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=

internal/agent/controller/app/app_handler.go

@@ -16,8 +16,10 @@ import (
 	appModule "forge.cadoles.com/arcad/edge/pkg/module/app"
 	"forge.cadoles.com/arcad/edge/pkg/module/blob"
 	"forge.cadoles.com/arcad/edge/pkg/module/cast"
+	"forge.cadoles.com/arcad/edge/pkg/module/fetch"
 	fetchModule "forge.cadoles.com/arcad/edge/pkg/module/fetch"
 	netModule "forge.cadoles.com/arcad/edge/pkg/module/net"
+	"forge.cadoles.com/arcad/edge/pkg/module/rpc"
 	shareModule "forge.cadoles.com/arcad/edge/pkg/module/share"
 	"forge.cadoles.com/arcad/edge/pkg/storage"
 	"forge.cadoles.com/arcad/edge/pkg/storage/driver"
@@ -29,6 +31,7 @@ import (
 	"gitlab.com/wpetit/goweb/logger"
 
 	// Register storage drivers
+	_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/cache"
 	_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc"
 	_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
 )
@@ -85,7 +88,12 @@ func (c *Controller) getHandlerOptions(ctx context.Context, appKey string, specs
 		mounts = append(mounts, authMount)
 	}
 
-	mounts = append(mounts, appModule.Mount(c.appRepository))
+	mounts = append(
+		mounts,
+		appModule.Mount(c.appRepository),
+		blob.Mount(10<<(10*2)), // 10Mb
+		fetch.Mount(),
+	)
 
 	deps := Dependencies{
 		Bus:           memory.NewBus(),
@@ -299,7 +307,7 @@ func (c *Controller) getAppModules(deps Dependencies) []app.ServerModuleFactory
 		cast.CastModuleFactory(),
 		module.LifecycleModuleFactory(),
 		netModule.ModuleFactory(deps.Bus),
-		module.RPCModuleFactory(deps.Bus),
+		rpc.ModuleFactory(deps.Bus),
 		module.StoreModuleFactory(deps.DocumentStore),
 		blob.ModuleFactory(deps.Bus, deps.BlobStore),
 		authModuleFactory(deps.KeySet),

internal/agent/controller/app/server.go

@@ -14,8 +14,8 @@ import (
 	"gitlab.com/wpetit/goweb/logger"
 
 	"forge.cadoles.com/arcad/edge/pkg/bundle"
-	"github.com/go-chi/chi/middleware"
 	"github.com/go-chi/chi/v5"
+	"github.com/go-chi/chi/v5/middleware"
 	"github.com/pkg/errors"
 
 	_ "forge.cadoles.com/Cadoles/emissary/internal/imports/passwd"

internal/config/database.go

@@ -15,6 +15,6 @@ type DatabaseConfig struct {
 func NewDefaultDatabaseConfig() DatabaseConfig {
 	return DatabaseConfig{
 		Driver: "sqlite",
-		DSN:    "sqlite://emissary.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
+		DSN:    "sqlite://emissary.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=150000&_pragma=journal_mode=WAL",
 	}
 }

internal/config/environment.go

@@ -10,7 +10,41 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-var reVar = regexp.MustCompile(`^\${(\w+)}$`)
+var (
+	interpolationRegExp          = regexp.MustCompile(`^\${((?P<varName>\w+)|((?P<varNameWithDefault>\w+):-(?P<defaultValue>[^}]+)))}$`)
+	varNameGroupIndex            = interpolationRegExp.SubexpIndex("varName")
+	varNameWithDefaultGroupIndex = interpolationRegExp.SubexpIndex("varNameWithDefault")
+	defaultValueGroupIndex       = interpolationRegExp.SubexpIndex("defaultValue")
+)
+
+func interpolate(str string, getValueFunc func(name string) string) string {
+	for _, match := range interpolationRegExp.FindAllStringSubmatch(str, -1) {
+		varName := match[varNameWithDefaultGroupIndex]
+		if varName == "" {
+			varName = match[varNameGroupIndex]
+		}
+
+		if varName == "" {
+			continue
+		}
+
+		defaultValue := ""
+		if defaultValueGroupIndex < len(match) {
+			defaultValue = match[defaultValueGroupIndex]
+		}
+
+		str = getValueFunc(varName)
+		if str == "" {
+			str = defaultValue
+		}
+	}
+
+	return str
+}
+
+func interpolateEnv(str string) string {
+	return interpolate(str, os.Getenv)
+}
 
 type InterpolatedString string
 
@@ -21,11 +55,7 @@ func (is *InterpolatedString) UnmarshalYAML(value *yaml.Node) error {
 		return errors.WithStack(err)
 	}
 
-	if match := reVar.FindStringSubmatch(str); len(match) > 0 {
+	*is = InterpolatedString(interpolateEnv(str))
-		*is = InterpolatedString(os.Getenv(match[1]))
-	} else {
-		*is = InterpolatedString(str)
-	}
 
 	return nil
 }
@@ -39,9 +69,7 @@ func (ii *InterpolatedInt) UnmarshalYAML(value *yaml.Node) error {
 		return errors.Wrapf(err, "could not decode value '%v' (line '%d') into string", value.Value, value.Line)
 	}
 
-	if match := reVar.FindStringSubmatch(str); len(match) > 0 {
+	str = interpolateEnv(str)
-		str = os.Getenv(match[1])
-	}
 
 	intVal, err := strconv.ParseInt(str, 10, 32)
 	if err != nil {
@@ -62,9 +90,7 @@ func (ib *InterpolatedBool) UnmarshalYAML(value *yaml.Node) error {
 		return errors.Wrapf(err, "could not decode value '%v' (line '%d') into string", value.Value, value.Line)
 	}
 
-	if match := reVar.FindStringSubmatch(str); len(match) > 0 {
+	str = interpolateEnv(str)
-		str = os.Getenv(match[1])
-	}
 
 	boolVal, err := strconv.ParseBool(str)
 	if err != nil {
@@ -91,9 +117,7 @@ func (im *InterpolatedMap) UnmarshalYAML(value *yaml.Node) error {
 			continue
 		}
 
-		if match := reVar.FindStringSubmatch(strVal); len(match) > 0 {
+		strVal = interpolateEnv(strVal)
-			strVal = os.Getenv(match[1])
-		}
 
 		data[key] = strVal
 	}
@@ -113,9 +137,7 @@ func (iss *InterpolatedStringSlice) UnmarshalYAML(value *yaml.Node) error {
 	}
 
 	for index, value := range data {
-		if match := reVar.FindStringSubmatch(value); len(match) > 0 {
+		value = interpolateEnv(value)
-			value = os.Getenv(match[1])
-		}
 
 		data[index] = value
 	}
@@ -134,9 +156,7 @@ func (id *InterpolatedDuration) UnmarshalYAML(value *yaml.Node) error {
 		return errors.Wrapf(err, "could not decode value '%v' (line '%d') into string", value.Value, value.Line)
 	}
 
-	if match := reVar.FindStringSubmatch(str); len(match) > 0 {
+	str = interpolateEnv(str)
-		str = os.Getenv(match[1])
-	}
 
 	duration, err := time.ParseDuration(str)
 	if err != nil {

internal/config/environment_test.go

@@ -0,0 +1,63 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+)
+
+type interpolateTestCase struct {
+	String   string
+	Data     map[string]string
+	Expected string
+}
+
+var interpolateTestCases = []interpolateTestCase{
+	{
+		String: "${foo}",
+		Data: map[string]string{
+			"foo": "bar",
+		},
+		Expected: "bar",
+	},
+	{
+		String:   "${hello:-world}",
+		Data:     map[string]string{},
+		Expected: "world",
+	},
+	{
+		String:   "${hello:-}",
+		Data:     map[string]string{},
+		Expected: "${hello:-}",
+	},
+	{
+		String:   "foo",
+		Data:     map[string]string{},
+		Expected: "foo",
+	},
+	{
+		String:   "",
+		Data:     map[string]string{},
+		Expected: "",
+	},
+}
+
+func TestInterpolate(t *testing.T) {
+	for idx, tc := range interpolateTestCases {
+		func(idx int, tc interpolateTestCase) {
+			t.Run(fmt.Sprintf("Case_%d", idx), func(t *testing.T) {
+				result := interpolate(tc.String, func(name string) string {
+					value, exists := tc.Data[name]
+					if !exists {
+						return ""
+					}
+
+					return value
+				})
+
+				if e, g := tc.Expected, result; e != g {
+					t.Errorf("result: expected '%v', got '%v'", tc.Expected, result)
+				}
+			})
+		}(idx, tc)
+	}
+}

internal/datastore/sqlite/agent_repository.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"strings"
 	"time"
 
 	"forge.cadoles.com/Cadoles/emissary/internal/datastore"
@@ -15,12 +16,13 @@ import (
 )
 
 type AgentRepository struct {
-	db *sql.DB
+	db                         *sql.DB
+	sqliteBusyRetryMaxAttempts int
 }
 
 // DeleteSpec implements datastore.AgentRepository.
 func (r *AgentRepository) DeleteSpec(ctx context.Context, agentID datastore.AgentID, name string) error {
-	err := r.withTx(ctx, func(tx *sql.Tx) error {
+	err := r.withTxRetry(ctx, func(tx *sql.Tx) error {
 		exists, err := r.agentExists(ctx, tx, agentID)
 		if err != nil {
 			return errors.WithStack(err)
@@ -49,7 +51,7 @@ func (r *AgentRepository) DeleteSpec(ctx context.Context, agentID datastore.Agen
 func (r *AgentRepository) GetSpecs(ctx context.Context, agentID datastore.AgentID) ([]*datastore.Spec, error) {
 	specs := make([]*datastore.Spec, 0)
 
-	err := r.withTx(ctx, func(tx *sql.Tx) error {
+	err := r.withTxRetry(ctx, func(tx *sql.Tx) error {
 		exists, err := r.agentExists(ctx, tx, agentID)
 		if err != nil {
 			return errors.WithStack(err)
@@ -108,7 +110,7 @@ func (r *AgentRepository) GetSpecs(ctx context.Context, agentID datastore.AgentI
 func (r *AgentRepository) UpdateSpec(ctx context.Context, agentID datastore.AgentID, name string, revision int, data map[string]any) (*datastore.Spec, error) {
 	spec := &datastore.Spec{}
 
-	err := r.withTx(ctx, func(tx *sql.Tx) error {
+	err := r.withTxRetry(ctx, func(tx *sql.Tx) error {
 		exists, err := r.agentExists(ctx, tx, agentID)
 		if err != nil {
 			return errors.WithStack(err)
@@ -167,7 +169,7 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
 	agents := make([]*datastore.Agent, 0)
 	count := 0
 
-	err := r.withTx(ctx, func(tx *sql.Tx) error {
+	err := r.withTxRetry(ctx, func(tx *sql.Tx) error {
 		query := `SELECT id, label, thumbprint, status, contacted_at, created_at, updated_at FROM agents`
 
 		limit := 10
@@ -220,7 +222,7 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
 
 		logger.Debug(ctx, "executing query", logger.F("query", query), logger.F("args", args))
 
-		rows, err := r.db.QueryContext(ctx, query, args...)
+		rows, err := tx.QueryContext(ctx, query, args...)
 		if err != nil {
 			return errors.WithStack(err)
 		}
@@ -272,7 +274,7 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
 func (r *AgentRepository) Create(ctx context.Context, thumbprint string, keySet jwk.Set, metadata map[string]any) (*datastore.Agent, error) {
 	agent := &datastore.Agent{}
 
-	err := r.withTx(ctx, func(tx *sql.Tx) error {
+	err := r.withTxRetry(ctx, func(tx *sql.Tx) error {
 		query := `SELECT count(id) FROM agents WHERE thumbprint = $1`
 		row := tx.QueryRowContext(ctx, query, thumbprint)
 
@@ -331,15 +333,15 @@ func (r *AgentRepository) Create(ctx context.Context, thumbprint string, keySet
 
 // Delete implements datastore.AgentRepository
 func (r *AgentRepository) Delete(ctx context.Context, id datastore.AgentID) error {
-	err := r.withTx(ctx, func(tx *sql.Tx) error {
+	err := r.withTxRetry(ctx, func(tx *sql.Tx) error {
 		query := `DELETE FROM agents WHERE id = $1`
-		_, err := r.db.ExecContext(ctx, query, id)
+		_, err := tx.ExecContext(ctx, query, id)
 		if err != nil {
 			return errors.WithStack(err)
 		}
 
 		query = `DELETE FROM specs WHERE agent_id = $1`
-		_, err = r.db.ExecContext(ctx, query, id)
+		_, err = tx.ExecContext(ctx, query, id)
 		if err != nil {
 			return errors.WithStack(err)
 		}
@@ -359,14 +361,14 @@ func (r *AgentRepository) Get(ctx context.Context, id datastore.AgentID) (*datas
 		ID: id,
 	}
 
-	err := r.withTx(ctx, func(tx *sql.Tx) error {
+	err := r.withTxRetry(ctx, func(tx *sql.Tx) error {
 		query := `
 		SELECT "id", "label", "thumbprint", "keyset", "metadata", "status", "contacted_at", "created_at", "updated_at"
 		FROM agents
 		WHERE id = $1
 		`
 
-		row := r.db.QueryRowContext(ctx, query, id)
+		row := tx.QueryRowContext(ctx, query, id)
 
 		metadata := JSONMap{}
 		contactedAt := sql.NullTime{}
@@ -410,7 +412,7 @@ func (r *AgentRepository) Update(ctx context.Context, id datastore.AgentID, opts
 
 	agent := &datastore.Agent{}
 
-	err := r.withTx(ctx, func(tx *sql.Tx) error {
+	err := r.withTxRetry(ctx, func(tx *sql.Tx) error {
 		query := `
 			UPDATE agents SET id = $1
 		`
@@ -534,6 +536,59 @@ func (r *AgentRepository) agentExists(ctx context.Context, tx *sql.Tx, agentID d
 	return true, nil
 }
 
+func (r *AgentRepository) withTxRetry(ctx context.Context, fn func(*sql.Tx) error) error {
+	attempts := 0
+	max := r.sqliteBusyRetryMaxAttempts
+
+	ctx = logger.With(ctx, logger.F("max", max))
+
+	var err error
+	for {
+		ctx = logger.With(ctx)
+
+		if attempts >= max {
+			logger.Debug(ctx, "transaction retrying failed", logger.F("attempts", attempts))
+
+			return errors.Wrapf(err, "transaction failed after %d attempts", max)
+		}
+
+		err = r.withTx(ctx, fn)
+		if err != nil {
+			if !strings.Contains(err.Error(), "(5) (SQLITE_BUSY)") {
+				return errors.WithStack(err)
+			}
+
+			err = errors.WithStack(err)
+
+			logger.Warn(ctx, "database is busy", logger.E(err))
+
+			wait := time.Duration(8<<(attempts+1)) * time.Millisecond
+
+			logger.Debug(
+				ctx, "database is busy, waiting before retrying transaction",
+				logger.F("wait", wait.String()),
+				logger.F("attempts", attempts),
+			)
+
+			timer := time.NewTimer(wait)
+			select {
+			case <-timer.C:
+				attempts++
+				continue
+
+			case <-ctx.Done():
+				if err := ctx.Err(); err != nil {
+					return errors.WithStack(err)
+				}
+
+				return nil
+			}
+		}
+
+		return nil
+	}
+}
+
 func (r *AgentRepository) withTx(ctx context.Context, fn func(*sql.Tx) error) error {
 	tx, err := r.db.BeginTx(ctx, nil)
 	if err != nil {
@@ -562,8 +617,8 @@ func (r *AgentRepository) withTx(ctx context.Context, fn func(*sql.Tx) error) er
 	return nil
 }
 
-func NewAgentRepository(db *sql.DB) *AgentRepository {
+func NewAgentRepository(db *sql.DB, sqliteBusyRetryMaxAttempts int) *AgentRepository {
-	return &AgentRepository{db}
+	return &AgentRepository{db, sqliteBusyRetryMaxAttempts}
 }
 
 var _ datastore.AgentRepository = &AgentRepository{}

internal/datastore/sqlite/agent_repository_test.go

@@ -40,7 +40,7 @@ func TestSQLiteAgentRepository(t *testing.T) {
 		t.Fatalf("%+v", errors.WithStack(err))
 	}
 
-	repo := NewAgentRepository(db)
+	repo := NewAgentRepository(db, 5)
 
 	testsuite.TestAgentRepository(t, repo)
 }

internal/server/agent_api.go

@@ -9,7 +9,7 @@ import (
 	"forge.cadoles.com/Cadoles/emissary/internal/agent/metadata"
 	"forge.cadoles.com/Cadoles/emissary/internal/datastore"
 	"forge.cadoles.com/Cadoles/emissary/internal/jwk"
-	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/v5"
 	"github.com/pkg/errors"
 	"gitlab.com/wpetit/goweb/api"
 	"gitlab.com/wpetit/goweb/logger"

internal/server/server.go

@@ -18,8 +18,8 @@ import (
 	"forge.cadoles.com/Cadoles/emissary/internal/jwk"
 	"github.com/antonmedv/expr"
 	"github.com/antonmedv/expr/vm"
-	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/v5"
-	"github.com/go-chi/chi/middleware"
+	"github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/cors"
 	"github.com/lestrrat-go/jwx/v2/jwa"
 	"github.com/lestrrat-go/jwx/v2/jwt"

internal/server/spec_api.go

@@ -6,7 +6,7 @@ import (
 
 	"forge.cadoles.com/Cadoles/emissary/internal/datastore"
 	"forge.cadoles.com/Cadoles/emissary/internal/spec"
-	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/v5"
 	"github.com/pkg/errors"
 	"gitlab.com/wpetit/goweb/api"
 	"gitlab.com/wpetit/goweb/logger"

internal/setup/repository.go

@@ -59,7 +59,7 @@ func NewAgentRepository(ctx context.Context, conf config.DatabaseConfig) (datast
 			return nil, errors.WithStack(err)
 		}
 
-		agentRepository = sqlite.NewAgentRepository(db)
+		agentRepository = sqlite.NewAgentRepository(db, 5)
 
 	default:
 		return nil, errors.Errorf("unsupported database driver '%s'", driver)

misc/docker/agent.yml

@@ -0,0 +1,32 @@
+logger:
+    level: ${EMISSARY_AGENT_LOGGER_LEVEL:-1}
+    format: ${EMISSARY_AGENT_LOGGER_FORMAT:-human}
+sentry:
+    dsn: ${EMISSARY_AGENT_SENTRY_DSN}
+agent:
+    serverUrl: ${EMISSARY_AGENT_SERVER_URL:-http://127.0.0.1:3000}
+    privateKeyPath: ${EMISSARY_AGENT_PRIVATE_KEY_PATH:-/data/agent-key.json}
+    reconciliationInterval: ${EMISSARY_AGENT_RECONCILIATION_INTERVAL:-30}
+    controllers:
+        persistence:
+            enabled: true
+            stateFile: ${EMISSARY_AGENT_CONTROLLERS_PERSISTENCE_STATE_FILE:-/data/state.json}
+        spec:
+            enabled: true
+        proxy:
+            enabled: true
+        uci:
+            enabled: false
+        app:
+            enabled: true
+            dataDir: ${EMISSARY_AGENT_CONTROLLERS_APP_DATA_DIR:-/data/apps/data}
+            downloadDir: ${EMISSARY_AGENT_CONTROLLERS_APP_DOWNLOAD_DIR:-/data/apps/bundles}
+        sysupgrade:
+            enabled: false
+        mdns:
+            enabled: true
+    collectors:
+        - name: uname
+          command: uname
+          args:
+            - -a

misc/docker/docker-agent-wrapper.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+# Generate machine id if not exists
+if [ ! -f /etc/machine-id ]; then
+    cat /proc/sys/kernel/random/uuid > /etc/machine-id
+fi
+
+exec $@

misc/docker/server.yml

@@ -0,0 +1,35 @@
+logger:
+    level: ${EMISSARY_SERVER_LOGGER_LEVEL:-1}
+    format: ${EMISSARY_SERVER_LOGGER_FORMAT:-human}
+sentry:
+    dsn: ${EMISSARY_SERVER_SENTRY_DSN}
+server:
+    http:
+        host: ${EMISSARY_SERVER_HTTP_HOST:-0.0.0.0}
+        port: ${EMISSARY_SERVER_HTTP_HOST:-3000}
+    database:
+        driver: ${EMISSARY_SERVER_DATABASE_DRIVER:-sqlite}
+        dsn: ${EMISSARY_SERVER_DATABASE_DSN:-sqlite:///data/emissary.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=150000&_pragma=journal_mode=WAL}
+    cors:
+        allowedOrigins:
+            - ${EMISSARY_SERVER_CORS_ALLOWED_ORIGINS:-http://localhost:3001}
+        allowCredentials: ${EMISSARY_SERVER_CORS_ALLOW_CREDENTIALS:-true}
+        allowMethods:
+            - POST
+            - GET
+            - PUT
+            - DELETE
+        allowedHeaders:
+            - Origin
+            - Accept
+            - Content-Type
+            - Authorization
+            - Sentry-Trace
+        debug: ${EMISSARY_SERVER_CORS_DEBUG:-false}
+    auth:
+        local:
+            privateKeyPath: ${EMISSARY_SERVER_AUTH_LOCAL_PRIVATE_KEY_PATH:-/data/server-key.json}
+        remote:
+            jwksUrl: "${EMISSARY_SERVER_AUTH_REMOTE_JWKS_URL}"
+        roleExtractionRules:
+            - "${EMISSARY_SERVER_AUTH_ROLE_EXTRACTION_RULES_0:-jwt.role != nil ? str(jwt.role) : ''}"

misc/packaging/openrc/emissary-agent.logrotate.conf

@@ -1,6 +1,8 @@
 /var/log/emissary/agent.log {
 	missingok
 	sharedscripts
+	compress
+	rotate 7
 	postrotate
 		/etc/init.d/emissary-agent restart
 	endscript

misc/packaging/openrc/emissary-server.logrotate.conf

@@ -1,6 +1,8 @@
 /var/log/emissary/server.log {
 	missingok
 	sharedscripts
+	compress
+	rotate 7
 	postrotate
 		/etc/init.d/emissary-server restart
 	endscript

misc/spec-samples/app.emissary.cadoles.com.json

@@ -12,8 +12,8 @@
             }
         },
         "edge.sdk.client.test": {
-            "url": "https://emissary.cadol.es/files/apps/edge.sdk.client.test_2023.10.11-stable.943.4d064de.zip",
+            "url": "https://emissary.cadol.es/files/apps/edge.sdk.client.test_2023.10.22-stable.847.22a3326.zip",
-            "sha256sum": "5419054e09342f5b9834771c087385b0fb425ebdbca3d91aa69eab98d909ca57",
+            "sha256sum": "37aa6c4af57b647dc5cae4ec7d36f4289d99c1f090e533997e0d59188a0a04f4",
             "address": ":8084",
             "format": "zip",
             "storage": {