Merge pull request 'feat: bootstrap default proxy and layers from configuration' (#17) from proxy-bootstrap into develop

Reviewed-on: #17

.goreleaser.yaml

@@ -1,148 +1,143 @@
 project_name: bouncer
 before:
   hooks:
-  - go mod tidy
+    - go mod tidy
-  - go generate ./...
+    - go generate ./...
 builds:
-- id: bouncer
+  - id: bouncer
-  env:
+    env:
-  - CGO_ENABLED=0
+      - CGO_ENABLED=0
-  ldflags:
+    ldflags:
-  - -s
+      - -s
-  - -w
+      - -w
-  - -X 'main.GitRef={{ .Commit }}'
+      - -X 'main.GitRef={{ .Commit }}'
-  - -X 'main.ProjectVersion={{ .Version }}'
+      - -X 'main.ProjectVersion={{ .Version }}'
-  - -X 'main.BuildDate={{ .Date }}'
+      - -X 'main.BuildDate={{ .Date }}'
-  - -X 'main.DefaultConfigPath=/etc/bouncer/config.yml'
+      - -X 'main.DefaultConfigPath=/etc/bouncer/config.yml'
-  gcflags:
+    gcflags:
-  - -trimpath="${PWD}"
+      - -trimpath="${PWD}"
-  asmflags:
+    asmflags:
-  - -trimpath="${PWD}"
+      - -trimpath="${PWD}"
-  goos:
+    goos:
-  - linux
+      - linux
-  goarch:
+    goarch:
-  - amd64
+      - amd64
-  - arm64
+      - arm64
-  - "386"
+      - "386"
-  main: ./cmd/bouncer
+    main: ./cmd/bouncer
 archives:
-- id: bouncer
+  - id: bouncer
-  builds: ["bouncer"]
+    builds: ["bouncer"]
-  name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
+    name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
-  files:
+    files:
-  - README.md
+      - README.md
-  - misc/packaging/common/config.yml
+      - misc/packaging/common/config.yml
 checksum:
-  name_template: 'checksums.txt'
+  name_template: "checksums.txt"
 snapshot:
   name_template: "{{ .Version }}"
 changelog:
   sort: asc
   filters:
     exclude:
-    - '^docs:'
+      - "^docs:"
-    - '^test:'
+      - "^test:"
 nfpms:
-- id: bouncer-bin
+  - id: bouncer-bin
-  builds:
+    builds:
-  - "bouncer"
+      - "bouncer"
-  package_name: bouncer-bin
+    package_name: bouncer-bin
-  homepage: https://forge.cadoles.com/Cadoles/bouncer
+    homepage: https://forge.cadoles.com/Cadoles/bouncer
-  maintainer: Cadoles <contact@cadoles.com>
+    maintainer: Cadoles <contact@cadoles.com>
-  description: |-
+    description: |-
-    reverse proxy server with dynamic queuing management - binaries
+      reverse proxy server with dynamic queuing management - binaries
-  license: AGPL-3.0
+    license: AGPL-3.0
-  formats:
+    formats:
-  - apk
+      - apk
-  - deb
+      - deb
-  - rpm
+      - rpm
-  - archlinux
+    contents:
-  contents:
+      - src: misc/packaging/common/config.yml
-  - src: misc/packaging/common/config.yml
+        dst: /etc/bouncer/config.yml
-    dst: /etc/bouncer/config.yml
+        type: config
-    type: config
+      - src: layers
-  - src: layers
+        dst: /etc/bouncer/layers
-    dst: /etc/bouncer/layers
+        type: config
-    type: config
+      - dst: /etc/bouncer/bootstrap.d
-- id: bouncer-admin
+        type: dir
-  meta: true
+        file_info:
-  package_name: bouncer-admin
+          mode: 0700
-  homepage: https://forge.cadoles.com/Cadoles/bouncer
+  - id: bouncer-admin
-  maintainer: Cadoles <contact@cadoles.com>
+    meta: true
-  dependencies:
+    package_name: bouncer-admin
-  - bouncer-bin
+    homepage: https://forge.cadoles.com/Cadoles/bouncer
-  description: |-
+    maintainer: Cadoles <contact@cadoles.com>
-    reverse proxy server with dynamic queuing management - administration service
+    dependencies:
-  license: AGPL-3.0
+      - bouncer-bin
-  formats:
+    description: |-
-  - apk
+      reverse proxy server with dynamic queuing management - administration service
-  - deb
+    license: AGPL-3.0
-  - rpm
+    formats:
-  - archlinux
+      - apk
-  contents:
+      - deb
-  - src: misc/packaging/systemd/bouncer-admin.systemd.service
+      - rpm
-    dst: /usr/lib/systemd/system/bouncer-admin.service
+    contents:
-    packager: deb
+      - src: misc/packaging/systemd/bouncer-admin.systemd.service
-  - src: misc/packaging/systemd/bouncer-admin.systemd.service
+        dst: /usr/lib/systemd/system/bouncer-admin.service
-    dst: /usr/lib/systemd/system/bouncer-admin.service
+        packager: deb
-    packager: rpm
+      - src: misc/packaging/systemd/bouncer-admin.systemd.service
-  - src: misc/packaging/systemd/bouncer-admin.systemd.service
+        dst: /usr/lib/systemd/system/bouncer-admin.service
-    dst: /usr/lib/systemd/system/bouncer-admin.service
+        packager: rpm
-    packager: archlinux
+      - src: misc/packaging/openrc/bouncer-admin.openrc.sh
-  - src: misc/packaging/openrc/bouncer-admin.openrc.sh
+        dst: /etc/init.d/bouncer-admin
-    dst: /etc/init.d/bouncer-admin
+        file_info:
-    file_info:
+          mode: 0755
-      mode: 0755
+        packager: apk
-    packager: apk
+      - dst: /usr/share/bouncer
-  - dst: /usr/share/bouncer
+        type: dir
-    type: dir
+        file_info:
-    file_info:
+          mode: 0700
-      mode: 0700
+      - dst: /var/log/bouncer
-  - dst: /var/log/bouncer
+        type: dir
-    type: dir
+        file_info:
-    file_info:
+          mode: 0700
-      mode: 0700
+        packager: apk
-    packager: apk
+    scripts:
-  scripts:
+      postinstall: "misc/packaging/common/postinstall-bouncer-admin.sh"
-    postinstall: "misc/packaging/common/postinstall-bouncer-admin.sh"
+  - id: bouncer-proxy
-- id: bouncer-proxy
+    meta: true
-  meta: true
+    dependencies:
-  dependencies:
+      - bouncer-bin
-  - bouncer-bin
+    package_name: bouncer-proxy
-  package_name: bouncer-proxy
+    homepage: https://forge.cadoles.com/Cadoles/bouncer
-  homepage: https://forge.cadoles.com/Cadoles/bouncer
+    maintainer: Cadoles <contact@cadoles.com>
-  maintainer: Cadoles <contact@cadoles.com>
+    description: |-
-  description: |-
+      reverse proxy server with dynamic queuing management - proxy service
-    reverse proxy server with dynamic queuing management - proxy service
+    license: AGPL-3.0
-  license: AGPL-3.0
+    formats:
-  formats:
+      - apk
-  - apk
+      - deb
-  - deb
+      - rpm
-  - rpm
+    contents:
-  - archlinux
+      - src: misc/packaging/systemd/bouncer-proxy.systemd.service
-  contents:
+        dst: /usr/lib/systemd/system/bouncer-proxy.service
-  - src: misc/packaging/systemd/bouncer-proxy.systemd.service
+        packager: deb
-    dst: /usr/lib/systemd/system/bouncer-proxy.service
+      - src: misc/packaging/systemd/bouncer-proxy.systemd.service
-    packager: deb
+        dst: /usr/lib/systemd/system/bouncer-proxy.service
-  - src: misc/packaging/systemd/bouncer-proxy.systemd.service
+        packager: rpm
-    dst: /usr/lib/systemd/system/bouncer-proxy.service
+      - src: misc/packaging/openrc/bouncer-proxy.openrc.sh
-    packager: rpm
+        dst: /etc/init.d/bouncer-proxy
-  - src: misc/packaging/systemd/bouncer-proxy.systemd.service
+        file_info:
-    dst: /usr/lib/systemd/system/bouncer-proxy.service
+          mode: 0755
-    packager: archlinux
+        packager: apk
-  - src: misc/packaging/openrc/bouncer-proxy.openrc.sh
+      - dst: /usr/share/bouncer
-    dst: /etc/init.d/bouncer-proxy
+        type: dir
-    file_info:
+        file_info:
-      mode: 0755
+          mode: 0700
-    packager: apk
+      - dst: /var/log/bouncer
-  - dst: /usr/share/bouncer
+        type: dir
-    type: dir
+        file_info:
-    file_info:
+          mode: 0700
-      mode: 0700
+        packager: apk
-  - dst: /var/log/bouncer
+    scripts:
-    type: dir
+      postinstall: "misc/packaging/common/postinstall-bouncer-proxy.sh"
-    file_info:
-      mode: 0700
-    packager: apk
-  scripts:
-    postinstall: "misc/packaging/common/postinstall-bouncer-proxy.sh"

Dockerfile

@@ -9,10 +9,15 @@ RUN mkdir -p /usr/local/bin \
     && wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64 \
     && chmod +x /usr/local/bin/yq
 
-COPY . /src
-
 WORKDIR /src
 
+COPY go.mod .
+COPY go.sum .
+
+RUN go mod download
+
+COPY . /src
+
 RUN make GORELEASER_ARGS='build --rm-dist --single-target --snapshot' goreleaser
 
 # Patch config

doc/README.md

@@ -19,6 +19,7 @@
 ### Utilisation
 
 - [(FR) - Ajouter un layer de type "file d'attente"](./fr/tutorials/add-queue-layer.md)
+- [(FR) - Amorçage d'un serveur Bouncer via la configuration](./fr/tutorials/bootstrapping.md)
 
 ### Développement
 

doc/fr/tutorials/bootstrapping.md

@@ -0,0 +1,47 @@
+# Amorçage d'un serveur Bouncer via la configuration
+
+Il est possible d'amorcer des données par défaut (i.e. des "proxies" et "layers" associés) via la configuration du serveur d'administration.
+
+> **Attention** Ce mécanisme de modifiera pas des proxies déjà existants dans la base de données du serveur Bouncer. Autrement dit, si un proxy est déjà pré-existant lors du démarrage du serveur Bouncer, il ne sera pas modifié.
+
+La définition des proxies et layers par défaut s'effectue dans la section `bootstrap` du fichier de configuration. Deux possibilités pour définir les proxys à charger par défaut:
+
+- Utiliser un répertoire contenant des fichiers YAML (un par proxy) en définissant le chemin du répertoire via l'attribut `bootstrap.dir`;
+- Définir directement la liste des proxies via l'attribut `bootstrap.proxies`.
+
+```yaml
+# Configuration d'une série de proxy/layers
+# à créer par défaut par le serveur d'administration
+bootstrap:
+  # Répertoire contenant les définitions de proxy à créer
+  # par défaut. Les fichiers seront récupérés si ils
+  # correspondent au patron de nommage suivant:
+  #
+  # <bootstrap_dir>/<proxy_name>.yml
+  #
+  # Voir ci-dessous pour les attributs possibles dans les fichiers.
+  #
+  # Si l'attribut est vide ou absent le chargement des fichiers
+  # est désactivé.
+  dir: /etc/bouncer/bootstrap.d
+
+  # Tableau associatif de définition de proxies à créer par
+  # défaut par le serveur d'administration.
+  # Si `proxies` et `dir` sont tous les deux définis, les fichiers
+  # présents dans le répertoire `dir` surchargeront les valeurs définies
+  # dans `proxies`.
+  #
+  # Par défaut vide.
+  proxies:
+  #     my-proxy:
+  #         enabled: true                       # Activer/désactiver le proxy
+  #         from: ["*"]                         # Filtre d'origine d'activation du proxy
+  #         to: "https://example.net"           # Destination du proxy
+  #         weight: 0                           # Priorité du proxy
+  #         layers:                             # Layers associés au proxy
+  #             my-layer:
+  #                 type: queue                 # Type du proxy
+  #                 enabled: false              # Activer/désactiver le layer
+  #                 weight: 0                   # Priorité du layer
+  #                 options: {"capacity": 100}  # Options associées au layer
+```

go.mod

@@ -5,6 +5,7 @@ go 1.20
 require (
 	forge.cadoles.com/Cadoles/go-proxy v0.0.0-20230701194111-c6b3d482cca6
 	github.com/Masterminds/sprig/v3 v3.2.3
+	github.com/bsm/redislock v0.9.4
 	github.com/btcsuite/btcd/btcutil v1.1.3
 	github.com/drone/envsubst v1.0.3
 	github.com/getsentry/sentry-go v0.22.0

go.sum

@@ -84,6 +84,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
 github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y=
+github.com/bsm/redislock v0.9.4 h1:X/Wse1DPpiQgHbVYRE9zv6m070UcKoOGekgvpNhiSvw=
+github.com/bsm/redislock v0.9.4/go.mod h1:Epf7AJLiSFwLCiZcfi6pWFO/8eAYrYpQXFxEDPoDeAk=
 github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
 github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M=
 github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY=

internal/admin/init.go

@@ -2,12 +2,22 @@ package admin
 
 import (
 	"context"
+	"time"
 
+	"forge.cadoles.com/cadoles/bouncer/internal/config"
+	"forge.cadoles.com/cadoles/bouncer/internal/schema"
 	"forge.cadoles.com/cadoles/bouncer/internal/setup"
+	"forge.cadoles.com/cadoles/bouncer/internal/store"
+	"github.com/bsm/redislock"
 	"github.com/pkg/errors"
+	"gitlab.com/wpetit/goweb/logger"
 )
 
 func (s *Server) initRepositories(ctx context.Context) error {
+	if err := s.initRedisClient(ctx); err != nil {
+		return errors.WithStack(err)
+	}
+
 	if err := s.initLayerRepository(ctx); err != nil {
 		return errors.WithStack(err)
 	}
@@ -19,8 +29,16 @@ func (s *Server) initRepositories(ctx context.Context) error {
 	return nil
 }
 
+func (s *Server) initRedisClient(ctx context.Context) error {
+	client := setup.NewRedisClient(ctx, s.redisConfig)
+
+	s.redisClient = client
+
+	return nil
+}
+
 func (s *Server) initLayerRepository(ctx context.Context) error {
-	layerRepository, err := setup.NewLayerRepository(ctx, s.redisConfig)
+	layerRepository, err := setup.NewLayerRepository(ctx, s.redisClient)
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -31,7 +49,7 @@ func (s *Server) initLayerRepository(ctx context.Context) error {
 }
 
 func (s *Server) initProxyRepository(ctx context.Context) error {
-	proxyRepository, err := setup.NewProxyRepository(ctx, s.redisConfig)
+	proxyRepository, err := setup.NewProxyRepository(ctx, s.redisClient)
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -40,3 +58,112 @@ func (s *Server) initProxyRepository(ctx context.Context) error {
 
 	return nil
 }
+
+const bootstrapLockKey = "bouncer-bootstrap"
+
+func (s *Server) bootstrapProxies(ctx context.Context) error {
+	if err := s.validateBootstrap(ctx); err != nil {
+		return errors.Wrap(err, "could not validate bootstrapped proxies")
+	}
+
+	proxyRepo := s.proxyRepository
+	layerRepo := s.layerRepository
+
+	locker := redislock.New(s.redisClient)
+
+	backoff := redislock.ExponentialBackoff(time.Second, time.Duration(s.bootstrapConfig.LockTimeout)*2)
+
+	logger.Debug(ctx, "acquiring proxies bootstrap lock", logger.F("lockTimeout", s.bootstrapConfig.LockTimeout))
+
+	lock, err := locker.Obtain(ctx, bootstrapLockKey, time.Duration(s.bootstrapConfig.LockTimeout), &redislock.Options{
+		RetryStrategy: backoff,
+	})
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	defer func() {
+		if err := lock.Release(ctx); err != nil {
+			logger.Error(ctx, "could not release lock", logger.E(errors.WithStack(err)))
+		}
+	}()
+
+	logger.Info(ctx, "bootstrapping proxies")
+
+	for proxyName, proxyConfig := range s.bootstrapConfig.Proxies {
+		_, err := s.proxyRepository.GetProxy(ctx, proxyName)
+		if !errors.Is(err, store.ErrNotFound) {
+			if err != nil {
+				return errors.WithStack(err)
+			}
+
+			logger.Info(ctx, "ignoring existing proxy", logger.F("proxyName", proxyName))
+			continue
+		}
+
+		logger.Info(ctx, "creating proxy", logger.F("proxyName", proxyName))
+
+		if _, err := proxyRepo.CreateProxy(ctx, proxyName, string(proxyConfig.To), proxyConfig.From...); err != nil {
+			return errors.WithStack(err)
+		}
+
+		_, err = proxyRepo.UpdateProxy(
+			ctx, proxyName,
+			store.WithProxyUpdateEnabled(bool(proxyConfig.Enabled)),
+			store.WithProxyUpdateWeight(int(proxyConfig.Weight)),
+		)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+
+		for layerName, layerConfig := range proxyConfig.Layers {
+			layerType := store.LayerType(layerConfig.Type)
+			layerOptions := store.LayerOptions(layerConfig.Options)
+
+			if _, err := layerRepo.CreateLayer(ctx, proxyName, layerName, layerType, layerOptions); err != nil {
+				return errors.WithStack(err)
+			}
+
+			_, err := layerRepo.UpdateLayer(
+				ctx,
+				proxyName, layerName,
+				store.WithLayerUpdateEnabled(bool(layerConfig.Enabled)),
+				store.WithLayerUpdateOptions(layerOptions),
+				store.WithLayerUpdateWeight(int(layerConfig.Weight)),
+			)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+		}
+	}
+
+	return nil
+}
+
+const validateErrMessage = "could not validate proxy '%s': could not validate layer '%s'"
+
+func (s *Server) validateBootstrap(ctx context.Context) error {
+	for proxyName, proxyConf := range s.bootstrapConfig.Proxies {
+		for layerName, layerConf := range proxyConf.Layers {
+			layerType := store.LayerType(layerConf.Type)
+			if !setup.LayerTypeExists(layerType) {
+				return errors.Errorf(validateErrMessage+": could not find layer type '%s'", proxyName, layerName, layerType)
+			}
+
+			layerOptionsSchema, err := setup.GetLayerOptionsSchema(layerType)
+			if err != nil {
+				return errors.Wrapf(err, validateErrMessage, proxyName, layerName)
+			}
+
+			rawOptions := func(opts config.InterpolatedMap) map[string]any {
+				return opts
+			}(layerConf.Options)
+
+			if err := schema.Validate(ctx, layerOptionsSchema, rawOptions); err != nil {
+				return errors.Wrapf(err, validateErrMessage, proxyName, layerName)
+			}
+		}
+	}
+
+	return nil
+}

internal/admin/layer_route.go

@@ -51,15 +51,6 @@ func (s *Server) queryLayer(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
-func validateLayerName(v string) (store.LayerName, error) {
-	name, err := store.ValidateName(v)
-	if err != nil {
-		return "", errors.WithStack(err)
-	}
-
-	return store.LayerName(name), nil
-}
-
 type GetLayerResponse struct {
 	Layer *store.Layer `json:"layer"`
 }

internal/admin/option.go

@@ -5,8 +5,9 @@ import (
 )
 
 type Option struct {
-	ServerConfig config.AdminServerConfig
+	BootstrapConfig config.BootstrapConfig
-	RedisConfig  config.RedisConfig
+	ServerConfig    config.AdminServerConfig
+	RedisConfig     config.RedisConfig
 }
 
 type OptionFunc func(*Option)
@@ -29,3 +30,9 @@ func WithRedisConfig(conf config.RedisConfig) OptionFunc {
 		opt.RedisConfig = conf
 	}
 }
+
+func WithBootstrapConfig(conf config.BootstrapConfig) OptionFunc {
+	return func(opt *Option) {
+		opt.BootstrapConfig = conf
+	}
+}

internal/admin/proxy_route.go

@@ -114,6 +114,23 @@ func (s *Server) deleteProxy(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	layers, err := s.layerRepository.QueryLayers(ctx, proxyName)
+	if err != nil {
+		logAndCaptureError(ctx, "could not query proxy's layers", errors.WithStack(err))
+		api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
+
+		return
+	}
+
+	for _, layer := range layers {
+		if err := s.layerRepository.DeleteLayer(ctx, proxyName, layer.Name); err != nil {
+			logAndCaptureError(ctx, "could not delete layer", errors.WithStack(err))
+			api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
+
+			return
+		}
+	}
+
 	api.DataResponse(w, http.StatusOK, DeleteProxyResponse{
 		ProxyName: proxyName,
 	})

internal/admin/server.go

@@ -19,12 +19,15 @@ import (
 	"github.com/go-chi/cors"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/redis/go-redis/v9"
 	"gitlab.com/wpetit/goweb/logger"
 )
 
 type Server struct {
 	serverConfig    config.AdminServerConfig
 	redisConfig     config.RedisConfig
+	redisClient     redis.UniversalClient
+	bootstrapConfig config.BootstrapConfig
 	proxyRepository store.ProxyRepository
 	layerRepository store.LayerRepository
 }
@@ -53,6 +56,12 @@ func (s *Server) run(parentCtx context.Context, addrs chan net.Addr, errs chan e
 		return
 	}
 
+	if err := s.bootstrapProxies(ctx); err != nil {
+		errs <- errors.WithStack(err)
+
+		return
+	}
+
 	listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", s.serverConfig.HTTP.Host, s.serverConfig.HTTP.Port))
 	if err != nil {
 		errs <- errors.WithStack(err)
@@ -175,7 +184,8 @@ func NewServer(funcs ...OptionFunc) *Server {
 	}
 
 	return &Server{
-		serverConfig: opt.ServerConfig,
+		serverConfig:    opt.ServerConfig,
-		redisConfig:  opt.RedisConfig,
+		redisConfig:     opt.RedisConfig,
+		bootstrapConfig: opt.BootstrapConfig,
 	}
 }

internal/command/server/admin/run.go

@@ -68,6 +68,7 @@ func RunCommand() *cli.Command {
 			srv := admin.NewServer(
 				admin.WithServerConfig(conf.Admin),
 				admin.WithRedisConfig(conf.Redis),
+				admin.WithBootstrapConfig(conf.Bootstrap),
 			)
 
 			addrs, srvErrs := srv.Start(ctx.Context)

internal/config/bootstrap.go

@@ -0,0 +1,104 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"forge.cadoles.com/cadoles/bouncer/internal/store"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
+)
+
+type BootstrapConfig struct {
+	Proxies     map[store.ProxyName]BootstrapProxyConfig `yaml:"proxies"`
+	Dir         InterpolatedString                       `yaml:"dir"`
+	LockTimeout InterpolatedDuration                     `yaml:"lockTimeout"`
+}
+
+func (c *BootstrapConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	src := struct {
+		Proxies map[store.ProxyName]BootstrapProxyConfig `yaml:"proxies"`
+		Dir     InterpolatedString                       `yaml:"dir"`
+	}{
+		Proxies: make(map[store.ProxyName]BootstrapProxyConfig),
+		Dir:     "",
+	}
+
+	if err := unmarshal(&src); err != nil {
+		return errors.WithStack(err)
+	}
+
+	c.Proxies = src.Proxies
+	c.Dir = src.Dir
+
+	if src.Dir != "" {
+		proxies, err := loadBootstrapDir(string(src.Dir))
+		if err != nil {
+			return errors.Wrapf(err, "could not load bootstrap dir '%s'", src.Dir)
+		}
+
+		c.Proxies = overrideProxies(c.Proxies, proxies)
+	}
+
+	return nil
+}
+
+type BootstrapProxyConfig struct {
+	Enabled InterpolatedBool                         `yaml:"enabled"`
+	Weight  InterpolatedInt                          `yaml:"weight"`
+	To      InterpolatedString                       `yaml:"to"`
+	From    InterpolatedStringSlice                  `yaml:"from"`
+	Layers  map[store.LayerName]BootstrapLayerConfig `yaml:"layers"`
+}
+
+type BootstrapLayerConfig struct {
+	Enabled InterpolatedBool   `yaml:"enabled"`
+	Type    InterpolatedString `yaml:"type"`
+	Weight  InterpolatedInt    `yaml:"weight"`
+	Options InterpolatedMap    `yaml:"options"`
+}
+
+func NewDefaultBootstrapConfig() BootstrapConfig {
+	return BootstrapConfig{
+		Dir:         "",
+		LockTimeout: *NewInterpolatedDuration(30 * time.Second),
+	}
+}
+
+func loadBootstrapDir(dir string) (map[store.ProxyName]BootstrapProxyConfig, error) {
+	pattern := filepath.Join(dir, "*.yml")
+
+	files, err := filepath.Glob(pattern)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	proxies := make(map[store.ProxyName]BootstrapProxyConfig)
+	for _, f := range files {
+		data, err := os.ReadFile(f)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not read file '%s'", f)
+		}
+
+		proxy := BootstrapProxyConfig{}
+
+		if err := yaml.Unmarshal(data, &proxy); err != nil {
+			return nil, errors.Wrapf(err, "could not unmarshal proxy")
+		}
+
+		name := store.ProxyName(strings.TrimSuffix(filepath.Base(f), filepath.Ext(f)))
+		proxies[name] = proxy
+	}
+
+	return proxies, nil
+}
+
+func overrideProxies(base map[store.ProxyName]BootstrapProxyConfig, proxies map[store.ProxyName]BootstrapProxyConfig) map[store.ProxyName]BootstrapProxyConfig {
+	for name, proxy := range proxies {
+		base[name] = proxy
+	}
+
+	return base
+}

internal/config/config.go

@@ -2,7 +2,7 @@ package config
 
 import (
 	"io"
-	"io/ioutil"
+	"os"
 
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
@@ -10,18 +10,19 @@ import (
 
 // Config definition
 type Config struct {
-	Admin  AdminServerConfig `yaml:"admin"`
+	Admin     AdminServerConfig `yaml:"admin"`
-	Proxy  ProxyServerConfig `yaml:"proxy"`
+	Proxy     ProxyServerConfig `yaml:"proxy"`
-	Redis  RedisConfig       `yaml:"redis"`
+	Redis     RedisConfig       `yaml:"redis"`
-	Logger LoggerConfig      `yaml:"logger"`
+	Logger    LoggerConfig      `yaml:"logger"`
-	Layers LayersConfig      `yaml:"layers"`
+	Layers    LayersConfig      `yaml:"layers"`
+	Bootstrap BootstrapConfig   `yaml:"bootstrap"`
 }
 
 // NewFromFile retrieves the configuration from the given file
 func NewFromFile(path string) (*Config, error) {
 	config := NewDefault()
 
-	data, err := ioutil.ReadFile(path)
+	data, err := os.ReadFile(path)
 	if err != nil {
 		return nil, errors.Wrapf(err, "could not read file '%s'", path)
 	}
@@ -43,11 +44,12 @@ func NewDumpDefault() *Config {
 // NewDefault return new default configuration
 func NewDefault() *Config {
 	return &Config{
-		Admin:  NewDefaultAdminServerConfig(),
+		Admin:     NewDefaultAdminServerConfig(),
-		Proxy:  NewDefaultProxyServerConfig(),
+		Proxy:     NewDefaultProxyServerConfig(),
-		Logger: NewDefaultLoggerConfig(),
+		Logger:    NewDefaultLoggerConfig(),
-		Redis:  NewDefaultRedisConfig(),
+		Redis:     NewDefaultRedisConfig(),
-		Layers: NewDefaultLayersConfig(),
+		Layers:    NewDefaultLayersConfig(),
+		Bootstrap: NewDefaultBootstrapConfig(),
 	}
 }
 

internal/proxy/init.go

@@ -5,22 +5,25 @@ import (
 
 	"forge.cadoles.com/cadoles/bouncer/internal/setup"
 	"github.com/pkg/errors"
+	"github.com/redis/go-redis/v9"
 )
 
 func (s *Server) initRepositories(ctx context.Context) error {
-	if err := s.initProxyRepository(ctx); err != nil {
+	client := setup.NewRedisClient(ctx, s.redisConfig)
+
+	if err := s.initProxyRepository(ctx, client); err != nil {
 		return errors.WithStack(err)
 	}
 
-	if err := s.initLayerRepository(ctx); err != nil {
+	if err := s.initLayerRepository(ctx, client); err != nil {
 		return errors.WithStack(err)
 	}
 
 	return nil
 }
 
-func (s *Server) initProxyRepository(ctx context.Context) error {
+func (s *Server) initProxyRepository(ctx context.Context, client redis.UniversalClient) error {
-	proxyRepository, err := setup.NewProxyRepository(ctx, s.redisConfig)
+	proxyRepository, err := setup.NewProxyRepository(ctx, client)
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -30,8 +33,8 @@ func (s *Server) initProxyRepository(ctx context.Context) error {
 	return nil
 }
 
-func (s *Server) initLayerRepository(ctx context.Context) error {
+func (s *Server) initLayerRepository(ctx context.Context, client redis.UniversalClient) error {
-	layerRepository, err := setup.NewLayerRepository(ctx, s.redisConfig)
+	layerRepository, err := setup.NewLayerRepository(ctx, client)
 	if err != nil {
 		return errors.WithStack(err)
 	}

internal/setup/proxy_repository.go

@@ -9,16 +9,17 @@ import (
 	"github.com/redis/go-redis/v9"
 )
 
-func NewProxyRepository(ctx context.Context, conf config.RedisConfig) (store.ProxyRepository, error) {
+func NewRedisClient(ctx context.Context, conf config.RedisConfig) redis.UniversalClient {
-	rdb := newRedisClient(conf)
+	return redis.NewUniversalClient(&redis.UniversalOptions{
-	return redisStore.NewProxyRepository(rdb), nil
-}
-
-func NewLayerRepository(ctx context.Context, conf config.RedisConfig) (store.LayerRepository, error) {
-	rdb := redis.NewUniversalClient(&redis.UniversalOptions{
 		Addrs:      conf.Adresses,
 		MasterName: string(conf.Master),
 	})
-
+}
-	return redisStore.NewLayerRepository(rdb), nil
+
+func NewProxyRepository(ctx context.Context, client redis.UniversalClient) (store.ProxyRepository, error) {
+	return redisStore.NewProxyRepository(client), nil
+}
+
+func NewLayerRepository(ctx context.Context, client redis.UniversalClient) (store.LayerRepository, error) {
+	return redisStore.NewLayerRepository(client), nil
 }

misc/k8s/README.md

@@ -4,57 +4,60 @@
 
 1. Generate the Docker configuration to enable image builds with Kaniko and communicate with reg.cadoles.com
 
-    ```shell
+   ```shell
-    docker login reg.cadoles.com
+   docker login reg.cadoles.com
-    mkdir -p misc/k8s/kustomization/base/secrets/dockerconfig
+   mkdir -p misc/k8s/kustomization/base/secrets/dockerconfig
-    docker --config misc/k8s/kustomization/base/secrets/dockerconfig login reg.cadoles.com
+   docker --config misc/k8s/kustomization/base/secrets/dockerconfig login reg.cadoles.com
-    mv misc/k8s/kustomization/base/secrets/dockerconfig/config.json misc/k8s/kustomization/base/secrets/dockerconfig/.dockerconfigjson
+   mv misc/k8s/kustomization/base/secrets/dockerconfig/config.json misc/k8s/kustomization/base/secrets/dockerconfig/.dockerconfigjson
-    mkdir -p misc/k8s/kustomization/overlays/dev/secrets/dockerconfig
+   mkdir -p misc/k8s/kustomization/overlays/dev/secrets/dockerconfig
-    cp misc/k8s/kustomization/base/secrets/dockerconfig/.dockerconfigjson misc/k8s/kustomization/overlays/dev/secrets/dockerconfig/.dockerconfigjson
+   cp misc/k8s/kustomization/base/secrets/dockerconfig/.dockerconfigjson misc/k8s/kustomization/overlays/dev/secrets/dockerconfig/.dockerconfigjson
-    ```
+   ```
 
 ## Getting started with Kind
 
 1. Create your [Kind](https://kind.sigs.k8s.io/) cluster
 
-    ```shell
+   ```shell
-    kind create cluster --config misc/k8s/kind/bouncer-cluster.yaml
+   kind create cluster --config misc/k8s/kind/bouncer-cluster.yaml
-    ```
+   ```
 
 2. Deploy required operators
 
-    ```shell
+   ```shell
-    kubectl apply -k misc/k8s/kind/cluster --server-side
+   kubectl apply -k misc/k8s/kind/cluster --server-side
-    ```
+   ```
 
 3. Deploy your Bouncer development environment
 
-    ```shell
+   ```shell
-    skaffold dev -p dev --cleanup=false --default-repo reg.cadoles.com/<YOUR_PERSONNAL_USER_NAME>
+   skaffold dev -p dev --cleanup=false --default-repo reg.cadoles.com/<YOUR_PERSONNAL_USER_NAME>
-    ```
+   ```
 
 ## Testing
 
+Bouncer will automatically create proxies based on the files present in the `misc/k8s/kustomization/overlays/dev/files/bouncer/bootstrap.d` folder.
+
+By default, with you host web browser, open http://localhost:9000, you should see the Cadoles website.
+
+### Using the admin API
+
 1. Open shell in bouncer-admin pod
 
-    ```shell
+   ```shell
-    kubectl exec -it -n bouncer-dev bouncer-admin-<suffix> -- /bin/sh
+   kubectl exec -it -n bouncer-dev bouncer-admin-<suffix> -- /bin/sh
-    ```
+   ```
 
 2. Create an authentication token
 
-    ```shell
+   ```shell
-    bouncer --config /etc/bouncer/config.yml auth create-token --role writer --subject $(whoami) > .bouncer-token
+   bouncer --config /etc/bouncer/config.yml auth create-token --role writer --subject $(whoami) > .bouncer-token
-    ```
+   ```
 
 3. Create a proxy and enable it
 
-    ```shell
+   ```shell
-    bouncer admin proxy create --proxy-to https://www.cadoles.com --proxy-name cadoles
+   bouncer admin proxy query
-    bouncer admin proxy update --proxy-name cadoles --proxy-enabled=true
+   ```
-    ```
-
-4. With you host web browser, open http://localhost:9000, you should see the Cadoles website.
 
 ## Benchmarking
 
@@ -63,4 +66,3 @@ You can use [`siege`](https://github.com/JoeDog/siege) to benchmark your instanc
 ```shell
 BASE_URL=http://localhost:9000 make siege
 ```
-

misc/k8s/kustomization/base/resources/bouncer-admin/files/config.yml

@@ -4,19 +4,19 @@ admin:
     port: 8081
   cors:
     allowedOrigins:
-    - http://localhost:3001
+      - http://localhost:3001
     allowCredentials: true
     allowMethods:
-    - POST
+      - POST
-    - GET
+      - GET
-    - PUT
+      - PUT
-    - DELETE
+      - DELETE
     allowedHeaders:
-    - Origin
+      - Origin
-    - Accept
+      - Accept
-    - Content-Type
+      - Content-Type
-    - Authorization
+      - Authorization
-    - Sentry-Trace
+      - Sentry-Trace
     debug: false
   auth:
     issuer: http://127.0.0.1:8081
@@ -28,9 +28,13 @@ admin:
 
 redis:
   addresses:
-  - rfs-bouncer-redis:${RFS_BOUNCER_REDIS_SERVICE_PORT}
+    - rfs-bouncer-redis:${RFS_BOUNCER_REDIS_SERVICE_PORT}
   master: mymaster
 
 logger:
-  level: 2
+  level: ${BOUNCER_LOG_LEVEL}
   format: human
+
+bootstrap:
+  dir: /etc/bouncer/bootstrap.d
+  lockTimeout: 30s

misc/k8s/kustomization/base/resources/bouncer-admin/kustomization.yaml

@@ -2,11 +2,15 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- ./resources/service.yaml
+  - ./resources/service.yaml
-- ./resources/deployment.yaml
+  - ./resources/deployment.yaml
 
 configMapGenerator:
-- name: bouncer-admin-config
+  - name: bouncer-admin-config
-  files:
+    files:
-  - ./files/config.yml
+      - ./files/config.yml
-  - ./files/admin-key.json
+      - ./files/admin-key.json
+  - name: bouncer-admin-bootstrap
+  - name: bouncer-admin-env
+    literals:
+      - BOUNCER_LOG_LEVEL=2

misc/k8s/kustomization/base/resources/bouncer-admin/resources/deployment.yaml

@@ -17,18 +17,35 @@ spec:
         io.kompose.service: bouncer-admin
     spec:
       containers:
-      - name: bouncer-admin
-        image: reg.cadoles.com/cadoles/bouncer:v2024.2.5-1602626
-        command: ["bouncer", "--debug", "-c", "/etc/bouncer/config.yml", "server", "admin", "run"]
-        imagePullPolicy: Always
-        resources: {}
-        ports:
         - name: bouncer-admin
-          containerPort: 8081
+          image: bouncer
-        volumeMounts:
+          command:
-        - mountPath: /etc/bouncer/
+            [
-          name: bouncer-admin-config
+              "bouncer",
+              "--debug",
+              "-c",
+              "/etc/bouncer/config.yml",
+              "server",
+              "admin",
+              "run",
+            ]
+          imagePullPolicy: Always
+          resources: {}
+          ports:
+            - name: bouncer-admin
+              containerPort: 8081
+          envFrom:
+            - configMapRef:
+                name: bouncer-admin-env
+          volumeMounts:
+            - mountPath: /etc/bouncer/
+              name: bouncer-admin-config
+            - mountPath: /etc/bouncer/bootstrap.d
+              name: bouncer-admin-bootstrap
       volumes:
-      - name: bouncer-admin-config
+        - name: bouncer-admin-config
-        configMap:
+          configMap:
-          name: bouncer-admin-config
+            name: bouncer-admin-config
+        - name: bouncer-admin-bootstrap
+          configMap:
+            name: bouncer-admin-bootstrap

misc/k8s/kustomization/base/resources/bouncer-server/files/config.yml

@@ -14,9 +14,9 @@ layers:
 
 redis:
   addresses:
-  - rfs-bouncer-redis:${RFS_BOUNCER_REDIS_SERVICE_PORT}
+    - rfs-bouncer-redis:${RFS_BOUNCER_REDIS_SERVICE_PORT}
   master: mymaster
 
 logger:
-  level: 2
+  level: ${BOUNCER_LOG_LEVEL}
   format: human

misc/k8s/kustomization/base/resources/bouncer-server/kustomization.yaml

@@ -2,10 +2,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- ./resources/service.yaml
+  - ./resources/service.yaml
-- ./resources/deployment.yaml
+  - ./resources/deployment.yaml
 
 configMapGenerator:
-- name: bouncer-server-config
+  - name: bouncer-server-config
-  files:
+    files:
-  - ./files/config.yml
+      - ./files/config.yml
+  - name: bouncer-server-env
+    literals:
+      - BOUNCER_LOG_LEVEL=2

misc/k8s/kustomization/base/resources/bouncer-server/resources/deployment.yaml

@@ -17,18 +17,29 @@ spec:
         io.kompose.service: bouncer-server
     spec:
       containers:
-      - name: bouncer-server
-        image: reg.cadoles.com/cadoles/bouncer:v2024.2.5-1602626
-        command: ["bouncer", "-c", "/etc/bouncer/config.yml", "server", "proxy", "run"]
-        imagePullPolicy: Always
-        resources: {}
-        ports:
         - name: bouncer-server
-          containerPort: 8080
+          image: bouncer
-        volumeMounts:
+          command:
-        - mountPath: /etc/bouncer/
+            [
-          name: bouncer-server-config
+              "bouncer",
+              "-c",
+              "/etc/bouncer/config.yml",
+              "server",
+              "proxy",
+              "run",
+            ]
+          imagePullPolicy: Always
+          envFrom:
+            - configMapRef:
+                name: bouncer-server-env
+          resources: {}
+          ports:
+            - name: bouncer-server
+              containerPort: 8080
+          volumeMounts:
+            - mountPath: /etc/bouncer/
+              name: bouncer-server-config
       volumes:
-      - name: bouncer-server-config
+        - name: bouncer-server-config
-        configMap:
+          configMap:
-          name: bouncer-server-config
+            name: bouncer-server-config

misc/k8s/kustomization/overlays/dev/files/bouncer/bootstrap.d/cadoles.yml

@@ -0,0 +1,11 @@
+from: ["*"]
+to: https://www.cadoles.com
+enabled: true
+weight: 0
+layers:
+  my-queue:
+    type: queue
+    enabled: true
+    weight: 0
+    options:
+      capacity: 10

misc/k8s/kustomization/overlays/dev/kustomization.yaml

@@ -3,16 +3,30 @@ kind: Kustomization
 namespace: bouncer-dev
 
 resources:
-- ../../base
+  - ../../base
 
 secretGenerator:
-- files:
+  - files:
-  - secrets/dockerconfig/.dockerconfigjson
+      - secrets/dockerconfig/.dockerconfigjson
-  name: regcred-dev
+    name: regcred-dev
-  type: kubernetes.io/dockerconfigjson
+    type: kubernetes.io/dockerconfigjson
 
 patches:
-- path: patches/add-registry-pull-secret.patch.yaml
+  - path: patches/add-registry-pull-secret.patch.yaml
-  target:
+    target:
-    kind: Deployment
+      kind: Deployment
-    version: v1
+      version: v1
+
+configMapGenerator:
+  - name: bouncer-admin-bootstrap
+    behavior: merge
+    files:
+      - ./files/bouncer/bootstrap.d/cadoles.yml
+  - name: bouncer-admin-env
+    behavior: merge
+    literals:
+      - BOUNCER_LOG_LEVEL=0
+  - name: bouncer-server-env
+    behavior: merge
+    literals:
+      - BOUNCER_LOG_LEVEL=0

misc/packaging/common/config.yml

@@ -15,19 +15,19 @@ admin:
   # est branché sur l'API d'administration.
   cors:
     allowedOrigins:
-    - http://localhost:8081
+      - http://localhost:8081
     allowCredentials: true
     allowMethods:
-    - POST
+      - POST
-    - GET
+      - GET
-    - PUT
+      - PUT
-    - DELETE
+      - DELETE
     allowedHeaders:
-    - Origin
+      - Origin
-    - Accept
+      - Accept
-    - Content-Type
+      - Content-Type
-    - Authorization
+      - Authorization
-    - Sentry-Trace
+      - Sentry-Trace
     debug: false
 
   # Authentification JWT
@@ -145,7 +145,7 @@ proxy:
 # - Mode "cluster": renseigner plusieurs adresses dans redis.addresses et laisser redis.master vide.
 redis:
   addresses:
-  - localhost:6379
+    - localhost:6379
   master: ""
   writeTimeout: 30s
   readTimeout: 30s
@@ -177,3 +177,36 @@ layers:
     # Répertoire contenant les templates
     templateDir: "/etc/bouncer/layers/circuitbreaker/templates"
 
+# Configuration d'une série de proxy/layers
+# à créer par défaut par le serveur d'administration
+bootstrap:
+  # Répertoire contenant les définitions de proxy à créer
+  # par défaut. Les fichiers seront récupérés si ils
+  # correspondent au patron de nommage suivant:
+  #
+  # <bootstrap_dir>/<proxy_name>.yml
+  #
+  # Si l'attribut est vide ou absent le chargement des fichiers
+  # est désactivé.
+  dir: /etc/bouncer/bootstrap.d
+  # Délai d'expiration du verrou distribué utilisé lors du chargement
+  # des définitions de proxy par défaut.
+  lockTimeout: 30s
+  # Tableau associatif de définition de proxies à créer par
+  # défaut par le serveur d'administration.
+  # Si `proxies` et `dir` sont tous les deux définis, les fichiers
+  # présents dans le répertoire `dir` surchargeront les valeurs définies
+  # dans `proxies`.
+  # Par défault non défini
+  proxies:
+  #     my-proxy:
+  #         enabled: true
+  #         from: ["*"]
+  #         to: "https://example.net"
+  #         weight: 0
+  #         layers:
+  #             my-layer:
+  #                 type: queue
+  #                 enabled: false
+  #                 weight: 0
+  #                 options: {"capacity": 100}

skaffold.yaml

@@ -7,16 +7,16 @@ metadata:
 manifests:
   kustomize:
     paths:
-    - misc/k8s/kustomization/base
+      - misc/k8s/kustomization/base
 
 profiles:
-- name: dev
+  - name: dev
-  manifests:
+    manifests:
-    kustomize:
+      kustomize:
-      paths:
+        paths:
-      - misc/k8s/kustomization/overlays/dev
+          - misc/k8s/kustomization/overlays/dev
-  activation:
+    activation:
-  - command: dev
+      - command: dev
 
 build:
   local:
@@ -26,28 +26,28 @@ build:
     sha256: {}
 
   artifacts:
-  - image: reg.cadoles.com/cadoles/bouncer
+    - image: bouncer
-    context: .
+      context: .
-    sync:
+      sync:
-      infer:
+        infer:
-      - cmd/**
+          - cmd/**
-      - internal/**
+          - internal/**
-      - layers/**
+          - layers/**
-      - misc/**
+          - misc/**
-    docker:
+      docker:
-      dockerfile: Dockerfile
+        dockerfile: Dockerfile
 
 deploy:
   statusCheckDeadlineSeconds: 600
 
 portForward:
-- resourceType: service
+  - resourceType: service
-  resourceName: bouncer-admin
+    resourceName: bouncer-admin
-  namespace: bouncer-dev
+    namespace: bouncer-dev
-  port: 8081
+    port: 8081
-  localPort: 9999
+    localPort: 9999
-- resourceType: service
+  - resourceType: service
-  resourceName: bouncer-server
+    resourceName: bouncer-server
-  namespace: bouncer-dev
+    namespace: bouncer-dev
-  port: 8080
+    port: 8080
-  localPort: 9000 # *Optional*
+    localPort: 9000 # *Optional*