Cadoles/bouncer
Cadoles
/
bouncer
10
0
Fork 0
Code Issues 2 Pull Requests 1 Projects Releases 3 Wiki Activity

feat: bootstrap default proxy and layers from configuration #17

Merged
wpetit merged 1 commits from proxy-bootstrap into develop 2024-03-26 17:30:22 +01:00
Conversation 0 Commits 1 Files Changed 27 +725 -312
27 changed files with 725 additions and 312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

265
.goreleaser.yaml
View File

@ -1,148 +1,143 @@
project_name: bouncer project_name: bouncer
before: before:
hooks: hooks:
- go mod tidy - go mod tidy
- go generate ./... - go generate ./...
builds: builds:
- id: bouncer - id: bouncer
env: env:
- CGO_ENABLED=0 - CGO_ENABLED=0
ldflags: ldflags:
- -s - -s
- -w - -w
- -X 'main.GitRef={{ .Commit }}' - -X 'main.GitRef={{ .Commit }}'
- -X 'main.ProjectVersion={{ .Version }}' - -X 'main.ProjectVersion={{ .Version }}'
- -X 'main.BuildDate={{ .Date }}' - -X 'main.BuildDate={{ .Date }}'
- -X 'main.DefaultConfigPath=/etc/bouncer/config.yml' - -X 'main.DefaultConfigPath=/etc/bouncer/config.yml'
gcflags: gcflags:
- -trimpath="${PWD}" - -trimpath="${PWD}"
asmflags: asmflags:
- -trimpath="${PWD}" - -trimpath="${PWD}"
goos: goos:
- linux - linux
goarch: goarch:
- amd64 - amd64
- arm64 - arm64
- "386" - "386"
main: ./cmd/bouncer main: ./cmd/bouncer
archives: archives:
- id: bouncer - id: bouncer
builds: ["bouncer"] builds: ["bouncer"]
name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}' name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
files: files:
- README.md - README.md
- misc/packaging/common/config.yml - misc/packaging/common/config.yml
checksum: checksum:
name_template: 'checksums.txt' name_template: "checksums.txt"
snapshot: snapshot:
name_template: "{{ .Version }}" name_template: "{{ .Version }}"
changelog: changelog:
sort: asc sort: asc
filters: filters:
exclude: exclude:
- '^docs:' - "^docs:"
- '^test:' - "^test:"
nfpms: nfpms:
- id: bouncer-bin - id: bouncer-bin
builds: builds:
- "bouncer" - "bouncer"
package_name: bouncer-bin package_name: bouncer-bin
homepage: https://forge.cadoles.com/Cadoles/bouncer homepage: https://forge.cadoles.com/Cadoles/bouncer
maintainer: Cadoles <contact@cadoles.com> maintainer: Cadoles <contact@cadoles.com>
description: |- description: |-
reverse proxy server with dynamic queuing management - binaries reverse proxy server with dynamic queuing management - binaries
license: AGPL-3.0 license: AGPL-3.0
formats: formats:
- apk - apk
- deb - deb
- rpm - rpm
- archlinux contents:
contents: - src: misc/packaging/common/config.yml
- src: misc/packaging/common/config.yml dst: /etc/bouncer/config.yml
dst: /etc/bouncer/config.yml type: config
type: config - src: layers
- src: layers dst: /etc/bouncer/layers
dst: /etc/bouncer/layers type: config
type: config - dst: /etc/bouncer/bootstrap.d
- id: bouncer-admin type: dir
meta: true file_info:
package_name: bouncer-admin mode: 0700
homepage: https://forge.cadoles.com/Cadoles/bouncer - id: bouncer-admin
maintainer: Cadoles <contact@cadoles.com> meta: true
dependencies: package_name: bouncer-admin
- bouncer-bin homepage: https://forge.cadoles.com/Cadoles/bouncer
description: |- maintainer: Cadoles <contact@cadoles.com>
reverse proxy server with dynamic queuing management - administration service dependencies:
license: AGPL-3.0 - bouncer-bin
formats: description: |-
- apk reverse proxy server with dynamic queuing management - administration service
- deb license: AGPL-3.0
- rpm formats:
- archlinux - apk
contents: - deb
- src: misc/packaging/systemd/bouncer-admin.systemd.service - rpm
dst: /usr/lib/systemd/system/bouncer-admin.service contents:
packager: deb - src: misc/packaging/systemd/bouncer-admin.systemd.service
- src: misc/packaging/systemd/bouncer-admin.systemd.service dst: /usr/lib/systemd/system/bouncer-admin.service
dst: /usr/lib/systemd/system/bouncer-admin.service packager: deb
packager: rpm - src: misc/packaging/systemd/bouncer-admin.systemd.service
- src: misc/packaging/systemd/bouncer-admin.systemd.service dst: /usr/lib/systemd/system/bouncer-admin.service
dst: /usr/lib/systemd/system/bouncer-admin.service packager: rpm
packager: archlinux - src: misc/packaging/openrc/bouncer-admin.openrc.sh
- src: misc/packaging/openrc/bouncer-admin.openrc.sh dst: /etc/init.d/bouncer-admin
dst: /etc/init.d/bouncer-admin file_info:
file_info: mode: 0755
mode: 0755 packager: apk
packager: apk - dst: /usr/share/bouncer
- dst: /usr/share/bouncer type: dir
type: dir file_info:
file_info: mode: 0700
mode: 0700 - dst: /var/log/bouncer
- dst: /var/log/bouncer type: dir
type: dir file_info:
file_info: mode: 0700
mode: 0700 packager: apk
packager: apk scripts:
scripts: postinstall: "misc/packaging/common/postinstall-bouncer-admin.sh"
postinstall: "misc/packaging/common/postinstall-bouncer-admin.sh" - id: bouncer-proxy
- id: bouncer-proxy meta: true
meta: true dependencies:
dependencies: - bouncer-bin
- bouncer-bin package_name: bouncer-proxy
package_name: bouncer-proxy homepage: https://forge.cadoles.com/Cadoles/bouncer
homepage: https://forge.cadoles.com/Cadoles/bouncer maintainer: Cadoles <contact@cadoles.com>
maintainer: Cadoles <contact@cadoles.com> description: |-
description: |- reverse proxy server with dynamic queuing management - proxy service
reverse proxy server with dynamic queuing management - proxy service license: AGPL-3.0
license: AGPL-3.0 formats:
formats: - apk
- apk - deb
- deb - rpm
- rpm contents:
- archlinux - src: misc/packaging/systemd/bouncer-proxy.systemd.service
contents: dst: /usr/lib/systemd/system/bouncer-proxy.service
- src: misc/packaging/systemd/bouncer-proxy.systemd.service packager: deb
dst: /usr/lib/systemd/system/bouncer-proxy.service - src: misc/packaging/systemd/bouncer-proxy.systemd.service
packager: deb dst: /usr/lib/systemd/system/bouncer-proxy.service
- src: misc/packaging/systemd/bouncer-proxy.systemd.service packager: rpm
dst: /usr/lib/systemd/system/bouncer-proxy.service - src: misc/packaging/openrc/bouncer-proxy.openrc.sh
packager: rpm dst: /etc/init.d/bouncer-proxy
- src: misc/packaging/systemd/bouncer-proxy.systemd.service file_info:
dst: /usr/lib/systemd/system/bouncer-proxy.service mode: 0755
packager: archlinux packager: apk
- src: misc/packaging/openrc/bouncer-proxy.openrc.sh - dst: /usr/share/bouncer
dst: /etc/init.d/bouncer-proxy type: dir
file_info: file_info:
mode: 0755 mode: 0700
packager: apk - dst: /var/log/bouncer
- dst: /usr/share/bouncer type: dir
type: dir file_info:
file_info: mode: 0700
mode: 0700 packager: apk
- dst: /var/log/bouncer scripts:
type: dir postinstall: "misc/packaging/common/postinstall-bouncer-proxy.sh"
file_info:
mode: 0700
packager: apk
scripts:
postinstall: "misc/packaging/common/postinstall-bouncer-proxy.sh"

9
Dockerfile
View File

@ -9,10 +9,15 @@ RUN mkdir -p /usr/local/bin \
&& wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64 \ && wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64 \
&& chmod +x /usr/local/bin/yq && chmod +x /usr/local/bin/yq
COPY . /src
WORKDIR /src WORKDIR /src
COPY go.mod .
COPY go.sum .
RUN go mod download
COPY . /src
RUN make GORELEASER_ARGS='build --rm-dist --single-target --snapshot' goreleaser RUN make GORELEASER_ARGS='build --rm-dist --single-target --snapshot' goreleaser
# Patch config # Patch config

1
doc/README.md
View File

@ -19,6 +19,7 @@
### Utilisation ### Utilisation
- [(FR) - Ajouter un layer de type "file d'attente"](./fr/tutorials/add-queue-layer.md) - [(FR) - Ajouter un layer de type "file d'attente"](./fr/tutorials/add-queue-layer.md)
- [(FR) - Amorçage d'un serveur Bouncer via la configuration](./fr/tutorials/bootstrapping.md)
### Développement ### Développement

47
doc/fr/tutorials/bootstrapping.md Normal file
View File

@ -0,0 +1,47 @@
# Amorçage d'un serveur Bouncer via la configuration
Il est possible d'amorcer des données par défaut (i.e. des "proxies" et "layers" associés) via la configuration du serveur d'administration.
> **Attention** Ce mécanisme de modifiera pas des proxies déjà existants dans la base de données du serveur Bouncer. Autrement dit, si un proxy est déjà pré-existant lors du démarrage du serveur Bouncer, il ne sera pas modifié.
La définition des proxies et layers par défaut s'effectue dans la section `bootstrap` du fichier de configuration. Deux possibilités pour définir les proxys à charger par défaut:
- Utiliser un répertoire contenant des fichiers YAML (un par proxy) en définissant le chemin du répertoire via l'attribut `bootstrap.dir`;
- Définir directement la liste des proxies via l'attribut `bootstrap.proxies`.
```yaml
# Configuration d'une série de proxy/layers
# à créer par défaut par le serveur d'administration
bootstrap:
# Répertoire contenant les définitions de proxy à créer
# par défaut. Les fichiers seront récupérés si ils
# correspondent au patron de nommage suivant:
#
# <bootstrap_dir>/<proxy_name>.yml
#
# Voir ci-dessous pour les attributs possibles dans les fichiers.
#
# Si l'attribut est vide ou absent le chargement des fichiers
# est désactivé.
dir: /etc/bouncer/bootstrap.d
# Tableau associatif de définition de proxies à créer par
# défaut par le serveur d'administration.
# Si `proxies` et `dir` sont tous les deux définis, les fichiers
# présents dans le répertoire `dir` surchargeront les valeurs définies
# dans `proxies`.
#
# Par défaut vide.
proxies:
# my-proxy:
# enabled: true # Activer/désactiver le proxy
# from: ["*"] # Filtre d'origine d'activation du proxy
# to: "https://example.net" # Destination du proxy
# weight: 0 # Priorité du proxy
# layers: # Layers associés au proxy
# my-layer:
# type: queue # Type du proxy
# enabled: false # Activer/désactiver le layer
# weight: 0 # Priorité du layer
# options: {"capacity": 100} # Options associées au layer
```

1
go.mod
View File

@ -5,6 +5,7 @@ go 1.20
require ( require (
forge.cadoles.com/Cadoles/go-proxy v0.0.0-20230701194111-c6b3d482cca6 forge.cadoles.com/Cadoles/go-proxy v0.0.0-20230701194111-c6b3d482cca6
github.com/Masterminds/sprig/v3 v3.2.3 github.com/Masterminds/sprig/v3 v3.2.3
github.com/bsm/redislock v0.9.4
github.com/btcsuite/btcd/btcutil v1.1.3 github.com/btcsuite/btcd/btcutil v1.1.3
github.com/drone/envsubst v1.0.3 github.com/drone/envsubst v1.0.3
github.com/getsentry/sentry-go v0.22.0 github.com/getsentry/sentry-go v0.22.0

2
go.sum
View File

@ -84,6 +84,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y=
github.com/bsm/redislock v0.9.4 h1:X/Wse1DPpiQgHbVYRE9zv6m070UcKoOGekgvpNhiSvw=
github.com/bsm/redislock v0.9.4/go.mod h1:Epf7AJLiSFwLCiZcfi6pWFO/8eAYrYpQXFxEDPoDeAk=
github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M= github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M=
github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY= github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY=

131
internal/admin/init.go
View File

@ -2,12 +2,22 @@ package admin
import ( import (
"context" "context"
"time"
"forge.cadoles.com/cadoles/bouncer/internal/config"
"forge.cadoles.com/cadoles/bouncer/internal/schema"
"forge.cadoles.com/cadoles/bouncer/internal/setup" "forge.cadoles.com/cadoles/bouncer/internal/setup"
"forge.cadoles.com/cadoles/bouncer/internal/store"
"github.com/bsm/redislock"
"github.com/pkg/errors" "github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
) )
func (s *Server) initRepositories(ctx context.Context) error { func (s *Server) initRepositories(ctx context.Context) error {
if err := s.initRedisClient(ctx); err != nil {
return errors.WithStack(err)
}
if err := s.initLayerRepository(ctx); err != nil { if err := s.initLayerRepository(ctx); err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }
@ -19,8 +29,16 @@ func (s *Server) initRepositories(ctx context.Context) error {
return nil return nil
} }
func (s *Server) initRedisClient(ctx context.Context) error {
client := setup.NewRedisClient(ctx, s.redisConfig)
s.redisClient = client
return nil
}
func (s *Server) initLayerRepository(ctx context.Context) error { func (s *Server) initLayerRepository(ctx context.Context) error {
layerRepository, err := setup.NewLayerRepository(ctx, s.redisConfig) layerRepository, err := setup.NewLayerRepository(ctx, s.redisClient)
if err != nil { if err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }
@ -31,7 +49,7 @@ func (s *Server) initLayerRepository(ctx context.Context) error {
} }
func (s *Server) initProxyRepository(ctx context.Context) error { func (s *Server) initProxyRepository(ctx context.Context) error {
proxyRepository, err := setup.NewProxyRepository(ctx, s.redisConfig) proxyRepository, err := setup.NewProxyRepository(ctx, s.redisClient)
if err != nil { if err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }
@ -40,3 +58,112 @@ func (s *Server) initProxyRepository(ctx context.Context) error {
return nil return nil
} }
const bootstrapLockKey = "bouncer-bootstrap"
func (s *Server) bootstrapProxies(ctx context.Context) error {
if err := s.validateBootstrap(ctx); err != nil {
return errors.Wrap(err, "could not validate bootstrapped proxies")
}
proxyRepo := s.proxyRepository
layerRepo := s.layerRepository
locker := redislock.New(s.redisClient)
backoff := redislock.ExponentialBackoff(time.Second, time.Duration(s.bootstrapConfig.LockTimeout)*2)
logger.Debug(ctx, "acquiring proxies bootstrap lock", logger.F("lockTimeout", s.bootstrapConfig.LockTimeout))
lock, err := locker.Obtain(ctx, bootstrapLockKey, time.Duration(s.bootstrapConfig.LockTimeout), &redislock.Options{
RetryStrategy: backoff,
})
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := lock.Release(ctx); err != nil {
logger.Error(ctx, "could not release lock", logger.E(errors.WithStack(err)))
}
}()
logger.Info(ctx, "bootstrapping proxies")
for proxyName, proxyConfig := range s.bootstrapConfig.Proxies {
_, err := s.proxyRepository.GetProxy(ctx, proxyName)
if !errors.Is(err, store.ErrNotFound) {
if err != nil {
return errors.WithStack(err)
}
logger.Info(ctx, "ignoring existing proxy", logger.F("proxyName", proxyName))
continue
}
logger.Info(ctx, "creating proxy", logger.F("proxyName", proxyName))
if _, err := proxyRepo.CreateProxy(ctx, proxyName, string(proxyConfig.To), proxyConfig.From...); err != nil {
return errors.WithStack(err)
}
_, err = proxyRepo.UpdateProxy(
ctx, proxyName,
store.WithProxyUpdateEnabled(bool(proxyConfig.Enabled)),
store.WithProxyUpdateWeight(int(proxyConfig.Weight)),
)
if err != nil {
return errors.WithStack(err)
}
for layerName, layerConfig := range proxyConfig.Layers {
layerType := store.LayerType(layerConfig.Type)
layerOptions := store.LayerOptions(layerConfig.Options)
if _, err := layerRepo.CreateLayer(ctx, proxyName, layerName, layerType, layerOptions); err != nil {
return errors.WithStack(err)
}
_, err := layerRepo.UpdateLayer(
ctx,
proxyName, layerName,
store.WithLayerUpdateEnabled(bool(layerConfig.Enabled)),
store.WithLayerUpdateOptions(layerOptions),
store.WithLayerUpdateWeight(int(layerConfig.Weight)),
)
if err != nil {
return errors.WithStack(err)
}
}
}
return nil
}
const validateErrMessage = "could not validate proxy '%s': could not validate layer '%s'"
func (s *Server) validateBootstrap(ctx context.Context) error {
for proxyName, proxyConf := range s.bootstrapConfig.Proxies {
for layerName, layerConf := range proxyConf.Layers {
layerType := store.LayerType(layerConf.Type)
if !setup.LayerTypeExists(layerType) {
return errors.Errorf(validateErrMessage+": could not find layer type '%s'", proxyName, layerName, layerType)
}
layerOptionsSchema, err := setup.GetLayerOptionsSchema(layerType)
if err != nil {
return errors.Wrapf(err, validateErrMessage, proxyName, layerName)
}
rawOptions := func(opts config.InterpolatedMap) map[string]any {
return opts
}(layerConf.Options)
if err := schema.Validate(ctx, layerOptionsSchema, rawOptions); err != nil {
return errors.Wrapf(err, validateErrMessage, proxyName, layerName)
}
}
}
return nil
}

9
internal/admin/layer_route.go
View File

@ -51,15 +51,6 @@ func (s *Server) queryLayer(w http.ResponseWriter, r *http.Request) {
}) })
} }
func validateLayerName(v string) (store.LayerName, error) {
name, err := store.ValidateName(v)
if err != nil {
return "", errors.WithStack(err)
}
return store.LayerName(name), nil
}
type GetLayerResponse struct { type GetLayerResponse struct {
Layer *store.Layer `json:"layer"` Layer *store.Layer `json:"layer"`
} }

11
internal/admin/option.go
View File

@ -5,8 +5,9 @@ import (
) )
type Option struct { type Option struct {
ServerConfig config.AdminServerConfig BootstrapConfig config.BootstrapConfig
RedisConfig config.RedisConfig ServerConfig config.AdminServerConfig
RedisConfig config.RedisConfig
} }
type OptionFunc func(*Option) type OptionFunc func(*Option)
@ -29,3 +30,9 @@ func WithRedisConfig(conf config.RedisConfig) OptionFunc {
opt.RedisConfig = conf opt.RedisConfig = conf
} }
} }
func WithBootstrapConfig(conf config.BootstrapConfig) OptionFunc {
return func(opt *Option) {
opt.BootstrapConfig = conf
}
}

17
internal/admin/proxy_route.go
View File

@ -114,6 +114,23 @@ func (s *Server) deleteProxy(w http.ResponseWriter, r *http.Request) {
return return
} }
layers, err := s.layerRepository.QueryLayers(ctx, proxyName)
if err != nil {
logAndCaptureError(ctx, "could not query proxy's layers", errors.WithStack(err))
api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
return
}
for _, layer := range layers {
if err := s.layerRepository.DeleteLayer(ctx, proxyName, layer.Name); err != nil {
logAndCaptureError(ctx, "could not delete layer", errors.WithStack(err))
api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
return
}
}
api.DataResponse(w, http.StatusOK, DeleteProxyResponse{ api.DataResponse(w, http.StatusOK, DeleteProxyResponse{
ProxyName: proxyName, ProxyName: proxyName,
}) })

14
internal/admin/server.go
View File

@ -19,12 +19,15 @@ import (
"github.com/go-chi/cors" "github.com/go-chi/cors"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/redis/go-redis/v9"
"gitlab.com/wpetit/goweb/logger" "gitlab.com/wpetit/goweb/logger"
) )
type Server struct { type Server struct {
serverConfig config.AdminServerConfig serverConfig config.AdminServerConfig
redisConfig config.RedisConfig redisConfig config.RedisConfig
redisClient redis.UniversalClient
bootstrapConfig config.BootstrapConfig
proxyRepository store.ProxyRepository proxyRepository store.ProxyRepository
layerRepository store.LayerRepository layerRepository store.LayerRepository
} }
@ -53,6 +56,12 @@ func (s *Server) run(parentCtx context.Context, addrs chan net.Addr, errs chan e
return return
} }
if err := s.bootstrapProxies(ctx); err != nil {
errs <- errors.WithStack(err)
return
}
listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", s.serverConfig.HTTP.Host, s.serverConfig.HTTP.Port)) listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", s.serverConfig.HTTP.Host, s.serverConfig.HTTP.Port))
if err != nil { if err != nil {
errs <- errors.WithStack(err) errs <- errors.WithStack(err)
@ -175,7 +184,8 @@ func NewServer(funcs ...OptionFunc) *Server {
} }
return &Server{ return &Server{
serverConfig: opt.ServerConfig, serverConfig: opt.ServerConfig,
redisConfig: opt.RedisConfig, redisConfig: opt.RedisConfig,
bootstrapConfig: opt.BootstrapConfig,
} }
} }

1
internal/command/server/admin/run.go
View File

@ -68,6 +68,7 @@ func RunCommand() *cli.Command {
srv := admin.NewServer( srv := admin.NewServer(
admin.WithServerConfig(conf.Admin), admin.WithServerConfig(conf.Admin),
admin.WithRedisConfig(conf.Redis), admin.WithRedisConfig(conf.Redis),
admin.WithBootstrapConfig(conf.Bootstrap),
) )
addrs, srvErrs := srv.Start(ctx.Context) addrs, srvErrs := srv.Start(ctx.Context)

104
internal/config/bootstrap.go Normal file
View File

@ -0,0 +1,104 @@
package config
import (
"os"
"path/filepath"
"strings"
"time"
"forge.cadoles.com/cadoles/bouncer/internal/store"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
)
type BootstrapConfig struct {
Proxies map[store.ProxyName]BootstrapProxyConfig `yaml:"proxies"`
Dir InterpolatedString `yaml:"dir"`
LockTimeout InterpolatedDuration `yaml:"lockTimeout"`
}
func (c *BootstrapConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
src := struct {
Proxies map[store.ProxyName]BootstrapProxyConfig `yaml:"proxies"`
Dir InterpolatedString `yaml:"dir"`
}{
Proxies: make(map[store.ProxyName]BootstrapProxyConfig),
Dir: "",
}
if err := unmarshal(&src); err != nil {
return errors.WithStack(err)
}
c.Proxies = src.Proxies
c.Dir = src.Dir
if src.Dir != "" {
proxies, err := loadBootstrapDir(string(src.Dir))
if err != nil {
return errors.Wrapf(err, "could not load bootstrap dir '%s'", src.Dir)
}
c.Proxies = overrideProxies(c.Proxies, proxies)
}
return nil
}
type BootstrapProxyConfig struct {
Enabled InterpolatedBool `yaml:"enabled"`
Weight InterpolatedInt `yaml:"weight"`
To InterpolatedString `yaml:"to"`
From InterpolatedStringSlice `yaml:"from"`
Layers map[store.LayerName]BootstrapLayerConfig `yaml:"layers"`
}
type BootstrapLayerConfig struct {
Enabled InterpolatedBool `yaml:"enabled"`
Type InterpolatedString `yaml:"type"`
Weight InterpolatedInt `yaml:"weight"`
Options InterpolatedMap `yaml:"options"`
}
func NewDefaultBootstrapConfig() BootstrapConfig {
return BootstrapConfig{
Dir: "",
LockTimeout: *NewInterpolatedDuration(30 * time.Second),
}
}
func loadBootstrapDir(dir string) (map[store.ProxyName]BootstrapProxyConfig, error) {
pattern := filepath.Join(dir, "*.yml")
files, err := filepath.Glob(pattern)
if err != nil {
return nil, errors.WithStack(err)
}
proxies := make(map[store.ProxyName]BootstrapProxyConfig)
for _, f := range files {
data, err := os.ReadFile(f)
if err != nil {
return nil, errors.Wrapf(err, "could not read file '%s'", f)
}
proxy := BootstrapProxyConfig{}
if err := yaml.Unmarshal(data, &proxy); err != nil {
return nil, errors.Wrapf(err, "could not unmarshal proxy")
}
name := store.ProxyName(strings.TrimSuffix(filepath.Base(f), filepath.Ext(f)))
proxies[name] = proxy
}
return proxies, nil
}
func overrideProxies(base map[store.ProxyName]BootstrapProxyConfig, proxies map[store.ProxyName]BootstrapProxyConfig) map[store.ProxyName]BootstrapProxyConfig {
for name, proxy := range proxies {
base[name] = proxy
}
return base
}

26
internal/config/config.go
View File

@ -2,7 +2,7 @@ package config
import ( import (
"io" "io"
"io/ioutil" "os"
"github.com/pkg/errors" "github.com/pkg/errors"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
@ -10,18 +10,19 @@ import (
// Config definition // Config definition
type Config struct { type Config struct {
Admin AdminServerConfig `yaml:"admin"` Admin AdminServerConfig `yaml:"admin"`
Proxy ProxyServerConfig `yaml:"proxy"` Proxy ProxyServerConfig `yaml:"proxy"`
Redis RedisConfig `yaml:"redis"` Redis RedisConfig `yaml:"redis"`
Logger LoggerConfig `yaml:"logger"` Logger LoggerConfig `yaml:"logger"`
Layers LayersConfig `yaml:"layers"` Layers LayersConfig `yaml:"layers"`
Bootstrap BootstrapConfig `yaml:"bootstrap"`
} }
// NewFromFile retrieves the configuration from the given file // NewFromFile retrieves the configuration from the given file
func NewFromFile(path string) (*Config, error) { func NewFromFile(path string) (*Config, error) {
config := NewDefault() config := NewDefault()
data, err := ioutil.ReadFile(path) data, err := os.ReadFile(path)
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "could not read file '%s'", path) return nil, errors.Wrapf(err, "could not read file '%s'", path)
} }
@ -43,11 +44,12 @@ func NewDumpDefault() *Config {
// NewDefault return new default configuration // NewDefault return new default configuration
func NewDefault() *Config { func NewDefault() *Config {
return &Config{ return &Config{
Admin: NewDefaultAdminServerConfig(), Admin: NewDefaultAdminServerConfig(),
Proxy: NewDefaultProxyServerConfig(), Proxy: NewDefaultProxyServerConfig(),
Logger: NewDefaultLoggerConfig(), Logger: NewDefaultLoggerConfig(),
Redis: NewDefaultRedisConfig(), Redis: NewDefaultRedisConfig(),
Layers: NewDefaultLayersConfig(), Layers: NewDefaultLayersConfig(),
Bootstrap: NewDefaultBootstrapConfig(),
} }
} }

15
internal/proxy/init.go
View File

@ -5,22 +5,25 @@ import (
"forge.cadoles.com/cadoles/bouncer/internal/setup" "forge.cadoles.com/cadoles/bouncer/internal/setup"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/redis/go-redis/v9"
) )
func (s *Server) initRepositories(ctx context.Context) error { func (s *Server) initRepositories(ctx context.Context) error {
if err := s.initProxyRepository(ctx); err != nil { client := setup.NewRedisClient(ctx, s.redisConfig)
if err := s.initProxyRepository(ctx, client); err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }
if err := s.initLayerRepository(ctx); err != nil { if err := s.initLayerRepository(ctx, client); err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }
return nil return nil
} }
func (s *Server) initProxyRepository(ctx context.Context) error { func (s *Server) initProxyRepository(ctx context.Context, client redis.UniversalClient) error {
proxyRepository, err := setup.NewProxyRepository(ctx, s.redisConfig) proxyRepository, err := setup.NewProxyRepository(ctx, client)
if err != nil { if err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }
@ -30,8 +33,8 @@ func (s *Server) initProxyRepository(ctx context.Context) error {
return nil return nil
} }
func (s *Server) initLayerRepository(ctx context.Context) error { func (s *Server) initLayerRepository(ctx context.Context, client redis.UniversalClient) error {
layerRepository, err := setup.NewLayerRepository(ctx, s.redisConfig) layerRepository, err := setup.NewLayerRepository(ctx, client)
if err != nil { if err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }

19
internal/setup/proxy_repository.go
View File

@ -9,16 +9,17 @@ import (
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
) )
func NewProxyRepository(ctx context.Context, conf config.RedisConfig) (store.ProxyRepository, error) { func NewRedisClient(ctx context.Context, conf config.RedisConfig) redis.UniversalClient {
rdb := newRedisClient(conf) return redis.NewUniversalClient(&redis.UniversalOptions{
return redisStore.NewProxyRepository(rdb), nil
}
func NewLayerRepository(ctx context.Context, conf config.RedisConfig) (store.LayerRepository, error) {
rdb := redis.NewUniversalClient(&redis.UniversalOptions{
Addrs: conf.Adresses, Addrs: conf.Adresses,
MasterName: string(conf.Master), MasterName: string(conf.Master),
}) })
}
return redisStore.NewLayerRepository(rdb), nil
func NewProxyRepository(ctx context.Context, client redis.UniversalClient) (store.ProxyRepository, error) {
return redisStore.NewProxyRepository(client), nil
}
func NewLayerRepository(ctx context.Context, client redis.UniversalClient) (store.LayerRepository, error) {
return redisStore.NewLayerRepository(client), nil
} }

62
misc/k8s/README.md
View File

@ -4,57 +4,60 @@
1. Generate the Docker configuration to enable image builds with Kaniko and communicate with reg.cadoles.com 1. Generate the Docker configuration to enable image builds with Kaniko and communicate with reg.cadoles.com
```shell ```shell
docker login reg.cadoles.com docker login reg.cadoles.com
mkdir -p misc/k8s/kustomization/base/secrets/dockerconfig mkdir -p misc/k8s/kustomization/base/secrets/dockerconfig
docker --config misc/k8s/kustomization/base/secrets/dockerconfig login reg.cadoles.com docker --config misc/k8s/kustomization/base/secrets/dockerconfig login reg.cadoles.com
mv misc/k8s/kustomization/base/secrets/dockerconfig/config.json misc/k8s/kustomization/base/secrets/dockerconfig/.dockerconfigjson mv misc/k8s/kustomization/base/secrets/dockerconfig/config.json misc/k8s/kustomization/base/secrets/dockerconfig/.dockerconfigjson
mkdir -p misc/k8s/kustomization/overlays/dev/secrets/dockerconfig mkdir -p misc/k8s/kustomization/overlays/dev/secrets/dockerconfig
cp misc/k8s/kustomization/base/secrets/dockerconfig/.dockerconfigjson misc/k8s/kustomization/overlays/dev/secrets/dockerconfig/.dockerconfigjson cp misc/k8s/kustomization/base/secrets/dockerconfig/.dockerconfigjson misc/k8s/kustomization/overlays/dev/secrets/dockerconfig/.dockerconfigjson
``` ```
## Getting started with Kind ## Getting started with Kind
1. Create your [Kind](https://kind.sigs.k8s.io/) cluster 1. Create your [Kind](https://kind.sigs.k8s.io/) cluster
```shell ```shell
kind create cluster --config misc/k8s/kind/bouncer-cluster.yaml kind create cluster --config misc/k8s/kind/bouncer-cluster.yaml
``` ```
2. Deploy required operators 2. Deploy required operators
```shell ```shell
kubectl apply -k misc/k8s/kind/cluster --server-side kubectl apply -k misc/k8s/kind/cluster --server-side
``` ```
3. Deploy your Bouncer development environment 3. Deploy your Bouncer development environment
```shell ```shell
skaffold dev -p dev --cleanup=false --default-repo reg.cadoles.com/<YOUR_PERSONNAL_USER_NAME> skaffold dev -p dev --cleanup=false --default-repo reg.cadoles.com/<YOUR_PERSONNAL_USER_NAME>
``` ```
## Testing ## Testing
Bouncer will automatically create proxies based on the files present in the `misc/k8s/kustomization/overlays/dev/files/bouncer/bootstrap.d` folder.
By default, with you host web browser, open http://localhost:9000, you should see the Cadoles website.
### Using the admin API
1. Open shell in bouncer-admin pod 1. Open shell in bouncer-admin pod
```shell ```shell
kubectl exec -it -n bouncer-dev bouncer-admin-<suffix> -- /bin/sh kubectl exec -it -n bouncer-dev bouncer-admin-<suffix> -- /bin/sh
``` ```
2. Create an authentication token 2. Create an authentication token
```shell ```shell
bouncer --config /etc/bouncer/config.yml auth create-token --role writer --subject $(whoami) > .bouncer-token bouncer --config /etc/bouncer/config.yml auth create-token --role writer --subject $(whoami) > .bouncer-token
``` ```
3. Create a proxy and enable it 3. Create a proxy and enable it
```shell ```shell
bouncer admin proxy create --proxy-to https://www.cadoles.com --proxy-name cadoles bouncer admin proxy query
bouncer admin proxy update --proxy-name cadoles --proxy-enabled=true ```
```
4. With you host web browser, open http://localhost:9000, you should see the Cadoles website.
## Benchmarking ## Benchmarking
@ -63,4 +66,3 @@ You can use [`siege`](https://github.com/JoeDog/siege) to benchmark your instanc
```shell ```shell
BASE_URL=http://localhost:9000 make siege BASE_URL=http://localhost:9000 make siege
``` ```

28
misc/k8s/kustomization/base/resources/bouncer-admin/files/config.yml
View File

@ -4,19 +4,19 @@ admin:
port: 8081 port: 8081
cors: cors:
allowedOrigins: allowedOrigins:
- http://localhost:3001 - http://localhost:3001
allowCredentials: true allowCredentials: true
allowMethods: allowMethods:
- POST - POST
- GET - GET
- PUT - PUT
- DELETE - DELETE
allowedHeaders: allowedHeaders:
- Origin - Origin
- Accept - Accept
- Content-Type - Content-Type
- Authorization - Authorization
- Sentry-Trace - Sentry-Trace
debug: false debug: false
auth: auth:
issuer: http://127.0.0.1:8081 issuer: http://127.0.0.1:8081
@ -28,9 +28,13 @@ admin:
redis: redis:
addresses: addresses:
- rfs-bouncer-redis:${RFS_BOUNCER_REDIS_SERVICE_PORT} - rfs-bouncer-redis:${RFS_BOUNCER_REDIS_SERVICE_PORT}
master: mymaster master: mymaster
logger: logger:
level: 2 level: ${BOUNCER_LOG_LEVEL}
format: human format: human
bootstrap:
dir: /etc/bouncer/bootstrap.d
lockTimeout: 30s

16
misc/k8s/kustomization/base/resources/bouncer-admin/kustomization.yaml
View File

@ -2,11 +2,15 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization kind: Kustomization
resources: resources:
- ./resources/service.yaml - ./resources/service.yaml
- ./resources/deployment.yaml - ./resources/deployment.yaml
configMapGenerator: configMapGenerator:
- name: bouncer-admin-config - name: bouncer-admin-config
files: files:
- ./files/config.yml - ./files/config.yml
- ./files/admin-key.json - ./files/admin-key.json
- name: bouncer-admin-bootstrap
- name: bouncer-admin-env
literals:
- BOUNCER_LOG_LEVEL=2

43
misc/k8s/kustomization/base/resources/bouncer-admin/resources/deployment.yaml
View File

@ -17,18 +17,35 @@ spec:
io.kompose.service: bouncer-admin io.kompose.service: bouncer-admin
spec: spec:
containers: containers:
- name: bouncer-admin
image: reg.cadoles.com/cadoles/bouncer:v2024.2.5-1602626
command: ["bouncer", "--debug", "-c", "/etc/bouncer/config.yml", "server", "admin", "run"]
imagePullPolicy: Always
resources: {}
ports:
- name: bouncer-admin - name: bouncer-admin
containerPort: 8081 image: bouncer
volumeMounts: command:
- mountPath: /etc/bouncer/ [
name: bouncer-admin-config "bouncer",
"--debug",
"-c",
"/etc/bouncer/config.yml",
"server",
"admin",
"run",
]
imagePullPolicy: Always
resources: {}
ports:
- name: bouncer-admin
containerPort: 8081
envFrom:
- configMapRef:
name: bouncer-admin-env
volumeMounts:
- mountPath: /etc/bouncer/
name: bouncer-admin-config
- mountPath: /etc/bouncer/bootstrap.d
name: bouncer-admin-bootstrap
volumes: volumes:
- name: bouncer-admin-config - name: bouncer-admin-config
configMap: configMap:
name: bouncer-admin-config name: bouncer-admin-config
- name: bouncer-admin-bootstrap
configMap:
name: bouncer-admin-bootstrap

4
misc/k8s/kustomization/base/resources/bouncer-server/files/config.yml
View File

@ -14,9 +14,9 @@ layers:
redis: redis:
addresses: addresses:
- rfs-bouncer-redis:${RFS_BOUNCER_REDIS_SERVICE_PORT} - rfs-bouncer-redis:${RFS_BOUNCER_REDIS_SERVICE_PORT}
master: mymaster master: mymaster
logger: logger:
level: 2 level: ${BOUNCER_LOG_LEVEL}
format: human format: human

13
misc/k8s/kustomization/base/resources/bouncer-server/kustomization.yaml
View File

@ -2,10 +2,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization kind: Kustomization
resources: resources:
- ./resources/service.yaml - ./resources/service.yaml
- ./resources/deployment.yaml - ./resources/deployment.yaml
configMapGenerator: configMapGenerator:
- name: bouncer-server-config - name: bouncer-server-config
files: files:
- ./files/config.yml - ./files/config.yml
- name: bouncer-server-env
literals:
- BOUNCER_LOG_LEVEL=2

37
misc/k8s/kustomization/base/resources/bouncer-server/resources/deployment.yaml
View File

@ -17,18 +17,29 @@ spec:
io.kompose.service: bouncer-server io.kompose.service: bouncer-server
spec: spec:
containers: containers:
- name: bouncer-server
image: reg.cadoles.com/cadoles/bouncer:v2024.2.5-1602626
command: ["bouncer", "-c", "/etc/bouncer/config.yml", "server", "proxy", "run"]
imagePullPolicy: Always
resources: {}
ports:
- name: bouncer-server - name: bouncer-server
containerPort: 8080 image: bouncer
volumeMounts: command:
- mountPath: /etc/bouncer/ [
name: bouncer-server-config "bouncer",
"-c",
"/etc/bouncer/config.yml",
"server",
"proxy",
"run",
]
imagePullPolicy: Always
envFrom:
- configMapRef:
name: bouncer-server-env
resources: {}
ports:
- name: bouncer-server
containerPort: 8080
volumeMounts:
- mountPath: /etc/bouncer/
name: bouncer-server-config
volumes: volumes:
- name: bouncer-server-config - name: bouncer-server-config
configMap: configMap:
name: bouncer-server-config name: bouncer-server-config

11
misc/k8s/kustomization/overlays/dev/files/bouncer/bootstrap.d/cadoles.yml Normal file
View File

@ -0,0 +1,11 @@
from: ["*"]
to: https://www.cadoles.com
enabled: true
weight: 0
layers:
my-queue:
type: queue
enabled: true
weight: 0
options:
capacity: 10

32
misc/k8s/kustomization/overlays/dev/kustomization.yaml
View File

@ -3,16 +3,30 @@ kind: Kustomization
namespace: bouncer-dev namespace: bouncer-dev
resources: resources:
- ../../base - ../../base
secretGenerator: secretGenerator:
- files: - files:
- secrets/dockerconfig/.dockerconfigjson - secrets/dockerconfig/.dockerconfigjson
name: regcred-dev name: regcred-dev
type: kubernetes.io/dockerconfigjson type: kubernetes.io/dockerconfigjson
patches: patches:
- path: patches/add-registry-pull-secret.patch.yaml - path: patches/add-registry-pull-secret.patch.yaml
target: target:
kind: Deployment kind: Deployment
version: v1 version: v1
configMapGenerator:
- name: bouncer-admin-bootstrap
behavior: merge
files:
- ./files/bouncer/bootstrap.d/cadoles.yml
- name: bouncer-admin-env
behavior: merge
literals:
- BOUNCER_LOG_LEVEL=0
- name: bouncer-server-env
behavior: merge
literals:
- BOUNCER_LOG_LEVEL=0

55
misc/packaging/common/config.yml
View File

@ -15,19 +15,19 @@ admin:
# est branché sur l'API d'administration. # est branché sur l'API d'administration.
cors: cors:
allowedOrigins: allowedOrigins:
- http://localhost:8081 - http://localhost:8081
allowCredentials: true allowCredentials: true
allowMethods: allowMethods:
- POST - POST
- GET - GET
- PUT - PUT
- DELETE - DELETE
allowedHeaders: allowedHeaders:
- Origin - Origin
- Accept - Accept
- Content-Type - Content-Type
- Authorization - Authorization
- Sentry-Trace - Sentry-Trace
debug: false debug: false
# Authentification JWT # Authentification JWT
@ -145,7 +145,7 @@ proxy:
# - Mode "cluster": renseigner plusieurs adresses dans redis.addresses et laisser redis.master vide. # - Mode "cluster": renseigner plusieurs adresses dans redis.addresses et laisser redis.master vide.
redis: redis:
addresses: addresses:
- localhost:6379 - localhost:6379
master: "" master: ""
writeTimeout: 30s writeTimeout: 30s
readTimeout: 30s readTimeout: 30s
@ -177,3 +177,36 @@ layers:
# Répertoire contenant les templates # Répertoire contenant les templates
templateDir: "/etc/bouncer/layers/circuitbreaker/templates" templateDir: "/etc/bouncer/layers/circuitbreaker/templates"
# Configuration d'une série de proxy/layers
# à créer par défaut par le serveur d'administration
bootstrap:
# Répertoire contenant les définitions de proxy à créer
# par défaut. Les fichiers seront récupérés si ils
# correspondent au patron de nommage suivant:
#
# <bootstrap_dir>/<proxy_name>.yml
#
# Si l'attribut est vide ou absent le chargement des fichiers
# est désactivé.
dir: /etc/bouncer/bootstrap.d
# Délai d'expiration du verrou distribué utilisé lors du chargement
# des définitions de proxy par défaut.
lockTimeout: 30s
# Tableau associatif de définition de proxies à créer par
# défaut par le serveur d'administration.
# Si `proxies` et `dir` sont tous les deux définis, les fichiers
# présents dans le répertoire `dir` surchargeront les valeurs définies
# dans `proxies`.
# Par défault non défini
proxies:
# my-proxy:
# enabled: true
# from: ["*"]
# to: "https://example.net"
# weight: 0
# layers:
# my-layer:
# type: queue
# enabled: false
# weight: 0
# options: {"capacity": 100}

56
skaffold.yaml
View File

@ -7,16 +7,16 @@ metadata:
manifests: manifests:
kustomize: kustomize:
paths: paths:
- misc/k8s/kustomization/base - misc/k8s/kustomization/base
profiles: profiles:
- name: dev - name: dev
manifests: manifests:
kustomize: kustomize:
paths: paths:
- misc/k8s/kustomization/overlays/dev - misc/k8s/kustomization/overlays/dev
activation: activation:
- command: dev - command: dev
build: build:
local: local:
@ -26,28 +26,28 @@ build:
sha256: {} sha256: {}
artifacts: artifacts:
- image: reg.cadoles.com/cadoles/bouncer - image: bouncer
context: . context: .
sync: sync:
infer: infer:
- cmd/** - cmd/**
- internal/** - internal/**
- layers/** - layers/**
- misc/** - misc/**
docker: docker:
dockerfile: Dockerfile dockerfile: Dockerfile
deploy: deploy:
statusCheckDeadlineSeconds: 600 statusCheckDeadlineSeconds: 600
portForward: portForward:
- resourceType: service - resourceType: service
resourceName: bouncer-admin resourceName: bouncer-admin
namespace: bouncer-dev namespace: bouncer-dev
port: 8081 port: 8081
localPort: 9999 localPort: 9999
- resourceType: service - resourceType: service
resourceName: bouncer-server resourceName: bouncer-server
namespace: bouncer-dev namespace: bouncer-dev
port: 8080 port: 8080
localPort: 9000 # *Optional* localPort: 9000 # *Optional*