Merge pull request 'feat: bootstrap default proxy and layers from configuration' (#17) from proxy-bootstrap into develop
All checks were successful
Cadoles/bouncer/pipeline/head This commit looks good

Reviewed-on: #17
This commit is contained in:
wpetit 2024-03-26 17:30:20 +01:00
commit 61012b07cd
27 changed files with 725 additions and 312 deletions

View File

@ -1,148 +1,143 @@
project_name: bouncer
before:
hooks:
- go mod tidy
- go generate ./...
- go mod tidy
- go generate ./...
builds:
- id: bouncer
env:
- CGO_ENABLED=0
ldflags:
- -s
- -w
- -X 'main.GitRef={{ .Commit }}'
- -X 'main.ProjectVersion={{ .Version }}'
- -X 'main.BuildDate={{ .Date }}'
- -X 'main.DefaultConfigPath=/etc/bouncer/config.yml'
gcflags:
- -trimpath="${PWD}"
asmflags:
- -trimpath="${PWD}"
goos:
- linux
goarch:
- amd64
- arm64
- "386"
main: ./cmd/bouncer
- id: bouncer
env:
- CGO_ENABLED=0
ldflags:
- -s
- -w
- -X 'main.GitRef={{ .Commit }}'
- -X 'main.ProjectVersion={{ .Version }}'
- -X 'main.BuildDate={{ .Date }}'
- -X 'main.DefaultConfigPath=/etc/bouncer/config.yml'
gcflags:
- -trimpath="${PWD}"
asmflags:
- -trimpath="${PWD}"
goos:
- linux
goarch:
- amd64
- arm64
- "386"
main: ./cmd/bouncer
archives:
- id: bouncer
builds: ["bouncer"]
name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
files:
- README.md
- misc/packaging/common/config.yml
- id: bouncer
builds: ["bouncer"]
name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
files:
- README.md
- misc/packaging/common/config.yml
checksum:
name_template: 'checksums.txt'
name_template: "checksums.txt"
snapshot:
name_template: "{{ .Version }}"
changelog:
sort: asc
filters:
exclude:
- '^docs:'
- '^test:'
- "^docs:"
- "^test:"
nfpms:
- id: bouncer-bin
builds:
- "bouncer"
package_name: bouncer-bin
homepage: https://forge.cadoles.com/Cadoles/bouncer
maintainer: Cadoles <contact@cadoles.com>
description: |-
reverse proxy server with dynamic queuing management - binaries
license: AGPL-3.0
formats:
- apk
- deb
- rpm
- archlinux
contents:
- src: misc/packaging/common/config.yml
dst: /etc/bouncer/config.yml
type: config
- src: layers
dst: /etc/bouncer/layers
type: config
- id: bouncer-admin
meta: true
package_name: bouncer-admin
homepage: https://forge.cadoles.com/Cadoles/bouncer
maintainer: Cadoles <contact@cadoles.com>
dependencies:
- bouncer-bin
description: |-
reverse proxy server with dynamic queuing management - administration service
license: AGPL-3.0
formats:
- apk
- deb
- rpm
- archlinux
contents:
- src: misc/packaging/systemd/bouncer-admin.systemd.service
dst: /usr/lib/systemd/system/bouncer-admin.service
packager: deb
- src: misc/packaging/systemd/bouncer-admin.systemd.service
dst: /usr/lib/systemd/system/bouncer-admin.service
packager: rpm
- src: misc/packaging/systemd/bouncer-admin.systemd.service
dst: /usr/lib/systemd/system/bouncer-admin.service
packager: archlinux
- src: misc/packaging/openrc/bouncer-admin.openrc.sh
dst: /etc/init.d/bouncer-admin
file_info:
mode: 0755
packager: apk
- dst: /usr/share/bouncer
type: dir
file_info:
mode: 0700
- dst: /var/log/bouncer
type: dir
file_info:
mode: 0700
packager: apk
scripts:
postinstall: "misc/packaging/common/postinstall-bouncer-admin.sh"
- id: bouncer-proxy
meta: true
dependencies:
- bouncer-bin
package_name: bouncer-proxy
homepage: https://forge.cadoles.com/Cadoles/bouncer
maintainer: Cadoles <contact@cadoles.com>
description: |-
reverse proxy server with dynamic queuing management - proxy service
license: AGPL-3.0
formats:
- apk
- deb
- rpm
- archlinux
contents:
- src: misc/packaging/systemd/bouncer-proxy.systemd.service
dst: /usr/lib/systemd/system/bouncer-proxy.service
packager: deb
- src: misc/packaging/systemd/bouncer-proxy.systemd.service
dst: /usr/lib/systemd/system/bouncer-proxy.service
packager: rpm
- src: misc/packaging/systemd/bouncer-proxy.systemd.service
dst: /usr/lib/systemd/system/bouncer-proxy.service
packager: archlinux
- src: misc/packaging/openrc/bouncer-proxy.openrc.sh
dst: /etc/init.d/bouncer-proxy
file_info:
mode: 0755
packager: apk
- dst: /usr/share/bouncer
type: dir
file_info:
mode: 0700
- dst: /var/log/bouncer
type: dir
file_info:
mode: 0700
packager: apk
scripts:
postinstall: "misc/packaging/common/postinstall-bouncer-proxy.sh"
- id: bouncer-bin
builds:
- "bouncer"
package_name: bouncer-bin
homepage: https://forge.cadoles.com/Cadoles/bouncer
maintainer: Cadoles <contact@cadoles.com>
description: |-
reverse proxy server with dynamic queuing management - binaries
license: AGPL-3.0
formats:
- apk
- deb
- rpm
contents:
- src: misc/packaging/common/config.yml
dst: /etc/bouncer/config.yml
type: config
- src: layers
dst: /etc/bouncer/layers
type: config
- dst: /etc/bouncer/bootstrap.d
type: dir
file_info:
mode: 0700
- id: bouncer-admin
meta: true
package_name: bouncer-admin
homepage: https://forge.cadoles.com/Cadoles/bouncer
maintainer: Cadoles <contact@cadoles.com>
dependencies:
- bouncer-bin
description: |-
reverse proxy server with dynamic queuing management - administration service
license: AGPL-3.0
formats:
- apk
- deb
- rpm
contents:
- src: misc/packaging/systemd/bouncer-admin.systemd.service
dst: /usr/lib/systemd/system/bouncer-admin.service
packager: deb
- src: misc/packaging/systemd/bouncer-admin.systemd.service
dst: /usr/lib/systemd/system/bouncer-admin.service
packager: rpm
- src: misc/packaging/openrc/bouncer-admin.openrc.sh
dst: /etc/init.d/bouncer-admin
file_info:
mode: 0755
packager: apk
- dst: /usr/share/bouncer
type: dir
file_info:
mode: 0700
- dst: /var/log/bouncer
type: dir
file_info:
mode: 0700
packager: apk
scripts:
postinstall: "misc/packaging/common/postinstall-bouncer-admin.sh"
- id: bouncer-proxy
meta: true
dependencies:
- bouncer-bin
package_name: bouncer-proxy
homepage: https://forge.cadoles.com/Cadoles/bouncer
maintainer: Cadoles <contact@cadoles.com>
description: |-
reverse proxy server with dynamic queuing management - proxy service
license: AGPL-3.0
formats:
- apk
- deb
- rpm
contents:
- src: misc/packaging/systemd/bouncer-proxy.systemd.service
dst: /usr/lib/systemd/system/bouncer-proxy.service
packager: deb
- src: misc/packaging/systemd/bouncer-proxy.systemd.service
dst: /usr/lib/systemd/system/bouncer-proxy.service
packager: rpm
- src: misc/packaging/openrc/bouncer-proxy.openrc.sh
dst: /etc/init.d/bouncer-proxy
file_info:
mode: 0755
packager: apk
- dst: /usr/share/bouncer
type: dir
file_info:
mode: 0700
- dst: /var/log/bouncer
type: dir
file_info:
mode: 0700
packager: apk
scripts:
postinstall: "misc/packaging/common/postinstall-bouncer-proxy.sh"

View File

@ -9,10 +9,15 @@ RUN mkdir -p /usr/local/bin \
&& wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64 \
&& chmod +x /usr/local/bin/yq
COPY . /src
WORKDIR /src
COPY go.mod .
COPY go.sum .
RUN go mod download
COPY . /src
RUN make GORELEASER_ARGS='build --rm-dist --single-target --snapshot' goreleaser
# Patch config

View File

@ -19,8 +19,9 @@
### Utilisation
- [(FR) - Ajouter un layer de type "file d'attente"](./fr/tutorials/add-queue-layer.md)
- [(FR) - Amorçage d'un serveur Bouncer via la configuration](./fr/tutorials/bootstrapping.md)
### Développement
- [(FR) - Démarrer avec les sources](./fr/tutorials/getting-started-with-sources.md)
- [(FR) - Créer son propre layer](./fr/tutorials/create-custom-layer.md)
- [(FR) - Créer son propre layer](./fr/tutorials/create-custom-layer.md)

View File

@ -0,0 +1,47 @@
# Amorçage d'un serveur Bouncer via la configuration
Il est possible d'amorcer des données par défaut (i.e. des "proxies" et "layers" associés) via la configuration du serveur d'administration.
> **Attention** Ce mécanisme de modifiera pas des proxies déjà existants dans la base de données du serveur Bouncer. Autrement dit, si un proxy est déjà pré-existant lors du démarrage du serveur Bouncer, il ne sera pas modifié.
La définition des proxies et layers par défaut s'effectue dans la section `bootstrap` du fichier de configuration. Deux possibilités pour définir les proxys à charger par défaut:
- Utiliser un répertoire contenant des fichiers YAML (un par proxy) en définissant le chemin du répertoire via l'attribut `bootstrap.dir`;
- Définir directement la liste des proxies via l'attribut `bootstrap.proxies`.
```yaml
# Configuration d'une série de proxy/layers
# à créer par défaut par le serveur d'administration
bootstrap:
# Répertoire contenant les définitions de proxy à créer
# par défaut. Les fichiers seront récupérés si ils
# correspondent au patron de nommage suivant:
#
# <bootstrap_dir>/<proxy_name>.yml
#
# Voir ci-dessous pour les attributs possibles dans les fichiers.
#
# Si l'attribut est vide ou absent le chargement des fichiers
# est désactivé.
dir: /etc/bouncer/bootstrap.d
# Tableau associatif de définition de proxies à créer par
# défaut par le serveur d'administration.
# Si `proxies` et `dir` sont tous les deux définis, les fichiers
# présents dans le répertoire `dir` surchargeront les valeurs définies
# dans `proxies`.
#
# Par défaut vide.
proxies:
# my-proxy:
# enabled: true # Activer/désactiver le proxy
# from: ["*"] # Filtre d'origine d'activation du proxy
# to: "https://example.net" # Destination du proxy
# weight: 0 # Priorité du proxy
# layers: # Layers associés au proxy
# my-layer:
# type: queue # Type du proxy
# enabled: false # Activer/désactiver le layer
# weight: 0 # Priorité du layer
# options: {"capacity": 100} # Options associées au layer
```

1
go.mod
View File

@ -5,6 +5,7 @@ go 1.20
require (
forge.cadoles.com/Cadoles/go-proxy v0.0.0-20230701194111-c6b3d482cca6
github.com/Masterminds/sprig/v3 v3.2.3
github.com/bsm/redislock v0.9.4
github.com/btcsuite/btcd/btcutil v1.1.3
github.com/drone/envsubst v1.0.3
github.com/getsentry/sentry-go v0.22.0

2
go.sum
View File

@ -84,6 +84,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y=
github.com/bsm/redislock v0.9.4 h1:X/Wse1DPpiQgHbVYRE9zv6m070UcKoOGekgvpNhiSvw=
github.com/bsm/redislock v0.9.4/go.mod h1:Epf7AJLiSFwLCiZcfi6pWFO/8eAYrYpQXFxEDPoDeAk=
github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M=
github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY=

View File

@ -2,12 +2,22 @@ package admin
import (
"context"
"time"
"forge.cadoles.com/cadoles/bouncer/internal/config"
"forge.cadoles.com/cadoles/bouncer/internal/schema"
"forge.cadoles.com/cadoles/bouncer/internal/setup"
"forge.cadoles.com/cadoles/bouncer/internal/store"
"github.com/bsm/redislock"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func (s *Server) initRepositories(ctx context.Context) error {
if err := s.initRedisClient(ctx); err != nil {
return errors.WithStack(err)
}
if err := s.initLayerRepository(ctx); err != nil {
return errors.WithStack(err)
}
@ -19,8 +29,16 @@ func (s *Server) initRepositories(ctx context.Context) error {
return nil
}
func (s *Server) initRedisClient(ctx context.Context) error {
client := setup.NewRedisClient(ctx, s.redisConfig)
s.redisClient = client
return nil
}
func (s *Server) initLayerRepository(ctx context.Context) error {
layerRepository, err := setup.NewLayerRepository(ctx, s.redisConfig)
layerRepository, err := setup.NewLayerRepository(ctx, s.redisClient)
if err != nil {
return errors.WithStack(err)
}
@ -31,7 +49,7 @@ func (s *Server) initLayerRepository(ctx context.Context) error {
}
func (s *Server) initProxyRepository(ctx context.Context) error {
proxyRepository, err := setup.NewProxyRepository(ctx, s.redisConfig)
proxyRepository, err := setup.NewProxyRepository(ctx, s.redisClient)
if err != nil {
return errors.WithStack(err)
}
@ -40,3 +58,112 @@ func (s *Server) initProxyRepository(ctx context.Context) error {
return nil
}
const bootstrapLockKey = "bouncer-bootstrap"
func (s *Server) bootstrapProxies(ctx context.Context) error {
if err := s.validateBootstrap(ctx); err != nil {
return errors.Wrap(err, "could not validate bootstrapped proxies")
}
proxyRepo := s.proxyRepository
layerRepo := s.layerRepository
locker := redislock.New(s.redisClient)
backoff := redislock.ExponentialBackoff(time.Second, time.Duration(s.bootstrapConfig.LockTimeout)*2)
logger.Debug(ctx, "acquiring proxies bootstrap lock", logger.F("lockTimeout", s.bootstrapConfig.LockTimeout))
lock, err := locker.Obtain(ctx, bootstrapLockKey, time.Duration(s.bootstrapConfig.LockTimeout), &redislock.Options{
RetryStrategy: backoff,
})
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := lock.Release(ctx); err != nil {
logger.Error(ctx, "could not release lock", logger.E(errors.WithStack(err)))
}
}()
logger.Info(ctx, "bootstrapping proxies")
for proxyName, proxyConfig := range s.bootstrapConfig.Proxies {
_, err := s.proxyRepository.GetProxy(ctx, proxyName)
if !errors.Is(err, store.ErrNotFound) {
if err != nil {
return errors.WithStack(err)
}
logger.Info(ctx, "ignoring existing proxy", logger.F("proxyName", proxyName))
continue
}
logger.Info(ctx, "creating proxy", logger.F("proxyName", proxyName))
if _, err := proxyRepo.CreateProxy(ctx, proxyName, string(proxyConfig.To), proxyConfig.From...); err != nil {
return errors.WithStack(err)
}
_, err = proxyRepo.UpdateProxy(
ctx, proxyName,
store.WithProxyUpdateEnabled(bool(proxyConfig.Enabled)),
store.WithProxyUpdateWeight(int(proxyConfig.Weight)),
)
if err != nil {
return errors.WithStack(err)
}
for layerName, layerConfig := range proxyConfig.Layers {
layerType := store.LayerType(layerConfig.Type)
layerOptions := store.LayerOptions(layerConfig.Options)
if _, err := layerRepo.CreateLayer(ctx, proxyName, layerName, layerType, layerOptions); err != nil {
return errors.WithStack(err)
}
_, err := layerRepo.UpdateLayer(
ctx,
proxyName, layerName,
store.WithLayerUpdateEnabled(bool(layerConfig.Enabled)),
store.WithLayerUpdateOptions(layerOptions),
store.WithLayerUpdateWeight(int(layerConfig.Weight)),
)
if err != nil {
return errors.WithStack(err)
}
}
}
return nil
}
const validateErrMessage = "could not validate proxy '%s': could not validate layer '%s'"
func (s *Server) validateBootstrap(ctx context.Context) error {
for proxyName, proxyConf := range s.bootstrapConfig.Proxies {
for layerName, layerConf := range proxyConf.Layers {
layerType := store.LayerType(layerConf.Type)
if !setup.LayerTypeExists(layerType) {
return errors.Errorf(validateErrMessage+": could not find layer type '%s'", proxyName, layerName, layerType)
}
layerOptionsSchema, err := setup.GetLayerOptionsSchema(layerType)
if err != nil {
return errors.Wrapf(err, validateErrMessage, proxyName, layerName)
}
rawOptions := func(opts config.InterpolatedMap) map[string]any {
return opts
}(layerConf.Options)
if err := schema.Validate(ctx, layerOptionsSchema, rawOptions); err != nil {
return errors.Wrapf(err, validateErrMessage, proxyName, layerName)
}
}
}
return nil
}

View File

@ -51,15 +51,6 @@ func (s *Server) queryLayer(w http.ResponseWriter, r *http.Request) {
})
}
func validateLayerName(v string) (store.LayerName, error) {
name, err := store.ValidateName(v)
if err != nil {
return "", errors.WithStack(err)
}
return store.LayerName(name), nil
}
type GetLayerResponse struct {
Layer *store.Layer `json:"layer"`
}

View File

@ -5,8 +5,9 @@ import (
)
type Option struct {
ServerConfig config.AdminServerConfig
RedisConfig config.RedisConfig
BootstrapConfig config.BootstrapConfig
ServerConfig config.AdminServerConfig
RedisConfig config.RedisConfig
}
type OptionFunc func(*Option)
@ -29,3 +30,9 @@ func WithRedisConfig(conf config.RedisConfig) OptionFunc {
opt.RedisConfig = conf
}
}
func WithBootstrapConfig(conf config.BootstrapConfig) OptionFunc {
return func(opt *Option) {
opt.BootstrapConfig = conf
}
}

View File

@ -114,6 +114,23 @@ func (s *Server) deleteProxy(w http.ResponseWriter, r *http.Request) {
return
}
layers, err := s.layerRepository.QueryLayers(ctx, proxyName)
if err != nil {
logAndCaptureError(ctx, "could not query proxy's layers", errors.WithStack(err))
api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
return
}
for _, layer := range layers {
if err := s.layerRepository.DeleteLayer(ctx, proxyName, layer.Name); err != nil {
logAndCaptureError(ctx, "could not delete layer", errors.WithStack(err))
api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
return
}
}
api.DataResponse(w, http.StatusOK, DeleteProxyResponse{
ProxyName: proxyName,
})

View File

@ -19,12 +19,15 @@ import (
"github.com/go-chi/cors"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/redis/go-redis/v9"
"gitlab.com/wpetit/goweb/logger"
)
type Server struct {
serverConfig config.AdminServerConfig
redisConfig config.RedisConfig
redisClient redis.UniversalClient
bootstrapConfig config.BootstrapConfig
proxyRepository store.ProxyRepository
layerRepository store.LayerRepository
}
@ -53,6 +56,12 @@ func (s *Server) run(parentCtx context.Context, addrs chan net.Addr, errs chan e
return
}
if err := s.bootstrapProxies(ctx); err != nil {
errs <- errors.WithStack(err)
return
}
listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", s.serverConfig.HTTP.Host, s.serverConfig.HTTP.Port))
if err != nil {
errs <- errors.WithStack(err)
@ -175,7 +184,8 @@ func NewServer(funcs ...OptionFunc) *Server {
}
return &Server{
serverConfig: opt.ServerConfig,
redisConfig: opt.RedisConfig,
serverConfig: opt.ServerConfig,
redisConfig: opt.RedisConfig,
bootstrapConfig: opt.BootstrapConfig,
}
}

View File

@ -68,6 +68,7 @@ func RunCommand() *cli.Command {
srv := admin.NewServer(
admin.WithServerConfig(conf.Admin),
admin.WithRedisConfig(conf.Redis),
admin.WithBootstrapConfig(conf.Bootstrap),
)
addrs, srvErrs := srv.Start(ctx.Context)

View File

@ -0,0 +1,104 @@
package config
import (
"os"
"path/filepath"
"strings"
"time"
"forge.cadoles.com/cadoles/bouncer/internal/store"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
)
type BootstrapConfig struct {
Proxies map[store.ProxyName]BootstrapProxyConfig `yaml:"proxies"`
Dir InterpolatedString `yaml:"dir"`
LockTimeout InterpolatedDuration `yaml:"lockTimeout"`
}
func (c *BootstrapConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
src := struct {
Proxies map[store.ProxyName]BootstrapProxyConfig `yaml:"proxies"`
Dir InterpolatedString `yaml:"dir"`
}{
Proxies: make(map[store.ProxyName]BootstrapProxyConfig),
Dir: "",
}
if err := unmarshal(&src); err != nil {
return errors.WithStack(err)
}
c.Proxies = src.Proxies
c.Dir = src.Dir
if src.Dir != "" {
proxies, err := loadBootstrapDir(string(src.Dir))
if err != nil {
return errors.Wrapf(err, "could not load bootstrap dir '%s'", src.Dir)
}
c.Proxies = overrideProxies(c.Proxies, proxies)
}
return nil
}
type BootstrapProxyConfig struct {
Enabled InterpolatedBool `yaml:"enabled"`
Weight InterpolatedInt `yaml:"weight"`
To InterpolatedString `yaml:"to"`
From InterpolatedStringSlice `yaml:"from"`
Layers map[store.LayerName]BootstrapLayerConfig `yaml:"layers"`
}
type BootstrapLayerConfig struct {
Enabled InterpolatedBool `yaml:"enabled"`
Type InterpolatedString `yaml:"type"`
Weight InterpolatedInt `yaml:"weight"`
Options InterpolatedMap `yaml:"options"`
}
func NewDefaultBootstrapConfig() BootstrapConfig {
return BootstrapConfig{
Dir: "",
LockTimeout: *NewInterpolatedDuration(30 * time.Second),
}
}
func loadBootstrapDir(dir string) (map[store.ProxyName]BootstrapProxyConfig, error) {
pattern := filepath.Join(dir, "*.yml")
files, err := filepath.Glob(pattern)
if err != nil {
return nil, errors.WithStack(err)
}
proxies := make(map[store.ProxyName]BootstrapProxyConfig)
for _, f := range files {
data, err := os.ReadFile(f)
if err != nil {
return nil, errors.Wrapf(err, "could not read file '%s'", f)
}
proxy := BootstrapProxyConfig{}
if err := yaml.Unmarshal(data, &proxy); err != nil {
return nil, errors.Wrapf(err, "could not unmarshal proxy")
}
name := store.ProxyName(strings.TrimSuffix(filepath.Base(f), filepath.Ext(f)))
proxies[name] = proxy
}
return proxies, nil
}
func overrideProxies(base map[store.ProxyName]BootstrapProxyConfig, proxies map[store.ProxyName]BootstrapProxyConfig) map[store.ProxyName]BootstrapProxyConfig {
for name, proxy := range proxies {
base[name] = proxy
}
return base
}

View File

@ -2,7 +2,7 @@ package config
import (
"io"
"io/ioutil"
"os"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
@ -10,18 +10,19 @@ import (
// Config definition
type Config struct {
Admin AdminServerConfig `yaml:"admin"`
Proxy ProxyServerConfig `yaml:"proxy"`
Redis RedisConfig `yaml:"redis"`
Logger LoggerConfig `yaml:"logger"`
Layers LayersConfig `yaml:"layers"`
Admin AdminServerConfig `yaml:"admin"`
Proxy ProxyServerConfig `yaml:"proxy"`
Redis RedisConfig `yaml:"redis"`
Logger LoggerConfig `yaml:"logger"`
Layers LayersConfig `yaml:"layers"`
Bootstrap BootstrapConfig `yaml:"bootstrap"`
}
// NewFromFile retrieves the configuration from the given file
func NewFromFile(path string) (*Config, error) {
config := NewDefault()
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
return nil, errors.Wrapf(err, "could not read file '%s'", path)
}
@ -43,11 +44,12 @@ func NewDumpDefault() *Config {
// NewDefault return new default configuration
func NewDefault() *Config {
return &Config{
Admin: NewDefaultAdminServerConfig(),
Proxy: NewDefaultProxyServerConfig(),
Logger: NewDefaultLoggerConfig(),
Redis: NewDefaultRedisConfig(),
Layers: NewDefaultLayersConfig(),
Admin: NewDefaultAdminServerConfig(),
Proxy: NewDefaultProxyServerConfig(),
Logger: NewDefaultLoggerConfig(),
Redis: NewDefaultRedisConfig(),
Layers: NewDefaultLayersConfig(),
Bootstrap: NewDefaultBootstrapConfig(),
}
}

View File

@ -5,22 +5,25 @@ import (
"forge.cadoles.com/cadoles/bouncer/internal/setup"
"github.com/pkg/errors"
"github.com/redis/go-redis/v9"
)
func (s *Server) initRepositories(ctx context.Context) error {
if err := s.initProxyRepository(ctx); err != nil {
client := setup.NewRedisClient(ctx, s.redisConfig)
if err := s.initProxyRepository(ctx, client); err != nil {
return errors.WithStack(err)
}
if err := s.initLayerRepository(ctx); err != nil {
if err := s.initLayerRepository(ctx, client); err != nil {
return errors.WithStack(err)
}
return nil
}
func (s *Server) initProxyRepository(ctx context.Context) error {
proxyRepository, err := setup.NewProxyRepository(ctx, s.redisConfig)
func (s *Server) initProxyRepository(ctx context.Context, client redis.UniversalClient) error {
proxyRepository, err := setup.NewProxyRepository(ctx, client)
if err != nil {
return errors.WithStack(err)
}
@ -30,8 +33,8 @@ func (s *Server) initProxyRepository(ctx context.Context) error {
return nil
}
func (s *Server) initLayerRepository(ctx context.Context) error {
layerRepository, err := setup.NewLayerRepository(ctx, s.redisConfig)
func (s *Server) initLayerRepository(ctx context.Context, client redis.UniversalClient) error {
layerRepository, err := setup.NewLayerRepository(ctx, client)
if err != nil {
return errors.WithStack(err)
}

View File

@ -9,16 +9,17 @@ import (
"github.com/redis/go-redis/v9"
)
func NewProxyRepository(ctx context.Context, conf config.RedisConfig) (store.ProxyRepository, error) {
rdb := newRedisClient(conf)
return redisStore.NewProxyRepository(rdb), nil
}
func NewLayerRepository(ctx context.Context, conf config.RedisConfig) (store.LayerRepository, error) {
rdb := redis.NewUniversalClient(&redis.UniversalOptions{
func NewRedisClient(ctx context.Context, conf config.RedisConfig) redis.UniversalClient {
return redis.NewUniversalClient(&redis.UniversalOptions{
Addrs: conf.Adresses,
MasterName: string(conf.Master),
})
return redisStore.NewLayerRepository(rdb), nil
}
func NewProxyRepository(ctx context.Context, client redis.UniversalClient) (store.ProxyRepository, error) {
return redisStore.NewProxyRepository(client), nil
}
func NewLayerRepository(ctx context.Context, client redis.UniversalClient) (store.LayerRepository, error) {
return redisStore.NewLayerRepository(client), nil
}

View File

@ -4,63 +4,65 @@
1. Generate the Docker configuration to enable image builds with Kaniko and communicate with reg.cadoles.com
```shell
docker login reg.cadoles.com
mkdir -p misc/k8s/kustomization/base/secrets/dockerconfig
docker --config misc/k8s/kustomization/base/secrets/dockerconfig login reg.cadoles.com
mv misc/k8s/kustomization/base/secrets/dockerconfig/config.json misc/k8s/kustomization/base/secrets/dockerconfig/.dockerconfigjson
mkdir -p misc/k8s/kustomization/overlays/dev/secrets/dockerconfig
cp misc/k8s/kustomization/base/secrets/dockerconfig/.dockerconfigjson misc/k8s/kustomization/overlays/dev/secrets/dockerconfig/.dockerconfigjson
```
```shell
docker login reg.cadoles.com
mkdir -p misc/k8s/kustomization/base/secrets/dockerconfig
docker --config misc/k8s/kustomization/base/secrets/dockerconfig login reg.cadoles.com
mv misc/k8s/kustomization/base/secrets/dockerconfig/config.json misc/k8s/kustomization/base/secrets/dockerconfig/.dockerconfigjson
mkdir -p misc/k8s/kustomization/overlays/dev/secrets/dockerconfig
cp misc/k8s/kustomization/base/secrets/dockerconfig/.dockerconfigjson misc/k8s/kustomization/overlays/dev/secrets/dockerconfig/.dockerconfigjson
```
## Getting started with Kind
1. Create your [Kind](https://kind.sigs.k8s.io/) cluster
```shell
kind create cluster --config misc/k8s/kind/bouncer-cluster.yaml
```
```shell
kind create cluster --config misc/k8s/kind/bouncer-cluster.yaml
```
2. Deploy required operators
```shell
kubectl apply -k misc/k8s/kind/cluster --server-side
```
```shell
kubectl apply -k misc/k8s/kind/cluster --server-side
```
3. Deploy your Bouncer development environment
```shell
skaffold dev -p dev --cleanup=false --default-repo reg.cadoles.com/<YOUR_PERSONNAL_USER_NAME>
```
```shell
skaffold dev -p dev --cleanup=false --default-repo reg.cadoles.com/<YOUR_PERSONNAL_USER_NAME>
```
## Testing
Bouncer will automatically create proxies based on the files present in the `misc/k8s/kustomization/overlays/dev/files/bouncer/bootstrap.d` folder.
By default, with you host web browser, open http://localhost:9000, you should see the Cadoles website.
### Using the admin API
1. Open shell in bouncer-admin pod
```shell
kubectl exec -it -n bouncer-dev bouncer-admin-<suffix> -- /bin/sh
```
```shell
kubectl exec -it -n bouncer-dev bouncer-admin-<suffix> -- /bin/sh
```
2. Create an authentication token
```shell
bouncer --config /etc/bouncer/config.yml auth create-token --role writer --subject $(whoami) > .bouncer-token
```
```shell
bouncer --config /etc/bouncer/config.yml auth create-token --role writer --subject $(whoami) > .bouncer-token
```
3. Create a proxy and enable it
```shell
bouncer admin proxy create --proxy-to https://www.cadoles.com --proxy-name cadoles
bouncer admin proxy update --proxy-name cadoles --proxy-enabled=true
```
4. With you host web browser, open http://localhost:9000, you should see the Cadoles website.
```shell
bouncer admin proxy query
```
## Benchmarking
You can use [`siege`](https://github.com/JoeDog/siege) to benchmark your instance with the Cadoles proxy.
```shell
BASE_URL=http://localhost:9000 make siege
BASE_URL=http://localhost:9000 make siege
```

View File

@ -4,19 +4,19 @@ admin:
port: 8081
cors:
allowedOrigins:
- http://localhost:3001
- http://localhost:3001
allowCredentials: true
allowMethods:
- POST
- GET
- PUT
- DELETE
- POST
- GET
- PUT
- DELETE
allowedHeaders:
- Origin
- Accept
- Content-Type
- Authorization
- Sentry-Trace
- Origin
- Accept
- Content-Type
- Authorization
- Sentry-Trace
debug: false
auth:
issuer: http://127.0.0.1:8081
@ -28,9 +28,13 @@ admin:
redis:
addresses:
- rfs-bouncer-redis:${RFS_BOUNCER_REDIS_SERVICE_PORT}
- rfs-bouncer-redis:${RFS_BOUNCER_REDIS_SERVICE_PORT}
master: mymaster
logger:
level: 2
level: ${BOUNCER_LOG_LEVEL}
format: human
bootstrap:
dir: /etc/bouncer/bootstrap.d
lockTimeout: 30s

View File

@ -2,11 +2,15 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./resources/service.yaml
- ./resources/deployment.yaml
- ./resources/service.yaml
- ./resources/deployment.yaml
configMapGenerator:
- name: bouncer-admin-config
files:
- ./files/config.yml
- ./files/admin-key.json
- name: bouncer-admin-config
files:
- ./files/config.yml
- ./files/admin-key.json
- name: bouncer-admin-bootstrap
- name: bouncer-admin-env
literals:
- BOUNCER_LOG_LEVEL=2

View File

@ -17,18 +17,35 @@ spec:
io.kompose.service: bouncer-admin
spec:
containers:
- name: bouncer-admin
image: reg.cadoles.com/cadoles/bouncer:v2024.2.5-1602626
command: ["bouncer", "--debug", "-c", "/etc/bouncer/config.yml", "server", "admin", "run"]
imagePullPolicy: Always
resources: {}
ports:
- name: bouncer-admin
containerPort: 8081
volumeMounts:
- mountPath: /etc/bouncer/
name: bouncer-admin-config
image: bouncer
command:
[
"bouncer",
"--debug",
"-c",
"/etc/bouncer/config.yml",
"server",
"admin",
"run",
]
imagePullPolicy: Always
resources: {}
ports:
- name: bouncer-admin
containerPort: 8081
envFrom:
- configMapRef:
name: bouncer-admin-env
volumeMounts:
- mountPath: /etc/bouncer/
name: bouncer-admin-config
- mountPath: /etc/bouncer/bootstrap.d
name: bouncer-admin-bootstrap
volumes:
- name: bouncer-admin-config
configMap:
name: bouncer-admin-config
- name: bouncer-admin-config
configMap:
name: bouncer-admin-config
- name: bouncer-admin-bootstrap
configMap:
name: bouncer-admin-bootstrap

View File

@ -14,9 +14,9 @@ layers:
redis:
addresses:
- rfs-bouncer-redis:${RFS_BOUNCER_REDIS_SERVICE_PORT}
- rfs-bouncer-redis:${RFS_BOUNCER_REDIS_SERVICE_PORT}
master: mymaster
logger:
level: 2
level: ${BOUNCER_LOG_LEVEL}
format: human

View File

@ -2,10 +2,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./resources/service.yaml
- ./resources/deployment.yaml
- ./resources/service.yaml
- ./resources/deployment.yaml
configMapGenerator:
- name: bouncer-server-config
files:
- ./files/config.yml
- name: bouncer-server-config
files:
- ./files/config.yml
- name: bouncer-server-env
literals:
- BOUNCER_LOG_LEVEL=2

View File

@ -17,18 +17,29 @@ spec:
io.kompose.service: bouncer-server
spec:
containers:
- name: bouncer-server
image: reg.cadoles.com/cadoles/bouncer:v2024.2.5-1602626
command: ["bouncer", "-c", "/etc/bouncer/config.yml", "server", "proxy", "run"]
imagePullPolicy: Always
resources: {}
ports:
- name: bouncer-server
containerPort: 8080
volumeMounts:
- mountPath: /etc/bouncer/
name: bouncer-server-config
image: bouncer
command:
[
"bouncer",
"-c",
"/etc/bouncer/config.yml",
"server",
"proxy",
"run",
]
imagePullPolicy: Always
envFrom:
- configMapRef:
name: bouncer-server-env
resources: {}
ports:
- name: bouncer-server
containerPort: 8080
volumeMounts:
- mountPath: /etc/bouncer/
name: bouncer-server-config
volumes:
- name: bouncer-server-config
configMap:
name: bouncer-server-config
- name: bouncer-server-config
configMap:
name: bouncer-server-config

View File

@ -0,0 +1,11 @@
from: ["*"]
to: https://www.cadoles.com
enabled: true
weight: 0
layers:
my-queue:
type: queue
enabled: true
weight: 0
options:
capacity: 10

View File

@ -3,16 +3,30 @@ kind: Kustomization
namespace: bouncer-dev
resources:
- ../../base
- ../../base
secretGenerator:
- files:
- secrets/dockerconfig/.dockerconfigjson
name: regcred-dev
type: kubernetes.io/dockerconfigjson
- files:
- secrets/dockerconfig/.dockerconfigjson
name: regcred-dev
type: kubernetes.io/dockerconfigjson
patches:
- path: patches/add-registry-pull-secret.patch.yaml
target:
kind: Deployment
version: v1
- path: patches/add-registry-pull-secret.patch.yaml
target:
kind: Deployment
version: v1
configMapGenerator:
- name: bouncer-admin-bootstrap
behavior: merge
files:
- ./files/bouncer/bootstrap.d/cadoles.yml
- name: bouncer-admin-env
behavior: merge
literals:
- BOUNCER_LOG_LEVEL=0
- name: bouncer-server-env
behavior: merge
literals:
- BOUNCER_LOG_LEVEL=0

View File

@ -1,7 +1,7 @@
# Configuration du service "admin"
admin:
http:
# Hôte d'écoute du service,
# Hôte d'écoute du service,
# 0.0.0.0 pour écouter sur toutes les interfaces
host: 127.0.0.1
# Port d'écoute du service
@ -15,19 +15,19 @@ admin:
# est branché sur l'API d'administration.
cors:
allowedOrigins:
- http://localhost:8081
- http://localhost:8081
allowCredentials: true
allowMethods:
- POST
- GET
- PUT
- DELETE
- POST
- GET
- PUT
- DELETE
allowedHeaders:
- Origin
- Accept
- Content-Type
- Authorization
- Sentry-Trace
- Origin
- Accept
- Content-Type
- Authorization
- Sentry-Trace
debug: false
# Authentification JWT
@ -71,7 +71,7 @@ admin:
# Configuration du service "proxy"
proxy:
http:
# Hôte d'écoute du service,
# Hôte d'écoute du service,
# 0.0.0.0 pour écouter sur toutes les interfaces
host: 0.0.0.0
# Port d'écoute du service
@ -145,7 +145,7 @@ proxy:
# - Mode "cluster": renseigner plusieurs adresses dans redis.addresses et laisser redis.master vide.
redis:
addresses:
- localhost:6379
- localhost:6379
master: ""
writeTimeout: 30s
readTimeout: 30s
@ -177,3 +177,36 @@ layers:
# Répertoire contenant les templates
templateDir: "/etc/bouncer/layers/circuitbreaker/templates"
# Configuration d'une série de proxy/layers
# à créer par défaut par le serveur d'administration
bootstrap:
# Répertoire contenant les définitions de proxy à créer
# par défaut. Les fichiers seront récupérés si ils
# correspondent au patron de nommage suivant:
#
# <bootstrap_dir>/<proxy_name>.yml
#
# Si l'attribut est vide ou absent le chargement des fichiers
# est désactivé.
dir: /etc/bouncer/bootstrap.d
# Délai d'expiration du verrou distribué utilisé lors du chargement
# des définitions de proxy par défaut.
lockTimeout: 30s
# Tableau associatif de définition de proxies à créer par
# défaut par le serveur d'administration.
# Si `proxies` et `dir` sont tous les deux définis, les fichiers
# présents dans le répertoire `dir` surchargeront les valeurs définies
# dans `proxies`.
# Par défault non défini
proxies:
# my-proxy:
# enabled: true
# from: ["*"]
# to: "https://example.net"
# weight: 0
# layers:
# my-layer:
# type: queue
# enabled: false
# weight: 0
# options: {"capacity": 100}

View File

@ -7,16 +7,16 @@ metadata:
manifests:
kustomize:
paths:
- misc/k8s/kustomization/base
- misc/k8s/kustomization/base
profiles:
- name: dev
manifests:
kustomize:
paths:
- misc/k8s/kustomization/overlays/dev
activation:
- command: dev
- name: dev
manifests:
kustomize:
paths:
- misc/k8s/kustomization/overlays/dev
activation:
- command: dev
build:
local:
@ -26,28 +26,28 @@ build:
sha256: {}
artifacts:
- image: reg.cadoles.com/cadoles/bouncer
context: .
sync:
infer:
- cmd/**
- internal/**
- layers/**
- misc/**
docker:
dockerfile: Dockerfile
- image: bouncer
context: .
sync:
infer:
- cmd/**
- internal/**
- layers/**
- misc/**
docker:
dockerfile: Dockerfile
deploy:
statusCheckDeadlineSeconds: 600
portForward:
- resourceType: service
resourceName: bouncer-admin
namespace: bouncer-dev
port: 8081
localPort: 9999
- resourceType: service
resourceName: bouncer-server
namespace: bouncer-dev
port: 8080
localPort: 9000 # *Optional*
- resourceType: service
resourceName: bouncer-admin
namespace: bouncer-dev
port: 8081
localPort: 9999
- resourceType: service
resourceName: bouncer-server
namespace: bouncer-dev
port: 8080
localPort: 9000 # *Optional*