Cadoles/bouncer
Cadoles
/
bouncer
10
0
Fork 0
Code Issues 2 Pull Requests Projects Releases 3 Wiki Activity

feat: bootstrap default proxy and layers from configuration
Cadoles/bouncer/pipeline/head This commit looks good Details
Cadoles/bouncer/pipeline/pr-develop This commit looks good Details

This commit is contained in:
wpetit 2024-03-25 18:13:54 +01:00
parent 734ed64e8e
commit f9b668642e
14 changed files with 340 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.goreleaser.yaml
View File

@ -63,6 +63,10 @@ nfpms:
- src: layers
dst: /etc/bouncer/layers
type: config
- dst: /etc/bouncer/bootstrap.d
type: dir
file_info:
mode: 0700
- id: bouncer-admin
meta: true
package_name: bouncer-admin

1
go.mod
View File

@ -5,6 +5,7 @@ go 1.20
require (
forge.cadoles.com/Cadoles/go-proxy v0.0.0-20230701194111-c6b3d482cca6
github.com/Masterminds/sprig/v3 v3.2.3
github.com/bsm/redislock v0.9.4
github.com/btcsuite/btcd/btcutil v1.1.3
github.com/drone/envsubst v1.0.3
github.com/getsentry/sentry-go v0.22.0

2
go.sum
View File

@ -84,6 +84,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y=
github.com/bsm/redislock v0.9.4 h1:X/Wse1DPpiQgHbVYRE9zv6m070UcKoOGekgvpNhiSvw=
github.com/bsm/redislock v0.9.4/go.mod h1:Epf7AJLiSFwLCiZcfi6pWFO/8eAYrYpQXFxEDPoDeAk=
github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M=
github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY=

131
internal/admin/init.go
View File

@ -2,12 +2,22 @@ package admin
import (
"context"
"time"
"forge.cadoles.com/cadoles/bouncer/internal/config"
"forge.cadoles.com/cadoles/bouncer/internal/schema"
"forge.cadoles.com/cadoles/bouncer/internal/setup"
"forge.cadoles.com/cadoles/bouncer/internal/store"
"github.com/bsm/redislock"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func (s *Server) initRepositories(ctx context.Context) error {
if err := s.initRedisClient(ctx); err != nil {
return errors.WithStack(err)
}
if err := s.initLayerRepository(ctx); err != nil {
return errors.WithStack(err)
}
@ -19,8 +29,16 @@ func (s *Server) initRepositories(ctx context.Context) error {
return nil
}
func (s *Server) initRedisClient(ctx context.Context) error {
client := setup.NewRedisClient(ctx, s.redisConfig)
s.redisClient = client
return nil
}
func (s *Server) initLayerRepository(ctx context.Context) error {
layerRepository, err := setup.NewLayerRepository(ctx, s.redisConfig)
layerRepository, err := setup.NewLayerRepository(ctx, s.redisClient)
if err != nil {
return errors.WithStack(err)
}
@ -31,7 +49,7 @@ func (s *Server) initLayerRepository(ctx context.Context) error {
}
func (s *Server) initProxyRepository(ctx context.Context) error {
proxyRepository, err := setup.NewProxyRepository(ctx, s.redisConfig)
proxyRepository, err := setup.NewProxyRepository(ctx, s.redisClient)
if err != nil {
return errors.WithStack(err)
}
@ -40,3 +58,112 @@ func (s *Server) initProxyRepository(ctx context.Context) error {
return nil
}
const bootstrapLockKey = "bouncer-bootstrap"
func (s *Server) bootstrapProxies(ctx context.Context) error {
if err := s.validateBootstrap(ctx); err != nil {
return errors.Wrap(err, "could not validate bootstrapped proxies")
}
proxyRepo := s.proxyRepository
layerRepo := s.layerRepository
locker := redislock.New(s.redisClient)
backoff := redislock.ExponentialBackoff(time.Second, time.Duration(s.bootstrapConfig.LockTimeout)*2)
logger.Debug(ctx, "acquiring proxies bootstrap lock", logger.F("lockTimeount", s.bootstrapConfig.LockTimeout))
lock, err := locker.Obtain(ctx, bootstrapLockKey, time.Duration(s.bootstrapConfig.LockTimeout), &redislock.Options{
RetryStrategy: backoff,
})
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := lock.Release(ctx); err != nil {
logger.Error(ctx, "could not release lock", logger.E(errors.WithStack(err)))
}
}()
logger.Info(ctx, "bootstrapping proxies")
for proxyName, proxyConfig := range s.bootstrapConfig.Proxies {
_, err := s.proxyRepository.GetProxy(ctx, proxyName)
if !errors.Is(err, store.ErrNotFound) {
if err != nil {
return errors.WithStack(err)
}
logger.Info(ctx, "ignoring existing proxy", logger.F("proxyName", proxyName))
continue
}
logger.Info(ctx, "creating proxy", logger.F("proxyName", proxyName))
if _, err := proxyRepo.CreateProxy(ctx, proxyName, string(proxyConfig.To), proxyConfig.From...); err != nil {
return errors.WithStack(err)
}
_, err = proxyRepo.UpdateProxy(
ctx, proxyName,
store.WithProxyUpdateEnabled(bool(proxyConfig.Enabled)),
store.WithProxyUpdateWeight(int(proxyConfig.Weight)),
)
if err != nil {
return errors.WithStack(err)
}
for layerName, layerConfig := range proxyConfig.Layers {
layerType := store.LayerType(layerConfig.Type)
layerOptions := store.LayerOptions(layerConfig.Options)
if _, err := layerRepo.CreateLayer(ctx, proxyName, layerName, layerType, layerOptions); err != nil {
return errors.WithStack(err)
}
_, err := layerRepo.UpdateLayer(
ctx,
proxyName, layerName,
store.WithLayerUpdateEnabled(bool(layerConfig.Enabled)),
store.WithLayerUpdateOptions(layerOptions),
store.WithLayerUpdateWeight(int(layerConfig.Weight)),
)
if err != nil {
return errors.WithStack(err)
}
}
}
return nil
}
const validateErrMessage = "could not validate proxy '%s': could not validate layer '%s'"
func (s *Server) validateBootstrap(ctx context.Context) error {
for proxyName, proxyConf := range s.bootstrapConfig.Proxies {
for layerName, layerConf := range proxyConf.Layers {
layerType := store.LayerType(layerConf.Type)
if !setup.LayerTypeExists(layerType) {
return errors.Errorf(validateErrMessage+": could not find layer type '%s'", proxyName, layerName, layerType)
}
layerOptionsSchema, err := setup.GetLayerOptionsSchema(layerType)
if err != nil {
return errors.Wrapf(err, validateErrMessage, proxyName, layerName)
}
rawOptions := func(opts config.InterpolatedMap) map[string]any {
return opts
}(layerConf.Options)
if err := schema.Validate(ctx, layerOptionsSchema, rawOptions); err != nil {
return errors.Wrapf(err, validateErrMessage, proxyName, layerName)
}
}
}
return nil
}

9
internal/admin/layer_route.go
View File

@ -51,15 +51,6 @@ func (s *Server) queryLayer(w http.ResponseWriter, r *http.Request) {
})
}
func validateLayerName(v string) (store.LayerName, error) {
name, err := store.ValidateName(v)
if err != nil {
return "", errors.WithStack(err)
}
return store.LayerName(name), nil
}
type GetLayerResponse struct {
Layer *store.Layer `json:"layer"`
}

11
internal/admin/option.go
View File

@ -5,8 +5,9 @@ import (
)
type Option struct {
ServerConfig config.AdminServerConfig
RedisConfig config.RedisConfig
BootstrapConfig config.BootstrapConfig
ServerConfig config.AdminServerConfig
RedisConfig config.RedisConfig
}
type OptionFunc func(*Option)
@ -29,3 +30,9 @@ func WithRedisConfig(conf config.RedisConfig) OptionFunc {
opt.RedisConfig = conf
}
}
func WithBootstrapConfig(conf config.BootstrapConfig) OptionFunc {
return func(opt *Option) {
opt.BootstrapConfig = conf
}
}

17
internal/admin/proxy_route.go
View File

@ -114,6 +114,23 @@ func (s *Server) deleteProxy(w http.ResponseWriter, r *http.Request) {
return
}
layers, err := s.layerRepository.QueryLayers(ctx, proxyName)
if err != nil {
logAndCaptureError(ctx, "could not query proxy's layers", errors.WithStack(err))
api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
return
}
for _, layer := range layers {
if err := s.layerRepository.DeleteLayer(ctx, proxyName, layer.Name); err != nil {
logAndCaptureError(ctx, "could not delete layer", errors.WithStack(err))
api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
return
}
}
api.DataResponse(w, http.StatusOK, DeleteProxyResponse{
ProxyName: proxyName,
})

14
internal/admin/server.go
View File

@ -19,12 +19,15 @@ import (
"github.com/go-chi/cors"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/redis/go-redis/v9"
"gitlab.com/wpetit/goweb/logger"
)
type Server struct {
serverConfig config.AdminServerConfig
redisConfig config.RedisConfig
redisClient redis.UniversalClient
bootstrapConfig config.BootstrapConfig
proxyRepository store.ProxyRepository
layerRepository store.LayerRepository
}
@ -53,6 +56,12 @@ func (s *Server) run(parentCtx context.Context, addrs chan net.Addr, errs chan e
return
}
if err := s.bootstrapProxies(ctx); err != nil {
errs <- errors.WithStack(err)
return
}
listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", s.serverConfig.HTTP.Host, s.serverConfig.HTTP.Port))
if err != nil {
errs <- errors.WithStack(err)
@ -175,7 +184,8 @@ func NewServer(funcs ...OptionFunc) *Server {
}
return &Server{
serverConfig: opt.ServerConfig,
redisConfig: opt.RedisConfig,
serverConfig: opt.ServerConfig,
redisConfig: opt.RedisConfig,
bootstrapConfig: opt.BootstrapConfig,
}
}

1
internal/command/server/admin/run.go
View File

@ -68,6 +68,7 @@ func RunCommand() *cli.Command {
srv := admin.NewServer(
admin.WithServerConfig(conf.Admin),
admin.WithRedisConfig(conf.Redis),
admin.WithBootstrapConfig(conf.Bootstrap),
)
addrs, srvErrs := srv.Start(ctx.Context)

104
internal/config/bootstrap.go Normal file
View File

@ -0,0 +1,104 @@
package config
import (
"os"
"path/filepath"
"strings"
"time"
"forge.cadoles.com/cadoles/bouncer/internal/store"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
)
type BootstrapConfig struct {
Proxies map[store.ProxyName]BootstrapProxyConfig `yaml:"proxies"`
Dir InterpolatedString `yaml:"dir"`
LockTimeout InterpolatedDuration `yaml:"lockTimeout"`
}
func (c *BootstrapConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
src := struct {
Proxies map[store.ProxyName]BootstrapProxyConfig `yaml:"proxies"`
Dir InterpolatedString `yaml:"dir"`
}{
Proxies: make(map[store.ProxyName]BootstrapProxyConfig),
Dir: "",
}
if err := unmarshal(&src); err != nil {
return errors.WithStack(err)
}
c.Proxies = src.Proxies
c.Dir = src.Dir
if src.Dir != "" {
proxies, err := loadBootstrapDir(string(src.Dir))
if err != nil {
return errors.Wrapf(err, "could not load bootstrap dir '%s'", src.Dir)
}
c.Proxies = overrideProxies(c.Proxies, proxies)
}
return nil
}
type BootstrapProxyConfig struct {
Enabled InterpolatedBool `yaml:"enabled"`
Weight InterpolatedInt `yaml:"weight"`
To InterpolatedString `yaml:"to"`
From InterpolatedStringSlice `yaml:"from"`
Layers map[store.LayerName]BootstrapLayerConfig `yaml:"layers"`
}
type BootstrapLayerConfig struct {
Enabled InterpolatedBool `yaml:"enabled"`
Type InterpolatedString `yaml:"type"`
Weight InterpolatedInt `yaml:"weight"`
Options InterpolatedMap `yaml:"options"`
}
func NewDefaultBootstrapConfig() BootstrapConfig {
return BootstrapConfig{
Dir: "",
LockTimeout: *NewInterpolatedDuration(30 * time.Second),
}
}
func loadBootstrapDir(dir string) (map[store.ProxyName]BootstrapProxyConfig, error) {
pattern := filepath.Join(dir, "*.yml")
files, err := filepath.Glob(pattern)
if err != nil {
return nil, errors.WithStack(err)
}
proxies := make(map[store.ProxyName]BootstrapProxyConfig)
for _, f := range files {
data, err := os.ReadFile(f)
if err != nil {
return nil, errors.Wrapf(err, "could not read file '%s'", f)
}
proxy := BootstrapProxyConfig{}
if err := yaml.Unmarshal(data, &proxy); err != nil {
return nil, errors.Wrapf(err, "could not unmarshal proxy")
}
name := store.ProxyName(strings.TrimSuffix(filepath.Base(f), filepath.Ext(f)))
proxies[name] = proxy
}
return proxies, nil
}
func overrideProxies(base map[store.ProxyName]BootstrapProxyConfig, proxies map[store.ProxyName]BootstrapProxyConfig) map[store.ProxyName]BootstrapProxyConfig {
for name, proxy := range proxies {
base[name] = proxy
}
return base
}

26
internal/config/config.go
View File

@ -2,7 +2,7 @@ package config
import (
"io"
"io/ioutil"
"os"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
@ -10,18 +10,19 @@ import (
// Config definition
type Config struct {
Admin AdminServerConfig `yaml:"admin"`
Proxy ProxyServerConfig `yaml:"proxy"`
Redis RedisConfig `yaml:"redis"`
Logger LoggerConfig `yaml:"logger"`
Layers LayersConfig `yaml:"layers"`
Admin AdminServerConfig `yaml:"admin"`
Proxy ProxyServerConfig `yaml:"proxy"`
Redis RedisConfig `yaml:"redis"`
Logger LoggerConfig `yaml:"logger"`
Layers LayersConfig `yaml:"layers"`
Bootstrap BootstrapConfig `yaml:"bootstrap"`
}
// NewFromFile retrieves the configuration from the given file
func NewFromFile(path string) (*Config, error) {
config := NewDefault()
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
return nil, errors.Wrapf(err, "could not read file '%s'", path)
}
@ -43,11 +44,12 @@ func NewDumpDefault() *Config {
// NewDefault return new default configuration
func NewDefault() *Config {
return &Config{
Admin: NewDefaultAdminServerConfig(),
Proxy: NewDefaultProxyServerConfig(),
Logger: NewDefaultLoggerConfig(),
Redis: NewDefaultRedisConfig(),
Layers: NewDefaultLayersConfig(),
Admin: NewDefaultAdminServerConfig(),
Proxy: NewDefaultProxyServerConfig(),
Logger: NewDefaultLoggerConfig(),
Redis: NewDefaultRedisConfig(),
Layers: NewDefaultLayersConfig(),
Bootstrap: NewDefaultBootstrapConfig(),
}
}

15
internal/proxy/init.go
View File

@ -5,22 +5,25 @@ import (
"forge.cadoles.com/cadoles/bouncer/internal/setup"
"github.com/pkg/errors"
"github.com/redis/go-redis/v9"
)
func (s *Server) initRepositories(ctx context.Context) error {
if err := s.initProxyRepository(ctx); err != nil {
client := setup.NewRedisClient(ctx, s.redisConfig)
if err := s.initProxyRepository(ctx, client); err != nil {
return errors.WithStack(err)
}
if err := s.initLayerRepository(ctx); err != nil {
if err := s.initLayerRepository(ctx, client); err != nil {
return errors.WithStack(err)
}
return nil
}
func (s *Server) initProxyRepository(ctx context.Context) error {
proxyRepository, err := setup.NewProxyRepository(ctx, s.redisConfig)
func (s *Server) initProxyRepository(ctx context.Context, client redis.UniversalClient) error {
proxyRepository, err := setup.NewProxyRepository(ctx, client)
if err != nil {
return errors.WithStack(err)
}
@ -30,8 +33,8 @@ func (s *Server) initProxyRepository(ctx context.Context) error {
return nil
}
func (s *Server) initLayerRepository(ctx context.Context) error {
layerRepository, err := setup.NewLayerRepository(ctx, s.redisConfig)
func (s *Server) initLayerRepository(ctx context.Context, client redis.UniversalClient) error {
layerRepository, err := setup.NewLayerRepository(ctx, client)
if err != nil {
return errors.WithStack(err)
}

18
internal/setup/proxy_repository.go
View File

@ -9,20 +9,16 @@ import (
"github.com/redis/go-redis/v9"
)
func NewProxyRepository(ctx context.Context, conf config.RedisConfig) (store.ProxyRepository, error) {
rdb := redis.NewUniversalClient(&redis.UniversalOptions{
func NewRedisClient(ctx context.Context, conf config.RedisConfig) redis.UniversalClient {
return redis.NewUniversalClient(&redis.UniversalOptions{
Addrs: conf.Adresses,
MasterName: string(conf.Master),
})
return redisStore.NewProxyRepository(rdb), nil
}
func NewProxyRepository(ctx context.Context, client redis.UniversalClient) (store.ProxyRepository, error) {
return redisStore.NewProxyRepository(client), nil
}
func NewLayerRepository(ctx context.Context, conf config.RedisConfig) (store.LayerRepository, error) {
rdb := redis.NewUniversalClient(&redis.UniversalOptions{
Addrs: conf.Adresses,
MasterName: string(conf.Master),
})
return redisStore.NewLayerRepository(rdb), nil
func NewLayerRepository(ctx context.Context, client redis.UniversalClient) (store.LayerRepository, error) {
return redisStore.NewLayerRepository(client), nil
}

31
misc/packaging/common/config.yml
View File

@ -179,4 +179,35 @@ layers:
circuitbreaker:
# Répertoire contenant les templates
templateDir: "/etc/bouncer/layers/circuitbreaker/templates"
# Configuration d'une série de proxy/layers
# à créer par défaut par le serveur d'administration
bootstrap:
# Répertoire contenant les définitions de proxy à créer
# par défaut. Les fichiers seront récupérés si ils
# correspondent au patron de nommage suivant:
#
# <bootstrap_dir>/<proxy_name>.yml
#
# Si l'attribut est vide ou absent le chargement des fichiers
# est désactivé.
dir: /etc/bouncer/bootstrap.d
# Tableau associatif de définition de proxies à créer par
# défaut par le serveur d'administration.
# Si `proxies` et `dir` sont tous les deux définis, les fichiers
# présents dans le répertoire `dir` surchargeront les valeurs définies
# dans `proxies`.
# Par défault non défini
# proxies:
# my-proxy:
# enabled: true
# from: ["*"]
# to: "https://example.net"
# weight: 0
# layers:
# my-layer:
# type: queue
# enabled: false
# weight: 0
# options: {"capacity": 100}