diff --git a/Makefile b/Makefile index 71e0fd0..9e29d03 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,8 @@ GOTEST_ARGS ?= -short OPENWRT_DEVICE ?= 192.168.1.1 SIEGE_URLS_FILE ?= misc/siege/urls.txt -SIEGE_CONCURRENCY ?= 100 +SIEGE_CONCURRENCY ?= 50 +SIEGE_DURATION ?= 1M data/bootstrap.d/dummy.yml: mkdir -p data/bootstrap.d @@ -114,7 +115,7 @@ grafterm: tools/grafterm/bin/grafterm siege: $(eval TMP := $(shell mktemp)) cat $(SIEGE_URLS_FILE) | envsubst > $(TMP) - siege -i -b -c $(SIEGE_CONCURRENCY) -f $(TMP) + siege -R ./misc/siege/siege.conf -i -b -c $(SIEGE_CONCURRENCY) -t $(SIEGE_DURATION) -f $(TMP) rm -rf $(TMP) tools/gitea-release/bin/gitea-release.sh: @@ -150,7 +151,7 @@ run-redis: -v $(PWD)/data/redis:/data \ -p 6379:6379 \ redis:alpine3.17 \ - redis-server --save 60 1 --loglevel warning + redis-server --save 60 1 --loglevel debug redis-shell: docker exec -it \ diff --git a/internal/admin/init.go b/internal/admin/init.go index fa5fd9b..e63479a 100644 --- a/internal/admin/init.go +++ b/internal/admin/init.go @@ -27,7 +27,7 @@ func (s *Server) initRepositories(ctx context.Context) error { } func (s *Server) initRedisClient(ctx context.Context) error { - client := setup.NewRedisClient(ctx, s.redisConfig) + client := setup.NewSharedClient(s.redisConfig) s.redisClient = client diff --git a/internal/config/redis.go b/internal/config/redis.go index 8d71422..d6e47ce 100644 --- a/internal/config/redis.go +++ b/internal/config/redis.go @@ -15,6 +15,8 @@ type RedisConfig struct { WriteTimeout InterpolatedDuration `yaml:"writeTimeout"` DialTimeout InterpolatedDuration `yaml:"dialTimeout"` LockMaxRetries InterpolatedInt `yaml:"lockMaxRetries"` + MaxRetries InterpolatedInt `yaml:"maxRetries"` + PingInterval InterpolatedDuration `yaml:"pingInterval"` } func NewDefaultRedisConfig() RedisConfig { @@ -25,5 +27,7 @@ func NewDefaultRedisConfig() RedisConfig { WriteTimeout: InterpolatedDuration(30 * time.Second), DialTimeout: InterpolatedDuration(30 * time.Second), LockMaxRetries: 10, + MaxRetries: 3, + PingInterval: InterpolatedDuration(30 * time.Second), } } diff --git a/internal/proxy/init.go b/internal/proxy/init.go index 8bcafdd..dd54ecd 100644 --- a/internal/proxy/init.go +++ b/internal/proxy/init.go @@ -9,7 +9,7 @@ import ( ) func (s *Server) initRepositories(ctx context.Context) error { - client := setup.NewRedisClient(ctx, s.redisConfig) + client := setup.NewSharedClient(s.redisConfig) if err := s.initProxyRepository(ctx, client); err != nil { return errors.WithStack(err) diff --git a/internal/setup/authn_oidc_layer.go b/internal/setup/authn_oidc_layer.go index 6ecbecf..5db66fe 100644 --- a/internal/setup/authn_oidc_layer.go +++ b/internal/setup/authn_oidc_layer.go @@ -23,7 +23,7 @@ func init() { } func setupAuthnOIDCLayer(conf *config.Config) (director.Layer, error) { - rdb := newRedisClient(conf.Redis) + rdb := NewSharedClient(conf.Redis) adapter := redis.NewStoreAdapter(rdb) store := session.NewStore(adapter) diff --git a/internal/setup/integrations.go b/internal/setup/integrations.go index 580b5af..23675d9 100644 --- a/internal/setup/integrations.go +++ b/internal/setup/integrations.go @@ -27,7 +27,7 @@ func SetupIntegrations(ctx context.Context, conf *config.Config) ([]integration. } func setupKubernetesIntegration(ctx context.Context, conf *config.Config) (*kubernetes.Integration, error) { - client := newRedisClient(conf.Redis) + client := NewSharedClient(conf.Redis) locker := redis.NewLocker(client, 10) integration := kubernetes.NewIntegration( diff --git a/internal/setup/lock.go b/internal/setup/lock.go index d560c70..df776f2 100644 --- a/internal/setup/lock.go +++ b/internal/setup/lock.go @@ -9,7 +9,7 @@ import ( ) func SetupLocker(ctx context.Context, conf *config.Config) (lock.Locker, error) { - client := newRedisClient(conf.Redis) + client := NewSharedClient(conf.Redis) locker := redis.NewLocker(client, int(conf.Redis.LockMaxRetries)) return locker, nil } diff --git a/internal/setup/proxy_repository.go b/internal/setup/proxy_repository.go index 352d009..97a4be3 100644 --- a/internal/setup/proxy_repository.go +++ b/internal/setup/proxy_repository.go @@ -3,19 +3,11 @@ package setup import ( "context" - "forge.cadoles.com/cadoles/bouncer/internal/config" "forge.cadoles.com/cadoles/bouncer/internal/store" redisStore "forge.cadoles.com/cadoles/bouncer/internal/store/redis" "github.com/redis/go-redis/v9" ) -func NewRedisClient(ctx context.Context, conf config.RedisConfig) redis.UniversalClient { - return redis.NewUniversalClient(&redis.UniversalOptions{ - Addrs: conf.Adresses, - MasterName: string(conf.Master), - }) -} - func NewProxyRepository(ctx context.Context, client redis.UniversalClient) (store.ProxyRepository, error) { return redisStore.NewProxyRepository(client, redisStore.DefaultTxMaxAttempts, redisStore.DefaultTxBaseDelay), nil } diff --git a/internal/setup/queue_layer.go b/internal/setup/queue_layer.go index 8b97f70..863ebc5 100644 --- a/internal/setup/queue_layer.go +++ b/internal/setup/queue_layer.go @@ -35,6 +35,6 @@ func setupQueueLayer(conf *config.Config) (director.Layer, error) { } func newQueueAdapter(redisConf config.RedisConfig) (queue.Adapter, error) { - rdb := newRedisClient(redisConf) + rdb := NewSharedClient(redisConf) return queueRedis.NewAdapter(rdb, 2), nil } diff --git a/internal/setup/redis.go b/internal/setup/redis.go index 3ec583a..e171fcc 100644 --- a/internal/setup/redis.go +++ b/internal/setup/redis.go @@ -1,14 +1,38 @@ package setup import ( + "context" + "strings" + "sync" "time" "forge.cadoles.com/cadoles/bouncer/internal/config" + "github.com/pkg/errors" "github.com/redis/go-redis/v9" + "gitlab.com/wpetit/goweb/logger" ) +var clients sync.Map + +func NewSharedClient(conf config.RedisConfig) redis.UniversalClient { + key := strings.Join(conf.Adresses, "|") + "|" + string(conf.Master) + + value, exists := clients.Load(key) + if exists { + if client, ok := (value).(redis.UniversalClient); ok { + return client + } + } + + client := newRedisClient(conf) + + clients.Store(key, client) + + return client +} + func newRedisClient(conf config.RedisConfig) redis.UniversalClient { - return redis.NewUniversalClient(&redis.UniversalOptions{ + client := redis.NewUniversalClient(&redis.UniversalOptions{ Addrs: conf.Adresses, MasterName: string(conf.Master), ReadTimeout: time.Duration(conf.ReadTimeout), @@ -16,5 +40,33 @@ func newRedisClient(conf config.RedisConfig) redis.UniversalClient { DialTimeout: time.Duration(conf.DialTimeout), RouteByLatency: true, ContextTimeoutEnabled: true, + MaxRetries: int(conf.MaxRetries), }) + + go func() { + ctx := logger.With(context.Background(), + logger.F("adresses", conf.Adresses), + logger.F("master", conf.Master), + ) + + timer := time.NewTicker(time.Duration(conf.PingInterval)) + defer timer.Stop() + + connected := true + + for range timer.C { + if _, err := client.Ping(ctx).Result(); err != nil { + logger.Error(ctx, "redis disconnected", logger.E(errors.WithStack(err))) + connected = false + continue + } + + if !connected { + logger.Info(ctx, "redis reconnected") + connected = true + } + } + }() + + return client } diff --git a/internal/store/redis/helper.go b/internal/store/redis/helper.go index 60db019..954b171 100644 --- a/internal/store/redis/helper.go +++ b/internal/store/redis/helper.go @@ -91,12 +91,14 @@ func WithRetry(ctx context.Context, client redis.UniversalClient, key string, fn continue } - return err + return errors.WithStack(err) } return nil } + logger.Error(ctx, "redis error", logger.E(errors.WithStack(err))) + return errors.WithStack(redis.TxFailedErr) } diff --git a/misc/packaging/common/config.yml b/misc/packaging/common/config.yml index 1c1559c..85d454d 100644 --- a/misc/packaging/common/config.yml +++ b/misc/packaging/common/config.yml @@ -190,6 +190,8 @@ redis: writeTimeout: 30s readTimeout: 30s dialTimeout: 30s + maxRetries: 3 + pingInterval: 30s # Configuration des logs logger: diff --git a/misc/siege/siege.conf b/misc/siege/siege.conf new file mode 100644 index 0000000..5f90b9d --- /dev/null +++ b/misc/siege/siege.conf @@ -0,0 +1,79 @@ +# Updated by Siege %_VERSION%, %_DATE% +# Copyright 2000-2016 by %_AUTHOR% +# +# Siege configuration file -- edit as necessary +# For more information about configuring and running this program, +# visit: http://www.joedog.org/ + +# +# +# Verbose mode: With this feature enabled, siege will print the +# result of each transaction to stdout. (Enabled by default) +# +# ex: verbose = true|false +# +verbose = true + +# +# Color mode: This option works in conjunction with verbose mode. +# It tells siege whether or not it should display its output in +# color-coded output. (Enabled by default) +# +# ex: color = on | off +# +color = on + +# +# Cache revalidation. Siege supports cache revalidation for both ETag +# and Last-modified headers. If a copy is still fresh, the server +# responds with 304. While this feature is required for HTTP/1.1, it +# may not be welcomed for load testing. We allow you to breach the +# protocol and turn off caching +# +# HTTP/1.1 200 0.00 secs: 2326 bytes ==> /apache_pb.gif +# HTTP/1.1 304 0.00 secs: 0 bytes ==> /apache_pb.gif +# HTTP/1.1 304 0.00 secs: 0 bytes ==> /apache_pb.gif +# +# Siege also supports Cache-control headers. Consider this server +# response: Cache-Control: max-age=3 +# That tells siege to cache the file for three seconds. While it +# doesn't actually store the file, it will logically grab it from +# its cache. In verbose output, it designates a cached resource +# with (c): +# +# HTTP/1.1 200 0.25 secs: 159 bytes ==> GET /expires/ +# HTTP/1.1 200 1.48 secs: 498419 bytes ==> GET /expires/Otter_in_Southwold.jpg +# HTTP/1.1 200 0.24 secs: 159 bytes ==> GET /expires/ +# HTTP/1.1 200(C) 0.00 secs: 0 bytes ==> GET /expires/Otter_in_Southwold.jpg +# +# NOTE: with color enabled, cached URLs appear in green +# +# ex: cache = true +# +cache = true + +# +# Cookie support: by default siege accepts cookies. This directive is +# available to disable that support. Set cookies to 'false' to refuse +# cookies. Set it to 'true' to accept them. The default value is true. +# If you want to maintain state with the server, then this MUST be set +# to true. +# +# ex: cookies = false +# +cookies = true + +# +# Failures: This is the number of total connection failures allowed +# before siege aborts. Connection failures (timeouts, socket failures, +# etc.) are combined with 400 and 500 level errors in the final stats, +# but those errors do not count against the abort total. If you set +# this total to 10, then siege will abort after ten socket timeouts, +# but it will NOT abort after ten 404s. This is designed to prevent a +# run-away mess on an unattended siege. +# +# The default value is 1024 +# +# ex: failures = 50 +# +failures = -1 \ No newline at end of file