feat: use shared redis client to maximize pooling usage (#39)

Makefile

@@ -17,7 +17,8 @@ GOTEST_ARGS ?= -short
 OPENWRT_DEVICE ?= 192.168.1.1
 
 SIEGE_URLS_FILE ?= misc/siege/urls.txt
-SIEGE_CONCURRENCY ?= 100
+SIEGE_CONCURRENCY ?= 50
+SIEGE_DURATION ?= 1M
 
 data/bootstrap.d/dummy.yml:
 	mkdir -p data/bootstrap.d
@@ -114,7 +115,7 @@ grafterm: tools/grafterm/bin/grafterm
 siege:
 	$(eval TMP := $(shell mktemp))
 	cat $(SIEGE_URLS_FILE) | envsubst > $(TMP)
-	siege -i -b -c $(SIEGE_CONCURRENCY) -f $(TMP)
+	siege -R ./misc/siege/siege.conf -i -b -c $(SIEGE_CONCURRENCY) -t $(SIEGE_DURATION) -f $(TMP)
 	rm -rf $(TMP)
 
 tools/gitea-release/bin/gitea-release.sh:
@@ -150,7 +151,7 @@ run-redis:
 		-v $(PWD)/data/redis:/data \
 		-p 6379:6379 \
 		redis:alpine3.17 \
-		redis-server --save 60 1 --loglevel warning
+		redis-server --save 60 1 --loglevel debug
 
 redis-shell:
 	docker exec -it \

internal/admin/init.go

@@ -27,7 +27,7 @@ func (s *Server) initRepositories(ctx context.Context) error {
 }
 
 func (s *Server) initRedisClient(ctx context.Context) error {
-	client := setup.NewRedisClient(ctx, s.redisConfig)
+	client := setup.NewSharedClient(s.redisConfig)
 
 	s.redisClient = client
 

internal/config/redis.go

@@ -15,6 +15,8 @@ type RedisConfig struct {
 	WriteTimeout   InterpolatedDuration    `yaml:"writeTimeout"`
 	DialTimeout    InterpolatedDuration    `yaml:"dialTimeout"`
 	LockMaxRetries InterpolatedInt         `yaml:"lockMaxRetries"`
+	MaxRetries     InterpolatedInt         `yaml:"maxRetries"`
+	PingInterval   InterpolatedDuration    `yaml:"pingInterval"`
 }
 
 func NewDefaultRedisConfig() RedisConfig {
@@ -25,5 +27,7 @@ func NewDefaultRedisConfig() RedisConfig {
 		WriteTimeout:   InterpolatedDuration(30 * time.Second),
 		DialTimeout:    InterpolatedDuration(30 * time.Second),
 		LockMaxRetries: 10,
+		MaxRetries:     3,
+		PingInterval:   InterpolatedDuration(30 * time.Second),
 	}
 }

internal/proxy/init.go

@@ -9,7 +9,7 @@ import (
 )
 
 func (s *Server) initRepositories(ctx context.Context) error {
-	client := setup.NewRedisClient(ctx, s.redisConfig)
+	client := setup.NewSharedClient(s.redisConfig)
 
 	if err := s.initProxyRepository(ctx, client); err != nil {
 		return errors.WithStack(err)

internal/setup/authn_oidc_layer.go

@@ -23,7 +23,7 @@ func init() {
 }
 
 func setupAuthnOIDCLayer(conf *config.Config) (director.Layer, error) {
-	rdb := newRedisClient(conf.Redis)
+	rdb := NewSharedClient(conf.Redis)
 	adapter := redis.NewStoreAdapter(rdb)
 	store := session.NewStore(adapter)
 

internal/setup/integrations.go

@@ -27,7 +27,7 @@ func SetupIntegrations(ctx context.Context, conf *config.Config) ([]integration.
 }
 
 func setupKubernetesIntegration(ctx context.Context, conf *config.Config) (*kubernetes.Integration, error) {
-	client := newRedisClient(conf.Redis)
+	client := NewSharedClient(conf.Redis)
 	locker := redis.NewLocker(client, 10)
 
 	integration := kubernetes.NewIntegration(

internal/setup/lock.go

@@ -9,7 +9,7 @@ import (
 )
 
 func SetupLocker(ctx context.Context, conf *config.Config) (lock.Locker, error) {
-	client := newRedisClient(conf.Redis)
+	client := NewSharedClient(conf.Redis)
 	locker := redis.NewLocker(client, int(conf.Redis.LockMaxRetries))
 	return locker, nil
 }

internal/setup/proxy_repository.go

@@ -3,19 +3,11 @@ package setup
 import (
 	"context"
 
-	"forge.cadoles.com/cadoles/bouncer/internal/config"
 	"forge.cadoles.com/cadoles/bouncer/internal/store"
 	redisStore "forge.cadoles.com/cadoles/bouncer/internal/store/redis"
 	"github.com/redis/go-redis/v9"
 )
 
-func NewRedisClient(ctx context.Context, conf config.RedisConfig) redis.UniversalClient {
-	return redis.NewUniversalClient(&redis.UniversalOptions{
-		Addrs:      conf.Adresses,
-		MasterName: string(conf.Master),
-	})
-}
-
 func NewProxyRepository(ctx context.Context, client redis.UniversalClient) (store.ProxyRepository, error) {
 	return redisStore.NewProxyRepository(client, redisStore.DefaultTxMaxAttempts, redisStore.DefaultTxBaseDelay), nil
 }

internal/setup/queue_layer.go

@@ -35,6 +35,6 @@ func setupQueueLayer(conf *config.Config) (director.Layer, error) {
 }
 
 func newQueueAdapter(redisConf config.RedisConfig) (queue.Adapter, error) {
-	rdb := newRedisClient(redisConf)
+	rdb := NewSharedClient(redisConf)
 	return queueRedis.NewAdapter(rdb, 2), nil
 }

internal/setup/redis.go

@@ -1,14 +1,38 @@
 package setup
 
 import (
+	"context"
+	"strings"
+	"sync"
 	"time"
 
 	"forge.cadoles.com/cadoles/bouncer/internal/config"
+	"github.com/pkg/errors"
 	"github.com/redis/go-redis/v9"
+	"gitlab.com/wpetit/goweb/logger"
 )
 
+var clients sync.Map
+
+func NewSharedClient(conf config.RedisConfig) redis.UniversalClient {
+	key := strings.Join(conf.Adresses, "|") + "|" + string(conf.Master)
+
+	value, exists := clients.Load(key)
+	if exists {
+		if client, ok := (value).(redis.UniversalClient); ok {
+			return client
+		}
+	}
+
+	client := newRedisClient(conf)
+
+	clients.Store(key, client)
+
+	return client
+}
+
 func newRedisClient(conf config.RedisConfig) redis.UniversalClient {
-	return redis.NewUniversalClient(&redis.UniversalOptions{
+	client := redis.NewUniversalClient(&redis.UniversalOptions{
 		Addrs:                 conf.Adresses,
 		MasterName:            string(conf.Master),
 		ReadTimeout:           time.Duration(conf.ReadTimeout),
@@ -16,5 +40,33 @@ func newRedisClient(conf config.RedisConfig) redis.UniversalClient {
 		DialTimeout:           time.Duration(conf.DialTimeout),
 		RouteByLatency:        true,
 		ContextTimeoutEnabled: true,
+		MaxRetries:            int(conf.MaxRetries),
 	})
+
+	go func() {
+		ctx := logger.With(context.Background(),
+			logger.F("adresses", conf.Adresses),
+			logger.F("master", conf.Master),
+		)
+
+		timer := time.NewTicker(time.Duration(conf.PingInterval))
+		defer timer.Stop()
+
+		connected := true
+
+		for range timer.C {
+			if _, err := client.Ping(ctx).Result(); err != nil {
+				logger.Error(ctx, "redis disconnected", logger.E(errors.WithStack(err)))
+				connected = false
+				continue
+			}
+
+			if !connected {
+				logger.Info(ctx, "redis reconnected")
+				connected = true
+			}
+		}
+	}()
+
+	return client
 }

internal/store/redis/helper.go

@@ -91,12 +91,14 @@ func WithRetry(ctx context.Context, client redis.UniversalClient, key string, fn
 				continue
 			}
 
-			return err
+			return errors.WithStack(err)
 		}
 
 		return nil
 	}
 
+	logger.Error(ctx, "redis error", logger.E(errors.WithStack(err)))
+
 	return errors.WithStack(redis.TxFailedErr)
 }
 

misc/packaging/common/config.yml

@@ -190,6 +190,8 @@ redis:
   writeTimeout: 30s
   readTimeout: 30s
   dialTimeout: 30s
+  maxRetries: 3
+  pingInterval: 30s
 
 # Configuration des logs
 logger:

misc/siege/siege.conf

@@ -0,0 +1,79 @@
+# Updated by Siege %_VERSION%, %_DATE%
+# Copyright 2000-2016 by %_AUTHOR%
+# 
+# Siege configuration file -- edit as necessary
+# For more information about configuring and running this program, 
+# visit: http://www.joedog.org/
+
+#
+#
+# Verbose mode: With this feature enabled, siege will print the
+# result of each transaction to stdout. (Enabled by default)
+#
+# ex: verbose = true|false
+#
+verbose = true
+
+#
+# Color mode: This option works in conjunction with verbose mode.
+# It tells siege whether or not it should display its output in
+# color-coded output. (Enabled by default)
+#
+# ex: color = on | off
+# 
+color = on
+
+# 
+# Cache revalidation. Siege supports cache revalidation for both ETag
+# and Last-modified headers. If a copy is still fresh, the server 
+# responds with 304. While this feature is required for HTTP/1.1, it 
+# may not be welcomed for load testing. We allow you to breach the 
+# protocol and turn off caching
+#
+# HTTP/1.1 200   0.00 secs:    2326 bytes ==> /apache_pb.gif
+# HTTP/1.1 304   0.00 secs:       0 bytes ==> /apache_pb.gif
+# HTTP/1.1 304   0.00 secs:       0 bytes ==> /apache_pb.gif
+#
+# Siege also supports Cache-control headers. Consider this server 
+# response: Cache-Control: max-age=3
+# That tells siege to cache the file for three seconds. While it
+# doesn't actually store the file, it will logically grab it from
+# its cache. In verbose output, it designates a cached resource 
+# with (c):
+#
+# HTTP/1.1 200     0.25 secs:     159 bytes ==> GET  /expires/
+# HTTP/1.1 200     1.48 secs:  498419 bytes ==> GET  /expires/Otter_in_Southwold.jpg
+# HTTP/1.1 200     0.24 secs:     159 bytes ==> GET  /expires/
+# HTTP/1.1 200(C)  0.00 secs:       0 bytes ==> GET  /expires/Otter_in_Southwold.jpg
+#
+# NOTE: with color enabled, cached URLs appear in green
+# 
+# ex: cache = true
+#
+cache = true
+
+#
+# Cookie support: by default siege accepts cookies. This directive is
+# available to disable that support. Set cookies to 'false' to refuse 
+# cookies. Set it to 'true' to accept them. The default value is true. 
+# If you want to maintain state with the server, then this MUST be set 
+# to true.
+#
+# ex: cookies = false
+#
+cookies = true 
+
+#
+# Failures: This is the number of total connection failures allowed 
+# before siege aborts. Connection failures (timeouts, socket failures,
+# etc.) are combined with 400 and 500 level errors in the final stats, 
+# but those errors do not count against the abort total.  If you set 
+# this total to 10, then siege will abort after ten socket timeouts, 
+# but it will NOT abort after ten 404s. This is designed to prevent a 
+# run-away mess on an unattended siege. 
+#
+# The default value is 1024
+#
+# ex: failures = 50
+#
+failures = -1