feat: add limited retry mechanism to prevent startup error if redis is not ready

doc/fr/references/admin_api.md

@@ -22,9 +22,8 @@ Où:
 
 - `"<subject>"` est une chaîne de caractère arbitraire ayant pour objectif d'identifier de manière unique l'utilisateur associé au jeton;
 - `"<role>"` peut prendre une des deux valeurs `reader` ou `writer` correspondant aux droits suivants respectifs:
-    - droit en lecture sur l'ensemble des entités (proxy, layer);
-    - droit en lecture ET en écriture sur l'ensemble des entités.
-
+  - droit en lecture sur l'ensemble des entités (proxy, layer);
+  - droit en lecture ET en écriture sur l'ensemble des entités.
 
 ## Points d'entrée
 
@@ -34,29 +33,29 @@ Créer un nouveau proxy
 
 #### Exemple de corps de requête
 
-```json5
+```json
 {
-  "name": "myproxy",                // OBLIGATOIRE - Nom du proxy
-  "to": "https://www.cadoles.com",  // OBLIGATOIRE - Site distant ciblé par le proxy
-  "from": ["*"]                     // OPTIONNEL - Liste de patrons de filtrage associés au proxy
+  "name": "myproxy", // OBLIGATOIRE - Nom du proxy
+  "to": "https://www.cadoles.com", // OBLIGATOIRE - Site distant ciblé par le proxy
+  "from": ["*"] // OPTIONNEL - Liste de patrons de filtrage associés au proxy
 }
 ```
 
 #### Exemple de résultat
 
-```json5
+```json
 {
-    "data": {
-        "proxy": {
-            "name": "myproxy",
-            "weight": 0,
-            "enabled": false,
-            "to": "https://www.cadoles.com",
-            "from": ["*"],
-            "createdAt": "2018-12-10T13:45:00.000Z",
-            "updatedAt": "2018-12-10T13:45:00.000Z"
-        }
+  "data": {
+    "proxy": {
+      "name": "myproxy",
+      "weight": 0,
+      "enabled": false,
+      "to": "https://www.cadoles.com",
+      "from": ["*"],
+      "createdAt": "2018-12-10T13:45:00.000Z",
+      "updatedAt": "2018-12-10T13:45:00.000Z"
     }
+  }
 }
 ```
 
@@ -74,19 +73,19 @@ Récupérer les informations complètes sur un proxy
 
 #### Exemple de résultat
 
-```json5
+```json
 {
-    "data": {
-        "proxy": {
-            "name": "myproxy",
-            "weight": 0,
-            "enabled": false,
-            "to": "https://www.cadoles.com",
-            "from": ["*"],
-            "createdAt": "2018-12-10T13:45:00.000Z",
-            "updatedAt": "2018-12-10T13:45:00.000Z"
-        }
+  "data": {
+    "proxy": {
+      "name": "myproxy",
+      "weight": 0,
+      "enabled": false,
+      "to": "https://www.cadoles.com",
+      "from": ["*"],
+      "createdAt": "2018-12-10T13:45:00.000Z",
+      "updatedAt": "2018-12-10T13:45:00.000Z"
     }
+  }
 }
 ```
 
@@ -100,30 +99,30 @@ Modifier un proxy
 
 #### Exemple de corps de requête
 
-```json5
+```json
 {
-  "to": "https://www.cadoles.com",  // OPTIONNEL - Site distant ciblé par le proxy
+  "to": "https://www.cadoles.com", // OPTIONNEL - Site distant ciblé par le proxy
   "from": ["mylocalproxydomain:*"], // OPTIONNEL - Liste de patrons de filtrage associés au proxy
-  "weight": 100,                    // OPTIONNEL - Poids à associer au proxy
-  "enabled": true,                  // OPTIONNEL - Activer/désactiver le proxy
+  "weight": 100, // OPTIONNEL - Poids à associer au proxy
+  "enabled": true // OPTIONNEL - Activer/désactiver le proxy
 }
 ```
 
 #### Exemple de résultat
 
-```json5
+```json
 {
-    "data": {
-        "proxy": {
-            "name": "myproxy",
-            "weight": 100,
-            "enabled": true,
-            "to": "https://www.cadoles.com",
-            "from": ["mylocalproxydomain:*"],
-            "createdAt": "2018-12-10T13:45:00.000Z",
-            "updatedAt": "2020-10-02T15:09:00.000Z"
-        }
+  "data": {
+    "proxy": {
+      "name": "myproxy",
+      "weight": 100,
+      "enabled": true,
+      "to": "https://www.cadoles.com",
+      "from": ["mylocalproxydomain:*"],
+      "createdAt": "2018-12-10T13:45:00.000Z",
+      "updatedAt": "2020-10-02T15:09:00.000Z"
     }
+  }
 }
 ```
 
@@ -141,17 +140,17 @@ Lister les proxies existants
 
 #### Exemple de résultat
 
-```json5
+```json
 {
-    "data": {
-        "proxies": [
-            {
-                "name": "myproxy",
-                "weight": 0,
-                "enabled": false,
-            }
-        ]
-    }
+  "data": {
+    "proxies": [
+      {
+        "name": "myproxy",
+        "weight": 0,
+        "enabled": false
+      }
+    ]
+  }
 }
 ```
 
@@ -169,11 +168,11 @@ Supprimer le proxy
 
 #### Exemple de résultat
 
-```json5
+```json
 {
-    "data": {
-        "proxyName": "myproxy"
-    }
+  "data": {
+    "proxyName": "myproxy"
+  }
 }
 ```
 

internal/admin/bootstrap.go

@@ -22,7 +22,7 @@ func (s *Server) bootstrapProxies(ctx context.Context) error {
 	layerRepo := s.layerRepository
 
 	lockTimeout := time.Duration(s.bootstrapConfig.LockTimeout)
-	locker := redis.NewLocker(s.redisClient)
+	locker := redis.NewLocker(s.redisClient, int(s.bootstrapConfig.MaxConnectionRetries))
 
 	err := locker.WithLock(ctx, "bouncer-admin-bootstrap", lockTimeout, func(ctx context.Context) error {
 		logger.Info(ctx, "bootstrapping proxies")

internal/config/bootstrap.go

@@ -12,9 +12,10 @@ import (
 )
 
 type BootstrapConfig struct {
-	Proxies     map[store.ProxyName]BootstrapProxyConfig `yaml:"proxies"`
-	Dir         InterpolatedString                       `yaml:"dir"`
-	LockTimeout InterpolatedDuration                     `yaml:"lockTimeout"`
+	Proxies              map[store.ProxyName]BootstrapProxyConfig `yaml:"proxies"`
+	Dir                  InterpolatedString                       `yaml:"dir"`
+	LockTimeout          InterpolatedDuration                     `yaml:"lockTimeout"`
+	MaxConnectionRetries InterpolatedInt                          `yaml:"maxRetries"`
 }
 
 func (c *BootstrapConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -62,8 +63,9 @@ type BootstrapLayerConfig struct {
 
 func NewDefaultBootstrapConfig() BootstrapConfig {
 	return BootstrapConfig{
-		Dir:         "",
-		LockTimeout: *NewInterpolatedDuration(30 * time.Second),
+		Dir:                  "",
+		LockTimeout:          *NewInterpolatedDuration(30 * time.Second),
+		MaxConnectionRetries: 10,
 	}
 }
 

internal/config/redis.go

@@ -9,19 +9,21 @@ const (
 )
 
 type RedisConfig struct {
-	Adresses     InterpolatedStringSlice `yaml:"addresses"`
-	Master       InterpolatedString      `yaml:"master"`
-	ReadTimeout  InterpolatedDuration    `yaml:"readTimeout"`
-	WriteTimeout InterpolatedDuration    `yaml:"writeTimeout"`
-	DialTimeout  InterpolatedDuration    `yaml:"dialTimeout"`
+	Adresses       InterpolatedStringSlice `yaml:"addresses"`
+	Master         InterpolatedString      `yaml:"master"`
+	ReadTimeout    InterpolatedDuration    `yaml:"readTimeout"`
+	WriteTimeout   InterpolatedDuration    `yaml:"writeTimeout"`
+	DialTimeout    InterpolatedDuration    `yaml:"dialTimeout"`
+	LockMaxRetries InterpolatedInt         `yaml:"lockMaxRetries"`
 }
 
 func NewDefaultRedisConfig() RedisConfig {
 	return RedisConfig{
-		Adresses:     InterpolatedStringSlice{"localhost:6379"},
-		Master:       "",
-		ReadTimeout:  InterpolatedDuration(30 * time.Second),
-		WriteTimeout: InterpolatedDuration(30 * time.Second),
-		DialTimeout:  InterpolatedDuration(30 * time.Second),
+		Adresses:       InterpolatedStringSlice{"localhost:6379"},
+		Master:         "",
+		ReadTimeout:    InterpolatedDuration(30 * time.Second),
+		WriteTimeout:   InterpolatedDuration(30 * time.Second),
+		DialTimeout:    InterpolatedDuration(30 * time.Second),
+		LockMaxRetries: 10,
 	}
 }

internal/lock/redis/locker.go

@@ -12,8 +12,8 @@ import (
 )
 
 type Locker struct {
-	client  redis.UniversalClient
-	timeout time.Duration
+	client     redis.UniversalClient
+	maxRetries int
 }
 
 // WithLock implements lock.Locker.
@@ -26,33 +26,41 @@ func (l *Locker) WithLock(ctx context.Context, key string, timeout time.Duration
 
 	logger.Debug(ctx, "acquiring lock")
 
-	lock, err := locker.Obtain(ctx, key, timeout, &redislock.Options{
-		RetryStrategy: backoff,
-	})
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	logger.Debug(ctx, "lock obtained")
-
-	defer func() {
-		if err := lock.Release(ctx); err != nil {
-			logger.Error(ctx, "could not release lock", logger.E(errors.WithStack(err)))
+	err := retryWithBackoff(ctx, l.maxRetries, func(ctx context.Context) error {
+		lock, err := locker.Obtain(ctx, key, timeout, &redislock.Options{
+			RetryStrategy: backoff,
+		})
+		if err != nil {
+			return errors.WithStack(err)
 		}
 
-		logger.Debug(ctx, "lock released")
-	}()
+		logger.Debug(ctx, "lock obtained")
 
-	if err := fn(ctx); err != nil {
+		defer func() {
+			if err := lock.Release(ctx); err != nil {
+				logger.Error(ctx, "could not release lock", logger.E(errors.WithStack(err)))
+			}
+
+			logger.Debug(ctx, "lock released")
+		}()
+
+		if err := fn(ctx); err != nil {
+			return errors.WithStack(err)
+		}
+
+		return nil
+	})
+	if err != nil {
 		return errors.WithStack(err)
 	}
 
 	return nil
 }
 
-func NewLocker(client redis.UniversalClient) *Locker {
+func NewLocker(client redis.UniversalClient, maxRetries int) *Locker {
 	return &Locker{
-		client: client,
+		client:     client,
+		maxRetries: maxRetries,
 	}
 }
 

internal/lock/redis/retry.go

@@ -0,0 +1,42 @@
+package redis
+
+import (
+	"context"
+	"time"
+
+	"github.com/pkg/errors"
+	"gitlab.com/wpetit/goweb/logger"
+)
+
+const (
+	baseWatchBackoffDelay = time.Millisecond * 500
+	maxDelay              = time.Minute * 10
+)
+
+func retryWithBackoff(ctx context.Context, attempts int, fn func(ctx context.Context) error) error {
+	backoffDelay := baseWatchBackoffDelay
+	count := 0
+
+	for {
+		err := fn(ctx)
+		if err == nil {
+			return nil
+		}
+
+		err = errors.WithStack(err)
+
+		count++
+		if count >= attempts {
+			return errors.Wrapf(err, "execution failed after %d attempts", attempts)
+		}
+
+		logger.Error(ctx, "error while executing func, retrying with backoff", logger.E(err), logger.F("backoffDelay", backoffDelay), logger.F("remainingAttempts", attempts-count))
+
+		time.Sleep(backoffDelay)
+
+		backoffDelay *= 2
+		if backoffDelay > maxDelay {
+			backoffDelay = maxDelay
+		}
+	}
+}

internal/proxy/director/director.go

@@ -40,13 +40,14 @@ MAIN:
 				logger.F("from", from),
 			)
 			if matches := wildcard.Match(url.String(), from); !matches {
-				logger.Debug(
-					ctx, "proxy's from matched",
-					logger.F("from", from),
-				)
 				continue
 			}
 
+			logger.Debug(
+				ctx, "proxy's from matched",
+				logger.F("from", from),
+			)
+
 			match = p
 			break MAIN
 		}

internal/setup/integrations.go

@@ -28,7 +28,7 @@ func SetupIntegrations(ctx context.Context, conf *config.Config) ([]integration.
 
 func setupKubernetesIntegration(ctx context.Context, conf *config.Config) (*kubernetes.Integration, error) {
 	client := newRedisClient(conf.Redis)
-	locker := redis.NewLocker(client)
+	locker := redis.NewLocker(client, 10)
 
 	integration := kubernetes.NewIntegration(
 		kubernetes.WithReaderTokenSecret(string(conf.Integrations.Kubernetes.ReaderTokenSecret)),

internal/setup/lock.go

@@ -10,6 +10,6 @@ import (
 
 func SetupLocker(ctx context.Context, conf *config.Config) (lock.Locker, error) {
 	client := newRedisClient(conf.Redis)
-	locker := redis.NewLocker(client)
+	locker := redis.NewLocker(client, int(conf.Redis.LockMaxRetries))
 	return locker, nil
 }