feat: add limited retry mechanism to prevent startup error if redis is not ready

internal/lock/redis/locker.go

@@ -12,8 +12,8 @@ import (
 )
 
 type Locker struct {
-	client  redis.UniversalClient
-	timeout time.Duration
+	client     redis.UniversalClient
+	maxRetries int
 }
 
 // WithLock implements lock.Locker.
@@ -26,33 +26,41 @@ func (l *Locker) WithLock(ctx context.Context, key string, timeout time.Duration
 
 	logger.Debug(ctx, "acquiring lock")
 
-	lock, err := locker.Obtain(ctx, key, timeout, &redislock.Options{
-		RetryStrategy: backoff,
-	})
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	logger.Debug(ctx, "lock obtained")
-
-	defer func() {
-		if err := lock.Release(ctx); err != nil {
-			logger.Error(ctx, "could not release lock", logger.E(errors.WithStack(err)))
+	err := retryWithBackoff(ctx, l.maxRetries, func(ctx context.Context) error {
+		lock, err := locker.Obtain(ctx, key, timeout, &redislock.Options{
+			RetryStrategy: backoff,
+		})
+		if err != nil {
+			return errors.WithStack(err)
 		}
 
-		logger.Debug(ctx, "lock released")
-	}()
+		logger.Debug(ctx, "lock obtained")
 
-	if err := fn(ctx); err != nil {
+		defer func() {
+			if err := lock.Release(ctx); err != nil {
+				logger.Error(ctx, "could not release lock", logger.E(errors.WithStack(err)))
+			}
+
+			logger.Debug(ctx, "lock released")
+		}()
+
+		if err := fn(ctx); err != nil {
+			return errors.WithStack(err)
+		}
+
+		return nil
+	})
+	if err != nil {
 		return errors.WithStack(err)
 	}
 
 	return nil
 }
 
-func NewLocker(client redis.UniversalClient) *Locker {
+func NewLocker(client redis.UniversalClient, maxRetries int) *Locker {
 	return &Locker{
-		client: client,
+		client:     client,
+		maxRetries: maxRetries,
 	}
 }
 

internal/lock/redis/retry.go

@@ -0,0 +1,42 @@
+package redis
+
+import (
+	"context"
+	"time"
+
+	"github.com/pkg/errors"
+	"gitlab.com/wpetit/goweb/logger"
+)
+
+const (
+	baseWatchBackoffDelay = time.Millisecond * 500
+	maxDelay              = time.Minute * 10
+)
+
+func retryWithBackoff(ctx context.Context, attempts int, fn func(ctx context.Context) error) error {
+	backoffDelay := baseWatchBackoffDelay
+	count := 0
+
+	for {
+		err := fn(ctx)
+		if err == nil {
+			return nil
+		}
+
+		err = errors.WithStack(err)
+
+		count++
+		if count >= attempts {
+			return errors.Wrapf(err, "execution failed after %d attempts", attempts)
+		}
+
+		logger.Error(ctx, "error while executing func, retrying with backoff", logger.E(err), logger.F("backoffDelay", backoffDelay), logger.F("remainingAttempts", attempts-count))
+
+		time.Sleep(backoffDelay)
+
+		backoffDelay *= 2
+		if backoffDelay > maxDelay {
+			backoffDelay = maxDelay
+		}
+	}
+}