feat: rewrite bus to prevent deadlocks

pkg/storage/driver/rpc/client/client_pool.go

@@ -5,6 +5,7 @@ import (
 	"net/url"
 	"strconv"
 	"sync"
+	"time"
 
 	"github.com/jackc/puddle/v2"
 	"github.com/keegancsmith/rpc"
@@ -74,21 +75,59 @@ func WithPooledClient(serverURL *url.URL) WithClientFunc {
 			return errors.WithStack(err)
 		}
 
-		clientResource, err := pool.Acquire(ctx)
-		if err != nil {
-			return errors.WithStack(err)
-		}
+		attempts := 0
+		max := 5
 
-		if err := fn(ctx, clientResource.Value()); err != nil {
-			if errors.Is(err, rpc.ErrShutdown) {
-				clientResource.Destroy()
+		for {
+			if attempts >= max {
+				logger.Debug(ctx, "rpc client call retrying failed", logger.F("attempts", attempts))
+
+				return errors.Wrapf(err, "rpc client call failed after %d attempts", max)
 			}
 
-			return errors.WithStack(err)
+			clientResource, err := pool.Acquire(ctx)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+
+			client := clientResource.Value()
+
+			if err := fn(ctx, client); err != nil {
+				if errors.Is(err, rpc.ErrShutdown) {
+					clientResource.Destroy()
+
+					wait := time.Duration(8<<(attempts+1)) * time.Millisecond
+
+					logger.Warn(
+						ctx, "rpc client connection is shutdown, retrying",
+						logger.F("attempts", attempts),
+						logger.F("max", max),
+						logger.F("delay", wait),
+					)
+
+					timer := time.NewTimer(wait)
+					select {
+					case <-timer.C:
+						attempts++
+						continue
+
+					case <-ctx.Done():
+						if err := ctx.Err(); err != nil {
+							return errors.WithStack(err)
+						}
+
+						return nil
+					}
+				}
+
+				clientResource.Release()
+
+				return errors.WithStack(err)
+			}
+
+			clientResource.Release()
+
+			return nil
 		}
-
-		clientResource.Release()
-
-		return nil
 	}
 }

pkg/storage/driver/rpc/server/blob/new_blob_reader.go

@@ -45,12 +45,12 @@ func (s *Service) NewBlobReader(ctx context.Context, args *NewBlobReaderArgs, re
 func (s *Service) getOpenedReader(id ReaderID) (io.ReadSeekCloser, error) {
 	raw, exists := s.readers.Load(id)
 	if !exists {
-		return nil, errors.Errorf("could not find writer '%s'", id)
+		return nil, errors.Errorf("could not find reader '%s'", id)
 	}
 
 	reader, ok := raw.(io.ReadSeekCloser)
 	if !ok {
-		return nil, errors.Errorf("unexpected type '%T' for writer", raw)
+		return nil, errors.Errorf("unexpected type '%T' for reader", raw)
 	}
 
 	return reader, nil