feat(k8s): use secret as shared source for admin private key

internal/admin/init.go

@@ -3,8 +3,11 @@ package admin
 import (
 	"context"
 
+	"forge.cadoles.com/cadoles/bouncer/internal/integration"
+	"forge.cadoles.com/cadoles/bouncer/internal/jwk"
 	"forge.cadoles.com/cadoles/bouncer/internal/setup"
 	"github.com/pkg/errors"
+	"gitlab.com/wpetit/goweb/logger"
 )
 
 func (s *Server) initRepositories(ctx context.Context) error {
@@ -52,3 +55,34 @@ func (s *Server) initProxyRepository(ctx context.Context) error {
 
 	return nil
 }
+
+func (s *Server) initPrivateKey(ctx context.Context) error {
+	localKey, err := jwk.LoadOrGenerate(string(s.serverConfig.Auth.PrivateKey), jwk.DefaultKeySize)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	ctx = integration.WithPrivateKey(ctx, localKey)
+
+	key, err := integration.RunOnKeyLoad(ctx, s.integrations)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	if key != nil {
+		s.privateKey = key
+	} else {
+		s.privateKey = localKey
+	}
+
+	logger.Info(ctx, "using private key", logger.F("keyID", s.privateKey.KeyID()))
+
+	publicKeys, err := jwk.PublicKeySet(s.privateKey)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	s.publicKeys = publicKeys
+
+	return nil
+}

internal/admin/server.go

@@ -35,6 +35,9 @@ type Server struct {
 	bootstrapConfig config.BootstrapConfig
 	proxyRepository store.ProxyRepository
 	layerRepository store.LayerRepository
+
+	privateKey jwk.Key
+	publicKeys jwk.Set
 }
 
 func (s *Server) Start(ctx context.Context) (<-chan net.Addr, <-chan error) {
@@ -67,6 +70,15 @@ func (s *Server) run(parentCtx context.Context, addrs chan net.Addr, errs chan e
 		return
 	}
 
+	if err := s.initPrivateKey(ctx); err != nil {
+		errs <- errors.WithStack(err)
+
+		return
+	}
+
+	ctx = integration.WithPrivateKey(ctx, s.privateKey)
+	ctx = integration.WithPublicKeySet(ctx, s.publicKeys)
+
 	if err := integration.RunOnStartup(ctx, s.integrations); err != nil {
 		errs <- errors.WithStack(err)
 
@@ -96,20 +108,6 @@ func (s *Server) run(parentCtx context.Context, addrs chan net.Addr, errs chan e
 		}
 	}()
 
-	key, err := jwk.LoadOrGenerate(string(s.serverConfig.Auth.PrivateKey), jwk.DefaultKeySize)
-	if err != nil {
-		errs <- errors.WithStack(err)
-
-		return
-	}
-
-	keys, err := jwk.PublicKeySet(key)
-	if err != nil {
-		errs <- errors.WithStack(err)
-
-		return
-	}
-
 	router := chi.NewRouter()
 
 	if s.serverConfig.HTTP.UseRealIP {
@@ -160,7 +158,7 @@ func (s *Server) run(parentCtx context.Context, addrs chan net.Addr, errs chan e
 	router.Route("/api/v1", func(r chi.Router) {
 		r.Group(func(r chi.Router) {
 			r.Use(auth.Middleware(
-				jwt.NewAuthenticator(keys, string(s.serverConfig.Auth.Issuer), jwt.DefaultAcceptableSkew),
+				jwt.NewAuthenticator(s.publicKeys, string(s.serverConfig.Auth.Issuer), jwt.DefaultAcceptableSkew),
 			))
 
 			r.Route("/proxies", func(r chi.Router) {

internal/config/integrations.go

@@ -9,10 +9,14 @@ type IntegrationsConfig struct {
 func NewDefaultIntegrationsConfig() IntegrationsConfig {
 	return IntegrationsConfig{
 		Kubernetes: KubernetesConfig{
-			Enabled:           false,
-			WriterTokenSecret: "",
-			ReaderTokenSecret: "",
-			LockTimeout:       *NewInterpolatedDuration(30 * time.Second),
+			Enabled:                    false,
+			WriterTokenSecret:          "",
+			WriterTokenSecretNamespace: "",
+			ReaderTokenSecretNamespace: "",
+			PrivateKeySecret:           "",
+			PrivateKeySecretNamespace:  "",
+			ReaderTokenSecret:          "",
+			LockTimeout:                *NewInterpolatedDuration(30 * time.Second),
 		},
 	}
 }
@@ -23,5 +27,7 @@ type KubernetesConfig struct {
 	WriterTokenSecretNamespace InterpolatedString   `yaml:"writerTokenSecretNamespace"`
 	ReaderTokenSecret          InterpolatedString   `yaml:"readerTokenSecret"`
 	ReaderTokenSecretNamespace InterpolatedString   `yaml:"readerTokenSecretNamespace"`
+	PrivateKeySecret           InterpolatedString   `yaml:"privateKeySecret"`
+	PrivateKeySecretNamespace  InterpolatedString   `yaml:"privateKeySecretNamespace"`
 	LockTimeout                InterpolatedDuration `yaml:"lockTimeout"`
 }

internal/integration/context.go

@@ -0,0 +1,49 @@
+package integration
+
+import (
+	"context"
+
+	"forge.cadoles.com/cadoles/bouncer/internal/jwk"
+	"github.com/pkg/errors"
+)
+
+var (
+	ErrNotFound = errors.New("not found")
+)
+
+type contextKey string
+
+const (
+	ctxPublicKeySet contextKey = "public-key-set"
+	ctxPrivateKey   contextKey = "private-key"
+)
+
+func CtxPublicKeySet(ctx context.Context) (jwk.Set, error) {
+	return ctxValue[jwk.Set](ctx, ctxPublicKeySet)
+}
+
+func WithPublicKeySet(ctx context.Context, set jwk.Set) context.Context {
+	return context.WithValue(ctx, ctxPublicKeySet, set)
+}
+
+func CtxPrivateKey(ctx context.Context) (jwk.Key, error) {
+	return ctxValue[jwk.Key](ctx, ctxPrivateKey)
+}
+
+func WithPrivateKey(ctx context.Context, key jwk.Key) context.Context {
+	return context.WithValue(ctx, ctxPrivateKey, key)
+}
+
+func ctxValue[T any](ctx context.Context, key contextKey) (T, error) {
+	raw := ctx.Value(key)
+	if raw == nil {
+		return *new(T), errors.WithStack(ErrNotFound)
+	}
+
+	value, ok := raw.(T)
+	if !ok {
+		return *new(T), errors.Errorf("unexpected value type '%T'", raw)
+	}
+
+	return value, nil
+}

internal/integration/integration.go

@@ -3,6 +3,7 @@ package integration
 import (
 	"context"
 
+	"forge.cadoles.com/cadoles/bouncer/internal/jwk"
 	"github.com/pkg/errors"
 )
 
@@ -15,6 +16,11 @@ type OnStartup interface {
 	OnStartup(ctx context.Context) error
 }
 
+type OnKeyLoad interface {
+	Integration
+	OnKeyLoad(ctx context.Context) (jwk.Key, error)
+}
+
 func RunOnStartup(ctx context.Context, integrations []Integration) error {
 	for _, it := range integrations {
 		onStartup, ok := it.(OnStartup)
@@ -29,3 +35,23 @@ func RunOnStartup(ctx context.Context, integrations []Integration) error {
 
 	return nil
 }
+
+func RunOnKeyLoad(ctx context.Context, integrations []Integration) (jwk.Key, error) {
+	for _, it := range integrations {
+		onKeyLoad, ok := it.(OnKeyLoad)
+		if !ok {
+			continue
+		}
+
+		key, err := onKeyLoad.OnKeyLoad(ctx)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		if key != nil {
+			return key, nil
+		}
+	}
+
+	return nil, nil
+}

internal/integration/kubernetes/integration.go

@@ -2,8 +2,7 @@ package kubernetes
 
 import (
 	"context"
-	"crypto"
-	"fmt"
+	"encoding/json"
 	"os"
 
 	"forge.cadoles.com/cadoles/bouncer/internal/auth/jwt"
@@ -28,6 +27,38 @@ type Integration struct {
 	Options *Options
 }
 
+// OnKeyLoad implements integration.OnKeyLoad.
+func (i *Integration) OnKeyLoad(ctx context.Context) (jwk.Key, error) {
+	locker := i.Options.Locker
+	timeout := i.Options.LockTimeout
+
+	var key jwk.Key
+	err := locker.WithLock(ctx, "bouncer-kubernetes-onkeyload", timeout, func(ctx context.Context) error {
+		client, err := i.getClient()
+		if err != nil {
+			return errors.WithStack(err)
+		}
+
+		if i.Options.PrivateKeySecret != "" {
+			sharedPrivateKey, err := i.getSharedPrivateKey(ctx, client, i.Options.PrivateKeySecretNamespace, i.Options.PrivateKeySecret)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+
+			if sharedPrivateKey != nil {
+				key = sharedPrivateKey
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return key, nil
+}
+
 // Integration implements integration.OnStartup.
 func (i *Integration) Integration() {}
 
@@ -36,12 +67,7 @@ func (i *Integration) OnStartup(ctx context.Context) error {
 	locker := i.Options.Locker
 	timeout := i.Options.LockTimeout
 	err := locker.WithLock(ctx, "bouncer-kubernetes-onstartup", timeout, func(ctx context.Context) error {
-		config, err := rest.InClusterConfig()
-		if err != nil {
-			return errors.WithStack(err)
-		}
-
-		client, err := kubernetes.NewForConfig(config)
+		client, err := i.getClient()
 		if err != nil {
 			return errors.WithStack(err)
 		}
@@ -67,6 +93,20 @@ func (i *Integration) OnStartup(ctx context.Context) error {
 	return nil
 }
 
+func (i *Integration) getClient() (*kubernetes.Clientset, error) {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return client, nil
+}
+
 const (
 	annotationPublicKey = "bouncer.cadoles.com/public-key"
 )
@@ -90,23 +130,6 @@ func (i *Integration) upsertTokenSecret(ctx context.Context, client *kubernetes.
 
 	logger.Debug(ctx, "generating new token")
 
-	key, err := jwk.LoadOrGenerate(i.Options.PrivateKey, jwk.DefaultKeySize)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	publicKey, err := key.PublicKey()
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	publicKeyThumbprint, err := publicKey.Thumbprint(crypto.SHA256)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	publicKeyHash := fmt.Sprintf("%x", publicKeyThumbprint)
-
 	alreadyExists := true
 	secret, err := client.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{})
 	if err != nil {
@@ -117,8 +140,23 @@ func (i *Integration) upsertTokenSecret(ctx context.Context, client *kubernetes.
 		}
 	}
 
+	privateKey, err := integration.CtxPrivateKey(ctx)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	keySet, err := integration.CtxPublicKeySet(ctx)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	publicKeyThumbprint, err := getKeySetThumbprint(keySet)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
 	if !alreadyExists {
-		token, err := jwt.GenerateToken(ctx, key, i.Options.Issuer, subject, role)
+		token, err := jwt.GenerateToken(ctx, privateKey, i.Options.Issuer, subject, role)
 		if err != nil {
 			return errors.WithStack(err)
 		}
@@ -128,7 +166,7 @@ func (i *Integration) upsertTokenSecret(ctx context.Context, client *kubernetes.
 			ObjectMeta: metav1.ObjectMeta{
 				Name: name,
 				Annotations: map[string]string{
-					annotationPublicKey: publicKeyHash,
+					annotationPublicKey: publicKeyThumbprint,
 				},
 			},
 			StringData: map[string]string{
@@ -143,8 +181,8 @@ func (i *Integration) upsertTokenSecret(ctx context.Context, client *kubernetes.
 		}
 	} else {
 		existingPublicKeyHash, exists := secret.Annotations[annotationPublicKey]
-		if !exists || publicKeyHash != existingPublicKeyHash {
-			token, err := jwt.GenerateToken(ctx, key, i.Options.Issuer, subject, role)
+		if !exists || publicKeyThumbprint != existingPublicKeyHash {
+			token, err := jwt.GenerateToken(ctx, privateKey, i.Options.Issuer, subject, role)
 			if err != nil {
 				return errors.WithStack(err)
 			}
@@ -157,7 +195,7 @@ func (i *Integration) upsertTokenSecret(ctx context.Context, client *kubernetes.
 				secret.Annotations = make(map[string]string)
 			}
 
-			secret.Annotations[annotationPublicKey] = publicKeyHash
+			secret.Annotations[annotationPublicKey] = publicKeyThumbprint
 
 			logger.Info(ctx, "updating token secret")
 
@@ -172,6 +210,66 @@ func (i *Integration) upsertTokenSecret(ctx context.Context, client *kubernetes.
 	return nil
 }
 
+func (i *Integration) getSharedPrivateKey(ctx context.Context, client *kubernetes.Clientset, namespace string, name string) (jwk.Key, error) {
+	if namespace == "" {
+		defaultNamespace, err := i.getCurrentNamespace()
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		namespace = defaultNamespace
+	}
+
+	ctx = logger.With(ctx,
+		logger.F("secretNamespace", namespace),
+		logger.F("secretName", name),
+	)
+
+	logger.Debug(ctx, "searching shared private key from secret")
+
+	secret, err := client.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{})
+	if err != nil && !k8serr.IsNotFound(err) {
+		return nil, errors.WithStack(err)
+	}
+
+	rawPrivateKey, exists := secret.Data["key"]
+
+	if exists && len(rawPrivateKey) != 0 {
+		key, err := jwk.ParseKey(rawPrivateKey)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		return key, nil
+	}
+
+	localKey, err := integration.CtxPrivateKey(ctx)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	rawLocalKey, err := json.Marshal(localKey)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	secret = &v1.Secret{
+		Type: v1.SecretTypeOpaque,
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Data: map[string][]byte{
+			"key": rawLocalKey,
+		},
+	}
+
+	if _, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return localKey, nil
+}
+
 func (i *Integration) getCurrentNamespace() (string, error) {
 	namespace, err := os.ReadFile(namespaceFile)
 	if err != nil {
@@ -191,4 +289,5 @@ func NewIntegration(funcs ...OptionFunc) *Integration {
 
 var (
 	_ integration.OnStartup = &Integration{}
+	_ integration.OnKeyLoad = &Integration{}
 )

internal/integration/kubernetes/keyset.go

@@ -0,0 +1,41 @@
+package kubernetes
+
+import (
+	"bytes"
+	"crypto"
+	"crypto/sha256"
+	"fmt"
+	"slices"
+
+	"forge.cadoles.com/cadoles/bouncer/internal/jwk"
+	"github.com/pkg/errors"
+)
+
+func getKeySetThumbprint(set jwk.Set) (string, error) {
+	data := make([][]byte, 0, set.Len())
+
+	for i := 0; i < set.Len(); i++ {
+		key, exists := set.Key(i)
+		if !exists {
+			continue
+		}
+
+		thumbprint, err := key.Thumbprint(crypto.SHA256)
+		if err != nil {
+			return "", errors.WithStack(err)
+		}
+
+		data = append(data, thumbprint)
+	}
+
+	slices.SortFunc(data, bytes.Compare)
+
+	hash := sha256.New()
+	for _, d := range data {
+		if _, err := hash.Write(d); err != nil {
+			return "", errors.WithStack(err)
+		}
+	}
+
+	return fmt.Sprintf("%x", hash.Sum(nil)), nil
+}

internal/integration/kubernetes/options.go

@@ -12,7 +12,8 @@ type Options struct {
 	WriterTokenSecretNamespace string
 	ReaderTokenSecret          string
 	ReaderTokenSecretNamespace string
-	PrivateKey                 string
+	PrivateKeySecret           string
+	PrivateKeySecretNamespace  string
 	Issuer                     string
 	Locker                     lock.Locker
 	LockTimeout                time.Duration
@@ -26,7 +27,8 @@ func NewOptions(funcs ...OptionFunc) *Options {
 		WriterTokenSecretNamespace: "",
 		ReaderTokenSecret:          "",
 		ReaderTokenSecretNamespace: "",
-		PrivateKey:                 "",
+		PrivateKeySecret:           "",
+		PrivateKeySecretNamespace:  "",
 		Issuer:                     "",
 		Locker:                     memory.NewLocker(),
 		LockTimeout:                30 * time.Second,
@@ -62,9 +64,15 @@ func WithReaderTokenSecretNamespace(namespace string) OptionFunc {
 	}
 }
 
-func WithPrivateKey(privateKeyFile string) OptionFunc {
+func WithPrivateKeySecret(secretName string) OptionFunc {
 	return func(opts *Options) {
-		opts.PrivateKey = privateKeyFile
+		opts.PrivateKeySecret = secretName
+	}
+}
+
+func WithPrivateKeySecretNamespace(namespace string) OptionFunc {
+	return func(opts *Options) {
+		opts.PrivateKeySecretNamespace = namespace
 	}
 }
 

internal/jwk/jwk.go

@@ -10,6 +10,7 @@ import (
 	"github.com/lestrrat-go/jwx/v2/jwa"
 	"github.com/lestrrat-go/jwx/v2/jwk"
 	"github.com/lestrrat-go/jwx/v2/jws"
+	"github.com/oklog/ulid/v2"
 
 	"github.com/pkg/errors"
 )
@@ -23,8 +24,9 @@ type (
 )
 
 var (
-	FromRaw = jwk.FromRaw
-	NewSet  = jwk.NewSet
+	FromRaw  = jwk.FromRaw
+	NewSet   = jwk.NewSet
+	ParseKey = jwk.ParseKey
 )
 
 const AlgorithmKey = jwk.AlgorithmKey
@@ -95,6 +97,12 @@ func Generate(size int) (jwk.Key, error) {
 		return nil, errors.WithStack(err)
 	}
 
+	keyID := ulid.Make().String()
+
+	if err := key.Set(jwk.KeyIDKey, keyID); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
 	return key, nil
 }
 

internal/setup/integrations.go

@@ -35,8 +35,9 @@ func setupKubernetesIntegration(ctx context.Context, conf *config.Config) (*kube
 		kubernetes.WithReaderTokenSecretNamespace(string(conf.Integrations.Kubernetes.ReaderTokenSecretNamespace)),
 		kubernetes.WithWriterTokenSecret(string(conf.Integrations.Kubernetes.WriterTokenSecret)),
 		kubernetes.WithWriterTokenSecretNamespace(string(conf.Integrations.Kubernetes.WriterTokenSecretNamespace)),
+		kubernetes.WithPrivateKeySecret(string(conf.Integrations.Kubernetes.PrivateKeySecret)),
+		kubernetes.WithPrivateKeySecretNamespace(string(conf.Integrations.Kubernetes.PrivateKeySecretNamespace)),
 		kubernetes.WithIssuer(string(conf.Admin.Auth.Issuer)),
-		kubernetes.WithPrivateKey(string(conf.Admin.Auth.PrivateKey)),
 		kubernetes.WithLocker(locker),
 		kubernetes.WithLockTimeout(time.Duration(conf.Integrations.Kubernetes.LockTimeout)),
 	)