feat: use sentry tags instead of context for better observability

internal/command/server/admin/run.go

@@ -12,10 +12,6 @@ import (
 	"gitlab.com/wpetit/goweb/logger"
 )
 
-const (
-	flagPrintDefaultToken = "print-default-token"
-)
-
 func RunCommand() *cli.Command {
 	flags := common.Flags()
 
@@ -32,6 +28,8 @@ func RunCommand() *cli.Command {
 			logger.SetFormat(logger.Format(conf.Logger.Format))
 			logger.SetLevel(logger.Level(conf.Logger.Level))
 
+			logger.Debug(ctx.Context, "using config", logger.F("config", conf))
+
 			projectVersion := ctx.String("projectVersion")
 
 			if conf.Proxy.Sentry.DSN != "" {

internal/command/server/proxy/run.go

@@ -29,6 +29,8 @@ func RunCommand() *cli.Command {
 			logger.SetFormat(logger.Format(conf.Logger.Format))
 			logger.SetLevel(logger.Level(conf.Logger.Level))
 
+			logger.Debug(ctx.Context, "using config", logger.F("config", conf))
+
 			projectVersion := ctx.String("projectVersion")
 
 			if conf.Proxy.Sentry.DSN != "" {
@@ -49,7 +51,7 @@ func RunCommand() *cli.Command {
 				proxy.WithServerConfig(conf.Proxy),
 				proxy.WithRedisConfig(conf.Redis),
 				proxy.WithDirectorLayers(layers...),
-				proxy.WithDirectorCacheTTL(time.Duration(conf.Proxy.Cache.TTL)),
+				proxy.WithDirectorCacheTTL(time.Duration(*conf.Proxy.Cache.TTL)),
 			)
 
 			addrs, srvErrs := srv.Start(ctx.Context)

internal/config/environment.go

@@ -19,7 +19,7 @@ func (is *InterpolatedString) UnmarshalYAML(value *yaml.Node) error {
 		return errors.WithStack(err)
 	}
 
-	str, err := envsubst.EvalEnv(str)
+	str, err := envsubst.Eval(str, getEnv)
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -38,7 +38,7 @@ func (ii *InterpolatedInt) UnmarshalYAML(value *yaml.Node) error {
 		return errors.Wrapf(err, "could not decode value '%v' (line '%d') into string", value.Value, value.Line)
 	}
 
-	str, err := envsubst.EvalEnv(str)
+	str, err := envsubst.Eval(str, getEnv)
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -62,7 +62,7 @@ func (ifl *InterpolatedFloat) UnmarshalYAML(value *yaml.Node) error {
 		return errors.Wrapf(err, "could not decode value '%v' (line '%d') into string", value.Value, value.Line)
 	}
 
-	str, err := envsubst.EvalEnv(str)
+	str, err := envsubst.Eval(str, getEnv)
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -86,7 +86,7 @@ func (ib *InterpolatedBool) UnmarshalYAML(value *yaml.Node) error {
 		return errors.Wrapf(err, "could not decode value '%v' (line '%d') into string", value.Value, value.Line)
 	}
 
-	str, err := envsubst.EvalEnv(str)
+	str, err := envsubst.Eval(str, getEnv)
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -101,9 +101,10 @@ func (ib *InterpolatedBool) UnmarshalYAML(value *yaml.Node) error {
 	return nil
 }
 
+var getEnv = os.Getenv
+
 type InterpolatedMap struct {
-	Data   map[string]any
+	Data map[string]any
-	getEnv func(string) string
 }
 
 func (im *InterpolatedMap) UnmarshalYAML(value *yaml.Node) error {
@@ -113,10 +114,6 @@ func (im *InterpolatedMap) UnmarshalYAML(value *yaml.Node) error {
 		return errors.Wrapf(err, "could not decode value '%v' (line '%d') into map", value.Value, value.Line)
 	}
 
-	if im.getEnv == nil {
-		im.getEnv = os.Getenv
-	}
-
 	interpolated, err := im.interpolateRecursive(data)
 	if err != nil {
 		return errors.WithStack(err)
@@ -140,7 +137,7 @@ func (im InterpolatedMap) interpolateRecursive(data any) (any, error) {
 		}
 
 	case string:
-		value, err := envsubst.Eval(typ, im.getEnv)
+		value, err := envsubst.Eval(typ, getEnv)
 		if err != nil {
 			return nil, errors.WithStack(err)
 		}
@@ -171,7 +168,7 @@ func (iss *InterpolatedStringSlice) UnmarshalYAML(value *yaml.Node) error {
 	}
 
 	for index, value := range data {
-		value, err := envsubst.EvalEnv(value)
+		value, err := envsubst.Eval(value, getEnv)
 		if err != nil {
 			return errors.WithStack(err)
 		}
@@ -193,7 +190,7 @@ func (id *InterpolatedDuration) UnmarshalYAML(value *yaml.Node) error {
 		return errors.Wrapf(err, "could not decode value '%v' (line '%d') into string", value.Value, value.Line)
 	}
 
-	str, err := envsubst.EvalEnv(str)
+	str, err := envsubst.Eval(str, getEnv)
 	if err != nil {
 		return errors.WithStack(err)
 	}

internal/config/environment_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
@@ -65,7 +66,7 @@ func TestInterpolatedMap(t *testing.T) {
 			var interpolatedMap InterpolatedMap
 
 			if tc.Env != nil {
-				interpolatedMap.getEnv = func(key string) string {
+				getEnv = func(key string) string {
 					return tc.Env[key]
 				}
 			}
@@ -80,3 +81,54 @@ func TestInterpolatedMap(t *testing.T) {
 		})
 	}
 }
+
+func TestInterpolatedDuration(t *testing.T) {
+	type testCase struct {
+		Path   string
+		Env    map[string]string
+		Assert func(t *testing.T, parsed *InterpolatedDuration)
+	}
+
+	testCases := []testCase{
+		{
+			Path: "testdata/environment/interpolated-duration.yml",
+			Env: map[string]string{
+				"MY_DURATION": "30s",
+			},
+			Assert: func(t *testing.T, parsed *InterpolatedDuration) {
+				if e, g := 30*time.Second, parsed; e != time.Duration(*g) {
+					t.Errorf("parsed: expected '%v', got '%v'", e, g)
+				}
+			},
+		},
+	}
+
+	for idx, tc := range testCases {
+		t.Run(fmt.Sprintf("Case #%d", idx), func(t *testing.T) {
+			data, err := os.ReadFile(tc.Path)
+			if err != nil {
+				t.Fatalf("%+v", errors.WithStack(err))
+			}
+
+			if tc.Env != nil {
+				getEnv = func(key string) string {
+					return tc.Env[key]
+				}
+			}
+
+			config := struct {
+				Duration *InterpolatedDuration `yaml:"duration"`
+			}{
+				Duration: NewInterpolatedDuration(-1),
+			}
+
+			if err := yaml.Unmarshal(data, &config); err != nil {
+				t.Fatalf("%+v", errors.WithStack(err))
+			}
+
+			if tc.Assert != nil {
+				tc.Assert(t, config.Duration)
+			}
+		})
+	}
+}

internal/config/proxy_server.go

@@ -113,12 +113,12 @@ func NewDefaultDialConfig() DialConfig {
 }
 
 type CacheConfig struct {
-	TTL InterpolatedDuration `yaml:"ttl"`
+	TTL *InterpolatedDuration `yaml:"ttl"`
 }
 
 func NewDefaultCacheConfig() CacheConfig {
 	return CacheConfig{
-		TTL: *NewInterpolatedDuration(time.Second * 30),
+		TTL: NewInterpolatedDuration(time.Second * 30),
 	}
 }
 

internal/config/testdata/environment/interpolated-duration.yml

@@ -0,0 +1 @@
+duration: ${MY_DURATION}

internal/proxy/director/director.go

@@ -7,8 +7,8 @@ import (
 
 	"forge.cadoles.com/Cadoles/go-proxy"
 	"forge.cadoles.com/Cadoles/go-proxy/wildcard"
-	"forge.cadoles.com/cadoles/bouncer/internal/cache"
 	"forge.cadoles.com/cadoles/bouncer/internal/store"
+	"forge.cadoles.com/cadoles/bouncer/internal/syncx"
 	"github.com/getsentry/sentry-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -20,16 +20,18 @@ type Director struct {
 	layerRepository store.LayerRepository
 	layerRegistry   *LayerRegistry
 
-	proxyCache cache.Cache[string, []*store.Proxy]
+	cachedProxies *syncx.CachedResource[string, []*store.Proxy]
-	layerCache cache.Cache[string, []*store.Layer]
+	cachedLayers  *syncx.CachedResource[string, []*store.Layer]
 
 	handleError HandleErrorFunc
 }
 
+const proxiesCacheKey = "proxies"
+
 func (d *Director) rewriteRequest(r *http.Request) (*http.Request, error) {
 	ctx := r.Context()
 
-	proxies, err := d.getProxies(ctx)
+	proxies, _, err := d.cachedProxies.Get(ctx, proxiesCacheKey)
 	if err != nil {
 		return r, errors.WithStack(err)
 	}
@@ -54,7 +56,7 @@ func (d *Director) rewriteRequest(r *http.Request) (*http.Request, error) {
 
 			metricProxyRequestsTotal.With(prometheus.Labels{metricLabelProxy: string(p.Name)}).Add(1)
 
-			proxyLayers, err := d.getLayers(proxyCtx, p.Name)
+			proxyLayers, _, err := d.cachedLayers.Get(proxyCtx, string(p.Name))
 			if err != nil {
 				return r, errors.WithStack(err)
 			}
@@ -78,9 +80,10 @@ func (d *Director) rewriteRequest(r *http.Request) (*http.Request, error) {
 			r = r.WithContext(proxyCtx)
 
 			if sentryScope, _ := SentryScope(ctx); sentryScope != nil {
-				sentryScope.SetContext("bouncer", sentry.Context{
+				sentryScope.SetTags(map[string]string{
-					"proxy_name":   p.Name,
+					"bouncer.proxy.name":        string(p.Name),
-					"proxy_target": r.URL.String(),
+					"bouncer.proxy.target.url":  r.URL.String(),
+					"bouncer.proxy.target.host": r.URL.Host,
 				})
 			}
 
@@ -94,15 +97,7 @@ func (d *Director) rewriteRequest(r *http.Request) (*http.Request, error) {
 	return r, nil
 }
 
-const proxiesCacheKey = "proxies"
+func (d *Director) getProxies(ctx context.Context, key string) ([]*store.Proxy, error) {
-
-func (d *Director) getProxies(ctx context.Context) ([]*store.Proxy, error) {
-	proxies, exists := d.proxyCache.Get(proxiesCacheKey)
-	if exists {
-		logger.Debug(ctx, "using cached proxies")
-		return proxies, nil
-	}
-
 	logger.Debug(ctx, "querying fresh proxies")
 
 	headers, err := d.proxyRepository.QueryProxy(ctx, store.WithProxyQueryEnabled(true))
@@ -112,7 +107,7 @@ func (d *Director) getProxies(ctx context.Context) ([]*store.Proxy, error) {
 
 	sort.Sort(store.ByProxyWeight(headers))
 
-	proxies = make([]*store.Proxy, 0, len(headers))
+	proxies := make([]*store.Proxy, 0, len(headers))
 
 	for _, h := range headers {
 		if !h.Enabled {
@@ -127,19 +122,11 @@ func (d *Director) getProxies(ctx context.Context) ([]*store.Proxy, error) {
 		proxies = append(proxies, proxy)
 	}
 
-	d.proxyCache.Set(proxiesCacheKey, proxies)
-
 	return proxies, nil
 }
 
-func (d *Director) getLayers(ctx context.Context, proxyName store.ProxyName) ([]*store.Layer, error) {
+func (d *Director) getLayers(ctx context.Context, rawProxyName string) ([]*store.Layer, error) {
-	cacheKey := "layers-" + string(proxyName)
+	proxyName := store.ProxyName(rawProxyName)
-
-	layers, exists := d.layerCache.Get(cacheKey)
-	if exists {
-		logger.Debug(ctx, "using cached layers")
-		return layers, nil
-	}
 
 	logger.Debug(ctx, "querying fresh layers")
 
@@ -150,7 +137,7 @@ func (d *Director) getLayers(ctx context.Context, proxyName store.ProxyName) ([]
 
 	sort.Sort(store.ByLayerWeight(headers))
 
-	layers = make([]*store.Layer, 0, len(headers))
+	layers := make([]*store.Layer, 0, len(headers))
 
 	for _, h := range headers {
 		if !h.Enabled {
@@ -165,8 +152,6 @@ func (d *Director) getLayers(ctx context.Context, proxyName store.ProxyName) ([]
 		layers = append(layers, layer)
 	}
 
-	d.layerCache.Set(cacheKey, layers)
-
 	return layers, nil
 }
 
@@ -278,12 +263,15 @@ func New(proxyRepository store.ProxyRepository, layerRepository store.LayerRepos
 
 	registry := NewLayerRegistry(opts.Layers...)
 
-	return &Director{
+	director := &Director{
 		proxyRepository: proxyRepository,
 		layerRepository: layerRepository,
 		layerRegistry:   registry,
-		proxyCache:      opts.ProxyCache,
-		layerCache:      opts.LayerCache,
 		handleError:     opts.HandleError,
 	}
+
+	director.cachedProxies = syncx.NewCachedResource(opts.ProxyCache, director.getProxies)
+	director.cachedLayers = syncx.NewCachedResource(opts.LayerCache, director.getLayers)
+
+	return director
 }

internal/proxy/director/layer/authn/oidc/authenticator.go

@@ -13,10 +13,12 @@ import (
 	"time"
 
 	"forge.cadoles.com/Cadoles/go-proxy/wildcard"
-	"forge.cadoles.com/cadoles/bouncer/internal/cache"
+	"forge.cadoles.com/cadoles/bouncer/internal/cache/memory"
+	"forge.cadoles.com/cadoles/bouncer/internal/cache/ttl"
 	"forge.cadoles.com/cadoles/bouncer/internal/proxy/director"
 	"forge.cadoles.com/cadoles/bouncer/internal/proxy/director/layer/authn"
 	"forge.cadoles.com/cadoles/bouncer/internal/store"
+	"forge.cadoles.com/cadoles/bouncer/internal/syncx"
 	"github.com/coreos/go-oidc/v3/oidc"
 	"github.com/gorilla/sessions"
 	"github.com/pkg/errors"
@@ -25,10 +27,10 @@ import (
 )
 
 type Authenticator struct {
-	store             sessions.Store
+	store              sessions.Store
-	httpTransport     *http.Transport
+	httpTransport      *http.Transport
-	httpClientTimeout time.Duration
+	httpClientTimeout  time.Duration
-	oidcProviderCache cache.Cache[string, *oidc.Provider]
+	cachedOIDCProvider *syncx.CachedResource[string, *oidc.Provider]
 }
 
 func (a *Authenticator) PreAuthentication(w http.ResponseWriter, r *http.Request, layer *store.Layer) error {
@@ -54,7 +56,7 @@ func (a *Authenticator) PreAuthentication(w http.ResponseWriter, r *http.Request
 		return errors.WithStack(err)
 	}
 
-	client, err := a.getClient(options, loginCallbackURL.String())
+	client, err := a.getClient(ctx, options, loginCallbackURL.String())
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -160,7 +162,7 @@ func (a *Authenticator) Authenticate(w http.ResponseWriter, r *http.Request, lay
 		return nil, errors.WithStack(err)
 	}
 
-	client, err := a.getClient(options, loginCallbackURL.String())
+	client, err := a.getClient(ctx, options, loginCallbackURL.String())
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
@@ -362,9 +364,7 @@ func (a *Authenticator) templatize(rawTemplate string, proxyName store.ProxyName
 	return raw.String(), nil
 }
 
-func (a *Authenticator) getClient(options *LayerOptions, redirectURL string) (*Client, error) {
+func (a *Authenticator) getClient(ctx context.Context, options *LayerOptions, redirectURL string) (*Client, error) {
-	ctx := context.Background()
-
 	transport := a.httpTransport.Clone()
 
 	if options.OIDC.TLSInsecureSkipVerify {
@@ -375,28 +375,24 @@ func (a *Authenticator) getClient(options *LayerOptions, redirectURL string) (*C
 		transport.TLSClientConfig.InsecureSkipVerify = true
 	}
 
+	if options.OIDC.SkipIssuerVerification {
+		ctx = oidc.InsecureIssuerURLContext(ctx, options.OIDC.IssuerURL)
+	}
+
 	httpClient := &http.Client{
 		Timeout:   a.httpClientTimeout,
 		Transport: transport,
 	}
 
-	provider, exists := a.oidcProviderCache.Get(options.OIDC.IssuerURL)
+	ctx = oidc.ClientContext(ctx, httpClient)
-	if !exists {
-		var err error
-		ctx = oidc.ClientContext(ctx, httpClient)
 
-		if options.OIDC.SkipIssuerVerification {
+	if options.OIDC.SkipIssuerVerification {
-			ctx = oidc.InsecureIssuerURLContext(ctx, options.OIDC.IssuerURL)
+		ctx = oidc.InsecureIssuerURLContext(ctx, options.OIDC.IssuerURL)
-		}
+	}
 
-		logger.Debug(ctx, "refreshing oidc provider", logger.F("issuerURL", options.OIDC.IssuerURL))
+	provider, _, err := a.cachedOIDCProvider.Get(ctx, options.OIDC.IssuerURL)
-
+	if err != nil {
-		provider, err = oidc.NewProvider(ctx, options.OIDC.IssuerURL)
+		return nil, errors.Wrap(err, "could not retrieve oidc provider")
-		if err != nil {
-			return nil, errors.Wrap(err, "could not create oidc provider")
-		}
-
-		a.oidcProviderCache.Set(options.OIDC.IssuerURL, provider)
 	}
 
 	client := NewClient(
@@ -411,6 +407,17 @@ func (a *Authenticator) getClient(options *LayerOptions, redirectURL string) (*C
 	return client, nil
 }
 
+func (a *Authenticator) getOIDCProvider(ctx context.Context, issuerURL string) (*oidc.Provider, error) {
+	logger.Debug(ctx, "refreshing oidc provider", logger.F("issuerURL", issuerURL))
+
+	provider, err := oidc.NewProvider(ctx, issuerURL)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create oidc provider")
+	}
+
+	return provider, nil
+}
+
 const defaultCookieNamePrefix = "_bouncer_authn_oidc"
 
 func (a *Authenticator) getCookieName(cookieName string, proxyName store.ProxyName, layerName store.LayerName) string {
@@ -421,6 +428,25 @@ func (a *Authenticator) getCookieName(cookieName string, proxyName store.ProxyNa
 	return strings.ToLower(fmt.Sprintf("%s_%s_%s", defaultCookieNamePrefix, proxyName, layerName))
 }
 
+func NewAuthenticator(httpTransport *http.Transport, clientTimeout time.Duration, store sessions.Store, oidcProviderCacheTimeout time.Duration) *Authenticator {
+	authenticator := &Authenticator{
+		httpTransport:     httpTransport,
+		httpClientTimeout: clientTimeout,
+		store:             store,
+	}
+
+	authenticator.cachedOIDCProvider = syncx.NewCachedResource(
+		ttl.NewCache(
+			memory.NewCache[string, *oidc.Provider](),
+			memory.NewCache[string, time.Time](),
+			oidcProviderCacheTimeout,
+		),
+		authenticator.getOIDCProvider,
+	)
+
+	return authenticator
+}
+
 var (
 	_ authn.PreAuthentication = &Authenticator{}
 	_ authn.Authenticator     = &Authenticator{}

internal/proxy/director/layer/authn/oidc/layer.go

@@ -1,13 +1,8 @@
 package oidc
 
 import (
-	"time"
-
-	"forge.cadoles.com/cadoles/bouncer/internal/cache/memory"
-	"forge.cadoles.com/cadoles/bouncer/internal/cache/ttl"
 	"forge.cadoles.com/cadoles/bouncer/internal/proxy/director/layer/authn"
 	"forge.cadoles.com/cadoles/bouncer/internal/store"
-	"github.com/coreos/go-oidc/v3/oidc"
 	"github.com/gorilla/sessions"
 )
 
@@ -15,14 +10,11 @@ const LayerType store.LayerType = "authn-oidc"
 
 func NewLayer(store sessions.Store, funcs ...OptionFunc) *authn.Layer {
 	opts := NewOptions(funcs...)
-	return authn.NewLayer(LayerType, &Authenticator{
+	authenticator := NewAuthenticator(
-		httpTransport:     opts.HTTPTransport,
+		opts.HTTPTransport,
-		httpClientTimeout: opts.HTTPClientTimeout,
+		opts.HTTPClientTimeout,
-		store:             store,
+		store,
-		oidcProviderCache: ttl.NewCache(
+		opts.OIDCProviderCacheTimeout,
-			memory.NewCache[string, *oidc.Provider](),
+	)
-			memory.NewCache[string, time.Time](),
+	return authn.NewLayer(LayerType, authenticator, opts.AuthnOptions...)
-			opts.OIDCProviderCacheTimeout,
-		),
-	}, opts.AuthnOptions...)
 }

internal/syncx/cached_resource.go

@@ -0,0 +1,63 @@
+package syncx
+
+import (
+	"context"
+	"sync"
+
+	"forge.cadoles.com/cadoles/bouncer/internal/cache"
+	"github.com/pkg/errors"
+)
+
+type RefreshFunc[K comparable, V any] func(ctx context.Context, key K) (V, error)
+
+type CachedResource[K comparable, V any] struct {
+	cache   cache.Cache[K, V]
+	lock    sync.RWMutex
+	refresh RefreshFunc[K, V]
+}
+
+func (r *CachedResource[K, V]) Clear() {
+	r.cache.Clear()
+}
+
+func (r *CachedResource[K, V]) Get(ctx context.Context, key K) (V, bool, error) {
+	value, exists := r.cache.Get(key)
+	if exists {
+		return value, false, nil
+	}
+
+	locked := r.lock.TryLock()
+	if !locked {
+		r.lock.RLock()
+
+		value, exists := r.cache.Get(key)
+		if exists {
+			r.lock.RUnlock()
+			return value, false, nil
+		}
+
+		r.lock.RUnlock()
+	}
+
+	if !locked {
+		r.lock.Lock()
+	}
+
+	defer r.lock.Unlock()
+
+	value, err := r.refresh(ctx, key)
+	if err != nil {
+		return *new(V), false, errors.WithStack(err)
+	}
+
+	r.cache.Set(key, value)
+
+	return value, true, nil
+}
+
+func NewCachedResource[K comparable, V any](cache cache.Cache[K, V], refresh RefreshFunc[K, V]) *CachedResource[K, V] {
+	return &CachedResource[K, V]{
+		cache:   cache,
+		refresh: refresh,
+	}
+}

internal/syncx/cached_resource_test.go

@@ -0,0 +1,66 @@
+package syncx
+
+import (
+	"context"
+	"math"
+	"sync"
+	"testing"
+	"time"
+
+	"forge.cadoles.com/cadoles/bouncer/internal/cache/memory"
+	"forge.cadoles.com/cadoles/bouncer/internal/cache/ttl"
+	"github.com/pkg/errors"
+)
+
+func TestCachedResource(t *testing.T) {
+	refreshCalls := 0
+	cacheTTL := 1*time.Second + 500*time.Millisecond
+	duration := 2 * time.Second
+
+	expectedCalls := math.Ceil(float64(duration) / float64(cacheTTL))
+
+	resource := NewCachedResource(
+		ttl.NewCache(
+			memory.NewCache[string, string](),
+			memory.NewCache[string, time.Time](),
+			cacheTTL,
+		),
+		func(ctx context.Context, key string) (string, error) {
+			refreshCalls++
+			return "bar", nil
+		},
+	)
+
+	concurrents := 50
+	key := "foo"
+
+	var wg sync.WaitGroup
+
+	wg.Add(concurrents)
+
+	for i := range concurrents {
+		go func(i int) {
+			done := time.After(duration)
+
+			defer wg.Done()
+			for {
+				select {
+				case <-done:
+					return
+				default:
+					value, fresh, err := resource.Get(context.Background(), key)
+					if err != nil {
+						t.Errorf("%+v", errors.WithStack(err))
+					}
+					t.Logf("resource retrieved for goroutine #%d: (%s, %s, %v)", i, key, value, fresh)
+				}
+			}
+		}(i)
+	}
+
+	wg.Wait()
+
+	if e, g := int(expectedCalls), refreshCalls; e != g {
+		t.Errorf("refreshCalls: expected '%d', got '%d'", e, g)
+	}
+}