feat: add basic prometheus metrics integration

internal/proxy/director/layer/queue/queue.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"html/template"
+	"math/rand"
 	"net/http"
 	"path/filepath"
 	"sync"
@@ -16,6 +17,7 @@ import (
 	"github.com/Masterminds/sprig/v3"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"gitlab.com/wpetit/goweb/logger"
 )
 
@@ -30,7 +32,9 @@ type Queue struct {
 	loadOnce    sync.Once
 	tmpl        *template.Template
 
-	refreshJobRunning uint32
+	refreshJobRunning       uint32
+	updateMetricsJobRunning uint32
+	postKeepAliveDebouncer  *DebouncerMap
 }
 
 // LayerType implements director.MiddlewareLayer
@@ -52,6 +56,8 @@ func (q *Queue) Middleware(layer *store.Layer) proxy.Middleware {
 				return
 			}
 
+			defer q.updateMetrics(ctx, layer.Proxy, layer.Name, options)
+
 			cookieName := q.getCookieName(layer.Name)
 
 			cookie, err := r.Cookie(cookieName)
@@ -72,8 +78,6 @@ func (q *Queue) Middleware(layer *store.Layer) proxy.Middleware {
 			sessionID := cookie.Value
 			queueName := string(layer.Name)
 
-			q.refreshQueue(queueName, options.KeepAlive)
-
 			rank, err := q.adapter.Touch(ctx, queueName, sessionID)
 			if err != nil {
 				logger.Error(ctx, "could not retrieve session rank", logger.E(errors.WithStack(err)))
@@ -102,6 +106,30 @@ func (q *Queue) Middleware(layer *store.Layer) proxy.Middleware {
 	}
 }
 
+func (q *Queue) updateSessionsMetric(ctx context.Context, proxyName store.ProxyName, layerName store.LayerName) {
+	if !atomic.CompareAndSwapUint32(&q.updateMetricsJobRunning, 0, 1) {
+		return
+	}
+
+	defer atomic.StoreUint32(&q.updateMetricsJobRunning, 0)
+
+	queueName := string(layerName)
+
+	status, err := q.adapter.Status(ctx, queueName)
+	if err != nil {
+		logger.Error(ctx, "could not retrieve queue status", logger.E(errors.WithStack(err)))
+
+		return
+	}
+
+	metricQueueSessions.With(
+		prometheus.Labels{
+			metricLabelLayer: string(layerName),
+			metricLabelProxy: string(proxyName),
+		},
+	).Set(float64(status.Sessions))
+}
+
 func (q *Queue) renderQueuePage(w http.ResponseWriter, r *http.Request, queueName string, options *LayerOptions, rank int64) {
 	ctx := r.Context()
 
@@ -135,20 +163,22 @@ func (q *Queue) renderQueuePage(w http.ResponseWriter, r *http.Request, queueNam
 		return
 	}
 
+	refreshRate := time.Duration(int64(options.KeepAlive.Seconds()/2)) * time.Second
+
 	templateData := struct {
 		QueueName       string
 		LayerOptions    *LayerOptions
 		Rank            int64
 		CurrentSessions int64
 		MaxSessions     int64
-		RefreshRate     int64
+		RefreshRate     time.Duration
 	}{
 		QueueName:       queueName,
 		LayerOptions:    options,
 		Rank:            rank + 1,
 		CurrentSessions: status.Sessions,
 		MaxSessions:     options.Capacity,
-		RefreshRate:     5,
+		RefreshRate:     refreshRate,
 	}
 
 	w.WriteHeader(http.StatusServiceUnavailable)
@@ -161,24 +191,55 @@ func (q *Queue) renderQueuePage(w http.ResponseWriter, r *http.Request, queueNam
 	}
 }
 
-func (q *Queue) refreshQueue(queueName string, keepAlive time.Duration) {
+func (q *Queue) refreshQueue(ctx context.Context, layerName store.LayerName, keepAlive time.Duration) {
 	if !atomic.CompareAndSwapUint32(&q.refreshJobRunning, 0, 1) {
 		return
 	}
 
-	go func() {
-		defer atomic.StoreUint32(&q.refreshJobRunning, 0)
+	defer atomic.StoreUint32(&q.refreshJobRunning, 0)
 
-		ctx, cancel := context.WithTimeout(context.Background(), keepAlive*2)
-		defer cancel()
+	if err := q.adapter.Refresh(ctx, string(layerName), keepAlive); err != nil {
+		logger.Error(ctx, "could not refresh queue",
+			logger.E(errors.WithStack(err)),
+			logger.F("queue", layerName),
+		)
+	}
+}
 
-		if err := q.adapter.Refresh(ctx, queueName, keepAlive); err != nil {
-			logger.Error(ctx, "could not refresh queue",
-				logger.E(errors.WithStack(err)),
-				logger.F("queue", queueName),
-			)
-		}
-	}()
+func (q *Queue) updateMetrics(ctx context.Context, proxyName store.ProxyName, layerName store.LayerName, options *LayerOptions) {
+	// Update queue capacity metric
+	metricQueueCapacity.With(
+		prometheus.Labels{
+			metricLabelLayer: string(layerName),
+			metricLabelProxy: string(proxyName),
+		},
+	).Set(float64(options.Capacity))
+
+	// Refresh queue data and metrics
+	q.refreshQueue(ctx, layerName, options.KeepAlive)
+	q.updateSessionsMetric(ctx, proxyName, layerName)
+
+	// (Re)schedule an update job after session ttl + semi-random time padding
+	// to update metrics after last session expiration
+	randDuration := rand.Int63n(int64(options.KeepAlive))
+	timePadding := options.KeepAlive/2 + time.Duration(randDuration)
+	after := options.KeepAlive + timePadding
+
+	debouncingKey := fmt.Sprintf("%s/%s", proxyName, layerName)
+
+	q.postKeepAliveDebouncer.Do(debouncingKey, after, func() {
+		ctx := logger.With(
+			context.Background(),
+			logger.F("proxy", proxyName),
+			logger.F("layer", layerName),
+			logger.F("after", after),
+		)
+
+		logger.Info(ctx, "running post keep alive refresh job")
+
+		q.refreshQueue(ctx, layerName, options.KeepAlive)
+		q.updateSessionsMetric(ctx, proxyName, layerName)
+	})
 }
 
 func (q *Queue) getCookieName(layerName store.LayerName) string {
@@ -192,9 +253,10 @@ func New(adapter Adapter, funcs ...OptionFunc) *Queue {
 	}
 
 	return &Queue{
-		adapter:          adapter,
-		templateDir:      opts.TemplateDir,
-		defaultKeepAlive: opts.DefaultKeepAlive,
+		adapter:                adapter,
+		templateDir:            opts.TemplateDir,
+		defaultKeepAlive:       opts.DefaultKeepAlive,
+		postKeepAliveDebouncer: NewDebouncerMap(),
 	}
 }