From 56609ec316439d628e8d87937c31f676fa843ee5 Mon Sep 17 00:00:00 2001 From: William Petit Date: Fri, 30 Jun 2023 10:26:27 -0600 Subject: [PATCH] feat: add basic prometheus metrics integration --- go.mod | 12 ++- go.sum | 21 ++++ internal/admin/server.go | 20 ++++ internal/command/common/load_config.go | 5 - internal/command/config/dump.go | 2 + internal/config/admin_server.go | 14 +-- internal/config/logger.go | 2 +- internal/config/metrics.go | 35 ++++++ internal/config/proxy_server.go | 6 +- internal/proxy/director/director.go | 3 + .../proxy/director/layer/queue/debouncer.go | 73 +++++++++++++ .../proxy/director/layer/queue/metrics.go | 31 ++++++ internal/proxy/director/layer/queue/queue.go | 102 ++++++++++++++---- internal/proxy/director/metrics.go | 20 ++++ internal/proxy/server.go | 43 ++++++-- layers/queue/templates/queue.gohtml | 2 +- 16 files changed, 344 insertions(+), 47 deletions(-) create mode 100644 internal/config/metrics.go create mode 100644 internal/proxy/director/layer/queue/debouncer.go create mode 100644 internal/proxy/director/layer/queue/metrics.go create mode 100644 internal/proxy/director/metrics.go diff --git a/go.mod b/go.mod index 0464115..d579702 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/continuity v0.3.0 // indirect @@ -30,18 +31,24 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/kr/pretty v0.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opencontainers/runc v1.1.5 // indirect + github.com/prometheus/client_golang v1.16.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/qri-io/jsonpointer v0.1.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect @@ -52,6 +59,7 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect google.golang.org/genproto v0.0.0-20220314164441-57ef72a4c106 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) @@ -88,7 +96,7 @@ require ( go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.8.0 // indirect golang.org/x/mod v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect + golang.org/x/sys v0.8.0 // indirect golang.org/x/term v0.7.0 // indirect golang.org/x/tools v0.7.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go.sum b/go.sum index 3184cf7..08de0cb 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUS github.com/alecthomas/kong-hcl v0.1.8-0.20190615233001-b21fea9723c8/go.mod h1:MRgZdU3vrFd05IQ89AxUZ0aYdF39BYoNFa324SodPCA= github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= @@ -230,6 +232,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -309,6 +313,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -347,6 +352,8 @@ github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp9 github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -379,6 +386,7 @@ github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuh github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4aNE4= github.com/ory/dockertest/v3 v3.10.0/go.mod h1:nr57ZbRWMqfsdGdFNLHz5jjNdDb7VVFnzAeW1n5N1Lg= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -386,7 +394,15 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/qri-io/jsonpointer v0.1.1 h1:prVZBZLL6TW5vsSB9fFHFAMBLI4b0ri5vribQlTJiBA= github.com/qri-io/jsonpointer v0.1.1/go.mod h1:DnJPaYgiKu56EuDp8TU5wFLdZIcAnb/uH9v37ZaMV64= github.com/qri-io/jsonschema v0.2.1 h1:NNFoKms+kut6ABPf6xiKNM5214jzxAhDBrPHCJ97Wg0= @@ -398,6 +414,7 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -665,6 +682,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -891,6 +910,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/internal/admin/server.go b/internal/admin/server.go index e312c94..683145c 100644 --- a/internal/admin/server.go +++ b/internal/admin/server.go @@ -16,6 +16,7 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/cors" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus/promhttp" "gitlab.com/wpetit/goweb/logger" ) @@ -101,6 +102,25 @@ func (s *Server) run(parentCtx context.Context, addrs chan net.Addr, errs chan e router.Use(corsMiddleware.Handler) + if s.serverConfig.Metrics.Enabled { + metrics := s.serverConfig.Metrics + + logger.Info(ctx, "enabling metrics", logger.F("endpoint", metrics.Endpoint)) + + router.Group(func(r chi.Router) { + if metrics.BasicAuth != nil { + logger.Info(ctx, "enabling authentication on metrics endpoint") + + r.Use(middleware.BasicAuth( + "metrics", + metrics.BasicAuth.CredentialsMap(), + )) + } + + r.Handle(string(metrics.Endpoint), promhttp.Handler()) + }) + } + router.Route("/api/v1", func(r chi.Router) { r.Group(func(r chi.Router) { r.Use(auth.Middleware( diff --git a/internal/command/common/load_config.go b/internal/command/common/load_config.go index 0536f32..fc16461 100644 --- a/internal/command/common/load_config.go +++ b/internal/command/common/load_config.go @@ -4,7 +4,6 @@ import ( "forge.cadoles.com/cadoles/bouncer/internal/config" "github.com/pkg/errors" "github.com/urfave/cli/v2" - "gitlab.com/wpetit/goweb/logger" ) func LoadConfig(ctx *cli.Context) (*config.Config, error) { @@ -16,15 +15,11 @@ func LoadConfig(ctx *cli.Context) (*config.Config, error) { ) if configFile != "" { - logger.Info(ctx.Context, "loading config", logger.F("config", configFile)) - conf, err = config.NewFromFile(configFile) if err != nil { return nil, errors.Wrapf(err, "Could not load config file '%s'", configFile) } } else { - logger.Info(ctx.Context, "using default config") - conf = config.NewDefault() } diff --git a/internal/command/config/dump.go b/internal/command/config/dump.go index b14f63c..6fef44e 100644 --- a/internal/command/config/dump.go +++ b/internal/command/config/dump.go @@ -18,6 +18,8 @@ func Dump() *cli.Command { Usage: "Dump the current configuration", Flags: flags, Action: func(ctx *cli.Context) error { + logger.SetLevel(logger.LevelError) + conf, err := common.LoadConfig(ctx) if err != nil { return errors.Wrap(err, "Could not load configuration") diff --git a/internal/config/admin_server.go b/internal/config/admin_server.go index faf1ffa..35b90b4 100644 --- a/internal/config/admin_server.go +++ b/internal/config/admin_server.go @@ -1,16 +1,18 @@ package config type AdminServerConfig struct { - HTTP HTTPConfig `yaml:"http"` - CORS CORSConfig `yaml:"cors"` - Auth AuthConfig `yaml:"auth"` + HTTP HTTPConfig `yaml:"http"` + CORS CORSConfig `yaml:"cors"` + Auth AuthConfig `yaml:"auth"` + Metrics MetricsConfig `yaml:"metrics"` } func NewDefaultAdminServerConfig() AdminServerConfig { return AdminServerConfig{ - HTTP: NewHTTPConfig("127.0.0.1", 8081), - CORS: NewDefaultCORSConfig(), - Auth: NewDefaultAuthConfig(), + HTTP: NewHTTPConfig("127.0.0.1", 8081), + CORS: NewDefaultCORSConfig(), + Auth: NewDefaultAuthConfig(), + Metrics: NewDefaultMetricsConfig(), } } diff --git a/internal/config/logger.go b/internal/config/logger.go index a524695..254232b 100644 --- a/internal/config/logger.go +++ b/internal/config/logger.go @@ -9,7 +9,7 @@ type LoggerConfig struct { func NewDefaultLoggerConfig() LoggerConfig { return LoggerConfig{ - Level: InterpolatedInt(logger.LevelInfo), + Level: InterpolatedInt(logger.LevelError), Format: InterpolatedString(logger.FormatHuman), } } diff --git a/internal/config/metrics.go b/internal/config/metrics.go new file mode 100644 index 0000000..18f86ed --- /dev/null +++ b/internal/config/metrics.go @@ -0,0 +1,35 @@ +package config + +import "fmt" + +type MetricsConfig struct { + Enabled InterpolatedBool `yaml:"enabled"` + Endpoint InterpolatedString `yaml:"endpoint"` + BasicAuth *BasicAuthConfig `yaml:"basicAuth"` +} + +type BasicAuthConfig struct { + Credentials *InterpolatedMap `yaml:"credentials"` +} + +func (c *BasicAuthConfig) CredentialsMap() map[string]string { + if c.Credentials == nil { + return map[string]string{} + } + + credentials := make(map[string]string, len(*c.Credentials)) + + for k, v := range *c.Credentials { + credentials[k] = fmt.Sprintf("%v", v) + } + + return credentials +} + +func NewDefaultMetricsConfig() MetricsConfig { + return MetricsConfig{ + Enabled: true, + Endpoint: "/.bouncer/metrics", + BasicAuth: nil, + } +} diff --git a/internal/config/proxy_server.go b/internal/config/proxy_server.go index 7bdc44c..f0b9e32 100644 --- a/internal/config/proxy_server.go +++ b/internal/config/proxy_server.go @@ -1,11 +1,13 @@ package config type ProxyServerConfig struct { - HTTP HTTPConfig `yaml:"http"` + HTTP HTTPConfig `yaml:"http"` + Metrics MetricsConfig `yaml:"metrics"` } func NewDefaultProxyServerConfig() ProxyServerConfig { return ProxyServerConfig{ - HTTP: NewHTTPConfig("0.0.0.0", 8080), + HTTP: NewHTTPConfig("0.0.0.0", 8080), + Metrics: NewDefaultMetricsConfig(), } } diff --git a/internal/proxy/director/director.go b/internal/proxy/director/director.go index 6d5749d..654a182 100644 --- a/internal/proxy/director/director.go +++ b/internal/proxy/director/director.go @@ -10,6 +10,7 @@ import ( "forge.cadoles.com/Cadoles/go-proxy/wildcard" "forge.cadoles.com/cadoles/bouncer/internal/store" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "gitlab.com/wpetit/goweb/logger" ) @@ -59,6 +60,8 @@ MAIN: logger.F("remoteAddr", r.RemoteAddr), ) + metricProxyRequestsTotal.With(prometheus.Labels{metricLabelProxy: string(match.Name)}).Add(1) + ctx = withProxy(ctx, match) layers, err := d.getLayers(ctx, match.Name) diff --git a/internal/proxy/director/layer/queue/debouncer.go b/internal/proxy/director/layer/queue/debouncer.go new file mode 100644 index 0000000..ff298f2 --- /dev/null +++ b/internal/proxy/director/layer/queue/debouncer.go @@ -0,0 +1,73 @@ +package queue + +import ( + "sync" + "time" + + "github.com/pkg/errors" +) + +type DebouncerMap struct { + debouncers sync.Map +} + +func NewDebouncerMap() *DebouncerMap { + return &DebouncerMap{ + debouncers: sync.Map{}, + } +} + +func (m *DebouncerMap) Do(key string, after time.Duration, fn func()) { + newDebouncer := NewDebouncer(after) + rawDebouncer, loaded := m.debouncers.LoadOrStore(key, newDebouncer) + + debouncer, ok := rawDebouncer.(*Debouncer) + if !ok { + panic(errors.Errorf("unexpected debouncer value, expected '%T', got '%T'", newDebouncer, rawDebouncer)) + } + + if loaded { + debouncer.Update(after) + } + + debouncer.Do(fn) +} + +func NewDebouncer(after time.Duration) *Debouncer { + return &Debouncer{after: after} +} + +type Debouncer struct { + mu sync.Mutex + after time.Duration + timer *time.Timer + fn func() +} + +func (d *Debouncer) Do(fn func()) { + d.mu.Lock() + defer d.mu.Unlock() + + if d.timer != nil { + d.timer.Stop() + } + + d.fn = fn + d.timer = time.AfterFunc(d.after, d.fn) +} + +func (d *Debouncer) Update(after time.Duration) { + d.mu.Lock() + defer d.mu.Unlock() + + if after == d.after { + return + } + + if d.timer != nil { + d.timer.Stop() + } + + d.after = after + d.timer = time.AfterFunc(d.after, d.fn) +} diff --git a/internal/proxy/director/layer/queue/metrics.go b/internal/proxy/director/layer/queue/metrics.go new file mode 100644 index 0000000..1f5f462 --- /dev/null +++ b/internal/proxy/director/layer/queue/metrics.go @@ -0,0 +1,31 @@ +package queue + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + metricNamespace = "bouncer_layer_queue" + metricLabelProxy = "proxy" + metricLabelLayer = "layer" +) + +var ( + metricQueueSessions = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "sessions", + Help: "Bouncer's queue layer current sessions", + Namespace: metricNamespace, + }, + []string{metricLabelProxy, metricLabelLayer}, + ) + metricQueueCapacity = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "capacity", + Help: "Bouncer's queue layer capacity", + Namespace: metricNamespace, + }, + []string{metricLabelProxy, metricLabelLayer}, + ) +) diff --git a/internal/proxy/director/layer/queue/queue.go b/internal/proxy/director/layer/queue/queue.go index 652d5a1..c6e1d5a 100644 --- a/internal/proxy/director/layer/queue/queue.go +++ b/internal/proxy/director/layer/queue/queue.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "html/template" + "math/rand" "net/http" "path/filepath" "sync" @@ -16,6 +17,7 @@ import ( "github.com/Masterminds/sprig/v3" "github.com/google/uuid" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "gitlab.com/wpetit/goweb/logger" ) @@ -30,7 +32,9 @@ type Queue struct { loadOnce sync.Once tmpl *template.Template - refreshJobRunning uint32 + refreshJobRunning uint32 + updateMetricsJobRunning uint32 + postKeepAliveDebouncer *DebouncerMap } // LayerType implements director.MiddlewareLayer @@ -52,6 +56,8 @@ func (q *Queue) Middleware(layer *store.Layer) proxy.Middleware { return } + defer q.updateMetrics(ctx, layer.Proxy, layer.Name, options) + cookieName := q.getCookieName(layer.Name) cookie, err := r.Cookie(cookieName) @@ -72,8 +78,6 @@ func (q *Queue) Middleware(layer *store.Layer) proxy.Middleware { sessionID := cookie.Value queueName := string(layer.Name) - q.refreshQueue(queueName, options.KeepAlive) - rank, err := q.adapter.Touch(ctx, queueName, sessionID) if err != nil { logger.Error(ctx, "could not retrieve session rank", logger.E(errors.WithStack(err))) @@ -102,6 +106,30 @@ func (q *Queue) Middleware(layer *store.Layer) proxy.Middleware { } } +func (q *Queue) updateSessionsMetric(ctx context.Context, proxyName store.ProxyName, layerName store.LayerName) { + if !atomic.CompareAndSwapUint32(&q.updateMetricsJobRunning, 0, 1) { + return + } + + defer atomic.StoreUint32(&q.updateMetricsJobRunning, 0) + + queueName := string(layerName) + + status, err := q.adapter.Status(ctx, queueName) + if err != nil { + logger.Error(ctx, "could not retrieve queue status", logger.E(errors.WithStack(err))) + + return + } + + metricQueueSessions.With( + prometheus.Labels{ + metricLabelLayer: string(layerName), + metricLabelProxy: string(proxyName), + }, + ).Set(float64(status.Sessions)) +} + func (q *Queue) renderQueuePage(w http.ResponseWriter, r *http.Request, queueName string, options *LayerOptions, rank int64) { ctx := r.Context() @@ -135,20 +163,22 @@ func (q *Queue) renderQueuePage(w http.ResponseWriter, r *http.Request, queueNam return } + refreshRate := time.Duration(int64(options.KeepAlive.Seconds()/2)) * time.Second + templateData := struct { QueueName string LayerOptions *LayerOptions Rank int64 CurrentSessions int64 MaxSessions int64 - RefreshRate int64 + RefreshRate time.Duration }{ QueueName: queueName, LayerOptions: options, Rank: rank + 1, CurrentSessions: status.Sessions, MaxSessions: options.Capacity, - RefreshRate: 5, + RefreshRate: refreshRate, } w.WriteHeader(http.StatusServiceUnavailable) @@ -161,24 +191,55 @@ func (q *Queue) renderQueuePage(w http.ResponseWriter, r *http.Request, queueNam } } -func (q *Queue) refreshQueue(queueName string, keepAlive time.Duration) { +func (q *Queue) refreshQueue(ctx context.Context, layerName store.LayerName, keepAlive time.Duration) { if !atomic.CompareAndSwapUint32(&q.refreshJobRunning, 0, 1) { return } - go func() { - defer atomic.StoreUint32(&q.refreshJobRunning, 0) + defer atomic.StoreUint32(&q.refreshJobRunning, 0) - ctx, cancel := context.WithTimeout(context.Background(), keepAlive*2) - defer cancel() + if err := q.adapter.Refresh(ctx, string(layerName), keepAlive); err != nil { + logger.Error(ctx, "could not refresh queue", + logger.E(errors.WithStack(err)), + logger.F("queue", layerName), + ) + } +} - if err := q.adapter.Refresh(ctx, queueName, keepAlive); err != nil { - logger.Error(ctx, "could not refresh queue", - logger.E(errors.WithStack(err)), - logger.F("queue", queueName), - ) - } - }() +func (q *Queue) updateMetrics(ctx context.Context, proxyName store.ProxyName, layerName store.LayerName, options *LayerOptions) { + // Update queue capacity metric + metricQueueCapacity.With( + prometheus.Labels{ + metricLabelLayer: string(layerName), + metricLabelProxy: string(proxyName), + }, + ).Set(float64(options.Capacity)) + + // Refresh queue data and metrics + q.refreshQueue(ctx, layerName, options.KeepAlive) + q.updateSessionsMetric(ctx, proxyName, layerName) + + // (Re)schedule an update job after session ttl + semi-random time padding + // to update metrics after last session expiration + randDuration := rand.Int63n(int64(options.KeepAlive)) + timePadding := options.KeepAlive/2 + time.Duration(randDuration) + after := options.KeepAlive + timePadding + + debouncingKey := fmt.Sprintf("%s/%s", proxyName, layerName) + + q.postKeepAliveDebouncer.Do(debouncingKey, after, func() { + ctx := logger.With( + context.Background(), + logger.F("proxy", proxyName), + logger.F("layer", layerName), + logger.F("after", after), + ) + + logger.Info(ctx, "running post keep alive refresh job") + + q.refreshQueue(ctx, layerName, options.KeepAlive) + q.updateSessionsMetric(ctx, proxyName, layerName) + }) } func (q *Queue) getCookieName(layerName store.LayerName) string { @@ -192,9 +253,10 @@ func New(adapter Adapter, funcs ...OptionFunc) *Queue { } return &Queue{ - adapter: adapter, - templateDir: opts.TemplateDir, - defaultKeepAlive: opts.DefaultKeepAlive, + adapter: adapter, + templateDir: opts.TemplateDir, + defaultKeepAlive: opts.DefaultKeepAlive, + postKeepAliveDebouncer: NewDebouncerMap(), } } diff --git a/internal/proxy/director/metrics.go b/internal/proxy/director/metrics.go new file mode 100644 index 0000000..025124a --- /dev/null +++ b/internal/proxy/director/metrics.go @@ -0,0 +1,20 @@ +package director + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + metricNamespace = "bouncer_proxy_director" + metricLabelProxy = "proxy" +) + +var metricProxyRequestsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_requests_total", + Help: "Bouncer proxy total requests", + Namespace: metricNamespace, + }, + []string{metricLabelProxy}, +) diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 98c2ef6..ee5adbe 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -15,6 +15,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus/promhttp" "gitlab.com/wpetit/goweb/logger" ) @@ -84,18 +85,40 @@ func (s *Server) run(parentCtx context.Context, addrs chan net.Addr, errs chan e ) router.Use(middleware.RequestLogger(bouncerChi.NewLogFormatter())) - router.Use(director.Middleware()) - handler := proxy.New( - proxy.WithRequestTransformers( - director.RequestTransformer(), - ), - proxy.WithResponseTransformers( - director.ResponseTransformer(), - ), - ) + if s.serverConfig.Metrics.Enabled { + metrics := s.serverConfig.Metrics - router.Handle("/*", handler) + logger.Info(ctx, "enabling metrics", logger.F("endpoint", metrics.Endpoint)) + + router.Group(func(r chi.Router) { + if metrics.BasicAuth != nil { + logger.Info(ctx, "enabling authentication on metrics endpoint") + + r.Use(middleware.BasicAuth( + "metrics", + metrics.BasicAuth.CredentialsMap(), + )) + } + + r.Handle(string(metrics.Endpoint), promhttp.Handler()) + }) + } + + router.Group(func(r chi.Router) { + r.Use(director.Middleware()) + + handler := proxy.New( + proxy.WithRequestTransformers( + director.RequestTransformer(), + ), + proxy.WithResponseTransformers( + director.ResponseTransformer(), + ), + ) + + r.Handle("/*", handler) + }) if err := http.Serve(listener, router); err != nil && !errors.Is(err, net.ErrClosed) { errs <- errors.WithStack(err) diff --git a/layers/queue/templates/queue.gohtml b/layers/queue/templates/queue.gohtml index af6cb5c..3dc7b07 100644 --- a/layers/queue/templates/queue.gohtml +++ b/layers/queue/templates/queue.gohtml @@ -5,7 +5,7 @@ Veuillez patienter - {{ .QueueName }} - +