From f1c107c43d25e841fa0a3a0807448d2a473e4724 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Fri, 28 Mar 2025 16:10:10 -0700 Subject: [PATCH 1/3] new svcutil package, with MetricsMiddleware --- util/svcutil/metrics_middleware.go | 99 ++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 util/svcutil/metrics_middleware.go diff --git a/util/svcutil/metrics_middleware.go b/util/svcutil/metrics_middleware.go new file mode 100644 index 00000000..10c5e08d --- /dev/null +++ b/util/svcutil/metrics_middleware.go @@ -0,0 +1,99 @@ +package svcutil + +import ( + "errors" + "net/http" + "strconv" + "time" + + "github.com/labstack/echo/v4" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var reqSz = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "http_request_size_bytes", + Help: "A histogram of request sizes for requests.", + Buckets: prometheus.ExponentialBuckets(100, 10, 8), +}, []string{"code", "method", "path"}) + +var reqDur = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "http_request_duration_seconds", + Help: "A histogram of latencies for requests.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), +}, []string{"code", "method", "path"}) + +var reqCnt = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "http_requests_total", + Help: "A counter for requests to the wrapped handler.", +}, []string{"code", "method", "path"}) + +var resSz = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "http_response_size_bytes", + Help: "A histogram of response sizes for requests.", + Buckets: prometheus.ExponentialBuckets(100, 10, 8), +}, []string{"code", "method", "path"}) + +// MetricsMiddleware defines handler function for metrics middleware +func MetricsMiddleware(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + path := c.Path() + if path == "/metrics" || path == "/_health" { + return next(c) + } + + start := time.Now() + requestSize := computeApproximateRequestSize(c.Request()) + + err := next(c) + + status := c.Response().Status + if err != nil { + var httpError *echo.HTTPError + if errors.As(err, &httpError) { + status = httpError.Code + } + if status == 0 || status == http.StatusOK { + status = http.StatusInternalServerError + } + } + + elapsed := float64(time.Since(start)) / float64(time.Second) + + statusStr := strconv.Itoa(status) + method := c.Request().Method + + responseSize := float64(c.Response().Size) + + reqDur.WithLabelValues(statusStr, method, path).Observe(elapsed) + reqCnt.WithLabelValues(statusStr, method, path).Inc() + reqSz.WithLabelValues(statusStr, method, path).Observe(float64(requestSize)) + resSz.WithLabelValues(statusStr, method, path).Observe(responseSize) + + return err + } +} + +func computeApproximateRequestSize(r *http.Request) int { + s := 0 + if r.URL != nil { + s = len(r.URL.Path) + } + + s += len(r.Method) + s += len(r.Proto) + for name, values := range r.Header { + s += len(name) + for _, value := range values { + s += len(value) + } + } + s += len(r.Host) + + // N.B. r.Form and r.MultipartForm are assumed to be included in r.URL. + + if r.ContentLength != -1 { + s += int(r.ContentLength) + } + return s +} -- 2.51.2 From 0178d878ca096a30a2cdb29ac7a1372e7b59faa8 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Fri, 28 Mar 2025 16:10:48 -0700 Subject: [PATCH 2/3] swap in svcutil version of MetricsMiddleware for bgs and splitter --- bgs/bgs.go | 3 +- bgs/metrics.go | 93 -------------------------------------------- splitter/splitter.go | 4 +- 3 files changed, 4 insertions(+), 96 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 7eea8206..a75e2c52 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -24,6 +24,7 @@ import ( "github.com/bluesky-social/indigo/indexer" "github.com/bluesky-social/indigo/models" "github.com/bluesky-social/indigo/repomgr" + "github.com/bluesky-social/indigo/util/svcutil" "github.com/bluesky-social/indigo/xrpc" lru "github.com/hashicorp/golang-lru/v2" "golang.org/x/sync/semaphore" @@ -237,7 +238,7 @@ func (bgs *BGS) StartWithListener(listen net.Listener) error { e.File("/dash/*", "public/index.html") e.Static("/assets", "public/assets") - e.Use(MetricsMiddleware) + e.Use(svcutil.MetricsMiddleware) e.HTTPErrorHandler = func(err error, ctx echo.Context) { switch err := err.(type) { diff --git a/bgs/metrics.go b/bgs/metrics.go index 5ff362a1..519cd2e3 100644 --- a/bgs/metrics.go +++ b/bgs/metrics.go @@ -1,12 +1,6 @@ package bgs import ( - "errors" - "net/http" - "strconv" - "time" - - "github.com/labstack/echo/v4" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) @@ -68,29 +62,6 @@ var newUsersDiscovered = promauto.NewCounter(prometheus.CounterOpts{ Help: "The total number of new users discovered directly from the firehose (not from refs)", }) -var reqSz = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "http_request_size_bytes", - Help: "A histogram of request sizes for requests.", - Buckets: prometheus.ExponentialBuckets(100, 10, 8), -}, []string{"code", "method", "path"}) - -var reqDur = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "http_request_duration_seconds", - Help: "A histogram of latencies for requests.", - Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), -}, []string{"code", "method", "path"}) - -var reqCnt = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "http_requests_total", - Help: "A counter for requests to the wrapped handler.", -}, []string{"code", "method", "path"}) - -var resSz = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "http_response_size_bytes", - Help: "A histogram of response sizes for requests.", - Buckets: prometheus.ExponentialBuckets(100, 10, 8), -}, []string{"code", "method", "path"}) - var userLookupDuration = promauto.NewHistogram(prometheus.HistogramOpts{ Name: "relay_user_lookup_duration", Help: "A histogram of user lookup latencies", @@ -102,67 +73,3 @@ var newUserDiscoveryDuration = promauto.NewHistogram(prometheus.HistogramOpts{ Help: "A histogram of new user discovery latencies", Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), }) - -// MetricsMiddleware defines handler function for metrics middleware -func MetricsMiddleware(next echo.HandlerFunc) echo.HandlerFunc { - return func(c echo.Context) error { - path := c.Path() - if path == "/metrics" || path == "/_health" { - return next(c) - } - - start := time.Now() - requestSize := computeApproximateRequestSize(c.Request()) - - err := next(c) - - status := c.Response().Status - if err != nil { - var httpError *echo.HTTPError - if errors.As(err, &httpError) { - status = httpError.Code - } - if status == 0 || status == http.StatusOK { - status = http.StatusInternalServerError - } - } - - elapsed := float64(time.Since(start)) / float64(time.Second) - - statusStr := strconv.Itoa(status) - method := c.Request().Method - - responseSize := float64(c.Response().Size) - - reqDur.WithLabelValues(statusStr, method, path).Observe(elapsed) - reqCnt.WithLabelValues(statusStr, method, path).Inc() - reqSz.WithLabelValues(statusStr, method, path).Observe(float64(requestSize)) - resSz.WithLabelValues(statusStr, method, path).Observe(responseSize) - - return err - } -} - -func computeApproximateRequestSize(r *http.Request) int { - s := 0 - if r.URL != nil { - s = len(r.URL.Path) - } - - s += len(r.Method) - s += len(r.Proto) - for name, values := range r.Header { - s += len(name) - for _, value := range values { - s += len(value) - } - } - s += len(r.Host) - - // N.B. r.Form and r.MultipartForm are assumed to be included in r.URL. - - if r.ContentLength != -1 { - s += int(r.ContentLength) - } - return s -} diff --git a/splitter/splitter.go b/splitter/splitter.go index 93f96538..1aa90af8 100644 --- a/splitter/splitter.go +++ b/splitter/splitter.go @@ -21,11 +21,11 @@ import ( "github.com/bluesky-social/indigo/api/atproto" comatproto "github.com/bluesky-social/indigo/api/atproto" - "github.com/bluesky-social/indigo/bgs" "github.com/bluesky-social/indigo/events" "github.com/bluesky-social/indigo/events/pebblepersist" "github.com/bluesky-social/indigo/events/schedulers/sequential" "github.com/bluesky-social/indigo/util" + "github.com/bluesky-social/indigo/util/svcutil" "github.com/bluesky-social/indigo/xrpc" "github.com/gorilla/websocket" "github.com/labstack/echo/v4" @@ -211,7 +211,7 @@ func (s *Splitter) StartWithListener(listen net.Listener) error { } */ - e.Use(bgs.MetricsMiddleware) + e.Use(svcutil.MetricsMiddleware) e.HTTPErrorHandler = func(err error, ctx echo.Context) { switch err := err.(type) { -- 2.51.2 From 7e2494b0c3efff9045268a2034c0c3fc4db9a8e0 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Fri, 28 Mar 2025 16:15:38 -0700 Subject: [PATCH 3/3] use svcutil for collectionsdir --- cmd/collectiondir/metrics.go | 52 ------------------------------------ cmd/collectiondir/serve.go | 3 ++- 2 files changed, 2 insertions(+), 53 deletions(-) diff --git a/cmd/collectiondir/metrics.go b/cmd/collectiondir/metrics.go index 3d1af066..993e39af 100644 --- a/cmd/collectiondir/metrics.go +++ b/cmd/collectiondir/metrics.go @@ -1,13 +1,8 @@ package main import ( - "errors" - "github.com/labstack/echo/v4" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "net/http" - "strconv" - "time" ) var firehoseReceivedCounter = promauto.NewCounter(prometheus.CounterOpts{ @@ -52,50 +47,3 @@ var statsCalculations = promauto.NewHistogram(prometheus.HistogramOpts{ Help: "how long it takes to calculate total stats", Buckets: prometheus.ExponentialBuckets(0.01, 2, 13), }) - -var reqDur = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "http_request_duration_seconds", - Help: "A histogram of latencies for requests.", - Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), -}, []string{"code", "method", "path"}) - -var reqCnt = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "http_requests_total", - Help: "A counter for requests to the wrapped handler.", -}, []string{"code", "method", "path"}) - -// MetricsMiddleware defines handler function for metrics middleware -// TODO: reunify with bgs/metrics.go ? -func MetricsMiddleware(next echo.HandlerFunc) echo.HandlerFunc { - return func(c echo.Context) error { - path := c.Path() - if path == "/metrics" || path == "/_health" { - return next(c) - } - - start := time.Now() - - err := next(c) - - status := c.Response().Status - if err != nil { - var httpError *echo.HTTPError - if errors.As(err, &httpError) { - status = httpError.Code - } - if status == 0 || status == http.StatusOK { - status = http.StatusInternalServerError - } - } - - elapsed := float64(time.Since(start)) / float64(time.Second) - - statusStr := strconv.Itoa(status) - method := c.Request().Method - - reqDur.WithLabelValues(statusStr, method, path).Observe(elapsed) - reqCnt.WithLabelValues(statusStr, method, path).Inc() - - return err - } -} diff --git a/cmd/collectiondir/serve.go b/cmd/collectiondir/serve.go index b8e2c506..fba58441 100644 --- a/cmd/collectiondir/serve.go +++ b/cmd/collectiondir/serve.go @@ -25,6 +25,7 @@ import ( comatproto "github.com/bluesky-social/indigo/api/atproto" "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/util/svcutil" "github.com/bluesky-social/indigo/xrpc" "github.com/labstack/echo/v4" @@ -405,7 +406,7 @@ func (cs *collectionServer) StartApiServer(ctx context.Context, addr string) err e := echo.New() e.HideBanner = true - e.Use(MetricsMiddleware) + e.Use(svcutil.MetricsMiddleware) e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ AllowOrigins: []string{"*"}, AllowHeaders: []string{echo.HeaderOrigin, echo.HeaderContentType, echo.HeaderAccept, echo.HeaderAuthorization}, -- 2.51.2