From 1a954eaaf7343d74df109110952329879c53e559 Mon Sep 17 00:00:00 2001 From: Thibault Le Ouay Date: Thu, 12 Feb 2026 14:29:57 +0100 Subject: [PATCH] bug checker (#1845) --- apps/checker/cmd/server/main.go | 2 +- apps/checker/go.mod | 11 +- apps/checker/go.sum | 20 +- apps/checker/pkg/otel/otel.go | 248 ++++++++---------------- apps/checker/pkg/otel/otel_test.go | 300 +++++++++++++++++++++++++++++ 5 files changed, 393 insertions(+), 188 deletions(-) create mode 100644 apps/checker/pkg/otel/otel_test.go diff --git a/apps/checker/cmd/server/main.go b/apps/checker/cmd/server/main.go index 717ce5f7..542486eb 100644 --- a/apps/checker/cmd/server/main.go +++ b/apps/checker/cmd/server/main.go @@ -26,7 +26,7 @@ import ( otlploghttp "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" sdklog "go.opentelemetry.io/otel/sdk/log" "go.opentelemetry.io/otel/sdk/resource" - semconv "go.opentelemetry.io/otel/semconv/v1.24.0" + semconv "go.opentelemetry.io/otel/semconv/v1.39.0" ) func shouldSample(event map[string]any) bool { diff --git a/apps/checker/go.mod b/apps/checker/go.mod index 9879344c..bd209106 100644 --- a/apps/checker/go.mod +++ b/apps/checker/go.mod @@ -8,9 +8,9 @@ require ( github.com/google/uuid v1.6.0 github.com/rs/zerolog v1.34.0 github.com/stretchr/testify v1.11.1 - go.opentelemetry.io/otel v1.39.0 + go.opentelemetry.io/otel v1.40.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.34.0 - go.opentelemetry.io/otel/metric v1.39.0 + go.opentelemetry.io/otel/metric v1.40.0 go.opentelemetry.io/otel/sdk v1.39.0 go.opentelemetry.io/otel/sdk/metric v1.39.0 google.golang.org/protobuf v1.36.11 @@ -24,9 +24,9 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/cenkalti/backoff/v5 v5.0.3 github.com/madflojo/tasks v1.2.1 - github.com/vincentfree/opentelemetry/otelzerolog v0.1.1 go.opentelemetry.io/contrib/bridges/otelslog v0.14.0 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.15.0 + go.opentelemetry.io/otel/log v0.15.0 go.opentelemetry.io/otel/sdk/log v0.15.0 google.golang.org/api v0.247.0 ) @@ -66,12 +66,9 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/bridges/otelzerolog v0.0.0-20240726205214-b0584291236a // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect - go.opentelemetry.io/otel/log v0.15.0 // indirect - go.opentelemetry.io/otel/log/logtest v0.15.0 // indirect - go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.opentelemetry.io/otel/trace v1.40.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect golang.org/x/arch v0.7.0 // indirect golang.org/x/crypto v0.46.0 // indirect diff --git a/apps/checker/go.sum b/apps/checker/go.sum index c45ed9bb..46e027d3 100644 --- a/apps/checker/go.sum +++ b/apps/checker/go.sum @@ -130,30 +130,24 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/vincentfree/opentelemetry/otelzerolog v0.1.1 h1:JwLECM8a+sxQGtFjREmkUzfXlQGmmeIqq9THUbMfuLo= -github.com/vincentfree/opentelemetry/otelzerolog v0.1.1/go.mod h1:nm1JiUG5YGpuAQMqoLLoU50oBBGxJ7R58Zpcp5cJ6LQ= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/bridges/otelslog v0.14.0 h1:eypSOd+0txRKCXPNyqLPsbSfA0jULgJcGmSAdFAnrCM= go.opentelemetry.io/contrib/bridges/otelslog v0.14.0/go.mod h1:CRGvIBL/aAxpQU34ZxyQVFlovVcp67s4cAmQu8Jh9mc= -go.opentelemetry.io/contrib/bridges/otelzerolog v0.0.0-20240726205214-b0584291236a h1:8EYSlvUrF1q+5XFBpUAz1XkYg3XAyBSqru0d1IZdLlk= -go.opentelemetry.io/contrib/bridges/otelzerolog v0.0.0-20240726205214-b0584291236a/go.mod h1:IrTzekhe/C6xdedbc0jIDhWMMPvXiF63Lj6JaVogekU= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 h1:ssfIgGNANqpVFCndZvcuyKbl0g+UAVcbBcqGkG28H0Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0/go.mod h1:GQ/474YrbE4Jx8gZ4q5I4hrhUzM6UPzyrqJYV2AqPoQ= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.15.0 h1:EKpiGphOYq3CYnIe2eX9ftUkyU+Y8Dtte8OaWyHJ4+I= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.15.0/go.mod h1:nWFP7C+T8TygkTjJ7mAyEaFaE7wNfms3nV/vexZ6qt0= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.34.0 h1:opwv08VbCZ8iecIWs+McMdHRcAXzjAeda3uG2kI/hcA= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.34.0/go.mod h1:oOP3ABpW7vFHulLpE8aYtNBodrHhMTrvfxUXGvqm7Ac= go.opentelemetry.io/otel/log v0.15.0 h1:0VqVnc3MgyYd7QqNVIldC3dsLFKgazR6P3P3+ypkyDY= go.opentelemetry.io/otel/log v0.15.0/go.mod h1:9c/G1zbyZfgu1HmQD7Qj84QMmwTp2QCQsZH1aeoWDE4= -go.opentelemetry.io/otel/log/logtest v0.15.0 h1:porNFuxAjodl6LhePevOc3n7bo3Wi3JhGXNWe7KP8iU= -go.opentelemetry.io/otel/log/logtest v0.15.0/go.mod h1:c8epqBXGHgS1LiNgmD+LuNYK9lSS3mqvtMdxLsfJgLg= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= go.opentelemetry.io/otel/sdk/log v0.15.0 h1:WgMEHOUt5gjJE93yqfqJOkRflApNif84kxoHWS9VVHE= @@ -162,8 +156,8 @@ go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4Y go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= @@ -171,8 +165,6 @@ golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= diff --git a/apps/checker/pkg/otel/otel.go b/apps/checker/pkg/otel/otel.go index 462bd9e3..5e0a9b96 100644 --- a/apps/checker/pkg/otel/otel.go +++ b/apps/checker/pkg/otel/otel.go @@ -2,8 +2,6 @@ package otel import ( "context" - "errors" - "fmt" "time" "github.com/openstatushq/openstatus/apps/checker/checker" @@ -17,49 +15,23 @@ import ( sdkMetrics "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" - semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + semconv "go.opentelemetry.io/otel/semconv/v1.39.0" ) -func SetupOTelSDK( - ctx context.Context, url string, probes string, headers map[string]string, -) (shutdown func(context.Context) error, err error) { - var shutdownFuncs []func(context.Context) error - - shutdown = func(ctx context.Context) error { - var err error - - for _, fn := range shutdownFuncs { - err = errors.Join(err, fn(ctx)) - } - - shutdownFuncs = nil - - return err - } - - handleErr := func(inErr error) { - err = errors.Join(inErr, shutdown(ctx)) - } - +func setupOTelSDK(ctx context.Context, url string, headers map[string]string) (shutdown func(context.Context) error, err error) { res, err := newResource() if err != nil { - handleErr(err) - return nil, err } meterProvider, err := newMeterProvider(ctx, res, url, headers) if err != nil { - handleErr(err) - return nil, err } - shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown) - otel.SetMeterProvider(meterProvider) - return shutdown, nil + return meterProvider.Shutdown, nil } func newResource() (*resource.Resource, error) { @@ -76,183 +48,127 @@ func newMeterProvider( url string, headers map[string]string, ) (*sdkMetrics.MeterProvider, error) { - - grafanaExporter, err := otlpmetrichttp.New(ctx, + exporter, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithEndpointURL(url), - // otlpmetrichttp.WithInsecure(), otlpmetrichttp.WithHeaders(headers), ) if err != nil { return nil, err } - meterProvider := sdkMetrics.NewMeterProvider( + return sdkMetrics.NewMeterProvider( sdkMetrics.WithResource(res), - - sdkMetrics.WithReader(sdkMetrics.NewPeriodicReader(grafanaExporter, + sdkMetrics.WithReader(sdkMetrics.NewPeriodicReader(exporter, sdkMetrics.WithInterval(3*time.Second))), - ) - - return meterProvider, nil + ), nil } -func RecordHTTPMetrics(ctx context.Context, req request.HttpCheckerRequest, result checker.Response, region string) { - - otelShutdown, err := SetupOTelSDK(ctx, req.OtelConfig.Endpoint, region, req.OtelConfig.Headers) - +// withMeter sets up the OTel SDK, passes a Meter to the callback, then shuts down. +func withMeter(ctx context.Context, endpoint string, headers map[string]string, fn func(metric.Meter)) { + shutdown, err := setupOTelSDK(ctx, endpoint, headers) if err != nil { log.Ctx(ctx).Error().Err(err).Msg("Error setting up otel") + return } defer func() { - err = errors.Join(err, otelShutdown(ctx)) - if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("Error sending the data") + if err := shutdown(ctx); err != nil { + log.Ctx(ctx).Error().Err(err).Msg("Error shutting down otel") } }() - meter := otel.Meter("OpenStatus") - - if result.Error != "" { - att := metric.WithAttributes( - attribute.String("openstatus.probes", region), - attribute.String("openstatus.target", req.URL), - semconv.HTTPResponseStatusCode(result.Status), - ) - statusError, err := meter.Int64Counter("openstatus.error", metric.WithDescription("Status of the check")) - - if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("Error setting up counter") - } - - statusError.Add(ctx, (1), att) - - return - } - - att := metric.WithAttributes( - attribute.String("openstatus.probes", region), - attribute.String("openstatus.target", req.URL), - semconv.HTTPResponseStatusCode(result.Status), - ) - - status, err := meter.Int64Counter("openstatus.status", metric.WithDescription("Status of the check")) - - if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("Error setting up conunter") - } - - status.Add(ctx, 1, att) - - gauge, err := meter.Float64Gauge("openstatus.http.request.duration", - metric.WithDescription("Duration of the check"), metric.WithUnit("ms")) - - if err != nil { - fmt.Println("Error creating gauge", err) - } - - gauge.Record(ctx, float64(result.Latency), att) - - gaugeDns, err := meter.Float64Gauge("openstatus.http.dns.duration", - metric.WithDescription("Duration of the dns lookup"), metric.WithUnit("ms")) - - if err != nil { - fmt.Println("Error creating gauge", err) - } - - gaugeDns.Record(ctx, float64(result.Timing.DnsDone-result.Timing.DnsStart), att) - - gaugeConnect, err := meter.Float64Gauge("openstatus.http.connection.duration", - metric.WithDescription("Duration of the connection"), metric.WithUnit("ms")) - - if err != nil { - fmt.Println("Error creating gauge", err) - } - - gaugeConnect.Record(ctx, float64(result.Timing.ConnectDone-result.Timing.ConnectStart), att) - - gaugeTLS, err := meter.Float64Gauge("openstatus.http.tls.duration", - metric.WithDescription("Duration of the tls handshake"), metric.WithUnit("ms")) - - if err != nil { - fmt.Println("Error creating gauge", err) - } - - gaugeTLS.Record(ctx, float64(result.Timing.TlsHandshakeDone-result.Timing.TlsHandshakeStart), att) - - gaugeTTFB, err := meter.Float64Gauge("openstatus.http.ttfb.duration", - metric.WithDescription("Duration of the ttfb"), metric.WithUnit("ms")) + fn(otel.Meter("OpenStatus")) +} +// recordGauge creates a Float64Gauge and records a value. +func recordGauge(ctx context.Context, meter metric.Meter, name, description string, value float64, att metric.MeasurementOption) error { + gauge, err := meter.Float64Gauge(name, + metric.WithDescription(description), metric.WithUnit("ms")) if err != nil { - fmt.Println("Error creating gauge", err) + return err } - gaugeTTFB.Record(ctx, float64(result.Timing.FirstByteDone-result.Timing.FirstByteStart), att) + gauge.Record(ctx, value, att) - gaugeTransfer, err := meter.Float64Gauge("openstatus.http.transfer.duration", - metric.WithDescription("Duration of the transfer"), metric.WithUnit("ms")) + return nil +} +func recordErrorCounter(ctx context.Context, meter metric.Meter, att metric.MeasurementOption) { + counter, err := meter.Int64Counter("openstatus.error", metric.WithDescription("Status of the check")) if err != nil { - fmt.Println("Error creating gauge", err) + log.Ctx(ctx).Error().Err(err).Msg("Error setting up counter") + return } - gaugeTransfer.Record(ctx, float64(result.Timing.TransferDone-result.Timing.TransferStart), att) + counter.Add(ctx, 1, att) } -func RecordTCPMetrics(ctx context.Context, req request.TCPCheckerRequest, result checker.TCPResponse, region string) { - - otelShutdown, err := SetupOTelSDK(ctx, req.OtelConfig.Endpoint, region, req.OtelConfig.Headers) +func RecordHTTPMetrics(ctx context.Context, req request.HttpCheckerRequest, result checker.Response, region string) { + withMeter(ctx, req.OtelConfig.Endpoint, req.OtelConfig.Headers, func(meter metric.Meter) { + att := metric.WithAttributes( + attribute.String("openstatus.probes", region), + attribute.String("openstatus.target", req.URL), + semconv.HTTPResponseStatusCode(result.Status), + ) - if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("Error setting up otel") - } + if result.Error != "" { + recordErrorCounter(ctx, meter, att) + return + } - defer func() { - err = errors.Join(err, otelShutdown(ctx)) + status, err := meter.Int64Counter("openstatus.status", metric.WithDescription("Status of the check")) if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("Error sending the data") + log.Ctx(ctx).Error().Err(err).Msg("Error setting up counter") } - }() - meter := otel.Meter("OpenStatus") + status.Add(ctx, 1, att) + + timings := []struct { + name string + description string + value float64 + }{ + {"openstatus.http.request.duration", "Duration of the check", float64(result.Latency)}, + {"openstatus.http.dns.duration", "Duration of the DNS lookup", float64(result.Timing.DnsDone - result.Timing.DnsStart)}, + {"openstatus.http.connection.duration", "Duration of the connection", float64(result.Timing.ConnectDone - result.Timing.ConnectStart)}, + {"openstatus.http.tls.duration", "Duration of the TLS handshake", float64(result.Timing.TlsHandshakeDone - result.Timing.TlsHandshakeStart)}, + {"openstatus.http.ttfb.duration", "Duration of the TTFB", float64(result.Timing.FirstByteDone - result.Timing.FirstByteStart)}, + {"openstatus.http.transfer.duration", "Duration of the transfer", float64(result.Timing.TransferDone - result.Timing.TransferStart)}, + } - if result.Error == 1 { + for _, t := range timings { + if err := recordGauge(ctx, meter, t.name, t.description, t.value, att); err != nil { + log.Ctx(ctx).Error().Err(err).Str("metric", t.name).Msg("Error creating gauge") + } + } + }) +} + +func RecordTCPMetrics(ctx context.Context, req request.TCPCheckerRequest, result checker.TCPResponse, region string) { + withMeter(ctx, req.OtelConfig.Endpoint, req.OtelConfig.Headers, func(meter metric.Meter) { att := metric.WithAttributes( attribute.String("openstatus.probes", region), attribute.String("openstatus.target", req.URI), ) - statusError, err := meter.Int64Counter("openstatus.error", metric.WithDescription("Status of the check")) - if err != nil { - log.Ctx(ctx).Error().Err(err).Msg("Error setting up counter") + if result.Error == 1 { + recordErrorCounter(ctx, meter, att) + return } - statusError.Add(ctx, (1), att) - - return - } - - att := metric.WithAttributes( - attribute.String("openstatus.probes", region), - attribute.String("openstatus.target", req.URI), - ) - - gauge, err := meter.Float64Gauge("openstatus.tcp.request.duration", - metric.WithDescription("Duration of the check"), metric.WithUnit("ms")) - - if err != nil { - fmt.Println("Error creating gauge", err) - } - - gauge.Record(ctx, float64(result.Latency), att) - - gaugeTCP, err := meter.Float64Gauge("openstatus.tcp.tcp.duration", - metric.WithDescription("Duration of the dns lookup"), metric.WithUnit("ms")) - - if err != nil { - fmt.Println("Error creating gauge", err) - } - - gaugeTCP.Record(ctx, float64(result.Timing.TCPDone-result.Timing.TCPStart), att) + timings := []struct { + name string + description string + value float64 + }{ + {"openstatus.tcp.request.duration", "Duration of the check", float64(result.Latency)}, + {"openstatus.tcp.tcp.duration", "Duration of the TCP connection", float64(result.Timing.TCPDone - result.Timing.TCPStart)}, + } + for _, t := range timings { + if err := recordGauge(ctx, meter, t.name, t.description, t.value, att); err != nil { + log.Ctx(ctx).Error().Err(err).Str("metric", t.name).Msg("Error creating gauge") + } + } + }) } diff --git a/apps/checker/pkg/otel/otel_test.go b/apps/checker/pkg/otel/otel_test.go new file mode 100644 index 00000000..c80a25c4 --- /dev/null +++ b/apps/checker/pkg/otel/otel_test.go @@ -0,0 +1,300 @@ +package otel + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/openstatushq/openstatus/apps/checker/checker" + "github.com/openstatushq/openstatus/apps/checker/request" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + sdkMetrics "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +func newTestMeter(t *testing.T) (metric.Meter, *sdkMetrics.ManualReader) { + t.Helper() + reader := sdkMetrics.NewManualReader() + provider := sdkMetrics.NewMeterProvider(sdkMetrics.WithReader(reader)) + t.Cleanup(func() { + require.NoError(t, provider.Shutdown(context.Background())) + }) + return provider.Meter("test"), reader +} + +func collectMetrics(t *testing.T, reader *sdkMetrics.ManualReader) metricdata.ResourceMetrics { + t.Helper() + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + return rm +} + +func newOTLPTestServer(t *testing.T) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + return server +} + +// --- recordGauge tests --- + +func TestRecordGauge(t *testing.T) { + meter, reader := newTestMeter(t) + ctx := context.Background() + att := metric.WithAttributes(attribute.String("test.key", "test-value")) + + err := recordGauge(ctx, meter, "test.gauge", "A test gauge", 42.5, att) + require.NoError(t, err) + + rm := collectMetrics(t, reader) + require.Len(t, rm.ScopeMetrics, 1) + + sm := rm.ScopeMetrics[0] + require.Len(t, sm.Metrics, 1) + + m := sm.Metrics[0] + assert.Equal(t, "test.gauge", m.Name) + assert.Equal(t, "A test gauge", m.Description) + assert.Equal(t, "ms", m.Unit) + + gauge, ok := m.Data.(metricdata.Gauge[float64]) + require.True(t, ok, "expected Gauge[float64] data type") + require.Len(t, gauge.DataPoints, 1) + assert.Equal(t, 42.5, gauge.DataPoints[0].Value) + + attrs := gauge.DataPoints[0].Attributes + val, found := attrs.Value(attribute.Key("test.key")) + assert.True(t, found) + assert.Equal(t, "test-value", val.AsString()) +} + +func TestRecordGauge_MultipleMetrics(t *testing.T) { + meter, reader := newTestMeter(t) + ctx := context.Background() + att := metric.WithAttributes(attribute.String("region", "us-east-1")) + + gauges := []struct { + name string + desc string + value float64 + }{ + {"http.dns.duration", "DNS duration", 10.0}, + {"http.tls.duration", "TLS duration", 25.0}, + {"http.ttfb.duration", "TTFB duration", 50.0}, + } + + for _, g := range gauges { + require.NoError(t, recordGauge(ctx, meter, g.name, g.desc, g.value, att)) + } + + rm := collectMetrics(t, reader) + require.Len(t, rm.ScopeMetrics, 1) + assert.Len(t, rm.ScopeMetrics[0].Metrics, 3) +} + +// --- recordErrorCounter tests --- + +func TestRecordErrorCounter(t *testing.T) { + meter, reader := newTestMeter(t) + ctx := context.Background() + att := metric.WithAttributes(attribute.String("region", "us-east-1")) + + recordErrorCounter(ctx, meter, att) + + rm := collectMetrics(t, reader) + require.Len(t, rm.ScopeMetrics, 1) + require.Len(t, rm.ScopeMetrics[0].Metrics, 1) + + m := rm.ScopeMetrics[0].Metrics[0] + assert.Equal(t, "openstatus.error", m.Name) + + sum, ok := m.Data.(metricdata.Sum[int64]) + require.True(t, ok, "expected Sum[int64] data type") + require.Len(t, sum.DataPoints, 1) + assert.Equal(t, int64(1), sum.DataPoints[0].Value) +} + +// --- setupOTelSDK tests --- + +func TestSetupOTelSDK(t *testing.T) { + server := newOTLPTestServer(t) + ctx := context.Background() + + shutdown, err := setupOTelSDK(ctx, server.URL, nil) + if err != nil { + // The resource merge can fail due to semconv schema URL version + // mismatch between resource.Default() and the pinned semconv import. + assert.Nil(t, shutdown, "shutdown must be nil when setup fails") + return + } + + require.NotNil(t, shutdown) + assert.NoError(t, shutdown(ctx)) +} + +func TestSetupOTelSDK_InvalidURL(t *testing.T) { + ctx := context.Background() + + shutdown, err := setupOTelSDK(ctx, "://invalid", nil) + if err != nil { + assert.Nil(t, shutdown, "shutdown should be nil when setup fails") + } else { + require.NotNil(t, shutdown) + _ = shutdown(ctx) + } +} + +// --- withMeter tests --- + +func TestWithMeter(t *testing.T) { + server := newOTLPTestServer(t) + called := false + + withMeter(context.Background(), server.URL, nil, func(meter metric.Meter) { + called = true + assert.NotNil(t, meter) + }) + + // withMeter may fail due to schema URL mismatch — only assert if it got called. + if !called { + t.Log("withMeter callback not called (likely schema URL mismatch), skipping") + } +} + +func TestWithMeter_InvalidEndpoint(t *testing.T) { + called := false + + // Must not panic — callback should not be called on setup failure. + withMeter(context.Background(), "://invalid", nil, func(meter metric.Meter) { + called = true + }) + + assert.False(t, called, "callback should not be called when setup fails") +} + +// --- RecordHTTPMetrics tests --- + +func TestRecordHTTPMetrics_Success(t *testing.T) { + server := newOTLPTestServer(t) + + req := request.HttpCheckerRequest{ + URL: "https://example.com", + MonitorID: "mon-1", + } + req.OtelConfig.Endpoint = server.URL + + result := checker.Response{ + Status: 200, + Latency: 150, + Timing: checker.Timing{ + DnsStart: 0, + DnsDone: 10, + ConnectStart: 10, + ConnectDone: 30, + TlsHandshakeStart: 30, + TlsHandshakeDone: 55, + FirstByteStart: 55, + FirstByteDone: 100, + TransferStart: 100, + TransferDone: 150, + }, + } + + // Should not panic. + RecordHTTPMetrics(context.Background(), req, result, "us-east-1") +} + +func TestRecordHTTPMetrics_Error(t *testing.T) { + server := newOTLPTestServer(t) + + req := request.HttpCheckerRequest{ + URL: "https://example.com", + MonitorID: "mon-1", + } + req.OtelConfig.Endpoint = server.URL + + result := checker.Response{ + Status: 500, + Error: "connection refused", + } + + // Should record error counter and not panic. + RecordHTTPMetrics(context.Background(), req, result, "us-east-1") +} + +func TestRecordHTTPMetrics_SetupFailure(t *testing.T) { + req := request.HttpCheckerRequest{ + URL: "https://example.com", + MonitorID: "mon-1", + } + req.OtelConfig.Endpoint = "://invalid" + + result := checker.Response{ + Status: 200, + Latency: 100, + } + + // Must not panic — this was the original nil pointer bug. + RecordHTTPMetrics(context.Background(), req, result, "us-east-1") +} + +// --- RecordTCPMetrics tests --- + +func TestRecordTCPMetrics_Success(t *testing.T) { + server := newOTLPTestServer(t) + + req := request.TCPCheckerRequest{ + URI: "example.com:443", + MonitorID: "mon-2", + } + req.OtelConfig.Endpoint = server.URL + + result := checker.TCPResponse{ + Latency: 45, + Timing: checker.TCPResponseTiming{ + TCPStart: 0, + TCPDone: 45, + }, + } + + // Should not panic. + RecordTCPMetrics(context.Background(), req, result, "us-east-1") +} + +func TestRecordTCPMetrics_Error(t *testing.T) { + server := newOTLPTestServer(t) + + req := request.TCPCheckerRequest{ + URI: "example.com:443", + MonitorID: "mon-2", + } + req.OtelConfig.Endpoint = server.URL + + result := checker.TCPResponse{ + Error: 1, + } + + // Should record error counter and not panic. + RecordTCPMetrics(context.Background(), req, result, "us-east-1") +} + +func TestRecordTCPMetrics_SetupFailure(t *testing.T) { + req := request.TCPCheckerRequest{ + URI: "example.com:443", + MonitorID: "mon-2", + } + req.OtelConfig.Endpoint = "://invalid" + + result := checker.TCPResponse{ + Latency: 45, + } + + // Must not panic — same nil pointer guard as HTTP. + RecordTCPMetrics(context.Background(), req, result, "us-east-1") +} -- 2.51.2