From 7fb1ec27c0ad957612ff9f9453dcb07b17843a32 Mon Sep 17 00:00:00 2001 From: Thibault Le Ouay Date: Sat, 10 Oct 2026 21:36:52 +0200 Subject: [PATCH] analytics: throttle event api (#2888) --- apps/server/src/libs/api-request-sampling.ts | 19 ++ .../server/src/libs/middlewares/track.test.ts | 42 +++ apps/server/src/libs/middlewares/track.ts | 8 +- .../interceptors/__tests__/tracking.test.ts | 75 +++++ .../src/routes/rpc/interceptors/tracking.ts | 17 +- deno.lock | 266 ++++++++++++++++++ 6 files changed, 422 insertions(+), 5 deletions(-) create mode 100644 apps/server/src/libs/api-request-sampling.ts diff --git a/apps/server/src/libs/api-request-sampling.ts b/apps/server/src/libs/api-request-sampling.ts new file mode 100644 index 000000000..d6de4348a --- /dev/null +++ b/apps/server/src/libs/api-request-sampling.ts @@ -0,0 +1,19 @@ +import type { CliInfo } from "@/libs/cli-telemetry"; + +/** Share of non-CLI `api_request` events sent to OpenPanel. */ +export const API_REQUEST_SAMPLE_RATE = 0.1; + +/** + * The sample rate to stamp on this request's `api_request` event, or + * `undefined` when the event is dropped. CLI requests are always kept so CLI + * usage stays exact; everything else is sampled at `API_REQUEST_SAMPLE_RATE`. + * Volumes are recovered by weighting each event by `1 / sampleRate`. + */ +export function apiRequestSampleRate( + cli: CliInfo | undefined, +): number | undefined { + if (cli) return 1; + return Math.random() < API_REQUEST_SAMPLE_RATE + ? API_REQUEST_SAMPLE_RATE + : undefined; +} diff --git a/apps/server/src/libs/middlewares/track.test.ts b/apps/server/src/libs/middlewares/track.test.ts index bf21cb126..6c735b8dc 100644 --- a/apps/server/src/libs/middlewares/track.test.ts +++ b/apps/server/src/libs/middlewares/track.test.ts @@ -1,10 +1,13 @@ import { Events } from "@openstatus/analytics"; import type { Workspace } from "@openstatus/db/src/schema"; import { + afterEach, beforeEach, describe, expect, type mock, + type Stub, + stub, test, } from "@openstatus/test-utils"; import { Hono } from "hono"; @@ -63,9 +66,17 @@ function cliCommandCalls() { } describe("apiTrackMiddleware", () => { + let random: Stub; + beforeEach(() => { mockSetupAnalytics.mockClear(); mockTrack.mockClear(); + // Keep every sampled request unless a test says otherwise. + random = stub(Math, "random", () => 0); + }); + + afterEach(() => { + random.restore(); }); test("fires api_request for GET requests with the route pattern", async () => { @@ -81,9 +92,39 @@ describe("apiTrackMiddleware", () => { service: "v1", method: "GET /monitor/:id", success: false, + sampleRate: 0.1, }); }); + test("drops non-CLI requests outside the sample", async () => { + random.restore(); + random = stub(Math, "random", () => 0.5); + + const res = await makeApp().request("/whoami", { + headers: { "user-agent": "curl/8" }, + }); + await flush(); + + expect(res.status).toBe(200); + expect(mockSetupAnalytics).not.toHaveBeenCalled(); + expect(mockTrack).not.toHaveBeenCalled(); + }); + + test("always tracks CLI requests", async () => { + random.restore(); + random = stub(Math, "random", () => 0.99); + + await makeApp().request("/whoami", { + headers: cliHeaders(crypto.randomUUID()), + }); + await flush(); + + expect(mockTrack).toHaveBeenCalledWith( + expect.objectContaining({ ...Events.ApiRequest, sampleRate: 1 }), + ); + expect(cliCommandCalls()).toHaveLength(1); + }); + test("skips requests that match no route", async () => { const res = await makeApp().request("/nope", { headers: { "user-agent": "curl/8" }, @@ -119,6 +160,7 @@ describe("apiTrackMiddleware", () => { service: "v1", method: "GET /whoami", success: true, + sampleRate: 1, cliCommand: "whoami", cliVersion: "1.3.2", }); diff --git a/apps/server/src/libs/middlewares/track.ts b/apps/server/src/libs/middlewares/track.ts index 628f48d43..784b6819b 100644 --- a/apps/server/src/libs/middlewares/track.ts +++ b/apps/server/src/libs/middlewares/track.ts @@ -9,6 +9,7 @@ import type { Context, Next } from "hono"; import { matchedRoutes } from "hono/route"; import { apiAnalyticsIdentity } from "@/libs/analytics-identity"; +import { apiRequestSampleRate } from "@/libs/api-request-sampling"; import { parseCliHeaders, trackCliCommand } from "@/libs/cli-telemetry"; import type { Variables } from "@/types"; @@ -53,7 +54,8 @@ export function trackMiddleware(event: EventProps, eventProps?: string[]) { * pattern (`GET /v1/monitor/:id`), never the raw URL, to keep it low-cardinality. * * Requests from the openstatus CLI also carry `cliCommand`/`cliVersion`, and - * the first request of each CLI run fires one `cli_command`. + * the first request of each CLI run fires one `cli_command`. CLI requests are + * always tracked; the rest are sampled (see `apiRequestSampleRate`). * * Mount after `authMiddleware`; requests it rejects carry no workspace and are * skipped, as are requests that match no route. Per-route domain events stay with `trackMiddleware`. @@ -72,6 +74,9 @@ export function apiTrackMiddleware() { if (!route) return; const cli = parseCliHeaders(c.req.raw.headers); + const sampleRate = apiRequestSampleRate(cli); + if (sampleRate === undefined) return; + const success = c.res.status.toString().startsWith("2") && !c.error; setupAnalytics(apiAnalyticsIdentity(workspace, c.req.raw.headers)) @@ -82,6 +87,7 @@ export function apiTrackMiddleware() { service: "v1", method: `${c.req.method} ${route.path}`, success, + sampleRate, ...(cli ? { cliCommand: cli.command } : {}), ...(cli?.version ? { cliVersion: cli.version } : {}), }), diff --git a/apps/server/src/routes/rpc/interceptors/__tests__/tracking.test.ts b/apps/server/src/routes/rpc/interceptors/__tests__/tracking.test.ts index baa056999..b803671ec 100644 --- a/apps/server/src/routes/rpc/interceptors/__tests__/tracking.test.ts +++ b/apps/server/src/routes/rpc/interceptors/__tests__/tracking.test.ts @@ -4,10 +4,13 @@ import { MonitorService } from "@openstatus/proto/monitor/v1"; // @ts-nocheck — ConnectRPC's deep generic types (AnyFn, UnaryResponse, etc.) // are incompatible with the test mocks. All runtime behavior is correct. import { + afterEach, beforeEach, describe, expect, mock, + type Stub, + stub, test, } from "@openstatus/test-utils"; @@ -84,9 +87,17 @@ function createMockRequest( } describe("trackingInterceptor", () => { + let random: Stub; + beforeEach(() => { mockSetupAnalytics.mockClear(); mockTrack.mockClear(); + // Keep every sampled request unless a test says otherwise. + random = stub(Math, "random", () => 0); + }); + + afterEach(() => { + random.restore(); }); test("tracks mapped event on success", async () => { @@ -121,6 +132,7 @@ describe("trackingInterceptor", () => { service: "openstatus.monitor.v1.MonitorService", method: "DeleteMonitor", success: true, + sampleRate: 0.1, }); expect(mockTrack).toHaveBeenCalledWith({ ...Events.DeleteMonitor, @@ -184,6 +196,7 @@ describe("trackingInterceptor", () => { service: "openstatus.monitor.v1.MonitorService", method: "ListMonitors", success: true, + sampleRate: 0.1, }); }); @@ -205,6 +218,7 @@ describe("trackingInterceptor", () => { service: "openstatus.monitor.v1.MonitorService", method: "DeleteMonitor", success: false, + sampleRate: 0.1, }); }); @@ -233,6 +247,7 @@ describe("trackingInterceptor", () => { service: "openstatus.monitor.v1.MonitorService", method: "ListMonitors", success: true, + sampleRate: 1, cliCommand: "monitors list", cliVersion: "1.3.2", }); @@ -250,6 +265,66 @@ describe("trackingInterceptor", () => { }); }); + test("drops non-CLI api_request outside the sample", async () => { + random.restore(); + random = stub(Math, "random", () => 0.5); + + const interceptor = trackingInterceptor(); + const req = createMockRequest( + "openstatus.monitor.v1.MonitorService", + "ListMonitors", + ); + + await interceptor(mockNext({}))(req as never); + await flush(); + + expect(mockSetupAnalytics).not.toHaveBeenCalled(); + expect(mockTrack).not.toHaveBeenCalled(); + }); + + test("keeps the domain event when api_request is sampled out", async () => { + random.restore(); + random = stub(Math, "random", () => 0.5); + + const interceptor = trackingInterceptor(); + const req = createMockRequest( + "openstatus.monitor.v1.MonitorService", + "DeleteMonitor", + ); + + await interceptor(mockNext({}))(req as never); + await flush(); + + expect(mockTrack).toHaveBeenCalledTimes(1); + expect(mockTrack).toHaveBeenCalledWith({ ...Events.DeleteMonitor }); + }); + + test("always tracks CLI api_request", async () => { + random.restore(); + random = stub(Math, "random", () => 0.99); + + const interceptor = trackingInterceptor(); + const req = createMockRequest( + "openstatus.monitor.v1.MonitorService", + "ListMonitors", + {}, + { + headers: { + "user-agent": "openstatus-cli/v1.3.2 (darwin; arm64)", + "x-openstatus-cli-command": "monitors list", + "x-openstatus-cli-invocation": crypto.randomUUID(), + }, + }, + ); + + await interceptor(mockNext({}))(req as never); + await flush(); + + expect(mockTrack).toHaveBeenCalledWith( + expect.objectContaining({ ...Events.ApiRequest, sampleRate: 1 }), + ); + }); + test("skips tracking when no RPC context", async () => { const interceptor = trackingInterceptor(); const req = createMockRequest( diff --git a/apps/server/src/routes/rpc/interceptors/tracking.ts b/apps/server/src/routes/rpc/interceptors/tracking.ts index 82438545e..7dba19698 100644 --- a/apps/server/src/routes/rpc/interceptors/tracking.ts +++ b/apps/server/src/routes/rpc/interceptors/tracking.ts @@ -8,6 +8,7 @@ import { } from "@openstatus/analytics"; import { apiAnalyticsIdentity } from "../../../libs/analytics-identity"; +import { apiRequestSampleRate } from "../../../libs/api-request-sampling"; import { parseCliHeaders, trackCliCommand } from "../../../libs/cli-telemetry"; import { RPC_CONTEXT_KEY } from "./auth"; @@ -171,6 +172,8 @@ export const RPC_EVENT_MAP: Record = { * * Requests from the openstatus CLI also carry `cliCommand`/`cliVersion` on * `api_request`, and the first request of each CLI run fires one `cli_command`. + * CLI requests always fire `api_request`; the rest are sampled (see + * `apiRequestSampleRate`). Domain events are never sampled. * * Must be placed after authInterceptor (needs workspace context) and * validationInterceptor (requests it rejects never reach here, so they are not @@ -209,17 +212,21 @@ function trackRpcCall(req: TrackedRequest, success: boolean) { const key = `${req.service.typeName}/${req.method.name}`; const mapping = success ? RPC_EVENT_MAP[key] : undefined; const cli = parseCliHeaders(req.header); + const sampleRate = apiRequestSampleRate(cli); - const events: (EventProps & Record)[] = [ - { + const events: (EventProps & Record)[] = []; + + if (sampleRate !== undefined) { + events.push({ ...Events.ApiRequest, service: req.service.typeName, method: req.method.name, success, + sampleRate, ...(cli ? { cliCommand: cli.command } : {}), ...(cli?.version ? { cliVersion: cli.version } : {}), - }, - ]; + }); + } if (mapping) { const input = mapping.normalizeInput?.(req.message) ?? req.message; @@ -227,6 +234,8 @@ function trackRpcCall(req: TrackedRequest, success: boolean) { events.push({ ...additionalProps, ...mapping.event }); } + if (events.length === 0) return; + setupAnalytics(apiAnalyticsIdentity(rpcCtx.workspace, req.header)) .then((analytics) => Promise.all([ diff --git a/deno.lock b/deno.lock index c4704b002..5cb6c59e7 100644 --- a/deno.lock +++ b/deno.lock @@ -58,6 +58,272 @@ "dependencies": [ "npm:turbo@2.11.2" ] + }, + "members": { + "apps/dashboard": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "apps/server": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "apps/status-page": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "apps/web": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/ai": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/api": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/assertions": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/db": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/emails": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/header-analysis": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/importers": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/notifications/base": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/notifications/bird-whatsapp": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/notifications/discord": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/notifications/google-chat": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/notifications/grafana-oncall": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/notifications/ms-teams": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/notifications/ntfy": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/notifications/opsgenie": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/notifications/pagerduty": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/notifications/slack": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/notifications/telegram": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/notifications/twillio-sms": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/notifications/webhook": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/services": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/status-fetcher": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/subscriptions": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/test-utils": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/theme-store": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/tinybird": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/tracker": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/upstash": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + }, + "packages/utils": { + "packageJson": { + "dependencies": [ + "npm:@jsr/std__expect@^1.0.19", + "npm:@jsr/std__testing@^1.0.19" + ] + } + } } } } -- 2.51.2