From 218fa6170abc5f2ac1165b46ec9c51810ce39cb3 Mon Sep 17 00:00:00 2001 From: Maximilian Kaske <56969857+mxkaske@users.noreply.github.com> Date: Fri, 20 Sep 2024 11:31:24 +0200 Subject: [PATCH] fix: timestamp and timing (#1009) * fix: timestamp and timing * ci: apply automated fixes * fix: test * ci: apply automated fixes * fix: struct --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- apps/checker/handlers/ping.go | 29 +++-- apps/server/src/v1/check/post.test.ts | 8 +- apps/server/src/v1/check/post.ts | 173 +++++++++++--------------- apps/server/src/v1/check/schema.ts | 2 +- 4 files changed, 98 insertions(+), 114 deletions(-) diff --git a/apps/checker/handlers/ping.go b/apps/checker/handlers/ping.go index a3913512b..5168b2903 100644 --- a/apps/checker/handlers/ping.go +++ b/apps/checker/handlers/ping.go @@ -14,15 +14,15 @@ import ( ) type PingResponse struct { - Body string `json:"body,omitempty"` - Headers string `json:"headers,omitempty"` - Region string `json:"region"` - RequestId int64 `json:"requestId,omitempty"` - WorkspaceId int64 `json:"workspaceId,omitempty"` - Latency int64 `json:"latency"` - Time int64 `json:"time"` - Status int `json:"status,omitempty"` - Timing checker.Timing `json:"timing"` + Body string `json:"body,omitempty"` + Headers string `json:"headers,omitempty"` + Region string `json:"region"` + RequestId int64 `json:"requestId,omitempty"` + WorkspaceId int64 `json:"workspaceId,omitempty"` + Latency int64 `json:"latency"` + Timestamp int64 `json:"timestamp"` + Status int `json:"status,omitempty"` + Timing string `json:"timing,omitempty"` } type Response struct { @@ -34,7 +34,7 @@ type Response struct { RequestId int64 `json:"requestId,omitempty"` WorkspaceId int64 `json:"workspaceId,omitempty"` Latency int64 `json:"latency"` - Time int64 `json:"time"` + Timestamp int64 `json:"timestamp"` Timing checker.Timing `json:"timing"` Status int `json:"status,omitempty"` } @@ -112,6 +112,11 @@ func (h Handler) PingRegionHandler(c *gin.Context) { return fmt.Errorf("unable to ping: %w", err) } + timingAsString, err := json.Marshal(r.Timing) + if err != nil { + return fmt.Errorf("error while parsing timing data %s: %w", req.URL, err) + } + headersAsString, err := json.Marshal(r.Headers) if err != nil { return nil @@ -124,8 +129,8 @@ func (h Handler) PingRegionHandler(c *gin.Context) { Latency: r.Latency, Body: r.Body, Headers: string(headersAsString), - Time: r.Timestamp, - Timing: r.Timing, + Timestamp: r.Timestamp, + Timing: string(timingAsString), Region: h.Region, } diff --git a/apps/server/src/v1/check/post.test.ts b/apps/server/src/v1/check/post.test.ts index 183a5b282..3006c4040 100644 --- a/apps/server/src/v1/check/post.test.ts +++ b/apps/server/src/v1/check/post.test.ts @@ -24,7 +24,7 @@ test("Create a single check ", async () => { mockFetch.mockReturnValue( Promise.resolve( new Response( - '{"status":200,"latency":100,"body":"Hello World","headers":{"Content-Type":"application/json"},"time":1234567890,"timing":{"dnsStart":1,"dnsDone":2,"connectStart":3,"connectDone":4,"tlsHandshakeStart":5,"tlsHandshakeDone":6,"firstByteStart":7,"firstByteDone":8,"transferStart":9,"transferDone":10},"region":"ams"}', + '{"status":200,"latency":100,"body":"Hello World","headers":{"Content-Type":"application/json"},"timestamp":1234567890,"timing":{"dnsStart":1,"dnsDone":2,"connectStart":3,"connectDone":4,"tlsHandshakeStart":5,"tlsHandshakeDone":6,"firstByteStart":7,"firstByteDone":8,"transferStart":9,"transferDone":10},"region":"ams"}', { status: 200, headers: { "content-type": "application/json" } }, ), ), @@ -65,7 +65,7 @@ test("Create a single check ", async () => { latency: 100, region: "ams", status: 200, - time: 1234567890, + timestamp: 1234567890, timing: { connectDone: 4, connectStart: 3, @@ -93,7 +93,7 @@ test.todo("Create a multiple check ", async () => { mockFetch.mockReturnValue( Promise.resolve( new Response( - '{"status":200,"latency":100,"body":"Hello World","headers":{"Content-Type":"application/json"},"time":1234567890,"timing":{"dnsStart":1,"dnsDone":2,"connectStart":3,"connectDone":4,"tlsHandshakeStart":5,"tlsHandshakeDone":6,"firstByteStart":7,"firstByteDone":8,"transferStart":9,"transferDone":10},"region":"ams"}', + '{"status":200,"latency":100,"body":"Hello World","headers":{"Content-Type":"application/json"},"timestamp":1234567890,"timing":{"dnsStart":1,"dnsDone":2,"connectStart":3,"connectDone":4,"tlsHandshakeStart":5,"tlsHandshakeDone":6,"firstByteStart":7,"firstByteDone":8,"transferStart":9,"transferDone":10},"region":"ams"}', { status: 200, headers: { "content-type": "application/json" } }, ), ), @@ -134,7 +134,7 @@ test.todo("Create a multiple check ", async () => { latency: 100, region: "ams", status: 200, - time: 1234567890, + timestamp: 1234567890, timing: { connectDone: 4, connectStart: 3, diff --git a/apps/server/src/v1/check/post.ts b/apps/server/src/v1/check/post.ts index 306e0d928..4badd35b7 100644 --- a/apps/server/src/v1/check/post.ts +++ b/apps/server/src/v1/check/post.ts @@ -1,4 +1,4 @@ -import { createRoute, z } from "@hono/zod-openapi"; +import { createRoute, type z } from "@hono/zod-openapi"; import { db } from "@openstatus/db"; import { check } from "@openstatus/db/src/schema/check"; @@ -60,7 +60,9 @@ export function registerPostCheck(api: typeof checkAPI) { }) .returning() .get(); + const result = []; + for (let count = 0; count < input.runCount; count++) { const currentFetch = []; for (const region of input.regions) { @@ -94,120 +96,46 @@ export function registerPostCheck(api: typeof checkAPI) { const allResults = await Promise.allSettled(currentFetch); result.push(...allResults); } + + const fulfilledRequest: z.infer[] = []; + const filteredResult = result.filter((r) => r.status === "fulfilled"); - const fulfilledRequest = []; for await (const r of filteredResult) { if (r.status !== "fulfilled") throw new Error("No value"); const json = await r.value.json(); - fulfilledRequest.push(ResponseSchema.parse(json)); + const parsed = ResponseSchema.safeParse(json); + + if (!parsed.success) { + console.error(parsed.error.errors); + throw new Error(`Failed to parse response: ${parsed.error.errors}`); + } + + fulfilledRequest.push(parsed.data); } let aggregatedResponse = null; - if (aggregated) { - // This is ugly - const dnsArray = fulfilledRequest.map( - (r) => r.timing.dnsDone - r.timing.dnsStart, - ); - const connectArray = fulfilledRequest.map( - (r) => r.timing.connectDone - r.timing.connectStart, - ); - const tlsArray = fulfilledRequest.map( - (r) => r.timing.tlsHandshakeDone - r.timing.tlsHandshakeStart, - ); - const firstArray = fulfilledRequest.map( - (r) => r.timing.firstByteDone - r.timing.firstByteStart, - ); - const transferArray = fulfilledRequest.map( - (r) => r.timing.transferDone - r.timing.transferStart, - ); - const latencyArray = fulfilledRequest.map((r) => r.latency); - - const dnsPercentile = percentile([50, 75, 95, 99], dnsArray) as number[]; - const connectPercentile = percentile( - [50, 75, 95, 99], - connectArray, - ) as number[]; - const tlsPercentile = percentile([50, 75, 95, 99], tlsArray) as number[]; - const firstPercentile = percentile( - [50, 75, 95, 99], - firstArray, - ) as number[]; - - const transferPercentile = percentile( - [50, 75, 95, 99], - transferArray, - ) as number[]; - const latencyPercentile = percentile( - [50, 75, 95, 99], - latencyArray, - ) as number[]; - - const aggregatedDNS = AggregatedResponseSchema.parse({ - p50: dnsPercentile[0], - p75: dnsPercentile[1], - p95: dnsPercentile[2], - p99: dnsPercentile[3], - min: Math.min(...dnsArray), - max: Math.max(...dnsArray), - }); - const aggregatedConnect = AggregatedResponseSchema.parse({ - p50: connectPercentile[0], - p75: connectPercentile[1], - p95: connectPercentile[2], - p99: connectPercentile[3], - min: Math.min(...connectArray), - max: Math.max(...connectArray), - }); - const aggregatedTls = AggregatedResponseSchema.parse({ - p50: tlsPercentile[0], - p75: tlsPercentile[1], - p95: tlsPercentile[2], - p99: tlsPercentile[3], - min: Math.min(...tlsArray), - max: Math.max(...tlsArray), - }); - const aggregatedFirst = AggregatedResponseSchema.parse({ - p50: firstPercentile[0], - p75: firstPercentile[1], - p95: firstPercentile[2], - p99: firstPercentile[3], - min: Math.min(...firstArray), - max: Math.max(...firstArray), - }); - const aggregatedTransfer = AggregatedResponseSchema.parse({ - p50: transferPercentile[0], - p75: transferPercentile[1], - p95: transferPercentile[2], - p99: transferPercentile[3], - min: Math.min(...transferArray), - max: Math.max(...transferArray), - }); - const aggregatedLatency = AggregatedResponseSchema.parse({ - p50: latencyPercentile[0], - p75: latencyPercentile[1], - p95: latencyPercentile[2], - p99: latencyPercentile[3], - min: Math.min(...latencyArray), - max: Math.max(...latencyArray), - }); + if (aggregated) { + const { dns, connect, tls, firstByte, transfer, latency } = + getTiming(fulfilledRequest); aggregatedResponse = AggregatedResult.parse({ - dns: aggregatedDNS, - connect: aggregatedConnect, - tls: aggregatedTls, - firstByte: aggregatedFirst, - transfer: aggregatedTransfer, - latency: aggregatedLatency, + dns: getAggregate(dns), + connect: getAggregate(connect), + tls: getAggregate(tls), + firstByte: getAggregate(firstByte), + transfer: getAggregate(transfer), + latency: getAggregate(latency), }); } + const allTimings = fulfilledRequest.map((r) => r.timing); const lastResponse = fulfilledRequest[fulfilledRequest.length - 1]; const responseResult = CheckPostResponseSchema.parse({ id: newCheck.id, - raw: allTimings, + raw: allTimings, // TODO: we should return the region here as well! response: lastResponse, aggregated: aggregatedResponse ? aggregatedResponse : undefined, }); @@ -215,3 +143,54 @@ export function registerPostCheck(api: typeof checkAPI) { return c.json(responseResult, 200); }); } + +// This is a helper function to get the timing of the request + +type ReturnGetTiming = Record< + "dns" | "connect" | "tls" | "firstByte" | "transfer" | "latency", + number[] +>; + +function getTiming(data: z.infer[]): ReturnGetTiming { + return data.reduce( + (prev, curr) => { + prev.dns.push(curr.timing.dnsDone - curr.timing.dnsStart); + prev.connect.push(curr.timing.connectDone - curr.timing.connectStart); + prev.tls.push( + curr.timing.tlsHandshakeDone - curr.timing.tlsHandshakeStart, + ); + prev.firstByte.push( + curr.timing.firstByteDone - curr.timing.firstByteStart, + ); + prev.transfer.push(curr.timing.transferDone - curr.timing.transferStart); + prev.latency.push(curr.latency); + return prev; + }, + { + dns: [], + connect: [], + tls: [], + firstByte: [], + transfer: [], + latency: [], + } as ReturnGetTiming, + ); +} + +function getAggregate(data: number[]) { + const parsed = AggregatedResponseSchema.safeParse({ + p50: percentile(50, data), + p75: percentile(75, data), + p95: percentile(95, data), + p99: percentile(99, data), + min: Math.min(...data), + max: Math.max(...data), + }); + + if (!parsed.success) { + console.error(parsed.error.errors); + throw new Error(`Failed to parse response: ${parsed.error.errors}`); + } + + return parsed.data; +} diff --git a/apps/server/src/v1/check/schema.ts b/apps/server/src/v1/check/schema.ts index cb78e2cab..d6aadf81e 100644 --- a/apps/server/src/v1/check/schema.ts +++ b/apps/server/src/v1/check/schema.ts @@ -75,7 +75,7 @@ export const AggregatedResponseSchema = z }); export const ResponseSchema = z.object({ - time: z + timestamp: z .number() .openapi({ description: "The timestamp of the response in UTC" }), status: z -- 2.54.0