From a78e339c0c797a562f48e14d83df24230dec96ca Mon Sep 17 00:00:00 2001 From: Maximilian Kaske Date: Thu, 15 Oct 2026 13:50:10 +0200 Subject: [PATCH] fix(tinybird): return the local calendar date from the hourly status pipes - `day` is now a `YYYY-MM-DD` date re-keyed as UTC midnight, the convention the 45d pipes and every day-keyed consumer already use, instead of the epoch-ms instant of local midnight that shifted non-UTC days backwards - enforce the whole-hour timezone rule in the endpoints too (January and July offsets), not only in the TypeScript client - share `status60mParameters`/`status60mShape` across the five getters and memoize the zone check - test the schemas directly; extend the server tinybird double --- .../src/libs/test/doubles/tinybird.mock.ts | 15 ++ .../endpoint__dns_status_60m__v0.pipe | 15 +- .../endpoint__grpc_status_60m__v0.pipe | 15 +- .../endpoint__http_status_60m__v0.pipe | 15 +- .../endpoint__icmp_status_60m__v0.pipe | 15 +- .../endpoint__tcp_status_60m__v0.pipe | 15 +- packages/tinybird/src/client.test.ts | 140 ++++++++++++++---- packages/tinybird/src/client.ts | 128 +++++++--------- 8 files changed, 242 insertions(+), 116 deletions(-) diff --git a/apps/server/src/libs/test/doubles/tinybird.mock.ts b/apps/server/src/libs/test/doubles/tinybird.mock.ts index 2cf71dab6..a0aa672ad 100644 --- a/apps/server/src/libs/test/doubles/tinybird.mock.ts +++ b/apps/server/src/libs/test/doubles/tinybird.mock.ts @@ -17,6 +17,21 @@ export class OSTinybird { get legacy_tcpStatus45d() { return empty; } + get httpStatus60m() { + return empty; + } + get tcpStatus60m() { + return empty; + } + get dnsStatus60m() { + return empty; + } + get icmpStatus60m() { + return empty; + } + get grpcStatus60m() { + return empty; + } get httpListBiweekly() { return (params: unknown) => { calls.httpListBiweekly.push(params); diff --git a/packages/tinybird/endpoints/endpoint__dns_status_60m__v0.pipe b/packages/tinybird/endpoints/endpoint__dns_status_60m__v0.pipe index 573964394..a7673fc10 100644 --- a/packages/tinybird/endpoints/endpoint__dns_status_60m__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__dns_status_60m__v0.pipe @@ -2,10 +2,10 @@ NODE endpoint SQL > % +WITH {{ String(timezone, 'UTC') }} AS tz SELECT - -- epoch ms of local midnight in the requested zone: an unambiguous instant, - -- unlike a zone-less "YYYY-MM-DD 00:00:00" string the client would misread as UTC - toUnixTimestamp(toStartOfDay(toTimeZone(time, {{ String(timezone, 'UTC') }}))) * 1000 AS day, + -- calendar date of the local day; the client re-keys it as UTC midnight like the 45d pipes + toDate(toTimeZone(time, tz)) AS day, monitorId, countMerge(count) AS count, countMerge(success) AS ok, @@ -13,6 +13,15 @@ SELECT countMerge(degraded) AS degraded FROM mv__dns_status_60m__v0 WHERE monitorId IN {{ Array(monitorIds, 'String', '1,666') }} + -- hourly buckets only regroup cleanly into days when every offset the zone + -- uses is a whole hour; January and July cover both sides of DST + AND ( + SELECT throwIf( + timeZoneOffset(toTimeZone(makeDateTime(toYear(now()), 1, 1, 0, 0, 0), tz)) % 3600 != 0 + OR timeZoneOffset(toTimeZone(makeDateTime(toYear(now()), 7, 1, 0, 0, 0), tz)) % 3600 != 0, + 'timezone must be an IANA name whose UTC offset is a whole hour' + ) + ) = 0 GROUP BY day, monitorId ORDER BY day DESC diff --git a/packages/tinybird/endpoints/endpoint__grpc_status_60m__v0.pipe b/packages/tinybird/endpoints/endpoint__grpc_status_60m__v0.pipe index 3b8e297da..b14797a31 100644 --- a/packages/tinybird/endpoints/endpoint__grpc_status_60m__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__grpc_status_60m__v0.pipe @@ -2,10 +2,10 @@ NODE endpoint SQL > % +WITH {{ String(timezone, 'UTC') }} AS tz SELECT - -- epoch ms of local midnight in the requested zone: an unambiguous instant, - -- unlike a zone-less "YYYY-MM-DD 00:00:00" string the client would misread as UTC - toUnixTimestamp(toStartOfDay(toTimeZone(time, {{ String(timezone, 'UTC') }}))) * 1000 AS day, + -- calendar date of the local day; the client re-keys it as UTC midnight like the 45d pipes + toDate(toTimeZone(time, tz)) AS day, monitorId, countMerge(count) AS count, countMerge(success) AS ok, @@ -13,6 +13,15 @@ SELECT countMerge(degraded) AS degraded FROM mv__grpc_status_60m__v0 WHERE monitorId IN {{ Array(monitorIds, 'String', '1,666') }} + -- hourly buckets only regroup cleanly into days when every offset the zone + -- uses is a whole hour; January and July cover both sides of DST + AND ( + SELECT throwIf( + timeZoneOffset(toTimeZone(makeDateTime(toYear(now()), 1, 1, 0, 0, 0), tz)) % 3600 != 0 + OR timeZoneOffset(toTimeZone(makeDateTime(toYear(now()), 7, 1, 0, 0, 0), tz)) % 3600 != 0, + 'timezone must be an IANA name whose UTC offset is a whole hour' + ) + ) = 0 GROUP BY day, monitorId ORDER BY day DESC diff --git a/packages/tinybird/endpoints/endpoint__http_status_60m__v0.pipe b/packages/tinybird/endpoints/endpoint__http_status_60m__v0.pipe index cfecddcfd..06c566d5f 100644 --- a/packages/tinybird/endpoints/endpoint__http_status_60m__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__http_status_60m__v0.pipe @@ -2,10 +2,10 @@ NODE endpoint SQL > % +WITH {{ String(timezone, 'UTC') }} AS tz SELECT - -- epoch ms of local midnight in the requested zone: an unambiguous instant, - -- unlike a zone-less "YYYY-MM-DD 00:00:00" string the client would misread as UTC - toUnixTimestamp(toStartOfDay(toTimeZone(time, {{ String(timezone, 'UTC') }}))) * 1000 AS day, + -- calendar date of the local day; the client re-keys it as UTC midnight like the 45d pipes + toDate(toTimeZone(time, tz)) AS day, monitorId, countMerge(count) AS count, countMerge(success) AS ok, @@ -13,6 +13,15 @@ SELECT countMerge(degraded) AS degraded FROM mv__http_status_60m__v0 WHERE monitorId IN {{ Array(monitorIds, 'String', '1,666') }} + -- hourly buckets only regroup cleanly into days when every offset the zone + -- uses is a whole hour; January and July cover both sides of DST + AND ( + SELECT throwIf( + timeZoneOffset(toTimeZone(makeDateTime(toYear(now()), 1, 1, 0, 0, 0), tz)) % 3600 != 0 + OR timeZoneOffset(toTimeZone(makeDateTime(toYear(now()), 7, 1, 0, 0, 0), tz)) % 3600 != 0, + 'timezone must be an IANA name whose UTC offset is a whole hour' + ) + ) = 0 GROUP BY day, monitorId ORDER BY day DESC diff --git a/packages/tinybird/endpoints/endpoint__icmp_status_60m__v0.pipe b/packages/tinybird/endpoints/endpoint__icmp_status_60m__v0.pipe index 6ba14da90..61e454ee0 100644 --- a/packages/tinybird/endpoints/endpoint__icmp_status_60m__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__icmp_status_60m__v0.pipe @@ -2,10 +2,10 @@ NODE endpoint SQL > % +WITH {{ String(timezone, 'UTC') }} AS tz SELECT - -- epoch ms of local midnight in the requested zone: an unambiguous instant, - -- unlike a zone-less "YYYY-MM-DD 00:00:00" string the client would misread as UTC - toUnixTimestamp(toStartOfDay(toTimeZone(time, {{ String(timezone, 'UTC') }}))) * 1000 AS day, + -- calendar date of the local day; the client re-keys it as UTC midnight like the 45d pipes + toDate(toTimeZone(time, tz)) AS day, monitorId, countMerge(count) AS count, countMerge(success) AS ok, @@ -13,6 +13,15 @@ SELECT countMerge(degraded) AS degraded FROM mv__icmp_status_60m__v0 WHERE monitorId IN {{ Array(monitorIds, 'String', '1,666') }} + -- hourly buckets only regroup cleanly into days when every offset the zone + -- uses is a whole hour; January and July cover both sides of DST + AND ( + SELECT throwIf( + timeZoneOffset(toTimeZone(makeDateTime(toYear(now()), 1, 1, 0, 0, 0), tz)) % 3600 != 0 + OR timeZoneOffset(toTimeZone(makeDateTime(toYear(now()), 7, 1, 0, 0, 0), tz)) % 3600 != 0, + 'timezone must be an IANA name whose UTC offset is a whole hour' + ) + ) = 0 GROUP BY day, monitorId ORDER BY day DESC diff --git a/packages/tinybird/endpoints/endpoint__tcp_status_60m__v0.pipe b/packages/tinybird/endpoints/endpoint__tcp_status_60m__v0.pipe index 3f7e50df6..df0bfe047 100644 --- a/packages/tinybird/endpoints/endpoint__tcp_status_60m__v0.pipe +++ b/packages/tinybird/endpoints/endpoint__tcp_status_60m__v0.pipe @@ -2,10 +2,10 @@ NODE endpoint SQL > % +WITH {{ String(timezone, 'UTC') }} AS tz SELECT - -- epoch ms of local midnight in the requested zone: an unambiguous instant, - -- unlike a zone-less "YYYY-MM-DD 00:00:00" string the client would misread as UTC - toUnixTimestamp(toStartOfDay(toTimeZone(time, {{ String(timezone, 'UTC') }}))) * 1000 AS day, + -- calendar date of the local day; the client re-keys it as UTC midnight like the 45d pipes + toDate(toTimeZone(time, tz)) AS day, monitorId, countMerge(count) AS count, countMerge(success) AS ok, @@ -13,6 +13,15 @@ SELECT countMerge(degraded) AS degraded FROM mv__tcp_status_60m__v0 WHERE monitorId IN {{ Array(monitorIds, 'String', '1,666') }} + -- hourly buckets only regroup cleanly into days when every offset the zone + -- uses is a whole hour; January and July cover both sides of DST + AND ( + SELECT throwIf( + timeZoneOffset(toTimeZone(makeDateTime(toYear(now()), 1, 1, 0, 0, 0), tz)) % 3600 != 0 + OR timeZoneOffset(toTimeZone(makeDateTime(toYear(now()), 7, 1, 0, 0, 0), tz)) % 3600 != 0, + 'timezone must be an IANA name whose UTC offset is a whole hour' + ) + ) = 0 GROUP BY day, monitorId ORDER BY day DESC diff --git a/packages/tinybird/src/client.test.ts b/packages/tinybird/src/client.test.ts index 826228e0c..f6f156c23 100644 --- a/packages/tinybird/src/client.test.ts +++ b/packages/tinybird/src/client.test.ts @@ -1,7 +1,14 @@ import { expect } from "@std/expect"; import { describe, test } from "@std/testing/bdd"; +import { z } from "zod"; -import { noopFlagSchema, OSTinybird } from "./client"; +import { + isWholeHourTimeZone, + noopFlagSchema, + OSTinybird, + status60mParameters, + status60mShape, +} from "./client"; describe("noopFlagSchema", () => { test("treats the string 'false' as disabled, not as a truthy string", () => { @@ -165,38 +172,117 @@ const status60mGetters = [ "grpcStatus60m", ] as const; -describe("hourly status pipes", () => { - const tb = new OSTinybird({ token: "a-token" }); +describe("isWholeHourTimeZone", () => { + test("accepts zones whose standard and DST offsets are both whole hours", () => { + for (const tz of ["UTC", "Etc/GMT+1", "Asia/Tokyo", "Europe/Berlin"]) { + expect(isWholeHourTimeZone(tz)).toBe(true); + } + expect(isWholeHourTimeZone("America/New_York")).toBe(true); + expect(isWholeHourTimeZone("Australia/Sydney")).toBe(true); + }); - for (const name of status60mGetters) { - for (const timezone of [ - "Asia/Kolkata", - "Asia/Kathmandu", - "Australia/Lord_Howe", - "Mars/Olympus", + test("rejects half- and quarter-hour offsets, on either side of DST", () => { + expect(isWholeHourTimeZone("Asia/Kolkata")).toBe(false); + expect(isWholeHourTimeZone("Asia/Kathmandu")).toBe(false); + expect(isWholeHourTimeZone("Pacific/Chatham")).toBe(false); + // +10:30 standard, +11:00 daylight: whole-hour for half the year only + expect(isWholeHourTimeZone("Australia/Lord_Howe")).toBe(false); + }); + + test("rejects unknown zones and fixed offsets instead of throwing", () => { + expect(isWholeHourTimeZone("Mars/Olympus")).toBe(false); + expect(isWholeHourTimeZone("")).toBe(false); + }); +}); + +describe("status60mParameters", () => { + test("defaults timezone to UTC", () => { + expect(status60mParameters.parse({ monitorIds: ["1"] })).toEqual({ + monitorIds: ["1"], + timezone: "UTC", + }); + }); + + test("forwards a whole-hour timezone untouched", () => { + expect( + status60mParameters.parse({ + monitorIds: ["1", "2"], + timezone: "Europe/Berlin", + }).timezone, + ).toBe("Europe/Berlin"); + }); + + test("names the whole-hour requirement when rejecting", () => { + const result = status60mParameters.safeParse({ + monitorIds: ["1"], + timezone: "Asia/Kolkata", + }); + expect(result.success).toBe(false); + expect(JSON.stringify(result.error?.issues)).toContain("whole hour"); + }); +}); + +describe("status60mShape", () => { + const shape = status60mShape(z.coerce.string()); + + test("re-keys the local calendar date as UTC midnight, like the 45d pipes", () => { + const row = shape.parse({ + day: "2026-10-14", + count: 10, + ok: 9, + degraded: 0, + error: 1, + monitorId: 1, + }); + expect(row.day).toBe("2026-10-14T00:00:00.000Z"); + // the key every day-indexed consumer derives must be the local date itself + expect(new Date(row.day).toISOString().split("T")[0]).toBe("2026-10-14"); + expect(row.monitorId).toBe("1"); + }); + + test("rejects a datetime or epoch `day`, which would re-introduce the UTC shift", () => { + for (const day of [ + "2026-10-13 22:00:00", + "2026-10-13T22:00:00.000Z", + 1760392800000, ]) { - test(`${name} rejects ${timezone}, whose days don't start on a UTC hour`, async () => { - await expect( - tb[name]({ monitorIds: ["1"], timezone }), - ).rejects.toThrow(); - }); + expect( + shape.safeParse({ + day, + count: 0, + ok: 0, + degraded: 0, + error: 0, + monitorId: "1", + }).success, + ).toBe(false); } + }); - test(`${name} accepts whole-hour timezones, with or without DST`, async () => { - for (const timezone of [ - "UTC", - "Asia/Tokyo", - "Europe/Berlin", - "America/New_York", - ]) { - expect(await tb[name]({ monitorIds: ["1"], timezone })).toEqual({ - meta: [], - data: [], - }); - } + test("defaults missing counters to 0", () => { + expect(shape.parse({ day: "2026-01-01", monitorId: "1" })).toMatchObject({ + count: 0, + ok: 0, + degraded: 0, + error: 0, }); + }); +}); - test(`${name} defaults to UTC`, async () => { +describe("hourly status pipes", () => { + const tb = new OSTinybird({ token: "a-token" }); + + for (const name of status60mGetters) { + test(`${name} validates timezone before the request`, async () => { + await expect( + tb[name]({ monitorIds: ["1"], timezone: "Asia/Kolkata" }), + ).rejects.toThrow("whole hour"); + }); + + test(`${name} resolves for whole-hour zones and when timezone is omitted`, async () => { + expect( + await tb[name]({ monitorIds: ["1"], timezone: "Europe/Berlin" }), + ).toEqual({ meta: [], data: [] }); expect(await tb[name]({ monitorIds: ["1"] })).toEqual({ meta: [], data: [], diff --git a/packages/tinybird/src/client.ts b/packages/tinybird/src/client.ts index 3b6ce9483..bce249a11 100644 --- a/packages/tinybird/src/client.ts +++ b/packages/tinybird/src/client.ts @@ -16,8 +16,13 @@ const REVALIDATE = env.NODE_ENV === "development" ? DEV_CACHE : 0; // The 60m pipes group UTC-hour buckets into local days, which only lines up // when every offset the zone uses is a whole number of hours. January and -// July cover both sides of DST. -function isWholeHourTimeZone(timeZone: string): boolean { +// July cover both sides of DST. The endpoints enforce the same rule server-side. +const wholeHourTimeZoneCache = new Map(); + +export function isWholeHourTimeZone(timeZone: string): boolean { + const cached = wholeHourTimeZoneCache.get(timeZone); + if (cached !== undefined) return cached; + let format: Intl.DateTimeFormat; try { format = new Intl.DateTimeFormat("en-US", { @@ -29,20 +34,45 @@ function isWholeHourTimeZone(timeZone: string): boolean { throw error; } const year = new Date().getUTCFullYear(); - return [0, 6].every((month) => { + const result = [0, 6].every((month) => { const offset = format .formatToParts(new Date(Date.UTC(year, month, 1))) .find((p) => p.type === "timeZoneName")?.value; return offset === "GMT" || /^GMT[+-]\d{2}:00$/.test(offset ?? ""); }); + wholeHourTimeZoneCache.set(timeZone, result); + return result; } -const wholeHourTimeZone = z - .string() - .refine( - isWholeHourTimeZone, - "timezone must be an IANA name whose UTC offset is a whole hour", - ); +export const status60mParameters = z.object({ + monitorIds: z.string().array(), + timezone: z + .string() + .refine( + isWholeHourTimeZone, + "timezone must be an IANA name whose UTC offset is a whole hour", + ) + .prefault("UTC"), +}); + +/** + * Same columns as the 45d pipes. `day` is the local calendar date re-keyed as + * UTC midnight, the convention day-keyed consumers already read — a Berlin + * 2026-10-14 stays "2026-10-14", not the 22:00Z instant of local midnight. + */ +export function status60mShape(monitorId: z.ZodType) { + return z.object({ + day: z + .string() + .regex(/^\d{4}-\d{2}-\d{2}$/) + .transform((val) => new Date(`${val}T00:00:00.000Z`).toISOString()), + count: z.number().prefault(0), + ok: z.number().prefault(0), + degraded: z.number().prefault(0), + error: z.number().prefault(0), + monitorId, + }); +} // Daily-aggregate columns shared by the external-status history pipes (page- // level and per-component). One shape so the `day` GMT-parse can't drift. @@ -837,26 +867,12 @@ export class OSTinybird { }); } - /** - * Hourly status buckets grouped into days of `timezone` at read time. Returns - * the same columns as the 45d pipe; `day` arrives as epoch ms of the local - * midnight (an instant), not a zone-less date string. - */ + /** Hourly buckets regrouped into days of `timezone`; see {@link status60mShape}. */ public get httpStatus60m() { return this.tb.buildPipe({ pipe: "endpoint__http_status_60m__v0", - parameters: z.object({ - monitorIds: z.string().array(), - timezone: wholeHourTimeZone.prefault("UTC"), - }), - data: z.object({ - day: z.number().transform((ms) => new Date(ms).toISOString()), - count: z.number().prefault(0), - ok: z.number().prefault(0), - degraded: z.number().prefault(0), - error: z.number().prefault(0), - monitorId: z.string(), - }), + parameters: status60mParameters, + data: status60mShape(z.string()), opts: { next: { revalidate: REVALIDATE } }, }); } @@ -1428,21 +1444,12 @@ export class OSTinybird { }); } + /** Hourly buckets regrouped into days of `timezone`; see {@link status60mShape}. */ public get tcpStatus60m() { return this.tb.buildPipe({ pipe: "endpoint__tcp_status_60m__v0", - parameters: z.object({ - monitorIds: z.string().array(), - timezone: wholeHourTimeZone.prefault("UTC"), - }), - data: z.object({ - day: z.number().transform((ms) => new Date(ms).toISOString()), - count: z.number().prefault(0), - ok: z.number().prefault(0), - degraded: z.number().prefault(0), - error: z.number().prefault(0), - monitorId: z.coerce.string(), - }), + parameters: status60mParameters, + data: status60mShape(z.coerce.string()), opts: { next: { revalidate: REVALIDATE } }, }); } @@ -1847,21 +1854,12 @@ export class OSTinybird { }); } + /** Hourly buckets regrouped into days of `timezone`; see {@link status60mShape}. */ public get icmpStatus60m() { return this.tb.buildPipe({ pipe: "endpoint__icmp_status_60m__v0", - parameters: z.object({ - monitorIds: z.string().array(), - timezone: wholeHourTimeZone.prefault("UTC"), - }), - data: z.object({ - day: z.number().transform((ms) => new Date(ms).toISOString()), - count: z.number().prefault(0), - ok: z.number().prefault(0), - degraded: z.number().prefault(0), - error: z.number().prefault(0), - monitorId: z.coerce.string(), - }), + parameters: status60mParameters, + data: status60mShape(z.coerce.string()), opts: { next: { revalidate: REVALIDATE } }, }); } @@ -2337,21 +2335,12 @@ export class OSTinybird { }); } + /** Hourly buckets regrouped into days of `timezone`; see {@link status60mShape}. */ public get grpcStatus60m() { return this.tb.buildPipe({ pipe: "endpoint__grpc_status_60m__v0", - parameters: z.object({ - monitorIds: z.string().array(), - timezone: wholeHourTimeZone.prefault("UTC"), - }), - data: z.object({ - day: z.number().transform((ms) => new Date(ms).toISOString()), - count: z.number().prefault(0), - ok: z.number().prefault(0), - degraded: z.number().prefault(0), - error: z.number().prefault(0), - monitorId: z.coerce.string(), - }), + parameters: status60mParameters, + data: status60mShape(z.coerce.string()), opts: { next: { revalidate: REVALIDATE } }, }); } @@ -3642,21 +3631,12 @@ export class OSTinybird { }); } + /** Hourly buckets regrouped into days of `timezone`; see {@link status60mShape}. */ public get dnsStatus60m() { return this.tb.buildPipe({ pipe: "endpoint__dns_status_60m__v0", - parameters: z.object({ - monitorIds: z.string().array(), - timezone: wholeHourTimeZone.prefault("UTC"), - }), - data: z.object({ - day: z.number().transform((ms) => new Date(ms).toISOString()), - count: z.number().prefault(0), - ok: z.number().prefault(0), - degraded: z.number().prefault(0), - error: z.number().prefault(0), - monitorId: z.coerce.string(), - }), + parameters: status60mParameters, + data: status60mShape(z.coerce.string()), opts: { next: { revalidate: REVALIDATE } }, }); } -- 2.54.0