Something went wrong. Try again.
[READ-ONLY] Mirror of https://github.com/openstatusHQ/openstatus. 🫖 Status page with uptime monitoring & API monitoring as code 🫖 openstatus.dev
bun drizzle-orm monitoring monitoring-as-code nextjs observability on-call open-source shadcn-ui status-page statuspage synthetic-monitoring tinybird turso uptime uptime-checker uptime-monitor
Something went wrong. Try again.
9.0 kB · 286 lines
TypeScript
at incident-api
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287import { and, db as defaultDb, eq, inArray, isNull } from "@openstatus/db";import { frozenMonitorUptime, monitor, selectWorkspaceSchema, workspace,} from "@openstatus/db/src/schema";
import type { DB } from "../context";import { type ComputeCountRow, computeMonitorMonth, monthRange, previousMonth,} from "./compute";import { freezeMonitorMonth } from "./freeze";
export type StatusPipeFn = (params: { monitorIds: string[];}) => Promise<{ data: ComputeCountRow[] }>;
// only these job types have a 45d status pipe; others (udp/ssl) have no// counts on the live status page either and are skippedexport type UptimeFreezePipes = Record< "http" | "tcp" | "dns" | "icmp" | "grpc", StatusPipeFn>;
export type ChunkFailure = { jobType: string; monitorIds: string[]; error: unknown;};
// zod-bird sends monitorIds as one comma-joined GET query param — unbounded// batches risk URL-length 400sconst TB_CHUNK_SIZE = 200;const TB_ATTEMPTS = 3;// spacing between chunk requests: keeps the monthly sweep from bursting TB// (zod-bird already absorbs 429/5xx with its own internal retries)const TB_THROTTLE_MS = 250;
// the status pipes look back a fixed 45 days; past monthStart + 45d the// earliest month days return no rows and would freeze as permanent zerosexport const FREEZE_CUTOFF_MS = 45 * 86_400_000;
function chunk<T>(items: T[], size: number): T[][] { if (size <= 0) throw new Error(`chunk size must be positive, got ${size}`); const out: T[][] = []; for (let i = 0; i < items.length; i += size) { out.push(items.slice(i, i + size)); } return out;}
function hasStatusPipe( jobType: string | null | undefined,): jobType is keyof UptimeFreezePipes { return ( jobType === "http" || jobType === "tcp" || jobType === "dns" || jobType === "icmp" || jobType === "grpc" );}
/** * Fetch daily counts for all monitors, chunked and retried. Monitors whose * chunk still fails after retries land in `failedMonitorIds` — they are * skipped (never frozen with silent zeros) and picked up by a re-run. */export async function fetchFreezeCounts(args: { monitorIdsByJobType: Map<string, Set<string>>; pipes: UptimeFreezePipes; chunkSize?: number; attempts?: number; throttleMs?: number; sleep?: (ms: number) => Promise<void>; onChunkFailure?: (failure: ChunkFailure) => void;}): Promise<{ counts: ComputeCountRow[]; failedMonitorIds: Set<string> }> { const chunkSize = args.chunkSize ?? TB_CHUNK_SIZE; const attempts = args.attempts ?? TB_ATTEMPTS; const throttleMs = args.throttleMs ?? TB_THROTTLE_MS; const sleep = args.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)));
const counts: ComputeCountRow[] = []; const failedMonitorIds = new Set<string>(); let firstChunk = true;
for (const [jobType, ids] of args.monitorIdsByJobType) { if (!hasStatusPipe(jobType)) continue; const pipe = args.pipes[jobType]; for (const monitorIds of chunk([...ids], chunkSize)) { if (!firstChunk && throttleMs > 0) await sleep(throttleMs); firstChunk = false; let lastError: unknown; let done = false; for (let attempt = 0; attempt < attempts && !done; attempt++) { try { if (attempt > 0) await sleep(1000 * 2 ** (attempt - 1)); const res = await pipe({ monitorIds }); counts.push(...res.data); done = true; } catch (e) { lastError = e; } } if (!done) { for (const id of monitorIds) failedMonitorIds.add(id); args.onChunkFailure?.({ jobType, monitorIds, error: lastError }); } } }
return { counts, failedMonitorIds };}
export type RunUptimeFreezeResult = { month: string; frozen: number; alreadyFrozen: number; skipped: number; failures: string[];};
/** * Freeze the previous calendar month for every non-deleted monitor. * Pipes (and optionally db/now) are injected so the cron stays a thin * env/Sentry wrapper and this orchestration is unit-testable. */export async function runUptimeFreeze(args: { pipes: UptimeFreezePipes; now?: Date; db?: DB; monitorIds?: number[]; // scope a run (tests, manual retries); default all sleep?: (ms: number) => Promise<void>; onChunkFailure?: (failure: ChunkFailure) => void;}): Promise<RunUptimeFreezeResult> { const db = args.db ?? defaultDb; const now = args.now ?? new Date(); const month = previousMonth(now); const monthStart = new Date(monthRange(month).start);
// refuse instead of writing wrong zeros into the write-once table; the // failure surfaces via the cron's reportBackgroundError if (now.getTime() - monthStart.getTime() >= FREEZE_CUTOFF_MS) { return { month, frozen: 0, alreadyFrozen: 0, skipped: 0, failures: [ `month ${month}: past the 45d Tinybird window — freezing now would write permanent zero-count days`, ], }; }
// all non-deleted monitors, regardless of page attachment — months without // counts are dropped by computeMonitorMonth anyway const monitorRows = await db .select({ id: monitor.id, workspaceId: monitor.workspaceId, jobType: monitor.jobType, active: monitor.active, updatedAt: monitor.updatedAt, }) .from(monitor) .where( and( isNull(monitor.deletedAt), args.monitorIds ? inArray(monitor.id, args.monitorIds) : undefined, ), ); // inactive + untouched since before the month ⇒ it was inactive the whole // month (pausing bumps updatedAt), so skip the TB fetch entirely; inactive // but updated during/after the month may have run part of it — check TB const monitors = monitorRows.flatMap((m) => m.workspaceId === null || (!m.active && m.updatedAt !== null && m.updatedAt < monthStart) ? [] : [{ ...m, workspaceId: m.workspaceId }], );
// re-runs only work the monitors that failed last time: those with a frozen // row for this month are excluded before any TB fetch (the insert's // onConflictDoNothing stays as the race backstop) const frozenRows = await db .select({ monitorId: frozenMonitorUptime.monitorId }) .from(frozenMonitorUptime) .where( args.monitorIds ? and( eq(frozenMonitorUptime.month, month), inArray(frozenMonitorUptime.monitorId, args.monitorIds), ) : eq(frozenMonitorUptime.month, month), ); const frozenMonitorIds = new Set(frozenRows.map((r) => r.monitorId));
const pending = monitors.filter((m) => !frozenMonitorIds.has(m.id)); let alreadyFrozen = monitors.length - pending.length;
const monitorIdsByJobType = new Map<string, Set<string>>(); for (const m of pending) { const ids = monitorIdsByJobType.get(m.jobType) ?? new Set<string>(); ids.add(String(m.id)); monitorIdsByJobType.set(m.jobType, ids); }
const { counts, failedMonitorIds } = await fetchFreezeCounts({ monitorIdsByJobType, pipes: args.pipes, sleep: args.sleep, onChunkFailure: args.onChunkFailure, });
// avoid an O(pending²) sweep: computeMonitorMonth scans linearly per call const countsByMonitorId = new Map<string, ComputeCountRow[]>(); for (const row of counts) { const rows = countsByMonitorId.get(row.monitorId); if (rows) rows.push(row); else countsByMonitorId.set(row.monitorId, [row]); }
const workspaceIds = [...new Set(pending.map((m) => m.workspaceId))]; const workspaceRows = workspaceIds.length > 0 ? await db .select() .from(workspace) .where(inArray(workspace.id, workspaceIds)) : []; const workspacesById = new Map( workspaceRows.map((w) => [w.id, selectWorkspaceSchema.parse(w)]), );
let frozen = 0; let skipped = 0; const failures: string[] = [];
for (const m of pending) { if (failedMonitorIds.has(String(m.id))) { skipped++; failures.push(`monitor ${m.id}: tinybird counts unavailable`); continue; }
// no counts in the month (paused, created later, or no status pipe for // the job type): nothing to freeze — silent, not a failure const computed = computeMonitorMonth({ month, monitorId: m.id, counts: countsByMonitorId.get(String(m.id)) ?? [], }); if (!computed) continue;
const ws = workspacesById.get(m.workspaceId); if (!ws) { failures.push(`monitor ${m.id}: workspace ${m.workspaceId} not found`); continue; } const ctx = { workspace: ws, actor: { type: "system" as const, job: "uptime-freeze" }, db, };
try { const inserted = await freezeMonitorMonth({ ctx, input: { monitorId: m.id, month, days: computed.days }, }); if (inserted) frozen++; else alreadyFrozen++; } catch (e) { failures.push( `monitor ${m.id}: ${e instanceof Error ? e.message : String(e)}`, ); } }
return { month, frozen, alreadyFrozen, skipped, failures };}