import { getLogger } from "@logtape/logtape"; import { type SQL, and, db, eq, sql } from "@openstatus/db"; import type { NotificationOutboxPayload, MonitorStatus, NotificationProvider, } from "@openstatus/db/src/schema"; import { monitorTransition, notificationOutbox, notificationOutboxEventType, monitorIncidentTable, monitor, monitorStatusTable, notification, notificationsToMonitors, selectMonitorSchema, } from "@openstatus/db/src/schema"; import { withBusyRetry } from "@openstatus/services"; import { checkerAudit } from "../utils/audit-log"; const logger = getLogger(["workflow"]); import { quorumCountSql, quorumGuardSql } from "./quorum"; export type EventType = (typeof notificationOutboxEventType)[number]; export const EVENT_TYPE: Record = { active: "recovery", degraded: "degraded", error: "alert", }; export type TransitionInput = { monitorId: number; region: string; status: MonitorStatus; cronTimestamp: number; statusCode?: number; message?: string; latency?: number; deadlineSeconds: number; /** Monitors outside this gate are still delivered by the inline sender. */ rolloutPct: number; }; export type OutboxRowRef = { id: number; notificationId: number; provider: NotificationProvider; /** `pending` means the drainer owns delivery; `settled` means the inline sender does. */ deliveryStatus: "pending" | "settled"; }; export type TransitionResult = | { kind: "unchanged" } | { kind: "monitor-missing" } | { kind: "evaluated"; transitioned: boolean; quorumCount: number; regionCount: number; affectedRegions: string[]; outboxRows: OutboxRowRef[]; incidentId: number | null; incidentCreatedId: number | null; incidentResolvedIds: number[]; }; /** * A Cloud Tasks retry can land after a later check already reported. The stored * `cron_timestamp` only advances on a status change, so it cannot bound this on * its own. */ export function isStaleCheck( cronTimestamp: number, maxAgeMs: number, now = Date.now(), ): boolean { return now - cronTimestamp > maxAgeMs; } type JournalRow = { quorum_count: number; transitioned: number }; type OutboxInsertRow = { id: number; notification_id: number; incident_id: number | null; provider: NotificationProvider; delivery_status: "pending" | "settled"; }; function journalStatement( input: TransitionInput, regionsJson: string, regionCount: number, guard: SQL, ) { const count = quorumCountSql({ toStatus: input.status, regionsJson, }); const transitioning = sql`${guard} AND ${monitor.status} <> ${input.status}`; return db.all(sql` INSERT INTO ${monitorTransition} (monitor_id, region, cron_timestamp, from_status, to_status, quorum_count, region_count, transitioned, outbox_rows, created_at) SELECT ${monitor.id}, ${input.region}, ${input.cronTimestamp}, ${monitor.status}, ${input.status}, ${count}, ${regionCount}, CASE WHEN ${transitioning} THEN 1 ELSE 0 END, CASE WHEN ${transitioning} THEN (SELECT count(*) FROM ${notificationsToMonitors} WHERE ${notificationsToMonitors.monitorId} = ${monitor.id}) ELSE 0 END, unixepoch() FROM ${monitor} WHERE ${monitor.id} = ${input.monitorId} RETURNING quorum_count, transitioned `); } function createIncidentStatement(input: TransitionInput, guard: SQL) { return db.all<{ id: number }>(sql` INSERT INTO ${monitorIncidentTable} (monitor_id, workspace_id, started_at) SELECT ${monitor.id}, ${monitor.workspaceId}, ${Math.floor(input.cronTimestamp / 1000)} FROM ${monitor} WHERE ${monitor.id} = ${input.monitorId} AND ${monitor.status} <> ${input.status} AND ${guard} AND NOT EXISTS ( SELECT 1 FROM ${monitorIncidentTable} WHERE ${monitorIncidentTable.monitorId} = ${monitor.id} AND ${monitorIncidentTable.resolvedAt} IS NULL) ON CONFLICT DO NOTHING RETURNING id `); } function resolveIncidentStatement(input: TransitionInput, guard: SQL) { return db.all<{ id: number }>(sql` UPDATE ${monitorIncidentTable} SET resolved_at = ${Math.floor(input.cronTimestamp / 1000)}, auto_resolved = 1 WHERE ${monitorIncidentTable.monitorId} = ${input.monitorId} AND ${monitorIncidentTable.resolvedAt} IS NULL AND EXISTS ( SELECT 1 FROM ${monitor} WHERE ${monitor.id} = ${input.monitorId} AND ${monitor.status} <> ${input.status} AND ${guard}) RETURNING id `); } function outboxStatement( input: TransitionInput, payload: NotificationOutboxPayload, guard: SQL, ) { const dedupPrefix = `${input.cronTimestamp}:${input.monitorId}:${input.status}:`; // Outside the rollout the inline sender owns this delivery, so the row is // written already consumed: it still feeds the shadow diff, but raising // OUTBOX_ROLLOUT_PCT can never make the drainer re-send it. const owned = sql`(${monitor.id} % 100) < ${input.rolloutPct}`; return db.all(sql` INSERT INTO ${notificationOutbox} (dedup_key, monitor_id, workspace_id, notification_id, provider, event_type, from_status, to_status, cron_timestamp, incident_id, payload, delivery_status, outcome, next_attempt_at, deadline_at, created_at) SELECT ${dedupPrefix} || ${notification.id}, ${monitor.id}, ${monitor.workspaceId}, ${notification.id}, ${notification.provider}, ${EVENT_TYPE[input.status]}, ${monitor.status}, ${input.status}, ${input.cronTimestamp}, (SELECT id FROM ${monitorIncidentTable} WHERE ${monitorIncidentTable.monitorId} = ${monitor.id} AND ${monitorIncidentTable.resolvedAt} IS NULL ORDER BY id DESC LIMIT 1), ${JSON.stringify(payload)}, CASE WHEN ${owned} THEN 'pending' ELSE 'settled' END, CASE WHEN ${owned} THEN NULL ELSE 'inline' END, unixepoch(), unixepoch() + ${input.deadlineSeconds}, unixepoch() FROM ${monitor} JOIN ${notificationsToMonitors} ON ${notificationsToMonitors.monitorId} = ${monitor.id} JOIN ${notification} ON ${notification.id} = ${notificationsToMonitors.notificationId} WHERE ${monitor.id} = ${input.monitorId} AND ${monitor.status} <> ${input.status} AND ${guard} ON CONFLICT (dedup_key) DO NOTHING RETURNING id, notification_id, incident_id, provider, delivery_status `); } function casStatement(input: TransitionInput, guard: SQL) { return db.all<{ id: number }>(sql` UPDATE ${monitor} SET status = ${input.status}, updated_at = unixepoch() WHERE ${monitor.id} = ${input.monitorId} AND ${monitor.status} <> ${input.status} AND ${guard} RETURNING id `); } /** * Writes the region status and, only when that changed something, evaluates the * monitor transition as one atomic batch. An outbox row exists iff the * compare-and-swap matched, so a notification is owed exactly once. */ export async function applyStatusTransition( input: TransitionInput, ): Promise { const changed = await withBusyRetry(() => db.all<{ region: string }>(sql` INSERT INTO ${monitorStatusTable} (monitor_id, region, status, cron_timestamp, updated_at) VALUES (${input.monitorId}, ${input.region}, ${input.status}, ${input.cronTimestamp}, unixepoch()) ON CONFLICT (monitor_id, region) DO UPDATE SET status = excluded.status, cron_timestamp = excluded.cron_timestamp, updated_at = unixepoch() WHERE ${monitorStatusTable.status} <> excluded.status AND excluded.cron_timestamp > ${monitorStatusTable.cronTimestamp} RETURNING region `), ); if (changed.length === 0) return { kind: "unchanged" }; return evaluateTransition(input); } /** * The transition half, without the region write. Exported as the repair entry * point: the region write and this evaluation are separate transactions, so a * crash or a failed batch between them leaves `monitor.status` behind, and the * fast path means replaying the same check will not re-evaluate it. */ export async function evaluateTransition( input: TransitionInput, ): Promise { const [monitorRows, statusRows] = await withBusyRetry(() => db.batch([ db.select().from(monitor).where(eq(monitor.id, input.monitorId)), db .select({ region: monitorStatusTable.region }) .from(monitorStatusTable) .where( and( eq(monitorStatusTable.monitorId, input.monitorId), eq(monitorStatusTable.status, input.status), ), ), ]), ); const parsed = selectMonitorSchema.safeParse(monitorRows[0]); if (!parsed.success) return { kind: "monitor-missing" }; const regions = parsed.data.regions; const regionsJson = JSON.stringify(regions); const regionCount = regions.length; const configuredRegions = new Set(regions); const affectedRegions = statusRows .map((row) => row.region) .filter((region) => configuredRegions.has(region)); const payload: NotificationOutboxPayload = { regions: affectedRegions, statusCode: input.statusCode, message: input.message, latency: input.latency, }; const guard = quorumGuardSql({ toStatus: input.status, regionsJson, regionCount, }); const journal = journalStatement(input, regionsJson, regionCount, guard); const outbox = outboxStatement(input, payload, guard); const cas = casStatement(input, guard); let journalRows: JournalRow[]; let outboxRows: OutboxInsertRow[]; let incidentRows: { id: number }[] = []; // All three statuses enqueue notifications; only the incident statement moves. // The outbox row reads incident_id by subquery, so it must run after the // incident is created but before an existing one is resolved. if (input.status === "error") { const [journalResult, incidentResult, outboxResult] = await withBusyRetry( () => db.batch([journal, createIncidentStatement(input, guard), outbox, cas]), ); journalRows = journalResult; outboxRows = outboxResult; incidentRows = incidentResult; } else { const [journalResult, outboxResult, incidentResult] = await withBusyRetry( () => db.batch([ journal, outbox, resolveIncidentStatement(input, guard), cas, ]), ); journalRows = journalResult; outboxRows = outboxResult; incidentRows = incidentResult; } const incidentCreatedId = input.status === "error" ? (incidentRows[0]?.id ?? null) : null; const incidentResolvedIds = input.status === "error" ? [] : incidentRows.map((row) => row.id); // Published here rather than in the route so drift repair keeps the trail too. await publishIncidentAudit(input, incidentCreatedId, incidentResolvedIds); return { kind: "evaluated", transitioned: (journalRows[0]?.transitioned ?? 0) === 1, quorumCount: journalRows[0]?.quorum_count ?? 0, regionCount, affectedRegions, outboxRows: outboxRows.map((row) => ({ id: row.id, notificationId: row.notification_id, provider: row.provider, deliveryStatus: row.delivery_status, })), incidentId: outboxRows[0]?.incident_id ?? null, incidentCreatedId, incidentResolvedIds, }; } async function publishIncidentAudit( input: TransitionInput, createdId: number | null, resolvedIds: number[], ): Promise { const targets = [{ id: String(input.monitorId), type: "monitor" as const }]; const entries: Promise[] = []; if (createdId !== null) { entries.push( checkerAudit.publishAuditLog({ id: `monitor:${input.monitorId}`, action: "incident.created", targets, metadata: { cronTimestamp: input.cronTimestamp, incidentId: createdId }, }), ); } for (const incidentId of resolvedIds) { entries.push( checkerAudit.publishAuditLog({ id: `monitor:${input.monitorId}`, action: "incident.resolved", targets, metadata: { cronTimestamp: input.cronTimestamp, incidentId }, }), ); } // Best-effort: the incident is already committed, and failing here would make // Cloud Tasks retry a transition that has landed. try { await Promise.all(entries); } catch (error) { logger.warn("Failed to publish incident audit log", { monitor_id: input.monitorId, error_message: error instanceof Error ? error.message : String(error), }); } }