Something went wrong. Try again.
[READ-ONLY] Mirror of https://github.com/openstatusHQ/openstatus. 🫖 Status page with uptime monitoring & API monitoring as code 🫖 openstatus.dev
bun drizzle-orm monitoring monitoring-as-code nextjs observability on-call open-source shadcn-ui status-page statuspage synthetic-monitoring tinybird turso uptime uptime-checker uptime-monitor
Something went wrong. Try again.
9.3 kB · 355 lines
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356import { getLogger } from "@logtape/logtape";import { and, db, eq, inArray, isNull, schema } from "@openstatus/db";import { incidentTable } from "@openstatus/db/src/schema";import { monitorRegions } from "@openstatus/db/src/schema/constants";import { monitorStatusSchema, selectMonitorSchema,} from "@openstatus/db/src/schema/monitors/validation";import { Hono } from "hono";import { z } from "zod";
import { env } from "../env";import type { Env } from "../index";import { checkerAudit } from "../utils/audit-log";import { triggerNotifications, upsertMonitorStatus } from "./alerting";
export const checkerRoute = new Hono<Env>();
const payloadSchema = z.object({ monitorId: z.string(), message: z.string().optional(), statusCode: z.number().optional(), region: z.enum(monitorRegions), cronTimestamp: z.number(), status: monitorStatusSchema, latency: z.number().optional(),});
const logger = getLogger(["workflow"]);
/** * Finds an open incident (not resolved and not acknowledged) for the given monitor. */async function findOpenIncident(monitorId: number) { return db .select() .from(incidentTable) .where( and( eq(incidentTable.monitorId, monitorId), isNull(incidentTable.resolvedAt), ), ) .get();}
/** * Resolves an open incident by setting resolvedAt and autoResolved flag. */async function resolveIncident(params: { monitorId: string; cronTimestamp: number;}) { const { monitorId, cronTimestamp } = params; const incident = await findOpenIncident(Number(monitorId));
if (!incident || incident.resolvedAt) { return null; }
logger.info("Recovering incident", { incident_id: incident.id, monitor_id: monitorId, });
await db .update(incidentTable) .set({ resolvedAt: new Date(cronTimestamp), autoResolved: true, }) .where(eq(incidentTable.id, incident.id)) .run();
await checkerAudit.publishAuditLog({ id: `monitor:${monitorId}`, action: "incident.resolved", targets: [{ id: monitorId, type: "monitor" }], metadata: { cronTimestamp, incidentId: incident.id }, });
return incident;}
checkerRoute.post("/updateStatus", async (c) => { const auth = c.req.header("Authorization"); if (auth !== `Basic ${env().CRON_SECRET}`) { logger.error("Unauthorized"); return c.text("Unauthorized", 401); }
const event = c.get("event"); const json = await c.req.json();
const result = payloadSchema.safeParse(json);
if (!result.success) { return c.text("Unprocessable Entity", 422); }
const { monitorId, message, region, statusCode, cronTimestamp, status, latency, } = result.data;
logger.info("Updating monitor status", { monitor_id: monitorId, region, status, status_code: statusCode, cron_timestamp: cronTimestamp, latency_ms: latency, });
// First we upsert the monitor status await upsertMonitorStatus({ monitorId: monitorId, status, region: region, });
const currentMonitor = await db .select() .from(schema.monitor) .where(eq(schema.monitor.id, Number(monitorId))) .get();
const monitor = selectMonitorSchema.parse(currentMonitor); const numberOfRegions = monitor.regions.length;
// Fetch all affected regions for notifications (single query) const affectedRegions = await db .select({ region: schema.monitorStatusTable.region }) .from(schema.monitorStatusTable) .where( and( eq(schema.monitorStatusTable.monitorId, monitor.id), eq(schema.monitorStatusTable.status, status), inArray(schema.monitorStatusTable.region, monitor.regions), ), ) .all();
const affectedRegionsList = affectedRegions.map((r) => r.region); const affectedRegionCount = affectedRegionsList.length;
event.status_update = { status: result.data.status, message: result.data.message, region: result.data.region, status_code: result.data.statusCode, cron_timestamp: result.data.cronTimestamp, latency_ms: result.data.latency, affectedRegionsCount: affectedRegionCount, monitorId: monitor.id, };
if (affectedRegionCount === 0) { return c.json({ success: true }, 200); }
// audit log the current state of the ping
switch (status) { case "active": await checkerAudit.publishAuditLog({ id: `monitor:${monitorId}`, action: "monitor.recovered", targets: [{ id: monitorId, type: "monitor" }], metadata: { region, statusCode: statusCode ?? -1, cronTimestamp, latency, }, }); break; case "degraded": await checkerAudit.publishAuditLog({ id: `monitor:${monitorId}`, action: "monitor.degraded", targets: [{ id: monitorId, type: "monitor" }], metadata: { region, statusCode: statusCode ?? -1, cronTimestamp, latency, }, }); break; case "error": await checkerAudit.publishAuditLog({ id: `monitor:${monitorId}`, action: "monitor.failed", targets: [{ id: monitorId, type: "monitor" }], metadata: { region, statusCode: statusCode ?? -1, message, cronTimestamp, latency, }, }); break; }
let triggeredNotifications: { notificationId: number; provider: string }[] = [];
if (affectedRegionCount >= numberOfRegions / 2 || numberOfRegions === 1) { switch (status) { case "active": { if (monitor.status === "active") { break; }
logger.info("Monitor status changed to active", { monitor_id: monitor.id, workspace_id: monitor.workspaceId, }); await db .update(schema.monitor) .set({ status: "active" }) .where(eq(schema.monitor.id, monitor.id));
let incident = null; if (monitor.status === "error") { incident = await resolveIncident({ monitorId, cronTimestamp }); }
triggeredNotifications = await triggerNotifications({ monitorId, statusCode, message, notifType: "recovery", cronTimestamp, regions: affectedRegionsList, latency, incidentId: incident?.id, });
break; } case "degraded": if (monitor.status === "degraded") { break; }
logger.info("Monitor status changed to degraded", { monitor_id: monitor.id, workspace_id: monitor.workspaceId, });
await db .update(schema.monitor) .set({ status: "degraded" }) .where(eq(schema.monitor.id, monitor.id));
let incident = null; if (monitor.status === "error") { incident = await resolveIncident({ monitorId, cronTimestamp, }); }
triggeredNotifications = await triggerNotifications({ monitorId, statusCode, message, notifType: "degraded", cronTimestamp, latency, regions: affectedRegionsList, incidentId: incident?.id, });
break; case "error": if (monitor.status === "error") { break; }
logger.info("Monitor status changed to error", { monitor_id: monitor.id, workspace_id: monitor.workspaceId, });
await db .update(schema.monitor) .set({ status: "error" }) .where(eq(schema.monitor.id, monitor.id));
try { const existingIncident = await findOpenIncident(Number(monitorId)); if (existingIncident) { logger.info("Already in incident", { incident_id: existingIncident.id, }); break; }
const [newIncident] = await db .insert(incidentTable) .values({ monitorId: Number(monitorId), workspaceId: monitor.workspaceId, startedAt: new Date(cronTimestamp), }) .returning();
if (!newIncident?.id) { break; }
await checkerAudit.publishAuditLog({ id: `monitor:${monitorId}`, action: "incident.created", targets: [{ id: monitorId, type: "monitor" }], metadata: { cronTimestamp, incidentId: newIncident.id }, });
triggeredNotifications = await triggerNotifications({ monitorId, statusCode, message, notifType: "alert", cronTimestamp, latency, regions: affectedRegionsList, incidentId: newIncident.id, }); } catch (error) { logger.warning("Failed to create incident", { error }); }
break; default: logger.error("should not happen"); break; } }
(event.status_update as Record<string, unknown>).notificationTriggered = triggeredNotifications.length > 0; (event.status_update as Record<string, unknown>).notifications = triggeredNotifications;
return c.text("Ok", 200);});