Something went wrong. Try again.
[READ-ONLY] Mirror of https://github.com/openstatusHQ/openstatus. 🫖 Status page with uptime monitoring & API monitoring as code 🫖 openstatus.dev
bun drizzle-orm monitoring monitoring-as-code nextjs observability on-call open-source shadcn-ui status-page statuspage synthetic-monitoring tinybird turso uptime uptime-checker uptime-monitor
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208import { and, db as defaultDb, desc, eq, inArray } from "@openstatus/db";import { type Incident, type IncidentSeverity, incident, incidentEvent, slackUser, selectWorkspaceSchema, workspace,} from "@openstatus/db/src/schema";
import type { DB, ServiceContext } from "../context";import { type SlackConnection, getSlackConnection,} from "../integration/slack-connection";import type { SlackClientFactory } from "./slack-flow";
const HOUR = 60 * 60 * 1000;
export const STALE_AFTER: Record<IncidentSeverity, number> = { critical: HOUR, major: 4 * HOUR, minor: 24 * HOUR,};
/** * Which reminder window an incident is in: 0 once it's stale, then 1, 2, … * each twice as long as the last. `null` while it's still fresh. */export function reminderWindow( severity: IncidentSeverity, lastActivity: Date, now: Date,): number | null { const threshold = STALE_AFTER[severity]; const elapsed = now.getTime() - lastActivity.getTime(); if (elapsed < threshold) return null; return Math.floor(Math.log2(elapsed / threshold));}
export type ReminderTarget = | { kind: "channel"; channel: string } | { kind: "dm"; slackUserId: string; role: "commander" | "declarer" };
export type ReminderResult = { incidentId: number; target: ReminderTarget | null; /** `false` when there was nobody to remind or Slack rejected the post. */ sent: boolean;};
/** `true` when this caller owns the key; used so each window fires once. */export type ClaimOnce = (key: string, ttlSeconds: number) => Promise<boolean>;export type ReleaseClaim = (key: string) => Promise<void>;
// The Slack client already retries network, 5xx and 429 errors; anything else// (channel_not_found, not_in_channel, …) would fail again on the next tick.function isPermanentSlackError(error: unknown): boolean { return ( typeof error === "object" && error !== null && "code" in error && error.code === "slack_webapi_platform_error" );}
async function reminderConnection( db: DB, workspaceId: number,): Promise<SlackConnection | null> { const parsed = selectWorkspaceSchema.safeParse( await db .select() .from(workspace) .where(eq(workspace.id, workspaceId)) .get(), ); if (!parsed.success) return null; const ws = parsed.data; if (!ws.limits["slack-agent"]) return null; const ctx: ServiceContext = { workspace: ws, actor: { type: "system", job: "incident-reminders" }, db, }; const connection = await getSlackConnection({ ctx }); if (!connection || connection.missingScopes.includes("chat:write")) { return null; } return connection;}
async function dmTarget( db: DB, row: Incident, teamId: string,): Promise<ReminderTarget | null> { for (const [role, userId] of [ ["commander", row.commanderId], ["declarer", row.declaredBy], ] as const) { if (userId === null) continue; const mapping = await db .select({ slackUserId: slackUser.slackUserId }) .from(slackUser) .where( and( eq(slackUser.workspaceId, row.workspaceId), eq(slackUser.slackTeamId, teamId), eq(slackUser.userId, userId), ), ) .get(); if (mapping) return { kind: "dm", slackUserId: mapping.slackUserId, role }; } return null;}
/** * Nudges open and mitigated incidents that went quiet: the bound channel, * else a DM to the commander, else to the declarer. Safe to run on every * machine at once: each window is claimed before anything is sent. */export async function remindStaleIncidents(args: { now?: Date; clientFor: SlackClientFactory; claim: ClaimOnce; release: ReleaseClaim; dashboardUrl: string; db?: DB;}): Promise<ReminderResult[]> { const db = args.db ?? defaultDb; const now = args.now ?? new Date(); const candidates = await db .select() .from(incident) .where(inArray(incident.status, ["open", "mitigated"])) .all();
const results: ReminderResult[] = []; const connections = new Map<number, SlackConnection | null>(); for (const row of candidates) { if (row.closedAt) continue; if (!connections.has(row.workspaceId)) { connections.set( row.workspaceId, await reminderConnection(db, row.workspaceId), ); } const connection = connections.get(row.workspaceId); if (!connection) continue;
const last = await db .select({ id: incidentEvent.id, createdAt: incidentEvent.createdAt }) .from(incidentEvent) .where(eq(incidentEvent.incidentId, row.id)) .orderBy(desc(incidentEvent.createdAt), desc(incidentEvent.id)) .get(); const lastActivity = last?.createdAt ?? row.declaredAt; const window = reminderWindow(row.severity, lastActivity, now); if (window === null) continue;
// Claimed before resolving the target so a skip is reported once per window. const key = `incident:reminder:${row.id}:${last?.id ?? 0}:${window}`; if (!(await args.claim(key, 14 * 24 * 60 * 60))) continue;
const target: ReminderTarget | null = row.slackChannelId && row.slackTeamId === connection.teamId ? { kind: "channel", channel: row.slackChannelId } : await dmTarget(db, row, connection.teamId); if (!target) { console.warn("incident reminder skipped: nobody to remind", { incidentId: row.id, }); results.push({ incidentId: row.id, target: null, sent: false }); continue; }
const quiet = Math.round((now.getTime() - lastActivity.getTime()) / HOUR); const url = `${args.dashboardUrl}/incidents/${row.id}`; const text = target.kind === "channel" ? `:hourglass: No update on *${row.title}* for ${quiet}h. Post a note, change its status, or resolve it. <${url}|Open in openstatus>` : `:hourglass: You are the ${target.role} of *${row.title}* (${row.severity}), which has had no update for ${quiet}h. <${url}|Open in openstatus>`; const sent = await args .clientFor(connection.botToken) .chat.postMessage({ channel: target.kind === "channel" ? target.channel : target.slackUserId, text, }) .then( () => true, async (error) => { console.warn("incident reminder failed", { incidentId: row.id, error, }); if (!isPermanentSlackError(error)) await args.release(key); return false; }, ); results.push({ incidentId: row.id, target, sent }); } return results;}