diff --git a/src/db.ts b/src/db.ts index c279606..ba9dd36 100644 --- a/src/db.ts +++ b/src/db.ts @@ -324,6 +324,52 @@ export async function getRecentIncidents(db: D1Database, days: number): Promise< return rows.results as unknown as Incident[]; } +export async function getRecentResolvedIncidentsWithUpdates(db: D1Database, days: number): Promise { + const since = Math.floor(Date.now() / 1000) - days * 24 * 60 * 60; + const rows = await db + .prepare( + `SELECT i.*, u.id as update_id, u.status as update_status, u.message as update_message, u.created_at as update_created_at + FROM incidents i + LEFT JOIN incident_updates u ON u.incident_id = i.id + WHERE i.status = 'resolved' AND i.resolved_at >= ? + ORDER BY i.resolved_at DESC, u.created_at ASC`, + ) + .bind(since) + .all(); + + const incidentMap = new Map(); + for (const row of rows.results) { + const id = row.id as number; + if (!incidentMap.has(id)) { + incidentMap.set(id, { + id, + service_id: row.service_id as string, + title: row.title as string, + status: row.status as string, + severity: row.severity as string, + triage_report: row.triage_report as string | null, + github_repo: row.github_repo as string | null, + github_issue_number: row.github_issue_number as number | null, + started_at: row.started_at as number, + resolved_at: row.resolved_at as number | null, + created_at: row.created_at as number, + updated_at: row.updated_at as number, + updates: [], + }); + } + if (row.update_id) { + incidentMap.get(id)!.updates.push({ + id: row.update_id as number, + incident_id: id, + status: row.update_status as string, + message: row.update_message as string, + created_at: row.update_created_at as number, + }); + } + } + return Array.from(incidentMap.values()); +} + export async function getIncident(db: D1Database, id: number): Promise { const incident = await db .prepare("SELECT * FROM incidents WHERE id = ?") @@ -376,3 +422,15 @@ export async function getRecentlyResolvedIncident( .first(); return (row as unknown as Incident) ?? null; } + +export async function getRecentlyResolvedIncidents( + db: D1Database, + withinSeconds: number, +): Promise { + const since = Math.floor(Date.now() / 1000) - withinSeconds; + const rows = await db + .prepare("SELECT * FROM incidents WHERE status = 'resolved' AND resolved_at >= ? ORDER BY resolved_at DESC") + .bind(since) + .all(); + return rows.results as unknown as Incident[]; +} diff --git a/src/github.ts b/src/github.ts index db6986c..7c91384 100644 --- a/src/github.ts +++ b/src/github.ts @@ -81,31 +81,6 @@ export async function closeIssue( } } -export async function editIssueBody( - token: string, - owner: string, - repo: string, - issueNumber: number, - body: string, -): Promise { - const res = await fetch( - `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}`, - { - method: "PATCH", - headers: { - Authorization: `Bearer ${token}`, - Accept: "application/vnd.github+json", - "User-Agent": "infra-status-worker", - }, - body: JSON.stringify({ body }), - }, - ); - if (!res.ok) { - const text = await res.text(); - throw new Error(`GitHub edit issue failed: ${res.status} ${text}`); - } -} - interface GitHubIssue { state: string; body: string | null; @@ -161,14 +136,29 @@ export async function syncGitHubIncidents( const issue = await fetchIssue(token, parsed.owner, parsed.repo, incident.github_issue_number); if (issue.state === "closed" && incident.status !== "resolved") { const now = Math.floor(Date.now() / 1000); + // Fetch latest comments to find the closing message + const kvKey = `gh_sync:${incident.id}:last`; + const lastSeen = await kv.get(kvKey); + const comments = await fetchComments(token, parsed.owner, parsed.repo, incident.github_issue_number, lastSeen ?? undefined); + const human = comments.filter((c) => c.user.type !== "Bot" && !c.body.startsWith("Automated incident detected") && !c.body.startsWith("## Triage Report") && !c.body.startsWith("Service recovered automatically")); + + // Use the last human comment as the resolve message, or fall back to generic + const resolveMsg = human.length > 0 ? human[human.length - 1].body : "Issue closed on GitHub"; + + // Add any earlier human comments as investigating updates + for (const comment of human.slice(0, -1)) { + await addIncidentUpdate(db, incident.id, "investigating", comment.body); + } + await updateIncident(db, incident.id, { status: "resolved", resolved_at: now }); - await addIncidentUpdate(db, incident.id, "resolved", "Issue closed on GitHub"); - continue; - } + await addIncidentUpdate(db, incident.id, "resolved", resolveMsg); - // Sync issue body edits back to triage_report - if (issue.body && issue.body !== incident.triage_report) { - await updateIncident(db, incident.id, { triage_report: issue.body }); + // Track sync position + if (comments.length > 0) { + const latest = comments[comments.length - 1].created_at; + await kv.put(kvKey, latest, { expirationTtl: 86400 * 7 }); + } + continue; } // Sync new comments since last check diff --git a/src/health.ts b/src/health.ts index 036ea1e..deb324f 100644 --- a/src/health.ts +++ b/src/health.ts @@ -4,9 +4,11 @@ const SLOW_THRESHOLD_MS = 3000; export type Status = "up" | "degraded" | "misconfigured" | "timeout" | "down" | "unknown"; -interface HealthResult { +export interface HealthResult { status: Status; latency_ms: number; + status_code?: number; + error?: string; } export async function checkHealth(service: Service): Promise { @@ -24,24 +26,25 @@ export async function checkHealth(service: Service): Promise { const latency_ms = Date.now() - start; if (res.status >= 400 && res.status < 500) { - return { status: "misconfigured", latency_ms }; + return { status: "misconfigured", latency_ms, status_code: res.status }; } if (res.status === 502 || res.status === 504) { - return { status: "down", latency_ms }; + return { status: "down", latency_ms, status_code: res.status }; } if (res.status >= 500) { - return { status: "degraded", latency_ms }; + return { status: "degraded", latency_ms, status_code: res.status }; } if (res.status >= 200 && res.status < 300) { if (latency_ms > SLOW_THRESHOLD_MS) { - return { status: "degraded", latency_ms }; + return { status: "degraded", latency_ms, status_code: res.status }; } - return { status: "up", latency_ms }; + return { status: "up", latency_ms, status_code: res.status }; } - return { status: "down", latency_ms }; + return { status: "down", latency_ms, status_code: res.status }; } catch (err) { const latency_ms = Date.now() - start; const isTimeout = err instanceof DOMException && err.name === "TimeoutError"; - return { status: isTimeout ? "timeout" : "down", latency_ms }; + const error = isTimeout ? "Request timed out after 10s" : err instanceof Error ? err.message : "Unknown error"; + return { status: isTimeout ? "timeout" : "down", latency_ms, error }; } } diff --git a/src/index.ts b/src/index.ts index 8d01862..8031a0f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,7 @@ import type { Env } from "./types"; import { getManifest } from "./manifest"; import { checkHealth } from "./health"; -import { insertPing, getLatestPing, pruneOldPings, createIncident, updateIncident, addIncidentUpdate, getActiveIncidentForService, getActiveIncidents, getRecentlyResolvedIncident, setIncidentGitHub } from "./db"; +import { insertPing, getLatestPing, pruneOldPings, createIncident, updateIncident, addIncidentUpdate, getActiveIncidentForService, getActiveIncidents, getRecentlyResolvedIncident, getRecentlyResolvedIncidents, setIncidentGitHub } from "./db"; import { refreshDevices } from "./tailscale"; import { handleStatusRoute } from "./routes/status"; import { handleFavicon } from "./routes/favicon"; @@ -127,7 +127,7 @@ export default { try { const issueNumber = await createIssue(env.GITHUB_TOKEN, parsed.owner, parsed.repo, { title: `${svc.name} is ${result.status}`, - body: `Automated incident detected by [infra.dunkirk.sh](https://infra.dunkirk.sh)\n\n**Service:** ${svc.name}\n**Health URL:** ${svc.health_url}\n**Status:** ${result.status}\n**Detected at:** ${new Date().toISOString()}\n\n---\n*Comments on this issue will appear on the status page. Close the issue to resolve the incident.*`, + body: `Automated incident detected by [infra.dunkirk.sh](https://infra.dunkirk.sh)\n\n**Service:** ${svc.name}\n**Health URL:** ${svc.health_url}\n**Status:** ${result.status}${result.status_code ? ` (HTTP ${result.status_code})` : ""}${result.error ? ` — ${result.error}` : ""}\n**Latency:** ${result.latency_ms}ms\n**Detected at:** ${new Date().toISOString()}\n\n---\n*Comments on this issue will appear on the status page. Close the issue to resolve the incident.*`, assignees: env.GITHUB_ASSIGNEE ? [env.GITHUB_ASSIGNEE] : [], labels: ["incident"], }); @@ -188,7 +188,9 @@ export default { // Sync GitHub issue comments/state back to incidents if (env.GITHUB_TOKEN) { const active = await getActiveIncidents(env.DB); - await syncGitHubIncidents(env.DB, env.KV, env.GITHUB_TOKEN, active); + const recentlyResolved = await getRecentlyResolvedIncidents(env.DB, 86400 * 7); + const toSync = [...active, ...recentlyResolved]; + await syncGitHubIncidents(env.DB, env.KV, env.GITHUB_TOKEN, toSync); } }, } satisfies ExportedHandler; diff --git a/src/routes/incidents.ts b/src/routes/incidents.ts index a2e5a85..d079594 100644 --- a/src/routes/incidents.ts +++ b/src/routes/incidents.ts @@ -7,7 +7,7 @@ import { updateIncident, addIncidentUpdate, } from "../db"; -import { commentOnIssue, editIssueBody, parseRepo } from "../github"; +import { commentOnIssue, parseRepo } from "../github"; function authCheck(request: Request, env: Env): boolean { const auth = request.headers.get("Authorization"); @@ -55,14 +55,15 @@ export async function handleIncidentRoute( return Response.json({ error: "unauthorized" }, { status: 401 }); } const id = parseInt(singleMatch[1]); - const body = await request.json<{ status?: string; triage_report?: string }>(); + const body = await request.json<{ status?: string; triage_report?: string; summary?: string }>(); const updateData: { status?: string; triage_report?: string; resolved_at?: number } = {}; if (body.status) updateData.status = body.status; if (body.triage_report !== undefined) updateData.triage_report = body.triage_report; if (body.status === "resolved") updateData.resolved_at = Math.floor(Date.now() / 1000); await updateIncident(env.DB, id, updateData); if (body.status) { - await addIncidentUpdate(env.DB, id, body.status, body.triage_report ?? `Status changed to ${body.status}`); + const timelineMsg = body.summary ?? body.triage_report ?? `Status changed to ${body.status}`; + await addIncidentUpdate(env.DB, id, body.status, timelineMsg); } // Sync triage report to GitHub issue @@ -71,9 +72,7 @@ export async function handleIncidentRoute( const parsed = parseRepo(`https://github.com/${incident.github_repo}`); if (parsed) { if (body.triage_report) { - // Post triage report as comment and update issue body commentOnIssue(env.GITHUB_TOKEN, parsed.owner, parsed.repo, incident.github_issue_number, `## Triage Report\n\n${body.triage_report}`).catch(() => {}); - editIssueBody(env.GITHUB_TOKEN, parsed.owner, parsed.repo, incident.github_issue_number, body.triage_report).catch(() => {}); } else if (body.status) { commentOnIssue(env.GITHUB_TOKEN, parsed.owner, parsed.repo, incident.github_issue_number, `Status changed to **${body.status}**`).catch(() => {}); } diff --git a/src/routes/index.ts b/src/routes/index.ts index 7c41c60..55486bb 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -1,12 +1,12 @@ import type { Env } from "../types"; import { getManifest } from "../manifest"; -import { getAllLatestPings, getAllUptime7d, getOverallUptimeDays, getLastCheckTime, getActiveIncidentsWithUpdates, getActiveIncidents, getRecentIncidents } from "../db"; +import { getAllLatestPings, getAllUptime7d, getOverallUptimeDays, getLastCheckTime, getActiveIncidentsWithUpdates, getActiveIncidents, getRecentResolvedIncidentsWithUpdates } from "../db"; import { getDeviceStatus } from "../tailscale"; import { getOverallStatus } from "../overall"; import { COMMIT_SHA } from "../version"; export async function handleIndex(env: Env): Promise { - const [manifest, latestPings, uptimes, lastCheck, uptimeDays, activeIncidentsWithUpdates, activeIncidentsList, recentIncidents] = await Promise.all([ + const [manifest, latestPings, uptimes, lastCheck, uptimeDays, activeIncidentsWithUpdates, activeIncidentsList, resolvedIncidents] = await Promise.all([ getManifest(env), getAllLatestPings(env.DB), getAllUptime7d(env.DB), @@ -14,7 +14,7 @@ export async function handleIndex(env: Env): Promise { getOverallUptimeDays(env.DB, 90), getActiveIncidentsWithUpdates(env.DB), getActiveIncidents(env.DB), - getRecentIncidents(env.DB, 7), + getRecentResolvedIncidentsWithUpdates(env.DB, 7), ]); const machineOnline = new Map(); @@ -46,7 +46,6 @@ export async function handleIndex(env: Env): Promise { manifest, latestPings, activeIncidents: activeIncidentsList, machineOnline, }); const activeIncidents = activeIncidentsWithUpdates; - const resolvedIncidents = recentIncidents.filter((i) => i.status === "resolved"); const html = ` @@ -105,36 +104,38 @@ export async function handleIndex(env: Env): Promise { .uptime-bar .day.down { background: #e74c3c; } .uptime-bar .day.none { background: #21262d; } .incidents { margin-bottom: 1.5rem; } - .incident-banner { background: #2d1b1b; border: 1px solid #e74c3c; border-radius: 6px; padding: 0.75rem; margin-bottom: 0.5rem; } - .incident-banner.major { border-color: #f39c12; background: #2d2517; } - .incident-banner.minor { border-color: #8b949e; background: #21262d; } - .incident-title { font-size: 0.85rem; font-weight: 500; margin-bottom: 0.25rem; } - .incident-meta { font-size: 0.7rem; color: #8b949e; display: flex; gap: 0.75rem; } - .incident-status { text-transform: uppercase; letter-spacing: 0.05em; } + .incident-banner { padding: 0.5rem 0; border-bottom: 1px solid #21262d; } + .incident-banner:last-child { border-bottom: none; } + .incident-header { display: flex; align-items: baseline; gap: 0.5rem; flex-wrap: wrap; font-size: 0.8rem; } + .incident-status { text-transform: uppercase; letter-spacing: 0.05em; font-weight: 600; } .incident-status.investigating { color: #e74c3c; } .incident-status.identified { color: #f39c12; } - .incident-status.monitoring { color: #3498db; } - .incident-triage { margin-top: 0.5rem; } - .incident-triage summary { font-size: 0.75rem; color: #8b949e; cursor: pointer; } - .incident-triage pre { font-size: 0.7rem; color: #c9d1d9; background: #161b22; padding: 0.5rem; border-radius: 4px; margin-top: 0.25rem; white-space: pre-wrap; word-break: break-word; max-height: 300px; overflow-y: auto; } - .incident-timeline { margin-top: 0.5rem; padding-left: 0.75rem; border-left: 2px solid #21262d; } + .incident-title { font-weight: 500; } + .incident-time { color: #8b949e; } + .incident-timeline { margin-top: 0.25rem; padding-left: 0.75rem; } .timeline-entry { padding: 0.25rem 0 0.25rem 0.5rem; font-size: 0.7rem; position: relative; } - .timeline-entry::before { content: ''; position: absolute; left: -0.75rem; top: 0.55rem; width: 6px; height: 6px; border-radius: 50%; background: #30363d; transform: translateX(-2px); } + .timeline-entry::before { content: ''; position: absolute; left: -0.75rem; top: 50%; width: 6px; height: 6px; border-radius: 50%; background: #30363d; transform: translate(-2px, -50%); z-index: 1; } + .timeline-entry::after { content: ''; position: absolute; left: calc(-0.75rem - 1px); top: 0; bottom: 0; width: 2px; background: #21262d; } + .timeline-entry:first-child::after { top: 50%; } + .timeline-entry:last-child::after { bottom: 50%; } + .timeline-entry:only-child::after { display: none; } .timeline-entry.investigating::before { background: #e74c3c; } .timeline-entry.identified::before { background: #f39c12; } - .timeline-entry.monitoring::before { background: #3498db; } .timeline-entry.resolved::before { background: #2ecc71; } .timeline-status { text-transform: uppercase; letter-spacing: 0.04em; font-size: 0.6rem; font-weight: 600; } .timeline-status.investigating { color: #e74c3c; } .timeline-status.identified { color: #f39c12; } - .timeline-status.monitoring { color: #3498db; } .timeline-status.resolved { color: #2ecc71; } .timeline-time { color: #484f58; margin-right: 0.5rem; } .timeline-msg { color: #8b949e; } + .timeline-msg a { color: #8b949e; text-decoration: underline; } + .timeline-msg code { font-size: 0.65rem; background: #161b22; padding: 0.1rem 0.25rem; border-radius: 3px; } .resolved-incidents { margin-top: 0.75rem; } - .resolved-header { font-size: 0.7rem; text-transform: uppercase; letter-spacing: 0.08em; color: #8b949e; margin-bottom: 0.25rem; } - .resolved-item { font-size: 0.75rem; color: #8b949e; padding: 0.25rem 0; border-bottom: 1px solid #21262d; } + .resolved-header { font-size: 0.7rem; text-transform: uppercase; letter-spacing: 0.08em; color: #8b949e; margin-bottom: 0.5rem; } + .resolved-item { font-size: 0.75rem; color: #8b949e; padding: 0.4rem 0; border-bottom: 1px solid #21262d; } .resolved-item:last-child { border-bottom: none; } + .resolved-item-header { margin-bottom: 0.25rem; } + .resolved-item .incident-timeline { margin-top: 0.25rem; } footer { margin-top: auto; padding-top: 1rem; border-top: 1px solid #21262d; font-size: 0.7rem; color: #8b949e; } .footer-meta { display: flex; justify-content: space-between; } footer a { color: #8b949e; text-decoration: none; } @@ -146,14 +147,12 @@ export async function handleIndex(env: Env): Promise {

infra.dunkirk.sh

${overallText}

${activeIncidents.length > 0 ? `
-${activeIncidents.map((i) => `
-
${esc(i.title)}
-
+${activeIncidents.map((i) => `
+
${i.status} - ${esc(i.service_id)} - started loading + ${esc(i.title)} + started loading
- ${i.triage_report ? `
triage report
${esc(i.triage_report)}
` : ""} ${i.updates.length > 0 ? `
${i.updates.map((u) => `
${new Date(u.created_at * 1000).toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit", hour12: false, timeZone: "America/New_York" })} @@ -191,10 +190,19 @@ ${clients.map((m) => `` : ""} ${resolvedIncidents.length > 0 ? `
recent incidents
-${resolvedIncidents.map((i) => `
${esc(i.title)} — resolved loading
`).join("\n")} +${resolvedIncidents.map((i) => `
+
${esc(i.title)} — resolved loading
+ ${i.updates.length > 0 ? `
+${i.updates.map((u) => `
+ ${new Date(u.created_at * 1000).toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit", hour12: false, timeZone: "America/New_York" })} + ${esc(u.status)} + ${esc(u.message)} +
`).join("\n")} +
` : ""} +
`).join("\n")}
` : ""}
-
${lastCheckISO ? `updated loading` : "no checks yet"}${COMMIT_SHA}
+
${lastCheckISO ? `updated loading` : "no checks yet"}${COMMIT_SHA}