diff --git a/migrations/0003_add_incidents.sql b/migrations/0003_add_incidents.sql new file mode 100644 index 0000000..e334c66 --- /dev/null +++ b/migrations/0003_add_incidents.sql @@ -0,0 +1,24 @@ +CREATE TABLE incidents ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + service_id TEXT NOT NULL, + title TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('investigating', 'identified', 'monitoring', 'resolved')), + severity TEXT NOT NULL CHECK (severity IN ('critical', 'major', 'minor')), + triage_report TEXT, + started_at INTEGER NOT NULL, + resolved_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE incident_updates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + incident_id INTEGER NOT NULL REFERENCES incidents(id), + status TEXT NOT NULL, + message TEXT NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE INDEX idx_incidents_service ON incidents (service_id, status); +CREATE INDEX idx_incidents_active ON incidents (status) WHERE status != 'resolved'; +CREATE INDEX idx_incident_updates ON incident_updates (incident_id, created_at DESC); diff --git a/migrations/0004_add_github_fields.sql b/migrations/0004_add_github_fields.sql new file mode 100644 index 0000000..d78f090 --- /dev/null +++ b/migrations/0004_add_github_fields.sql @@ -0,0 +1,2 @@ +ALTER TABLE incidents ADD COLUMN github_repo TEXT; +ALTER TABLE incidents ADD COLUMN github_issue_number INTEGER; diff --git a/src/db.ts b/src/db.ts index 33ee48c..37e5bac 100644 --- a/src/db.ts +++ b/src/db.ts @@ -156,3 +156,151 @@ export async function pruneOldPings( .bind(cutoff) .run(); } + +import type { Incident, IncidentUpdate, IncidentWithUpdates } from "./types"; + +export async function createIncident( + db: D1Database, + data: { service_id: string; title: string; severity: "critical" | "major" | "minor"; github_repo?: string; github_issue_number?: number }, +): Promise { + const now = Math.floor(Date.now() / 1000); + const result = await db + .prepare( + "INSERT INTO incidents (service_id, title, status, severity, github_repo, github_issue_number, started_at, created_at, updated_at) VALUES (?, ?, 'investigating', ?, ?, ?, ?, ?, ?)", + ) + .bind(data.service_id, data.title, data.severity, data.github_repo ?? null, data.github_issue_number ?? null, now, now, now) + .run(); + const id = result.meta.last_row_id; + await db + .prepare( + "INSERT INTO incident_updates (incident_id, status, message, created_at) VALUES (?, 'investigating', 'Incident detected automatically', ?)", + ) + .bind(id, now) + .run(); + return id as number; +} + +export async function updateIncident( + db: D1Database, + id: number, + data: { status?: string; triage_report?: string; resolved_at?: number }, +): Promise { + const sets: string[] = []; + const values: unknown[] = []; + if (data.status) { sets.push("status = ?"); values.push(data.status); } + if (data.triage_report !== undefined) { sets.push("triage_report = ?"); values.push(data.triage_report); } + if (data.resolved_at) { sets.push("resolved_at = ?"); values.push(data.resolved_at); } + sets.push("updated_at = ?"); + values.push(Math.floor(Date.now() / 1000)); + values.push(id); + await db + .prepare(`UPDATE incidents SET ${sets.join(", ")} WHERE id = ?`) + .bind(...values) + .run(); +} + +export async function addIncidentUpdate( + db: D1Database, + incident_id: number, + status: string, + message: string, +): Promise { + const now = Math.floor(Date.now() / 1000); + await db + .prepare("INSERT INTO incident_updates (incident_id, status, message, created_at) VALUES (?, ?, ?, ?)") + .bind(incident_id, status, message, now) + .run(); +} + +export async function getActiveIncidents(db: D1Database): Promise { + const rows = await db + .prepare("SELECT * FROM incidents WHERE status != 'resolved' ORDER BY created_at DESC") + .all(); + return rows.results as unknown as Incident[]; +} + +export async function getActiveIncidentsWithUpdates(db: D1Database): Promise { + const incidents = await getActiveIncidents(db); + return Promise.all( + incidents.map(async (incident) => { + const updates = await db + .prepare("SELECT * FROM incident_updates WHERE incident_id = ? ORDER BY created_at ASC") + .bind(incident.id) + .all(); + return { ...incident, updates: updates.results as unknown as IncidentUpdate[] }; + }), + ); +} + +export async function getActiveIncidentForService( + db: D1Database, + service_id: string, +): Promise { + const row = await db + .prepare("SELECT * FROM incidents WHERE service_id = ? AND status != 'resolved' ORDER BY created_at DESC LIMIT 1") + .bind(service_id) + .first(); + return (row as unknown as Incident) ?? null; +} + +export async function getRecentIncidents(db: D1Database, days: number): Promise { + const since = Math.floor(Date.now() / 1000) - days * 24 * 60 * 60; + const rows = await db + .prepare("SELECT * FROM incidents WHERE resolved_at >= ? OR status != 'resolved' ORDER BY created_at DESC") + .bind(since) + .all(); + return rows.results as unknown as Incident[]; +} + +export async function getIncident(db: D1Database, id: number): Promise { + const incident = await db + .prepare("SELECT * FROM incidents WHERE id = ?") + .bind(id) + .first(); + if (!incident) return null; + const updates = await db + .prepare("SELECT * FROM incident_updates WHERE incident_id = ? ORDER BY created_at ASC") + .bind(id) + .all(); + return { + ...(incident as unknown as Incident), + updates: updates.results as unknown as IncidentUpdate[], + }; +} + +export async function getIncidentByGitHubIssue( + db: D1Database, + repo: string, + issueNumber: number, +): Promise { + const row = await db + .prepare("SELECT * FROM incidents WHERE github_repo = ? AND github_issue_number = ? LIMIT 1") + .bind(repo, issueNumber) + .first(); + return (row as unknown as Incident) ?? null; +} + +export async function setIncidentGitHub( + db: D1Database, + id: number, + repo: string, + issueNumber: number, +): Promise { + await db + .prepare("UPDATE incidents SET github_repo = ?, github_issue_number = ?, updated_at = ? WHERE id = ?") + .bind(repo, issueNumber, Math.floor(Date.now() / 1000), id) + .run(); +} + +export async function getRecentlyResolvedIncident( + db: D1Database, + service_id: string, + withinSeconds: number, +): Promise { + const since = Math.floor(Date.now() / 1000) - withinSeconds; + const row = await db + .prepare("SELECT * FROM incidents WHERE service_id = ? AND status = 'resolved' AND resolved_at >= ? ORDER BY resolved_at DESC LIMIT 1") + .bind(service_id, since) + .first(); + return (row as unknown as Incident) ?? null; +} diff --git a/src/github.ts b/src/github.ts new file mode 100644 index 0000000..db6986c --- /dev/null +++ b/src/github.ts @@ -0,0 +1,193 @@ +// GitHub Issues integration for incidents + +import type { Incident } from "./types"; +import { updateIncident, addIncidentUpdate } from "./db"; + +export function parseRepo(repoUrl: string): { owner: string; repo: string } | null { + const match = repoUrl.match(/github\.com\/([^/]+)\/([^/]+)/); + if (!match) return null; + return { owner: match[1], repo: match[2].replace(/\.git$/, "") }; +} + +export async function createIssue( + token: string, + owner: string, + repo: string, + opts: { title: string; body: string; assignees?: string[]; labels?: string[] }, +): Promise { + const res = await fetch(`https://api.github.com/repos/${owner}/${repo}/issues`, { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "User-Agent": "infra-status-worker", + }, + body: JSON.stringify(opts), + }); + if (!res.ok) { + const text = await res.text(); + throw new Error(`GitHub create issue failed: ${res.status} ${text}`); + } + const data = await res.json<{ number: number }>(); + return data.number; +} + +export async function commentOnIssue( + token: string, + owner: string, + repo: string, + issueNumber: number, + body: string, +): Promise { + const res = await fetch( + `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}/comments`, + { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "User-Agent": "infra-status-worker", + }, + body: JSON.stringify({ body }), + }, + ); + if (!res.ok) { + const text = await res.text(); + throw new Error(`GitHub comment failed: ${res.status} ${text}`); + } +} + +export async function closeIssue( + token: string, + owner: string, + repo: string, + issueNumber: number, +): Promise { + const res = await fetch( + `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}`, + { + method: "PATCH", + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "User-Agent": "infra-status-worker", + }, + body: JSON.stringify({ state: "closed" }), + }, + ); + if (!res.ok) { + const text = await res.text(); + throw new Error(`GitHub close issue failed: ${res.status} ${text}`); + } +} + +export async function editIssueBody( + token: string, + owner: string, + repo: string, + issueNumber: number, + body: string, +): Promise { + const res = await fetch( + `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}`, + { + method: "PATCH", + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "User-Agent": "infra-status-worker", + }, + body: JSON.stringify({ body }), + }, + ); + if (!res.ok) { + const text = await res.text(); + throw new Error(`GitHub edit issue failed: ${res.status} ${text}`); + } +} + +interface GitHubIssue { + state: string; + body: string | null; +} + +interface GitHubComment { + id: number; + body: string; + created_at: string; + user: { login: string; type: string }; +} + +async function fetchIssue(token: string, owner: string, repo: string, issueNumber: number): Promise { + const res = await fetch(`https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}`, { + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "User-Agent": "infra-status-worker", + }, + }); + if (!res.ok) throw new Error(`GitHub fetch issue failed: ${res.status}`); + return res.json(); +} + +async function fetchComments(token: string, owner: string, repo: string, issueNumber: number, since?: string): Promise { + let url = `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}/comments?per_page=50`; + if (since) url += `&since=${since}`; + const res = await fetch(url, { + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "User-Agent": "infra-status-worker", + }, + }); + if (!res.ok) throw new Error(`GitHub fetch comments failed: ${res.status}`); + return res.json(); +} + +export async function syncGitHubIncidents( + db: D1Database, + kv: KVNamespace, + token: string, + incidents: Incident[], +): Promise { + for (const incident of incidents) { + if (!incident.github_repo || !incident.github_issue_number) continue; + + const parsed = parseRepo(`https://github.com/${incident.github_repo}`); + if (!parsed) continue; + + try { + // Check issue state (closed = resolved) and sync body edits + const issue = await fetchIssue(token, parsed.owner, parsed.repo, incident.github_issue_number); + if (issue.state === "closed" && incident.status !== "resolved") { + const now = Math.floor(Date.now() / 1000); + await updateIncident(db, incident.id, { status: "resolved", resolved_at: now }); + await addIncidentUpdate(db, incident.id, "resolved", "Issue closed on GitHub"); + continue; + } + + // Sync issue body edits back to triage_report + if (issue.body && issue.body !== incident.triage_report) { + await updateIncident(db, incident.id, { triage_report: issue.body }); + } + + // Sync new comments since last check + const kvKey = `gh_sync:${incident.id}:last`; + const lastSeen = await kv.get(kvKey); + const comments = await fetchComments(token, parsed.owner, parsed.repo, incident.github_issue_number, lastSeen ?? undefined); + + // Filter to human comments only (skip bots and our own posts) + const human = comments.filter((c) => c.user.type !== "Bot" && !c.body.startsWith("Automated incident detected") && !c.body.startsWith("## Triage Report") && !c.body.startsWith("Service recovered automatically")); + + for (const comment of human) { + await addIncidentUpdate(db, incident.id, incident.status, comment.body); + } + + // Track last comment time so we don't re-import + if (comments.length > 0) { + const latest = comments[comments.length - 1].created_at; + await kv.put(kvKey, latest, { expirationTtl: 86400 * 7 }); + } + } catch (_) {} // best effort, don't block other syncs + } +} diff --git a/src/index.ts b/src/index.ts index d626d2b..8d01862 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,13 +1,15 @@ import type { Env } from "./types"; import { getManifest } from "./manifest"; import { checkHealth } from "./health"; -import { insertPing, pruneOldPings } from "./db"; +import { insertPing, getLatestPing, pruneOldPings, createIncident, updateIncident, addIncidentUpdate, getActiveIncidentForService, getActiveIncidents, getRecentlyResolvedIncident, setIncidentGitHub } from "./db"; import { refreshDevices } from "./tailscale"; import { handleStatusRoute } from "./routes/status"; import { handleFavicon } from "./routes/favicon"; import { handleUptime } from "./routes/uptime"; import { handleBadgeRoute } from "./routes/badge"; import { handleIndex } from "./routes/index"; +import { handleIncidentRoute } from "./routes/incidents"; +import { createIssue, commentOnIssue, closeIssue, parseRepo, syncGitHubIncidents } from "./github"; import { schemas } from "./schemas"; async function handleRequest(request: Request, env: Env): Promise { @@ -54,6 +56,11 @@ async function handleRequest(request: Request, env: Env): Promise { if (badge) return badge; } + if (path.startsWith("/api/incidents")) { + const res = await handleIncidentRoute(request, env, path); + if (res) return res; + } + return new Response("Not Found", { status: 404 }); } @@ -63,8 +70,8 @@ export default { return new Response(null, { headers: { "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Methods": "GET, OPTIONS", - "Access-Control-Allow-Headers": "Content-Type", + "Access-Control-Allow-Methods": "GET, POST, PATCH, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type, Authorization", }, }); } @@ -81,16 +88,107 @@ export default { refreshDevices(env), ]); - const checks = Object.values(manifest).flatMap((machine) => - machine.services + const checks = Object.values(manifest).flatMap((machine) => { + const triageUrl = machine.triage_url; + return machine.services .filter((svc) => svc.health_url) .map(async (svc) => { + const previous = await getLatestPing(env.DB, svc.name); const result = await checkHealth(svc); await insertPing(env.DB, svc.name, result.status, result.latency_ms); - }), - ); + + const isDown = result.status === "down" || result.status === "timeout"; + const wasUp = !previous || previous.status === "up" || previous.status === "degraded"; + + if (isDown) { + // Track consecutive failures in KV for flap prevention + const failKey = `triage:${svc.name}:failures`; + const current = parseInt((await env.KV.get(failKey)) ?? "0"); + const failures = current + 1; + await env.KV.put(failKey, String(failures), { expirationTtl: 1800 }); + + // Only trigger after 2 consecutive failures (10 min of downtime) + if (failures >= 2) { + const existing = await getActiveIncidentForService(env.DB, svc.name); + if (!existing) { + // Check cooldown: no incident resolved in last 15 min + const recent = await getRecentlyResolvedIncident(env.DB, svc.name, 900); + if (!recent) { + const id = await createIncident(env.DB, { + service_id: svc.name, + title: `${svc.name} is ${result.status}`, + severity: "major", + }); + + // Create GitHub issue on the service's repo + if (env.GITHUB_TOKEN && svc.repository) { + const parsed = parseRepo(svc.repository); + if (parsed) { + try { + const issueNumber = await createIssue(env.GITHUB_TOKEN, parsed.owner, parsed.repo, { + title: `${svc.name} is ${result.status}`, + body: `Automated incident detected by [infra.dunkirk.sh](https://infra.dunkirk.sh)\n\n**Service:** ${svc.name}\n**Health URL:** ${svc.health_url}\n**Status:** ${result.status}\n**Detected at:** ${new Date().toISOString()}\n\n---\n*Comments on this issue will appear on the status page. Close the issue to resolve the incident.*`, + assignees: env.GITHUB_ASSIGNEE ? [env.GITHUB_ASSIGNEE] : [], + labels: ["incident"], + }); + await setIncidentGitHub(env.DB, id, `${parsed.owner}/${parsed.repo}`, issueNumber); + } catch (_) {} // best effort + } + } + + // Fire webhook to triage agent (non-blocking) + if (triageUrl && env.TRIAGE_AUTH_TOKEN) { + fetch(triageUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${env.TRIAGE_AUTH_TOKEN}`, + }, + body: JSON.stringify({ + incident_id: id, + service_id: svc.name, + service_name: svc.name, + health_url: svc.health_url, + callback_url: `https://infra.dunkirk.sh/api/incidents/${id}`, + }), + }).catch(() => {}); // fire and forget + } + } + } + } + } else { + // Service is up — clear failure counter + await env.KV.delete(`triage:${svc.name}:failures`); + + // Auto-resolve active incidents + const active = await getActiveIncidentForService(env.DB, svc.name); + if (active) { + await updateIncident(env.DB, active.id, { + status: "resolved", + resolved_at: Math.floor(Date.now() / 1000), + }); + await addIncidentUpdate(env.DB, active.id, "resolved", "Service recovered automatically"); + + // Close the GitHub issue + if (env.GITHUB_TOKEN && active.github_repo && active.github_issue_number) { + const parsed = parseRepo(`https://github.com/${active.github_repo}`); + if (parsed) { + commentOnIssue(env.GITHUB_TOKEN, parsed.owner, parsed.repo, active.github_issue_number, "Service recovered automatically. Closing issue.").catch(() => {}); + closeIssue(env.GITHUB_TOKEN, parsed.owner, parsed.repo, active.github_issue_number).catch(() => {}); + } + } + } + } + }); + }); await Promise.all(checks); await pruneOldPings(env.DB, 365); + + // Sync GitHub issue comments/state back to incidents + if (env.GITHUB_TOKEN) { + const active = await getActiveIncidents(env.DB); + await syncGitHubIncidents(env.DB, env.KV, env.GITHUB_TOKEN, active); + } }, } satisfies ExportedHandler; diff --git a/src/overall.ts b/src/overall.ts index a55b058..19d7cb0 100644 --- a/src/overall.ts +++ b/src/overall.ts @@ -1,6 +1,6 @@ import type { Env } from "./types"; import { getManifest } from "./manifest"; -import { getLatestPing } from "./db"; +import { getLatestPing, getActiveIncidents } from "./db"; import { getDeviceStatus } from "./tailscale"; export type OverallGrade = "up" | "degraded" | "down"; @@ -42,7 +42,11 @@ export async function getOverallStatus(env: Env): Promise { s === "misconfigured", ); - if (onFire) return { grade: "down", label: "On fire" }; - if (hasDegraded) return { grade: "degraded", label: "Some systems degraded" }; + const activeIncidents = await getActiveIncidents(env.DB); + const hasCritical = activeIncidents.some((i) => i.severity === "critical"); + const hasMajor = activeIncidents.some((i) => i.severity === "major"); + + if (hasCritical || onFire) return { grade: "down", label: "On fire" }; + if (hasMajor || hasDegraded) return { grade: "degraded", label: "Some systems degraded" }; return { grade: "up", label: "All systems operational" }; } diff --git a/src/routes/incidents.ts b/src/routes/incidents.ts new file mode 100644 index 0000000..a2e5a85 --- /dev/null +++ b/src/routes/incidents.ts @@ -0,0 +1,101 @@ +import type { Env } from "../types"; +import { + getActiveIncidents, + getRecentIncidents, + getIncident, + createIncident, + updateIncident, + addIncidentUpdate, +} from "../db"; +import { commentOnIssue, editIssueBody, parseRepo } from "../github"; + +function authCheck(request: Request, env: Env): boolean { + const auth = request.headers.get("Authorization"); + if (!auth || !env.TRIAGE_AUTH_TOKEN) return false; + return auth === `Bearer ${env.TRIAGE_AUTH_TOKEN}`; +} + +export async function handleIncidentRoute( + request: Request, + env: Env, + path: string, +): Promise { + // GET /api/incidents + if (path === "/api/incidents" && request.method === "GET") { + const active = await getActiveIncidents(env.DB); + const recent = await getRecentIncidents(env.DB, 7); + const resolved = recent.filter((i) => i.status === "resolved"); + return Response.json({ active, recent_resolved: resolved }); + } + + // POST /api/incidents + if (path === "/api/incidents" && request.method === "POST") { + if (!authCheck(request, env)) { + return Response.json({ error: "unauthorized" }, { status: 401 }); + } + const body = await request.json<{ service_id: string; title: string; severity: "critical" | "major" | "minor" }>(); + if (!body.service_id || !body.title || !body.severity) { + return Response.json({ error: "missing fields" }, { status: 400 }); + } + const id = await createIncident(env.DB, body); + return Response.json({ id }, { status: 201 }); + } + + // GET /api/incidents/:id + const singleMatch = path.match(/^\/api\/incidents\/(\d+)$/); + if (singleMatch && request.method === "GET") { + const incident = await getIncident(env.DB, parseInt(singleMatch[1])); + if (!incident) return Response.json({ error: "not found" }, { status: 404 }); + return Response.json(incident); + } + + // PATCH /api/incidents/:id + if (singleMatch && request.method === "PATCH") { + if (!authCheck(request, env)) { + return Response.json({ error: "unauthorized" }, { status: 401 }); + } + const id = parseInt(singleMatch[1]); + const body = await request.json<{ status?: string; triage_report?: string }>(); + const updateData: { status?: string; triage_report?: string; resolved_at?: number } = {}; + if (body.status) updateData.status = body.status; + if (body.triage_report !== undefined) updateData.triage_report = body.triage_report; + if (body.status === "resolved") updateData.resolved_at = Math.floor(Date.now() / 1000); + await updateIncident(env.DB, id, updateData); + if (body.status) { + await addIncidentUpdate(env.DB, id, body.status, body.triage_report ?? `Status changed to ${body.status}`); + } + + // Sync triage report to GitHub issue + const incident = await getIncident(env.DB, id); + if (env.GITHUB_TOKEN && incident?.github_repo && incident.github_issue_number) { + const parsed = parseRepo(`https://github.com/${incident.github_repo}`); + if (parsed) { + if (body.triage_report) { + // Post triage report as comment and update issue body + commentOnIssue(env.GITHUB_TOKEN, parsed.owner, parsed.repo, incident.github_issue_number, `## Triage Report\n\n${body.triage_report}`).catch(() => {}); + editIssueBody(env.GITHUB_TOKEN, parsed.owner, parsed.repo, incident.github_issue_number, body.triage_report).catch(() => {}); + } else if (body.status) { + commentOnIssue(env.GITHUB_TOKEN, parsed.owner, parsed.repo, incident.github_issue_number, `Status changed to **${body.status}**`).catch(() => {}); + } + } + } + + return Response.json({ ok: true }); + } + + // POST /api/incidents/:id/updates + const updatesMatch = path.match(/^\/api\/incidents\/(\d+)\/updates$/); + if (updatesMatch && request.method === "POST") { + if (!authCheck(request, env)) { + return Response.json({ error: "unauthorized" }, { status: 401 }); + } + const body = await request.json<{ status: string; message: string }>(); + if (!body.status || !body.message) { + return Response.json({ error: "missing fields" }, { status: 400 }); + } + await addIncidentUpdate(env.DB, parseInt(updatesMatch[1]), body.status, body.message); + return Response.json({ ok: true }, { status: 201 }); + } + + return null; +} diff --git a/src/routes/index.ts b/src/routes/index.ts index 0f85a21..c0f5f75 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -1,6 +1,6 @@ import type { Env } from "../types"; import { getManifest } from "../manifest"; -import { getLatestPing, getUptime7d, getOverallUptimeDays, getLastCheckTime } from "../db"; +import { getLatestPing, getUptime7d, getOverallUptimeDays, getLastCheckTime, getActiveIncidentsWithUpdates, getRecentIncidents } from "../db"; import { getDeviceStatus } from "../tailscale"; import { getOverallStatus } from "../overall"; import { COMMIT_SHA } from "../version"; @@ -40,6 +40,9 @@ export async function handleIndex(env: Env): Promise { const { grade: overallClass, label: overallText } = await getOverallStatus(env); const uptimeDays = await getOverallUptimeDays(env.DB, 90); + const activeIncidents = await getActiveIncidentsWithUpdates(env.DB); + const recentIncidents = await getRecentIncidents(env.DB, 7); + const resolvedIncidents = recentIncidents.filter((i) => i.status === "resolved"); const html = ` @@ -97,6 +100,37 @@ export async function handleIndex(env: Env): Promise { .uptime-bar .day.degraded { background: #f39c12; } .uptime-bar .day.down { background: #e74c3c; } .uptime-bar .day.none { background: #21262d; } + .incidents { margin-bottom: 1.5rem; } + .incident-banner { background: #2d1b1b; border: 1px solid #e74c3c; border-radius: 6px; padding: 0.75rem; margin-bottom: 0.5rem; } + .incident-banner.major { border-color: #f39c12; background: #2d2517; } + .incident-banner.minor { border-color: #8b949e; background: #21262d; } + .incident-title { font-size: 0.85rem; font-weight: 500; margin-bottom: 0.25rem; } + .incident-meta { font-size: 0.7rem; color: #8b949e; display: flex; gap: 0.75rem; } + .incident-status { text-transform: uppercase; letter-spacing: 0.05em; } + .incident-status.investigating { color: #e74c3c; } + .incident-status.identified { color: #f39c12; } + .incident-status.monitoring { color: #3498db; } + .incident-triage { margin-top: 0.5rem; } + .incident-triage summary { font-size: 0.75rem; color: #8b949e; cursor: pointer; } + .incident-triage pre { font-size: 0.7rem; color: #c9d1d9; background: #161b22; padding: 0.5rem; border-radius: 4px; margin-top: 0.25rem; white-space: pre-wrap; word-break: break-word; max-height: 300px; overflow-y: auto; } + .incident-timeline { margin-top: 0.5rem; padding-left: 0.75rem; border-left: 2px solid #21262d; } + .timeline-entry { padding: 0.25rem 0 0.25rem 0.5rem; font-size: 0.7rem; position: relative; } + .timeline-entry::before { content: ''; position: absolute; left: -0.75rem; top: 0.55rem; width: 6px; height: 6px; border-radius: 50%; background: #30363d; transform: translateX(-2px); } + .timeline-entry.investigating::before { background: #e74c3c; } + .timeline-entry.identified::before { background: #f39c12; } + .timeline-entry.monitoring::before { background: #3498db; } + .timeline-entry.resolved::before { background: #2ecc71; } + .timeline-status { text-transform: uppercase; letter-spacing: 0.04em; font-size: 0.6rem; font-weight: 600; } + .timeline-status.investigating { color: #e74c3c; } + .timeline-status.identified { color: #f39c12; } + .timeline-status.monitoring { color: #3498db; } + .timeline-status.resolved { color: #2ecc71; } + .timeline-time { color: #484f58; margin-right: 0.5rem; } + .timeline-msg { color: #8b949e; } + .resolved-incidents { margin-top: 0.75rem; } + .resolved-header { font-size: 0.7rem; text-transform: uppercase; letter-spacing: 0.08em; color: #8b949e; margin-bottom: 0.25rem; } + .resolved-item { font-size: 0.75rem; color: #8b949e; padding: 0.25rem 0; border-bottom: 1px solid #21262d; } + .resolved-item:last-child { border-bottom: none; } footer { margin-top: auto; padding-top: 1rem; border-top: 1px solid #21262d; font-size: 0.7rem; color: #8b949e; } .footer-meta { display: flex; justify-content: space-between; } footer a { color: #8b949e; text-decoration: none; } @@ -107,6 +141,24 @@ export async function handleIndex(env: Env): Promise {
${uptimeDays.map((d) => `
`).join("")}

infra.dunkirk.sh

${overallText}

+${activeIncidents.length > 0 ? `
+${activeIncidents.map((i) => `
+
${esc(i.title)}
+
+ ${i.status} + ${esc(i.service_id)} + started loading +
+ ${i.triage_report ? `
triage report
${esc(i.triage_report)}
` : ""} + ${i.updates.length > 0 ? `
+${i.updates.map((u) => `
+ ${new Date(u.created_at * 1000).toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit", hour12: false, timeZone: "America/New_York" })} + ${esc(u.status)} + ${esc(u.message)} +
`).join("\n")} +
` : ""} +
`).join("\n")} +
` : ""} ${servers .map( (m) => `
@@ -133,6 +185,10 @@ ${clients.length > 0 ? `
${clients.map((m) => `${esc(m.name)}`).join("\n")}
` : ""} +${resolvedIncidents.length > 0 ? `
+
recent incidents
+${resolvedIncidents.map((i) => `
${esc(i.title)} — resolved loading
`).join("\n")} +
` : ""} diff --git a/src/types.ts b/src/types.ts index 6eacdc6..db24bc8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -17,6 +17,7 @@ export interface Machine { hostname: string; tailscale_host: string; type: "server" | "client"; + triage_url: string | null; services: Service[]; } @@ -31,8 +32,38 @@ export interface UptimeResponse { }[]; } +export interface Incident { + id: number; + service_id: string; + title: string; + status: "investigating" | "identified" | "monitoring" | "resolved"; + severity: "critical" | "major" | "minor"; + triage_report: string | null; + github_repo: string | null; + github_issue_number: number | null; + started_at: number; + resolved_at: number | null; + created_at: number; + updated_at: number; +} + +export interface IncidentUpdate { + id: number; + incident_id: number; + status: string; + message: string; + created_at: number; +} + +export interface IncidentWithUpdates extends Incident { + updates: IncidentUpdate[]; +} + export interface Env { DB: D1Database; KV: KVNamespace; TAILSCALE_API_KEY?: string; + TRIAGE_AUTH_TOKEN?: string; + GITHUB_TOKEN?: string; + GITHUB_ASSIGNEE?: string; } diff --git a/wrangler.toml b/wrangler.toml index 0b43241..b9072fb 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -14,3 +14,6 @@ migrations_dir = "./migrations" [[kv_namespaces]] binding = "KV" id = "c4daf2e998ea45e1aad9efb6636b66df" + +[vars] +GITHUB_ASSIGNEE = "taciturnaxolotl"