diff --git a/migrations/0002_add_statuses.sql b/migrations/0002_add_statuses.sql new file mode 100644 index 0000000..a440223 --- /dev/null +++ b/migrations/0002_add_statuses.sql @@ -0,0 +1,17 @@ +-- Drop the old CHECK constraint and add new one with additional statuses +-- SQLite doesn't support ALTER CONSTRAINT, so we recreate the table +CREATE TABLE pings_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + service_id TEXT NOT NULL, + timestamp INTEGER NOT NULL, + status TEXT NOT NULL CHECK (status IN ('up', 'degraded', 'misconfigured', 'down', 'unknown')), + latency_ms INTEGER +); + +INSERT INTO pings_new SELECT * FROM pings; + +DROP TABLE pings; + +ALTER TABLE pings_new RENAME TO pings; + +CREATE INDEX idx_pings_service_ts ON pings (service_id, timestamp DESC); diff --git a/src/health.ts b/src/health.ts index ca78efa..520ebc7 100644 --- a/src/health.ts +++ b/src/health.ts @@ -1,13 +1,17 @@ import type { Service } from "./types"; +const SLOW_THRESHOLD_MS = 3000; + +export type Status = "up" | "degraded" | "misconfigured" | "down" | "unknown"; + interface HealthResult { - status: "up" | "degraded" | "down"; + status: Status; latency_ms: number; } export async function checkHealth(service: Service): Promise { if (!service.health_url) { - return { status: "unknown" as "down", latency_ms: 0 }; + return { status: "unknown", latency_ms: 0 }; } const start = Date.now(); @@ -19,12 +23,18 @@ export async function checkHealth(service: Service): Promise { }); const latency_ms = Date.now() - start; - if (res.status >= 200 && res.status < 300) { - return { status: "up", latency_ms }; + if (res.status >= 400 && res.status < 500) { + return { status: "misconfigured", latency_ms }; } if (res.status >= 500) { return { status: "degraded", latency_ms }; } + if (res.status >= 200 && res.status < 300) { + if (latency_ms > SLOW_THRESHOLD_MS) { + return { status: "degraded", latency_ms }; + } + return { status: "up", latency_ms }; + } return { status: "down", latency_ms }; } catch { return { status: "down", latency_ms: Date.now() - start }; diff --git a/src/routes/badge.ts b/src/routes/badge.ts index 4d0202f..926f6ec 100644 --- a/src/routes/badge.ts +++ b/src/routes/badge.ts @@ -6,6 +6,8 @@ import { getDeviceStatus } from "../tailscale"; const COLORS: Record = { up: "#3cc068", degraded: "#f0ad4e", + misconfigured: "#9b59b6", + partial: "#f0ad4e", down: "#e05d44", unknown: "#9f9f9f", }; @@ -13,6 +15,8 @@ const COLORS: Record = { const STATUS_LABELS: Record = { up: "operational", degraded: "degraded", + misconfigured: "misconfigured", + partial: "partial", down: "down", unknown: "unknown", }; @@ -149,7 +153,10 @@ function esc(s: string): string { } function worstStatus(statuses: string[]): string { - if (statuses.includes("down")) return "down"; + if (statuses.length === 0) return "unknown"; + if (statuses.every((s) => s === "down")) return "down"; + if (statuses.includes("down")) return "partial"; + if (statuses.includes("misconfigured")) return "misconfigured"; if (statuses.includes("degraded")) return "degraded"; if (statuses.includes("unknown")) return "unknown"; return "up"; diff --git a/src/routes/index.ts b/src/routes/index.ts index 0a6c0e7..eb7823d 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -37,13 +37,23 @@ export async function handleIndex(env: Env): Promise { const servers = machines.filter((m) => m.type === "server"); const clients = machines.filter((m) => m.type === "client"); - const allUp = machines - .filter((m) => m.type === "server") - .every( - (m) => - m.online && - m.services.every((s) => s.status === "up" || s.status === "unknown"), - ); + const activeServers = servers.filter((m) => m.services.length > 0); + const anyServerOffline = activeServers.some((m) => !m.online); + const svcStatuses = activeServers.flatMap((m) => m.services.map((s) => s.status)); + const downCount = svcStatuses.filter((s) => s === "down").length; + const downRatio = svcStatuses.length > 0 ? downCount / svcStatuses.length : 0; + const onFire = anyServerOffline || downRatio >= 0.4; + const hasDegraded = + svcStatuses.includes("down") || + svcStatuses.includes("degraded") || + svcStatuses.includes("misconfigured") || + svcStatuses.includes("partial"); + const overallClass = onFire ? "down" : hasDegraded ? "degraded" : "up"; + const overallText = onFire + ? "On fire" + : hasDegraded + ? "Some systems degraded" + : "All systems operational"; const html = ` @@ -61,6 +71,7 @@ export async function handleIndex(env: Env): Promise { .dot.up { background: #2ecc71; } .dot.degraded { background: #f39c12; } .dot.down { background: #e74c3c; } + .dot.misconfigured { background: #9b59b6; } .dot.unknown { background: #8b949e; } .dot.online { background: #2ecc71; } .dot.offline { background: #e74c3c; } @@ -88,11 +99,11 @@ export async function handleIndex(env: Env): Promise {

infra.dunkirk.sh

-

${allUp ? "All systems operational" : "Some systems degraded"}

+

${overallText}

${servers .map( (m) => `
-
${esc(m.name)}${esc(m.type)}
+
${esc(m.name)}${esc(m.type)}
${m.services.length === 0 ? `
no services
` : m.services .map( (s) => `
@@ -112,7 +123,7 @@ ${m.services.length === 0 ? `
no services
` : m.ser ${clients.length > 0 ? `
devices
-${clients.map((m) => `${esc(m.name)}`).join("\n")} +${clients.map((m) => `${esc(m.name)}`).join("\n")}
` : ""}
${lastCheckISO ? `updated loading` : "no checks yet"}${COMMIT_SHA}
diff --git a/src/routes/status.ts b/src/routes/status.ts index ec724d3..4685f61 100644 --- a/src/routes/status.ts +++ b/src/routes/status.ts @@ -6,13 +6,16 @@ import { getDeviceStatus } from "../tailscale"; const JSON_HEADERS = { "Access-Control-Allow-Origin": "*" }; function worstStatus(statuses: string[]): string { - if (statuses.includes("down")) return "down"; + if (statuses.length === 0) return "unknown"; + if (statuses.every((s) => s === "down")) return "down"; + if (statuses.includes("down")) return "partial"; + if (statuses.includes("misconfigured")) return "misconfigured"; if (statuses.includes("degraded")) return "degraded"; if (statuses.includes("unknown")) return "unknown"; return "up"; } -// GET /api/status or /api/status/overall +// GET /api/status/overall async function overallStatus(env: Env): Promise { const manifest = await getManifest(env); const allServices = Object.values(manifest).flatMap((m) => m.services); @@ -47,6 +50,52 @@ async function overallStatus(env: Env): Promise { ); } +// GET /api/status +async function fullStatus(env: Env): Promise { + const manifest = await getManifest(env); + + const machines = await Promise.all( + Object.entries(manifest).map(async ([name, machine]) => { + const online = await getDeviceStatus(env, machine.tailscale_host); + const services = await Promise.all( + machine.services.map(async (svc) => { + const ping = await getLatestPing(env.DB, svc.name); + const uptime = await getUptime7d(env.DB, svc.name); + return { + id: svc.name, + status: (ping?.status ?? "unknown") as string, + latency_ms: ping?.latency_ms ?? null, + uptime_7d: uptime, + }; + }), + ); + const svcStatuses = services.map((s) => s.status); + return { + name, + hostname: machine.hostname, + type: machine.type, + online, + status: online ? worstStatus(svcStatuses) : "down", + services, + }; + }), + ); + + const allStatuses = machines + .filter((m) => m.type === "server") + .flatMap((m) => [m.online ? "up" : "down", ...m.services.map((s) => s.status)]); + const status = worstStatus(allStatuses); + + return Response.json( + { + ok: status === "up", + status, + machines, + }, + { headers: JSON_HEADERS }, + ); +} + // GET /api/status/service/:id async function serviceStatus(env: Env, id: string): Promise { const ping = await getLatestPing(env.DB, id); @@ -110,7 +159,11 @@ export async function handleStatusRoute( env: Env, path: string, ): Promise { - if (path === "/api/status" || path === "/api/status/overall") { + if (path === "/api/status") { + return fullStatus(env); + } + + if (path === "/api/status/overall") { return overallStatus(env); }