#!/usr/bin/env node // ac-fleet — what commit is every box actually running? // // Reachability has never been the hard question; provenance is. On 2026-08-08 // session-server served a four-hour-old commit while lith, the CDN, the oven // and its own health gate all read green, and the only thing that surfaced it // was somebody typing about it in laklok. // // ac-fleet the fleet, and how far each box is behind knot // ac-fleet --json same, machine-readable // // Asks silo (which polls the fleet continuously) and falls back to polling the // services directly — silo going down should not also blind you to the fleet. // The "behind" column is computed HERE, because this machine holds the history // and neither silo nor the boxes do. import { execFile } from "node:child_process"; import { join } from "node:path"; const JSON_OUT = process.argv.includes("--json"); const REPO = join(import.meta.dirname, ".."); const SILO = "https://silo.aesthetic.computer/api/fleet/drift"; // What each service actually reads. Not a guess: session-server's list is its // own sparse-checkout set, which is the box declaring which files it needs. // // This is what makes the tool worth reading. Measured against knot's tip, every // box goes red the moment anyone commits anything, and a signal that is always // red is one you stop looking at — the exact failure this was built to escape. // A box is behind when it is missing a commit that touched ITS paths. const FLEET = [ { service: "session-server", health: "https://session-server.aesthetic.computer/health", paths: ["session-server/", "shared/", "system/public/aesthetic.computer/lib/"] }, { service: "silo", health: "https://silo.aesthetic.computer/health", paths: ["silo/"] }, { service: "oven", health: "https://oven.aesthetic.computer/health", paths: ["oven/"] }, { service: "lith", health: "https://aesthetic.computer/api/health", paths: ["lith/", "system/", "shared/"] }, ]; const git = (args) => new Promise((resolve) => { execFile("git", args, { cwd: REPO, timeout: 15000 }, (err, out) => resolve(err ? null : out.trim())); }); async function viaSilo() { const r = await fetch(SILO, { signal: AbortSignal.timeout(20000) }); if (!r.ok) throw new Error(`silo HTTP ${r.status}`); return { ...(await r.json()), source: "silo" }; } // Same shape silo returns, assembled locally. async function direct() { const tip = async (remote) => { const line = await git(["ls-remote", remote, "refs/heads/main"]); return line ? line.split(/\s+/)[0] : null; }; const [knot, mirror] = await Promise.all([tip("origin"), tip("github")]); const services = await Promise.all(FLEET.map(async ({ service, health }) => { try { // A degraded service still answers with its sha (silo returns 503 when // mongo is unreachable), so read the body before judging the status — // otherwise "unhealthy" would erase "and here is what it is running". const r = await fetch(health, { signal: AbortSignal.timeout(8000) }); const body = await r.json().catch(() => null); const sha = typeof body?.sha === "string" ? body.sha : null; // A healthy answer with no sha means the service hasn't adopted the // contract yet — say that, rather than print a 200 as if it were a fault. if (!sha) return { service, sha: null, note: r.ok ? "no provenance yet" : `HTTP ${r.status}` }; return { service, sha, dirty: body?.dirty === true, degraded: r.ok ? undefined : `degraded (HTTP ${r.status})` }; } catch (e) { return { service, sha: null, note: e.name === "TimeoutError" ? "timeout" : "unreachable" }; } })); return { knot, mirror, mirrorInSync: knot && mirror ? knot === mirror : null, services, checkedAt: new Date().toISOString(), source: "direct" }; } let report; try { report = await viaSilo(); } catch (e) { if (!JSON_OUT) console.log(`(silo unavailable: ${e.message} — polling directly)\n`); report = await direct(); } // The verdict is reached HERE, never taken from silo: silo has no checkout, so // the best it can say is whether a sha equals knot's tip, and that reads every // unrelated commit as a fleet-wide outage. for (const s of report.services) { if (!s.sha || !report.knot) continue; const paths = FLEET.find((f) => f.service === s.service)?.paths ?? []; const range = `${s.sha}..${report.knot}`; const total = await git(["rev-list", "--count", range]); const mine = paths.length ? await git(["rev-list", "--count", range, "--", ...paths]) : total; if (mine === null) { s.current = null; s.note ??= "history unavailable"; continue; } s.behind = Number(mine); s.unrelated = Number(total ?? 0) - s.behind; // A box running files that are in no commit is never "current", however // flattering its sha looks — that sha names a commit it does not contain. s.current = s.behind === 0 && !s.dirty; } if (JSON_OUT) { console.log(JSON.stringify(report, null, 2)); process.exit(0); } const short = (s) => (s ? s.slice(0, 9) : "—"); console.log(`\n🛰 fleet provenance · knot ${short(report.knot)} · via ${report.source}\n`); const mirrorIcon = report.mirrorInSync === true ? "✅" : report.mirrorInSync === null ? "⚠️ " : "❌"; console.log(` ${mirrorIcon} github mirror ${short(report.mirror)}` + (report.mirrorInSync === false ? " DRIFTED — pull-based deploys will be stale" : "")); console.log(""); let stale = 0; for (const s of report.services) { const icon = s.current === true ? "✅" : s.sha ? "❌" : "⚠️ "; if (s.sha && s.current !== true) stale++; // "current" means nothing it depends on has moved — unrelated commits are // reported quietly so the count stays honest without turning the row red. const detail = s.dirty ? "running uncommitted files — sha is not the truth" : s.current === true ? (s.unrelated ? `current (${s.unrelated} unrelated)` : "current") : s.behind !== undefined ? `${s.behind} commit${s.behind === 1 ? "" : "s"} behind on its own paths` : s.note || "drifted"; console.log(` ${icon} ${s.service.padEnd(16)} ${short(s.sha).padEnd(11)} ${detail}`); } const unknown = report.services.filter((s) => !s.sha).length; console.log(`\n ${report.services.length - stale - unknown}/${report.services.length} current` + (stale ? ` · ${stale} STALE` : "") + (unknown ? ` · ${unknown} not reporting` : "") + "\n"); process.exit(stale > 0 || report.mirrorInSync === false ? 1 : 0);