diff --git a/src/api/relay.ts b/src/api/relay.ts index 0172f21..5a6cfb7 100644 --- a/src/api/relay.ts +++ b/src/api/relay.ts @@ -68,11 +68,13 @@ async function listRecords( return values; } -// pre-~v1.x records carry a host field with a tid rkey; newer ones drop the -// field entirely and the rkey IS the hostname (at://did/sh.tangled.knot/knot.example.com) -const KnotRecord = z.object({ host: z.string().min(1).optional() }); +// service records (knots, spindles) share a shape: pre-~v1.x ones carry a host +// field with a tid rkey; newer ones drop the field and the rkey IS the hostname +// (at://did/sh.tangled.knot/knot.example.com, at://did/sh.tangled.spindle/spindle.example.com) +const ServiceRecord = z.object({ host: z.string().min(1).optional() }); const RepoRecord = z.object({ knot: z.string().min(1), + spindle: z.string().optional(), description: z.string().optional(), createdAt: z.string().optional(), }); @@ -80,20 +82,22 @@ const RepoRecord = z.object({ export type RepoRef = { name: string; knot: string; + spindle?: string; description?: string; createdAt?: string; }; const host = (h: string) => h.trim().toLowerCase(); -// hosts this owner has verified as knots (malformed records skipped, not fatal) -export async function fetchKnotHosts( +// hosts this owner has verified for a service collection (malformed records skipped, not fatal) +export async function fetchServiceHosts( pds: string, did: string, + collection: "sh.tangled.knot" | "sh.tangled.spindle", ): Promise { - const records = await listRecords(pds, did, "sh.tangled.knot"); + const records = await listRecords(pds, did, collection); return records.flatMap((r) => { - const parsed = KnotRecord.safeParse(r.value); + const parsed = ServiceRecord.safeParse(r.value); if (!parsed.success) return []; const h = parsed.data.host ?? r.uri.split("/").pop() ?? ""; return h.includes(".") || h.includes(":") ? [host(h)] : []; // tid rkey without host field = junk @@ -107,6 +111,13 @@ export async function fetchRepos(pds: string, did: string): Promise { const parsed = RepoRecord.safeParse(r.value); if (!parsed.success) return []; const name = r.uri.split("/").pop() ?? ""; - return [{ ...parsed.data, knot: host(parsed.data.knot), name }]; + return [ + { + ...parsed.data, + knot: host(parsed.data.knot), + spindle: parsed.data.spindle ? host(parsed.data.spindle) : undefined, + name, + }, + ]; }); } diff --git a/src/cli.ts b/src/cli.ts index e122d77..5e851a2 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -20,6 +20,7 @@ const usage = `usage: --as who you are (or {"handle":"..."} in ~/.config/fid/config.json) fid stats network aggregates: versions, hosting, top knots fid repos what repos live on a knot + fid spindles the ci runner fleet fid watch live network board (q to quit) fid --version print the version `; @@ -80,6 +81,9 @@ switch (command) { args._[1] === undefined ? undefined : String(args._[1]), ); break; + case "spindles": + await (await import("./commands/spindles.js")).run(args); + break; case "watch": await (await import("./commands/watch.js")).run(args); break; diff --git a/src/commands/check.ts b/src/commands/check.ts index 19c85e4..9288622 100644 --- a/src/commands/check.ts +++ b/src/commands/check.ts @@ -5,7 +5,7 @@ import { styleText } from "node:util"; import * as p from "@clack/prompts"; import { resolveDid, resolveHandle } from "../api/identity.js"; import { fetchMotd, isLocalHost, probeKnot } from "../api/knot.js"; -import { fetchKnotHosts } from "../api/relay.js"; +import { fetchServiceHosts } from "../api/relay.js"; import { assessKnot, versionOf } from "../core/health.js"; type Flags = { json?: boolean; mine?: boolean; as?: string }; @@ -35,7 +35,9 @@ async function targets(flags: Flags, hostArg?: string): Promise { const did = await resolveHandle(who); const id = await resolveDid(did); if (!id.pds) throw new Error(`no pds found for ${who}`); - const hosts = [...new Set(await fetchKnotHosts(id.pds, did))]; + const hosts = [ + ...new Set(await fetchServiceHosts(id.pds, did, "sh.tangled.knot")), + ]; if (hosts.length === 0) throw new Error(`${who} owns no knots (no sh.tangled.knot records)`); return hosts; diff --git a/src/commands/spindles.ts b/src/commands/spindles.ts new file mode 100644 index 0000000..0352a9d --- /dev/null +++ b/src/commands/spindles.ts @@ -0,0 +1,69 @@ +import { styleText } from "node:util"; +import * as p from "@clack/prompts"; +import { loadNetwork, type Network } from "../core/network.js"; + +const dim = (s: string) => styleText("dim", s); + +export async function run(flags: { + json?: boolean; + all?: boolean; + fresh?: boolean; +}) { + if (!flags.json) p.intro(styleText("green", "fiddling with knots")); + const spin = flags.json ? null : p.spinner(); + spin?.start("assembling the network"); + let network: Network; + try { + network = await loadNetwork(!!flags.fresh, (msg) => spin?.message(msg)); + } catch (err) { + spin?.stop("upstream failure", 1); + console.error(err instanceof Error ? err.message : err); + process.exit(2); + } + spin?.stop(`network assembled ${dim(`(${network.generatedAt})`)}`); + + if (flags.json) { + console.log(JSON.stringify(network.spindles, null, 2)); + return; + } + + const shown = flags.all + ? network.spindles + : network.spindles.filter((s) => !s.local); + const hidden = network.spindles.length - shown.length; + + const status = (s: (typeof shown)[number]) => { + if (s.local) return dim("local"); + if (s.probe?.reachable) + return styleText("green", `up ${s.probe.latencyMs ?? "?"}ms`); + return styleText("red", "down"); + }; + + const rows = shown.map((s) => [ + s.host, + String(s.repoCount), + status(s), + s.owners[0] + ? `@${s.owners[0].handle}${s.owners.length > 1 ? dim(` +${s.owners.length - 1}`) : ""}` + : dim("—"), + ]); + + const header = ["HOST", "REPOS", "STATUS", "OWNER"]; + // biome-ignore lint/suspicious/noControlCharactersInRegex: stripping our own ansi colors for width math + const visible = (s: string) => s.replace(/\x1b\[\d+m/g, "").length; + const widths = header.map((h, i) => + Math.max(h.length, ...rows.map((r) => visible(r[i]))), + ); + const line = (cells: string[]) => + cells.map((c, i) => c + " ".repeat(widths[i] - visible(c))).join(" "); + + p.log.message([dim(line(header)), ...rows.map((r) => line(r))].join("\n")); + + const reachable = shown.filter((s) => s.probe?.reachable).length; + p.outro( + dim( + `${shown.length} spindles · ${reachable} reachable` + + (hidden ? ` · ${hidden} local/dev hidden (--all)` : ""), + ), + ); +} diff --git a/src/commands/stats.ts b/src/commands/stats.ts index f4f91b5..0dd1f2a 100644 --- a/src/commands/stats.ts +++ b/src/commands/stats.ts @@ -77,7 +77,7 @@ export async function run(flags: { json?: boolean; fresh?: boolean }) { const { knots, types, repos } = stats; const growth = growthLine(stats, previous); p.log.message( - `${knots.total} knots · ${repos.total} repos · ${repos.owners} owners` + + `${knots.total} knots · ${repos.total} repos · ${repos.owners} owners · ${stats.spindles.total} spindles` + (growth ? `\n${dim(growth)}` : ""), ); diff --git a/src/core/network.ts b/src/core/network.ts index badd7dd..aa00e93 100644 --- a/src/core/network.ts +++ b/src/core/network.ts @@ -1,7 +1,7 @@ import { resolveDid, type Identity } from "../api/identity.js"; import { - fetchKnotHosts, fetchRepos, + fetchServiceHosts, listDidsByCollection, type RepoRef, } from "../api/relay.js"; @@ -25,8 +25,17 @@ export type Knot = { export type RepoInfo = RepoRef & { owner: { did: string; handle: string } }; +export type Spindle = { + host: string; + owners: { did: string; handle: string }[]; + repoCount: number; // repos configured to run ci on this spindle + local: boolean; + probe: KnotProbe | null; // only reachable/latencyMs are meaningful here +}; + export type Network = { knots: Knot[]; + spindles: Spindle[]; repos: RepoInfo[]; totalRepos: number; repoOwners: number; @@ -34,9 +43,12 @@ export type Network = { generatedAt: string; }; +type ServiceOwners = { identity: Identity; hosts: string[] }[]; + // pure assembly: data in, data out — the io lives in assembleNetwork below export function groupNetwork(input: { - knotOwners: { identity: Identity; hosts: string[] }[]; + knotOwners: ServiceOwners; + spindleOwners: ServiceOwners; repos: RepoInfo[]; // every repo record on the network repoOwners: number; probes: Map; @@ -74,11 +86,35 @@ export function groupNetwork(input: { } for (const repo of input.repos) knot(repo.knot).repoCount++; - const knots = [...byHost.values()].sort( - (a, b) => b.repoCount - a.repoCount || a.host.localeCompare(b.host), - ); + const bySpindle = new Map(); + const spindle = (host: string): Spindle => { + let s = bySpindle.get(host); + if (!s) { + s = { + host, + owners: [], + repoCount: 0, + local: isLocalHost(host), + probe: input.probes.get(host) ?? null, + }; + bySpindle.set(host, s); + } + return s; + }; + for (const { identity, hosts } of input.spindleOwners) { + for (const host of new Set(hosts)) { + spindle(host).owners.push({ did: identity.did, handle: identity.handle }); + } + } + for (const repo of input.repos) { + if (repo.spindle) spindle(repo.spindle).repoCount++; + } + + const byCount = (a: { repoCount: number; host: string }, b: typeof a) => + b.repoCount - a.repoCount || a.host.localeCompare(b.host); return { - knots, + knots: [...byHost.values()].sort(byCount), + spindles: [...bySpindle.values()].sort(byCount), repos: input.repos, totalRepos: input.repos.length, repoOwners: input.repoOwners, @@ -93,8 +129,9 @@ export async function assembleNetwork( const progress = onProgress ?? (() => {}); progress("enumerating the network via relay…"); - const [knotDids, repoDids] = await Promise.all([ + const [knotDids, spindleDids, repoDids] = await Promise.all([ listDidsByCollection("sh.tangled.knot"), + listDidsByCollection("sh.tangled.spindle"), listDidsByCollection("sh.tangled.repo", (n) => progress(`enumerating repo owners… ${n}`), ), @@ -127,7 +164,12 @@ export async function assembleNetwork( const knotOwnersRaw = await hydrate( knotDids, "reading knot records", - fetchKnotHosts, + (p, d) => fetchServiceHosts(p, d, "sh.tangled.knot"), + ); + const spindleOwnersRaw = await hydrate( + spindleDids, + "reading spindle records", + (p, d) => fetchServiceHosts(p, d, "sh.tangled.spindle"), ); const repoOwnersRaw = await hydrate( repoDids, @@ -135,10 +177,10 @@ export async function assembleNetwork( fetchRepos, ); - const knotOwners = knotOwnersRaw.map(({ identity, result }) => ({ - identity, - hosts: result ?? [], - })); + const toOwners = (raw: typeof knotOwnersRaw) => + raw.map(({ identity, result }) => ({ identity, hosts: result ?? [] })); + const knotOwners = toOwners(knotOwnersRaw); + const spindleOwners = toOwners(spindleOwnersRaw); const repos: RepoInfo[] = repoOwnersRaw.flatMap(({ identity, result }) => (result ?? []).map((r) => ({ ...r, @@ -150,6 +192,7 @@ export async function assembleNetwork( ...new Set( [ ...knotOwners.flatMap((k) => k.hosts), + ...spindleOwners.flatMap((s) => s.hosts), ...repos.map((r) => r.knot), ].filter((h) => !isLocalHost(h)), ), @@ -170,6 +213,7 @@ export async function assembleNetwork( return groupNetwork({ knotOwners, + spindleOwners, repos, repoOwners: repoDids.length, probes, @@ -180,8 +224,8 @@ export async function loadNetwork( fresh: boolean, onProgress?: (msg: string) => void, ): Promise { - // key bumped when the snapshot shape changes — v1 lacked `repos` - return cached("network2", fresh ? 0 : 5 * 60 * 1000, () => + // key bumped when the snapshot shape changes — v2 lacked `spindles` + return cached("network3", fresh ? 0 : 5 * 60 * 1000, () => assembleNetwork(onProgress), ); } diff --git a/src/core/stats.ts b/src/core/stats.ts index 37b2b0a..3371543 100644 --- a/src/core/stats.ts +++ b/src/core/stats.ts @@ -10,6 +10,7 @@ export type Stats = { }; types: { managed: number; selfHosted: number }; repos: { total: number; owners: number }; + spindles: { total: number; reachable: number }; topKnots: { host: string; repoCount: number; version: string }[]; histogram: { label: string; count: number }[]; }; @@ -45,6 +46,10 @@ export function computeStats(network: Network): Stats { selfHosted: pub.filter((k) => !k.managed).length, }, repos: { total: network.totalRepos, owners: network.repoOwners }, + spindles: { + total: network.spindles.filter((s) => !s.local).length, + reachable: network.spindles.filter((s) => s.probe?.reachable).length, + }, // network.knots is already sorted by repoCount desc (groupNetwork) topKnots: pub.slice(0, 10).map((k) => ({ host: k.host, diff --git a/tests/network.test.ts b/tests/network.test.ts index f9de90b..9391da4 100644 --- a/tests/network.test.ts +++ b/tests/network.test.ts @@ -1,11 +1,17 @@ import { readFileSync } from "node:fs"; import { afterEach, expect, test, vi } from "vitest"; -import { fetchKnotHosts, fetchRepos } from "../src/api/relay.js"; +import { fetchRepos, fetchServiceHosts } from "../src/api/relay.js"; import { groupNetwork, type RepoInfo } from "../src/core/network.js"; -const repo = (name: string, knot: string, handle = "someone"): RepoInfo => ({ +const repo = ( + name: string, + knot: string, + handle = "someone", + spindle?: string, +): RepoInfo => ({ name, knot, + spindle, owner: { did: `did:plc:${handle}`, handle }, }); @@ -21,10 +27,16 @@ test("groupNetwork groups repos by knot host and classifies versions", () => { hosts: ["knot.krasovs.ky"], }, ], + spindleOwners: [ + { + identity: { did: "did:plc:c", handle: "tangled.org" }, + hosts: ["spindle.tangled.sh"], + }, + ], repos: [ - repo("alpha", "knot1.tangled.sh"), + repo("alpha", "knot1.tangled.sh", "someone", "spindle.tangled.sh"), repo("beta", "knot1.tangled.sh"), - repo("gamma", "knot.krasovs.ky"), + repo("gamma", "knot.krasovs.ky", "someone", "spindle.tangled.sh"), repo("delta", "ghost.example.com"), ], repoOwners: 3, @@ -59,11 +71,17 @@ test("groupNetwork groups repos by knot host and classifies versions", () => { expect(local).toMatchObject({ local: true, probe: null }); expect(network.totalRepos).toBe(4); expect(network.repos.map((r) => r.name)).toContain("alpha"); + expect(network.spindles).toHaveLength(1); + expect(network.spindles[0]).toMatchObject({ + host: "spindle.tangled.sh", + repoCount: 2, + owners: [{ did: "did:plc:c", handle: "tangled.org" }], + }); }); afterEach(() => vi.unstubAllGlobals()); -test("fetchKnotHosts handles both lexicon shapes: host field and rkey-as-host", async () => { +test("fetchServiceHosts handles both lexicon shapes: host field and rkey-as-host", async () => { const fixture = JSON.parse( readFileSync( new URL("./fixtures/listRecords.knot.json", import.meta.url), @@ -74,7 +92,11 @@ test("fetchKnotHosts handles both lexicon shapes: host field and rkey-as-host", "fetch", vi.fn(async () => new Response(JSON.stringify(fixture))), ); - const hosts = await fetchKnotHosts("https://pds.example", "did:plc:x"); + const hosts = await fetchServiceHosts( + "https://pds.example", + "did:plc:x", + "sh.tangled.knot", + ); // old shape uses value.host (normalized), new shape uses the rkey, tid-rkey junk is dropped expect(hosts).toEqual(["knot.example.com", "localhost:5557"]); }); diff --git a/tests/stats.test.ts b/tests/stats.test.ts index 97092c4..68278ca 100644 --- a/tests/stats.test.ts +++ b/tests/stats.test.ts @@ -40,6 +40,29 @@ const net: Network = { version: "unknown", }), ], + spindles: [ + { + host: "spindle.tangled.sh", + owners: [], + repoCount: 9, + local: false, + probe: { reachable: true, xrpcOk: false, latencyMs: 80 }, + }, + { + host: "spindle.dead.dev", + owners: [], + repoCount: 0, + local: false, + probe: { reachable: false, xrpcOk: false }, + }, + { + host: "localhost:6555", + owners: [], + repoCount: 0, + local: true, + probe: null, + }, + ], repos: [], totalRepos: 169, repoOwners: 42, @@ -58,6 +81,7 @@ test("computeStats classifies knots and buckets repos", () => { }); expect(s.types).toEqual({ managed: 1, selfHosted: 4 }); expect(s.repos).toEqual({ total: 169, owners: 42 }); + expect(s.spindles).toEqual({ total: 2, reachable: 1 }); // local knot excluded everywhere; zero-repo knots excluded from the histogram expect(s.histogram).toEqual([ { label: "1", count: 1 }, diff --git a/tests/watch.test.ts b/tests/watch.test.ts index 500c08d..7dba76e 100644 --- a/tests/watch.test.ts +++ b/tests/watch.test.ts @@ -36,6 +36,7 @@ const network: Network = { version: "unknown", }, ], + spindles: [], repos: [], totalRepos: 43, repoOwners: 2,