diff --git a/package.json b/package.json index 0fd080e..0d3f62f 100644 --- a/package.json +++ b/package.json @@ -1,33 +1,37 @@ { - "name": "fid", - "version": "0.0.0", - "description": "a network observatory for tangled — enumerate every knot, check their health, watch the federation", - "type": "module", - "bin": { - "fid": "./dist/cli.js" - }, - "files": [ - "dist" - ], - "engines": { - "node": ">=20" - }, - "scripts": { - "dev": "tsx src/cli.ts", - "build": "tsc", - "test": "vitest run" - }, - "dependencies": { - "@bomb.sh/args": "^0.3.1", - "@bomb.sh/tty": "^0.8.0", - "@clack/prompts": "^0.11.0", - "zod": "^4.0.0" - }, - "devDependencies": { - "@biomejs/biome": "2.5.5", - "@types/node": "^24.0.0", - "tsx": "^4.19.0", - "typescript": "^5.8.0", - "vitest": "^3.2.0" - } + "name": "fid", + "version": "0.0.0", + "description": "a network observatory for tangled — enumerate every knot, check their health, watch the federation", + "type": "module", + "bin": { + "fid": "./dist/cli.js" + }, + "files": [ + "dist" + ], + "engines": { + "node": ">=20" + }, + "scripts": { + "dev": "tsx src/cli.ts", + "build": "tsc", + "test": "vitest run", + "format": "biome format --write .", + "lint": "biome lint .", + "lint:fix": "biome lint --write .", + "check": "biome check --write ." + }, + "dependencies": { + "@bomb.sh/args": "^0.3.1", + "@bomb.sh/tty": "^0.8.0", + "@clack/prompts": "^0.11.0", + "zod": "^4.0.0" + }, + "devDependencies": { + "@biomejs/biome": "2.5.5", + "@types/node": "^24.0.0", + "tsx": "^4.19.0", + "typescript": "^5.8.0", + "vitest": "^3.2.0" + } } diff --git a/src/api/identity.ts b/src/api/identity.ts index c0f6c2f..f3f57a1 100644 --- a/src/api/identity.ts +++ b/src/api/identity.ts @@ -1,31 +1,34 @@ -import { z } from 'zod' -import { cached } from '../util.js' +import { z } from "zod"; +import { cached } from "../util.js"; const DidDoc = z.object({ - alsoKnownAs: z.array(z.string()).default([]), - service: z - .array(z.object({ id: z.string(), serviceEndpoint: z.string() })) - .default([]), -}) + alsoKnownAs: z.array(z.string()).default([]), + service: z + .array(z.object({ id: z.string(), serviceEndpoint: z.string() })) + .default([]), +}); -export type Identity = { did: string; handle: string; pds?: string } +export type Identity = { did: string; handle: string; pds?: string }; -const DAY = 24 * 60 * 60 * 1000 +const DAY = 24 * 60 * 60 * 1000; export async function resolveDid(did: string): Promise { - return cached(`did-${did}`, DAY, async () => { - const url = did.startsWith('did:web:') - ? `https://${did.slice('did:web:'.length)}/.well-known/did.json` - : `https://plc.directory/${did}` - try { - const res = await fetch(url, { signal: AbortSignal.timeout(10_000) }) - if (!res.ok) return { did, handle: did } - const doc = DidDoc.parse(await res.json()) - const handle = doc.alsoKnownAs.find((a) => a.startsWith('at://'))?.slice(5) ?? did - const pds = doc.service.find((s) => s.id.endsWith('atproto_pds'))?.serviceEndpoint - return { did, handle, pds } - } catch { - return { did, handle: did } // broken identity shouldn't sink the whole list - } - }) + return cached(`did-${did}`, DAY, async () => { + const url = did.startsWith("did:web:") + ? `https://${did.slice("did:web:".length)}/.well-known/did.json` + : `https://plc.directory/${did}`; + try { + const res = await fetch(url, { signal: AbortSignal.timeout(10_000) }); + if (!res.ok) return { did, handle: did }; + const doc = DidDoc.parse(await res.json()); + const handle = + doc.alsoKnownAs.find((a) => a.startsWith("at://"))?.slice(5) ?? did; + const pds = doc.service.find((s) => + s.id.endsWith("atproto_pds"), + )?.serviceEndpoint; + return { did, handle, pds }; + } catch { + return { did, handle: did }; // broken identity shouldn't sink the whole list + } + }); } diff --git a/src/api/knot.ts b/src/api/knot.ts index defcf88..45fb9e7 100644 --- a/src/api/knot.ts +++ b/src/api/knot.ts @@ -1,44 +1,51 @@ export type KnotProbe = { - reachable: boolean - xrpcOk: boolean // 200 on a v1.15+ xrpc endpoint - status?: number - latencyMs?: number -} + reachable: boolean; + xrpcOk: boolean; // 200 on a v1.15+ xrpc endpoint + status?: number; + latencyMs?: number; +}; // v1.15+ knots serve xrpc directly; older ones 404 it but still answer http. // a knot that answers http but not xrpc is pre-v1.15 → silent-data-loss territory. export async function probeKnot(hostname: string): Promise { - const t0 = performance.now() - try { - const res = await fetch(`https://${hostname}/xrpc/sh.tangled.knot.listMembers`, { - signal: AbortSignal.timeout(5000), - }) - return { - reachable: true, - xrpcOk: res.ok, - status: res.status, - latencyMs: Math.round(performance.now() - t0), - } - } catch { - return { reachable: false, xrpcOk: false } - } + const t0 = performance.now(); + try { + const res = await fetch( + `https://${hostname}/xrpc/sh.tangled.knot.listMembers`, + { + signal: AbortSignal.timeout(5000), + }, + ); + return { + reachable: true, + xrpcOk: res.ok, + status: res.status, + latencyMs: Math.round(performance.now() - t0), + }; + } catch { + return { reachable: false, xrpcOk: false }; + } } // managed = hosted by tangled.org; keep the heuristic here, it may need refinement export function isManaged(hostname: string): boolean { - return hostname === 'tangled.sh' || hostname.endsWith('.tangled.sh') || hostname.endsWith('.tangled.org') + return ( + hostname === "tangled.sh" || + hostname.endsWith(".tangled.sh") || + hostname.endsWith(".tangled.org") + ); } // dev/localhost/private junk records people leave behind — not probeable from the internet export function isLocalHost(hostname: string): boolean { - const h = hostname.split(':')[0] - return ( - !h.includes('.') || - h === 'localhost' || - /^(127|10|0)\./.test(h) || - /^192\.168\./.test(h) || - /^172\.(1[6-9]|2\d|3[01])\./.test(h) || - h.endsWith('.local') || - h.endsWith('.internal') - ) + const h = hostname.split(":")[0]; + return ( + !h.includes(".") || + h === "localhost" || + /^(127|10|0)\./.test(h) || + /^192\.168\./.test(h) || + /^172\.(1[6-9]|2\d|3[01])\./.test(h) || + h.endsWith(".local") || + h.endsWith(".internal") + ); } diff --git a/src/api/relay.ts b/src/api/relay.ts index 08e00dd..dfb94d0 100644 --- a/src/api/relay.ts +++ b/src/api/relay.ts @@ -1,90 +1,102 @@ -import { z } from 'zod' +import { z } from "zod"; -const RELAY = process.env.FID_RELAY ?? 'https://relay1.us-west.bsky.network' +const RELAY = process.env.FID_RELAY ?? "https://relay1.us-west.bsky.network"; const DidPage = z.object({ - repos: z.array(z.object({ did: z.string() })), - cursor: z.string().optional(), -}) + repos: z.array(z.object({ did: z.string() })), + cursor: z.string().optional(), +}); // every account holding >=1 record in `collection` — the network IS the registry export async function listDidsByCollection( - collection: string, - onProgress?: (count: number) => void, + collection: string, + onProgress?: (count: number) => void, ): Promise { - const dids: string[] = [] - let cursor: string | undefined - do { - const url = new URL('/xrpc/com.atproto.sync.listReposByCollection', RELAY) - url.searchParams.set('collection', collection) - url.searchParams.set('limit', '1000') - if (cursor) url.searchParams.set('cursor', cursor) - const res = await fetch(url, { signal: AbortSignal.timeout(30_000) }) - if (!res.ok) { - throw new Error(`relay ${RELAY} answered ${res.status} for ${collection} — upstream is down, not the knots`) - } - const page = DidPage.parse(await res.json()) - dids.push(...page.repos.map((r) => r.did)) - cursor = page.repos.length > 0 ? page.cursor : undefined - onProgress?.(dids.length) - } while (cursor) - return dids + const dids: string[] = []; + let cursor: string | undefined; + do { + const url = new URL("/xrpc/com.atproto.sync.listReposByCollection", RELAY); + url.searchParams.set("collection", collection); + url.searchParams.set("limit", "1000"); + if (cursor) url.searchParams.set("cursor", cursor); + const res = await fetch(url, { signal: AbortSignal.timeout(30_000) }); + if (!res.ok) { + throw new Error( + `relay ${RELAY} answered ${res.status} for ${collection} — upstream is down, not the knots`, + ); + } + const page = DidPage.parse(await res.json()); + dids.push(...page.repos.map((r) => r.did)); + cursor = page.repos.length > 0 ? page.cursor : undefined; + onProgress?.(dids.length); + } while (cursor); + return dids; } const RecordsPage = z.object({ - records: z.array(z.object({ uri: z.string(), value: z.unknown() })), - cursor: z.string().optional(), -}) + records: z.array(z.object({ uri: z.string(), value: z.unknown() })), + cursor: z.string().optional(), +}); -type Record = { uri: string; value: unknown } +type Record = { uri: string; value: unknown }; -async function listRecords(pds: string, repo: string, collection: string): Promise { - const values: Record[] = [] - let cursor: string | undefined - let pages = 0 - do { - const url = new URL('/xrpc/com.atproto.repo.listRecords', pds) - url.searchParams.set('repo', repo) - url.searchParams.set('collection', collection) - url.searchParams.set('limit', '100') - if (cursor) url.searchParams.set('cursor', cursor) - let page: z.infer - try { - const res = await fetch(url, { signal: AbortSignal.timeout(15_000) }) - if (!res.ok) return values - page = RecordsPage.parse(await res.json()) - } catch { - return values // dead or garbage-spewing PDS: this owner is just unhydrated, not fatal - } - values.push(...page.records) - cursor = page.records.length > 0 ? page.cursor : undefined - } while (cursor && ++pages < 50) // ponytail: page cap so a rogue PDS can't loop us forever - return values +async function listRecords( + pds: string, + repo: string, + collection: string, +): Promise { + const values: Record[] = []; + let cursor: string | undefined; + let pages = 0; + do { + const url = new URL("/xrpc/com.atproto.repo.listRecords", pds); + url.searchParams.set("repo", repo); + url.searchParams.set("collection", collection); + url.searchParams.set("limit", "100"); + if (cursor) url.searchParams.set("cursor", cursor); + let page: z.infer; + try { + const res = await fetch(url, { signal: AbortSignal.timeout(15_000) }); + if (!res.ok) return values; + page = RecordsPage.parse(await res.json()); + } catch { + return values; // dead or garbage-spewing PDS: this owner is just unhydrated, not fatal + } + values.push(...page.records); + cursor = page.records.length > 0 ? page.cursor : undefined; + } while (cursor && ++pages < 50); // ponytail: page cap so a rogue PDS can't loop us forever + return values; } // pre-~v1.x records carry a host field with a tid rkey; newer ones drop the // field entirely and the rkey IS the hostname (at://did/sh.tangled.knot/knot.example.com) -const KnotRecord = z.object({ host: z.string().min(1).optional() }) -const RepoRecord = z.object({ knot: z.string().min(1) }) +const KnotRecord = z.object({ host: z.string().min(1).optional() }); +const RepoRecord = z.object({ knot: z.string().min(1) }); -const host = (h: string) => h.trim().toLowerCase() +const host = (h: string) => h.trim().toLowerCase(); // hosts this owner has verified as knots (malformed records skipped, not fatal) -export async function fetchKnotHosts(pds: string, did: string): Promise { - const records = await listRecords(pds, did, 'sh.tangled.knot') - return records.flatMap((r) => { - const parsed = KnotRecord.safeParse(r.value) - if (!parsed.success) return [] - const h = parsed.data.host ?? r.uri.split('/').pop()! - return h.includes('.') || h.includes(':') ? [host(h)] : [] // tid rkey without host field = junk - }) +export async function fetchKnotHosts( + pds: string, + did: string, +): Promise { + const records = await listRecords(pds, did, "sh.tangled.knot"); + return records.flatMap((r) => { + const parsed = KnotRecord.safeParse(r.value); + if (!parsed.success) return []; + const h = parsed.data.host ?? r.uri.split("/").pop() ?? ""; + return h.includes(".") || h.includes(":") ? [host(h)] : []; // tid rkey without host field = junk + }); } // knot host of each repo record this owner holds — one entry per repo -export async function fetchRepoKnots(pds: string, did: string): Promise { - const records = await listRecords(pds, did, 'sh.tangled.repo') - return records.flatMap((r) => { - const parsed = RepoRecord.safeParse(r.value) - return parsed.success ? [host(parsed.data.knot)] : [] - }) +export async function fetchRepoKnots( + pds: string, + did: string, +): Promise { + const records = await listRecords(pds, did, "sh.tangled.repo"); + return records.flatMap((r) => { + const parsed = RepoRecord.safeParse(r.value); + return parsed.success ? [host(parsed.data.knot)] : []; + }); } diff --git a/src/banner.ts b/src/banner.ts index 909e344..25bf673 100644 --- a/src/banner.ts +++ b/src/banner.ts @@ -40,6 +40,7 @@ function cursorRow(): Promise { }; const timer = setTimeout(() => done(null), 250); const onData = (buf: Buffer) => { + // biome-ignore lint/suspicious/noControlCharactersInRegex: parsing a DSR escape response const m = /\x1b\[(\d+);\d+R/.exec(buf.toString("utf8")); if (m) done(Number(m[1])); }; diff --git a/src/cli.ts b/src/cli.ts index 84ee90b..eae6a9c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,11 +1,11 @@ #!/usr/bin/env node -import { parse } from '@bomb.sh/args' -import { animateBanner, staticBanner } from './banner.js' +import { parse } from "@bomb.sh/args"; +import { animateBanner, staticBanner } from "./banner.js"; const args = parse(process.argv.slice(2), { - boolean: ['json', 'all', 'fresh', 'help'], - alias: { h: 'help' }, -}) + boolean: ["json", "all", "fresh", "help"], + alias: { h: "help" }, +}); const usage = `${staticBanner} usage: @@ -13,22 +13,22 @@ usage: --json machine-readable output --all include localhost/dev junk knots --fresh skip the 5-minute cache -` +`; -const [command] = args._ +const [command] = args._; if (args.help || !command) { - console.log(usage) - process.exit(command ? 0 : 1) + console.log(usage); + process.exit(command ? 0 : 1); } -if (!args.json) await animateBanner() // keep --json pipe-clean +if (!args.json) await animateBanner(); // keep --json pipe-clean switch (command) { - case 'list': - await (await import('./commands/list.js')).run(args) - break - default: - console.error(`unknown command: ${command}\n`) - console.log(usage) - process.exit(1) + case "list": + await (await import("./commands/list.js")).run(args); + break; + default: + console.error(`unknown command: ${command}\n`); + console.log(usage); + process.exit(1); } diff --git a/src/commands/list.ts b/src/commands/list.ts index 4b9c025..93940e5 100644 --- a/src/commands/list.ts +++ b/src/commands/list.ts @@ -1,66 +1,79 @@ -import * as p from '@clack/prompts' -import { assembleNetwork, type Network } from '../core/network.js' -import { cached } from '../util.js' +import * as p from "@clack/prompts"; +import { assembleNetwork, type Network } from "../core/network.js"; +import { cached } from "../util.js"; -const dim = (s: string) => `\x1b[2m${s}\x1b[0m` -const yellow = (s: string) => `\x1b[33m${s}\x1b[0m` -const green = (s: string) => `\x1b[32m${s}\x1b[0m` -const red = (s: string) => `\x1b[31m${s}\x1b[0m` +const dim = (s: string) => `\x1b[2m${s}\x1b[0m`; +const yellow = (s: string) => `\x1b[33m${s}\x1b[0m`; +const green = (s: string) => `\x1b[32m${s}\x1b[0m`; +const red = (s: string) => `\x1b[31m${s}\x1b[0m`; -export async function run(flags: { json?: boolean; all?: boolean; fresh?: boolean }) { - if (!flags.json) p.intro(green('fiddling with knots')) - const spin = flags.json ? null : p.spinner() - spin?.start('assembling the network') - let network: Network - try { - network = await cached('network', flags.fresh ? 0 : 5 * 60 * 1000, () => - assembleNetwork((msg) => spin?.message(msg)), - ) - } catch (err) { - spin?.stop('upstream failure', 1) - console.error(err instanceof Error ? err.message : err) - process.exit(2) - } - spin?.stop(`network assembled ${dim(`(${network.generatedAt})`)}`) +export async function run(flags: { + json?: boolean; + all?: boolean; + fresh?: boolean; +}) { + if (!flags.json) p.intro(green("fiddling with knots")); + const spin = flags.json ? null : p.spinner(); + spin?.start("assembling the network"); + let network: Network; + try { + network = await cached("network", flags.fresh ? 0 : 5 * 60 * 1000, () => + assembleNetwork((msg) => spin?.message(msg)), + ); + } catch (err) { + spin?.stop("upstream failure", 1); + console.error(err instanceof Error ? err.message : err); + process.exit(2); + } + spin?.stop(`network assembled ${dim(`(${network.generatedAt})`)}`); - if (flags.json) { - console.log(JSON.stringify(network, null, 2)) - return - } + if (flags.json) { + console.log(JSON.stringify(network, null, 2)); + return; + } - const shown = flags.all ? network.knots : network.knots.filter((k) => !k.local) - const hidden = network.knots.length - shown.length + const shown = flags.all + ? network.knots + : network.knots.filter((k) => !k.local); + const hidden = network.knots.length - shown.length; - const versionLabel = (k: (typeof shown)[number]) => { - if (k.local) return dim('local') - if (!k.probe?.reachable) return red('down') - if (k.version === 'v1.15+') return green('v1.15+') - return yellow('pre-v1.15 ⚠') // pre-v1.14 knots silently drop pushes/prs/issues - } + const versionLabel = (k: (typeof shown)[number]) => { + if (k.local) return dim("local"); + if (!k.probe?.reachable) return red("down"); + if (k.version === "v1.15+") return green("v1.15+"); + return yellow("pre-v1.15 ⚠"); // pre-v1.14 knots silently drop pushes/prs/issues + }; - const rows = shown.map((k) => [ - k.host, - String(k.repoCount), - versionLabel(k), - k.local ? dim('—') : k.managed ? 'managed' : 'self-hosted', - k.owners[0] ? `@${k.owners[0].handle}${k.owners.length > 1 ? dim(` +${k.owners.length - 1}`) : ''}` : dim('—'), - ]) + const rows = shown.map((k) => [ + k.host, + String(k.repoCount), + versionLabel(k), + k.local ? dim("—") : k.managed ? "managed" : "self-hosted", + k.owners[0] + ? `@${k.owners[0].handle}${k.owners.length > 1 ? dim(` +${k.owners.length - 1}`) : ""}` + : dim("—"), + ]); - const header = ['HOST', 'REPOS', 'VERSION', 'TYPE', 'OWNER'] - const visible = (s: string) => s.replace(/\x1b\[\d+m/g, '').length - const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => visible(r[i])))) - const line = (cells: string[]) => - cells.map((c, i) => c + ' '.repeat(widths[i] - visible(c))).join(' ') + const header = ["HOST", "REPOS", "VERSION", "TYPE", "OWNER"]; + // biome-ignore lint/suspicious/noControlCharactersInRegex: stripping our own ansi colors for width math + const visible = (s: string) => s.replace(/\x1b\[\d+m/g, "").length; + const widths = header.map((h, i) => + Math.max(h.length, ...rows.map((r) => visible(r[i]))), + ); + const line = (cells: string[]) => + cells.map((c, i) => c + " ".repeat(widths[i] - visible(c))).join(" "); - p.log.message([dim(line(header)), ...rows.map((r) => line(r))].join('\n')) + p.log.message([dim(line(header)), ...rows.map((r) => line(r))].join("\n")); - const reachable = shown.filter((k) => k.probe?.reachable).length - const stale = shown.filter((k) => !k.local && k.probe?.reachable && k.version !== 'v1.15+').length - p.outro( - dim( - `${shown.length} knots · ${reachable} reachable · ${stale} pre-v1.15 · ` + - `${network.totalRepos} repos across ${network.repoOwners} owners` + - (hidden ? ` · ${hidden} local/dev knots hidden (--all)` : ''), - ), - ) + const reachable = shown.filter((k) => k.probe?.reachable).length; + const stale = shown.filter( + (k) => !k.local && k.probe?.reachable && k.version !== "v1.15+", + ).length; + p.outro( + dim( + `${shown.length} knots · ${reachable} reachable · ${stale} pre-v1.15 · ` + + `${network.totalRepos} repos across ${network.repoOwners} owners` + + (hidden ? ` · ${hidden} local/dev knots hidden (--all)` : ""), + ), + ); } diff --git a/src/core/network.ts b/src/core/network.ts index 2350853..197cf27 100644 --- a/src/core/network.ts +++ b/src/core/network.ts @@ -1,124 +1,166 @@ -import { resolveDid, type Identity } from '../api/identity.js' -import { fetchKnotHosts, fetchRepoKnots, listDidsByCollection } from '../api/relay.js' -import { isLocalHost, isManaged, probeKnot, type KnotProbe } from '../api/knot.js' -import { pLimit } from '../util.js' +import { resolveDid, type Identity } from "../api/identity.js"; +import { + fetchKnotHosts, + fetchRepoKnots, + listDidsByCollection, +} from "../api/relay.js"; +import { + isLocalHost, + isManaged, + probeKnot, + type KnotProbe, +} from "../api/knot.js"; +import { pLimit } from "../util.js"; export type Knot = { - host: string - owners: { did: string; handle: string }[] - repoCount: number - managed: boolean - local: boolean // localhost/private junk record, not probeable - probe: KnotProbe | null // null = not probed (local) - version: 'v1.15+' | 'pre-v1.15' | 'unknown' -} + host: string; + owners: { did: string; handle: string }[]; + repoCount: number; + managed: boolean; + local: boolean; // localhost/private junk record, not probeable + probe: KnotProbe | null; // null = not probed (local) + version: "v1.15+" | "pre-v1.15" | "unknown"; +}; export type Network = { - knots: Knot[] - totalRepos: number - repoOwners: number - knotOwners: number - generatedAt: string -} + knots: Knot[]; + totalRepos: number; + repoOwners: number; + knotOwners: number; + generatedAt: string; +}; // pure assembly: data in, data out — the io lives in assembleNetwork below export function groupNetwork(input: { - knotOwners: { identity: Identity; hosts: string[] }[] - repoKnots: string[] // one host entry per repo record on the network - repoOwners: number - probes: Map + knotOwners: { identity: Identity; hosts: string[] }[]; + repoKnots: string[]; // one host entry per repo record on the network + repoOwners: number; + probes: Map; }): Network { - const byHost = new Map() - const knot = (host: string): Knot => { - let k = byHost.get(host) - if (!k) { - const local = isLocalHost(host) - const probe = input.probes.get(host) ?? null - k = { - host, - owners: [], - repoCount: 0, - managed: isManaged(host), - local, - probe, - version: probe ? (probe.xrpcOk ? 'v1.15+' : probe.reachable ? 'pre-v1.15' : 'unknown') : 'unknown', - } - byHost.set(host, k) - } - return k - } + const byHost = new Map(); + const knot = (host: string): Knot => { + let k = byHost.get(host); + if (!k) { + const local = isLocalHost(host); + const probe = input.probes.get(host) ?? null; + k = { + host, + owners: [], + repoCount: 0, + managed: isManaged(host), + local, + probe, + version: probe + ? probe.xrpcOk + ? "v1.15+" + : probe.reachable + ? "pre-v1.15" + : "unknown" + : "unknown", + }; + byHost.set(host, k); + } + return k; + }; - for (const { identity, hosts } of input.knotOwners) { - for (const host of new Set(hosts)) { - knot(host).owners.push({ did: identity.did, handle: identity.handle }) - } - } - for (const host of input.repoKnots) knot(host).repoCount++ + for (const { identity, hosts } of input.knotOwners) { + for (const host of new Set(hosts)) { + knot(host).owners.push({ did: identity.did, handle: identity.handle }); + } + } + for (const host of input.repoKnots) knot(host).repoCount++; - const knots = [...byHost.values()].sort( - (a, b) => b.repoCount - a.repoCount || a.host.localeCompare(b.host), - ) - return { - knots, - totalRepos: input.repoKnots.length, - repoOwners: input.repoOwners, - knotOwners: input.knotOwners.length, - generatedAt: new Date().toISOString(), - } + const knots = [...byHost.values()].sort( + (a, b) => b.repoCount - a.repoCount || a.host.localeCompare(b.host), + ); + return { + knots, + totalRepos: input.repoKnots.length, + repoOwners: input.repoOwners, + knotOwners: input.knotOwners.length, + generatedAt: new Date().toISOString(), + }; } -export async function assembleNetwork(onProgress?: (msg: string) => void): Promise { - const progress = onProgress ?? (() => {}) +export async function assembleNetwork( + onProgress?: (msg: string) => void, +): Promise { + const progress = onProgress ?? (() => {}); - progress('enumerating the network via relay…') - const [knotDids, repoDids] = await Promise.all([ - listDidsByCollection('sh.tangled.knot'), - listDidsByCollection('sh.tangled.repo', (n) => progress(`enumerating repo owners… ${n}`)), - ]) + progress("enumerating the network via relay…"); + const [knotDids, repoDids] = await Promise.all([ + listDidsByCollection("sh.tangled.knot"), + listDidsByCollection("sh.tangled.repo", (n) => + progress(`enumerating repo owners… ${n}`), + ), + ]); - // resolution + pds reads fan out across many hosts — go wide, cache makes reruns cheap - const limit = pLimit(32) - let done = 0 - const hydrate = async (dids: string[], what: string, fn: (id: Identity) => Promise) => { - done = 0 - return Promise.all( - dids.map((did) => - limit(async () => { - const identity = await resolveDid(did) - const result = identity.pds ? await fn(identity) : null - if (++done % 50 === 0 || done === dids.length) progress(`${what} ${done}/${dids.length}`) - return { identity, result } - }), - ), - ) - } + // resolution + pds reads fan out across many hosts — go wide, cache makes reruns cheap + const limit = pLimit(32); + let done = 0; + const hydrate = async ( + dids: string[], + what: string, + fn: (pds: string, did: string) => Promise, + ) => { + done = 0; + return Promise.all( + dids.map((did) => + limit(async () => { + const identity = await resolveDid(did); + const result = identity.pds + ? await fn(identity.pds, identity.did) + : null; + if (++done % 50 === 0 || done === dids.length) + progress(`${what} ${done}/${dids.length}`); + return { identity, result }; + }), + ), + ); + }; - const knotOwnersRaw = await hydrate(knotDids, 'reading knot records', (id) => - fetchKnotHosts(id.pds!, id.did), - ) - const repoOwnersRaw = await hydrate(repoDids, 'reading repo records', (id) => - fetchRepoKnots(id.pds!, id.did), - ) + const knotOwnersRaw = await hydrate( + knotDids, + "reading knot records", + fetchKnotHosts, + ); + const repoOwnersRaw = await hydrate( + repoDids, + "reading repo records", + fetchRepoKnots, + ); - const knotOwners = knotOwnersRaw.map(({ identity, result }) => ({ identity, hosts: result ?? [] })) - const repoKnots = repoOwnersRaw.flatMap(({ result }) => result ?? []) + const knotOwners = knotOwnersRaw.map(({ identity, result }) => ({ + identity, + hosts: result ?? [], + })); + const repoKnots = repoOwnersRaw.flatMap(({ result }) => result ?? []); - const publicHosts = [ - ...new Set([...knotOwners.flatMap((k) => k.hosts), ...repoKnots].filter((h) => !isLocalHost(h))), - ] - const probeLimit = pLimit(16) - let probed = 0 - const probes = new Map( - await Promise.all( - publicHosts.map((host) => - probeLimit(async () => { - const p = await probeKnot(host) - progress(`probing knots ${++probed}/${publicHosts.length}`) - return [host, p] as const - }), - ), - ), - ) + const publicHosts = [ + ...new Set( + [...knotOwners.flatMap((k) => k.hosts), ...repoKnots].filter( + (h) => !isLocalHost(h), + ), + ), + ]; + const probeLimit = pLimit(16); + let probed = 0; + const probes = new Map( + await Promise.all( + publicHosts.map((host) => + probeLimit(async () => { + const p = await probeKnot(host); + progress(`probing knots ${++probed}/${publicHosts.length}`); + return [host, p] as const; + }), + ), + ), + ); - return groupNetwork({ knotOwners, repoKnots, repoOwners: repoDids.length, probes }) + return groupNetwork({ + knotOwners, + repoKnots, + repoOwners: repoDids.length, + probes, + }); } diff --git a/src/util.ts b/src/util.ts index 6701ded..36bdef7 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,38 +1,45 @@ -import { mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs' -import { homedir } from 'node:os' -import { join } from 'node:path' +import { mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; export function pLimit(concurrency: number) { - let active = 0 - const queue: (() => void)[] = [] - const next = () => { - active-- - queue.shift()?.() - } - return async (fn: () => Promise): Promise => { - if (active >= concurrency) await new Promise((r) => queue.push(r)) - active++ - try { - return await fn() - } finally { - next() - } - } + let active = 0; + const queue: (() => void)[] = []; + const next = () => { + active--; + queue.shift()?.(); + }; + return async (fn: () => Promise): Promise => { + if (active >= concurrency) await new Promise((r) => queue.push(r)); + active++; + try { + return await fn(); + } finally { + next(); + } + }; } -const cacheDir = join(process.env.XDG_CACHE_HOME ?? join(homedir(), '.cache'), 'fid') +const cacheDir = join( + process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache"), + "fid", +); -export async function cached(key: string, ttlMs: number, fn: () => Promise): Promise { - const file = join(cacheDir, key.replace(/[^a-z0-9._-]/gi, '_') + '.json') - try { - if (Date.now() - statSync(file).mtimeMs < ttlMs) { - return JSON.parse(readFileSync(file, 'utf8')) as T - } - } catch { - // miss - } - const value = await fn() - mkdirSync(cacheDir, { recursive: true }) - writeFileSync(file, JSON.stringify(value)) - return value +export async function cached( + key: string, + ttlMs: number, + fn: () => Promise, +): Promise { + const file = join(cacheDir, `${key.replace(/[^a-z0-9._-]/gi, "_")}.json`); + try { + if (Date.now() - statSync(file).mtimeMs < ttlMs) { + return JSON.parse(readFileSync(file, "utf8")) as T; + } + } catch { + // miss + } + const value = await fn(); + mkdirSync(cacheDir, { recursive: true }); + writeFileSync(file, JSON.stringify(value)); + return value; } diff --git a/tests/fixtures/listRecords.knot.json b/tests/fixtures/listRecords.knot.json index fa2a790..dbcc160 100644 --- a/tests/fixtures/listRecords.knot.json +++ b/tests/fixtures/listRecords.knot.json @@ -1,29 +1,29 @@ { - "records": [ - { - "uri": "at://did:plc:qfpnj4og54vl56wngdriaxug/sh.tangled.knot/3lq35uwofw522", - "cid": "bafyreigsnszayizzgcb7ondewbx37rshqnmw56kbyqbt5hdfivctizo4x4", - "value": { - "host": "Knot.Example.Com ", - "$type": "sh.tangled.knot", - "createdAt": "2025-05-26T13:19:31+01:00" - } - }, - { - "uri": "at://did:plc:hwevmowznbiukdf6uk5dwrrq/sh.tangled.knot/localhost:5557", - "cid": "bafyreicwjyzjtzakjuaat32a4kl5yhcem4jegodnjrwnzafsfqqrksvzc4", - "value": { - "$type": "sh.tangled.knot", - "createdAt": "2025-08-23T23:04:41+03:00" - } - }, - { - "uri": "at://did:plc:junk/sh.tangled.knot/3lq35uwofw999", - "cid": "bafyreicwjyzjtzakjuaat32a4kl5yhcem4jegodnjrwnzafsfqqrksvzc5", - "value": { - "$type": "sh.tangled.knot", - "createdAt": "2025-08-23T23:04:41+03:00" - } - } - ] + "records": [ + { + "uri": "at://did:plc:qfpnj4og54vl56wngdriaxug/sh.tangled.knot/3lq35uwofw522", + "cid": "bafyreigsnszayizzgcb7ondewbx37rshqnmw56kbyqbt5hdfivctizo4x4", + "value": { + "host": "Knot.Example.Com ", + "$type": "sh.tangled.knot", + "createdAt": "2025-05-26T13:19:31+01:00" + } + }, + { + "uri": "at://did:plc:hwevmowznbiukdf6uk5dwrrq/sh.tangled.knot/localhost:5557", + "cid": "bafyreicwjyzjtzakjuaat32a4kl5yhcem4jegodnjrwnzafsfqqrksvzc4", + "value": { + "$type": "sh.tangled.knot", + "createdAt": "2025-08-23T23:04:41+03:00" + } + }, + { + "uri": "at://did:plc:junk/sh.tangled.knot/3lq35uwofw999", + "cid": "bafyreicwjyzjtzakjuaat32a4kl5yhcem4jegodnjrwnzafsfqqrksvzc5", + "value": { + "$type": "sh.tangled.knot", + "createdAt": "2025-08-23T23:04:41+03:00" + } + } + ] } diff --git a/tests/fixtures/listRecords.repo.json b/tests/fixtures/listRecords.repo.json index 381fd85..b33b3ad 100644 --- a/tests/fixtures/listRecords.repo.json +++ b/tests/fixtures/listRecords.repo.json @@ -1,34 +1,34 @@ { - "records": [ - { - "uri": "at://did:plc:qfpnj4og54vl56wngdriaxug/sh.tangled.repo/valley-sans", - "cid": "bafyreias2yphdcgjzvkw5r7vkogldce4afyekblpmbrbvuicuq5k5fmte4", - "value": { - "knot": "knot1.tangled.sh", - "$type": "sh.tangled.repo", - "repoDid": "did:plc:jqvj7h4wsn3l2rs57bhpeqbe", - "spindle": "spindle.tangled.sh", - "createdAt": "2026-05-28T22:18:55Z", - "description": "copy of https://github.com/HelsinkiTypeStudio/valley-sans with nix builds" - } - }, - { - "uri": "at://did:plc:qfpnj4og54vl56wngdriaxug/sh.tangled.repo/test-microvms", - "cid": "bafyreibr3idpiq3f3f722oimfzfqohqilkqn7fnvczsqur57ppdl7o5fi4", - "value": { - "knot": "localhost:6444", - "$type": "sh.tangled.repo", - "repoDid": "did:plc:4eu5c2igbphcxqexpkrqcxrv", - "createdAt": "2026-06-17T15:12:00+01:00" - } - }, - { - "uri": "at://did:plc:qfpnj4og54vl56wngdriaxug/sh.tangled.repo/malformed", - "cid": "bafyreibr3idpiq3f3f722oimfzfqohqilkqn7fnvczsqur57ppdl7o5fi5", - "value": { - "$type": "sh.tangled.repo", - "createdAt": "2026-06-17T15:12:00+01:00" - } - } - ] + "records": [ + { + "uri": "at://did:plc:qfpnj4og54vl56wngdriaxug/sh.tangled.repo/valley-sans", + "cid": "bafyreias2yphdcgjzvkw5r7vkogldce4afyekblpmbrbvuicuq5k5fmte4", + "value": { + "knot": "knot1.tangled.sh", + "$type": "sh.tangled.repo", + "repoDid": "did:plc:jqvj7h4wsn3l2rs57bhpeqbe", + "spindle": "spindle.tangled.sh", + "createdAt": "2026-05-28T22:18:55Z", + "description": "copy of https://github.com/HelsinkiTypeStudio/valley-sans with nix builds" + } + }, + { + "uri": "at://did:plc:qfpnj4og54vl56wngdriaxug/sh.tangled.repo/test-microvms", + "cid": "bafyreibr3idpiq3f3f722oimfzfqohqilkqn7fnvczsqur57ppdl7o5fi4", + "value": { + "knot": "localhost:6444", + "$type": "sh.tangled.repo", + "repoDid": "did:plc:4eu5c2igbphcxqexpkrqcxrv", + "createdAt": "2026-06-17T15:12:00+01:00" + } + }, + { + "uri": "at://did:plc:qfpnj4og54vl56wngdriaxug/sh.tangled.repo/malformed", + "cid": "bafyreibr3idpiq3f3f722oimfzfqohqilkqn7fnvczsqur57ppdl7o5fi5", + "value": { + "$type": "sh.tangled.repo", + "createdAt": "2026-06-17T15:12:00+01:00" + } + } + ] } diff --git a/tests/network.test.ts b/tests/network.test.ts index d393f17..caad0cd 100644 --- a/tests/network.test.ts +++ b/tests/network.test.ts @@ -1,51 +1,91 @@ -import { readFileSync } from 'node:fs' -import { afterEach, expect, test, vi } from 'vitest' -import { fetchKnotHosts, fetchRepoKnots } from '../src/api/relay.js' -import { groupNetwork } from '../src/core/network.js' +import { readFileSync } from "node:fs"; +import { afterEach, expect, test, vi } from "vitest"; +import { fetchKnotHosts, fetchRepoKnots } from "../src/api/relay.js"; +import { groupNetwork } from "../src/core/network.js"; -test('groupNetwork groups repos by knot host and classifies versions', () => { - const network = groupNetwork({ - knotOwners: [ - { identity: { did: 'did:plc:a', handle: 'oppi.li' }, hosts: ['knot1.tangled.sh', 'localhost:5555'] }, - { identity: { did: 'did:plc:b', handle: 'krasovs.ky' }, hosts: ['knot.krasovs.ky'] }, - ], - repoKnots: ['knot1.tangled.sh', 'knot1.tangled.sh', 'knot.krasovs.ky', 'ghost.example.com'], - repoOwners: 3, - probes: new Map([ - ['knot1.tangled.sh', { reachable: true, xrpcOk: true, status: 200, latencyMs: 40 }], - ['knot.krasovs.ky', { reachable: true, xrpcOk: false, status: 404, latencyMs: 90 }], - ['ghost.example.com', { reachable: false, xrpcOk: false }], - ]), - }) +test("groupNetwork groups repos by knot host and classifies versions", () => { + const network = groupNetwork({ + knotOwners: [ + { + identity: { did: "did:plc:a", handle: "oppi.li" }, + hosts: ["knot1.tangled.sh", "localhost:5555"], + }, + { + identity: { did: "did:plc:b", handle: "krasovs.ky" }, + hosts: ["knot.krasovs.ky"], + }, + ], + repoKnots: [ + "knot1.tangled.sh", + "knot1.tangled.sh", + "knot.krasovs.ky", + "ghost.example.com", + ], + repoOwners: 3, + probes: new Map([ + [ + "knot1.tangled.sh", + { reachable: true, xrpcOk: true, status: 200, latencyMs: 40 }, + ], + [ + "knot.krasovs.ky", + { reachable: true, xrpcOk: false, status: 404, latencyMs: 90 }, + ], + ["ghost.example.com", { reachable: false, xrpcOk: false }], + ]), + }); - expect(network.knots.map((k) => k.host)).toEqual([ - 'knot1.tangled.sh', // 2 repos, sorted first - 'ghost.example.com', - 'knot.krasovs.ky', - 'localhost:5555', - ]) - const [knot1, ghost, krasovsky, local] = network.knots - expect(knot1).toMatchObject({ repoCount: 2, managed: true, version: 'v1.15+' }) - expect(knot1.owners).toEqual([{ did: 'did:plc:a', handle: 'oppi.li' }]) - expect(krasovsky).toMatchObject({ managed: false, version: 'pre-v1.15' }) - expect(ghost).toMatchObject({ owners: [], version: 'unknown' }) - expect(local).toMatchObject({ local: true, probe: null }) - expect(network.totalRepos).toBe(4) -}) + expect(network.knots.map((k) => k.host)).toEqual([ + "knot1.tangled.sh", // 2 repos, sorted first + "ghost.example.com", + "knot.krasovs.ky", + "localhost:5555", + ]); + const [knot1, ghost, krasovsky, local] = network.knots; + expect(knot1).toMatchObject({ + repoCount: 2, + managed: true, + version: "v1.15+", + }); + expect(knot1.owners).toEqual([{ did: "did:plc:a", handle: "oppi.li" }]); + expect(krasovsky).toMatchObject({ managed: false, version: "pre-v1.15" }); + expect(ghost).toMatchObject({ owners: [], version: "unknown" }); + expect(local).toMatchObject({ local: true, probe: null }); + expect(network.totalRepos).toBe(4); +}); -afterEach(() => vi.unstubAllGlobals()) +afterEach(() => vi.unstubAllGlobals()); -test('fetchKnotHosts handles both lexicon shapes: host field and rkey-as-host', async () => { - const fixture = JSON.parse(readFileSync(new URL('./fixtures/listRecords.knot.json', import.meta.url), 'utf8')) - vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify(fixture)))) - const hosts = await fetchKnotHosts('https://pds.example', 'did:plc:x') - // old shape uses value.host (normalized), new shape uses the rkey, tid-rkey junk is dropped - expect(hosts).toEqual(['knot.example.com', 'localhost:5557']) -}) +test("fetchKnotHosts handles both lexicon shapes: host field and rkey-as-host", async () => { + const fixture = JSON.parse( + readFileSync( + new URL("./fixtures/listRecords.knot.json", import.meta.url), + "utf8", + ), + ); + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(JSON.stringify(fixture))), + ); + const hosts = await fetchKnotHosts("https://pds.example", "did:plc:x"); + // old shape uses value.host (normalized), new shape uses the rkey, tid-rkey junk is dropped + expect(hosts).toEqual(["knot.example.com", "localhost:5557"]); +}); -test('fetchRepoKnots parses real pds output, skips malformed records', async () => { - const fixture = JSON.parse(readFileSync(new URL('./fixtures/listRecords.repo.json', import.meta.url), 'utf8')) - vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify(fixture)))) - const knots = await fetchRepoKnots('https://pds.example', 'did:plc:qfpnj4og54vl56wngdriaxug') - expect(knots).toEqual(['knot1.tangled.sh', 'localhost:6444']) -}) +test("fetchRepoKnots parses real pds output, skips malformed records", async () => { + const fixture = JSON.parse( + readFileSync( + new URL("./fixtures/listRecords.repo.json", import.meta.url), + "utf8", + ), + ); + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(JSON.stringify(fixture))), + ); + const knots = await fetchRepoKnots( + "https://pds.example", + "did:plc:qfpnj4og54vl56wngdriaxug", + ); + expect(knots).toEqual(["knot1.tangled.sh", "localhost:6444"]); +}); diff --git a/tsconfig.json b/tsconfig.json index b027e5c..08f5151 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,13 +1,13 @@ { - "compilerOptions": { - "target": "ES2022", - "types": ["node"], - "module": "NodeNext", - "moduleResolution": "NodeNext", - "strict": true, - "outDir": "dist", - "rootDir": "src", - "skipLibCheck": true - }, - "include": ["src"] + "compilerOptions": { + "target": "ES2022", + "types": ["node"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "outDir": "dist", + "rootDir": "src", + "skipLibCheck": true + }, + "include": ["src"] }