import { createHash } from "node:crypto"; import { execFileSync } from "node:child_process"; import { readFileSync, renameSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; import "../src/env.ts"; import { knowledgeContentDigest, knowledgeRoute, loadKnowledgeGraph, type KnowledgeEntry, } from "../src/knowledge.ts"; import { nextStandardSiteRkey, shouldValidateStandardSiteWrite, type KnowledgeAtprotoManifest, } from "../src/knowledge-atproto.ts"; import { portableKnowledgeMarkdown, portableMarkdownLink, } from "../src/knowledge-charts/portable.ts"; import { loadKnowledgePolicy, scanKnowledgeDraft } from "./knowledge-policy.ts"; const CAMERON_DID = "did:plc:gfrmhdmjvxn2sjedzboeudef"; const PUBLICATION_COLLECTION = "site.standard.publication"; const DOCUMENT_COLLECTION = "site.standard.document"; const PUBLICATION_URL = "https://cameron.stream/knowledge"; const PUBLICATION_NAME = "Knowledge"; const PUBLICATION_DESCRIPTION = "Linked technical notes, subject maps, lessons, and dated synthesis from Cameron."; const MANIFEST_PATH = resolve(process.cwd(), "knowledge/atproto-manifest.json"); interface BlobRef { $type: "blob"; ref: { $link: string }; mimeType: string; size: number; } interface PublicationRecord { $type: typeof PUBLICATION_COLLECTION; url: string; name: string; description: string; icon: BlobRef; basicTheme: { $type: "site.standard.theme.basic"; background: Color; foreground: Color; accent: Color; accentForeground: Color; }; preferences: { showInDiscover: false }; } interface Color { $type: "site.standard.theme.color#rgb"; r: number; g: number; b: number; } interface StandardDocumentRecord { $type: typeof DOCUMENT_COLLECTION; site: string; title: string; description: string; publishedAt: string; updatedAt: string; path: string; tags: string[]; textContent: string; content: { $type: "site.standard.content.markdown"; text: string; version: "1.0"; }; } interface RemoteRecord { uri: string; cid: string; value: T; } type PlanAction = "create" | "update" | "unchanged" | "adopt" | "conflict"; interface RecordPlan { collection: string; rkey: string; uri: string; action: PlanAction; reason: string; record: T; recordDigest: string; remoteCid?: string; } function canonicalJson(value: unknown): string { if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; if (value && typeof value === "object") { return `{${Object.entries(value as Record) .sort(([a], [b]) => a.localeCompare(b)) .map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`) .join(",")}}`; } return JSON.stringify(value); } function digestRecord(record: unknown): string { return `sha256:${createHash("sha256").update(canonicalJson(record)).digest("hex")}`; } function loadManifest(): KnowledgeAtprotoManifest { const manifest = JSON.parse(readFileSync(MANIFEST_PATH, "utf8")) as KnowledgeAtprotoManifest; if (manifest.version !== 2) throw new Error("Unsupported Knowledge ATProto manifest"); if (manifest.publication && manifest.publication.showInDiscover !== false) { throw new Error("Refusing a Knowledge manifest without explicit showInDiscover=false"); } return manifest; } function writeManifest(manifest: KnowledgeAtprotoManifest): void { const temporary = `${MANIFEST_PATH}.tmp`; writeFileSync(temporary, `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); renameSync(temporary, MANIFEST_PATH); } function desiredPublication(): PublicationRecord { const color = (r: number, g: number, b: number): Color => ({ $type: "site.standard.theme.color#rgb", r, g, b, }); return { $type: PUBLICATION_COLLECTION, url: PUBLICATION_URL, name: PUBLICATION_NAME, description: PUBLICATION_DESCRIPTION, icon: { $type: "blob", ref: { $link: "bafkreibo3puyp2y32k33cp57em6ihxkoqgxfvrvbrmejnkdsjia5su3imi" }, mimeType: "image/webp", size: 3680, }, basicTheme: { $type: "site.standard.theme.basic", background: color(15, 15, 15), foreground: color(232, 228, 234), accent: color(180, 151, 191), accentForeground: color(255, 255, 255), }, preferences: { showInDiscover: false }, }; } function publicMarkdown(entry: KnowledgeEntry): string { const portableBody = portableKnowledgeMarkdown(entry.body, entry.slug); const absoluteLinks = portableBody.replace( /\]\(\/knowledge\/([a-z0-9-]+(?:\/[a-z0-9-]+)*)([^)]*)\)/g, "](https://cameron.stream/knowledge/$1$2)", ); const video = entry.youtubeVideoId ? `${portableMarkdownLink("Watch this episode on YouTube", `https://www.youtube.com/watch?v=${entry.youtubeVideoId}`)}\n\n` : ""; const sources = entry.sources.length === 0 ? "" : `\n\n## Sources\n\n${entry.sources .map((source) => `- ${portableMarkdownLink(source.title, source.url)}`) .join("\n")}`; return `${video}${absoluteLinks}${sources}`; } function stripMarkdown(markdown: string): string { return markdown .replace(/```[\s\S]*?```/g, "") .replace(/`([^`]+)`/g, "$1") .replace(/!\[[^\]]*\]\([^)]+\)/g, "") .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") .replace(/<[^>]+>/g, "") .replace(/^#{1,6}\s+/gm, "") .replace(/^\s*>\s?/gm, "") .replace(/^\s*[-*+]\s+/gm, "") .replace(/^\s*\d+\.\s+/gm, "") .replace(/\*\*([^*]+)\*\*/g, "$1") .replace(/\*([^*]+)\*/g, "$1") .replace(/__([^_]+)__/g, "$1") .replace(/_([^_]+)_/g, "$1") .replace(/\n{3,}/g, "\n\n") .trim(); } function desiredDocument(entry: KnowledgeEntry, publicationUri: string): StandardDocumentRecord { const markdown = publicMarkdown(entry); const record: StandardDocumentRecord = { $type: DOCUMENT_COLLECTION, site: publicationUri, title: entry.title, description: entry.summary, publishedAt: (entry.publishedAt ?? entry.updated).toISOString(), updatedAt: entry.updated.toISOString(), path: `/${knowledgeRoute(entry)}`, tags: [...new Set(["knowledge", entry.kind, ...entry.topics])], textContent: stripMarkdown(markdown).slice(0, 100_000), content: { $type: "site.standard.content.markdown", text: markdown, version: "1.0", }, }; const bytes = Buffer.byteLength(JSON.stringify(record)); if (bytes > 95_000) throw new Error(`${entry.slug}: Standard.site record is too large (${bytes} bytes)`); return record; } async function resolvePds(): Promise { const response = await fetch(`https://plc.directory/${CAMERON_DID}`); if (!response.ok) throw new Error(`Could not resolve Cameron DID: ${response.status}`); const document = await response.json() as { service?: Array<{ id?: string; type?: string; serviceEndpoint?: string }>; }; const pds = document.service?.find((service) => service.id === "#atproto_pds" || service.type === "AtprotoPersonalDataServer" )?.serviceEndpoint; if (!pds) throw new Error("Cameron DID has no ATProto PDS endpoint"); return pds.replace(/\/$/, ""); } async function listRecords(pds: string, collection: string): Promise>> { const records: Array> = []; let cursor: string | undefined; do { const query = new URLSearchParams({ repo: CAMERON_DID, collection, limit: "100", ...(cursor ? { cursor } : {}), }); const response = await fetch(`${pds}/xrpc/com.atproto.repo.listRecords?${query}`); if (!response.ok) throw new Error(`listRecords ${collection} failed: ${response.status}`); const page = await response.json() as { records: Array>; cursor?: string }; records.push(...page.records); cursor = page.cursor; } while (cursor); return records; } async function getRecord( pds: string, collection: string, rkey: string, ): Promise | null> { const query = new URLSearchParams({ repo: CAMERON_DID, collection, rkey }); const response = await fetch(`${pds}/xrpc/com.atproto.repo.getRecord?${query}`); if (response.status === 400) { const body = await response.json().catch(() => ({})) as { error?: string }; if (body.error === "RecordNotFound") return null; } if (!response.ok) throw new Error(`getRecord ${collection}/${rkey} failed: ${response.status}`); return await response.json() as RemoteRecord; } function rkeyFromUri(uri: string): string { return uri.slice(uri.lastIndexOf("/") + 1); } function recordUri(collection: string, rkey: string): string { return `at://${CAMERON_DID}/${collection}/${rkey}`; } function classify(options: { collection: string; desired: T; manifestUri?: string; manifestCid?: string; discovered: Array>; createRkey?: string; }): RecordPlan { const { collection, desired, manifestUri, manifestCid, discovered } = options; if (discovered.length > 1) { return { collection, rkey: "conflict", uri: "conflict", action: "conflict", reason: "multiple remote records match the same canonical object", record: desired, recordDigest: digestRecord(desired), }; } const remote = discovered[0]; const desiredDigest = digestRecord(desired); if (!remote) { if (manifestUri) { return { collection, rkey: rkeyFromUri(manifestUri), uri: manifestUri, action: "conflict", reason: "manifest expects a remote record that is missing", record: desired, recordDigest: desiredDigest, }; } const rkey = options.createRkey ?? nextStandardSiteRkey(); return { collection, rkey, uri: recordUri(collection, rkey), action: "create", reason: "no matching remote record exists", record: desired, recordDigest: desiredDigest, }; } const remoteDigest = digestRecord(remote.value); if (remoteDigest === desiredDigest) { return { collection, rkey: rkeyFromUri(remote.uri), uri: remote.uri, action: manifestUri ? "unchanged" : "adopt", reason: manifestUri ? "remote record matches the manifest and local source" : "matching remote record can be adopted", record: desired, recordDigest: desiredDigest, remoteCid: remote.cid, }; } if (!manifestUri || manifestUri !== remote.uri || manifestCid !== remote.cid) { return { collection, rkey: rkeyFromUri(remote.uri), uri: remote.uri, action: "conflict", reason: "remote record differs and is not owned by the current manifest CID", record: desired, recordDigest: desiredDigest, remoteCid: remote.cid, }; } return { collection, rkey: rkeyFromUri(remote.uri), uri: remote.uri, action: "update", reason: "approved local source changed while the remote CID still matches the manifest", record: desired, recordDigest: desiredDigest, remoteCid: remote.cid, }; } async function createSession(pds: string): Promise<{ accessJwt: string; did: string }> { const password = process.env.CAMERON_BSKY_APP_PASSWORD; if (!password) throw new Error("CAMERON_BSKY_APP_PASSWORD is required for --apply"); const response = await fetch(`${pds}/xrpc/com.atproto.server.createSession`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ identifier: CAMERON_DID, password }), }); if (!response.ok) throw new Error(`createSession failed: ${response.status} ${await response.text()}`); const session = await response.json() as { accessJwt: string; did: string }; if (session.did !== CAMERON_DID) { throw new Error(`Refusing wrong identity: authenticated ${session.did}, expected ${CAMERON_DID}`); } return session; } function toWrite(plan: RecordPlan): Record | undefined { if (plan.action === "create") { return { $type: "com.atproto.repo.applyWrites#create", collection: plan.collection, rkey: plan.rkey, value: plan.record, }; } if (plan.action === "update") { return { $type: "com.atproto.repo.applyWrites#update", collection: plan.collection, rkey: plan.rkey, value: plan.record, swapRecord: plan.remoteCid, }; } return undefined; } async function applyWrites( pds: string, accessJwt: string, plans: Array>, ): Promise { const operations = plans.flatMap((plan) => { const write = toWrite(plan); if (!write || (plan.action !== "create" && plan.action !== "update")) return []; return [{ write, validate: shouldValidateStandardSiteWrite({ collection: plan.collection, action: plan.action, rkey: plan.rkey, }), }]; }); if (operations.length === 0) return { unchanged: true }; const batches: Array<{ validate: boolean; writes: Array> }> = []; for (const operation of operations) { const current = batches.at(-1); const candidate = [...(current?.writes ?? []), operation.write]; const bytes = Buffer.byteLength(JSON.stringify({ repo: CAMERON_DID, writes: candidate })); if ( !current || current.validate !== operation.validate || candidate.length > 20 || bytes > 500_000 ) { batches.push({ validate: operation.validate, writes: [operation.write] }); } else { current.writes = candidate; } } const receipts = []; for (let index = 0; index < batches.length; index++) { const batch = batches[index]; const response = await fetch(`${pds}/xrpc/com.atproto.repo.applyWrites`, { method: "POST", headers: { Authorization: `Bearer ${accessJwt}`, "Content-Type": "application/json" }, body: JSON.stringify({ repo: CAMERON_DID, writes: batch.writes, validate: batch.validate }), }); if (!response.ok) { throw new Error( `applyWrites batch ${index + 1}/${batches.length} failed: ${response.status} ${await response.text()}`, ); } receipts.push({ validate: batch.validate, response: await response.json() }); } return { batches: receipts }; } function assertOriginMain(): void { const status = execFileSync("git", ["status", "--porcelain"], { encoding: "utf8" }); if (status.trim()) throw new Error("--from-origin-main requires a clean Git worktree"); const branch = execFileSync("git", ["branch", "--show-current"], { encoding: "utf8" }).trim(); if (branch !== "main") throw new Error(`--from-origin-main requires branch main, found ${branch}`); const head = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(); const remote = execFileSync("git", ["rev-parse", "origin/main"], { encoding: "utf8" }).trim(); if (head !== remote) throw new Error("--from-origin-main requires HEAD == origin/main"); } function validateEntry(entry: KnowledgeEntry): void { const policy = loadKnowledgePolicy(); const blocking = scanKnowledgeDraft(entry.body, policy).filter((finding) => finding.severity === "block"); if (blocking.length > 0) { throw new Error(`${entry.slug}: blocking policy findings: ${JSON.stringify(blocking)}`); } if (entry.kind === "person" && !policy.allowedPeopleSlugs.includes(entry.slug)) { throw new Error(`${entry.slug}: person entry is not explicitly allowlisted`); } if (!entry.reviewedContentDigest || knowledgeContentDigest(entry) !== entry.reviewedContentDigest) { throw new Error(`${entry.slug}: approved source no longer matches its reviewed digest`); } if (!entry.reviewReceiptDigest) { throw new Error(`${entry.slug}: approved source has no review receipt digest`); } } async function main(): Promise { const apply = process.argv.includes("--apply"); const includeRecords = process.argv.includes("--json"); const all = process.argv.includes("--all"); const fromOriginMain = process.argv.includes("--from-origin-main"); const slugIndex = process.argv.indexOf("--slug"); const slug = slugIndex >= 0 ? process.argv[slugIndex + 1] : undefined; if (Boolean(slug) === all) { throw new Error("Choose exactly one of --slug or --all"); } if (apply && !fromOriginMain) throw new Error("--apply requires --from-origin-main"); if (apply) assertOriginMain(); const graph = await loadKnowledgeGraph(); if (graph.errors.length > 0) throw new Error(`knowledge graph errors: ${graph.errors.join("; ")}`); const entries = all ? graph.entries.filter((entry) => !entry.draft).sort((left, right) => left.slug.localeCompare(right.slug)) : [graph.bySlug.get(slug!)].filter((entry): entry is KnowledgeEntry => Boolean(entry && !entry.draft)); if (entries.length === 0) throw new Error(`approved Knowledge entry not found: ${slug}`); entries.forEach(validateEntry); const manifest = loadManifest(); const entrySlugs = new Set(graph.entries.filter((entry) => !entry.draft).map((entry) => entry.slug)); const orphanSlugs = Object.keys(manifest.entries).filter((entrySlug) => !entrySlugs.has(entrySlug)); if (orphanSlugs.length > 0) { throw new Error(`manifest has orphaned Knowledge records; explicit withdrawal required: ${orphanSlugs.join(", ")}`); } const pds = await resolvePds(); const publications = await listRecords(pds, PUBLICATION_COLLECTION); const publicationMatches = manifest.publication ? (await Promise.all([ getRecord(pds, PUBLICATION_COLLECTION, rkeyFromUri(manifest.publication.uri)), ])).filter((record): record is RemoteRecord => Boolean(record)) : publications.filter((record) => record.value.url === PUBLICATION_URL); const publicationRecord = desiredPublication(); const publicationPlan = classify({ collection: PUBLICATION_COLLECTION, desired: publicationRecord, manifestUri: manifest.publication?.uri, manifestCid: manifest.publication?.cid, discovered: publicationMatches, }); if (publicationRecord.preferences.showInDiscover !== false) { throw new Error("Knowledge publication must explicitly set showInDiscover=false"); } const allDocuments = await listRecords(pds, DOCUMENT_COLLECTION); const documentPlans = await Promise.all(entries.map(async (entry) => { const documentRecord = desiredDocument(entry, publicationPlan.uri); const manifestEntry = manifest.entries[entry.slug]; const matchingDocuments = allDocuments.filter( (record) => record.value.site === publicationPlan.uri && record.value.path === `/${knowledgeRoute(entry)}`, ); const documents = manifestEntry ? (await Promise.all([ getRecord(pds, DOCUMENT_COLLECTION, rkeyFromUri(manifestEntry.uri)), ])).filter((record): record is RemoteRecord => Boolean(record)) : matchingDocuments.length > 0 ? matchingDocuments : []; return classify({ collection: DOCUMENT_COLLECTION, desired: documentRecord, manifestUri: manifestEntry?.uri, manifestCid: manifestEntry?.cid, discovered: documents, createRkey: !manifestEntry && documents.length === 0 ? nextStandardSiteRkey() : undefined, }); })); const plans: Array> = [publicationPlan, ...documentPlans]; const conflicts = plans.filter((plan) => plan.action === "conflict"); const report = { mode: apply ? "apply" : "dry-run", did: CAMERON_DID, pds, discoveryInvariant: { field: "site.standard.publication.preferences.showInDiscover", value: false, scope: "dedicated Knowledge publication", }, publication: includeRecords ? publicationPlan : { ...publicationPlan, record: undefined }, documents: documentPlans.map((documentPlan) => includeRecords ? documentPlan : { ...documentPlan, record: undefined } ), counts: Object.fromEntries( ["create", "update", "unchanged", "adopt", "conflict"].map((action) => [ action, plans.filter((plan) => plan.action === action).length, ]), ), }; if (!apply) { console.log(JSON.stringify(report, null, 2)); return; } if (conflicts.length > 0) throw new Error(`refusing conflicts: ${JSON.stringify(conflicts)}`); const session = await createSession(pds); const writeReceipt = await applyWrites(pds, session.accessJwt, plans); const verifiedPublication = await getRecord(pds, PUBLICATION_COLLECTION, publicationPlan.rkey); if (!verifiedPublication || digestRecord(verifiedPublication.value) !== publicationPlan.recordDigest) { throw new Error("post-write Knowledge publication verification failed"); } if (verifiedPublication.value.preferences?.showInDiscover !== false) { throw new Error("post-write discovery suppression verification failed"); } const syncedAt = new Date().toISOString(); let manifestChanged = false; if ( !manifest.publication || manifest.publication.uri !== verifiedPublication.uri || manifest.publication.cid !== verifiedPublication.cid || manifest.publication.recordDigest !== publicationPlan.recordDigest || manifest.publication.pds !== pds ) { manifest.publication = { uri: verifiedPublication.uri, cid: verifiedPublication.cid, recordDigest: publicationPlan.recordDigest, showInDiscover: false, pds, syncedAt, }; manifestChanged = true; } const verifiedDocuments = []; for (let index = 0; index < documentPlans.length; index++) { const entry = entries[index]; const documentPlan = documentPlans[index]; const verifiedDocument = await getRecord( pds, DOCUMENT_COLLECTION, documentPlan.rkey, ); if (!verifiedDocument || digestRecord(verifiedDocument.value) !== documentPlan.recordDigest) { throw new Error(`${entry.slug}: post-write Knowledge document verification failed`); } if ( verifiedDocument.value.site !== verifiedPublication.uri || verifiedDocument.value.path !== `/${knowledgeRoute(entry)}` ) { throw new Error(`${entry.slug}: post-write document publication/path verification failed`); } const previous = manifest.entries[entry.slug]; if ( !previous || previous.uri !== verifiedDocument.uri || previous.cid !== verifiedDocument.cid || previous.recordDigest !== documentPlan.recordDigest || previous.reviewedContentDigest !== entry.reviewedContentDigest || previous.reviewReceiptDigest !== entry.reviewReceiptDigest || previous.pds !== pds ) { manifest.entries[entry.slug] = { uri: verifiedDocument.uri, cid: verifiedDocument.cid, recordDigest: documentPlan.recordDigest, reviewedContentDigest: entry.reviewedContentDigest!, reviewReceiptDigest: entry.reviewReceiptDigest!, pds, syncedAt, }; manifestChanged = true; } verifiedDocuments.push({ slug: entry.slug, uri: verifiedDocument.uri, cid: verifiedDocument.cid, canonicalUrl: `${PUBLICATION_URL}/${knowledgeRoute(entry)}`, }); } if (manifestChanged) writeManifest(manifest); console.log(JSON.stringify({ ...report, receipt: { write: writeReceipt, publication: { uri: verifiedPublication.uri, cid: verifiedPublication.cid, showInDiscover: false }, documents: verifiedDocuments, manifest: MANIFEST_PATH, manifestChanged, }, }, null, 2)); } main().catch((error) => { console.error(error instanceof Error ? error.message : error); process.exit(1); });