Something went wrong. Try again.
My website. cameron.stream
Something went wrong. Try again.
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663import { createHash } from "node:crypto";import { execFileSync } from "node:child_process";import { readFileSync, renameSync, writeFileSync } from "node:fs";import { resolve } from "node:path";import "../src/env.ts";import { knowledgeContentDigest, knowledgeRoute, loadKnowledgeGraph, type KnowledgeEntry,} from "../src/knowledge.ts";import { nextStandardSiteRkey, shouldValidateStandardSiteWrite, type KnowledgeAtprotoManifest,} from "../src/knowledge-atproto.ts";import { portableKnowledgeMarkdown, portableMarkdownLink,} from "../src/knowledge-charts/portable.ts";import { loadKnowledgePolicy, scanKnowledgeDraft } from "./knowledge-policy.ts";
const CAMERON_DID = "did:plc:gfrmhdmjvxn2sjedzboeudef";const PUBLICATION_COLLECTION = "site.standard.publication";const DOCUMENT_COLLECTION = "site.standard.document";const PUBLICATION_URL = "https://cameron.stream/knowledge";const PUBLICATION_NAME = "Knowledge";const PUBLICATION_DESCRIPTION = "Linked technical notes, subject maps, lessons, and dated synthesis from Cameron.";const MANIFEST_PATH = resolve(process.cwd(), "knowledge/atproto-manifest.json");
interface BlobRef { $type: "blob"; ref: { $link: string }; mimeType: string; size: number;}
interface PublicationRecord { $type: typeof PUBLICATION_COLLECTION; url: string; name: string; description: string; icon: BlobRef; basicTheme: { $type: "site.standard.theme.basic"; background: Color; foreground: Color; accent: Color; accentForeground: Color; }; preferences: { showInDiscover: false };}
interface Color { $type: "site.standard.theme.color#rgb"; r: number; g: number; b: number;}
interface StandardDocumentRecord { $type: typeof DOCUMENT_COLLECTION; site: string; title: string; description: string; publishedAt: string; updatedAt: string; path: string; tags: string[]; textContent: string; content: { $type: "site.standard.content.markdown"; text: string; version: "1.0"; };}
interface RemoteRecord<T> { uri: string; cid: string; value: T;}
type PlanAction = "create" | "update" | "unchanged" | "adopt" | "conflict";
interface RecordPlan<T> { collection: string; rkey: string; uri: string; action: PlanAction; reason: string; record: T; recordDigest: string; remoteCid?: string;}
function canonicalJson(value: unknown): string { if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; if (value && typeof value === "object") { return `{${Object.entries(value as Record<string, unknown>) .sort(([a], [b]) => a.localeCompare(b)) .map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`) .join(",")}}`; } return JSON.stringify(value);}
function digestRecord(record: unknown): string { return `sha256:${createHash("sha256").update(canonicalJson(record)).digest("hex")}`;}
function loadManifest(): KnowledgeAtprotoManifest { const manifest = JSON.parse(readFileSync(MANIFEST_PATH, "utf8")) as KnowledgeAtprotoManifest; if (manifest.version !== 2) throw new Error("Unsupported Knowledge ATProto manifest"); if (manifest.publication && manifest.publication.showInDiscover !== false) { throw new Error("Refusing a Knowledge manifest without explicit showInDiscover=false"); } return manifest;}
function writeManifest(manifest: KnowledgeAtprotoManifest): void { const temporary = `${MANIFEST_PATH}.tmp`; writeFileSync(temporary, `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); renameSync(temporary, MANIFEST_PATH);}
function desiredPublication(): PublicationRecord { const color = (r: number, g: number, b: number): Color => ({ $type: "site.standard.theme.color#rgb", r, g, b, }); return { $type: PUBLICATION_COLLECTION, url: PUBLICATION_URL, name: PUBLICATION_NAME, description: PUBLICATION_DESCRIPTION, icon: { $type: "blob", ref: { $link: "bafkreibo3puyp2y32k33cp57em6ihxkoqgxfvrvbrmejnkdsjia5su3imi" }, mimeType: "image/webp", size: 3680, }, basicTheme: { $type: "site.standard.theme.basic", background: color(15, 15, 15), foreground: color(232, 228, 234), accent: color(180, 151, 191), accentForeground: color(255, 255, 255), }, preferences: { showInDiscover: false }, };}
function publicMarkdown(entry: KnowledgeEntry): string { const portableBody = portableKnowledgeMarkdown(entry.body, entry.slug); const absoluteLinks = portableBody.replace( /\]\(\/knowledge\/([a-z0-9-]+(?:\/[a-z0-9-]+)*)([^)]*)\)/g, "](https://cameron.stream/knowledge/$1$2)", ); const video = entry.youtubeVideoId ? `${portableMarkdownLink("Watch this episode on YouTube", `https://www.youtube.com/watch?v=${entry.youtubeVideoId}`)}\n\n` : ""; const sources = entry.sources.length === 0 ? "" : `\n\n## Sources\n\n${entry.sources .map((source) => `- ${portableMarkdownLink(source.title, source.url)}`) .join("\n")}`; return `${video}${absoluteLinks}${sources}`;}
function stripMarkdown(markdown: string): string { return markdown .replace(/```[\s\S]*?```/g, "") .replace(/`([^`]+)`/g, "$1") .replace(/!\[[^\]]*\]\([^)]+\)/g, "") .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") .replace(/<[^>]+>/g, "") .replace(/^#{1,6}\s+/gm, "") .replace(/^\s*>\s?/gm, "") .replace(/^\s*[-*+]\s+/gm, "") .replace(/^\s*\d+\.\s+/gm, "") .replace(/\*\*([^*]+)\*\*/g, "$1") .replace(/\*([^*]+)\*/g, "$1") .replace(/__([^_]+)__/g, "$1") .replace(/_([^_]+)_/g, "$1") .replace(/\n{3,}/g, "\n\n") .trim();}
function desiredDocument(entry: KnowledgeEntry, publicationUri: string): StandardDocumentRecord { const markdown = publicMarkdown(entry); const record: StandardDocumentRecord = { $type: DOCUMENT_COLLECTION, site: publicationUri, title: entry.title, description: entry.summary, publishedAt: (entry.publishedAt ?? entry.updated).toISOString(), updatedAt: entry.updated.toISOString(), path: `/${knowledgeRoute(entry)}`, tags: [...new Set(["knowledge", entry.kind, ...entry.topics])], textContent: stripMarkdown(markdown).slice(0, 100_000), content: { $type: "site.standard.content.markdown", text: markdown, version: "1.0", }, }; const bytes = Buffer.byteLength(JSON.stringify(record)); if (bytes > 95_000) throw new Error(`${entry.slug}: Standard.site record is too large (${bytes} bytes)`); return record;}
async function resolvePds(): Promise<string> { const response = await fetch(`https://plc.directory/${CAMERON_DID}`); if (!response.ok) throw new Error(`Could not resolve Cameron DID: ${response.status}`); const document = await response.json() as { service?: Array<{ id?: string; type?: string; serviceEndpoint?: string }>; }; const pds = document.service?.find((service) => service.id === "#atproto_pds" || service.type === "AtprotoPersonalDataServer" )?.serviceEndpoint; if (!pds) throw new Error("Cameron DID has no ATProto PDS endpoint"); return pds.replace(/\/$/, "");}
async function listRecords<T>(pds: string, collection: string): Promise<Array<RemoteRecord<T>>> { const records: Array<RemoteRecord<T>> = []; let cursor: string | undefined; do { const query = new URLSearchParams({ repo: CAMERON_DID, collection, limit: "100", ...(cursor ? { cursor } : {}), }); const response = await fetch(`${pds}/xrpc/com.atproto.repo.listRecords?${query}`); if (!response.ok) throw new Error(`listRecords ${collection} failed: ${response.status}`); const page = await response.json() as { records: Array<RemoteRecord<T>>; cursor?: string }; records.push(...page.records); cursor = page.cursor; } while (cursor); return records;}
async function getRecord<T>( pds: string, collection: string, rkey: string,): Promise<RemoteRecord<T> | null> { const query = new URLSearchParams({ repo: CAMERON_DID, collection, rkey }); const response = await fetch(`${pds}/xrpc/com.atproto.repo.getRecord?${query}`); if (response.status === 400) { const body = await response.json().catch(() => ({})) as { error?: string }; if (body.error === "RecordNotFound") return null; } if (!response.ok) throw new Error(`getRecord ${collection}/${rkey} failed: ${response.status}`); return await response.json() as RemoteRecord<T>;}
function rkeyFromUri(uri: string): string { return uri.slice(uri.lastIndexOf("/") + 1);}
function recordUri(collection: string, rkey: string): string { return `at://${CAMERON_DID}/${collection}/${rkey}`;}
function classify<T>(options: { collection: string; desired: T; manifestUri?: string; manifestCid?: string; discovered: Array<RemoteRecord<T>>; createRkey?: string;}): RecordPlan<T> { const { collection, desired, manifestUri, manifestCid, discovered } = options; if (discovered.length > 1) { return { collection, rkey: "conflict", uri: "conflict", action: "conflict", reason: "multiple remote records match the same canonical object", record: desired, recordDigest: digestRecord(desired), }; } const remote = discovered[0]; const desiredDigest = digestRecord(desired); if (!remote) { if (manifestUri) { return { collection, rkey: rkeyFromUri(manifestUri), uri: manifestUri, action: "conflict", reason: "manifest expects a remote record that is missing", record: desired, recordDigest: desiredDigest, }; } const rkey = options.createRkey ?? nextStandardSiteRkey(); return { collection, rkey, uri: recordUri(collection, rkey), action: "create", reason: "no matching remote record exists", record: desired, recordDigest: desiredDigest, }; } const remoteDigest = digestRecord(remote.value); if (remoteDigest === desiredDigest) { return { collection, rkey: rkeyFromUri(remote.uri), uri: remote.uri, action: manifestUri ? "unchanged" : "adopt", reason: manifestUri ? "remote record matches the manifest and local source" : "matching remote record can be adopted", record: desired, recordDigest: desiredDigest, remoteCid: remote.cid, }; } if (!manifestUri || manifestUri !== remote.uri || manifestCid !== remote.cid) { return { collection, rkey: rkeyFromUri(remote.uri), uri: remote.uri, action: "conflict", reason: "remote record differs and is not owned by the current manifest CID", record: desired, recordDigest: desiredDigest, remoteCid: remote.cid, }; } return { collection, rkey: rkeyFromUri(remote.uri), uri: remote.uri, action: "update", reason: "approved local source changed while the remote CID still matches the manifest", record: desired, recordDigest: desiredDigest, remoteCid: remote.cid, };}
async function createSession(pds: string): Promise<{ accessJwt: string; did: string }> { const password = process.env.CAMERON_BSKY_APP_PASSWORD; if (!password) throw new Error("CAMERON_BSKY_APP_PASSWORD is required for --apply"); const response = await fetch(`${pds}/xrpc/com.atproto.server.createSession`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ identifier: CAMERON_DID, password }), }); if (!response.ok) throw new Error(`createSession failed: ${response.status} ${await response.text()}`); const session = await response.json() as { accessJwt: string; did: string }; if (session.did !== CAMERON_DID) { throw new Error(`Refusing wrong identity: authenticated ${session.did}, expected ${CAMERON_DID}`); } return session;}
function toWrite<T>(plan: RecordPlan<T>): Record<string, unknown> | undefined { if (plan.action === "create") { return { $type: "com.atproto.repo.applyWrites#create", collection: plan.collection, rkey: plan.rkey, value: plan.record, }; } if (plan.action === "update") { return { $type: "com.atproto.repo.applyWrites#update", collection: plan.collection, rkey: plan.rkey, value: plan.record, swapRecord: plan.remoteCid, }; } return undefined;}
async function applyWrites( pds: string, accessJwt: string, plans: Array<RecordPlan<unknown>>,): Promise<unknown> { const operations = plans.flatMap((plan) => { const write = toWrite(plan); if (!write || (plan.action !== "create" && plan.action !== "update")) return []; return [{ write, validate: shouldValidateStandardSiteWrite({ collection: plan.collection, action: plan.action, rkey: plan.rkey, }), }]; }); if (operations.length === 0) return { unchanged: true }; const batches: Array<{ validate: boolean; writes: Array<Record<string, unknown>> }> = []; for (const operation of operations) { const current = batches.at(-1); const candidate = [...(current?.writes ?? []), operation.write]; const bytes = Buffer.byteLength(JSON.stringify({ repo: CAMERON_DID, writes: candidate })); if ( !current || current.validate !== operation.validate || candidate.length > 20 || bytes > 500_000 ) { batches.push({ validate: operation.validate, writes: [operation.write] }); } else { current.writes = candidate; } } const receipts = []; for (let index = 0; index < batches.length; index++) { const batch = batches[index]; const response = await fetch(`${pds}/xrpc/com.atproto.repo.applyWrites`, { method: "POST", headers: { Authorization: `Bearer ${accessJwt}`, "Content-Type": "application/json" }, body: JSON.stringify({ repo: CAMERON_DID, writes: batch.writes, validate: batch.validate }), }); if (!response.ok) { throw new Error( `applyWrites batch ${index + 1}/${batches.length} failed: ${response.status} ${await response.text()}`, ); } receipts.push({ validate: batch.validate, response: await response.json() }); } return { batches: receipts };}
function assertOriginMain(): void { const status = execFileSync("git", ["status", "--porcelain"], { encoding: "utf8" }); if (status.trim()) throw new Error("--from-origin-main requires a clean Git worktree"); const branch = execFileSync("git", ["branch", "--show-current"], { encoding: "utf8" }).trim(); if (branch !== "main") throw new Error(`--from-origin-main requires branch main, found ${branch}`); const head = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(); const remote = execFileSync("git", ["rev-parse", "origin/main"], { encoding: "utf8" }).trim(); if (head !== remote) throw new Error("--from-origin-main requires HEAD == origin/main");}
function validateEntry(entry: KnowledgeEntry): void { const policy = loadKnowledgePolicy(); const blocking = scanKnowledgeDraft(entry.body, policy).filter((finding) => finding.severity === "block"); if (blocking.length > 0) { throw new Error(`${entry.slug}: blocking policy findings: ${JSON.stringify(blocking)}`); } if (entry.kind === "person" && !policy.allowedPeopleSlugs.includes(entry.slug)) { throw new Error(`${entry.slug}: person entry is not explicitly allowlisted`); } if (!entry.reviewedContentDigest || knowledgeContentDigest(entry) !== entry.reviewedContentDigest) { throw new Error(`${entry.slug}: approved source no longer matches its reviewed digest`); } if (!entry.reviewReceiptDigest) { throw new Error(`${entry.slug}: approved source has no review receipt digest`); }}
async function main(): Promise<void> { const apply = process.argv.includes("--apply"); const includeRecords = process.argv.includes("--json"); const all = process.argv.includes("--all"); const fromOriginMain = process.argv.includes("--from-origin-main"); const slugIndex = process.argv.indexOf("--slug"); const slug = slugIndex >= 0 ? process.argv[slugIndex + 1] : undefined; if (Boolean(slug) === all) { throw new Error("Choose exactly one of --slug <slug> or --all"); } if (apply && !fromOriginMain) throw new Error("--apply requires --from-origin-main"); if (apply) assertOriginMain();
const graph = await loadKnowledgeGraph(); if (graph.errors.length > 0) throw new Error(`knowledge graph errors: ${graph.errors.join("; ")}`); const entries = all ? graph.entries.filter((entry) => !entry.draft).sort((left, right) => left.slug.localeCompare(right.slug)) : [graph.bySlug.get(slug!)].filter((entry): entry is KnowledgeEntry => Boolean(entry && !entry.draft)); if (entries.length === 0) throw new Error(`approved Knowledge entry not found: ${slug}`); entries.forEach(validateEntry);
const manifest = loadManifest(); const entrySlugs = new Set(graph.entries.filter((entry) => !entry.draft).map((entry) => entry.slug)); const orphanSlugs = Object.keys(manifest.entries).filter((entrySlug) => !entrySlugs.has(entrySlug)); if (orphanSlugs.length > 0) { throw new Error(`manifest has orphaned Knowledge records; explicit withdrawal required: ${orphanSlugs.join(", ")}`); } const pds = await resolvePds(); const publications = await listRecords<PublicationRecord>(pds, PUBLICATION_COLLECTION); const publicationMatches = manifest.publication ? (await Promise.all([ getRecord<PublicationRecord>(pds, PUBLICATION_COLLECTION, rkeyFromUri(manifest.publication.uri)), ])).filter((record): record is RemoteRecord<PublicationRecord> => Boolean(record)) : publications.filter((record) => record.value.url === PUBLICATION_URL);
const publicationRecord = desiredPublication(); const publicationPlan = classify({ collection: PUBLICATION_COLLECTION, desired: publicationRecord, manifestUri: manifest.publication?.uri, manifestCid: manifest.publication?.cid, discovered: publicationMatches, }); if (publicationRecord.preferences.showInDiscover !== false) { throw new Error("Knowledge publication must explicitly set showInDiscover=false"); }
const allDocuments = await listRecords<StandardDocumentRecord>(pds, DOCUMENT_COLLECTION); const documentPlans = await Promise.all(entries.map(async (entry) => { const documentRecord = desiredDocument(entry, publicationPlan.uri); const manifestEntry = manifest.entries[entry.slug]; const matchingDocuments = allDocuments.filter( (record) => record.value.site === publicationPlan.uri && record.value.path === `/${knowledgeRoute(entry)}`, ); const documents = manifestEntry ? (await Promise.all([ getRecord<StandardDocumentRecord>(pds, DOCUMENT_COLLECTION, rkeyFromUri(manifestEntry.uri)), ])).filter((record): record is RemoteRecord<StandardDocumentRecord> => Boolean(record)) : matchingDocuments.length > 0 ? matchingDocuments : []; return classify({ collection: DOCUMENT_COLLECTION, desired: documentRecord, manifestUri: manifestEntry?.uri, manifestCid: manifestEntry?.cid, discovered: documents, createRkey: !manifestEntry && documents.length === 0 ? nextStandardSiteRkey() : undefined, }); })); const plans: Array<RecordPlan<unknown>> = [publicationPlan, ...documentPlans];
const conflicts = plans.filter((plan) => plan.action === "conflict"); const report = { mode: apply ? "apply" : "dry-run", did: CAMERON_DID, pds, discoveryInvariant: { field: "site.standard.publication.preferences.showInDiscover", value: false, scope: "dedicated Knowledge publication", }, publication: includeRecords ? publicationPlan : { ...publicationPlan, record: undefined }, documents: documentPlans.map((documentPlan) => includeRecords ? documentPlan : { ...documentPlan, record: undefined } ), counts: Object.fromEntries( ["create", "update", "unchanged", "adopt", "conflict"].map((action) => [ action, plans.filter((plan) => plan.action === action).length, ]), ), }; if (!apply) { console.log(JSON.stringify(report, null, 2)); return; } if (conflicts.length > 0) throw new Error(`refusing conflicts: ${JSON.stringify(conflicts)}`);
const session = await createSession(pds); const writeReceipt = await applyWrites(pds, session.accessJwt, plans); const verifiedPublication = await getRecord<PublicationRecord>(pds, PUBLICATION_COLLECTION, publicationPlan.rkey); if (!verifiedPublication || digestRecord(verifiedPublication.value) !== publicationPlan.recordDigest) { throw new Error("post-write Knowledge publication verification failed"); } if (verifiedPublication.value.preferences?.showInDiscover !== false) { throw new Error("post-write discovery suppression verification failed"); } const syncedAt = new Date().toISOString(); let manifestChanged = false; if ( !manifest.publication || manifest.publication.uri !== verifiedPublication.uri || manifest.publication.cid !== verifiedPublication.cid || manifest.publication.recordDigest !== publicationPlan.recordDigest || manifest.publication.pds !== pds ) { manifest.publication = { uri: verifiedPublication.uri, cid: verifiedPublication.cid, recordDigest: publicationPlan.recordDigest, showInDiscover: false, pds, syncedAt, }; manifestChanged = true; } const verifiedDocuments = []; for (let index = 0; index < documentPlans.length; index++) { const entry = entries[index]; const documentPlan = documentPlans[index]; const verifiedDocument = await getRecord<StandardDocumentRecord>( pds, DOCUMENT_COLLECTION, documentPlan.rkey, ); if (!verifiedDocument || digestRecord(verifiedDocument.value) !== documentPlan.recordDigest) { throw new Error(`${entry.slug}: post-write Knowledge document verification failed`); } if ( verifiedDocument.value.site !== verifiedPublication.uri || verifiedDocument.value.path !== `/${knowledgeRoute(entry)}` ) { throw new Error(`${entry.slug}: post-write document publication/path verification failed`); } const previous = manifest.entries[entry.slug]; if ( !previous || previous.uri !== verifiedDocument.uri || previous.cid !== verifiedDocument.cid || previous.recordDigest !== documentPlan.recordDigest || previous.reviewedContentDigest !== entry.reviewedContentDigest || previous.reviewReceiptDigest !== entry.reviewReceiptDigest || previous.pds !== pds ) { manifest.entries[entry.slug] = { uri: verifiedDocument.uri, cid: verifiedDocument.cid, recordDigest: documentPlan.recordDigest, reviewedContentDigest: entry.reviewedContentDigest!, reviewReceiptDigest: entry.reviewReceiptDigest!, pds, syncedAt, }; manifestChanged = true; } verifiedDocuments.push({ slug: entry.slug, uri: verifiedDocument.uri, cid: verifiedDocument.cid, canonicalUrl: `${PUBLICATION_URL}/${knowledgeRoute(entry)}`, }); } if (manifestChanged) writeManifest(manifest); console.log(JSON.stringify({ ...report, receipt: { write: writeReceipt, publication: { uri: verifiedPublication.uri, cid: verifiedPublication.cid, showInDiscover: false }, documents: verifiedDocuments, manifest: MANIFEST_PATH, manifestChanged, }, }, null, 2));}
main().catch((error) => { console.error(error instanceof Error ? error.message : error); process.exit(1);});