import { createHash } from "node:crypto"; import { lstat, readdir, readFile, realpath } from "node:fs/promises"; import { resolve } from "node:path"; import matter from "gray-matter"; import { z } from "zod"; import { knowledgeChartDigestProjection, knowledgeChartReferences, } from "./knowledge-charts/references.ts"; const DigestSchema = z.string().regex(/^sha256:[a-f0-9]{64}$/); const PublicHttpUrlSchema = z.string().url().refine((value) => { if (value.includes("\\")) return false; try { const protocol = new URL(value).protocol; return protocol === "http:" || protocol === "https:"; } catch { return false; } }, "public source URL must use HTTP(S) without backslashes"); const KnowledgeSourceSchema = z.object({ title: z.string().trim().min(1), url: PublicHttpUrlSchema, }).strict(); const KnowledgeRouteSchema = z.string().regex( /^[a-z0-9]+(?:-[a-z0-9]+)*(?:\/[a-z0-9]+(?:-[a-z0-9]+)*)*$/, ); const YouTubeVideoIdSchema = z.string().regex( /^[A-Za-z0-9_-]{11}$/, "YouTube video ID must be exactly 11 URL-safe characters", ); const TechnicalPublicationAuthorizationSchema = z.object({ kind: z.literal("technical-publication-authorization"), authorizedBy: z.string().trim().min(1), recordedAt: z.coerce.date(), route: KnowledgeRouteSchema, scope: z.literal("technical-publication"), exactRenderReviewed: z.literal(false), receiptPath: z.string().regex(/^knowledge\/receipts\/technical-publication\/[a-z0-9-]+\.json$/), receiptDigest: DigestSchema, }).strict(); const KnowledgeMetadataSchema = z.object({ title: z.string().trim().min(1), slug: z.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/), route: KnowledgeRouteSchema.optional(), summary: z.string().trim().min(1).max(500), kind: z.enum([ "agent", "concept", "journal", "lesson", "map", "organization", "person", "place", "practice", "project", ]), status: z.enum(["active", "evolving", "historical"]), claimMode: z.enum(["factual", "perspective", "mixed"]), perspectiveOwner: z.string().trim().min(1).optional(), confidence: z.enum(["high", "medium", "low"]), topics: z.array(z.string().trim().min(1)).default([]), related: z.array(z.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/)).default([]), sources: z.array(KnowledgeSourceSchema).default([]), youtubeVideoId: YouTubeVideoIdSchema.optional(), aiAssisted: z.literal(true), generatedBy: z.string().trim().min(1), sourceDigest: DigestSchema.optional(), updated: z.coerce.date(), publishedAt: z.coerce.date().optional(), reviewStatus: z.enum(["draft", "approved"]), reviewBasis: z.enum(["exact-render-review", "technical-publication-authorization"]).optional(), reviewedBy: z.string().trim().min(1).optional(), reviewedAt: z.coerce.date().optional(), implementationReviewedBy: z.string().trim().min(1).optional(), implementationReviewedAt: z.coerce.date().optional(), publicationAuthorization: TechnicalPublicationAuthorizationSchema.optional(), reviewedContentDigest: DigestSchema.optional(), reviewReceiptDigest: DigestSchema.optional(), }); export type KnowledgeSource = z.infer; export type KnowledgeMetadata = z.infer; export interface KnowledgeEntry extends KnowledgeMetadata { body: string; draft: boolean; backlinks: string[]; chartReferences: string[]; } export interface KnowledgeGraph { entries: KnowledgeEntry[]; bySlug: Map; byRoute: Map; errors: string[]; } export function knowledgeDocumentClassLabel( kind: KnowledgeMetadata["kind"], ): string | undefined { if (kind === "journal") return "Journal"; if (kind === "lesson") return "Lesson"; if (kind === "map") return "Subject map"; return undefined; } export function knowledgeRoute(entry: Pick): string { return entry.route ?? entry.slug; } export function knowledgeHref(entry: Pick): string { return `/knowledge/${knowledgeRoute(entry)}`; } function assertPublishable(metadata: KnowledgeMetadata, sourceName: string): void { if (metadata.claimMode !== "factual" && !metadata.perspectiveOwner) { throw new Error(`${sourceName}: perspective entries require perspectiveOwner`); } if (metadata.claimMode !== "perspective" && metadata.sources.length === 0) { throw new Error(`${sourceName}: factual claims require at least one public source`); } if (metadata.reviewStatus === "approved") { if (!metadata.publishedAt || !metadata.reviewedContentDigest || !metadata.reviewReceiptDigest) { throw new Error( `${sourceName}: approved entries require publication time, content digest, and receipt digest`, ); } const basis = metadata.reviewBasis ?? "exact-render-review"; if (basis === "exact-render-review") { if (!metadata.reviewedBy || !metadata.reviewedAt) { throw new Error(`${sourceName}: exact-render review requires reviewedBy and reviewedAt`); } if (metadata.publicationAuthorization) { throw new Error(`${sourceName}: exact-render review cannot carry technical authorization metadata`); } } else { if (metadata.reviewedBy || metadata.reviewedAt) { throw new Error(`${sourceName}: technical authorization must not be represented as exact review`); } if (!metadata.implementationReviewedBy || !metadata.implementationReviewedAt) { throw new Error(`${sourceName}: technical authorization requires implementation review provenance`); } const authorization = metadata.publicationAuthorization; if (!authorization) { throw new Error(`${sourceName}: technical authorization requires a route-scoped receipt`); } if (authorization.route !== knowledgeRoute(metadata)) { throw new Error(`${sourceName}: publication authorization route does not match the entry route`); } if (authorization.receiptDigest !== metadata.reviewReceiptDigest) { throw new Error(`${sourceName}: publication authorization digest does not match the review receipt digest`); } } } } export function parseKnowledgeDocument( source: string, sourceName: string, draft: boolean, ): KnowledgeEntry { const parsed = matter(source); const metadata = KnowledgeMetadataSchema.parse(parsed.data); assertPublishable(metadata, sourceName); if (!draft && metadata.reviewStatus !== "approved") { throw new Error(`${sourceName}: published entries must be approved`); } if (draft && metadata.reviewStatus !== "draft") { throw new Error(`${sourceName}: staged entries must remain drafts`); } const body = parsed.content.trim(); if (!body) { throw new Error(`${sourceName}: entry body is empty`); } const chartReferences = knowledgeChartReferences(body, metadata.slug); return { ...metadata, body, draft, backlinks: [], chartReferences, }; } async function readDirectory( path: string, draft: boolean, ): Promise<{ entries: KnowledgeEntry[]; errors: string[] }> { let files: string[]; try { files = (await readdir(path)).filter((file) => file.endsWith(".md")).sort(); } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code === "ENOENT") return { entries: [], errors: [] }; throw error; } const realRoot = await realpath(path); const results = await Promise.all( files.map(async (file): Promise<{ entry?: KnowledgeEntry; error?: string }> => { try { const filePath = resolve(path, file); const stats = await lstat(filePath); if (stats.isSymbolicLink() || !stats.isFile()) { throw new Error(`${file}: knowledge inputs must be regular files`); } const realFile = await realpath(filePath); if (!realFile.startsWith(`${realRoot}/`)) { throw new Error(`${file}: resolved path escapes the knowledge directory`); } const source = await readFile(filePath, "utf8"); return { entry: parseKnowledgeDocument(source, file, draft) }; } catch (error) { return { error: error instanceof Error ? error.message : String(error) }; } }), ); return { entries: results.flatMap((result) => result.entry ? [result.entry] : []), errors: results.flatMap((result) => result.error ? [result.error] : []), }; } export function knowledgeContentDigest(entry: KnowledgeEntry): string { const charts = knowledgeChartDigestProjection(entry.body, entry.slug); const projection = { title: entry.title, slug: entry.slug, route: entry.route, summary: entry.summary, kind: entry.kind, status: entry.status, claimMode: entry.claimMode, perspectiveOwner: entry.perspectiveOwner, confidence: entry.confidence, topics: entry.topics, related: entry.related, sources: entry.sources, youtubeVideoId: entry.youtubeVideoId, aiAssisted: entry.aiAssisted, generatedBy: entry.generatedBy, sourceDigest: entry.sourceDigest, updated: entry.updated.toISOString(), body: entry.body, ...(charts.length > 0 ? { charts } : {}), }; return `sha256:${createHash("sha256").update(JSON.stringify(projection)).digest("hex")}`; } export async function loadKnowledgeGraph(options?: { root?: string; includeDrafts?: boolean; }): Promise { const root = options?.root ?? resolve(process.cwd(), "knowledge"); const includeDrafts = options?.includeDrafts ?? process.env.KNOWLEDGE_INCLUDE_DRAFTS === "1"; const errors: string[] = []; const load = async (directory: string, draft: boolean): Promise => { try { const result = await readDirectory(resolve(root, directory), draft); errors.push(...result.errors); return result.entries; } catch (error) { errors.push(error instanceof Error ? error.message : String(error)); return []; } }; const published = await load("published", false); const drafts = includeDrafts ? await load("staged", true) : []; const draftSlugs = new Set(drafts.map((entry) => entry.slug)); const entries = [ ...published.filter((entry) => !draftSlugs.has(entry.slug)), ...drafts, ]; const bySlug = new Map(); const byRoute = new Map(); for (const entry of entries) { const existing = bySlug.get(entry.slug); if (existing) { errors.push(`duplicate slug ${entry.slug}`); continue; } bySlug.set(entry.slug, entry); const route = knowledgeRoute(entry); const existingRoute = byRoute.get(route); if (existingRoute) { errors.push(`duplicate route ${route} (${existingRoute.slug}, ${entry.slug})`); continue; } byRoute.set(route, entry); } for (const entry of entries) { for (const relatedSlug of entry.related) { const related = bySlug.get(relatedSlug); if (!related) { errors.push(`${entry.slug}: related entry ${relatedSlug} does not exist`); continue; } related.backlinks.push(entry.slug); } } for (const entry of entries) entry.backlinks.sort(); entries.sort((a, b) => a.title.localeCompare(b.title)); return { entries, bySlug, byRoute, errors }; } export function formatKnowledgeDate(date: Date): string { return new Intl.DateTimeFormat("en-US", { month: "short", day: "numeric", year: "numeric", timeZone: "UTC", }).format(date); }