From 4fab8592de53087c0418637c8b97ff626aad9161 Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Sun, 12 Jul 2026 16:33:35 -0700 Subject: [PATCH] =?UTF-8?q?oven:=20crunch=20a=20private=20paper=20on=20dem?= =?UTF-8?q?and=20=E2=80=94=20upload,=20xelatex,=20get=20the=20pdf=20back?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the oven is the only box with the toolchain, but every path into it built `origin/main` and published to papers.aesthetic.computer. so a document that isn't in the repo — and mustn't be — had nowhere to go. `paper-crunch.mjs` takes an uploaded .tar.gz, builds it, hands the pdf back, and shreds the sources. it never writes to the site dir, never runs `cli.mjs publish|deploy`, never touches the git clone. it only *reads* the house .sty + webfonts out of the clone, because `ac-paper-essay.sty` loads fonts at `../../system/public/type/webfonts/` — so the builder pitches a sandbox two levels deep and links them in, and a bundle using the house styles just works. a run that fell back to nullfont is a failure, not a success (same sniff as `cli.mjs`). the bundle is untrusted: `..`/absolute/link entries refused before extraction, the gzip stream capped mid-flight against a bomb, shell-escape off, and a deadline that kills the process group. `papers/bin/crunch.mjs ` drives it from a laptop with no TeX. Co-Authored-By: Claude Opus 4.8 --- oven/.env.example | 3 + oven/README.md | 71 +++++++ oven/paper-crunch.mjs | 472 ++++++++++++++++++++++++++++++++++++++++++ oven/server.mjs | 101 +++++++++ papers/SCORE.md | 1 + papers/bin/crunch.mjs | 195 +++++++++++++++++ 6 files changed, 843 insertions(+) create mode 100644 oven/paper-crunch.mjs create mode 100755 papers/bin/crunch.mjs diff --git a/oven/.env.example b/oven/.env.example index 8f234a953..805fd6304 100644 --- a/oven/.env.example +++ b/oven/.env.example @@ -32,6 +32,9 @@ OS_BASE_BUILD_CWD=/opt/oven # OS_BASE_WORK_BASE=/tmp # OS_BASE_KEEP_ARTIFACTS=0 +# Paper crunch (on-demand private LaTeX; auth = OS_BUILD_ADMIN_KEY) +# CRUNCH_WORK_DIR=/tmp/oven-crunch + # OS base-image artifact destination (DigitalOcean Spaces) OS_SPACES_KEY=your_key_here OS_SPACES_SECRET=your_secret_here diff --git a/oven/README.md b/oven/README.md index f88c61a9f..56bca581b 100644 --- a/oven/README.md +++ b/oven/README.md @@ -147,6 +147,77 @@ New Netlify function: `system/netlify/functions/tape-bake-complete.mjs`: - MP4 remains in Spaces for backup/future use - ATProto gets blob, MongoDB gets both URLs and rkey +## LaTeX: two paths, and they are not the same + +The oven carries the full texlive/xelatex toolchain, so it is the only box that +can turn a `.tex` into a PDF. There are two ways to ask it, and the difference +matters: + +| | `papers-build` (`papers-builder.mjs`) | `paper-crunch` (`paper-crunch.mjs`) | +|---|---|---| +| **source** | whatever is on `origin/main` | a `.tar.gz` you upload | +| **committed?** | yes — builds from the git clone | no — never touches git | +| **published?** | yes — rsynced to `papers.aesthetic.computer` | **no — the PDF comes back to you and nowhere else** | +| **lifetime** | permanent | job + PDF reaped after 30 min | +| **trigger** | `papers-git-poller.mjs`, or POST | `node papers/bin/crunch.mjs ` | + +Crunch exists for the documents that must *not* be published — a private brief, +a draft, anything not in the repo. It never writes to +`system/public/papers.aesthetic.computer/`, never runs `papers/cli.mjs +publish|deploy`, and never touches the git clone. It only *reads* the house +`.sty` files and webfonts out of the clone, so a bundle using the AC styles +resolves its fonts. + +### Paper Crunch endpoints + +All of them take the admin key — including the reads, because the documents are +private and a 10-character job id is not a security boundary. Auth is the same +`OS_BUILD_ADMIN_KEY` as `/papers-build` (`Authorization: Bearer `). + +| Route | Purpose | +|---|---| +| `POST /paper-crunch` | Body is the gzipped tar. `x-crunch-tex` names the entrypoint (optional — otherwise the one `.tex` with a `\documentclass` wins), `x-crunch-name` labels the job. → `202 {jobId}`, `409` if busy. | +| `GET /paper-crunch` | Active + recent jobs, and the limits. | +| `GET /paper-crunch/:jobId` | Job snapshot. `?logs=1&tail=N` for the log. | +| `GET /paper-crunch/:jobId/stream` | SSE: `logs`, `status`, `complete`. | +| `GET /paper-crunch/:jobId/pdf` | The built PDF. | +| `POST /paper-crunch/:jobId/cancel` | Kill the build. | + +### The fonts + +`ac-paper-essay.sty` loads fonts by the relative path a *real* paper sees — +`Path=../../system/public/type/webfonts/`, which only resolves when the `.tex` +sits two levels below the repo root. An uploaded bundle has no such root, so the +builder pitches one: sources land in `/root/papers/src/`, the webfonts are +linked in at `/root/system/public/type/webfonts`, and the house `.sty` +files are linked in beside the sources and put on `TEXINPUTS`. A bundle can +therefore `\usepackage{ac-paper-essay}` without shipping a copy of it. + +A build whose fonts silently fell back to `nullfont` is a **failure**, not a +success — xelatex exits 0 and emits a stub PDF in that case, so the log is +sniffed for the fontspec/nullfont signature (same check as `papers/cli.mjs`). + +### The bundle is untrusted + +Tar entries with `..`, absolute paths, or links (sym/hard) are refused before +extraction. The gzip stream is capped mid-flight so a bomb cannot fill the +droplet. `xelatex` runs with shell-escape **off** and `openout_any=p`, under a +wall-clock deadline that kills the whole process group. The sources are shredded +as soon as the PDF is out; the job directory is reaped on a TTL. + +Limits live in `CRUNCH_LIMITS` (32 MB upload, 192 MB extracted, 180 s build, +30 min retention). Work lands in `/tmp/oven-crunch/` (`CRUNCH_WORK_DIR`). + +### The client + +```bash +node papers/bin/crunch.mjs [--tex ] [--out ] [--verbose] +``` + +Tars the directory (minus build artifacts), uploads it, streams the log, and +writes the PDF next to the source. Reads `OVEN_URL` (default +`https://oven.aesthetic.computer`) and `OS_BUILD_ADMIN_KEY` (else the vault). + ## Deployment Strategy ### Following Existing Patterns diff --git a/oven/paper-crunch.mjs b/oven/paper-crunch.mjs new file mode 100644 index 000000000..11d793493 --- /dev/null +++ b/oven/paper-crunch.mjs @@ -0,0 +1,472 @@ +// paper-crunch.mjs — On-demand xelatex for a private, uncommitted document. +// +// Upload a self-contained .tar.gz of a paper directory, get a PDF back. This +// is the *opposite* of papers-builder.mjs: nothing here is committed, nothing +// is deployed. It never writes to system/public/papers.aesthetic.computer/, +// never runs `papers/cli.mjs publish|deploy`, and never touches the git clone +// — it only *reads* the house .sty files and webfonts out of it, so a bundle +// that uses the AC styles resolves its fonts. That constraint is the whole +// reason this module exists: the oven is the only box with the toolchain, and +// some documents must never reach papers.aesthetic.computer. +// +// The bundle is untrusted. Entries are checked before extraction, the gzip +// stream is capped mid-flight, the build runs shell-escape off under a +// wall-clock deadline, and the sources are shredded the moment the PDF is out. + +import { promises as fs, existsSync, createWriteStream } from "fs"; +import path from "path"; +import zlib from "zlib"; +import { Readable, Transform } from "stream"; +import { pipeline } from "stream/promises"; +import { randomUUID } from "crypto"; +import { spawn, execFile } from "child_process"; +import { fileURLToPath } from "url"; + +const MAX_RECENT_JOBS = 20; +const MAX_LOG_LINES = 2000; + +const MAX_BUNDLE_BYTES = 32 * 1024 * 1024; // compressed upload +const MAX_EXTRACT_BYTES = 192 * 1024 * 1024; // decompressed tar +const BUILD_TIMEOUT_MS = 180_000; // whole 4-pass chain +const JOB_TTL_MS = 30 * 60 * 1000; // then the PDF is gone too + +const WORK_ROOT = process.env.CRUNCH_WORK_DIR || "/tmp/oven-crunch"; + +// The house .sty + webfonts live in the oven's git clone. Off the droplet +// (local dev) fall back to this repo, so a laptop with a TeX install can +// exercise the same sandbox. +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const ASSET_ROOT = [process.env.NATIVE_GIT_DIR, "/opt/oven/native-git", path.resolve(HERE, "..")] + .filter(Boolean) + .find((dir) => existsSync(path.join(dir, "system", "public", "type", "webfonts"))); + +const jobs = new Map(); +const jobOrder = []; +let activeJobId = null; + +function nowISO() { + return new Date().toISOString(); +} + +function stripAnsi(s) { + return String(s || "").replace(/\u001b\[[0-9;]*m/g, ""); +} + +function addLogLine(job, stream, line) { + const clean = stripAnsi(line).replace(/\r/g, "").trimEnd(); + if (!clean) return; + job.logs.push({ ts: nowISO(), stream, line: clean }); + if (job.logs.length > MAX_LOG_LINES) + job.logs.splice(0, job.logs.length - MAX_LOG_LINES); + job.updatedAt = nowISO(); +} + +function makeSnapshot(job, opts = {}) { + const { includeLogs = false, tail = 200 } = opts; + const snap = { + id: job.id, + name: job.name, + tex: job.tex, + status: job.status, + stage: job.stage, + percent: job.percent, + createdAt: job.createdAt, + startedAt: job.startedAt, + updatedAt: job.updatedAt, + finishedAt: job.finishedAt, + error: job.error, + pages: job.pages, + pdfBytes: job.pdfBytes, + pdfReady: !!job.pdfPath, + expiresAt: job.expiresAt, + logCount: job.logs.length, + elapsedMs: job.startedAt + ? (job.finishedAt ? Date.parse(job.finishedAt) : Date.now()) - + Date.parse(job.startedAt) + : 0, + }; + if (includeLogs) { + const start = Math.max(0, job.logs.length - Math.max(0, tail)); + snap.logs = job.logs.slice(start); + } + return snap; +} + +function wireStream(job, proc, streamName) { + let pending = ""; + const s = streamName === "stdout" ? proc.stdout : proc.stderr; + s.on("data", (chunk) => { + pending += chunk.toString(); + let idx; + while ((idx = pending.indexOf("\n")) >= 0) { + addLogLine(job, streamName, pending.slice(0, idx)); + pending = pending.slice(idx + 1); + } + }); + s.on("end", () => { + if (pending) addLogLine(job, streamName, pending); + }); +} + +function tar(args, cwd) { + return new Promise((resolve, reject) => { + execFile("tar", args, { cwd, timeout: 60_000, maxBuffer: 16 * 1024 * 1024 }, (err, stdout) => { + if (err) return reject(err); + resolve(stdout); + }); + }); +} + +// A tar entry we will not extract: anything that could land outside the job +// dir, plus links of any kind (a symlink to /etc would be readable by xelatex). +function rejectEntry(name) { + if (!name || name.startsWith("/") || /^[A-Za-z]:/.test(name)) return "absolute path"; + if (name.split("/").includes("..")) return "parent traversal"; + return null; +} + +// Decompress through a hard ceiling, so a gzip bomb can't fill the droplet. +async function gunzipCapped(bundle, tarPath) { + let seen = 0; + const ceiling = new Transform({ + transform(chunk, _enc, cb) { + seen += chunk.length; + if (seen > MAX_EXTRACT_BYTES) + return cb(new Error(`bundle expands past ${MAX_EXTRACT_BYTES} bytes`)); + cb(null, chunk); + }, + }); + await pipeline(Readable.from(bundle), zlib.createGunzip(), ceiling, createWriteStream(tarPath)); +} + +// Lay out a fake repo root, because ac-paper-*.sty loads fonts by the relative +// path a real paper sees: `../../system/public/type/webfonts/` from +// papers//. So the sources go two levels down and the webfonts hang off +// the sandbox root. The house styles are linked in beside them, so a bundle +// may \usepackage{ac-paper-essay} without shipping a copy. +async function pitchSandbox(job) { + const root = path.join(job.dir, "root"); + const papers = path.join(root, "papers"); + const src = path.join(papers, "src"); + await fs.mkdir(src, { recursive: true }); + + if (!ASSET_ROOT) { + addLogLine(job, "stderr", " WARN: no AC asset root — house fonts and styles unavailable"); + return { root, papers, src }; + } + + const type = path.join(root, "system", "public", "type"); + await fs.mkdir(type, { recursive: true }); + await fs.symlink( + path.join(ASSET_ROOT, "system", "public", "type", "webfonts"), + path.join(type, "webfonts"), + ); + + for (const sty of await fs.readdir(path.join(ASSET_ROOT, "papers"))) { + if (!sty.endsWith(".sty")) continue; + await fs.symlink(path.join(ASSET_ROOT, "papers", sty), path.join(papers, sty)); + } + addLogLine(job, "stdout", ` FONTS ${path.join(ASSET_ROOT, "system/public/type/webfonts")}`); + + return { root, papers, src }; +} + +async function unpack(job, src) { + const tarPath = path.join(job.dir, "bundle.tar"); + await gunzipCapped(job.bundle, tarPath); + job.bundle = null; // the upload buffer has served its purpose + + const names = (await tar(["-tf", tarPath])).split("\n").filter(Boolean); + if (!names.length) throw new Error("bundle is empty"); + for (const name of names) { + const why = rejectEntry(name); + if (why) throw new Error(`refusing bundle entry (${why}): ${name}`); + } + + // -tv leads each line with the mode string; `l`/`h` are sym/hard links. + for (const line of (await tar(["-tvf", tarPath])).split("\n")) { + if (line[0] === "l" || line[0] === "h") + throw new Error(`refusing bundle entry (link): ${line.trim()}`); + } + + await tar(["-xf", tarPath, "-C", src, "--no-same-owner"]); + await fs.rm(tarPath, { force: true }); + addLogLine(job, "stdout", ` UNPACK ${names.length} entries`); +} + +// The entrypoint is the one .tex with a \documentclass — unless the caller +// named it, in which case take them at their word. +async function findTex(src, wanted) { + const texs = (await fs.readdir(src)).filter((f) => f.endsWith(".tex")); + if (wanted) { + const named = wanted.endsWith(".tex") ? wanted : `${wanted}.tex`; + if (!texs.includes(named)) + throw new Error(`no ${named} in bundle (found: ${texs.join(", ") || "no .tex at all"})`); + return named; + } + if (!texs.length) throw new Error("no .tex at the top of the bundle"); + + const docs = []; + for (const t of texs) { + const body = await fs.readFile(path.join(src, t), "utf8").catch(() => ""); + if (body.includes("\\documentclass")) docs.push(t); + } + if (docs.length === 1) return docs[0]; + if (docs.length > 1) + throw new Error(`ambiguous entrypoint — pass --tex (candidates: ${docs.join(", ")})`); + throw new Error(`no .tex declares a \\documentclass (found: ${texs.join(", ")})`); +} + +function killGroup(proc) { + try { + process.kill(-proc.pid, "SIGKILL"); + } catch { + try { + proc.kill("SIGKILL"); + } catch {} + } +} + +function run(job, cmd, args, cwd, env, msLeft) { + return new Promise((resolve) => { + const proc = spawn(cmd, args, { + cwd, + env, + detached: true, // own process group, so a hung pass dies whole + stdio: ["ignore", "pipe", "pipe"], + }); + job.process = proc; + wireStream(job, proc, "stdout"); + wireStream(job, proc, "stderr"); + + const alarm = setTimeout(() => { + job.timedOut = true; + killGroup(proc); + }, Math.max(1, msLeft)); + + proc.on("error", (err) => { + clearTimeout(alarm); + job.process = null; + addLogLine(job, "stderr", ` ${cmd}: ${err.message}`); + resolve(127); + }); + proc.on("close", (code) => { + clearTimeout(alarm); + job.process = null; + resolve(code); + }); + }); +} + +// The silent failure: fontspec can't find a font, falls back to nullfont, and +// xelatex still exits 0 with a stub PDF. Same sniff as papers/cli.mjs. +async function logShowsBrokenBuild(logPath) { + const log = await fs.readFile(logPath, "utf8").catch(() => ""); + return log.includes("! Package fontspec Error") || log.includes("nullfont"); +} + +async function pdfPages(pdfPath) { + return new Promise((resolve) => { + execFile("pdfinfo", [pdfPath], { timeout: 10_000 }, (err, stdout) => { + if (err) return resolve(null); + const m = stdout.match(/^Pages:\s+(\d+)/m); + resolve(m ? parseInt(m[1], 10) : null); + }); + }); +} + +async function crunch(job) { + job.status = "running"; + job.startedAt = nowISO(); + job.stage = "unpack"; + job.percent = 5; + + const { papers, src } = await pitchSandbox(job); + await unpack(job, src); + + job.tex = await findTex(src, job.wantedTex); + const base = job.tex.replace(/\.tex$/, ""); + addLogLine(job, "stdout", ` CRUNCH ${job.tex}`); + + const env = { + ...process.env, + TERM: "dumb", + TEXINPUTS: `.:${papers}:`, + openout_any: "p", // xelatex writes inside the sandbox or not at all + shell_escape: "f", + }; + const deadline = Date.now() + BUILD_TIMEOUT_MS; + const xelatex = ["-interaction=nonstopmode", "-file-line-error", "-no-shell-escape", job.tex]; + + const passes = [ + ["xelatex", xelatex, "pass 1"], + ["bibtex", [base], "bibtex"], + ["xelatex", xelatex, "pass 2"], + ["xelatex", xelatex, "pass 3"], + ]; + + for (const [i, [cmd, args, label]] of passes.entries()) { + job.stage = label; + job.percent = 10 + Math.round((i / passes.length) * 80); + addLogLine(job, "stdout", ` ${label.toUpperCase()}`); + await run(job, cmd, args, src, env, deadline - Date.now()); + if (job.timedOut) throw new Error(`build exceeded ${BUILD_TIMEOUT_MS / 1000}s — killed`); + if (job.status === "cancelled") throw new Error("cancelled"); + // Exit codes are advisory here: xelatex returns non-zero on warnings and + // bibtex on a missing .bib. The PDF is the only success criterion. + } + + job.stage = "verify"; + job.percent = 95; + + const built = path.join(src, `${base}.pdf`); + if (!existsSync(built)) throw new Error(`no PDF produced — see the log for the LaTeX error`); + if (await logShowsBrokenBuild(path.join(src, `${base}.log`))) + throw new Error("fonts failed to load (fontspec/nullfont) — the PDF would be a stub"); + + const out = path.join(job.dir, "out.pdf"); + await fs.rename(built, out); + job.pdfPath = out; + job.pdfBytes = (await fs.stat(out)).size; + job.pages = await pdfPages(out); + addLogLine( + job, + "stdout", + ` OK ${base}.pdf — ${job.pdfBytes} bytes${job.pages ? `, ${job.pages}pp` : ""}`, + ); +} + +async function runCrunchJob(job) { + try { + await crunch(job); + job.status = "success"; + job.stage = "done"; + job.percent = 100; + } catch (err) { + job.status = job.status === "cancelled" ? "cancelled" : "failed"; + job.stage = job.status; + job.error = err.message || String(err); + addLogLine(job, "stderr", ` FAILED: ${job.error}`); + } finally { + job.finishedAt = nowISO(); + job.bundle = null; + job.expiresAt = new Date(Date.now() + JOB_TTL_MS).toISOString(); + // Shred the sources either way — a private document does not linger on the + // droplet. The PDF (already moved out) survives until the job is reaped. + await fs.rm(path.join(job.dir, "root"), { recursive: true, force: true }).catch(() => {}); + await fs.rm(path.join(job.dir, "bundle.tar"), { force: true }).catch(() => {}); + if (activeJobId === job.id) activeJobId = null; + } +} + +async function reap() { + const now = Date.now(); + for (const id of [...jobOrder]) { + const job = jobs.get(id); + if (!job || id === activeJobId) continue; + if (!job.expiresAt || Date.parse(job.expiresAt) > now) continue; + await fs.rm(job.dir, { recursive: true, force: true }).catch(() => {}); + jobs.delete(id); + jobOrder.splice(jobOrder.indexOf(id), 1); + } + // Anything on disk we've forgotten (a restart mid-job) goes too. + for (const name of await fs.readdir(WORK_ROOT).catch(() => [])) { + if (jobs.has(name)) continue; + await fs.rm(path.join(WORK_ROOT, name), { recursive: true, force: true }).catch(() => {}); + } +} + +setInterval(() => reap().catch(() => {}), 5 * 60 * 1000).unref(); + +export async function startPaperCrunch(bundle, options = {}) { + if (!Buffer.isBuffer(bundle) || !bundle.length) { + const err = new Error("empty bundle — POST a gzipped tar of the paper directory"); + err.code = "CRUNCH_BAD_BUNDLE"; + throw err; + } + if (bundle.length > MAX_BUNDLE_BYTES) { + const err = new Error(`bundle over ${MAX_BUNDLE_BYTES} bytes`); + err.code = "CRUNCH_BAD_BUNDLE"; + throw err; + } + if (activeJobId) { + const err = new Error(`Paper crunch already running: ${activeJobId}`); + err.code = "CRUNCH_BUSY"; + err.activeJobId = activeJobId; + throw err; + } + + const id = randomUUID().slice(0, 10); + const job = { + id, + name: String(options.name || "paper").replace(/[^\w.-]/g, "-").slice(0, 60), + wantedTex: options.tex ? path.basename(String(options.tex)) : null, + tex: null, + dir: path.join(WORK_ROOT, id), + bundle, + status: "queued", + stage: "queued", + percent: 0, + createdAt: nowISO(), + startedAt: null, + updatedAt: nowISO(), + finishedAt: null, + expiresAt: null, + process: null, + timedOut: false, + pdfPath: null, + pdfBytes: null, + pages: null, + error: null, + logs: [], + }; + + await fs.mkdir(job.dir, { recursive: true }); + + jobs.set(id, job); + jobOrder.unshift(id); + while (jobOrder.length > MAX_RECENT_JOBS) { + const old = jobOrder.pop(); + const stale = jobs.get(old); + if (old === activeJobId || !stale) continue; + jobs.delete(old); + fs.rm(stale.dir, { recursive: true, force: true }).catch(() => {}); + } + activeJobId = id; + runCrunchJob(job).catch(() => {}); + return makeSnapshot(job); +} + +export function getPaperCrunch(jobId, opts = {}) { + const job = jobs.get(jobId); + return job ? makeSnapshot(job, opts) : null; +} + +export function getPaperCrunchesSummary() { + return { + activeJobId, + active: activeJobId ? makeSnapshot(jobs.get(activeJobId)) : null, + recent: jobOrder + .map((id) => jobs.get(id)) + .filter(Boolean) + .map((j) => makeSnapshot(j)), + }; +} + +export function getPaperCrunchPdf(jobId) { + const job = jobs.get(jobId); + if (!job) return null; + if (!job.pdfPath || !existsSync(job.pdfPath)) return null; + return { path: job.pdfPath, name: `${job.name}.pdf`, bytes: job.pdfBytes }; +} + +export function cancelPaperCrunch(jobId) { + const job = jobs.get(jobId); + if (!job) return { ok: false, error: "not found" }; + if (job.status !== "running") return { ok: false, error: "not running" }; + job.status = "cancelled"; + if (job.process) killGroup(job.process); + return { ok: true }; +} + +export const CRUNCH_LIMITS = { MAX_BUNDLE_BYTES, MAX_EXTRACT_BYTES, BUILD_TIMEOUT_MS, JOB_TTL_MS }; diff --git a/oven/server.mjs b/oven/server.mjs index ac4870d39..c96da674d 100644 --- a/oven/server.mjs +++ b/oven/server.mjs @@ -22,6 +22,7 @@ import { startNativeBuild, getNativeBuild, getNativeBuildsSummary, cancelNativeB import { startPoller as startNativeGitPoller, getPollerStatus as getNativePollerStatus } from './native-git-poller.mjs'; import { startPapersBuild, getPapersBuild, getPapersBuildsSummary, cancelPapersBuild } from './papers-builder.mjs'; import { startPoller as startPapersGitPoller, getPollerStatus as getPapersPollerStatus } from './papers-git-poller.mjs'; +import { startPaperCrunch, getPaperCrunch, getPaperCrunchesSummary, getPaperCrunchPdf, cancelPaperCrunch, CRUNCH_LIMITS } from './paper-crunch.mjs'; import { startRecapBuild, getRecapBuild, getRecapBuildsSummary, cancelRecapBuild, getRecapMp4Path } from './recap-builder.mjs'; import { startPoller as startRecapGitPoller, getPollerStatus as getRecapPollerStatus } from './recap-git-poller.mjs'; import { join, dirname, basename } from 'path'; @@ -3924,6 +3925,106 @@ app.post('/papers-build/:jobId/cancel', requireOSBuildAdmin, (req, res) => { return res.json(result); }); +// ── Paper Crunch ─────────────────────────────────────────────────────────── +// On-demand xelatex for a private document that is NOT in the repo: POST a +// .tar.gz of the paper directory, stream the log, download the PDF. Nothing is +// committed and nothing reaches papers.aesthetic.computer — that is the whole +// point (see paper-crunch.mjs). Client: `node papers/bin/crunch.mjs `. +// +// Every route takes the admin key, including the reads — the documents are +// private, so a 10-char job id is not the thing standing between them and the +// world. Auth: same OS_BUILD_ADMIN_KEY used for /papers-build. + +app.post('/paper-crunch', requireOSBuildAdmin, express.raw({ type: '*/*', limit: CRUNCH_LIMITS.MAX_BUNDLE_BYTES }), async (req, res) => { + try { + const job = await startPaperCrunch(req.body, { + name: req.get('x-crunch-name'), + tex: req.get('x-crunch-tex'), + }); + addServerLog('info', '📐', `Paper crunch started: ${job.id} (${job.name})`); + return res.status(202).json({ ...job, jobId: job.id }); + } catch (err) { + if (err.code === 'CRUNCH_BUSY') { + return res.status(409).json({ error: err.message, activeJobId: err.activeJobId }); + } + if (err.code === 'CRUNCH_BAD_BUNDLE') { + return res.status(400).json({ error: err.message }); + } + return res.status(500).json({ error: err.message }); + } +}); + +app.get('/paper-crunch', requireOSBuildAdmin, (req, res) => { + res.json({ ...getPaperCrunchesSummary(), limits: CRUNCH_LIMITS }); +}); + +app.get('/paper-crunch/:jobId', requireOSBuildAdmin, (req, res) => { + const tail = Math.max(0, Math.min(2000, parseInt(req.query.tail, 10) || 200)); + const includeLogs = req.query.logs === '1' || req.query.logs === 'true'; + const job = getPaperCrunch(req.params.jobId, { includeLogs, tail }); + if (!job) return res.status(404).json({ error: 'Job not found' }); + return res.json(job); +}); + +app.get('/paper-crunch/:jobId/stream', requireOSBuildAdmin, (req, res) => { + const jobId = req.params.jobId; + const initial = getPaperCrunch(jobId, { includeLogs: true, tail: 500 }); + if (!initial) return res.status(404).json({ error: 'Job not found' }); + + res.set({ + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'X-Accel-Buffering': 'no', + }); + res.flushHeaders(); + + let sentLogs = 0; + const sendEvent = (type, data) => { + res.write(`event: ${type}\ndata: ${JSON.stringify(data)}\n\n`); + if (typeof res.flush === 'function') res.flush(); + }; + + if (Array.isArray(initial.logs) && initial.logs.length > 0) { + sendEvent('logs', { logs: initial.logs }); + sentLogs = initial.logs.length; + } + sendEvent('status', { id: initial.id, status: initial.status, stage: initial.stage, percent: initial.percent }); + + const timer = setInterval(() => { + const job = getPaperCrunch(jobId, { includeLogs: true, tail: 2000 }); + if (!job) { clearInterval(timer); res.end(); return; } + const logs = Array.isArray(job.logs) ? job.logs : []; + if (logs.length > sentLogs) { + sendEvent('logs', { logs: logs.slice(sentLogs) }); + sentLogs = logs.length; + } + sendEvent('status', { id: job.id, status: job.status, stage: job.stage, percent: job.percent, error: job.error }); + if (job.status === 'success' || job.status === 'failed' || job.status === 'cancelled') { + sendEvent('complete', { status: job.status, error: job.error, pages: job.pages, pdfBytes: job.pdfBytes }); + clearInterval(timer); + res.end(); + } + }, 1000); + + req.on('close', () => clearInterval(timer)); +}); + +app.get('/paper-crunch/:jobId/pdf', requireOSBuildAdmin, (req, res) => { + const pdf = getPaperCrunchPdf(req.params.jobId); + if (!pdf) return res.status(404).json({ error: 'No PDF for that job (unfinished, failed, or reaped)' }); + res.setHeader('Content-Type', 'application/pdf'); + res.setHeader('Content-Disposition', `attachment; filename="${pdf.name}"`); + res.sendFile(pdf.path); +}); + +app.post('/paper-crunch/:jobId/cancel', requireOSBuildAdmin, (req, res) => { + const result = cancelPaperCrunch(req.params.jobId); + if (!result.ok) return res.status(400).json(result); + addServerLog('info', '🛑', `Paper crunch cancel requested: ${req.params.jobId}`); + return res.json(result); +}); + // ── Recap Builds ─────────────────────────────────────────────────────────── // Builds a recap mp4 from a single audience config (e.g. jeffrey-73h-2026-05-02). // Auth: same OS_BUILD_ADMIN_KEY used for /native-build / /papers-build. diff --git a/papers/SCORE.md b/papers/SCORE.md index 680010247..010fbba38 100644 --- a/papers/SCORE.md +++ b/papers/SCORE.md @@ -162,6 +162,7 @@ The mill's code lives at the top of `papers/` and in [`bin/`](bin/). All scripts | Script | Purpose | |---|---| | [`bin/build-dossier.mjs`](bin/build-dossier.mjs) | Multi-pass xelatex + bibtex build for a single dossier or `--all`. Reports undefined citations. | +| [`bin/crunch.mjs`](bin/crunch.mjs) | Build a paper on the **oven** instead of locally — for a machine with no TeX, or a document that must stay private. Tars the directory, POSTs it to `/paper-crunch`, streams the log, writes the PDF. Never commits and never publishes: the crunched PDF comes back to you and goes nowhere near `papers.aesthetic.computer` (that is `cli.mjs publish`, which builds from `main`). House `.sty` + webfonts resolve inside the oven's sandbox. | | [`bin/gen-cover.mjs`](bin/gen-cover.mjs) | Generate the colored-pencil vignette cover illustration for a dossier from `figures/cover-prompt.txt` via OpenAI gpt-image-2 (1024×1024 square, faded edges). | | [`bin/gen-qrs.mjs`](bin/gen-qrs.mjs) | Generate per-paper QR-code PNG pointing to the deployed permalink at `papers.aesthetic.computer/.pdf`. Uses `qrencode` CLI. | | [`bin/migrate-cover.mjs`](bin/migrate-cover.mjs) | One-shot migration: rewrite an old-style cover block (4em pals + 15em hero) into the new vignette layout (pals top-left + QR top-right + TikZ-overlaid title floating over faded illustration). Idempotent. | diff --git a/papers/bin/crunch.mjs b/papers/bin/crunch.mjs new file mode 100755 index 000000000..85dca7f7c --- /dev/null +++ b/papers/bin/crunch.mjs @@ -0,0 +1,195 @@ +#!/usr/bin/env node +// crunch.mjs — build a paper on the oven, from a laptop with no TeX. +// +// Tars the directory, POSTs it to the oven's /paper-crunch, streams the log +// back, and drops the PDF next to the source. The document is never committed +// and never lands on papers.aesthetic.computer — this is for the private ones. +// (The public path is `cli.mjs publish`, which builds from main.) +// +// Usage: +// node papers/bin/crunch.mjs vault/some-brief +// node papers/bin/crunch.mjs vault/some-brief --tex brief.tex --out ~/brief.pdf +// node papers/bin/crunch.mjs vault/some-brief --verbose +// +// Env: +// OVEN_URL default https://oven.aesthetic.computer +// OS_BUILD_ADMIN_KEY else read from the vault (plain, then GPG) + +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync, writeFileSync, statSync } from "node:fs"; +import { resolve, dirname, join, basename } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO = resolve(HERE, "..", ".."); +const OVEN = (process.env.OVEN_URL || "https://oven.aesthetic.computer").replace(/\/$/, ""); + +// Build artifacts stay home — the oven makes its own (papers/.gitignore shapes). +const DROSS = ["*.pdf", "*.aux", "*.log", "*.out", "*.toc", "*.bbl", "*.blg", ".git", ".DS_Store"]; + +const argv = process.argv.slice(2); +const flags = {}; +const positional = []; +for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a.startsWith("--")) { + const key = a.slice(2); + if (key === "verbose") flags.verbose = true; + else flags[key] = argv[++i]; + } else positional.push(a); +} + +function die(msg, hint) { + console.error(` ✗ ${msg}`); + if (hint) console.error(` ${hint}`); + process.exit(1); +} + +// Key: the env var first, then the two places the rest of the toolchain keeps +// it (deploy.sh reads the plaintext; ac-os decrypts the .gpg and caches it). +// Never printed. +function adminKey() { + const fromEnv = (process.env.OS_BUILD_ADMIN_KEY || "").trim(); + if (fromEnv) return fromEnv; + + const cache = "/tmp/.ac-oven-admin-key"; + if (existsSync(cache)) { + const key = readFileSync(cache, "utf8").trim(); + if (key) return key; + } + + const plain = join(REPO, "aesthetic-computer-vault", "oven", "os-build-admin-key.txt"); + if (existsSync(plain)) { + const key = readFileSync(plain, "utf8").trim(); + if (key) return key; + } + + const gpg = `${plain}.gpg`; + if (existsSync(gpg)) { + try { + return execFileSync("gpg", ["--pinentry-mode", "loopback", "-d", gpg], { + stdio: ["inherit", "pipe", "ignore"], + encoding: "utf8", + }).trim(); + } catch {} + } + + die( + "no oven admin key", + "set OS_BUILD_ADMIN_KEY, or unlock the vault (aesthetic-computer-vault/oven/os-build-admin-key.txt)", + ); +} + +function bundle(dir) { + const args = ["-czf", "-", "-C", dir, ...DROSS.map((d) => `--exclude=${d}`), "."]; + return execFileSync("tar", args, { maxBuffer: 64 * 1024 * 1024 }); +} + +// xelatex says a great deal. Keep the stage markers, the errors, and the +// warnings that matter; --verbose for the whole flood. +function worthShowing(line) { + if (flags.verbose) return true; + if (/^\s{2}(CRUNCH|UNPACK|FONTS|PASS|BIBTEX|OK|FAILED|WARN)/.test(line)) return true; + if (/^!/.test(line) || /^l\.\d+/.test(line)) return true; + if (/^.*\.(tex|sty):\d+:/.test(line)) return true; + if (/LaTeX Warning: (Citation|Reference|There were undefined)/.test(line)) return true; + return false; +} + +async function ask(pathname, init = {}) { + const url = `${OVEN}${pathname}`; + try { + return await fetch(url, { + ...init, + headers: { Authorization: `Bearer ${KEY}`, ...(init.headers || {}) }, + }); + } catch (err) { + die(`oven unreachable at ${OVEN}`, err.message); + } +} + +// The tail of the log around the first LaTeX complaint — the two lines a +// human actually needs, not the 2000-line dump. +function complaint(lines) { + const at = lines.findIndex((l) => /^!/.test(l) || /\.(tex|sty):\d+:/.test(l)); + if (at < 0) return lines.slice(-12); + return lines.slice(at, at + 8); +} + +const src = positional[0]; +if (!src) { + console.error("usage: crunch.mjs [--tex ] [--out ] [--verbose]"); + process.exit(1); +} +const dir = resolve(src); +if (!existsSync(dir) || !statSync(dir).isDirectory()) die(`not a directory: ${src}`); + +const KEY = adminKey(); +const name = basename(dir); +const out = flags.out ? resolve(flags.out) : join(dir, `${flags.tex ? basename(flags.tex, ".tex") : name}.pdf`); + +const tarball = bundle(dir); +console.log(`▸ crunching ${name} on ${OVEN} (${(tarball.length / 1024).toFixed(0)}kb)`); + +const started = await ask("/paper-crunch", { + method: "POST", + headers: { + "Content-Type": "application/gzip", + "x-crunch-name": name, + ...(flags.tex ? { "x-crunch-tex": basename(flags.tex) } : {}), + }, + body: tarball, +}); + +if (started.status === 401) die("oven rejected the admin key", "is OS_BUILD_ADMIN_KEY the current one?"); +if (started.status === 409) { + const { activeJobId } = await started.json().catch(() => ({})); + die(`oven is already crunching (job ${activeJobId})`, "wait, or cancel it and retry"); +} +if (!started.ok) die(`oven refused the bundle (HTTP ${started.status})`, (await started.text()).slice(0, 300)); + +const { id } = await started.json(); + +const stream = await ask(`/paper-crunch/${id}/stream`); +if (!stream.ok) die(`cannot stream job ${id} (HTTP ${stream.status})`); + +const seen = []; +let outcome = null; +let pending = ""; + +for await (const chunk of stream.body) { + pending += Buffer.from(chunk).toString(); + let cut; + while ((cut = pending.indexOf("\n\n")) >= 0) { + const frame = pending.slice(0, cut); + pending = pending.slice(cut + 2); + + const type = frame.match(/^event: (.+)$/m)?.[1]; + const data = JSON.parse(frame.match(/^data: (.+)$/m)?.[1] || "{}"); + + if (type === "logs") { + for (const { line } of data.logs || []) { + seen.push(line); + if (worthShowing(line)) console.log(` ${line.trim()}`); + } + } else if (type === "complete") { + outcome = data; + } + } +} + +if (!outcome) die(`the oven hung up on job ${id}`, `check ${OVEN}/paper-crunch/${id}`); + +if (outcome.status !== "success") { + console.error(""); + for (const line of complaint(seen)) console.error(` ${line}`); + console.error(""); + die(outcome.error || `crunch ${outcome.status}`, flags.verbose ? null : "re-run with --verbose for the full log"); +} + +const pdf = await ask(`/paper-crunch/${id}/pdf`); +if (!pdf.ok) die(`no PDF came back (HTTP ${pdf.status})`); +writeFileSync(out, Buffer.from(await pdf.arrayBuffer())); + +const pages = outcome.pages ? `, ${outcome.pages}pp` : ""; +console.log(` ✓ ${out} (${(statSync(out).size / 1024).toFixed(0)}kb${pages})`); -- 2.51.2