diff --git a/oven/SETUP-LITH-RSYNC.md b/oven/SETUP-LITH-RSYNC.md new file mode 100644 index 000000000..33c2bbecf --- /dev/null +++ b/oven/SETUP-LITH-RSYNC.md @@ -0,0 +1,108 @@ +# SETUP: oven → lith rsync for papers + +One-time provisioning so `oven/papers-builder.mjs` can rsync built PDFs to +lith instead of committing them to git. + +## Why + +Phase 1 already stopped tracking xelatex outputs in source dirs +(`papers/arxiv-*/*.pdf`). Phase 2 stops the oven from committing the +**deployed** PDFs at `system/public/papers.aesthetic.computer/*.pdf` — +174 files that churn binary blobs into git on every build. They go to +lith via rsync over SSH instead. + +## Architecture + +``` +oven box lith box + ┌─────────────────────────────┐ ┌─────────────────────────────────┐ + │ papers/cli.mjs publish │ │ /opt/ac/system/public/ │ + │ → builds PDFs into │ rsync │ papers.aesthetic.computer/ │ + │ system/public/papers... │ ──ssh────▶ │ *.pdf, platter.html, ... │ + │ │ │ │ + │ git commit metadata.json │ push │ webhook on push → cache purge │ + │ + BUILDLOG.md only │ ──git────▶ │ Caddy serves the dir │ + └─────────────────────────────┘ └─────────────────────────────────┘ +``` + +Cache purge still works because the small metadata commit triggers lith's +existing GitHub webhook → `lith/webhook.sh` → per-file CDN purge. + +## One-time SSH key setup + +The vault's `home/.ssh/id_rsa` is your laptop's identity. For oven we want +a **dedicated** keypair — generated locally, public half authorized on lith, +private half copied to oven. Rotating the laptop key shouldn't break oven. + +From your laptop: + +```fish +# 1. Generate a dedicated ed25519 keypair (no passphrase — oven runs as root) +set keypath /tmp/oven-to-lith +ssh-keygen -t ed25519 -f $keypath -N "" -C "oven@aesthetic.computer → lith" + +# 2. Authorize the public key on lith (write-access to /opt/ac/system/public/papers.aesthetic.computer/) +ssh -i ~/.config/sops/age/ac-lith/id_rsa root@lith.aesthetic.computer "cat >> /root/.ssh/authorized_keys" < $keypath.pub + +# 3. Copy the private key to oven (find OVEN_HOST from oven/deploy.fish) +set oven_host (set -q OVEN_HOST; and echo $OVEN_HOST; or echo "") +scp $keypath root@$oven_host:/root/.ssh/oven-to-lith +ssh root@$oven_host "chmod 600 /root/.ssh/oven-to-lith" + +# 4. Pre-seed oven's known_hosts so rsync doesn't prompt on first run +ssh root@$oven_host "ssh-keyscan -t ed25519 lith.aesthetic.computer > /root/.ssh/oven-known-hosts" + +# 5. Wipe the local copy +rm $keypath $keypath.pub + +# 6. Smoke test from oven +ssh root@$oven_host "rsync -av --dry-run -e 'ssh -i /root/.ssh/oven-to-lith -o UserKnownHostsFile=/root/.ssh/oven-known-hosts' /tmp/ root@lith.aesthetic.computer:/tmp/oven-smoke/" +``` + +If step 6 prints rsync output without errors, the key works. + +## Deploy + +After verifying the key, sync the updated oven source and restart: + +```fish +oven/sync-source.sh # rsyncs oven/*.mjs to /opt/oven/ac-source/ +ssh root@$oven_host "systemctl restart oven" +``` + +Then trigger a papers build: + +```fish +curl -X POST https://oven.aesthetic.computer/papers-build +``` + +Watch the logs at `oven.aesthetic.computer/papers-build/?logs=true` — +you should see `RSYNC: pushing PDFs + platter to root@lith.aesthetic.computer` +followed by per-file transfer lines, then the small metadata commit + push. + +## After this is verified working + +Then it's safe to do the local cleanup commit: + +```fish +echo "system/public/papers.aesthetic.computer/*.pdf" >> .gitignore +echo "system/public/papers.aesthetic.computer/platter.html" >> .gitignore # also a build artifact +echo "system/public/papers.aesthetic.computer/index.html" >> .gitignore # also a build artifact +git ls-files system/public/papers.aesthetic.computer/ \ + | grep -E '\.(pdf|html)$' \ + | xargs git rm --cached +git commit -m "gitignore: stop tracking papers deploy artifacts (oven rsyncs them)" +git push +``` + +lith's webhook fires on this push, lith `git pull` removes the PDFs from +disk, and the next oven cycle (~60s later) restores them via rsync. +Expect a brief flicker on `papers.aesthetic.computer` URLs in the +interval — if that's unacceptable, trigger a manual oven rebuild +immediately after the push. + +## Rollback + +If rsync starts misbehaving, revert `oven/papers-builder.mjs` to the +pre-Phase-2 version (`commitAndPushPDFs`), sync-source + restart oven. +PDFs will start re-entering git again, but everything keeps working. diff --git a/oven/papers-builder.mjs b/oven/papers-builder.mjs index 367b01b4a..3811ed350 100644 --- a/oven/papers-builder.mjs +++ b/oven/papers-builder.mjs @@ -115,70 +115,91 @@ function git(args, cwd = GIT_REPO_DIR) { }); } -// After a successful publish, commit the built PDFs + index + metadata and push -// to origin so Netlify can deploy them via the papers.aesthetic.computer subdomain. -async function commitAndPushPDFs(job) { +// rsync the built PDFs + platter HTML directly to lith (papers.aesthetic.computer +// origin), bypassing git for binary artifacts. Then commit the tiny text-only +// source-of-truth files (metadata.json + BUILDLOG.md) so the lith cache-purge +// webhook still fires. +// +// See oven/SETUP-LITH-RSYNC.md for the one-time SSH key provisioning. +async function publishToLith(job) { const SITE_DIR = path.join(GIT_REPO_DIR, "system", "public", "papers.aesthetic.computer"); - const METADATA = path.join(GIT_REPO_DIR, "papers", "metadata.json"); + const LITH_HOST = process.env.LITH_PAPERS_HOST || "root@lith.aesthetic.computer"; + const LITH_DEST = process.env.LITH_PAPERS_DEST || "/opt/ac/system/public/papers.aesthetic.computer/"; + const SSH_KEY = process.env.LITH_SSH_KEY || "/root/.ssh/oven-to-lith"; + + addLogLine(job, "stdout", ` RSYNC: pushing PDFs + platter to ${LITH_HOST}...`); + job.stage = "rsync"; + job.percent = 94; + + try { + await new Promise((resolve, reject) => { + execFile( + "rsync", + [ + "-av", + "--delete-after", + "--include=*/", + "--include=*.pdf", + "--include=*.html", + "--exclude=*", + "-e", `ssh -i ${SSH_KEY} -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/oven-known-hosts`, + SITE_DIR + "/", + `${LITH_HOST}:${LITH_DEST}`, + ], + { timeout: 300_000 }, + (err, stdout, stderr) => { + if (err) { + err.stderr = stderr; + return reject(err); + } + for (const line of stdout.split("\n").slice(-8)) { + if (line.trim()) addLogLine(job, "stdout", " RSYNC: " + line); + } + resolve(); + }, + ); + }); + } catch (rsyncErr) { + addLogLine(job, "stderr", ` RSYNC FAILED: ${rsyncErr.message}${rsyncErr.stderr ? " | " + rsyncErr.stderr.trim() : ""}`); + throw rsyncErr; + } - addLogLine(job, "stdout", " GIT: staging built PDFs..."); + // Still commit metadata.json + BUILDLOG.md so the lith webhook fires and + // purges its caches. These are text files — no binary churn. job.stage = "git-push"; - job.percent = 96; + job.percent = 97; - // Configure git identity for the oven bot await git(["config", "user.email", "oven@aesthetic.computer"]); await git(["config", "user.name", "Oven (aesthetic.computer)"]); - // Only stage meaningful build outputs — not LaTeX intermediates (.log, .aux, - // .out, .fls, etc.) whose timestamps change every build even when PDFs don't. - await git(["add", "system/public/papers.aesthetic.computer/"]); await git(["add", "papers/metadata.json", "papers/BUILDLOG.md"]).catch(() => {}); - // Check if there are actually staged changes const status = await git(["diff", "--cached", "--name-only"]); if (!status) { - addLogLine(job, "stdout", " GIT: no changes to commit — PDFs unchanged"); + addLogLine(job, "stdout", " GIT: no metadata changes — rsync only"); return; } - const changedFiles = status.split("\n"); - const pdfCount = changedFiles.filter((f) => f.endsWith(".pdf")).length; - const msg = `[papers] oven auto-build: ${pdfCount} PDF${pdfCount !== 1 ? "s" : ""} updated`; - - addLogLine(job, "stdout", ` GIT: committing ${changedFiles.length} file(s)...`); + const msg = `[papers] oven auto-build: metadata + buildlog`; await git(["commit", "-m", msg]); - // Discard any remaining unstaged changes (xelatex leaves tracked - // .log/.toc/.aux/.fls files modified after each run, intermediate .pdfs - // in papers/arxiv-*/ may be dirty, and the papers-git-poller rewrites - // .last-papers-built-hash at the repo root). We only commit - // system/public/... + metadata.json + BUILDLOG.md, so anything still dirty - // is byproduct we don't want to preserve — and leaving it dirty blocks - // `git pull --rebase`. + // Discard byproduct (xelatex .log/.toc updates, dirty intermediate PDFs, + // .last-papers-built-hash) so pull --rebase can run cleanly. try { await git(["checkout", "--", "."]); } catch (cleanupErr) { - addLogLine( - job, - "stderr", - ` GIT: pre-rebase cleanup note: ${cleanupErr.message || cleanupErr}`, - ); + addLogLine(job, "stderr", ` GIT: pre-rebase cleanup note: ${cleanupErr.message || cleanupErr}`); } - // Pull any changes that landed while we were building (rebase our commit on top) - addLogLine(job, "stdout", " GIT: pulling latest before push..."); try { await git(["pull", "--rebase", "origin", "main"]); } catch (pullErr) { - // If rebase fails (conflict), abort and report — our PDFs are binary so this shouldn't happen try { await git(["rebase", "--abort"]); } catch {} throw pullErr; } - addLogLine(job, "stdout", " GIT: pushing to origin/main..."); - job.percent = 98; + job.percent = 99; await git(["push", "origin", "main"]); - addLogLine(job, "stdout", ` GIT: pushed — ${msg}`); } @@ -223,12 +244,12 @@ async function runPapersJob(job) { }); }); - // Commit and push built PDFs back to the repo for Netlify deployment + // rsync built PDFs to lith; commit metadata so cache-purge webhook fires try { - await commitAndPushPDFs(job); + await publishToLith(job); } catch (pushErr) { - // Git push failure is non-fatal — PDFs were still built successfully - addLogLine(job, "stderr", ` GIT PUSH FAILED: ${pushErr.message}${pushErr.stderr ? " | " + pushErr.stderr.trim() : ""}`); + // Non-fatal — PDFs were built successfully even if publish failed + addLogLine(job, "stderr", ` PUBLISH FAILED: ${pushErr.message}${pushErr.stderr ? " | " + pushErr.stderr.trim() : ""}`); } job.status = "success";