From 3ca41359f69427eb8d319ceb77ac2284b0f1ab9a Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Thu, 10 Sep 2026 12:24:16 -0400 Subject: [PATCH] assets: give the bucket a register, and ignore the scratch drawer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two hygiene fixes the traffic audit turned up. tmp/ was never ignored, so 130+ probe scripts and QA frame dumps sat in every `git status` on both machines — ~290 MB, 2,359 of them frame JPGs from two oskiewar oven smoke runs. That noise is how the arxiv-ios-so-far paper source went uncommitted long enough for its PDF to ship without it. Ignore the drawer, negate the two subtrees that are genuinely tracked (tmp/imagegen, tmp/laklok-touch-probe.mjs), and ignore .tmp-pr*/ agent worktrees. Untracked count on neo: 205 -> 57. toolchain/assets/sync-index.mjs is the assets counterpart to papers/sync-platter.mjs. There was no asset register at all: the bytes live in s3://assets-aesthetic-computer, pop/ASSETS.md covers policy for pop media only, and answering "what have we stored and where" required credentials, a network, and knowing the bucket's name. Now git holds the register and Spaces holds the bytes. Every prefix is listed whether or not anyone has described it, because the point is that nothing is invisible; an empty note is a prompt, not a resting state. The generator refuses to write when its parse disagrees with the bucket's own Total Objects / Total Size, since a confidently wrong register is worse than none — the same failure mode as the sfo3 sync that exits 0 while stranding files. --- .gitignore | 14 +++ toolchain/assets/sync-index.mjs | 205 ++++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 toolchain/assets/sync-index.mjs diff --git a/.gitignore b/.gitignore index 1525875c41..3985af0c71 100644 --- a/.gitignore +++ b/.gitignore @@ -564,3 +564,17 @@ crm/.cache/ # history; only ciphertext from that repository is ever pushed. /vault/ tmp/nopaint-sheets/ + +# tmp/ is the repo's scratch drawer: probe scripts, QA frame dumps, review +# captures. It accumulated 130+ untracked files and ~290 MB across two +# machines (2026-09-10), which made a real `git status` unreadable and hid +# actual work. Ignore the drawer, keep the two subtrees that are genuinely +# tracked, and keep tracking anything already committed (gitignore never +# untracks). Scratch worth keeping goes to the assets bucket, not here. +/tmp/* +!/tmp/imagegen/ +!/tmp/laklok-touch-probe.mjs +!/tmp/nopaint-sheets/ + +# agent/PR worktree scratch (client work checked out beside the repo) +/.tmp-pr*/ diff --git a/toolchain/assets/sync-index.mjs b/toolchain/assets/sync-index.mjs new file mode 100644 index 0000000000..fbadb9b75f --- /dev/null +++ b/toolchain/assets/sync-index.mjs @@ -0,0 +1,205 @@ +#!/usr/bin/env node +// Regenerate the asset register: assets/index.json + assets/INDEX.md. +// +// The bytes live in DigitalOcean Spaces; git holds the register. This is the +// assets counterpart to papers/sync-platter.mjs — before it existed, the only +// way to answer "what have we stored and where" was to list the bucket, which +// needs credentials, a network, and knowing the bucket's name. A traffic audit +// in September 2026 found the assets tree was the one part of the network with +// no register at all, so this file is the register. +// +// node toolchain/assets/sync-index.mjs # refresh from the bucket +// node toolchain/assets/sync-index.mjs --dry-run # print, write nothing +// +// Reads credentials the same way `npm run assets:sync:*` does (the ambient +// [default] DO Spaces key). Never prints a key, and never lists object bodies. + +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdirSync, writeFileSync, readFileSync, existsSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const execFileAsync = promisify(execFile); +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO = join(HERE, "..", ".."); +const OUT_DIR = join(REPO, "assets"); +const JSON_PATH = join(OUT_DIR, "index.json"); +const MD_PATH = join(OUT_DIR, "INDEX.md"); + +const BUCKET = process.env.SPACES_BUCKET || "assets-aesthetic-computer"; +const ENDPOINT = process.env.SPACES_ENDPOINT || "https://sfo3.digitaloceanspaces.com"; +const REGION = "sfo3"; +const DRY = process.argv.includes("--dry-run"); + +// Notes keyed by prefix. A prefix with no note still gets listed — the point of +// the register is that nothing is invisible — but an unnoted prefix is a prompt +// to write one rather than a permanent state. +const NOTES = { + "private/": "Not world-readable. Client work, grant material, and workspace archives.", + "private/vault-workspace/": "Personal vault workspace archives (grants, marketing drafts, internal reports).", + "private/client-assets/": "Client deliverables and source media under NDA.", + "private/regarde/": "REGARDE render workspaces (confidential).", + "private/day-workspaces/": "Dated whole-workspace snapshots.", + "private/pop-workspace/": "Pop track working media: raw vocal stems and alignment caches. Billable to regenerate.", + "private/pop-release/": "Mastered pop releases staged for distribution.", + "private/build-workspace/": "Build artifacts too large for git.", + "pop/": "Published pop media (beds, mixes) — see pop/ASSETS.md for the policy.", + "papers/": "Paper figures, readings audio, and cassette material.", + "whistlegraph/": "Whistlegraph archive and harvested media.", +}; + +const human = (n) => { + const u = ["B", "KB", "MB", "GB", "TB"]; + let i = 0, v = Number(n); + while (v >= 1024 && i < u.length - 1) { v /= 1024; i++; } + return `${v.toFixed(v >= 10 || i === 0 ? 0 : 1)} ${u[i]}`; +}; + +async function listBucket() { + const args = [ + "s3", "ls", `s3://${BUCKET}/`, "--recursive", "--summarize", + "--endpoint-url", ENDPOINT, "--region", REGION, + ]; + const { stdout } = await execFileAsync("aws", args, { maxBuffer: 512 * 1024 * 1024 }); + return stdout; +} + +// `aws s3 ls --recursive` lines are: DATE TIME SIZE KEY (key may contain spaces) +function parse(stdout) { + const objects = []; + let totalObjects = null, totalBytes = null; + for (const line of stdout.split("\n")) { + const t = line.trim(); + if (!t) continue; + let m = t.match(/^Total Objects:\s*(\d+)$/); + if (m) { totalObjects = Number(m[1]); continue; } + m = t.match(/^Total Size:\s*(\d+)$/); + if (m) { totalBytes = Number(m[1]); continue; } + m = t.match(/^(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2}:\d{2})\s+(\d+)\s+(.+)$/); + if (!m) continue; + objects.push({ date: m[1], bytes: Number(m[3]), key: m[4] }); + } + return { objects, totalObjects, totalBytes }; +} + +// Group one level deep, except under private/ where the useful unit is the +// second segment (private/regarde/ tells you something; private/ does not). +function groupKey(key) { + const parts = key.split("/"); + if (parts.length === 1) return "(bucket root)"; + if (parts[0] === "private" && parts.length > 2) return `private/${parts[1]}/`; + return `${parts[0]}/`; +} + +function build(objects) { + const groups = new Map(); + for (const o of objects) { + const g = groupKey(o.key); + const cur = groups.get(g) || { prefix: g, files: 0, bytes: 0, newest: "", oldest: "9999-99-99" }; + cur.files += 1; + cur.bytes += o.bytes; + if (o.date > cur.newest) cur.newest = o.date; + if (o.date < cur.oldest) cur.oldest = o.date; + groups.set(g, cur); + } + return [...groups.values()].sort((a, b) => b.bytes - a.bytes); +} + +function renderMd(reg) { + const L = []; + L.push("# Asset register"); + L.push(""); + L.push("Generated by [`toolchain/assets/sync-index.mjs`](../toolchain/assets/sync-index.mjs)."); + L.push("Do not hand-edit — rerun the generator."); + L.push(""); + L.push(`- Bucket: \`s3://${reg.bucket}\` (\`${reg.region}\`, DigitalOcean Spaces)`); + L.push(`- Mirrored locally to the gitignored \`system/public/assets/\` tree`); + L.push(`- Listed: **${reg.generated.slice(0, 10)}**`); + L.push(`- Contents: **${reg.totals.files.toLocaleString()} objects**, **${human(reg.totals.bytes)}**`); + L.push(""); + L.push("The bytes live in Spaces; this file is the register. Anything in the"); + L.push("bucket appears below whether or not anyone has described it — a prefix"); + L.push("with an empty note is a prompt to write one."); + L.push(""); + L.push("| Prefix | Files | Size | First | Last | Note |"); + L.push("|---|---:|---:|---|---|---|"); + for (const g of reg.prefixes) { + L.push(`| \`${g.prefix}\` | ${g.files.toLocaleString()} | ${human(g.bytes)} | ${g.oldest} | ${g.newest} | ${g.note || ""} |`); + } + L.push(""); + L.push("## Restoring"); + L.push(""); + L.push("```bash"); + L.push("# whole public tree (what npm run assets:sync:down does)"); + L.push(`aws s3 sync s3://${reg.bucket} system/public/assets \\`); + L.push(` --endpoint-url ${reg.endpoint}`); + L.push(""); + L.push("# one prefix"); + L.push(`aws s3 sync s3://${reg.bucket}/ \\`); + L.push(` --endpoint-url ${reg.endpoint} --region ${reg.region}`); + L.push("```"); + L.push(""); + L.push("## Pushing"); + L.push(""); + L.push("Spaces on sfo3 will exit 0 while silently stranding files under the"); + L.push("default 10-way concurrency, so large pushes go at 3 and get counted"); + L.push("afterwards. Private material must carry `--acl private`; the repo's"); + L.push("`assets:sync:up` bakes in `--acl public-read` and must never be reused"); + L.push("for it."); + L.push(""); + L.push("```bash"); + L.push("printf '[default]\\ns3 =\\n max_concurrent_requests = 3\\n' > /tmp/awscfg/config"); + L.push("AWS_CONFIG_FILE=/tmp/awscfg/config AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 \\"); + L.push(` aws s3 sync s3://${reg.bucket}/private// \\`); + L.push(` --endpoint-url ${reg.endpoint} --region ${reg.region} \\`); + L.push(" --acl private --exclude '*.DS_Store'"); + L.push(""); + L.push("node toolchain/assets/sync-index.mjs # then re-register"); + L.push("```"); + L.push(""); + return L.join("\n"); +} + +const stdout = await listBucket(); +const { objects, totalObjects, totalBytes } = parse(stdout); +if (!objects.length) { + console.error("assets: bucket listing returned no objects — refusing to write an empty register."); + process.exit(1); +} +const prefixes = build(objects).map((g) => ({ ...g, note: NOTES[g.prefix] || "" })); +const sumBytes = prefixes.reduce((a, g) => a + g.bytes, 0); +const sumFiles = prefixes.reduce((a, g) => a + g.files, 0); + +// The bucket's own summary is the check on our parse; a mismatch means the +// listing was truncated or the format moved, and a wrong register is worse +// than no register. +if (totalObjects != null && totalObjects !== sumFiles) { + console.error(`assets: parsed ${sumFiles} objects but bucket reports ${totalObjects} — not writing.`); + process.exit(1); +} +if (totalBytes != null && totalBytes !== sumBytes) { + console.error(`assets: parsed ${sumBytes} bytes but bucket reports ${totalBytes} — not writing.`); + process.exit(1); +} + +const reg = { + generated: new Date().toISOString(), + bucket: BUCKET, + endpoint: ENDPOINT, + region: REGION, + totals: { files: sumFiles, bytes: sumBytes }, + prefixes, +}; + +console.log(`assets: ${sumFiles.toLocaleString()} objects, ${human(sumBytes)}, ${prefixes.length} prefixes`); +for (const g of prefixes.slice(0, 12)) { + console.log(` ${human(g.bytes).padStart(8)} ${String(g.files).padStart(6)} ${g.prefix}`); +} +if (DRY) { console.log("\n(dry run — nothing written)"); process.exit(0); } + +mkdirSync(OUT_DIR, { recursive: true }); +writeFileSync(JSON_PATH, JSON.stringify(reg, null, 2) + "\n"); +writeFileSync(MD_PATH, renderMd(reg)); +console.log(`\nwrote ${JSON_PATH.replace(REPO + "/", "")}`); +console.log(`wrote ${MD_PATH.replace(REPO + "/", "")}`); -- 2.51.2