diff --git a/.gitignore b/.gitignore index 1525875c41..3985af0c71 100644 --- a/.gitignore +++ b/.gitignore @@ -564,3 +564,17 @@ crm/.cache/ # history; only ciphertext from that repository is ever pushed. /vault/ tmp/nopaint-sheets/ + +# tmp/ is the repo's scratch drawer: probe scripts, QA frame dumps, review +# captures. It accumulated 130+ untracked files and ~290 MB across two +# machines (2026-09-10), which made a real `git status` unreadable and hid +# actual work. Ignore the drawer, keep the two subtrees that are genuinely +# tracked, and keep tracking anything already committed (gitignore never +# untracks). Scratch worth keeping goes to the assets bucket, not here. +/tmp/* +!/tmp/imagegen/ +!/tmp/laklok-touch-probe.mjs +!/tmp/nopaint-sheets/ + +# agent/PR worktree scratch (client work checked out beside the repo) +/.tmp-pr*/ diff --git a/toolchain/assets/sync-index.mjs b/toolchain/assets/sync-index.mjs new file mode 100644 index 0000000000..fbadb9b75f --- /dev/null +++ b/toolchain/assets/sync-index.mjs @@ -0,0 +1,205 @@ +#!/usr/bin/env node +// Regenerate the asset register: assets/index.json + assets/INDEX.md. +// +// The bytes live in DigitalOcean Spaces; git holds the register. This is the +// assets counterpart to papers/sync-platter.mjs — before it existed, the only +// way to answer "what have we stored and where" was to list the bucket, which +// needs credentials, a network, and knowing the bucket's name. A traffic audit +// in September 2026 found the assets tree was the one part of the network with +// no register at all, so this file is the register. +// +// node toolchain/assets/sync-index.mjs # refresh from the bucket +// node toolchain/assets/sync-index.mjs --dry-run # print, write nothing +// +// Reads credentials the same way `npm run assets:sync:*` does (the ambient +// [default] DO Spaces key). Never prints a key, and never lists object bodies. + +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdirSync, writeFileSync, readFileSync, existsSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const execFileAsync = promisify(execFile); +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO = join(HERE, "..", ".."); +const OUT_DIR = join(REPO, "assets"); +const JSON_PATH = join(OUT_DIR, "index.json"); +const MD_PATH = join(OUT_DIR, "INDEX.md"); + +const BUCKET = process.env.SPACES_BUCKET || "assets-aesthetic-computer"; +const ENDPOINT = process.env.SPACES_ENDPOINT || "https://sfo3.digitaloceanspaces.com"; +const REGION = "sfo3"; +const DRY = process.argv.includes("--dry-run"); + +// Notes keyed by prefix. A prefix with no note still gets listed — the point of +// the register is that nothing is invisible — but an unnoted prefix is a prompt +// to write one rather than a permanent state. +const NOTES = { + "private/": "Not world-readable. Client work, grant material, and workspace archives.", + "private/vault-workspace/": "Personal vault workspace archives (grants, marketing drafts, internal reports).", + "private/client-assets/": "Client deliverables and source media under NDA.", + "private/regarde/": "REGARDE render workspaces (confidential).", + "private/day-workspaces/": "Dated whole-workspace snapshots.", + "private/pop-workspace/": "Pop track working media: raw vocal stems and alignment caches. Billable to regenerate.", + "private/pop-release/": "Mastered pop releases staged for distribution.", + "private/build-workspace/": "Build artifacts too large for git.", + "pop/": "Published pop media (beds, mixes) — see pop/ASSETS.md for the policy.", + "papers/": "Paper figures, readings audio, and cassette material.", + "whistlegraph/": "Whistlegraph archive and harvested media.", +}; + +const human = (n) => { + const u = ["B", "KB", "MB", "GB", "TB"]; + let i = 0, v = Number(n); + while (v >= 1024 && i < u.length - 1) { v /= 1024; i++; } + return `${v.toFixed(v >= 10 || i === 0 ? 0 : 1)} ${u[i]}`; +}; + +async function listBucket() { + const args = [ + "s3", "ls", `s3://${BUCKET}/`, "--recursive", "--summarize", + "--endpoint-url", ENDPOINT, "--region", REGION, + ]; + const { stdout } = await execFileAsync("aws", args, { maxBuffer: 512 * 1024 * 1024 }); + return stdout; +} + +// `aws s3 ls --recursive` lines are: DATE TIME SIZE KEY (key may contain spaces) +function parse(stdout) { + const objects = []; + let totalObjects = null, totalBytes = null; + for (const line of stdout.split("\n")) { + const t = line.trim(); + if (!t) continue; + let m = t.match(/^Total Objects:\s*(\d+)$/); + if (m) { totalObjects = Number(m[1]); continue; } + m = t.match(/^Total Size:\s*(\d+)$/); + if (m) { totalBytes = Number(m[1]); continue; } + m = t.match(/^(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2}:\d{2})\s+(\d+)\s+(.+)$/); + if (!m) continue; + objects.push({ date: m[1], bytes: Number(m[3]), key: m[4] }); + } + return { objects, totalObjects, totalBytes }; +} + +// Group one level deep, except under private/ where the useful unit is the +// second segment (private/regarde/ tells you something; private/ does not). +function groupKey(key) { + const parts = key.split("/"); + if (parts.length === 1) return "(bucket root)"; + if (parts[0] === "private" && parts.length > 2) return `private/${parts[1]}/`; + return `${parts[0]}/`; +} + +function build(objects) { + const groups = new Map(); + for (const o of objects) { + const g = groupKey(o.key); + const cur = groups.get(g) || { prefix: g, files: 0, bytes: 0, newest: "", oldest: "9999-99-99" }; + cur.files += 1; + cur.bytes += o.bytes; + if (o.date > cur.newest) cur.newest = o.date; + if (o.date < cur.oldest) cur.oldest = o.date; + groups.set(g, cur); + } + return [...groups.values()].sort((a, b) => b.bytes - a.bytes); +} + +function renderMd(reg) { + const L = []; + L.push("# Asset register"); + L.push(""); + L.push("Generated by [`toolchain/assets/sync-index.mjs`](../toolchain/assets/sync-index.mjs)."); + L.push("Do not hand-edit — rerun the generator."); + L.push(""); + L.push(`- Bucket: \`s3://${reg.bucket}\` (\`${reg.region}\`, DigitalOcean Spaces)`); + L.push(`- Mirrored locally to the gitignored \`system/public/assets/\` tree`); + L.push(`- Listed: **${reg.generated.slice(0, 10)}**`); + L.push(`- Contents: **${reg.totals.files.toLocaleString()} objects**, **${human(reg.totals.bytes)}**`); + L.push(""); + L.push("The bytes live in Spaces; this file is the register. Anything in the"); + L.push("bucket appears below whether or not anyone has described it — a prefix"); + L.push("with an empty note is a prompt to write one."); + L.push(""); + L.push("| Prefix | Files | Size | First | Last | Note |"); + L.push("|---|---:|---:|---|---|---|"); + for (const g of reg.prefixes) { + L.push(`| \`${g.prefix}\` | ${g.files.toLocaleString()} | ${human(g.bytes)} | ${g.oldest} | ${g.newest} | ${g.note || ""} |`); + } + L.push(""); + L.push("## Restoring"); + L.push(""); + L.push("```bash"); + L.push("# whole public tree (what npm run assets:sync:down does)"); + L.push(`aws s3 sync s3://${reg.bucket} system/public/assets \\`); + L.push(` --endpoint-url ${reg.endpoint}`); + L.push(""); + L.push("# one prefix"); + L.push(`aws s3 sync s3://${reg.bucket}/ \\`); + L.push(` --endpoint-url ${reg.endpoint} --region ${reg.region}`); + L.push("```"); + L.push(""); + L.push("## Pushing"); + L.push(""); + L.push("Spaces on sfo3 will exit 0 while silently stranding files under the"); + L.push("default 10-way concurrency, so large pushes go at 3 and get counted"); + L.push("afterwards. Private material must carry `--acl private`; the repo's"); + L.push("`assets:sync:up` bakes in `--acl public-read` and must never be reused"); + L.push("for it."); + L.push(""); + L.push("```bash"); + L.push("printf '[default]\\ns3 =\\n max_concurrent_requests = 3\\n' > /tmp/awscfg/config"); + L.push("AWS_CONFIG_FILE=/tmp/awscfg/config AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 \\"); + L.push(` aws s3 sync s3://${reg.bucket}/private// \\`); + L.push(` --endpoint-url ${reg.endpoint} --region ${reg.region} \\`); + L.push(" --acl private --exclude '*.DS_Store'"); + L.push(""); + L.push("node toolchain/assets/sync-index.mjs # then re-register"); + L.push("```"); + L.push(""); + return L.join("\n"); +} + +const stdout = await listBucket(); +const { objects, totalObjects, totalBytes } = parse(stdout); +if (!objects.length) { + console.error("assets: bucket listing returned no objects — refusing to write an empty register."); + process.exit(1); +} +const prefixes = build(objects).map((g) => ({ ...g, note: NOTES[g.prefix] || "" })); +const sumBytes = prefixes.reduce((a, g) => a + g.bytes, 0); +const sumFiles = prefixes.reduce((a, g) => a + g.files, 0); + +// The bucket's own summary is the check on our parse; a mismatch means the +// listing was truncated or the format moved, and a wrong register is worse +// than no register. +if (totalObjects != null && totalObjects !== sumFiles) { + console.error(`assets: parsed ${sumFiles} objects but bucket reports ${totalObjects} — not writing.`); + process.exit(1); +} +if (totalBytes != null && totalBytes !== sumBytes) { + console.error(`assets: parsed ${sumBytes} bytes but bucket reports ${totalBytes} — not writing.`); + process.exit(1); +} + +const reg = { + generated: new Date().toISOString(), + bucket: BUCKET, + endpoint: ENDPOINT, + region: REGION, + totals: { files: sumFiles, bytes: sumBytes }, + prefixes, +}; + +console.log(`assets: ${sumFiles.toLocaleString()} objects, ${human(sumBytes)}, ${prefixes.length} prefixes`); +for (const g of prefixes.slice(0, 12)) { + console.log(` ${human(g.bytes).padStart(8)} ${String(g.files).padStart(6)} ${g.prefix}`); +} +if (DRY) { console.log("\n(dry run — nothing written)"); process.exit(0); } + +mkdirSync(OUT_DIR, { recursive: true }); +writeFileSync(JSON_PATH, JSON.stringify(reg, null, 2) + "\n"); +writeFileSync(MD_PATH, renderMd(reg)); +console.log(`\nwrote ${JSON_PATH.replace(REPO + "/", "")}`); +console.log(`wrote ${MD_PATH.replace(REPO + "/", "")}`);