diff --git a/pop/big-pictures/cli.mjs b/pop/big-pictures/cli.mjs index b62487e4a..f5e468c91 100644 --- a/pop/big-pictures/cli.mjs +++ b/pop/big-pictures/cli.mjs @@ -169,6 +169,7 @@ const STRETCHED = `${OUT}/${SLUG}-pitched-stretched.mp3`; const PAD = `${OUT}/${SLUG}-pad.mp3`; const BED = `${OUT}/${SLUG}-bed.mp3`; const MIX = `${OUT}/${SLUG}-mix.mp3`; +const MIXSFX = `${OUT}/${SLUG}-mix-sfx.mp3`; const STAMPED = `${OUT}/${SLUG}-stamped.mp3`; const FINAL = `${OUT}/${SLUG}-final.mp3`; const STAMP_VOCAL = `${OUT}/ac-stamp-vocal.mp3`; @@ -195,6 +196,7 @@ if (flags.status) { tally("pad/bells", PAD); tally("bed (waltz)", BED); tally("mix", MIX); + tally("mix+sfx", MIXSFX); tally("stamped", STAMPED); tally("final", FINAL); tally("Desktop copy", DESK); @@ -289,6 +291,33 @@ function computeDuration() { return Math.ceil((beats * 60) / BPM); } +// Parse a `.sfx.txt` cue sidecar. Each non-comment line: +// at=SEC gain=G dur=S [loop] : sound description +// The head (before the colon) is whitespace-separated key=val tokens; +// everything after the colon is the prompt. +function parseSfxCues(path) { + const cues = []; + for (const raw of readFileSync(path, "utf8").split("\n")) { + const line = raw.trim(); + if (!line || line.startsWith("#")) continue; + const ci = line.indexOf(":"); + if (ci === -1) continue; + const prompt = line.slice(ci + 1).trim(); + if (!prompt) continue; + const cue = { at: 0, gain: 0.7, dur: null, loop: false, prompt }; + for (const tok of line.slice(0, ci).trim().split(/\s+/)) { + if (!tok) continue; + const [k, v] = tok.split("="); + if (k === "at") cue.at = Number(v) || 0; + else if (k === "gain") cue.gain = Number(v); + else if (k === "dur") cue.dur = Math.max(0.5, Math.min(30, Number(v))); + else if (k === "loop") cue.loop = true; + } + cues.push(cue); + } + return cues; +} + // ── pipeline ───────────────────────────────────────────────────────── const t0 = Date.now(); console.log(`━━━ ${SLUG} ━━━ bpm ${BPM} · transpose ${TRANSPOSE >= 0 ? "+" : ""}${TRANSPOSE}st · ${VOICE} · ${STRETCH ? "stretched" : "natural"}${STAMP ? " · stamp" : ""}\n`); @@ -428,7 +457,52 @@ step("7 · amix (vocal forward · vox 2.5 · pad 0.55 · bed 0.20)", () => { "-c:a", "libmp3lame", "-q:a", "2", MIX]); }); -let TO_FINAL = MIX; +// ── 7.5 · sfx overlay (optional) ───────────────────────────────────── +// If a `.sfx.txt` sidecar exists, render each cue through +// /api/sfx (ElevenLabs sound effects, content-hash cached by sfx.mjs) +// and lay it onto the mix at its placement time. Sidecar line format: +// at=SEC gain=G dur=S [loop] : a short sound description +// `at` defaults 0, `gain` defaults 0.7, `dur` omit = auto-length. +let MIXED = MIX; +const SFX_TXT = `${HERE}/${SLUG}.sfx.txt`; +if (existsSync(SFX_TXT)) { + step("7.5 · sfx (ElevenLabs sound effects)", () => { + const cues = parseSfxCues(SFX_TXT); + if (!cues.length) { console.log(" ↪ no cues in sidecar"); return; } + const sfxDir = `${OUT}/sfx/${SLUG}`; + mkdirSync(sfxDir, { recursive: true }); + const paths = []; + cues.forEach((c, i) => { + const p = `${sfxDir}/${String(i).padStart(2, "0")}.mp3`; + const args = ["bin/sfx.mjs", "--text", c.prompt, "--out", p]; + if (c.dur != null) args.push("--duration", String(c.dur)); + if (c.loop) args.push("--loop"); + if (FORCE) args.push("--force"); + run("node", args); + paths.push(p); + }); + // [0]=mix, [1..n]=cues, each delayed + gained, all amixed (no normalize). + const inputs = ["-i", MIX]; + for (const p of paths) inputs.push("-i", p); + const filters = []; + const labels = ["[0:a]"]; + cues.forEach((c, i) => { + const d = Math.max(0, Math.round(c.at * 1000)); + filters.push(`[${i + 1}:a]adelay=${d}|${d},volume=${c.gain}[s${i}]`); + labels.push(`[s${i}]`); + }); + const amix = `${labels.join("")}amix=inputs=${cues.length + 1}:duration=first:dropout_transition=0:normalize=0`; + run("ffmpeg", ["-y", "-loglevel", "error", ...inputs, + "-filter_complex", [...filters, amix].join(";"), + "-c:a", "libmp3lame", "-q:a", "2", MIXSFX]); + MIXED = MIXSFX; + console.log(` ✓ ${cues.length} sfx cue(s) layered`); + }); +} else { + console.log("▸ 7.5 · sfx (no .sfx.txt — skipped)\n"); +} + +let TO_FINAL = MIXED; if (STAMP) { step("8 · stamp (ac signoff · chipmunk + 4-bit crush)", () => { if (!existsSync(STAMP_VOCAL)) { @@ -445,7 +519,7 @@ if (STAMP) { } const filter = "[0:a]apad=pad_dur=0.5[song];[1:a]volume=1.0[stamp];[song][stamp]concat=n=2:v=0:a=1"; run("ffmpeg", ["-y", "-loglevel", "error", - "-i", MIX, "-i", STAMP_FX, + "-i", MIXED, "-i", STAMP_FX, "-filter_complex", filter, "-c:a", "libmp3lame", "-q:a", "2", STAMPED]); }); diff --git a/pop/big-pictures/phosphene.sfx b/pop/big-pictures/phosphene.sfx new file mode 100644 index 000000000..0bf8fbafa --- /dev/null +++ b/pop/big-pictures/phosphene.sfx @@ -0,0 +1,34 @@ +# phosphene — every sound generated by the ElevenLabs SFX endpoint. +# A psychedelic drift: warm drones, liquid blips, reverse swells, +# bells with long tails, all glued with phaser+chorus+echo. +# +# build: node pop/bin/sfx-compose.mjs phosphene +# (point SFX_ENDPOINT at sfx-local for offline rendering) + +@title phosphene +@glue swirl +@duration 84 + +# ── foundation: two long evolving beds + a slow pulse ──────────────── +at=0 gain=0.50 dur=28 loop fadein=6 fadeout=10 : deep warm analog synthesizer drone, slowly evolving and breathing, meditative, lush +at=0 gain=0.42 dur=26 loop fadein=4 fadeout=8 : slow pulsing low sub heartbeat, soft and round, calm tempo +at=2 gain=0.30 dur=24 loop fadein=6 fadeout=8 pan=-0.3 : distant warm tape hiss and gentle wind, ambient room tone + +# ── first bloom: bells, reverse swell, liquid ──────────────────────── +at=8 gain=0.40 dur=12 fadein=2 fadeout=5 pan=0.45 : shimmering glass bell chimes with long reverb tails, dreamy +at=14 gain=0.50 dur=8 fadein=1 fadeout=3 : reverse cymbal swell rising into a soft cushioned boom +at=20 gain=0.34 dur=14 fadein=2 fadeout=4 pan=0.5 : bubbling liquid water droplets and blips, playful, gentle + +# ── mid: vowel pad, sparkle arp, riser ─────────────────────────────── +at=28 gain=0.40 dur=20 loop fadein=5 fadeout=7 : warm choir-like vowel ahh pad, ethereal, soft and wide +at=34 gain=0.38 dur=12 fadein=1 fadeout=4 pan=0.3 : sparkling psychedelic synth arpeggio, glassy and twinkling +at=42 gain=0.48 dur=8 fadein=1 fadeout=2 : deep cinematic whoosh riser building gentle tension + +# ── late: tape warble, gong, morphing birds ────────────────────────── +at=46 gain=0.34 dur=16 fadein=3 fadeout=5 pan=-0.35 : granular wobbling tape warble texture, hazy and dreamy +at=52 gain=0.48 dur=10 fadein=0 fadeout=5 : gentle gong hit blooming into a wide shimmering reverb +at=58 gain=0.30 dur=12 fadein=2 fadeout=4 pan=0.4 : birdsong slowly morphing into soft electronic chirps + +# ── resolution: closing drone + final decay ────────────────────────── +at=62 gain=0.45 dur=24 loop fadein=6 fadeout=12 : warm resolving synthesizer drone, peaceful, gently descending +at=72 gain=0.44 dur=12 fadein=0 fadeout=8 : soft low boom with a long decaying crystalline shimmer diff --git a/pop/bin/sfx-compose.mjs b/pop/bin/sfx-compose.mjs new file mode 100644 index 000000000..a814b6835 --- /dev/null +++ b/pop/bin/sfx-compose.mjs @@ -0,0 +1,187 @@ +#!/usr/bin/env node +// sfx-compose.mjs — build a track whose EVERY sound is generated by the +// ElevenLabs SFX endpoint (/api/sfx). No synths, no samples, no vocals: +// just text-prompted sound effects layered on a timeline and glued with +// a psychedelic ffmpeg FX chain. +// +// Counterpart to big-pictures/cli.mjs (which builds jeffrey-vocal +// tracks). Here the "score" is an arrangement file of sound prompts. +// Each cue is content-hash cached by bin/sfx.mjs, so reruns are free. +// +// Arrangement file format (`.sfx`): +// @title phosphene # ID3 title (default: slug) +// @glue swirl # psychedelic glue: swirl | dub | none +// @duration 88 # hard trim (seconds); omit = auto +// +// # cue lines — head is key=val tokens, prompt follows the colon: +// # at=SEC gain=G dur=S [loop] fadein=S fadeout=S pan=-1..1 : prompt +// at=0 gain=0.5 dur=26 loop fadein=5 fadeout=8 : deep warm analog drone, evolving +// at=12 gain=0.6 dur=8 fadein=2 fadeout=4 pan=-0.5 : reverse cymbal swell rising +// +// Usage: +// node bin/sfx-compose.mjs big-pictures/phosphene.sfx +// node bin/sfx-compose.mjs phosphene # resolves big-pictures/.sfx +// node bin/sfx-compose.mjs phosphene --force # bust sfx + mix caches +// node bin/sfx-compose.mjs phosphene --no-finalize # stop at the raw mix + +import { spawnSync } from "node:child_process"; +import { readFileSync, copyFileSync, existsSync, mkdirSync, statSync } from "node:fs"; +import { resolve, dirname, basename } from "node:path"; +import { fileURLToPath } from "node:url"; +import { homedir } from "node:os"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const POP = resolve(HERE, ".."); +const OUT = `${POP}/big-pictures/out`; + +const argv = process.argv.slice(2); +const flags = {}; +const positional = []; +for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a.startsWith("--")) { + const k = a.slice(2); + const next = argv[i + 1]; + if (next !== undefined && !next.startsWith("--")) { flags[k] = next; i++; } + else flags[k] = true; + } else positional.push(a); +} + +if (!positional[0]) { + console.error("usage: node bin/sfx-compose.mjs [--force] [--no-finalize]"); + process.exit(1); +} + +// Resolve the arrangement file: explicit path, or big-pictures/.sfx. +let ARR = resolve(process.cwd(), positional[0]); +if (!existsSync(ARR)) { + const alt = `${POP}/big-pictures/${positional[0].replace(/\.sfx$/, "")}.sfx`; + if (existsSync(alt)) ARR = alt; + else { console.error(`✗ arrangement not found: ${positional[0]}`); process.exit(1); } +} +const SLUG = basename(ARR).replace(/\.sfx$/, ""); +const FORCE = flags.force === true; +const FINALIZE = flags["no-finalize"] !== true; + +// ── parse arrangement ───────────────────────────────────────────────── +const meta = { title: SLUG, glue: "swirl", duration: null }; +const cues = []; +for (const raw of readFileSync(ARR, "utf8").split("\n")) { + const line = raw.trim(); + if (!line || line.startsWith("#")) continue; + if (line.startsWith("@")) { + const sp = line.indexOf(" "); + const key = (sp === -1 ? line.slice(1) : line.slice(1, sp)).trim(); + const val = sp === -1 ? "" : line.slice(sp + 1).trim(); + if (key === "duration") meta.duration = Number(val) || null; + else meta[key] = val; + continue; + } + const ci = line.indexOf(":"); + if (ci === -1) continue; + const prompt = line.slice(ci + 1).trim(); + if (!prompt) continue; + const cue = { at: 0, gain: 0.7, dur: null, loop: false, fadein: 0, fadeout: 0, pan: 0, prompt }; + for (const tok of line.slice(0, ci).trim().split(/\s+/)) { + if (!tok) continue; + const [k, v] = tok.split("="); + if (k === "at") cue.at = Number(v) || 0; + else if (k === "gain") cue.gain = Number(v); + else if (k === "dur") cue.dur = Math.max(0.5, Math.min(30, Number(v))); + else if (k === "loop") cue.loop = true; + else if (k === "fadein") cue.fadein = Number(v) || 0; + else if (k === "fadeout") cue.fadeout = Number(v) || 0; + else if (k === "pan") cue.pan = Math.max(-1, Math.min(1, Number(v) || 0)); + } + cues.push(cue); +} +if (!cues.length) { console.error(`✗ no cues in ${ARR}`); process.exit(1); } + +const TITLE = flags.title || meta.title; +const MIX = `${OUT}/${SLUG}-sfxmix.mp3`; +const FINAL = `${OUT}/${SLUG}-final.mp3`; +const DESK = `${homedir()}/Desktop/${SLUG}.mp3`; +const sfxDir = `${OUT}/sfx/${SLUG}`; +mkdirSync(sfxDir, { recursive: true }); + +function run(cmd, args, cwd = POP) { + const r = spawnSync(cmd, args, { cwd, stdio: ["ignore", "inherit", "inherit"] }); + if (r.status !== 0) { console.error(`✗ ${cmd} ${args.join(" ")} failed (exit ${r.status})`); process.exit(1); } +} +function probeDur(p) { + const r = spawnSync("ffprobe", ["-v", "error", "-show_entries", "format=duration", + "-of", "default=nw=1:nk=1", p], { encoding: "utf8" }); + return Number((r.stdout || "0").trim()) || 0; +} + +console.log(`━━━ ${SLUG} ━━━ ${cues.length} sfx cues · glue=${meta.glue}${meta.duration ? ` · trim ${meta.duration}s` : ""}\n`); + +// ── 1 · render every cue through /api/sfx (cached) ─────────────────── +console.log("▸ 1 · render sfx cues"); +const stems = []; +cues.forEach((c, i) => { + const p = `${sfxDir}/${String(i).padStart(2, "0")}.mp3`; + const args = ["bin/sfx.mjs", "--text", c.prompt, "--out", p]; + if (c.dur != null) args.push("--duration", String(c.dur)); + if (c.loop) args.push("--loop"); + if (FORCE) args.push("--force"); + run("node", args); + stems.push(p); +}); + +// ── 2 · lay cues on a timeline + glue ──────────────────────────────── +console.log("\n▸ 2 · compose timeline + psychedelic glue"); +const inputs = []; +for (const p of stems) inputs.push("-i", p); + +const stemFilters = []; +const labels = []; +let trackEnd = 0; +cues.forEach((c, i) => { + const len = c.dur != null ? c.dur : probeDur(stems[i]); + trackEnd = Math.max(trackEnd, c.at + len); + const d = Math.max(0, Math.round(c.at * 1000)); + const chain = [`adelay=${d}|${d}`, `volume=${c.gain}`]; + if (c.fadein > 0) chain.push(`afade=t=in:st=${c.at}:d=${c.fadein}`); + if (c.fadeout > 0) chain.push(`afade=t=out:st=${(c.at + len - c.fadeout).toFixed(3)}:d=${c.fadeout}`); + // Stereo placement: pan a copy of the (mono/stereo) stem across the field. + if (c.pan !== 0) { + const l = (1 - Math.max(0, c.pan)).toFixed(3); + const rr = (1 + Math.min(0, c.pan)).toFixed(3); + chain.push(`pan=stereo|c0=${l}*c0|c1=${rr}*c1`); + } + stemFilters.push(`[${i}:a]${chain.join(",")}[s${i}]`); + labels.push(`[s${i}]`); +}); + +// Psychedelic glue presets applied to the full mix. +const GLUE = { + swirl: "aphaser=in_gain=0.5:out_gain=0.8:delay=3.0:decay=0.5:speed=0.4," + + "chorus=0.6:0.9:55|65|75:0.4|0.32|0.3:0.25|0.4|0.3:2|2.3|1.3," + + "aecho=0.8:0.7:90|330:0.4|0.25", + dub: "aecho=0.8:0.85:180|420|760:0.5|0.3|0.18,aphaser=speed=0.3", + none: "anull", +}[meta.glue] || "anull"; + +const dur = meta.duration || (trackEnd + 1); +// mix → glue → gentle loudness normalize → trim → 44.1k stereo +const post = `amix=inputs=${cues.length}:duration=longest:dropout_transition=0:normalize=0[mix];` + + `[mix]${GLUE},loudnorm=I=-15:TP=-1.5:LRA=11,atrim=duration=${dur.toFixed(2)},aresample=44100`; +const filter = `${stemFilters.join(";")};${labels.join("")}${post}`; + +run("ffmpeg", ["-y", "-loglevel", "error", ...inputs, + "-filter_complex", filter, "-c:a", "libmp3lame", "-q:a", "2", MIX]); +console.log(` ✓ ${MIX} (${(statSync(MIX).size / 1024 / 1024).toFixed(2)} MB · ~${dur.toFixed(0)}s)`); + +// ── 3 · finalize (ID3 + auto cover) ────────────────────────────────── +let result = MIX; +if (FINALIZE) { + console.log("\n▸ 3 · finalize (ID3 + cover)"); + run("node", ["bin/finalize.mjs", "--in", MIX.replace(POP + "/", ""), + "--slug", SLUG, "--title", TITLE, + "--out", FINAL.replace(POP + "/", ""), "--force"]); + result = FINAL; +} + +copyFileSync(result, DESK); +console.log(`\n✓ ~/Desktop/${SLUG}.mp3 ${(statSync(DESK).size / 1024 / 1024).toFixed(2)} MB`); diff --git a/pop/bin/sfx-local.mjs b/pop/bin/sfx-local.mjs new file mode 100644 index 000000000..d2427d3a0 --- /dev/null +++ b/pop/bin/sfx-local.mjs @@ -0,0 +1,100 @@ +#!/usr/bin/env node +// sfx-local.mjs — a LOCAL stand-in for /api/sfx (ElevenLabs SFX). +// +// Sibling of pop/chillwave/bin/say-local.mjs. Replicates the +// `generateSfx` branch of system/netlify/functions/sfx.js — hits the +// ElevenLabs text-to-sound-effects model directly and returns the audio +// bytes. No S3/Mongo cache layer (sfx.mjs already content-hash caches +// every cue locally), so this writes nothing to the prod CDN. +// +// Reads ELEVENLABS_API_KEY from the environment, then the vault +// (aesthetic-computer-vault/lith/.env or .devcontainer envs). +// +// Usage: +// node pop/bin/sfx-local.mjs # listens :8898 +// SFX_ENDPOINT=http://127.0.0.1:8898/api/sfx \ +// node pop/bin/sfx.mjs --text "warm analog drone" --duration 20 + +import { createServer } from "node:http"; +import { readFileSync, existsSync } from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO = resolve(HERE, "../.."); +const PORT = Number(process.env.SFX_LOCAL_PORT || 8898); + +const SFX_MODEL = "eleven_text_to_sound_v2"; // same as sfx.js +const DEFAULT_OUTPUT_FORMAT = "mp3_44100_128"; + +function loadKey() { + if (process.env.ELEVENLABS_API_KEY) return process.env.ELEVENLABS_API_KEY; + const candidates = [ + `${REPO}/aesthetic-computer-vault/lith/.env`, + `${REPO}/aesthetic-computer-vault/.devcontainer/envs/devcontainer.env`, + ]; + for (const path of candidates) { + if (!existsSync(path)) continue; + for (const line of readFileSync(path, "utf8").split("\n")) { + if (line.startsWith("ELEVENLABS_API_KEY=")) { + return line.slice("ELEVENLABS_API_KEY=".length).trim().replace(/^['"]|['"]$/g, ""); + } + } + } + throw new Error("ELEVENLABS_API_KEY not in env or vault"); +} +const KEY = loadKey(); + +function readBody(req) { + return new Promise((res, rej) => { + const chunks = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => { + try { res(JSON.parse(Buffer.concat(chunks).toString("utf8") || "{}")); } + catch (e) { rej(e); } + }); + req.on("error", rej); + }); +} + +const server = createServer(async (req, res) => { + if (req.method !== "POST" || !req.url.startsWith("/api/sfx")) { + res.writeHead(404).end("not found"); + return; + } + try { + const b = await readBody(req); + const text = (b.text || b.prompt || b.from || "").trim(); + if (!text) { res.writeHead(400).end("no text"); return; } + + const payload = { text, model_id: SFX_MODEL }; + if (typeof b.duration_seconds === "number") payload.duration_seconds = Math.max(0.5, Math.min(30, b.duration_seconds)); + if (typeof b.prompt_influence === "number") payload.prompt_influence = Math.max(0, Math.min(1, b.prompt_influence)); + if (b.loop === true) payload.loop = true; + + const fmt = b.output_format || DEFAULT_OUTPUT_FORMAT; + const url = `https://api.elevenlabs.io/v1/sound-generation?output_format=${encodeURIComponent(fmt)}`; + const r = await fetch(url, { + method: "POST", + headers: { "xi-api-key": KEY, "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }); + if (!r.ok) { + const err = await r.text(); + console.error(`✗ ElevenLabs SFX ${r.status}: ${err.slice(0, 300)}`); + res.writeHead(502, { "content-type": "text/plain" }).end(err.slice(0, 500)); + return; + } + const buf = Buffer.from(await r.arrayBuffer()); + res.writeHead(200, { "content-type": "audio/mpeg" }); + res.end(buf); + console.log(`✓ sfx · "${text.slice(0, 48)}${text.length > 48 ? "…" : ""}" · ${(buf.length / 1024) | 0} KB`); + } catch (e) { + console.error("✗", e.message); + res.writeHead(500, { "content-type": "text/plain" }).end(String(e.message)); + } +}); + +server.listen(PORT, "127.0.0.1", () => { + console.log(`▸ sfx-local (${SFX_MODEL}) → http://127.0.0.1:${PORT}/api/sfx`); +}); diff --git a/pop/bin/sfx.mjs b/pop/bin/sfx.mjs new file mode 100755 index 000000000..02e3185cb --- /dev/null +++ b/pop/bin/sfx.mjs @@ -0,0 +1,158 @@ +#!/usr/bin/env node +// sfx.mjs — POST a sound-effect description to /api/sfx, cache the stem. +// +// Sibling of `say.mjs` (which renders vocals). Where `say.mjs` reads a +// lyric file and hits ElevenLabs text-to-speech, this hits the +// text-to-sound-effects model (eleven_text_to_sound_v2) via the +// production /api/sfx proxy. Same content-hash local cache discipline — +// the endpoint costs real money per generation, so reruns are free. +// +// Two input modes: +// 1. inline prompt: node bin/sfx.mjs --text "distant thunder" --out out/thunder.mp3 +// 2. descriptions file (one cue per line, "# slug : prompt"): +// node bin/sfx.mjs ../big-pictures/plork.sfx.txt +// → renders each line to out/sfx/.mp3 +// +// Flags: +// --text "..." inline single prompt (skip the file) +// --out path.mp3 output path (single mode; default out/sfx/.mp3) +// --duration N clip length in seconds (0.5–30); omit = auto +// --influence N prompt_influence 0–1 (default server-side 0.3) +// --loop request a seamless loop (v2 only) +// --force bypass the local cache +// +// Descriptions-file line format (blank lines + #-only comment lines skip): +// slug : a short natural-language description of the sound +// thunder : distant thunder rolling over a wet city at night +// coin : bright 8-bit coin pickup, short and snappy +// A line with no "slug :" prefix uses a zero-padded index as the slug. + +import { writeFileSync, readFileSync, mkdirSync, existsSync } from "node:fs"; +import { resolve, dirname, basename } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createHash } from "node:crypto"; +import { homedir } from "node:os"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(HERE, ".."); + +const argv = process.argv.slice(2); +const flags = {}; +const positional = []; +for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a.startsWith("--")) { + const key = a.slice(2); + const next = argv[i + 1]; + if (next !== undefined && !next.startsWith("--")) { flags[key] = next; i++; } + else flags[key] = true; + } else positional.push(a); +} + +function expandHome(p) { + if (!p || typeof p !== "string") return p; + if (p === "~") return homedir(); + if (p.startsWith("~/")) return resolve(homedir(), p.slice(2)); + return p; +} + +const DURATION = flags.duration !== undefined ? Number(flags.duration) : null; // 0.5–30 s +const INFLUENCE = flags.influence !== undefined ? Number(flags.influence) : null; // 0–1 +const LOOP = flags.loop === true; +const FORCE = flags.force === true; + +const SFX_URL = process.env.SFX_ENDPOINT || "https://aesthetic.computer/api/sfx"; + +// Render one cue → mp3 at outPath. Returns true if it hit the network. +async function renderCue(text, outPath) { + const body = { text }; + if (DURATION !== null && Number.isFinite(DURATION)) body.duration_seconds = Math.max(0.5, Math.min(30, DURATION)); + if (INFLUENCE !== null && Number.isFinite(INFLUENCE)) body.prompt_influence = Math.max(0, Math.min(1, INFLUENCE)); + if (LOOP) body.loop = true; + + const inputHash = createHash("sha256").update(JSON.stringify(body)).digest("hex").slice(0, 16); + const hashFile = `${outPath}.hash`; + mkdirSync(dirname(outPath), { recursive: true }); + + if (!FORCE && existsSync(outPath) && existsSync(hashFile)) { + const cached = readFileSync(hashFile, "utf8").trim(); + if (cached === inputHash) { + const size = (readFileSync(outPath).length / 1024).toFixed(0); + console.log(`✓ ${outPath} cached (${size} KB · hash ${inputHash}) — skipping /api/sfx`); + return false; + } + } + + console.log(`→ POST /api/sfx · "${text.slice(0, 64)}${text.length > 64 ? "…" : ""}"` + + (DURATION !== null ? ` · ${DURATION}s` : "") + + (INFLUENCE !== null ? ` · influence=${INFLUENCE}` : "") + + (LOOP ? " · loop" : "")); + + const res = await fetch(SFX_URL, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + redirect: "follow", + }); + + if (!res.ok) { + console.error(`✗ /api/sfx returned ${res.status}: ${await res.text()}`); + process.exit(1); + } + + const buf = Buffer.from(await res.arrayBuffer()); + writeFileSync(outPath, buf); + writeFileSync(hashFile, inputHash + "\n"); + console.log(`✓ ${outPath} (${(buf.length / 1024).toFixed(0)} KB · hash ${inputHash})`); + return true; +} + +// ── Single inline prompt ─────────────────────────────────────────────── +if (flags.text) { + const text = String(flags.text).trim(); + if (!text) { console.error("✗ --text was empty"); process.exit(1); } + const slug = text.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40) || "sfx"; + const outPath = expandHome(flags.out) || `${ROOT}/big-pictures/out/sfx/${slug}.mp3`; + await renderCue(text, outPath); + process.exit(0); +} + +// ── Descriptions file (batch) ────────────────────────────────────────── +if (!positional[0]) { + console.error("usage: node bin/sfx.mjs | --text \"a sound\" [--out path.mp3] [--duration N] [--influence N] [--loop] [--force]"); + process.exit(1); +} + +const cuesPath = resolve(process.cwd(), positional[0]); +if (!existsSync(cuesPath)) { + console.error(`✗ descriptions file not found: ${cuesPath}`); + process.exit(1); +} + +const stem = basename(cuesPath).replace(/\.[^.]+$/, "").replace(/\.sfx$/, ""); +const OUT_DIR = expandHome(flags.out) || `${ROOT}/big-pictures/out/sfx/${stem}`; + +const lines = readFileSync(cuesPath, "utf8").split("\n"); +const cues = []; +let idx = 0; +for (const raw of lines) { + const line = raw.trim(); + if (!line || line.startsWith("#")) continue; + const m = line.match(/^([a-zA-Z0-9_-]+)\s*:\s*(.+)$/); + if (m) cues.push({ slug: m[1], text: m[2].trim() }); + else cues.push({ slug: String(idx).padStart(3, "0"), text: line }); + idx++; +} + +if (cues.length === 0) { + console.error(`✗ no cues found in ${cuesPath}`); + process.exit(1); +} + +console.log(`🔊 ${cues.length} cue(s) → ${OUT_DIR}`); +let rendered = 0; +for (const cue of cues) { + const outPath = `${OUT_DIR}/${cue.slug}.mp3`; + if (await renderCue(cue.text, outPath)) rendered++; +} +console.log(`✓ done — ${rendered} generated, ${cues.length - rendered} cached`); diff --git a/system/netlify/functions/sfx.js b/system/netlify/functions/sfx.js new file mode 100644 index 000000000..c97020e5e --- /dev/null +++ b/system/netlify/functions/sfx.js @@ -0,0 +1,263 @@ +// SFX - Sound-effects generation API via ElevenLabs. +// Sibling of `say.js` (TTS): same auth, DO Spaces cache, CDN redirect, +// and Mongo ledger patterns — but hits the text-to-sound-effects model +// instead of text-to-speech. +// +// ElevenLabs sound-generation (latest model: eleven_text_to_sound_v2): +// POST https://api.elevenlabs.io/v1/sound-generation +// body: { text, model_id, duration_seconds?, prompt_influence?, loop? } +// returns: binary audio (mp3) +// +// Usage from a piece / pipeline: +// POST /api/sfx { text: "distant thunder rolling over a city" } +// POST /api/sfx { text: "8-bit coin pickup", duration_seconds: 1.5, +// prompt_influence: 0.6, loop: false } +// +// Costs real money per generation — cached by content-hash to the CDN +// (sfx-cache/). Pass { bust: true } to regenerate. + +const crypto = require("crypto"); +const { S3Client, HeadObjectCommand, PutObjectCommand } = require("@aws-sdk/client-s3"); + +const SFX_MODEL = "eleven_text_to_sound_v2"; // latest text-to-sound model +// Higher-fidelity than the API default (mp3_22050_32) — paid-tier key. +const DEFAULT_OUTPUT_FORMAT = "mp3_44100_128"; + +// Initialize S3 client for Digital Ocean Spaces (same creds as say.js). +const s3 = new S3Client({ + endpoint: `https://${process.env.ART_ENDPOINT}`, + region: "us-east-1", // DO Spaces requires a region, but it's ignored + credentials: { + accessKeyId: process.env.ART_KEY, + secretAccessKey: process.env.ART_SECRET, + }, +}); + +const BUCKET = process.env.ART_SPACE_NAME; +const CDN_URL = "https://art.aesthetic.computer"; +const CACHE_PREFIX = "sfx-cache/"; + +// Cache key from every parameter that changes the generated audio. +function getCacheKey(spec, text) { + const parts = `${SFX_MODEL}:${spec}:${text}`; + const hash = crypto.createHash("sha256").update(parts).digest("hex"); + return `${CACHE_PREFIX}${hash}.mp3`; +} + +async function checkCache(key) { + try { + await s3.send(new HeadObjectCommand({ Bucket: BUCKET, Key: key })); + return `${CDN_URL}/${key}`; + } catch (err) { + if (err.name === "NotFound" || err.$metadata?.httpStatusCode === 404) { + return null; // Not cached + } + console.error("Cache check error:", err); + return null; + } +} + +// Ledger every generation into the `sfx` MongoDB collection — mirrors +// the `sayings` collection in say.js. Failures are swallowed. +async function recordSfx(entry) { + let database; + try { + const { connect } = await import("../../backend/database.mjs"); + database = await connect(); + const collection = database.db.collection("sfx"); + await collection.createIndex({ when: -1 }); + await collection.createIndex({ cacheKey: 1 }); + await collection.insertOne({ ...entry, when: new Date() }); + } catch (err) { + console.error("⚠️ sfx log failed:", err?.message || err); + } finally { + if (database) { + try { await database.disconnect(); } catch (_) {} + } + } +} + +async function saveToCache(key, audioBuffer, metadata = {}) { + try { + const cleanMeta = {}; + for (const [k, v] of Object.entries(metadata)) { + if (v == null) continue; + const str = String(v).slice(0, 1800); + cleanMeta[k] = Buffer.from(str, "utf8").toString("ascii").replace(/[\r\n]/g, " "); + } + + await s3.send(new PutObjectCommand({ + Bucket: BUCKET, + Key: key, + Body: audioBuffer, + ContentType: "audio/mpeg", + ACL: "public-read", + CacheControl: "public, max-age=31536000", // 1 year (audio doesn't change) + Metadata: cleanMeta, + })); + console.log(`✅ Cached SFX: ${CDN_URL}/${key}`); + return `${CDN_URL}/${key}`; + } catch (err) { + console.error("Cache write error:", err); + return null; + } +} + +// Generate a sound effect with ElevenLabs. +async function generateSfx(text, { durationSeconds, promptInfluence, loop, outputFormat }) { + const payload = { text, model_id: SFX_MODEL }; + if (durationSeconds != null) payload.duration_seconds = durationSeconds; + if (promptInfluence != null) payload.prompt_influence = promptInfluence; + if (loop != null) payload.loop = loop; + + const url = `https://api.elevenlabs.io/v1/sound-generation?output_format=${encodeURIComponent(outputFormat)}`; + + const response = await fetch(url, { + method: "POST", + headers: { + "xi-api-key": process.env.ELEVENLABS_API_KEY, + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + }); + + if (!response.ok) { + const err = await response.text(); + throw new Error(`ElevenLabs SFX API error ${response.status}: ${err}`); + } + + return Buffer.from(await response.arrayBuffer()); +} + +exports.handler = async (event) => { + const method = event.httpMethod; + const headers = corsHeaders(event); + + if (method === "OPTIONS") { + return { + statusCode: 200, + headers, + body: JSON.stringify({ message: "Success!" }), + }; + } else if (method !== "POST") { + return { + statusCode: 405, + headers, + body: JSON.stringify({ message: "Method Not Allowed" }), + }; + } + + const body = JSON.parse(event.body || "{}"); + + // The prompt describing the desired sound. Accept `text` (canonical), + // `prompt`, or `from` (parity with say.js). + const text = (body.text || body.prompt || body.from || "").trim(); + if (!text) { + return { + statusCode: 400, + headers, + body: JSON.stringify({ message: "Missing `text` (sound description)." }), + }; + } + + // duration_seconds: 0.5–30, or null to let the model auto-detect length. + const durationSeconds = (typeof body.duration_seconds === "number" || typeof body.duration === "number") + ? Math.max(0.5, Math.min(30, body.duration_seconds ?? body.duration)) + : null; + // prompt_influence: 0–1 (default 0.3). Higher = closer to the prompt. + const promptInfluence = (typeof body.prompt_influence === "number") + ? Math.max(0, Math.min(1, body.prompt_influence)) + : null; + // loop: produce a seamless loop (v2 only). + const loop = body.loop === true ? true : (body.loop === false ? false : null); + const outputFormat = body.output_format || DEFAULT_OUTPUT_FORMAT; + const bustCache = body.bust === true; + + // Cache key spec — every knob that changes the audio. + const durSuffix = durationSeconds != null ? `-d${durationSeconds}` : ""; + const piSuffix = promptInfluence != null ? `-pi${promptInfluence}` : ""; + const loopSuffix = loop === true ? "-loop" : ""; + const spec = `${outputFormat}${durSuffix}${piSuffix}${loopSuffix}`; + const cacheKey = getCacheKey(spec, text); + + try { + if (!bustCache) { + const cachedUrl = await checkCache(cacheKey); + if (cachedUrl) { + console.log(`🎯 SFX cache hit: ${cachedUrl}`); + await recordSfx({ text, spec, cacheKey, url: cachedUrl, cached: true }); + return { + statusCode: 302, + headers: { ...headers, Location: cachedUrl, "Cache-Control": "public, max-age=86400" }, + body: "", + }; + } + } else { + console.log(`🧹 SFX cache bust requested for: ${text.substring(0, 50)}...`); + } + + console.log(`🔄 SFX ${bustCache ? "regenerating" : "cache miss"}: ${text.substring(0, 50)}...`); + + const audioBuffer = await generateSfx(text, { durationSeconds, promptInfluence, loop, outputFormat }); + + if (!audioBuffer || audioBuffer.length === 0) { + return { + statusCode: 500, + headers, + body: JSON.stringify({ message: "Failed to generate sound effect." }), + }; + } + + console.log(`🔊 Generated SFX (${SFX_MODEL}): ${(audioBuffer.length / 1024).toFixed(0)} KB`); + + const cdnUrl = await saveToCache(cacheKey, audioBuffer, { + text, + model: SFX_MODEL, + spec, + ts: new Date().toISOString(), + }); + + if (cdnUrl) { + await recordSfx({ text, spec, cacheKey, url: cdnUrl, cached: false }); + return { + statusCode: 302, + headers: { ...headers, Location: cdnUrl }, + body: "", + }; + } + + // Fallback: return audio directly if caching failed. + return { + statusCode: 200, + headers: { + ...headers, + "Content-Disposition": 'inline; filename="sfx.mp3"', + "Content-Type": "audio/mpeg", + }, + body: audioBuffer.toString("base64"), + isBase64Encoded: true, + }; + } catch (error) { + console.error("SFX generation failed:", error); + return { + statusCode: 500, + headers, + body: JSON.stringify({ message: "An error has occurred.", error: error.message }), + }; + } +}; + +function corsHeaders(event) { + const dev = process.env.CONTEXT === "dev"; + const production = !dev; + let allowedOrigin = production ? "https://aesthetic.computer" : "*"; + if (event.headers.origin === "null") allowedOrigin = "*"; + + return { + "Access-Control-Allow-Methods": "GET,OPTIONS,PATCH,DELETE,POST,PUT", + "Access-Control-Allow-Origin": allowedOrigin, + "Access-Control-Allow-Credentials": true, + "Access-Control-Allow-Headers": + "X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, Content-Type, Date, X-Api-Version", + }; +}