Something went wrong. Try again.
Monorepo for Aesthetic.Computer aesthetic.computer
Something went wrong. Try again.
JavaScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270#!/usr/bin/env node// perword.mjs — generate per-WORD ElevenLabs takes (one API call per// lyric word) and concat them with tiny gaps between.//// Why: per-line takes ("that saved a wretch like me") give ElevenLabs// the whole phrase to perform, which means held phrase-end syllables// (`me*5` = 4.29s target) only get ~0.2s of natural source duration.// Pitchsnap then has to do 18× stretch, which exceeds WORLD's clean// range and produces garbled vowel sustain. The user can't hear "me"// in the rendered audio.//// Per-word takes give each syllable ~0.5-1.5s of natural ElevenLabs// prosody (a single word read aloud has natural release/decay), so// pitchsnap needs only 3-5× stretch — clean WORLD range, audible word.//// Trade-off: more API calls (26 instead of 4 for amazing's verse 1).// At jeffrey-pvc rates, ~$0.30 vs $0.05 per regen.//// Usage:// node bin/perword.mjs --slug amazing// [--gap-ms 80] [--style 0.55] [--stability 0.7] [--speed 0.7]
import { execSync, spawnSync } from "node:child_process";import { writeFileSync, readFileSync, existsSync, mkdtempSync, rmSync } from "node:fs";import { tmpdir } from "node:os";
const flags = {};for (let i = 0; i < process.argv.length; i++) { const a = process.argv[i]; if (!a.startsWith("--")) continue; const next = process.argv[i + 1]; if (next === undefined || next.startsWith("--")) flags[a.slice(2)] = true; else flags[a.slice(2)] = next;}
const SLUG = flags.slug || "amazing";const POP = "/Users/jas/aesthetic-computer/pop";const SCORE_PATH = `${POP}/big-pictures/${SLUG}.np`;const LYRIC_PATH = `${POP}/big-pictures/${SLUG}.txt`;const SECTION = (flags.section || "verse 1").toLowerCase();const BPM = Number(flags.bpm ?? 70);// 80ms was too tight — whisper merged single-char words ('i', 'a')// into adjacent words during the source-side segmentation, causing// pitchsnap to skip those score positions and shift every subsequent// word by one slot. 250ms minimum keeps word boundaries distinct.const GAP_MS = Number(flags["gap-ms"] ?? 250);const HELD_GAP_MS = Number(flags["held-gap-ms"] ?? 400);const STYLE = flags.style ?? "0.55";const STABILITY = flags.stability ?? "0.7";const SPEED = flags.speed ?? "0.7";// Validator + retry: each generated word must have natural duration// >= max(MIN_NATURAL_S, target_dur / MAX_STRETCH). For a *5 hold at// BPM=70, target=4.29s; with MAX_STRETCH=4 we need natural >= 1.07s.// Short *1 words just need >= 0.4s. Retries cycle through prosody// strategies until threshold hits or we run out.const MIN_NATURAL_S = Number(flags["min-natural"] ?? 0.40);const MAX_STRETCH = Number(flags["max-stretch"] ?? 4.0);const MAX_RETRIES = Number(flags["max-retries"] ?? 5);const OUT_PATH = flags.out ? flags.out : `${POP}/big-pictures/out/${SLUG}-perword.mp3`;
if (!existsSync(SCORE_PATH)) { console.error(`✗ score missing: ${SCORE_PATH}`); process.exit(1); }if (!existsSync(LYRIC_PATH)) { console.error(`✗ lyrics missing: ${LYRIC_PATH}`); process.exit(1); }
// ── Parse score: per-line list of {raw, weight} ───────────────────────const scoreLines = readFileSync(SCORE_PATH, "utf8").split("\n");const sStart = scoreLines.findIndex((l) => l.trim().toLowerCase() === SECTION);if (sStart < 0) { console.error(`✗ section '${SECTION}' missing in score`); process.exit(1); }const scoreSyls = []; // flat array of {raw, weight} for the sectionfor (let i = sStart + 1; i < scoreLines.length; i++) { const l = scoreLines[i].trim(); if (!l) break; if (l.startsWith("#")) continue; if (/^[a-z]+ \d/i.test(l)) break; for (const tok of l.split(/\s+/)) { const m = tok.match(/^[A-Ga-g][#b]?-?\d:(.+?)(?:\*(\d+(?:\.\d+)?))?$/); if (!m) continue; scoreSyls.push({ raw: m[1], weight: Number(m[2] ?? 1) }); }}
// Group syllables into words by hyphen markers — each lyric word maps// to one or more score syllables. Take MAX weight as the word's hold// (so multi-syllable words like 'a-ma-zing' inherit the held syllable's// weight if any).const wordWeights = [];let cur = null;for (const s of scoreSyls) { if (s.raw.startsWith("-") && cur) { cur.weight = Math.max(cur.weight, s.weight); } else { cur = { weight: s.weight }; wordWeights.push(cur); }}console.log(`→ score: ${scoreSyls.length} syllables grouped into ${wordWeights.length} words`);
// ── Parse lyric file: flat array of words ─────────────────────────────const lyricLines = readFileSync(LYRIC_PATH, "utf8").split("\n");const lStart = lyricLines.findIndex((l) => l.trim().toLowerCase() === SECTION);if (lStart < 0) { console.error(`✗ section '${SECTION}' missing in lyrics`); process.exit(1); }const lyricWords = [];for (let i = lStart + 1; i < lyricLines.length; i++) { const t = lyricLines[i].trim(); if (!t) break; if (/^[a-z]+ \d/i.test(t)) break; for (const w of t.split(/\s+/)) lyricWords.push(w);}console.log(`→ lyrics: ${lyricWords.length} words`);
if (lyricWords.length !== wordWeights.length) { console.warn(`⚠ word counts mismatch (lyrics=${lyricWords.length}, score=${wordWeights.length})`);}
// ── Helpers ──────────────────────────────────────────────────────────function probeDur(path) { try { const out = execSync( `ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 ${path}`, { encoding: "utf8" } ).trim(); return Number(out); } catch { return 0; }}
function elongateVowel(word, n) { const vowels = [...word.matchAll(/[aeiouy]+/gi)]; if (vowels.length === 0) return word + "e".repeat(n); const v = vowels[vowels.length - 1]; const vEnd = v.index + v[0].length; return word.slice(0, vEnd) + v[0].slice(-1).repeat(n) + word.slice(vEnd);}
// Retry strategies — each tweaks one or more knobs of the request.// Returns [{label, text, style, stability}, ...]function retryStrategies(word) { return [ { label: "ellipsis", text: `${word}...`, style: STYLE, stability: STABILITY }, { label: "double-vowel",text: elongateVowel(word, 2) + "...", style: STYLE, stability: STABILITY }, { label: "soft-low-stab",text: `${word}, ${word}.`, style: "0.45", stability: "0.55" }, { label: "many-dots", text: `${word}............`, style: "0.50", stability: STABILITY }, { label: "echo+vowel", text: `${elongateVowel(word, 4)}.`, style: "0.60", stability: "0.55" }, ];}
// ── Generate one ElevenLabs take per word with validate+retry ────────const tmp = mkdtempSync(`${tmpdir()}/perword-${SLUG}-`);const wordMp3s = [];for (let i = 0; i < lyricWords.length; i++) { const word = lyricWords[i].replace(/[.,!?;:]/g, ""); const wt = (wordWeights[i] && wordWeights[i].weight) ?? 1; const targetDur = wt * (60 / BPM); // Required natural duration: at least MIN_NATURAL_S, but if that's // not enough to keep stretch <= MAX_STRETCH, raise the bar. const needed = Math.max(MIN_NATURAL_S, targetDur / MAX_STRETCH);
// Try: original first, then ellipsis-augmented, then progressively // more aggressive prosody hints. Keep the LONGEST that meets `needed`, // or the longest of all attempts if none do. const wordOut = `${POP}/big-pictures/out/${SLUG}-w${String(i).padStart(2, "0")}.mp3`; const attempts = [ { label: "plain", text: word, style: STYLE, stability: STABILITY }, ...retryStrategies(word).slice(0, MAX_RETRIES), ]; let best = { dur: 0, attempt: -1, path: null, label: null }; for (let a = 0; a < attempts.length; a++) { const att = attempts[a]; const candPath = `${tmp}/${SLUG}-w${String(i).padStart(2, "0")}-a${a}.mp3`; const candFile = `${tmp}/${SLUG}-w${String(i).padStart(2, "0")}-a${a}.txt`; writeFileSync(candFile, `verse 1\n${att.text}\n`); const r = spawnSync("node", [ `${POP}/bin/say.mjs`, candFile, "--speed", String(SPEED), "--style", String(att.style), "--stability", String(att.stability), "--out", candPath, "--force", ], { stdio: ["ignore", "ignore", "inherit"] }); if (r.status !== 0) continue; const d = probeDur(candPath); if (d > best.dur) best = { dur: d, attempt: a, path: candPath, label: att.label }; if (d >= needed) break; // good enough — stop trying } if (!best.path) { console.error(`✗ all retries failed on "${word}"`); process.exit(1); }
// Copy winner to canonical location spawnSync("cp", [best.path, wordOut]); const ok = best.dur >= needed ? "✓" : "⚠"; console.log( `${ok} [${String(i + 1).padStart(2, " ")}/${lyricWords.length}] "${word.padEnd(8)}" hold=${wt}× ` + `target=${targetDur.toFixed(2)}s need=${needed.toFixed(2)}s ` + `got=${best.dur.toFixed(2)}s [a${best.attempt}:${best.label}]` ); wordMp3s.push({ path: wordOut, weight: wt });}
// ── Concat with per-word silence gaps ────────────────────────────────// Held words get longer LEADING silence so whisper keeps them as// separate utterances when aligning the concat.function silMp3(durS, name) { const path = `${tmp}/${name}.mp3`; spawnSync("ffmpeg", [ "-hide_banner", "-y", "-loglevel", "error", "-f", "lavfi", "-t", String(durS), "-i", "anullsrc=r=44100:cl=mono", path, ]); return path;}const shortSil = silMp3(GAP_MS / 1000, "sil-short");const heldSil = silMp3(HELD_GAP_MS / 1000, "sil-held");
const concatTxt = `${tmp}/concat.txt`;const lines = [];for (let i = 0; i < wordMp3s.length; i++) { const w = wordMp3s[i]; if (i > 0) { // Use the LONGER gap before any word that's part of a held syllable // (matches user pattern: held notes need clean entry boundary). lines.push(`file '${w.weight >= 3 ? heldSil : shortSil}'`); } lines.push(`file '${w.path}'`);}writeFileSync(concatTxt, lines.join("\n") + "\n");
const r = spawnSync("ffmpeg", [ "-hide_banner", "-y", "-loglevel", "error", "-f", "concat", "-safe", "0", "-i", concatTxt, "-c:a", "libmp3lame", "-q:a", "4", OUT_PATH,], { stdio: "inherit" });if (r.status !== 0) { console.error("✗ concat failed"); process.exit(1); }
rmSync(tmp, { recursive: true, force: true });const dur = execSync(`ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 ${OUT_PATH}`).toString().trim();console.log(`✓ ${OUT_PATH} (${Number(dur).toFixed(2)}s, ${lyricWords.length} words, gap=${GAP_MS}ms / held-gap=${HELD_GAP_MS}ms)`);
// ── Synthesize words.json from KNOWN word durations + gaps ───────────// Don't rely on whisper to segment — it routinely drops short words// like 'i' and 'a' even with generous inter-word gaps, which then// causes pitchsnap to skip score syllables and shift every subsequent// word by one slot ("me/i confusion"). Since we generated each word// as its own mp3 with known gaps, the boundaries are deterministic.const wordsJsonPath = OUT_PATH.replace(/\.mp3$/, "-words.json");const wordsJsonHash = wordsJsonPath + ".hash";const synthWords = [];let cursor_ms = 0;for (let i = 0; i < wordMp3s.length; i++) { const w = wordMp3s[i]; const wd_s = probeDur(w.path); const wd_ms = Math.round(wd_s * 1000); // Apply LEADING gap (this word's entry pause) if (i > 0) { const isHeld = w.weight >= 3; cursor_ms += isHeld ? HELD_GAP_MS : GAP_MS; } synthWords.push({ text: lyricWords[i].replace(/[.,!?;:]/g, ""), fromMs: cursor_ms, toMs: cursor_ms + wd_ms, }); cursor_ms += wd_ms;}writeFileSync(wordsJsonPath, JSON.stringify(synthWords, null, 2));// Drop any stale whisper hash so downstream callers don't think this// is a whisper-aligned file.if (existsSync(wordsJsonHash)) { try { execSync(`rm ${wordsJsonHash}`); } catch (_) {}}console.log(` ✓ synthesized ${synthWords.length} word boundaries → ${wordsJsonPath}`);