Something went wrong. Try again.
Monorepo for Aesthetic.Computer aesthetic.computer
Something went wrong. Try again.
JavaScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300#!/usr/bin/env node// score-render.mjs — score-as-truth audio render.//// Pivot from the previous "stretch a whole-verse take to fit the score// then realign whisper transcripts" pipeline. The score already knows// every word's start time, target duration, and target pitch — render// each word independently and place it at score time. No transcription// step needed; the score IS the alignment.//// Pipeline:// 1. Parse score (.np) → flat array of words with (start_s, dur_s,// target_midi). Group multi-syllable score tokens into one word.// 2. For each word, take its per-word clip from best-of-takes' picks// (already cached as ${slug}-take-*.mp3 + per-word selections).// Or just use ${slug}-perword.mp3 segments via the words.json// that perword.mjs synthesized.// 3. Rubberband each clip:// --time = target_dur / source_dur (stretch to score duration)// --pitch = target_midi - source_midi (semitone shift)// 4. Place each stretched clip at score_start_s using ffmpeg adelay.// 5. Mix all clips into one track (amix or just sequential delay+concat).//// Output: ${slug}-score-rendered.mp3 — clean, score-aligned, no whisper.//// Usage:// node bin/score-render.mjs --slug amazing// [--source perword|bestof] pick the per-word source set// [--bpm 70]// [--ref-note C3] source baritone reference// [--out path.mp3]
import { execSync, spawnSync } from "node:child_process";import { readFileSync, existsSync, mkdtempSync, rmSync } from "node:fs";import { resolve } from "node:path";import { tmpdir } from "node:os";
const flags = {};for (let i = 0; i < process.argv.length; i++) { const a = process.argv[i]; if (!a.startsWith("--")) continue; const next = process.argv[i + 1]; if (next === undefined || next.startsWith("--")) flags[a.slice(2)] = true; else flags[a.slice(2)] = next;}
const SLUG = flags.slug || "amazing";const POP = "/Users/jas/aesthetic-computer/pop";const SCORE_PATH = `${POP}/big-pictures/${SLUG}.np`;// `--section all` walks every `verse N` block in order, treating each// verse-marker as a phrase break (configurable via --inter-verse-beats).const SECTION = (flags.section || "verse 1").toLowerCase();const INTER_VERSE_BEATS = Number(flags["inter-verse-beats"] ?? 2);const BPM = Number(flags.bpm ?? 70);const REF_NOTE = flags["ref-note"] ?? "C3";const SOURCE_KIND = flags.source ?? "perword"; // perword | bestof// User feedback: pitches feel too high / not deep enough. Score is// mostly D3-D4 but jeffrey-pvc's natural baritone is C3-G3. Default// transpose -12 (one octave down) puts targets in his comfortable// range, gives the deep ballooned baritone instead of nasal tenor.const TRANSPOSE = Number(flags.transpose ?? -12);// True autotune: WORLD f0 replacement per clip (not rubberband -p// which only shifts the contour). Set false to use cheaper rubberband.const USE_WORLD = flags["no-world"] ? false : true;const OUT_PATH = flags.out || `${POP}/big-pictures/out/${SLUG}-score-rendered.mp3`;
if (!existsSync(SCORE_PATH)) { console.error(`✗ score missing: ${SCORE_PATH}`); process.exit(1); }
// ── Note → MIDI ──────────────────────────────────────────────────────const NOTE_BASE = { "C":0,"C#":1,"DB":1,"D":2,"D#":3,"EB":3,"E":4,"F":5, "F#":6,"GB":6,"G":7,"G#":8,"AB":8,"A":9,"A#":10,"BB":10,"B":11 };function noteToMidi(s) { const u = s.toUpperCase(); const oct = parseInt(u.slice(-1), 10); const name = u.slice(0, -1); return 12 * (oct + 1) + NOTE_BASE[name];}const REF_MIDI = noteToMidi(REF_NOTE);
// ── Parse score: flat list of {note, raw, weight} per syllable ───────const scoreLines = readFileSync(SCORE_PATH, "utf8").split("\n");const syllables = [];function parseRange(startIdx) { for (let i = startIdx; i < scoreLines.length; i++) { const l = scoreLines[i].trim(); if (!l) continue; if (l.startsWith("#")) continue; if (/^[a-z]+ \d+$/i.test(l)) break; for (const tok of l.split(/\s+/)) { const m = tok.match(/^([A-Ga-g][#b]?-?\d):(.+?)(?:\*(\d+(?:\.\d+)?))?$/); if (!m) continue; syllables.push({ note: m[1], raw: m[2], weight: Number(m[3] ?? 1) }); } }}if (SECTION === "all") { // Walk every `verse N` header in order; insert a rest weight between // verses so phrase breaks read musically. const headers = []; for (let i = 0; i < scoreLines.length; i++) { if (/^verse \d+$/i.test(scoreLines[i].trim())) headers.push(i); } if (!headers.length) { console.error("✗ section=all but no `verse N` headers found"); process.exit(1); } headers.forEach((h, idx) => { const before = syllables.length; parseRange(h + 1); if (idx < headers.length - 1 && syllables.length > before && INTER_VERSE_BEATS > 0) { // Mark a rest by inflating the previous syllable's weight (no audio // shifts; just gives the next word a later start_beat). syllables[syllables.length - 1].weight += INTER_VERSE_BEATS; } });} else { const sStart = scoreLines.findIndex((l) => l.trim().toLowerCase() === SECTION); if (sStart < 0) { console.error(`✗ section missing: ${SECTION}`); process.exit(1); } parseRange(sStart + 1);}console.log(`→ score: ${syllables.length} syllables`);
// Group syllables → words by hyphen-marker chains. Word's effective note// is its FIRST syllable's note (the attack); duration = sum of its// syllables' weights × beat_s.const beat_s = 60.0 / BPM;let beat_pos = 0;const words = [];let curWord = null;for (const s of syllables) { const startsContinuation = s.raw.startsWith("-"); if (startsContinuation && curWord) { curWord.weights.push(s.weight); curWord.notes.push(s.note); } else { if (curWord) words.push(curWord); curWord = { raw: s.raw.replace(/^-|-$/g, ""), notes: [s.note], weights: [s.weight], start_beat: beat_pos, }; } beat_pos += s.weight;}if (curWord) words.push(curWord);for (const w of words) { w.text = w.raw.replace(/[.,!?;:]/g, "").toLowerCase(); w.start_s = w.start_beat * beat_s; w.dur_s = w.weights.reduce((a, b) => a + b, 0) * beat_s; // Apply global TRANSPOSE so the score sits in jeffrey's baritone // range. All target_midis (and the per-syllable melody curve below) // shift by the same amount. w.target_midi = noteToMidi(w.notes[0]) + TRANSPOSE; w.target_midis_per_syllable = w.notes.map(n => noteToMidi(n) + TRANSPOSE); w.target_notes_per_syllable = w.notes.map(n => { const m = noteToMidi(n) + TRANSPOSE; const oct = Math.floor(m / 12) - 1; const name = ["C","C#","D","D#","E","F","F#","G","G#","A","A#","B"][m % 12]; return `${name}${oct}`; });}console.log(`→ ${words.length} words; total duration = ${(beat_pos * beat_s).toFixed(2)}s`);
// ── Locate per-word source clips ─────────────────────────────────────// Both perword.mjs and best-of-takes.mjs save individual word audio.// perword: ${slug}-perword.mp3 + ${slug}-perword-words.json (concat'd// single take per word, with synthesized boundaries).// best-of-takes: ${slug}-bestof.mp3 + ${slug}-bestof-words.json (best// pick per word from multiple takes).// Custom source overrides — point at any vocal stem + words sidecar.// Useful for the 7-verse cut where the source is the streaming TTS take// (amazing-7verse-vocal.mp3) rather than the per-word/best-of caches.const sourceMp3 = flags["source-mp3"] ? resolve(process.cwd(), flags["source-mp3"]) : `${POP}/big-pictures/out/${SLUG}-${SOURCE_KIND}.mp3`;const sourceWordsJson = flags["source-words"] ? resolve(process.cwd(), flags["source-words"]) : `${POP}/big-pictures/out/${SLUG}-${SOURCE_KIND}-words.json`;if (!existsSync(sourceMp3)) { console.error(`✗ source mp3 missing: ${sourceMp3}`); process.exit(1); }if (!existsSync(sourceWordsJson)) { console.error(`✗ source words missing: ${sourceWordsJson}`); process.exit(1); }const sourceWords = JSON.parse(readFileSync(sourceWordsJson, "utf8"));
if (sourceWords.length !== words.length) { console.warn(`⚠ source has ${sourceWords.length} words, score has ${words.length}; aligning by index`);}const n = Math.min(sourceWords.length, words.length);
// ── Source pitch (jeffrey-pvc baritone) ──────────────────────────────// Use REF_MIDI as the assumed source pitch. pitchsnap measures exact f0// per word; we just use a constant baritone reference for simplicity.// Per-word semitone shift = target_midi - REF_MIDI.
// ── Generate stretched + pitched per-word clips ──────────────────────const tmp = mkdtempSync(`${tmpdir()}/score-render-${SLUG}-`);const wordClips = [];for (let i = 0; i < n; i++) { const w = words[i]; const sw = sourceWords[i]; const src_dur = (sw.toMs - sw.fromMs) / 1000; const stretch = w.dur_s / Math.max(0.05, src_dur); const semitones = w.target_midi - REF_MIDI; // Extract this word's audio from the source mp3 const cutWav = `${tmp}/w${String(i).padStart(2, "0")}-cut.wav`; spawnSync("ffmpeg", [ "-hide_banner", "-y", "-loglevel", "error", "-ss", String(sw.fromMs / 1000), "-i", sourceMp3, "-t", String(src_dur), "-ar", "44100", "-ac", "1", cutWav, ]); // 1) Rubberband stretch (formant-preserving) to target duration — // NO pitch shift here; pitch is handled by WORLD next. const stretchedWav = `${tmp}/w${String(i).padStart(2, "0")}-stretch.wav`; spawnSync("rubberband", [ "-t", String(stretch), "--formant", cutWav, stretchedWav, ], { stdio: ["ignore", "ignore", "ignore"] }); if (!existsSync(stretchedWav)) { console.error(`✗ rubberband stretch failed on word ${i} '${w.text}'`); continue; } let rendWav = stretchedWav; // 2) WORLD f0 replacement (true autotune) — replaces the source f0 // contour with the target note(s) entirely, preserving formants. // Multi-syllable words pass --notes A,B,C with --weights for the // per-syllable melody curve. if (USE_WORLD) { const worldWav = `${tmp}/w${String(i).padStart(2, "0")}-world.wav`; const notesArg = w.target_notes_per_syllable.join(","); const weightsArg = w.weights.join(","); const r = spawnSync(`${POP}/.venv/bin/python`, [ `${POP}/bin/pitchsnap_world.py`, stretchedWav, worldWav, "--notes", notesArg, "--weights", weightsArg, "--retain", "1.0", ], { stdio: ["ignore", "ignore", "ignore"] }); if (r.status === 0 && existsSync(worldWav)) { rendWav = worldWav; } else { console.warn(` ! WORLD failed for '${w.text}', falling back to rubberband -p`); const rbWav = `${tmp}/w${String(i).padStart(2, "0")}-rb.wav`; spawnSync("rubberband", [ "-p", String(semitones), "--formant", stretchedWav, rbWav, ], { stdio: ["ignore", "ignore", "ignore"] }); if (existsSync(rbWav)) rendWav = rbWav; } } else if (semitones !== 0) { const rbWav = `${tmp}/w${String(i).padStart(2, "0")}-rb.wav`; spawnSync("rubberband", [ "-p", String(semitones), "--formant", stretchedWav, rbWav, ], { stdio: ["ignore", "ignore", "ignore"] }); if (existsSync(rbWav)) rendWav = rbWav; } wordClips.push({ text: w.text, start_s: w.start_s, dur_s: w.dur_s, src_dur, stretch, semitones, path: rendWav, }); console.log( ` [${String(i + 1).padStart(2)}/${n}] '${w.text.padEnd(8)}' ` + `start=${w.start_s.toFixed(2)}s dur=${w.dur_s.toFixed(2)}s ` + `src=${src_dur.toFixed(2)}s stretch=${stretch.toFixed(2)}× ` + `pitch=${semitones >= 0 ? "+" : ""}${semitones}st` );}
// ── Place each stretched clip at its score start_s and mix ───────────// Use ffmpeg with N inputs, each delayed via adelay, then amix=N. Or for// large N, do iterative mixdown to avoid command-line bloat.const totalDur = beat_pos * beat_s;console.log(`→ mixing ${wordClips.length} clips into ${totalDur.toFixed(2)}s base`);
// Build the filter graphconst inputs = wordClips.flatMap((c) => ["-i", c.path]);const filters = wordClips.map((c, idx) => { const delay_ms = Math.round(c.start_s * 1000); return `[${idx}:a]adelay=${delay_ms}|${delay_ms}[d${idx}]`;});const sumLabels = wordClips.map((_, idx) => `[d${idx}]`).join("");const filterGraph = filters.join(";") + `;${sumLabels}amix=inputs=${wordClips.length}:duration=longest:dropout_transition=0:normalize=0[out]`;
const r = spawnSync("ffmpeg", [ "-hide_banner", "-y", "-loglevel", "error", ...inputs, "-filter_complex", filterGraph, "-map", "[out]", "-c:a", "libmp3lame", "-q:a", "4", OUT_PATH,]);if (r.status !== 0) { console.error("✗ mix failed"); process.exit(1); }
const outDur = execSync(`ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 ${OUT_PATH}`).toString().trim();console.log(`✓ ${OUT_PATH} (${Number(outDur).toFixed(2)}s)`);
rmSync(tmp, { recursive: true, force: true });