diff --git a/recap/audience/jeffrey-24h-2026-05-08.mjs b/recap/audience/jeffrey-24h-2026-05-08.mjs --- a/recap/audience/jeffrey-24h-2026-05-08.mjs +++ b/recap/audience/jeffrey-24h-2026-05-08.mjs @@ -73,7 +73,12 @@ const rgb = cssColors[name]; if (!rgb) throw new Error(`colorAddress: unknown css color '${name}'`); const [r, g, b] = rgb; const hex = "#" + [r, g, b].map((c) => c.toString(16).padStart(2, "0")).join(""); - return { name, rgb, hex, caption: `rgb(${r}, ${g}, ${b})` }; + // Lift each channel 55% toward 255 — dark slide colors (indigo, + // blueviolet, etc) become punchy bright variants for the chapter + // title text, while already-bright colors stay close to themselves. + const lift = (c) => Math.min(255, c + Math.floor((255 - c) * 0.55)); + const brightHex = "#" + [lift(r), lift(g), lift(b)].map((c) => c.toString(16).padStart(2, "0")).join(""); + return { name, rgb, hex, brightHex, caption: `rgb(${r}, ${g}, ${b})` }; } const REAL = `\ @@ -238,14 +243,18 @@ "pop over": "popover", "Pop over": "popover", "pop overs": "popovers", "Note Pat": "notepat", "Notepat": "notepat", "Verovio": "verovio", "verbo": "verovio", + "Virovio": "verovio", "virovio": "verovio", // whisper hears Verovio as "Virovio" "Hockney-register": "hockney-register", "Hockney register": "hockney-register", "Hockney": "hockney", "Sage Jenson": "sage jenson", - "Sage": "sage", "Jenson": "jenson", + "Sage Jensen": "sage jenson", "sage Jensen": "sage jenson", // whisper hears Jenson as Jensen + "Sage": "sage", "Jenson": "jenson", "Jensen": "jenson", "GIPHY": "giphy", "Giphy": "giphy", + "giffy": "giphy", "Giffy": "giphy", // whisper hears GIPHY as "giffy" "Linked by Air": "linked by air", "KADIST": "kadist", "Kadist": "kadist", + "coddest": "kadist", "Coddest": "kadist", // whisper hears KADIST as "coddest" "SMK": "smk", "Parsons": "parsons", "UCLA": "ucla", "Yale": "yale", "Southern Oregon": "southern oregon", @@ -258,6 +267,8 @@ "Puppeteer": "puppeteer", "WASM": "wasm", "AC Native": "ac native", "AC-Native": "ac-native", "AC native": "ac native", + "act native": "ac-native", "Act Native": "ac-native", // whisper hears "ac-native" as "act native" + "act-native": "ac-native", "GPT-Image-2": "gpt-image-2", "GPT Image 2": "gpt-image-2", "GPT": "gpt", "DMG": "dmg", @@ -265,6 +276,9 @@ "CDN": "cdn", "QR": "qr", "WIP": "wip", "CCAT": "ccat", "CCat": "ccat", + "calarts cat": "calarts ccat", "Calarts cat": "calarts ccat", + "collarts cat": "calarts ccat", "Collarts cat": "calarts ccat", + "collarts": "calarts", "Collarts": "calarts", "Tech Director": "tech director", "PVC": "pvc", "CV": "cv", @@ -866,8 +880,8 @@
${hookHtml}
- - + + `; } @@ -1007,9 +1021,11 @@ const titleHtml = (title || "") .split("\n") .map((l) => `${l}`) .join(""); - const creamShadow = "2px 2px 0 rgba(0,0,0,0.95), -1px -1px 0 rgba(0,0,0,0.7), 0 0 18px rgba(0,0,0,0.55)"; - const capShadow = "1px 1px 0 rgba(0,0,0,0.92), 0 0 14px rgba(0,0,0,0.6)"; - const promptShadow = "1px 1px 0 rgba(0,0,0,0.95), 0 0 12px rgba(0,0,0,0.6)"; + // Sharper / higher-contrast text shadows — less blur, more solid + // black behind so the type punches through any photo background. + const creamShadow = "3px 3px 0 rgba(0,0,0,1), -2px -2px 0 rgba(0,0,0,1), 0 0 6px rgba(0,0,0,0.9)"; + const capShadow = "2px 2px 0 rgba(0,0,0,1), -1px -1px 0 rgba(0,0,0,0.95), 0 0 4px rgba(0,0,0,0.9)"; + const promptShadow = "2px 2px 0 rgba(0,0,0,1), 0 0 4px rgba(0,0,0,0.9)"; // PALS bug — top-LEFT, BELOW the chapter prompt. Rainbow drop shadow // tinted around the chapter color so each slide's branding takes a @@ -1046,18 +1062,18 @@ "Aesthetic.Computer>" with letters in deep purple and the dot in hot pink — same color treatment as the AC prompt itself. -->
- Aesthetic.Computer + Aesthetic.Computer
-
${titleHtml}
- ${cap ? `
${cap}
` : ""} +
${titleHtml}
+ ${cap ? `
${cap}
` : ""}
- - + + ${qrBlock} `; } diff --git a/recap/bin/beat.mjs b/recap/bin/beat.mjs new file mode 100644 --- /dev/null +++ b/recap/bin/beat.mjs @@ -0,0 +1,79 @@ +#!/usr/bin/env node +// beat.mjs — synthesize a low-key kick track from the waltz timing, +// one kick on every bar's beat 1. Mixed under the narration in +// compose.fish for a touch of musical pulse. +// +// Reads: recap/out/waltz-events.json (for bpm + totalSec) +// Writes: recap/out/beat.mp3 +// +// Kick: 60Hz sine, 0.18s, exponential amplitude decay. Quiet enough +// to sit under the narration without competing. +// +// Usage: node bin/beat.mjs + +import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(HERE, ".."); +const eventsPath = `${ROOT}/out/waltz-events.json`; +const rawPath = `${ROOT}/out/beat.f32.raw`; +const outPath = `${ROOT}/out/beat.mp3`; + +if (!existsSync(eventsPath)) { console.error(`✗ missing ${eventsPath}`); process.exit(1); } +const { bpm, totalSec } = JSON.parse(readFileSync(eventsPath, "utf8")); +const SR = 48000; +const beatSec = 60 / bpm; +const barSec = beatSec * 3; +const totalSamples = Math.ceil((totalSec + 1) * SR); +const out = new Float32Array(totalSamples); + +// Kick voice — 60 Hz sine, exponential pitch+amp decay (drum-like). +const kickDur = 0.18; +const kickGain = 0.35; +function placeKick(startSec) { + const start = Math.floor(startSec * SR); + const len = Math.floor(kickDur * SR); + for (let i = 0; i < len; i++) { + const dst = start + i; + if (dst < 0 || dst >= out.length) continue; + const t = i / SR; + // pitch sweep: 90Hz → 50Hz across the kick (helps it feel "round") + const f = 90 - 40 * (t / kickDur); + const env = Math.exp(-t / 0.06); // exponential amplitude decay + out[dst] += Math.sin(2 * Math.PI * f * t) * env * kickGain; + } +} + +// Place a kick on every bar's beat 1. +const nBars = Math.floor(totalSec / barSec); +for (let b = 0; b < nBars; b++) placeKick(b * barSec); + +// Soft clip / normalize to stay under -1. +let peak = 0; +for (let i = 0; i < out.length; i++) { + const a = Math.abs(out[i]); + if (a > peak) peak = a; +} +if (peak > 0.95) { + const k = 0.95 / peak; + for (let i = 0; i < out.length; i++) out[i] *= k; +} + +// Write float32 raw → ffmpeg → mp3 +const buf = Buffer.from(out.buffer); +writeFileSync(rawPath, buf); +const ff = "/opt/homebrew/opt/ffmpeg-full/bin/ffmpeg"; +const result = spawnSync(ff, [ + "-hide_banner", "-loglevel", "error", "-y", + "-f", "f32le", "-ar", String(SR), "-ac", "1", "-i", rawPath, + "-c:a", "libmp3lame", "-q:a", "4", + outPath, +], { stdio: "inherit" }); +if (result.status !== 0) { + console.error(`✗ ffmpeg encode failed`); + process.exit(result.status || 1); +} +console.log(`✓ ${outPath} · ${nBars} kicks · ${bpm} bpm`); diff --git a/recap/bin/build-filter.mjs b/recap/bin/build-filter.mjs --- a/recap/bin/build-filter.mjs +++ b/recap/bin/build-filter.mjs @@ -28,14 +28,12 @@ console.error("usage: build-filter.mjs "); process.exit(1); } -// Bottom chrome — piano fills y=1830..1920 flush bottom. Progress bar -// sits ABOVE the piano keys (NOT overlapping them) with a small gap. -// The PALS logo runs left-to-right ON TOP of the progress bar over the -// full episode, so the bar reads like a track and PALS is the runner. -const PROGRESS_Y = 1800; // 14 tall, sits y=1800..1814 — clear of piano (1830+) -const PROGRESS_H = 14; -const PALS_RUN_Y = 1700; // PALS bottom rests at y=1800 (= top of progress bar) -const PALS_RUN_H = 100; // 100×100 PNG sized at compose time +// Bottom chrome — piano flush bottom y=1830..1920. Progress bar sits +// FLUSH to the top of the piano keys (its bottom = piano top = 1830) +// with a solid full-width dark background "track" that the chapter +// segments fill into. +const PROGRESS_H = 16; +const PROGRESS_Y = 1830 - PROGRESS_H; // y=1814..1830, bottom flush with piano top const lines = []; // IMPORTANT: include `fps=25` here so the libass `subtitles=` filters @@ -46,9 +44,55 @@ // so animated overlays (waltz piano-roll) only flash on the rare frames // that happen to coincide with a slide transition. The earlier "do NOT // add fps" warning was about adding it AFTER the libass chain or to the // wrong stream — applied to [bg] up front, it's correct and necessary. -lines.push(`[0:v]format=yuv420p,scale=1080:1920,setsar=1,fps=25[bg]`); +// Inputs (per compose.fish): +// [0:v] photos.txt — raw jeffrey-photos at slide durations +// [1:v] chrome.txt — transparent chrome PNGs (chapter prompt / +// cap / QR / PALS) at slide durations +// [2:a] narration mp3 +// [3:a] waltz.mp3 (optional) +// [4:a] beat.mp3 (optional) +// +// PHOTO STREAM — handicam treatment. jeffrey-photos are 1024×1536 +// (2:3); the slide frame is 1080×1920 (9:16). We scale source up by +// height to 2112 PRESERVING ASPECT (avoiding the squashed look that +// hard-forcing 1188×2112 produced), then center-crop to a 1188×2112 +// canvas so the crop-pan can drift inside without exposing borders. +// Time-driven sine offsets on the inner crop produce a subtle hand- +// held camera shake on the portraits ONLY — chrome composited on top +// stays still. fps=50 + tmix=frames=12 + fps=25 gives ~0.24s temporal +// blend at chapter cuts. Final film grain (alls=14, temporal+uniform) +// breathes. +// Input #0 is photos.mov — pre-rendered in compose.fish to a continuous +// 1188×2112 yuv420p 25fps stream. The pre-render exists to dodge an +// ffmpeg deadlock: concat-demuxer-over-PNGs as a `-i` input alongside +// any audio `-i` AND the libass `subtitles=` filter downstream stalls +// at frame=0 forever (libass per-frame ticks starve the demuxer). +// Baking photos to a real video file fixes it. +// +// ffmpeg 8.1's crop filter has no `eval` option — x/y are re-evaluated +// per frame automatically because the option carries the `T` (timeline) +// flag. Setting `eval=frame` errors as "Option not found". +lines.push(`[0:v]crop=1080:1920:x='54+24*sin(t*0.6)+12*sin(t*1.7)':y='96+18*sin(t*0.4+1.2)+10*sin(t*1.3+0.5)'[bgPan]`); +lines.push(`[bgPan]tmix=frames=12,fps=25,noise=alls=14:allf=t+u[bgFilm]`); + +// CHROME STREAM — pre-rendered chrome.mov (qtrle, RGBA, 25fps, +// 1080x1920) created in compose.fish, read via `movie=` filter source +// (NOT a `-i` input). The latter deadlocks: ffmpeg's input scheduler +// stalls when the photo input is a concat-demuxer over slow PNGs AND +// the graph has libass `subtitles=` downstream — frame=0 forever. The +// `movie=` source bypasses the input scheduler entirely, so frames +// flow. +const CHROME_MOV = resolve(ROOT, "out/chrome.mov"); +const escMovie = CHROME_MOV.replace(/\\/g, "\\\\").replace(/:/g, "\\:"); +lines.push(`movie='${escMovie}',format=rgba,setsar=1[chr]`); + +// Composite chrome on top of the handicam'd photo. The chrome stays +// dead-still while the photo zooms and shakes underneath. +lines.push(`[bgFilm][chr]overlay=format=auto[v0]`); + +// Audio inputs renumbered: chrome moved from `-i` (slot 1) to a `movie=` +// filter source, so narration is now [1:a], waltz [2:a], beat [3:a]. lines.push(`[1:a]apad=whole_dur=${TOTAL}[a1]`); -lines.push(`[bg]null[v0]`); // Subtitles via libass — single filter pass, no pre-encode, no alpha // codec dance. fontsdir picks up the YWFT face shipped in the repo so // the .ass `Style: YWFTProcessing` resolves. The .ass path is escaped @@ -69,17 +113,33 @@ // the chapter starts (chapter-granular structure) AND a within-chapter // fill grows from 0 to the segment's full width across its duration. // Thin dark divider lines mark chapter boundaries. const segmentsPath = `${ROOT}/out/segments.json`; +const layoutsPath = `${ROOT}/out/layouts.json`; const segs = existsSync(segmentsPath) ? JSON.parse(readFileSync(segmentsPath, "utf8")).filter((s) => s.marker !== "__END__") : [{ name: "_default", startSec: 0, endSec: Number(TOTAL) }]; +// Per-segment chapter color from layouts.json — drives the progress +// bar fill hue so each chapter beat carries its own color. +const layoutsForColors = existsSync(layoutsPath) ? JSON.parse(readFileSync(layoutsPath, "utf8")) : {}; +function segColor(name) { + const c = layoutsForColors[name] && layoutsForColors[name].color; + if (c && c.hex) return c.hex.replace("#", "0x"); + return "0xff69b4"; // hot pink fallback +} const W = 1080; let prevTag = "v2"; +// Solid dark "track" background for the entire progress bar — sits +// behind the per-chapter segment fills so the bar shape reads even +// before any chapter has started. +lines.push(`[${prevTag}]drawbox=x=0:y=${PROGRESS_Y}:w=${W}:h=${PROGRESS_H}:color=0x0c1430@0.78:t=fill[ptrack]`); +prevTag = "ptrack"; for (let i = 0; i < segs.length; i++) { const s = segs[i]; const x0 = (s.startSec / Number(TOTAL)) * W; const segW = ((s.endSec - s.startSec) / Number(TOTAL)) * W; - const baseColor = ["0xff69b4@0.55", "0xff40a0@0.55", "0xff70d0@0.55", "0xff90c0@0.55", "0xff5099@0.55"][i % 5]; + const segHex = segColor(s.name); // 0xRRGGBB + const baseColor = `${segHex}@0.40`; // soft "passed-through" tint + const fillColor = segHex; // bright leading edge as time advances const nextTag = `prog${i}`; lines.push(`[${prevTag}]drawbox=x=${x0.toFixed(1)}:y=${PROGRESS_Y}:w=${segW.toFixed(1)}:h=${PROGRESS_H}:color=${baseColor}:t=fill:enable='gte(t,${s.startSec})'[${nextTag}]`); prevTag = nextTag; @@ -87,7 +147,7 @@ const fillTag = `pfill${i}`; const dur = s.endSec - s.startSec; const fillExpr = `min(${segW.toFixed(1)},${segW.toFixed(1)}*(t-${s.startSec})/${dur.toFixed(2)})`; - lines.push(`[${prevTag}]drawbox=x=${x0.toFixed(1)}:y=${PROGRESS_Y}:w='${fillExpr}':h=${PROGRESS_H}:color=0xff1493:t=fill:enable='between(t,${s.startSec},${s.endSec})'[${fillTag}]`); + lines.push(`[${prevTag}]drawbox=x=${x0.toFixed(1)}:y=${PROGRESS_Y}:w='${fillExpr}':h=${PROGRESS_H}:color=${fillColor}:t=fill:enable='between(t,${s.startSec},${s.endSec})'[${fillTag}]`); prevTag = fillTag; if (i < segs.length - 1) { diff --git a/recap/bin/compose.fish b/recap/bin/compose.fish --- a/recap/bin/compose.fish +++ b/recap/bin/compose.fish @@ -14,8 +14,15 @@ set -l ROOT (realpath (dirname (status -f))/..) set -l OUT $ROOT/out set -l TOTAL (cat $OUT/duration.txt) +# Prefer the sung version of the narration when sing.mjs has produced +# one — falls back to plain TTS otherwise. set -l AUDIO $OUT/recap.mp3 +if test -f $OUT/recap-sung.mp3 + set AUDIO $OUT/recap-sung.mp3 + echo " · using sung vocal: $OUT/recap-sung.mp3" +end set -l WALTZ $OUT/waltz.mp3 +set -l BEAT $OUT/beat.mp3 set -l SUBSASS $OUT/subs.ass set -l VIDEO $OUT/recap.mp4 set -l FILTER $OUT/filter.txt @@ -42,6 +49,54 @@ echo "✗ missing $SUBSASS — run bin/subtitle-track.mjs first" exit 1 end +# Pre-render photos.txt → photos.mov AND chrome.txt → chrome.mov. The +# main compose pass below uses libass `subtitles=` filter, which +# deadlocks ffmpeg's input scheduler at frame=0 forever when the photo +# input is a concat-demuxer over PNGs and there's any audio `-i` input +# (the per-frame libass ticks starve the demuxer). Baking the slide +# streams to continuous-frame mov files sidesteps it. Chrome.mov is +# read via `movie=` filter source (NOT `-i` input) because adding a +# *second* `-i` video input — even from a real video file — alongside +# concat-demuxer images also reliably deadlocks. + +# --- photos.mov: scaled to 1188×2112, sar=1, fps=25. Time-dependent +# crop/pan/shake/tmix/grain happens on top of this in the main pass. +set -l PHOTOS_MOV $OUT/photos.mov +set -l PHOTOS_HASH_FILE $OUT/photos.mov.hash +set -l PHOTOS_HASH (cat $OUT/photos.txt; for f in (grep "^file " $OUT/photos.txt | sed -E "s/^file '(.*)'\$/\1/"); shasum -a 256 $f; end | shasum -a 256 | awk '{print $1}') +if test -f $PHOTOS_MOV -a -f $PHOTOS_HASH_FILE -a "$PHOTOS_HASH" = (cat $PHOTOS_HASH_FILE 2>/dev/null) + echo "→ photos.mov cached (hash $PHOTOS_HASH) · skipping pre-render" +else + echo "→ pre-render photos.txt → photos.mov (h264, 1188×2112, fps=25)" + $FFMPEG -hide_banner -y -f concat -safe 0 -i $OUT/photos.txt \ + -vf "fps=25,format=yuv420p,scale=-2:2112,setsar=1,crop=1188:2112" \ + -c:v libx264 -preset ultrafast -crf 18 \ + $PHOTOS_MOV 2>&1 | tail -3 + or exit 1 + test -s $PHOTOS_MOV; or echo "✗ photos.mov failed to render"; or exit 1 + echo $PHOTOS_HASH > $PHOTOS_HASH_FILE +end + +# --- chrome.mov (qtrle, alpha preserved). +# +# Cache: hash chrome.txt + every PNG it lists. Skip the ~2 min qtrle +# encode unless any chrome PNG changed. +set -l CHROME_MOV $OUT/chrome.mov +set -l CHROME_HASH_FILE $OUT/chrome.mov.hash +set -l CHROME_HASH (cat $OUT/chrome.txt; for f in (grep "^file " $OUT/chrome.txt | sed -E "s/^file '(.*)'\$/\1/"); shasum -a 256 $f; end | shasum -a 256 | awk '{print $1}') +if test -f $CHROME_MOV -a -f $CHROME_HASH_FILE -a "$CHROME_HASH" = (cat $CHROME_HASH_FILE 2>/dev/null) + echo "→ chrome.mov cached (hash $CHROME_HASH) · skipping pre-render" +else + echo "→ pre-render chrome.txt → chrome.mov (qtrle alpha)" + $FFMPEG -hide_banner -y -f concat -safe 0 -i $OUT/chrome.txt \ + -vf "fps=25,format=rgba,scale=1080:1920,setsar=1" \ + -c:v qtrle \ + $CHROME_MOV 2>&1 | tail -3 + or exit 1 + test -s $CHROME_MOV; or echo "✗ chrome.mov failed to render"; or exit 1 + echo $CHROME_HASH > $CHROME_HASH_FILE +end + echo "→ ffmpeg compose · $TOTAL s · 1080x1920" # Build the filter graph in node. With the single-overlay subtitle track @@ -49,35 +104,54 @@ # the graph is short — three formatting filters on slides, an audio split # for the showwaves, and a final overlay of the subtitle stream. node $ROOT/bin/build-filter.mjs $TOTAL > $FILTER -# If a piano-waltz bed exists, append a mix into the same filter graph so -# we don't need a second -filter_complex flag (only the last one wins). -# Slides=0, narration=1, subs=2, waltz=3 — input order matters. +# Inputs (build-filter expects this exact order): +# 0 = photos.txt (raw jeffrey-photos at slide durations — gets the +# handicam crop / pan / shake / grain treatment) +# 1 = narration mp3 +# 2 = waltz.mp3 (optional) +# 3 = beat.mp3 (optional) +# Chrome.mov is read via `movie=` filter source inside the graph (NOT a +# `-i` input). Adding chrome as a `-i` input alongside the photos +# concat-demuxer + libass `subtitles=` filter deadlocks ffmpeg's input +# scheduler at frame=0 forever — `movie=` bypasses that scheduler. if test -f $WALTZ echo " + bed: $WALTZ (waltz, content-length, no loop)" - # waltz.mjs is auto-sized to the recap duration via out/duration.txt, - # so the bed plays once through and resolves with the narration. We - # apad to TOTAL to handle any tiny rounding gap at the tail, then atrim - # to the exact length. NO -stream_loop — looping the bed produced - # awkward seam jumps in the middle of the show. - printf ';[2:a]apad=whole_dur=%s,atrim=duration=%s,volume=0.42[bed];[a1][bed]amix=inputs=2:duration=first:dropout_transition=0:weights=1.0 0.55[mix]\n' "$TOTAL" "$TOTAL" >> $FILTER - $FFMPEG -hide_banner -y \ - -f concat -safe 0 -i $OUT/concat.txt \ - -i $AUDIO \ - -i $WALTZ \ - -filter_complex_script $FILTER \ - -map "[final]" -map "[mix]" \ - -c:v libx264 -preset ultrafast -crf 22 -pix_fmt yuv420p \ - -c:a aac -b:a 192k \ - -movflags +faststart \ - -t $TOTAL \ - $VIDEO + if test -f $BEAT + echo " + beat: $BEAT (kick on every waltz bar)" + printf ';[2:a]apad=whole_dur=%s,atrim=duration=%s,volume=0.42[bed];[3:a]apad=whole_dur=%s,atrim=duration=%s,volume=0.55[bk];[a1][bed][bk]amix=inputs=3:duration=first:dropout_transition=0:weights=1.0 0.55 0.45[mix]\n' "$TOTAL" "$TOTAL" "$TOTAL" "$TOTAL" >> $FILTER + $FFMPEG -hide_banner -y \ + -i $PHOTOS_MOV \ + -i $AUDIO \ + -i $WALTZ \ + -i $BEAT \ + -filter_complex_script $FILTER \ + -map "[final]" -map "[mix]" \ + -c:v libx264 -preset medium -crf 24 -maxrate 6000k -bufsize 12000k -tune grain -pix_fmt yuv420p \ + -c:a aac -b:a 192k \ + -movflags +faststart \ + -t $TOTAL \ + $VIDEO + else + printf ';[2:a]apad=whole_dur=%s,atrim=duration=%s,volume=0.42[bed];[a1][bed]amix=inputs=2:duration=first:dropout_transition=0:weights=1.0 0.55[mix]\n' "$TOTAL" "$TOTAL" >> $FILTER + $FFMPEG -hide_banner -y \ + -i $PHOTOS_MOV \ + -i $AUDIO \ + -i $WALTZ \ + -filter_complex_script $FILTER \ + -map "[final]" -map "[mix]" \ + -c:v libx264 -preset medium -crf 24 -maxrate 6000k -bufsize 12000k -tune grain -pix_fmt yuv420p \ + -c:a aac -b:a 192k \ + -movflags +faststart \ + -t $TOTAL \ + $VIDEO + end else $FFMPEG -hide_banner -y \ - -f concat -safe 0 -i $OUT/concat.txt \ + -i $PHOTOS_MOV \ -i $AUDIO \ -filter_complex_script $FILTER \ -map "[final]" -map "[a1]" \ - -c:v libx264 -preset ultrafast -crf 22 -pix_fmt yuv420p \ + -c:v libx264 -preset medium -crf 24 -maxrate 6000k -bufsize 12000k -tune grain -pix_fmt yuv420p \ -c:a aac -b:a 192k \ -movflags +faststart \ -t $TOTAL \ diff --git a/recap/bin/layout.mjs b/recap/bin/layout.mjs --- a/recap/bin/layout.mjs +++ b/recap/bin/layout.mjs @@ -29,6 +29,7 @@ process.exit(2); } const mod = await import(`${ROOT}/audience/${audienceName}.mjs`); +const audience = mod.audience || mod.default; const solveLayout = mod.solveLayout; if (typeof solveLayout !== "function") { console.error(`✗ audience '${audienceName}' does not export solveLayout — using default top placement`); @@ -43,12 +44,17 @@ for (const seg of segments) { const cvPath = `${ROOT}/out/cv/${seg.name}.json`; const cv = existsSync(cvPath) ? JSON.parse(readFileSync(cvPath, "utf8")) : null; const layout = solveLayout(cv); + // Pull the chapter color out of the audience config so downstream + // tools (subtitles.mjs, build-filter.mjs) can tint per-segment. + const slide = (audience && audience.slides && audience.slides[seg.name]) || null; + const colorAddress = slide && slide.colorAddress; layouts[seg.name] = { startSec: seg.startSec, endSec: seg.endSec, chapter: layout.chapter, subtitle: layout.subtitle, piano: layout.piano, + color: colorAddress || null, }; } diff --git a/recap/bin/nvidia-video.mjs b/recap/bin/nvidia-video.mjs new file mode 100644 --- /dev/null +++ b/recap/bin/nvidia-video.mjs @@ -0,0 +1,309 @@ +#!/usr/bin/env node +// nvidia-video — exercise NVIDIA NIM video gen on stills we already have. +// +// Two backends: +// --mode svd Stable Video Diffusion (sync). image → ~25-frame mp4. +// Input must be exactly 1024x576. We auto-crop a 1024x576 +// band out of the supplied still (default: top, where +// jeffrey's head usually is in the 1024x1536 portraits). +// --mode cosmos Cosmos predict1-7b text2world (async, NVCF). prompt → mp4. +// No image conditioning in this script — text only for now. +// +// Usage: +// node recap/bin/nvidia-video.mjs --mode svd \ +// --image recap/out/jeffrey-photos/01_title.png \ +// [--seed 42] [--cfg 1.8] [--crop top|center|bottom] \ +// [--out recap/out/nvidia-video/01_title.mp4] +// +// node recap/bin/nvidia-video.mjs --mode cosmos \ +// --prompt "a cinematic shot of ..." \ +// [--seed 4] [--out recap/out/nvidia-video/cosmos-test.mp4] + +import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { resolve, basename, dirname, join } from "node:path"; +import { tmpdir } from "node:os"; + +const NVIDIA_KEY = readFileSync( + "/Users/jas/aesthetic-computer/aesthetic-computer-vault/.env", + "utf8", +).match(/^NVIDIA_API_KEY=(\S+)/m)?.[1]; +if (!NVIDIA_KEY) { + console.error("✗ NVIDIA_API_KEY not in vault/.env"); + process.exit(1); +} + +const flags = {}; +for (let i = 2; i < process.argv.length; i++) { + const a = process.argv[i]; + if (a.startsWith("--")) flags[a.slice(2)] = process.argv[i + 1]; +} + +const mode = flags.mode || "svd"; +const outDefault = `recap/out/nvidia-video/${mode}-${Date.now()}.mp4`; +const out = resolve(flags.out || outDefault); +mkdirSync(dirname(out), { recursive: true }); + +if (mode === "svd") await runSvd(); +else if (mode === "cosmos") await runCosmos(); +else { + console.error(`unknown --mode ${mode}`); + process.exit(1); +} + +async function runSvd() { + const image = flags.image; + if (!image) { + console.error("✗ --image required for svd"); + process.exit(1); + } + const src = resolve(image); + if (!existsSync(src)) { + console.error(`✗ image not found: ${src}`); + process.exit(1); + } + + // Crop/resize to exactly 1024x576. SVD rejects anything else. + const cropped = join(tmpdir(), `svd-${Date.now()}-${basename(src)}`); + const cropMode = flags.crop || "top"; + cropTo1024x576(src, cropped, cropMode); + const bytes = readFileSync(cropped); + console.log(`→ cropped to 1024x576 (${cropMode}): ${cropped} (${bytes.length} B)`); + + // Inline base64 only works <200 KB. Bigger needs the NVCF asset upload path. + let imageField; + if (bytes.length < 195_000) { + imageField = `data:image/png;base64,${bytes.toString("base64")}`; + console.log(" using inline base64"); + } else { + const assetId = await uploadAsset(cropped, "image/png", "svd input"); + imageField = `data:image/png;asset_id,${assetId}`; + console.log(` uploaded asset_id=${assetId}`); + } + + const seed = Number.isInteger(+flags.seed) ? +flags.seed : 0; + const cfg = flags.cfg ? +flags.cfg : 1.8; + + console.log(`→ POST svd seed=${seed} cfg=${cfg}`); + const t0 = Date.now(); + const res = await fetch( + "https://ai.api.nvidia.com/v1/genai/stabilityai/stable-video-diffusion", + { + method: "POST", + headers: { + Authorization: `Bearer ${NVIDIA_KEY}`, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ + image: imageField, + seed, + cfg_scale: cfg, + motion_bucket_id: 127, + }), + }, + ); + const elapsed = ((Date.now() - t0) / 1000).toFixed(1); + if (!res.ok) { + console.error(`✗ svd ${res.status} after ${elapsed}s`); + console.error((await res.text()).slice(0, 600)); + process.exit(1); + } + const data = await res.json(); + const art = data.artifacts?.[0]; + if (!art?.base64) { + console.error("✗ no artifact in response", JSON.stringify(data).slice(0, 500)); + process.exit(1); + } + if (art.finishReason && art.finishReason !== "SUCCESS") { + console.error(`✗ finishReason=${art.finishReason} (safety filter?)`); + process.exit(1); + } + writeFileSync(out, Buffer.from(art.base64, "base64")); + console.log(`✓ svd ${elapsed}s → ${out}`); +} + +async function runCosmos() { + const prompt = flags.prompt; + if (!prompt) { + console.error("✗ --prompt required for cosmos"); + process.exit(1); + } + const seed = Number.isInteger(+flags.seed) ? +flags.seed : 4; + + // Cosmos predict1-7b uses Triton-style payload with a CLI-shaped command. + const url = + "https://ai.api.nvidia.com/v1/cosmos/nvidia/cosmos-predict1-7b"; + const body = { + inputs: [ + { + name: "command", + shape: [1], + datatype: "BYTES", + data: [ + `text2world --prompt="${prompt.replace(/"/g, '\\"')}" --seed=${seed}`, + ], + }, + ], + outputs: [{ name: "status", datatype: "BYTES", shape: [1] }], + }; + + console.log(`→ POST cosmos seed=${seed}`); + console.log(` prompt: ${prompt.slice(0, 120)}${prompt.length > 120 ? "…" : ""}`); + const t0 = Date.now(); + let res = await fetch(url, { + method: "POST", + headers: { + Authorization: `Bearer ${NVIDIA_KEY}`, + "Content-Type": "application/json", + Accept: "application/json", + "NVCF-POLL-SECONDS": "5", + }, + body: JSON.stringify(body), + }); + + let reqId = res.headers.get("nvcf-reqid"); + console.log(` initial status=${res.status} reqId=${reqId}`); + + while (res.status === 202) { + if (!reqId) { + console.error("✗ 202 without nvcf-reqid header"); + process.exit(1); + } + await new Promise((r) => setTimeout(r, 5000)); + res = await fetch( + `https://api.nvcf.nvidia.com/v2/nvcf/pexec/status/${reqId}`, + { + method: "GET", + headers: { + Authorization: `Bearer ${NVIDIA_KEY}`, + Accept: "application/json", + "NVCF-POLL-SECONDS": "5", + }, + redirect: "manual", + }, + ); + const elapsed = ((Date.now() - t0) / 1000).toFixed(0); + process.stdout.write(` poll ${elapsed}s status=${res.status}\n`); + reqId = res.headers.get("nvcf-reqid") || reqId; + } + + if (res.status === 302) { + const loc = res.headers.get("location"); + console.log(` ↪ redirect → ${loc?.slice(0, 100)}…`); + const zipRes = await fetch(loc); + const zipBuf = Buffer.from(await zipRes.arrayBuffer()); + const zipPath = out.replace(/\.mp4$/, ".zip"); + writeFileSync(zipPath, zipBuf); + console.log(` ✓ saved zip ${zipBuf.length} B → ${zipPath}`); + // Extract any .mp4 from the zip + const r = spawnSync("unzip", ["-o", "-d", dirname(out), zipPath], { + stdio: "inherit", + }); + if (r.status !== 0) console.error(" ⚠ unzip failed; inspect zip manually"); + return; + } + + if (!res.ok) { + console.error(`✗ cosmos ${res.status}`); + console.error((await res.text()).slice(0, 600)); + process.exit(1); + } + + // Some NVCF flows return JSON inline. + const ct = res.headers.get("content-type") || ""; + if (ct.includes("application/json")) { + const j = await res.json(); + console.log(" json response:", JSON.stringify(j).slice(0, 800)); + if (j.asset_url) { + const dl = await fetch(j.asset_url); + writeFileSync(out, Buffer.from(await dl.arrayBuffer())); + console.log(` ✓ ${out}`); + } + return; + } + + const buf = Buffer.from(await res.arrayBuffer()); + writeFileSync(out, buf); + console.log(` ✓ ${buf.length} B → ${out}`); +} + +function cropTo1024x576(src, dst, where /* top|center|bottom */) { + // Use sips: get height, then crop a 576-tall band, then enforce 1024 wide. + const info = spawnSync("sips", ["-g", "pixelWidth", "-g", "pixelHeight", src], { + encoding: "utf8", + }); + const w = +(/pixelWidth: (\d+)/.exec(info.stdout || "")?.[1] || 0); + const h = +(/pixelHeight: (\d+)/.exec(info.stdout || "")?.[1] || 0); + if (!w || !h) { + console.error("✗ sips could not read image dimensions"); + process.exit(1); + } + + // Step 1: scale long edge so the smaller dimension is at least 1024×576-fitting. + // Strategy: scale to width=1024, then crop a 576-tall band from the result. + const scaled = dst.replace(/\.png$/, ".scaled.png"); + spawnSync("sips", ["-z", String(Math.round((1024 / w) * h)), "1024", src, "--out", scaled], { + stdio: "ignore", + }); + const info2 = spawnSync("sips", ["-g", "pixelHeight", scaled], { encoding: "utf8" }); + const sh = +(/pixelHeight: (\d+)/.exec(info2.stdout || "")?.[1] || 0); + if (sh < 576) { + console.error(`✗ scaled height ${sh} < 576; image too wide for top crop`); + process.exit(1); + } + // Step 2: pick crop offset (sips --cropOffset is X Y, --crop is HEIGHT WIDTH). + let yOff; + if (where === "center") yOff = Math.floor((sh - 576) / 2); + else if (where === "bottom") yOff = sh - 576; + else yOff = 0; // top + + const r = spawnSync( + "sips", + [ + "--cropOffset", + "0", + String(yOff), + "-c", + "576", + "1024", + scaled, + "--out", + dst, + ], + { stdio: "ignore" }, + ); + if (r.status !== 0) { + console.error("✗ sips crop failed"); + process.exit(1); + } +} + +async function uploadAsset(path, contentType, description) { + // 1) ask for an upload URL + const auth = await fetch("https://api.nvcf.nvidia.com/v2/nvcf/assets", { + method: "POST", + headers: { + Authorization: `Bearer ${NVIDIA_KEY}`, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ contentType, description }), + }); + if (!auth.ok) { + console.error(`✗ asset auth ${auth.status}`, (await auth.text()).slice(0, 300)); + process.exit(1); + } + const { uploadUrl, assetId } = await auth.json(); + // 2) PUT the bytes + const put = await fetch(uploadUrl, { + method: "PUT", + headers: { "Content-Type": contentType, "x-amz-meta-nvcf-asset-description": description }, + body: readFileSync(path), + }); + if (!put.ok) { + console.error(`✗ asset put ${put.status}`, (await put.text()).slice(0, 300)); + process.exit(1); + } + return assetId; +} diff --git a/recap/bin/sing.mjs b/recap/bin/sing.mjs new file mode 100644 --- /dev/null +++ b/recap/bin/sing.mjs @@ -0,0 +1,264 @@ +#!/usr/bin/env node +// sing.mjs — generate an autotuned "sung" version of the recap +// narration using pop/bin/pitchsnap.mjs. +// +// Reads: +// recap/out/recap.mp3 — TTS narration (jeffrey-pvc) +// recap/out/words.json — whisper word alignment +// recap/out/waltz-events.json — bar / scale info for choosing notes +// +// Writes: +// recap/out/recap.np — generated .np score (one syllable per +// whisper word, melody from a simple +// 4-note cycle in the waltz's key) +// recap/out/recap-sung.mp3 — pitch-snapped vocal +// +// The melody walks G3 → B3 → A3 → D4 → repeating, transposed up 2 +// semitones during the waltz's lydian section and up 3 in dorian +// (matches audience.waltz.morph). Each word is one beat, snapped to +// the nearest 8th-note grid (light snap, preserves speech feel). +// +// Usage: node bin/sing.mjs + +import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(HERE, ".."); +const REPO = resolve(ROOT, ".."); + +const audienceName = process.argv[2]; +if (!audienceName) { console.error("usage: sing.mjs "); process.exit(2); } + +const wordsPath = `${ROOT}/out/words.json`; +// Prefer the stable whisper snapshot (transcribe.mjs writes this on +// each fresh run). Falls back to words.json — but words.json gets +// rewritten after sing, so reading from it on subsequent runs would +// feed us our own previous output and progressively eat words. +const whisperWordsPath = `${ROOT}/out/words.whisper.json`; +const inputWordsPath = existsSync(whisperWordsPath) ? whisperWordsPath : wordsPath; +const vocalPath = `${ROOT}/out/recap.mp3`; +const waltzPath = `${ROOT}/out/waltz-events.json`; +const npPath = `${ROOT}/out/recap.np`; +const outPath = `${ROOT}/out/recap-sung.mp3`; + +if (!existsSync(inputWordsPath)) { console.error(`✗ missing ${inputWordsPath} — run align.mjs first`); process.exit(1); } +if (!existsSync(vocalPath)) { console.error(`✗ missing ${vocalPath} — run tts.mjs first`); process.exit(1); } + +console.log(` ← reading words from ${inputWordsPath.replace(REPO + "/", "")} (${existsSync(whisperWordsPath) ? "stable whisper snapshot" : "current words.json"})`); +const words = JSON.parse(readFileSync(inputWordsPath, "utf8")); +const waltz = existsSync(waltzPath) ? JSON.parse(readFileSync(waltzPath, "utf8")) : null; +const BPM = (waltz && waltz.bpm) || 78; +const beatSec = 60 / BPM; +const barSec = beatSec * 3; + +// ── melody walk ──────────────────────────────────────────────────────── +// Major-pentatonic arpeggio pattern in jeffrey-pvc's baritone (~C3 +// ref). Each 8-step cycle traces a rise-and-fall arc so phrases have +// musical shape instead of looping monotone. +const BASE_CYCLE = ["G2", "B2", "D3", "G3", "D3", "B2", "A2", "G2"]; + +// Crude syllable counter — counts vowel clusters. Same idea pitchsnap +// uses internally, but mirrored here so OUR score tokens match what +// pitchsnap CLAIMS per word (otherwise words eat each other's pitches). +function syllableCount(word) { + if (!word) return 1; + const cleaned = word.toLowerCase().replace(/[^a-z]/g, ""); + if (!cleaned) return 1; + // Drop trailing silent 'e' (e.g., 'home', 'bone') + const stripped = cleaned.replace(/e$/, ""); + const groups = stripped.match(/[aeiouy]+/g) || []; + return Math.max(1, groups.length); +} + +// Split a word into roughly N syllable chunks for the .np score. +// Heuristic: split at vowel-cluster boundaries. Result tokens +// concatenate back to the original word. First chunk gets a trailing +// hyphen, middle chunks get hyphens both sides, last chunk gets a +// leading hyphen — same convention amazing.np uses. +function splitSyllables(word, n) { + if (n <= 1) return [word]; + const cleaned = word.toLowerCase(); + const matches = [...cleaned.matchAll(/[aeiouy]+[^aeiouy]*/g)]; + if (matches.length < n) return [word]; // fall back to single token + // Cluster adjacent vowel-groups into n chunks. + const chunks = []; + const per = matches.length / n; + let cursor = 0; + for (let k = 0; k < n; k++) { + const end = (k === n - 1) ? cleaned.length : matches[Math.min(matches.length - 1, Math.floor((k + 1) * per))].index; + const piece = cleaned.slice(cursor, end); + if (piece) chunks.push(piece); + cursor = end; + } + if (cursor < cleaned.length) chunks[chunks.length - 1] += cleaned.slice(cursor); + // Decorate with leading/trailing hyphens (pop convention). + return chunks.map((c, i) => { + let s = c; + if (i > 0) s = "-" + s; + if (i < chunks.length - 1) s = s + "-"; + return s; + }); +} +const NOTE_NAMES = ["C","C#","D","D#","E","F","F#","G","G#","A","A#","B"]; + +function noteToMidi(name) { + const m = name.match(/^([A-G])(#|b)?(\d+)$/); + if (!m) return 60; + const base = { C:0,D:2,E:4,F:5,G:7,A:9,B:11 }[m[1]]; + const acc = m[2] === "#" ? 1 : m[2] === "b" ? -1 : 0; + const oct = Number(m[3]); + return 12 * (oct + 1) + base + acc; +} +function midiToNote(midi) { + const oct = Math.floor(midi / 12) - 1; + const idx = midi % 12; + return `${NOTE_NAMES[idx]}${oct}`.replace("#", "#"); // keep sharps +} + +// Determine the transpose offset for a given start time, mapping into +// the audience.waltz.morph sections proportionally if available. +async function morphTransposeAt(secs) { + if (!waltz) return 0; + // Walk bars at the waltz BPM and figure out which morph section + // contains this second by cumulative bar count. + try { + const mod = await import(`${ROOT}/audience/${audienceName}.mjs`); + const audience = mod.audience || mod.default; + const morph = audience && audience.waltz && audience.waltz.morph; + if (!Array.isArray(morph) || morph.length === 0) return 0; + const totalBars = waltz.bars || (audience.waltz.bars) || Math.ceil((waltz.totalSec || 0) / barSec); + const totalWeight = morph.reduce((a, s) => a + (s.weight ?? 1), 0); + let cumBars = 0; + const barIdx = Math.floor(secs / barSec); + for (const s of morph) { + const bars = Math.max(1, Math.floor(((s.weight ?? 1) / totalWeight) * totalBars)); + cumBars += bars; + if (barIdx < cumBars) return Number(s.transpose ?? 0); + } + return Number(morph[morph.length - 1].transpose ?? 0); + } catch { return 0; } +} + +// ── build score lines ────────────────────────────────────────────────── +// One score line per phrase (long pause or sentence-end punctuation). +// Each WORD splits into its syllable count (matching pitchsnap's own +// `syllableCount()`), so the score has one token per syllable. This +// lets pitchsnap's syllable-aware claim-N-tokens-per-word logic work +// correctly — without it, multi-syllable words would steal pitches +// from following words. Last syllable of each phrase gets a long +// sustain (*4) so the phrase BREATHES — the single biggest thing +// that makes pitched speech read as "singing" instead of "metronome". +const PHRASE_END = /[.!?]$/; // sustains only on full-sentence ends +const SOFT_PHRASE_END = /[,—;:]$/; // commas → medium sustain +const PAUSE_MS = 350; +const SUSTAIN_HARD = 4; // *4 for full-sentence ends +const SUSTAIN_SOFT = 2; // *2 for commas / dashes +const lines = []; +let lineTokens = []; +let cycleIdx = 0; + +function flushLine(sustainAtEnd) { + if (!lineTokens.length) return; + if (sustainAtEnd && lineTokens.length) { + // Replace last token's *1 with *. + const last = lineTokens[lineTokens.length - 1]; + lineTokens[lineTokens.length - 1] = last.replace(/\*\d+$/, `*${sustainAtEnd}`); + } + lines.push(lineTokens.join(" ")); + lineTokens = []; +} + +let totalSyllables = 0; +for (let i = 0; i < words.length; i++) { + const w = words[i]; + const text = (w.text || "").trim(); + if (!text) continue; + // Strip outer punctuation but preserve a clean word for syllable splits. + const clean = text.replace(/[.!?,;:—()"'`]+$/g, "").replace(/^[—\-"'`]+/, ""); + if (!clean) continue; + const ns = syllableCount(clean); + const sylls = splitSyllables(clean, ns); + const transpose = await morphTransposeAt(w.fromMs / 1000); + + for (const syl of sylls) { + const cycleNote = BASE_CYCLE[cycleIdx % BASE_CYCLE.length]; + const midi = noteToMidi(cycleNote) + transpose; + const note = midiToNote(midi); + lineTokens.push(`${note}:${syl}*1`); + cycleIdx++; + totalSyllables++; + } + + // Phrase break on long pause OR sentence-end punctuation. + const next = words[i + 1]; + const longPause = next && next.fromMs - w.toMs > PAUSE_MS; + const hard = PHRASE_END.test(text) && lineTokens.length >= 3; + const soft = SOFT_PHRASE_END.test(text) && lineTokens.length >= 4; + if (hard || !next) flushLine(SUSTAIN_HARD); + else if (longPause || soft) flushLine(SUSTAIN_SOFT); +} +flushLine(SUSTAIN_HARD); +console.log(` syllable count: ${totalSyllables} across ${lines.length} phrases`); + +// ── write .np ────────────────────────────────────────────────────────── +const np = + `# auto-generated by recap/bin/sing.mjs from words.json + waltz events\n` + + `# bpm=${BPM} cycle=${BASE_CYCLE.join(" ")} ref-note=C3\n` + + `# sustains: hard sentence-end *${SUSTAIN_HARD}, soft pause *${SUSTAIN_SOFT}\n\n` + + `verse\n` + + lines.join("\n") + "\n"; +writeFileSync(npPath, np); +console.log(`✓ ${npPath} · ${totalSyllables} syllables · ${lines.length} phrases (with sustains)`); + +// ── invoke pitchsnap ─────────────────────────────────────────────────── +const pitchsnap = `${REPO}/pop/bin/pitchsnap.mjs`; +if (!existsSync(pitchsnap)) { console.error(`✗ missing ${pitchsnap}`); process.exit(1); } + +// Beat-mode singing: place each word at its cumulative-beat position +// from the score (not speech-time) so sustains (`*4`) actually hold. +// Without --beat-mode, sustains were ignored — words played out at +// natural speech rate with pitch overlay. With it, the score's beat +// values DRIVE the timeline. +console.log(`→ pitchsnap · vocal=${vocalPath.replace(REPO + "/", "")} score=${npPath.replace(REPO + "/", "")} bpm=${BPM} mode=beat curve=glide`); +const result = spawnSync("node", [ + pitchsnap, + "--vocal", vocalPath, + "--words", wordsPath, + "--score", npPath, + "--section", "verse", + "--bpm", String(BPM), + "--beat-mode", + "--curve", "glide", + "--ref-note", "C3", + "--out", outPath, +], { stdio: "inherit" }); + +if (result.status !== 0) { + console.error(`✗ pitchsnap exited ${result.status} — recap-sung.mp3 not produced`); + process.exit(result.status || 1); +} +console.log(`✓ ${outPath}`); + +// Re-emit words.json from the sung events.json so downstream steps +// (align, scout, slides, subtitles, waltz) see the stretched word +// timings and rebuild segments / chapter durations correctly. We +// preserve the original under words.original.json for reference. +const eventsJsonPath = `${ROOT}/out/recap-sung.events.json`; +if (existsSync(eventsJsonPath)) { + const ev = JSON.parse(readFileSync(eventsJsonPath, "utf8")); + if (Array.isArray(ev.events) && ev.events.length) { + if (!existsSync(`${ROOT}/out/words.original.json`) && existsSync(wordsPath)) { + writeFileSync(`${ROOT}/out/words.original.json`, readFileSync(wordsPath)); + } + const sungWords = ev.events.map((e) => ({ + text: e.text, + fromMs: Math.round(e.snappedStart * 1000), + toMs: Math.round((e.snappedStart + e.durSec) * 1000), + })); + writeFileSync(wordsPath, JSON.stringify(sungWords, null, 2)); + console.log(`✓ ${wordsPath} ← rewritten from sung events (${sungWords.length} words, total ${ev.totalDur.toFixed(2)}s)`); + } +} diff --git a/recap/bin/slides.mjs b/recap/bin/slides.mjs --- a/recap/bin/slides.mjs +++ b/recap/bin/slides.mjs @@ -175,22 +175,70 @@ await new Promise((r) => setTimeout(r, 400)); await new Promise((r) => setTimeout(r, 200)); const png = await page.screenshot({ type: "png", omitBackground: false }); writeFileSync(`${SLIDE_DIR}/${name}.png`, png); + + // Chrome-only sibling — same body but with the full-bleed photo + // stripped + transparent background. Used by compose.fish to + // overlay chrome on top of a separately-shaken photo stream so the + // chapter prompt / QR / PALS watermarks stay still while the + // portrait below them does the handicam crop. + const chromeBody = body.replace(/]*object-fit:\s*cover[^>]*\/?\s*>/g, ""); + const chromeHtml = `${chromeBody}`; + await page.setContent(chromeHtml, { waitUntil: "domcontentloaded", timeout: 90000 }); + await new Promise((r) => setTimeout(r, 200)); + const chromePng = await page.screenshot({ type: "png", omitBackground: true }); + mkdirSync(`${ROOT}/out/chrome`, { recursive: true }); + writeFileSync(`${ROOT}/out/chrome/${name}.png`, chromePng); + await page.close(); const seg = segments.find((s) => s.name === name); - console.log(`✓ ${name}.png · ${seg.durationSec.toFixed(2)}s (${seg.startSec}s → ${seg.endSec}s)`); + console.log(`✓ ${name}.png + chrome/${name}.png · ${seg.durationSec.toFixed(2)}s (${seg.startSec}s → ${seg.endSec}s)`); } await browser.close(); -// concat.txt with real durations +// concat.txt with real durations (full composited slides — kept for +// backward-compat / debugging). compose.fish now uses the photo + +// chrome split below. const lines = []; for (const seg of segments) { lines.push(`file '${SLIDE_DIR}/${seg.name}.png'`); lines.push(`duration ${seg.durationSec}`); } -// concat demuxer needs the last file repeated without duration for proper end lines.push(`file '${SLIDE_DIR}/${segments[segments.length - 1].name}.png'`); writeFileSync(`${ROOT}/out/concat.txt`, lines.join("\n") + "\n"); +// photos.txt — raw jeffrey-photos for chapters (the camera-shake +// stream operates on these), full end-slide for the static end card. +// A 1080×1920 black placeholder is used for any segment without a +// jeffrey-photos sibling. +import { spawnSync } from "node:child_process"; +const photosDir = `${ROOT}/out/jeffrey-photos`; +const blackPath = `${ROOT}/out/black-1080x1920.png`; +if (!existsSync(blackPath)) { + const ff = "/opt/homebrew/opt/ffmpeg-full/bin/ffmpeg"; + spawnSync(ff, ["-hide_banner","-loglevel","error","-y","-f","lavfi","-i","color=c=black:s=1080x1920:d=1","-frames:v","1",blackPath], { stdio: "inherit" }); +} +function photoFor(seg) { + const candidate = `${photosDir}/${seg.name}.png`; + if (existsSync(candidate)) return candidate; + return blackPath; +} +const photoLines = []; +for (const seg of segments) { + photoLines.push(`file '${photoFor(seg)}'`); + photoLines.push(`duration ${seg.durationSec}`); +} +photoLines.push(`file '${photoFor(segments[segments.length - 1])}'`); +writeFileSync(`${ROOT}/out/photos.txt`, photoLines.join("\n") + "\n"); + +// chrome.txt — the chrome PNGs for each slide. +const chromeLines = []; +for (const seg of segments) { + chromeLines.push(`file '${ROOT}/out/chrome/${seg.name}.png'`); + chromeLines.push(`duration ${seg.durationSec}`); +} +chromeLines.push(`file '${ROOT}/out/chrome/${segments[segments.length - 1].name}.png'`); +writeFileSync(`${ROOT}/out/chrome.txt`, chromeLines.join("\n") + "\n"); + const total = segments[segments.length - 1].endSec; writeFileSync(`${ROOT}/out/duration.txt`, String(total)); -console.log(`✓ ${ROOT}/out/concat.txt · total ${total}s`); +console.log(`✓ ${ROOT}/out/concat.txt + photos.txt + chrome.txt · total ${total}s`); diff --git a/recap/bin/subtitle-track.mjs b/recap/bin/subtitle-track.mjs --- a/recap/bin/subtitle-track.mjs +++ b/recap/bin/subtitle-track.mjs @@ -19,7 +19,7 @@ // drawing primitives (BorderStyle=4 = opaque box). // // Usage: node bin/subtitle-track.mjs [audience-name] -import { readFileSync, writeFileSync } from "node:fs"; +import { readFileSync, writeFileSync, existsSync } from "node:fs"; import { resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; @@ -28,6 +28,27 @@ const ROOT = resolve(HERE, ".."); const audienceName = process.argv[2] || "fia"; const subsPath = `${ROOT}/out/subs.json`; const assPath = `${ROOT}/out/subs.ass`; +const layoutsPath = `${ROOT}/out/layouts.json`; +const segmentsPath = `${ROOT}/out/segments.json`; + +// Per-chunk chapter color from layouts.json + segments.json — overrides +// the libass outline color per Dialogue line so the pill border follows +// the slide color story instead of being fixed magenta. +const layoutsForColors = existsSync(layoutsPath) ? JSON.parse(readFileSync(layoutsPath, "utf8")) : {}; +const segmentsForColors = existsSync(segmentsPath) ? JSON.parse(readFileSync(segmentsPath, "utf8")) : []; +function chapterColorAt(secs) { + for (const seg of segmentsForColors) { + if (secs >= seg.startSec && secs < seg.endSec) { + const layout = layoutsForColors[seg.name]; + if (layout && layout.color && layout.color.rgb) { + const [r, g, b] = layout.color.rgb; + return `&H${b.toString(16).padStart(2,"0")}${g.toString(16).padStart(2,"0")}${r.toString(16).padStart(2,"0")}&`; + } + return null; + } + } + return null; +} const subs = JSON.parse(readFileSync(subsPath, "utf8")); if (!subs.length) { @@ -88,7 +109,12 @@ .replace(/\{/g, "(") .replace(/\}/g, ")") .trim(); if (!text) continue; - lines.push(`Dialogue: 0,${assTime(c.startSec)},${assTime(c.endSec)},Default,,0,0,0,,${text}`); + // Per-segment chapter color override on the outline (\3c). Falls back + // to the Default style's outline (magenta) when the chunk lands + // outside any segment or before the layout step ran. + const chapHex = chapterColorAt(c.startSec); + const colorOverride = chapHex ? `{\\3c${chapHex}}` : ""; + lines.push(`Dialogue: 0,${assTime(c.startSec)},${assTime(c.endSec)},Default,,0,0,0,,${colorOverride}${text}`); } writeFileSync(assPath, lines.join("\n") + "\n"); diff --git a/recap/bin/subtitles.mjs b/recap/bin/subtitles.mjs --- a/recap/bin/subtitles.mjs +++ b/recap/bin/subtitles.mjs @@ -111,8 +111,22 @@ } } return PILL_Y_DEFAULT; } +// Per-segment chapter color → tints the pill's border + accent text so +// the subtitle reads as a continuation of the chapter color story. +function colorAt(startSec) { + for (const seg of segments) { + if (startSec >= seg.startSec && startSec < seg.endSec) { + const layout = layouts[seg.name]; + if (layout && layout.color && layout.color.hex) return layout.color; + break; + } + } + return { hex: "#ff69b4", brightHex: "#ff89d6", rgb: [255, 105, 180] }; +} +function rgbaStr(rgb, a) { return `rgba(${rgb[0]},${rgb[1]},${rgb[2]},${a})`; } -function buildCss(pillY) { +function buildCss(pillY, color) { + const borderRgba = rgbaStr(color.rgb, 0.65); return ` @font-face { font-family: 'ProcessingB'; @@ -133,9 +147,9 @@ justify-content: center; padding: 0 60px; } .pill { - background: rgba(16, 8, 32, 0.72); + background: rgba(16, 8, 32, 0.78); backdrop-filter: blur(2px); - border: 3px solid rgba(255, 105, 180, 0.55); + border: 3px solid ${borderRgba}; border-radius: 14px; padding: 18px 40px; max-width: 100%; @@ -148,7 +162,7 @@ letter-spacing: -1px; text-shadow: 0 2px 0 rgba(0,0,0,0.5); word-wrap: break-word; } -.pill em { font-style: normal; color: #ff70d0; } +.pill em { font-style: normal; color: ${color.brightHex || color.hex}; } `; } @@ -162,7 +176,8 @@ for (let i = 0; i < chunks.length; i++) { const c = chunks[i]; const file = `${SUB_DIR}/${String(i).padStart(3, "0")}.png`; const pillY = pillYAt(c.startMs / 1000); - const css = buildCss(pillY); + const color = colorAt(c.startMs / 1000); + const css = buildCss(pillY, color); const page = await browser.newPage(); await page.setViewport({ width: FRAME_W, height: FRAME_H, deviceScaleFactor: 1 }); const html = `
${escapeHtml(c.text)}
`; diff --git a/recap/bin/timeline.py b/recap/bin/timeline.py new file mode 100644 --- /dev/null +++ b/recap/bin/timeline.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +# timeline.py — recap visual-score PNG: words vs waltz beats vs audio. +# eye-validation: are subtitles ELONGATED to match held narration, or +# do they pop in/out as instant flashes? mirrors pop/bin/timeline.py. +# +# four panels top→bottom, sharing the time axis: +# 1. WORDS — subs.json windows, colored by duration class. +# short/medium/long/held — instant-pop words show red, +# sustained ones show mint. text rendered INSIDE box. +# 2. WALTZ — waltz-events.json drum/bell events on a midi y-axis, +# downbeats highlighted; bar-line ticks across width. +# 3. AUDIO — waltz.mp3 waveform + word-onset ticks (cyan). +# +# usage: +# .venv/bin/python recap/bin/timeline.py \ +# --subs recap/out/subs.json \ +# --events recap/out/waltz-events.json \ +# --audio recap/out/waltz.mp3 \ +# --out ~/Desktop/recap-timing.png + +import argparse, json, os, sys +import numpy as np +import matplotlib.pyplot as plt +import matplotlib.patches as mpatches +import matplotlib.patheffects as pe + +BG = "#0a0a14" +PANEL_BG = "#10101e" +GRID = "#1f1f30" +FG = "#f3f0d8" +DIM = "#7f7d68" +ACCENT = "#5fe8b8" # mint — long-held words / downbeat +GREEN = "#7fe070" # sustained +ORANGE = "#ff8a3d" # short +RED = "#ff5566" # instant-pop (under-elongated) +CYAN = "#5fd0ff" # waveform +PURPLE = "#c87dff" # melody (midi >= 60) +YELLOW = "#ffd84d" # downbeat (midi 36) + +plt.rcParams.update({ + "figure.facecolor": BG, + "axes.facecolor": PANEL_BG, + "axes.edgecolor": DIM, + "axes.labelcolor": FG, + "axes.titlecolor": FG, + "xtick.color": FG, + "ytick.color": FG, + "text.color": FG, + "font.family": "monospace", + "font.monospace": ["Menlo", "DejaVu Sans Mono", "Consolas", "Courier New"], + "font.weight": "bold", + "axes.labelweight": "bold", + "axes.titleweight": "bold", + "axes.linewidth": 1.4, + "xtick.major.width": 1.4, + "ytick.major.width": 1.4, + "xtick.major.size": 6, + "ytick.major.size": 6, +}) + +def stroke(width=3, fg=BG): + return [pe.withStroke(linewidth=width, foreground=fg)] + +def duration_color(dur): + if dur < 0.30: return RED # instant — bad: not elongated + if dur < 1.00: return ORANGE # short + if dur < 3.00: return GREEN # sustained — good + return ACCENT # held / drawn-out + +def duration_class(dur): + if dur < 0.30: return "instant" + if dur < 1.00: return "short" + if dur < 3.00: return "sustained" + return "held" + +def midi_label(m): + if m is None: return "" + octave = m // 12 - 1 + pitch = ["C","C#","D","D#","E","F","F#","G","G#","A","A#","B"][m % 12] + return f"{pitch}{octave}" + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--subs", default="recap/out/subs.json") + ap.add_argument("--events", default="recap/out/waltz-events.json") + ap.add_argument("--audio", default="recap/out/waltz.mp3") + ap.add_argument("--out", required=True) + ap.add_argument("--title", default=None) + ap.add_argument("--max-width", type=float, default=80.0, + help="cap on figure width in inches") + args = ap.parse_args() + + subs = json.load(open(args.subs)) + waltz = json.load(open(args.events)) + events = waltz.get("events", waltz) if isinstance(waltz, dict) else waltz + bpm = waltz.get("bpm", 78) if isinstance(waltz, dict) else 78 + beat_sec = waltz.get("beatSec", 60.0 / bpm) if isinstance(waltz, dict) else 60.0 / bpm + total = float(waltz.get("totalSec", max(s["endSec"] for s in subs) + 5)) + + # ── word duration stats ────────────────────────────────────────── + durs = [s["endSec"] - s["startSec"] for s in subs] + n = len(durs) + counts = {"instant":0, "short":0, "sustained":0, "held":0} + for d in durs: counts[duration_class(d)] += 1 + pct = {k: 100.0*v/max(1,n) for k,v in counts.items()} + mean_d = sum(durs)/max(1,n) + median_d = sorted(durs)[n//2] if n else 0 + + title = args.title or ( + f"RECAP TIMING {bpm} BPM {total:.0f}s {n} words " + f"mean {mean_d:.2f}s · median {median_d:.2f}s " + f"red {counts['instant']} orange {counts['short']} " + f"green {counts['sustained']} mint {counts['held']}" + ) + + # ── waveform load ──────────────────────────────────────────────── + try: + import librosa + y, sr = librosa.load(args.audio, sr=22050, mono=True) + print(f" audio: {args.audio} ({len(y)/sr:.1f}s)") + except Exception as e: + print(f" ! librosa failed: {e}") + y, sr = None, None + + # ── figure layout ──────────────────────────────────────────────── + panels_spec = [("words", 4.0), ("waltz", 2.6), ("audio", 2.4)] + height_ratios = [h for _, h in panels_spec] + fig_w = min(args.max_width, max(40.0, total * 0.18)) + fig_h = sum(height_ratios) * 1.05 + 0.6 + fig, axs = plt.subplots( + len(panels_spec), 1, figsize=(fig_w, fig_h), + gridspec_kw={"height_ratios": height_ratios, "hspace": 0.16}, + sharex=True, + ) + kind_to_ax = {kind: axs[i] for i, (kind, _) in enumerate(panels_spec)} + fig.suptitle(title, fontsize=24, fontweight="bold", color=FG, y=0.992, + path_effects=stroke(4)) + + # ── PANEL 1: WORDS ─────────────────────────────────────────────── + ax_w = kind_to_ax["words"] + # vertical-jitter rows so adjacent boxes don't visually collide + ROWS = 3 + for i, s in enumerate(subs): + x0 = s["startSec"]; x1 = s["endSec"]; dur = x1 - x0 + row = i % ROWS + y_low = 0.05 + row * 0.31 + col = duration_color(dur) + ax_w.add_patch(mpatches.FancyBboxPatch( + (x0, y_low), max(0.04, x1 - x0), 0.28, + boxstyle="round,pad=0.01,rounding_size=0.04", + facecolor=col, alpha=0.55, edgecolor=col, + linewidth=1.4, zorder=2)) + # text — size depends on box width-in-pixels + bar_px = (x1 - x0) / total * fig_w * 72 + size = 13 if bar_px > 28 else (10 if bar_px > 14 else 7) + ax_w.text((x0 + x1)/2, y_low + 0.14, s["text"], + ha="center", va="center", fontsize=size, + fontweight="bold", color=FG, zorder=3, + path_effects=stroke(2)) + # duration tag for the longer ones + if dur >= 1.5 and bar_px > 60: + ax_w.text((x0+x1)/2, y_low + 0.02, f"{dur:.1f}s", + ha="center", va="bottom", fontsize=8, + color=BG, zorder=4) + ax_w.set_yticks([]) + ax_w.set_ylim(0, 1) + ax_w.set_ylabel("WORDS", fontsize=18, labelpad=14, color=GREEN) + ax_w.tick_params(axis="x", labelsize=12, length=5) + ax_w.grid(axis="x", color=GRID, linestyle="-", linewidth=0.7) + ax_w.set_axisbelow(True) + # legend swatches + legend_items = [ + mpatches.Patch(color=RED, label=f"instant <0.3s ({counts['instant']})"), + mpatches.Patch(color=ORANGE, label=f"short <1.0s ({counts['short']})"), + mpatches.Patch(color=GREEN, label=f"sustained <3s ({counts['sustained']})"), + mpatches.Patch(color=ACCENT, label=f"held ≥3.0s ({counts['held']})"), + ] + ax_w.legend(handles=legend_items, loc="upper right", fontsize=11, + frameon=True, facecolor=PANEL_BG, edgecolor=DIM, + labelcolor=FG, ncol=4) + + # ── PANEL 2: WALTZ EVENTS ──────────────────────────────────────── + ax_b = kind_to_ax["waltz"] + midis = [int(e["midi"]) for e in events] + midi_min, midi_max = min(midis) - 1, max(midis) + 1 + for e in events: + x = e["startSec"]; w = e.get("durSec", beat_sec) + m = int(e["midi"]) + if m <= 40: + col = YELLOW # bass / downbeat + alpha = 0.55 + elif m <= 64: + col = CYAN + alpha = 0.42 + else: + col = PURPLE + alpha = 0.42 + ax_b.add_patch(mpatches.Rectangle( + (x, m - 0.42), w, 0.84, + facecolor=col, alpha=alpha, + edgecolor=col, linewidth=0.6, zorder=2)) + # bar lines (every 3 beats — waltz) + bar = 3 * beat_sec + t = 0 + while t < total: + ax_b.axvline(t, color=DIM, linewidth=0.7, alpha=0.5, zorder=1) + t += bar + ax_b.set_ylim(midi_min, midi_max) + ax_b.set_yticks([midi_min, (midi_min+midi_max)//2, midi_max]) + ax_b.set_yticklabels([midi_label(m) for m in + [midi_min, (midi_min+midi_max)//2, midi_max]], + fontsize=10) + ax_b.set_ylabel("WALTZ", fontsize=18, labelpad=14, color=PURPLE) + ax_b.tick_params(axis="x", labelsize=12, length=5) + ax_b.grid(axis="x", color=GRID, linestyle="-", linewidth=0.7) + ax_b.set_axisbelow(True) + ax_b.text(0.995, 0.94, + f"yellow=bass cyan=mid purple=high bar={bar:.2f}s ({bpm} BPM 3/4)", + transform=ax_b.transAxes, ha="right", va="top", + fontsize=10, color=FG, + bbox=dict(facecolor=BG, edgecolor=DIM, + boxstyle="round,pad=0.35")) + + # ── PANEL 3: AUDIO + word onsets ───────────────────────────────── + ax_a = kind_to_ax["audio"] + if y is not None: + # downsample for fast plotting + step = max(1, len(y) // 80000) + t = np.arange(0, len(y), step) / sr + ys = y[::step] + ax_a.plot(t, ys, color=CYAN, linewidth=0.5, alpha=0.85, zorder=2) + ax_a.fill_between(t, 0, ys, color=CYAN, alpha=0.18, zorder=2) + for s in subs: + ax_a.axvline(s["startSec"], color=GREEN, + linewidth=0.7, alpha=0.45, zorder=3) + ax_a.plot([], [], color=GREEN, linewidth=2.5, label="word start") + ax_a.plot([], [], color=CYAN, linewidth=2.5, label="waltz audio") + ax_a.legend(loc="upper right", fontsize=11, frameon=True, + facecolor=PANEL_BG, edgecolor=DIM, labelcolor=FG) + ax_a.set_xlim(0, total) + ax_a.set_ylim(-1.05, 1.05) + ax_a.set_xlabel("TIME (s)", fontsize=14, labelpad=8) + ax_a.set_ylabel("AUDIO", fontsize=18, labelpad=14, color=CYAN) + ax_a.tick_params(axis="x", labelsize=12, length=5) + ax_a.tick_params(axis="y", labelsize=10) + ax_a.grid(axis="x", color=GRID, linestyle="-", linewidth=0.7) + ax_a.set_axisbelow(True) + + plt.tight_layout(rect=[0.01, 0.005, 0.995, 0.97]) + plt.savefig(args.out, dpi=110, bbox_inches="tight", + facecolor=BG, edgecolor="none") + print(f" ✓ {args.out} ({fig_w:.0f}x{fig_h:.1f}in)") + +if __name__ == "__main__": + main() diff --git a/recap/bin/transcribe.mjs b/recap/bin/transcribe.mjs --- a/recap/bin/transcribe.mjs +++ b/recap/bin/transcribe.mjs @@ -64,5 +64,8 @@ .map((s) => ({ text: s.text.trim(), fromMs: s.offsets.from, toMs: s.offsets.to })) .filter((w) => w.text.length > 0); writeFileSync(wordsPath, JSON.stringify(words, null, 2)); +// Stable snapshot of fresh whisper output — sing.mjs reads from this +// so its score never gets fed back its own previous output. +writeFileSync(`${ROOT}/out/words.whisper.json`, JSON.stringify(words, null, 2)); writeFileSync(hashFile, inputHash + "\n"); console.log(`✓ ${wordsPath} · ${words.length} words · ${(words[words.length - 1].toMs / 1000).toFixed(2)}s · hash ${inputHash}`); diff --git a/recap/bin/trap.mjs b/recap/bin/trap.mjs new file mode 100644 --- /dev/null +++ b/recap/bin/trap.mjs @@ -0,0 +1,532 @@ +#!/usr/bin/env node +// trap.mjs — render a 4/4 trap bed for the pop/big-pictures lane. +// +// Bottom-up: drums are AC-native. Each drum's voice stack is read from +// `system/public/aesthetic.computer/lib/percussion.mjs` (the same kit web +// notepat plays through the AudioContext and fedac/native notepat plays +// through audio.c). We mock `sound.synth` as a node-side buffer mixer — +// the synthesis recipes themselves come from AC's percussion module +// verbatim, so the composition is identical to what the C path produces. +// +// Harmonic content (bass, chord stabs, sparse melody) reuses the piano +// sample bank + sinebells synth from waltz.mjs. +// +// Patterns: 16-step grids per bar (4/4, 16th-note resolution). Library +// adapted from artery/test-hiphop.mjs. +// +// Usage: +// node bin/trap.mjs # default +// node bin/trap.mjs jeffrey-24h-2026-05-01 # named audience +// node bin/trap.mjs --style trap --bars 16 --bpm 140 \ +// --voice sinebells --scale minor --seed plork-test \ +// --out ~/Desktop/trap-test.mp3 + +import { + readFileSync, + writeFileSync, + existsSync, + mkdirSync, + unlinkSync, +} from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; +import { homedir } from "node:os"; + +import { playPercussion } from "../../system/public/aesthetic.computer/lib/percussion.mjs"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(HERE, ".."); +const REPO = resolve(ROOT, ".."); + +// ── parse args ───────────────────────────────────────────────────────── +const argv = process.argv.slice(2); +const flags = {}; +const positional = []; +for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a.startsWith("--")) { + const key = a.slice(2); + const next = argv[i + 1]; + if (next !== undefined && !next.startsWith("--")) { + flags[key] = next; + i++; + } else { + flags[key] = true; + } + } else { + positional.push(a); + } +} +const audienceName = positional[0] || null; + +function expandHome(p) { + if (!p || typeof p !== "string") return p; + if (p === "~") return homedir(); + if (p.startsWith("~/")) return resolve(homedir(), p.slice(2)); + return p; +} + +// ── load audience config (optional) ─────────────────────────────────── +let audience = null; +let T = {}; +if (audienceName) { + const mod = await import(`${ROOT}/audience/${audienceName}.mjs`); + audience = mod.audience; + T = audience.trap || {}; +} + +const STYLE = flags.style || T.style || "trap"; +const VOICE = flags.voice || T.voice || "sinebells"; +const SEED_STR = flags.seed || T.seed || (audience?.name ?? audienceName ?? "trap-default"); +const BPM = Number(flags.bpm ?? T.bpm ?? 140); +const SCALE_NAME = flags.scale || T.scale || "minor"; +const PROGRESSION = parseProgression(flags.progression) || T.progression || [0, 5, 3, 4]; // i VI iv V +const VOICE_GAIN = Number(flags.gain ?? T.voiceGain ?? 0.18); +const DRUM_GAIN = Number(flags["drum-gain"] ?? T.drumGain ?? 0.85); +const DENSITY = Number(flags.density ?? T.density ?? 0.5); +const ROOT_OFFSET = Number(flags.transpose ?? T.transpose ?? 0); +const OUT_PATH = expandHome(flags.out) || `${ROOT}/out/trap.mp3`; + +const _beatSecPre = 60 / BPM; +const _barSecPre = _beatSecPre * 4; // 4/4 +let DURATION_SEC = null; +if (flags.duration !== undefined) DURATION_SEC = Number(flags.duration); +else if (T.duration !== undefined) DURATION_SEC = Number(T.duration); +const BARS = (() => { + if (flags.bars !== undefined) return Number(flags.bars); + if (T.bars !== undefined && DURATION_SEC === null) return Number(T.bars); + if (DURATION_SEC !== null) return Math.max(1, Math.ceil(DURATION_SEC / _barSecPre)); + return 16; +})(); + +function parseProgression(s) { + if (!s || s === true) return null; + return s.split(",").map((x) => Number(x.trim())); +} + +const SAMPLE_RATE = 48_000; + +// ── deterministic PRNG seeded by audience/style ─────────────────────── +function hashString(s) { + let h = 2166136261 >>> 0; + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i); + h = Math.imul(h, 16777619); + } + return h >>> 0; +} +function makeRng(seedStr) { + let s = hashString(seedStr) || 1; + return () => { + s ^= s << 13; s >>>= 0; + s ^= s >>> 17; s >>>= 0; + s ^= s << 5; s >>>= 0; + return (s >>> 0) / 0xffffffff; + }; +} +const rng = makeRng(SEED_STR); +// Separate RNG for noise sampling so per-hit noise stays deterministic +// without consuming entropy from the compositional rng. +const noiseRng = makeRng(SEED_STR + ":noise"); + +// ── musical theory ───────────────────────────────────────────────────── +const SCALES = { + major: [0, 2, 4, 5, 7, 9, 11], + minor: [0, 2, 3, 5, 7, 8, 10], + dorian: [0, 2, 3, 5, 7, 9, 10], + lydian: [0, 2, 4, 6, 7, 9, 11], +}; +const SCALE = SCALES[SCALE_NAME] || SCALES.minor; +const ROOT_MIDI = 60 + ROOT_OFFSET; + +function scaleNoteMidi(degree, octaveOffset = 0) { + const len = SCALE.length; + const idx = ((degree % len) + len) % len; + const octShift = Math.floor(degree / len); + return ROOT_MIDI + 12 * (octaveOffset + octShift) + SCALE[idx]; +} +function chordMidis(rootDegree, octaveOffset = 0) { + return [ + scaleNoteMidi(rootDegree, octaveOffset), + scaleNoteMidi(rootDegree + 2, octaveOffset), + scaleNoteMidi(rootDegree + 4, octaveOffset), + ]; +} + +// ── 16-step beat patterns (1 bar = 16 sixteenth-notes) ──────────────── +// Adapted from artery/test-hiphop.mjs. Each pattern: kick / snare / +// closed-hat / open-hat. 1 = hit, 0 = rest. +const BEAT_PATTERNS = { + trap: { + kick: [1,0,0,0, 0,0,1,0, 0,0,1,0, 0,0,0,0], + snare: [0,0,0,0, 1,0,0,0, 0,0,0,0, 1,0,0,0], + hat: [1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1], + hatOpen: [0,0,0,0, 0,0,0,1, 0,0,0,0, 0,0,0,1], + }, + drill: { + kick: [1,0,0,0, 0,0,0,1, 0,0,1,0, 0,0,0,0], + snare: [0,0,0,0, 1,0,0,0, 0,0,0,0, 1,0,0,0], + hat: [1,0,0,1, 0,1,0,0, 1,0,0,1, 0,1,0,0], + hatOpen: [0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0], + }, + "808": { + kick: [1,0,0,0, 0,0,0,0, 1,0,1,0, 0,0,0,0], + snare: [0,0,0,0, 1,0,0,0, 0,0,0,0, 1,0,0,0], + hat: [1,0,1,0, 1,0,1,0, 1,0,1,0, 1,0,1,1], + hatOpen: [0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0], + }, + lofi: { + kick: [1,0,0,0, 0,0,0,0, 0,0,1,0, 0,0,0,0], + snare: [0,0,0,0, 1,0,0,0, 0,0,0,0, 1,0,0,1], + hat: [0,0,1,0, 0,0,1,0, 0,0,1,0, 0,0,1,0], + hatOpen: [0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0], + }, + modern: { + kick: [1,0,0,0, 0,0,0,0, 1,0,0,0, 0,0,1,0], + snare: [0,0,0,0, 1,0,0,1, 0,0,0,0, 1,0,0,0], + hat: [1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1], + hatOpen: [0,0,0,0, 0,0,0,1, 0,0,0,0, 0,0,0,1], + }, +}; + +if (!BEAT_PATTERNS[STYLE]) { + console.error(`trap: unknown style '${STYLE}'. expected: ${Object.keys(BEAT_PATTERNS).join(" | ")}`); + process.exit(1); +} + +// ── voice: piano (sample bank) ──────────────────────────────────────── +const PIANO_SAMPLE_DIR = resolve(REPO, "fedac/native/samples/piano"); +const PIANO_ANCHORS = [21, 24, 27, 30, 33, 36, 39, 42, 45, 48, 51, 54, 57, 60, 63, 66, 69, 72, 75, 78, 81, 84, 87, 90, 93, 96]; +let pianoBank = null; + +function loadPianoBank() { + const bank = new Map(); + for (const m of PIANO_ANCHORS) { + const path = `${PIANO_SAMPLE_DIR}/${m}.raw`; + if (!existsSync(path)) throw new Error(`trap: missing piano anchor ${path}`); + const buf = readFileSync(path); + const f32 = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4); + bank.set(m, Float32Array.from(f32)); + } + console.log(`→ piano bank · ${bank.size} anchors loaded`); + return bank; +} + +function pianoAnchorFor(midi) { + let best = PIANO_ANCHORS[0]; + for (const a of PIANO_ANCHORS) if (Math.abs(a - midi) < Math.abs(best - midi)) best = a; + const ratio = Math.pow(2, (midi - best) / 12); + return { sample: pianoBank.get(best), ratio }; +} + +function mixEventPiano(ev, out) { + const { sample, ratio } = pianoAnchorFor(ev.midi); + const startIdx = Math.floor(ev.startSec * SAMPLE_RATE); + const durSamples = Math.floor(ev.durSec * SAMPLE_RATE); + const attack = Math.min(0.005 * SAMPLE_RATE, durSamples * 0.05); + const release = Math.min(0.08 * SAMPLE_RATE, durSamples * 0.5); + const lenOut = durSamples + Math.floor(release); + for (let i = 0; i < lenOut; i++) { + const dst = startIdx + i; + if (dst < 0 || dst >= out.length) continue; + const srcF = i * ratio; + const s0 = Math.floor(srcF); + const s1 = s0 + 1; + if (s1 >= sample.length) break; + const frac = srcF - s0; + const v = sample[s0] * (1 - frac) + sample[s1] * frac; + let env = 1; + if (i < attack) env = i / attack; + else if (i > durSamples) env = Math.max(0, 1 - (i - durSamples) / release); + out[dst] += v * env * ev.gain; + } +} + +// ── voice: sinebells ────────────────────────────────────────────────── +const BELL_PARTIALS = [ + { ratio: 0.5, amp: 0.28, decayT60: 5.5 }, + { ratio: 1.0, amp: 1.00, decayT60: 4.5 }, + { ratio: 2.0, amp: 0.32, decayT60: 2.6 }, + { ratio: 2.4, amp: 0.10, decayT60: 1.2 }, + { ratio: 3.0, amp: 0.09, decayT60: 1.0 }, + { ratio: 4.5, amp: 0.04, decayT60: 0.6 }, + { ratio: 5.4, amp: 0.02, decayT60: 0.4 }, +]; +const ATTACK_SEC = 0.012; +const BELL_RING_TAIL = 6.0; +const BELL_GAIN = 0.42; + +function midiToFreq(midi) { + return 440 * Math.pow(2, (midi - 69) / 12); +} + +function mixEventSinebell(ev, out) { + const startIdx = Math.floor(ev.startSec * SAMPLE_RATE); + const ringSamples = Math.floor((ev.durSec + BELL_RING_TAIL) * SAMPLE_RATE); + const attackS = ATTACK_SEC * SAMPLE_RATE; + const fundFreq = midiToFreq(ev.midi); + const twoPiOverSr = (2 * Math.PI) / SAMPLE_RATE; + const partials = BELL_PARTIALS.map((p) => ({ + omega: twoPiOverSr * fundFreq * p.ratio, + amp: p.amp, + decay: Math.exp(-Math.log(1000) / (p.decayT60 * SAMPLE_RATE)), + })); + for (let i = 0; i < ringSamples; i++) { + const dst = startIdx + i; + if (dst < 0 || dst >= out.length) continue; + let s = 0; + for (const p of partials) { + const env = p.amp * Math.pow(p.decay, i); + if (env < 1e-5) continue; + s += Math.sin(p.omega * i) * env; + } + let att = 1; + if (i < attackS) att = 0.5 - 0.5 * Math.cos((Math.PI * i) / attackS); + out[dst] += s * att * ev.gain * BELL_GAIN; + } +} + +// ── route harmonic events to chosen voice ───────────────────────────── +let mixHarmEvent; +if (VOICE === "piano") { + pianoBank = loadPianoBank(); + mixHarmEvent = mixEventPiano; +} else if (VOICE === "sinebells") { + console.log("→ harmonic voice · sinebells (no samples; pure synth)"); + mixHarmEvent = mixEventSinebell; +} else { + console.error(`trap: unknown voice '${VOICE}'. expected: piano | sinebells`); + process.exit(1); +} + +// ── drum mixer: AC-native via percussion.mjs ────────────────────────── +// Implements `sound.synth({type, tone, duration, volume, attack, decay, +// pan})` as a buffer-write into `out` at a fixed startSec. Each call to +// playPercussion fans out N synth calls — they all share startSec. +// +// Noise is shaped through a state-variable bandpass at `tone` (Q≈4), +// clamped at SR/6 for stability. Above the clamp the noise is still +// mixed but with a duller upper edge — adequate for the 8 kHz hi-hats +// in the AC kit. +function mixSynthVoice(out, startSec, params) { + const type = params?.type || "sine"; + const tone = Number(params?.tone) || 440; + const duration = Number(params?.duration); + const volume = Number(params?.volume ?? 1); + const attack = Math.max(0, Number(params?.attack ?? 0.001)); + const decay = Number.isFinite(params?.decay) + ? Math.max(0, Number(params.decay)) + : Math.max(0.001, (Number.isFinite(duration) ? duration : 0.05) - attack); + + if (!Number.isFinite(volume) || volume === 0) return; + if (!Number.isFinite(duration) || duration <= 0) return; + + const total = Math.max(duration, attack + decay); + const startIdx = Math.floor(startSec * SAMPLE_RATE); + const totalSamples = Math.ceil(total * SAMPLE_RATE); + const attSamples = Math.max(1, Math.floor(attack * SAMPLE_RATE)); + const decSamples = Math.max(1, Math.floor(decay * SAMPLE_RATE)); + + const omega = (2 * Math.PI * tone) / SAMPLE_RATE; + const phaseInc = tone / SAMPLE_RATE; + + // SVF bandpass for noise. Stable for fc < SR/6. + const fcSafe = Math.min(Math.max(40, tone), SAMPLE_RATE / 6); + const fParam = 2 * Math.sin(Math.PI * fcSafe / SAMPLE_RATE); + const dampParam = 1 / 4; + let svfLow = 0, svfBand = 0; + + for (let i = 0; i < totalSamples; i++) { + const dst = startIdx + i; + if (dst < 0 || dst >= out.length) continue; + + // Envelope: linear attack, exponential decay. + let env; + if (i < attSamples) { + env = i / attSamples; + } else { + const ti = i - attSamples; + if (ti >= decSamples) continue; + env = Math.exp(-3 * ti / decSamples); + } + + let s; + switch (type) { + case "sine": + s = Math.sin(omega * i); + break; + case "square": + s = Math.sin(omega * i) >= 0 ? 1 : -1; + break; + case "triangle": { + const ph = (i * phaseInc) % 1; + s = ph < 0.5 ? 4 * ph - 1 : 3 - 4 * ph; + break; + } + case "sawtooth": { + const ph = (i * phaseInc) % 1; + s = 2 * ph - 1; + break; + } + case "noise": + case "noise-white": { + const white = noiseRng() * 2 - 1; + const high = white - svfLow - dampParam * svfBand; + svfBand += fParam * high; + svfLow += fParam * svfBand; + s = svfBand * 1.5; + break; + } + default: + s = 0; + } + + out[dst] += s * env * volume; + } +} + +function makeBufferSynth(out, startSec) { + return { + synth: (params) => { + mixSynthVoice(out, startSec, params || {}); + return null; + }, + }; +} + +function fireDrum(out, startSec, letter, opts = {}) { + const sound = makeBufferSynth(out, startSec); + playPercussion(sound, letter, { phase: "both", ...opts }); +} + +// ── build event list ─────────────────────────────────────────────────── +const beatSec = 60 / BPM; +const barSec = beatSec * 4; // 4/4 +const stepSec = beatSec / 4; // 16th notes +const totalSec = barSec * BARS; +const pattern = BEAT_PATTERNS[STYLE]; + +const harmonicEvents = []; // { startSec, midi, gain, durSec } +const drumEvents = []; // { startSec, letter, volume } + +for (let bar = 0; bar < BARS; bar++) { + const barStart = bar * barSec; + const deg = PROGRESSION[bar % PROGRESSION.length]; + const triad = chordMidis(deg, 0); + const bass = scaleNoteMidi(deg, -2); + + // Drums: 16 steps per bar + for (let step = 0; step < 16; step++) { + const stepStart = barStart + step * stepSec; + if (pattern.kick[step]) { + const accent = step === 0 ? 1.15 : 1.0; + drumEvents.push({ startSec: stepStart, letter: "c", volume: accent * DRUM_GAIN }); + } + if (pattern.snare[step]) { + // Backbeat (steps 4 and 12) hits hardest. + const accent = (step === 4 || step === 12) ? 1.05 : 0.9; + drumEvents.push({ startSec: stepStart, letter: "d", volume: accent * DRUM_GAIN }); + } + if (pattern.hat[step]) { + // Trap hi-hat velocity variation: downbeat strong, off-beats softer. + const onBeat = step % 4 === 0; + const v = (onBeat ? 0.55 : 0.30) + rng() * 0.15; + drumEvents.push({ startSec: stepStart, letter: "g", volume: v * DRUM_GAIN }); + } + if (pattern.hatOpen[step]) { + drumEvents.push({ startSec: stepStart, letter: "a", volume: 0.55 * DRUM_GAIN }); + } + } + + // Bass on beat 1 — long sustain, octave low. + harmonicEvents.push({ startSec: barStart, midi: bass, gain: 0.55, durSec: barSec * 0.95 }); + + // Chord stabs on backbeats (beats 2 and 4) — short, soft. + for (const m of triad) { + harmonicEvents.push({ startSec: barStart + beatSec * 1, midi: m, gain: 0.22, durSec: beatSec * 0.6 }); + harmonicEvents.push({ startSec: barStart + beatSec * 3, midi: m, gain: 0.22, durSec: beatSec * 0.6 }); + } + + // Sparse melody — frequency tunable via DENSITY. + const wantMelody = rng() < (0.25 + DENSITY * 0.5); + if (wantMelody) { + const melDeg = deg + (rng() < 0.5 ? 4 : 2); + const melMidi = scaleNoteMidi(melDeg, 1); + const onset = rng() < 0.5 ? 0 : beatSec * 2; // top of bar or halfway + harmonicEvents.push({ + startSec: barStart + onset + beatSec * 0.05, + midi: melMidi, + gain: 0.32, + durSec: beatSec * 1.6, + }); + } +} + +console.log( + `→ trap · style=${STYLE} · voice=${VOICE} · ${BARS} bars · ${BPM} bpm · 4/4 · ${SCALE_NAME} · ` + + `${totalSec.toFixed(1)}s · ${drumEvents.length} drum hits · ${harmonicEvents.length} harm notes · seed=${SEED_STR}` +); + +// Export the deterministic event list for downstream tooling. +{ + const eventsPath = `${ROOT}/out/trap-events.json`; + const dir = OUT_PATH.replace(/\/[^/]+$/, ""); + mkdirSync(dir, { recursive: true }); + writeFileSync(eventsPath, JSON.stringify({ + style: STYLE, voice: VOICE, bpm: BPM, scale: SCALE_NAME, bars: BARS, + beatSec, barSec, stepSec, totalSec, seed: SEED_STR, + drum: drumEvents, harm: harmonicEvents, + }, null, 2)); + console.log(`→ events · ${eventsPath} (${drumEvents.length + harmonicEvents.length} total)`); +} + +// ── render ───────────────────────────────────────────────────────────── +const tailSec = VOICE === "sinebells" ? BELL_RING_TAIL : 1.0; +const totalSamples = Math.ceil((totalSec + tailSec) * SAMPLE_RATE); +const out = new Float32Array(totalSamples); + +for (const ev of harmonicEvents) mixHarmEvent(ev, out); +for (const ev of drumEvents) fireDrum(out, ev.startSec, ev.letter, { volume: ev.volume }); + +// Normalize to ~ -3 dBFS peak, then scale to voiceGain. +let peak = 0; +for (let i = 0; i < out.length; i++) { + const a = Math.abs(out[i]); + if (a > peak) peak = a; +} +if (peak > 0) { + const target = 0.7; + const norm = target / peak; + const finalGain = norm * VOICE_GAIN / 0.18; + for (let i = 0; i < out.length; i++) out[i] *= finalGain; +} + +// ── write ────────────────────────────────────────────────────────────── +const outDir = dirname(OUT_PATH); +mkdirSync(outDir, { recursive: true }); +const tag = audienceName || `${STYLE}-${SEED_STR}`; +const rawPath = `${outDir}/.${tag}-${VOICE}.f32.raw`; + +const buf = Buffer.alloc(out.length * 4); +for (let i = 0; i < out.length; i++) buf.writeFloatLE(out[i], i * 4); +writeFileSync(rawPath, buf); +console.log(`→ wrote ${rawPath} (${(buf.length / 1024 / 1024).toFixed(2)} MB f32 mono ${SAMPLE_RATE}Hz)`); + +const ff = spawnSync( + "ffmpeg", + [ + "-hide_banner", "-y", "-loglevel", "error", + "-f", "f32le", "-ar", String(SAMPLE_RATE), "-ac", "1", + "-i", rawPath, + "-c:a", "libmp3lame", "-q:a", "3", + OUT_PATH, + ], + { stdio: "inherit" } +); +if (ff.status !== 0) { + console.error("✗ ffmpeg failed"); + process.exit(1); +} +try { unlinkSync(rawPath); } catch {} +console.log(`✓ ${OUT_PATH}`); diff --git a/recap/bin/vocal.mjs b/recap/bin/vocal.mjs new file mode 100644 --- /dev/null +++ b/recap/bin/vocal.mjs @@ -0,0 +1,344 @@ +#!/usr/bin/env node +// vocal.mjs — AC-native formant-synthesized singing voice. EXPERIMENTAL. +// +// Status (2026-05-03): smoke-tested, dropped from the big-pictures +// vocal pipeline. The 3-formant synth at this fidelity reads as +// "melodic tones," not voice — getting real vocoder/talkbox character +// would need glottal pulse + F4/F5 + pitch jitter + breath + +// consonant articulation, a real research lane. Big-pictures vocal +// now routes through jeffrey-pvc via /api/say (see pop/RESEARCH-DIRECTION.md). +// +// This file stays in the repo as an experimental lane — may resurface +// as a *melodic instrument* layer (formant-shaped lead) rather than +// as a vocal substitute. +// +// Bottom-up vocal: same primitive bag as percussion.mjs. A sawtooth +// "glottal" source (rich harmonics, 1/n falloff) is shaped through 3 +// parallel 2-pole resonant filters tuned to a vowel's formant +// frequencies (F1/F2/F3), gated by a per-syllable envelope. Renders +// deterministically into a Float32 buffer and ffmpeg → mp3. +// +// Input: a phrase as [{ syl, vowel, pitch, dur, level? }, ...] +// vowel: key of VOWELS map (ah / eh / ih / ee / oh / oo / uh) +// pitch: Hz, MIDI number, or note string ("C3", "Eb3", "F#4") +// dur: seconds +// +// Smoke run (no args): renders "the music is real" on a 5-note pitch +// curve to recap/out/vocal.mp3. +// +// Score mode: --score .np reads a notepat-format score (matches +// papers/arxiv-folk-songs/folk-songs.tex §3) and renders the named +// section (default "hook"). Per-syllable vowels are inferred from the +// syllable text. Per-syllable durations are computed by splitting a +// bar's worth of time across each line, with the line-final syllable +// getting 1.5× weight. +// +// Usage: +// node bin/vocal.mjs # smoke phrase +// node bin/vocal.mjs --out ~/Desktop/vocal.mp3 +// node bin/vocal.mjs --over out/trap.mp3 # mix over trap bed +// node bin/vocal.mjs --score ../pop/big-pictures/plork.np \ +// --section hook --bpm 140 --octave 3 --over out/trap.mp3 + +import { + writeFileSync, readFileSync, mkdirSync, unlinkSync, existsSync, +} from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; +import { homedir } from "node:os"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(HERE, ".."); + +const SAMPLE_RATE = 48_000; + +// ── Vowel formants (Hz) — F1, F2, F3 with bandwidths and amps ──────── +// Approximate male/neutral speaker. Tweakable per syllable via override. +const VOWELS = { + ah: { F: [730, 1090, 2440], BW: [60, 90, 150], A: [1.0, 0.6, 0.3] }, // father, ah + eh: { F: [530, 1840, 2480], BW: [60, 90, 150], A: [1.0, 0.7, 0.3] }, // bet, eh + ih: { F: [390, 1990, 2550], BW: [50, 90, 150], A: [1.0, 0.7, 0.4] }, // bit, ih + ee: { F: [270, 2290, 3010], BW: [50, 100, 150], A: [1.0, 0.8, 0.4] }, // beat, real + oh: { F: [570, 840, 2410], BW: [60, 80, 150], A: [1.0, 0.5, 0.2] }, // boat + oo: { F: [300, 870, 2240], BW: [50, 80, 150], A: [1.0, 0.4, 0.2] }, // boot, music + uh: { F: [500, 1500, 2500], BW: [60, 90, 150], A: [1.0, 0.5, 0.25] }, // schwa, the +}; + +// ── Smoke phrase: "the music is real" ──────────────────────────────── +const SMOKE = [ + { syl: "the", vowel: "uh", pitch: "C3", dur: 0.18 }, + { syl: "mu", vowel: "oo", pitch: "D3", dur: 0.18 }, + { syl: "sic", vowel: "ih", pitch: "Eb3", dur: 0.22 }, + { syl: "is", vowel: "ih", pitch: "D3", dur: 0.20 }, + { syl: "real", vowel: "ee", pitch: "C3", dur: 0.55 }, +]; + +// ── Pitch parsing: Hz / MIDI / note name ────────────────────────────── +const NOTE_TO_SEMI = { c: 0, d: 2, e: 4, f: 5, g: 7, a: 9, b: 11 }; +function midiToFreq(midi) { return 440 * Math.pow(2, (midi - 69) / 12); } + +function parsePitch(p) { + if (typeof p === "number") return p > 20 ? p : midiToFreq(p); + if (typeof p !== "string") return 130.81; + const m = p.trim().toLowerCase().match(/^([a-g])([#b]?)(-?\d+)$/); + if (!m) { + const n = Number(p); + return Number.isFinite(n) ? (n > 20 ? n : midiToFreq(n)) : 130.81; + } + let semi = NOTE_TO_SEMI[m[1]]; + if (m[2] === "#") semi += 1; + if (m[2] === "b") semi -= 1; + const oct = parseInt(m[3], 10); + const midi = 12 * (oct + 1) + semi; + return midiToFreq(midi); +} + +// ── Render one syllable ─────────────────────────────────────────────── +// Sawtooth source → 3 parallel 2-pole resonant filters → sum. +// Filter: y[n] = gain*x[n] + a*y[n-1] + b*y[n-2] +// r = exp(-π*BW/sr), a = 2*r*cos(2π*fc/sr), b = -r² +// gain = 1 - r² to normalize peak. +// Envelope: 12ms attack, hold, 30ms release. +function renderSyllable(syl, out, startSec) { + const vowel = VOWELS[syl.vowel] || VOWELS.uh; + const freq = parsePitch(syl.pitch); + const dur = syl.dur ?? 0.2; + const level = syl.level ?? 1.0; + + const startIdx = Math.floor(startSec * SAMPLE_RATE); + const totalSamples = Math.ceil(dur * SAMPLE_RATE); + const attackSamples = Math.min(Math.floor(totalSamples / 3), Math.floor(0.012 * SAMPLE_RATE)); + const releaseSamples = Math.min(Math.floor(totalSamples / 3), Math.floor(0.030 * SAMPLE_RATE)); + const sustainEnd = totalSamples - releaseSamples; + + const phaseInc = freq / SAMPLE_RATE; + let phase = 0; + + const filters = vowel.F.map((fc, i) => { + const bw = vowel.BW[i]; + const r = Math.exp(-Math.PI * bw / SAMPLE_RATE); + const a = 2 * r * Math.cos(2 * Math.PI * fc / SAMPLE_RATE); + const b = -r * r; + const gain = 1 - r * r; + return { a, b, gain, amp: vowel.A[i], y1: 0, y2: 0 }; + }); + + for (let i = 0; i < totalSamples; i++) { + const dst = startIdx + i; + if (dst < 0 || dst >= out.length) continue; + + phase += phaseInc; + if (phase >= 1) phase -= 1; + const src = 2 * phase - 1; + + let s = 0; + for (const f of filters) { + const y = f.gain * src + f.a * f.y1 + f.b * f.y2; + f.y2 = f.y1; + f.y1 = y; + s += y * f.amp; + } + + let env; + if (i < attackSamples) env = i / attackSamples; + else if (i >= sustainEnd) env = Math.max(0, (totalSamples - i) / releaseSamples); + else env = 1; + + out[dst] += s * env * level; + } +} + +function renderPhrase(phrase) { + const total = phrase.reduce((t, s) => t + (s.dur ?? 0.2), 0); + const samples = Math.ceil((total + 0.3) * SAMPLE_RATE); + const out = new Float32Array(samples); + let t = 0; + for (const syl of phrase) { + renderSyllable(syl, out, t); + t += syl.dur ?? 0.2; + } + return out; +} + +// ── notepat (.np) score parser ──────────────────────────────────────── +// Format (folk-songs paper §3): NOTE:syllable whitespace-separated. +// Hyphens mark syllable continuation. Section headers are lowercase +// lines without colons. Comments start with '#'. +function parseNp(text) { + const sections = { _order: [] }; + let current = null; + const lines = text.split("\n"); + for (const raw of lines) { + const line = raw.trim(); + if (!line || line.startsWith("#")) continue; + // Section header: lowercase, no colon, words/digits/spaces only + if (!line.includes(":") && /^[a-z][a-z0-9 ]*$/.test(line)) { + current = line; + if (!sections[current]) { + sections[current] = []; + sections._order.push(current); + } + continue; + } + if (!current) { + current = "default"; + if (!sections[current]) { + sections[current] = []; + sections._order.push(current); + } + } + const tokens = line.split(/\s+/).filter(Boolean); + const lineSyllables = []; + for (const tok of tokens) { + const m = tok.match(/^([A-Ga-g][#b]?\d?):(.+)$/); + if (!m) continue; + const note = m[1].charAt(0).toUpperCase() + m[1].slice(1); + lineSyllables.push({ pitch: note, syl: m[2] }); + } + if (lineSyllables.length) sections[current].push(lineSyllables); + } + return sections; +} + +// Vowel heuristic: pick the most plausible vowel for a syllable's +// orthography. Order matters — check digraphs before single letters. +function pickVowel(rawSyl) { + const s = rawSyl.toLowerCase().replace(/^-+|-+$/g, "").replace(/[^a-z]/g, ""); + if (!s) return "uh"; + if (/ee|ea|ie$|y$/.test(s)) return "ee"; + if (/oo|ui|ue/.test(s)) return "oo"; + if (/oa|ow$|o[^aeiouy]?$/.test(s)) return "oh"; + if (/i/.test(s)) return "ih"; + if (/a/.test(s)) return "ah"; + if (/e/.test(s)) return "eh"; + if (/u/.test(s)) return "uh"; + if (/o/.test(s)) return "oh"; + return "uh"; +} + +// Time a single line: split lineSec across N syllables, line-final gets 1.5x. +function timeLine(line, lineSec, octave) { + const n = line.length; + if (!n) return []; + const units = (n - 1) + 1.5; + const unitSec = lineSec / units; + return line.map((tok, i) => ({ + syl: tok.syl, + pitch: /\d/.test(tok.pitch) ? tok.pitch : tok.pitch + String(octave), + vowel: pickVowel(tok.syl), + dur: i === n - 1 ? unitSec * 1.5 : unitSec, + })); +} + +// Build a phrase from a section by stretching each line to one bar. +function phraseFromSection(section, barSec, octave, gapSec = 0) { + const phrase = []; + for (let li = 0; li < section.length; li++) { + const timed = timeLine(section[li], barSec, octave); + phrase.push(...timed); + if (gapSec > 0 && li < section.length - 1) { + phrase.push({ syl: "", pitch: "C0", vowel: "uh", dur: gapSec, level: 0 }); + } + } + return phrase; +} + +// ── arg parse + main ────────────────────────────────────────────────── +const argv = process.argv.slice(2); +const flags = {}; +for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a.startsWith("--")) { + const key = a.slice(2); + const next = argv[i + 1]; + if (next !== undefined && !next.startsWith("--")) { flags[key] = next; i++; } + else flags[key] = true; + } +} + +function expandHome(p) { + if (!p || typeof p !== "string") return p; + if (p === "~") return homedir(); + if (p.startsWith("~/")) return resolve(homedir(), p.slice(2)); + return p; +} + +const OUT_PATH = expandHome(flags.out) || `${ROOT}/out/vocal.mp3`; + +let phrase; +if (flags.score) { + const scorePath = expandHome(flags.score); + if (!existsSync(scorePath)) { + console.error(`✗ --score file not found: ${scorePath}`); + process.exit(1); + } + const sections = parseNp(readFileSync(scorePath, "utf8")); + const sectionName = flags.section || "hook"; + if (!sections[sectionName]) { + console.error(`✗ section '${sectionName}' not in score. available: ${sections._order.join(", ")}`); + process.exit(1); + } + const bpm = Number(flags.bpm) || 140; + const barSec = (60 / bpm) * 4; + const octave = Number(flags.octave) || 3; + const gapSec = Number(flags.gap) || 0; + phrase = phraseFromSection(sections[sectionName], barSec, octave, gapSec); + console.log(`→ score · ${scorePath} · section=${sectionName} · ${sections[sectionName].length} lines @ ${bpm} BPM`); +} else { + phrase = SMOKE; +} + +console.log(`→ vocal · ${phrase.length} syllables · ${phrase.reduce((t,s)=>t+s.dur,0).toFixed(2)}s`); +console.log(" " + phrase.filter(s => s.syl).map(s => `${s.syl}[${s.vowel}/${s.pitch}]`).join(" ")); + +const out = renderPhrase(phrase); + +let peak = 0; +for (let i = 0; i < out.length; i++) { + const a = Math.abs(out[i]); + if (a > peak) peak = a; +} +if (peak > 0) { + const target = 0.7; + const norm = target / peak; + for (let i = 0; i < out.length; i++) out[i] *= norm; +} + +const outDir = dirname(OUT_PATH); +mkdirSync(outDir, { recursive: true }); +const rawPath = `${outDir}/.vocal-smoke.f32.raw`; +const buf = Buffer.alloc(out.length * 4); +for (let i = 0; i < out.length; i++) buf.writeFloatLE(out[i], i * 4); +writeFileSync(rawPath, buf); + +const ff = spawnSync("ffmpeg", [ + "-hide_banner", "-y", "-loglevel", "error", + "-f", "f32le", "-ar", String(SAMPLE_RATE), "-ac", "1", + "-i", rawPath, + "-c:a", "libmp3lame", "-q:a", "3", + OUT_PATH, +], { stdio: "inherit" }); +if (ff.status !== 0) { console.error("✗ ffmpeg failed"); process.exit(1); } +try { unlinkSync(rawPath); } catch {} +console.log(`✓ ${OUT_PATH}`); + +// Optional: mix over a trap bed (or any audio file). +if (flags.over) { + const overPath = expandHome(flags.over); + if (!existsSync(overPath)) { + console.error(`✗ --over file not found: ${overPath}`); + process.exit(1); + } + const mixOut = OUT_PATH.replace(/\.mp3$/, "-with-bed.mp3"); + console.log(`→ mixing vocal over ${overPath} → ${mixOut}`); + const mix = spawnSync("ffmpeg", [ + "-hide_banner", "-y", "-loglevel", "error", + "-i", overPath, "-i", OUT_PATH, + "-filter_complex", "[0:a]volume=0.55[bed];[1:a]volume=1.0[voc];[bed][voc]amix=inputs=2:duration=longest:dropout_transition=0[a]", + "-map", "[a]", "-c:a", "libmp3lame", "-q:a", "3", + mixOut, + ], { stdio: "inherit" }); + if (mix.status !== 0) { console.error("✗ ffmpeg mix failed"); process.exit(1); } + console.log(`✓ ${mixOut}`); +} diff --git a/recap/pipeline.fish b/recap/pipeline.fish --- a/recap/pipeline.fish +++ b/recap/pipeline.fish @@ -24,6 +24,14 @@ echo "▸ 2/8 transcribe + align" node bin/transcribe.mjs; or exit 1 node bin/align.mjs $AUDIENCE; or exit 1 +echo "▸ 2.5/8 sing (pitchsnap → recap-sung.mp3 + rewrites words.json)" +node bin/sing.mjs $AUDIENCE +or echo " ↳ sing step skipped or failed — compose will use plain narration" +# Re-align after sing rewrote words.json so segments.json reflects +# the stretched timeline. align caches on words.json hash so this +# only re-runs when sing actually changed something. +node bin/align.mjs $AUDIENCE --force; or exit 1 + echo "▸ 3/8 jeffrey-photos (gpt-image-2, cached per segment)" node bin/jeffrey-photos.mjs $AUDIENCE; or exit 1 @@ -71,6 +79,19 @@ or echo " ↳ waltz-overlay skipped (no events.json) — compose without piano bug" echo "▸ 8/8 compose" fish bin/compose.fish; or exit 1 + +echo "▸ 8.5/8 timeline (post-analysis PNG → desktop)" +set -l TIMELINE_OUT "$HOME/Desktop/recap-timing_"(date +%Y-%m-%d_%H%M%S)".png" +if test -x ../pop/.venv/bin/python + ../pop/.venv/bin/python bin/timeline.py \ + --subs out/subs.json \ + --events out/waltz-events.json \ + --audio out/waltz.mp3 \ + --out "$TIMELINE_OUT" + or echo " ↳ timeline step skipped or failed" +else + echo " ↳ no pop/.venv — skipping timeline (install librosa/matplotlib to enable)" +end echo "━━━ done · $ROOT/out/recap.mp4 ━━━" ls -lh $ROOT/out/recap.mp4