diff --git a/recap/bin/align.mjs b/recap/bin/align.mjs index 495338e595..e26214d451 100755 --- a/recap/bin/align.mjs +++ b/recap/bin/align.mjs @@ -4,18 +4,48 @@ // [{name, startSec, endSec, durationSec}, ...] // Each marker is normalized (lowercase, punctuation stripped) and matched as // a contiguous run of N words. Unmatched markers fail loud. -// Usage: node bin/align.mjs [audience-name] +// +// Caching: keyed on a hash of words.json + audience.segments. If the inputs +// are unchanged AND segments.json exists, skip alignment. Pass --force to +// bypass. +// +// Usage: +// node bin/align.mjs [audience-name] +// node bin/align.mjs jeffrey-73h-2026-05-02 --force -import { readFileSync, writeFileSync } from "node:fs"; +import { readFileSync, writeFileSync, existsSync } from "node:fs"; import { resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; +import { createHash } from "node:crypto"; const HERE = dirname(fileURLToPath(import.meta.url)); const ROOT = resolve(HERE, ".."); -const audienceName = process.argv[2] || "fia"; +const argv = process.argv.slice(2); +const force = argv.includes("--force"); +const audienceName = argv.find((a) => !a.startsWith("--")) || "fia"; const { audience } = await import(`${ROOT}/audience/${audienceName}.mjs`); const words = JSON.parse(readFileSync(`${ROOT}/out/words.json`, "utf8")); +const segmentsPath = `${ROOT}/out/segments.json`; +const hashFile = `${segmentsPath}.hash`; + +// Hash on words.json content + the segments[] markers/names. Trailing +// silence is included since it affects endMs computation. +const inputHash = createHash("sha256") + .update(JSON.stringify(words)) + .update(JSON.stringify(audience.segments.map((s) => ({ n: s.name, m: s.marker, t: s.trailingSilenceSec || 0 })))) + .digest("hex") + .slice(0, 16); + +if (!force && existsSync(segmentsPath) && existsSync(hashFile)) { + const cached = readFileSync(hashFile, "utf8").trim(); + if (cached === inputHash) { + const segments = JSON.parse(readFileSync(segmentsPath, "utf8")); + console.log(`✓ ${segmentsPath} cached · ${segments.length} segments · hash ${inputHash} — skipping align`); + process.exit(0); + } +} + const norm = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim(); const wordTokens = words.map((w) => norm(w.text)); const audioEndMs = words[words.length - 1].toMs; @@ -64,8 +94,9 @@ const segments = starts.map((s, i) => { }; }); -writeFileSync(`${ROOT}/out/segments.json`, JSON.stringify(segments, null, 2)); -console.log(`✓ ${ROOT}/out/segments.json`); +writeFileSync(segmentsPath, JSON.stringify(segments, null, 2)); +writeFileSync(hashFile, inputHash + "\n"); +console.log(`✓ ${segmentsPath} · hash ${inputHash}`); for (const s of segments) { console.log(` ${s.name.padEnd(18)} ${String(s.startSec).padStart(6)}s → ${String(s.endSec).padStart(6)}s (${s.durationSec.toFixed(2)}s) "${s.marker}"`); } diff --git a/recap/bin/build-filter.mjs b/recap/bin/build-filter.mjs index fd6fa4dd22..60a804cfad 100755 --- a/recap/bin/build-filter.mjs +++ b/recap/bin/build-filter.mjs @@ -1,33 +1,27 @@ #!/usr/bin/env node // build-filter.mjs — emit the ffmpeg filter_complex graph for compose.fish. -// Reads out/subs.json and stitches one overlay per subtitle chunk into the -// video chain so each chunk appears only between its [startSec, endSec]. -// Subs sit at the top of the frame so they don't collide with the slide's -// bottom-third title overlay; the waveform sits at the same y, layered behind -// the pill so it animates "through" the subtitle. +// +// Inputs (per compose.fish): +// [0:v] slide concat (PNG sequence) +// [1:a] narration mp3 +// [2:v] subtitle track concat (full-frame transparent PNG sequence — see +// subtitle-track.mjs); a single overlay onto the slide stream +// replaces the old 135-deep movie= chain. +// +// Subtitle PNGs are now full-frame 1080×1920 transparent images with the +// pill positioned at y=1690, so we just overlay [2:v] at (0,0). The +// concat demuxer at input #2 plays them with their stored durations, +// alternating with a fully-transparent blank frame for gaps. +// // Usage: node bin/build-filter.mjs (writes graph to stdout) -import { readFileSync } from "node:fs"; -import { resolve, dirname } from "node:path"; -import { fileURLToPath } from "node:url"; - -const HERE = dirname(fileURLToPath(import.meta.url)); -const ROOT = resolve(HERE, ".."); const TOTAL = process.argv[2]; if (!TOTAL) { console.error("usage: build-filter.mjs "); process.exit(1); } -const subs = JSON.parse(readFileSync(`${ROOT}/out/subs.json`, "utf8")); - -// Subtitle band lives just above the progress bar at the bottom of the frame, -// below the slide's title overlay. Sub PNGs are 1080×220 with the pill centered. -// Waveform is parked at the same vertical area so the pill sits in front of -// the dancing line. -// Centered horizontally — sub PNGs are 1080×220 so x=0. -const SUB_Y = 1690; -const WAVE_Y = 1752; // 1080×96 waveform centered behind the sub pill (~y 1752–1848) +const WAVE_Y = 1752; // y-band for the audio waveform under the subtitle pill const lines = []; lines.push(`[0:v]format=yuv420p,fps=30,scale=1080:1920,setsar=1[bg]`); @@ -35,19 +29,8 @@ lines.push(`[1:a]apad=whole_dur=${TOTAL},asplit=2[a1][a2]`); lines.push(`[a2]showwaves=s=1080x96:colors=0xff70d0|0x70f0e0:mode=cline:rate=30,format=rgba,colorchannelmixer=aa=0.55[wave]`); lines.push(`[bg][wave]overlay=x=0:y=${WAVE_Y}:format=auto[bg2]`); lines.push(`[bg2]drawbox=x=0:y=1912:w='iw*t/${TOTAL}':h=8:color=0xff69b4:t=fill[v0]`); - -let prev = "v0"; -for (let i = 0; i < subs.length; i++) { - const s = subs[i]; - const srcLabel = `s${i}`; - const nextLabel = `v${i + 1}`; - // movie filter loads PNG with alpha; format=rgba ensures alpha is preserved. - lines.push(`movie='${s.file}':loop=0,setpts=N/(FRAME_RATE*TB),format=rgba[${srcLabel}]`); - lines.push(`[${prev}][${srcLabel}]overlay=x=0:y=${SUB_Y}:format=auto:enable='between(t,${s.startSec},${s.endSec})'[${nextLabel}]`); - prev = nextLabel; -} - -// Final stream needs the canonical [final] label for compose.fish -map. -lines.push(`[${prev}]null[final]`); +// Single subtitle-track overlay (was a 135-deep movie= chain). +lines.push(`[2:v]format=rgba,fps=30,scale=1080:1920[subs]`); +lines.push(`[v0][subs]overlay=x=0:y=0:format=auto:shortest=0[final]`); process.stdout.write(lines.join(";\n") + "\n"); diff --git a/recap/bin/compose.fish b/recap/bin/compose.fish index 97e1a84e84..f53ec10184 100755 --- a/recap/bin/compose.fish +++ b/recap/bin/compose.fish @@ -1,14 +1,22 @@ #!/usr/bin/env fish # compose.fish — final ffmpeg pass: concat slides + audio (with trailing -# silence) + waveform + animated progress bar + word-synced subtitles -# (loaded as movie sources, overlaid with enable=between(t,a,b)). -# Reads out/concat.txt, out/recap.mp3, out/duration.txt, out/subs.json. +# silence) + waveform + animated progress bar + word-synced subtitles. +# +# Subtitles arrive as a single concat-demuxer track (out/subtitle-track.txt +# pointing to full-frame transparent PNGs with stored durations) — see +# subtitle-track.mjs. That replaces the 135-deep movie= overlay chain that +# bottlenecked the oven encode. Filter graph = a single overlay onto the +# slide stream. +# +# Reads out/concat.txt, out/recap.mp3, out/duration.txt, out/subs.json, +# out/subtitle-track.txt. set -l ROOT (realpath (dirname (status -f))/..) set -l OUT $ROOT/out set -l TOTAL (cat $OUT/duration.txt) set -l AUDIO $OUT/recap.mp3 set -l WALTZ $OUT/waltz.mp3 +set -l SUBTRACK $OUT/subtitle-track.txt set -l VIDEO $OUT/recap.mp4 set -l FILTER $OUT/filter.txt @@ -20,28 +28,29 @@ if not test -f $OUT/subs.json echo "✗ missing $OUT/subs.json — run bin/subtitles.mjs first" exit 1 end +if not test -f $SUBTRACK + echo "✗ missing $SUBTRACK — run bin/subtitle-track.mjs first" + exit 1 +end echo "→ ffmpeg compose · $TOTAL s · 1080x1920" -# Build the filter graph in node so we can splice in one overlay per subtitle -# chunk without fish escape gymnastics around brackets and quotes. +# Build the filter graph in node. With the single-overlay subtitle track +# the graph is short — three formatting filters on slides, an audio split +# for the showwaves, and a final overlay of the subtitle stream. node $ROOT/bin/build-filter.mjs $TOTAL > $FILTER # If a piano-waltz bed exists, append a mix into the same filter graph so # we don't need a second -filter_complex flag (only the last one wins). -# The waltz is already gain-staged by waltz.mjs (audience.waltz.voiceGain); -# we still clamp it lightly here so it sits well under the spoken track. +# Slides=0, narration=1, subs=2, waltz=3 — input order matters. if test -f $WALTZ echo " + bed: $WALTZ (waltz)" # printf — fish parses $TOTAL[bed] as a slice index; %s sidesteps that. - # NOTE: rely on `-stream_loop -1` at the input level for looping; do NOT - # also use `aloop=loop=-1:size=2e9` — that allocates a 2-billion-sample - # buffer (~24 GB worst case) which OOMs on the 8 GB machine. - # `atrim=duration=$TOTAL` is enough to cut the looped stream at length. - printf ';[2:a]volume=0.42,atrim=duration=%s[bed];[a1][bed]amix=inputs=2:duration=first:dropout_transition=0:weights=1.0 0.55[mix]\n' "$TOTAL" >> $FILTER + printf ';[3:a]volume=0.42,atrim=duration=%s[bed];[a1][bed]amix=inputs=2:duration=first:dropout_transition=0:weights=1.0 0.55[mix]\n' "$TOTAL" >> $FILTER ffmpeg -hide_banner -y \ -f concat -safe 0 -i $OUT/concat.txt \ -i $AUDIO \ + -f concat -safe 0 -i $SUBTRACK \ -stream_loop -1 -i $WALTZ \ -filter_complex_script $FILTER \ -map "[final]" -map "[mix]" \ @@ -54,6 +63,7 @@ else ffmpeg -hide_banner -y \ -f concat -safe 0 -i $OUT/concat.txt \ -i $AUDIO \ + -f concat -safe 0 -i $SUBTRACK \ -filter_complex_script $FILTER \ -map "[final]" -map "[a1]" \ -c:v libx264 -preset ultrafast -crf 22 -pix_fmt yuv420p \ diff --git a/recap/bin/subtitle-track.mjs b/recap/bin/subtitle-track.mjs new file mode 100644 index 0000000000..2815d50f10 --- /dev/null +++ b/recap/bin/subtitle-track.mjs @@ -0,0 +1,66 @@ +#!/usr/bin/env node +// subtitle-track.mjs — emit a concat-demuxer file (`out/subtitle-track.txt`) +// that sequences subtitle PNGs with explicit durations, so the main compose +// can include subtitles via a single `-f concat -i subtitle-track.txt` +// input + one overlay filter — instead of a 135-deep `movie=...` chain. +// +// Reads `out/subs.json` (timing + per-chunk PNG paths) and `out/subs/blank.png` +// (a fully-transparent 1080×1920 PNG produced by subtitles.mjs). +// +// The output is a plain concat-demuxer text file. ffmpeg picks it up at +// frame rate via: +// -f concat -safe 0 -i out/subtitle-track.txt +// The `duration` directive is honored on each entry. The very last `file` +// must be repeated (concat-demuxer quirk) so the final entry's duration +// applies. +// +// Usage: node bin/subtitle-track.mjs + +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(HERE, ".."); +const subsPath = `${ROOT}/out/subs.json`; +const blankPath = `${ROOT}/out/subs/blank.png`; +const outPath = `${ROOT}/out/subtitle-track.txt`; +const durPath = `${ROOT}/out/duration.txt`; + +const subs = JSON.parse(readFileSync(subsPath, "utf8")); +if (!subs.length) { + console.error(`✗ no subtitle chunks in ${subsPath}`); + process.exit(1); +} + +// Total video duration (so the track ends at the right moment). If +// duration.txt isn't around yet, fall back to the last subtitle endSec +// (slides.mjs will have set duration.txt before this runs in the pipeline). +let total; +try { total = parseFloat(readFileSync(durPath, "utf8")); } +catch { total = subs[subs.length - 1].endSec; } + +const lines = []; +let cursor = 0; +for (const s of subs) { + if (s.startSec > cursor + 0.001) { + // Gap before this chunk: blank. + lines.push(`file '${blankPath}'`); + lines.push(`duration ${(s.startSec - cursor).toFixed(3)}`); + } + lines.push(`file '${s.file}'`); + lines.push(`duration ${(s.endSec - s.startSec).toFixed(3)}`); + cursor = s.endSec; +} +// Trailing blank to fill the rest of the timeline. +if (total > cursor + 0.001) { + lines.push(`file '${blankPath}'`); + lines.push(`duration ${(total - cursor).toFixed(3)}`); +} +// Concat demuxer requires the last `file` line repeated for its duration +// to apply (https://trac.ffmpeg.org/wiki/Slideshow). +const lastFileLine = [...lines].reverse().find((l) => l.startsWith("file ")); +lines.push(lastFileLine); + +writeFileSync(outPath, lines.join("\n") + "\n"); +console.log(`✓ ${outPath} · ${subs.length} chunks · total ${total.toFixed(2)}s`); diff --git a/recap/bin/subtitles.mjs b/recap/bin/subtitles.mjs index e317d2008b..a03591f321 100755 --- a/recap/bin/subtitles.mjs +++ b/recap/bin/subtitles.mjs @@ -73,6 +73,16 @@ for (let i = 0; i < words.length; i++) { } } +// Full-frame PNGs (1080×1920, transparent except for the pill at y=1690). +// This replaces the older 1080×220 strip — by baking each subtitle into a +// full-frame transparent PNG, the compose step can stitch them via the +// concat demuxer (one timed PNG sequence) and overlay them as a single +// video stream, eliminating the 135-deep movie= filter chain that +// bottlenecked the oven encode (see feedback_recap_subtitles_required.md). +const FRAME_W = 1080; +const FRAME_H = 1920; +const PILL_Y_TOP = 1690; // matches the SUB_Y in build-filter.mjs + const cssTemplate = ` @font-face { font-family: 'ProcessingB'; @@ -80,8 +90,18 @@ const cssTemplate = ` unicode-range: U+0020-007E; } * { box-sizing: border-box; margin: 0; padding: 0; } -html, body { width: 1080px; height: 220px; background: transparent; -webkit-font-smoothing: antialiased; } -.wrap { width: 100%; height: 100%; display: flex; align-items: center; justify-content: center; padding: 0 60px; } +html, body { width: ${FRAME_W}px; height: ${FRAME_H}px; background: transparent; -webkit-font-smoothing: antialiased; } +.wrap { + position: absolute; + left: 0; + top: ${PILL_Y_TOP}px; + width: ${FRAME_W}px; + height: 220px; + display: flex; + align-items: center; + justify-content: center; + padding: 0 60px; +} .pill { background: rgba(16, 8, 32, 0.72); backdrop-filter: blur(2px); @@ -111,9 +131,9 @@ for (let i = 0; i < chunks.length; i++) { const c = chunks[i]; const file = `${SUB_DIR}/${String(i).padStart(3, "0")}.png`; const page = await browser.newPage(); - await page.setViewport({ width: 1080, height: 220, deviceScaleFactor: 1 }); + await page.setViewport({ width: FRAME_W, height: FRAME_H, deviceScaleFactor: 1 }); const html = `
${escapeHtml(c.text)}
`; - await page.setContent(html, { waitUntil: "networkidle0" }); + await page.setContent(html, { waitUntil: "domcontentloaded" }); await new Promise((r) => setTimeout(r, 80)); const png = await page.screenshot({ type: "png", omitBackground: true }); writeFileSync(file, png); @@ -125,10 +145,22 @@ for (let i = 0; i < chunks.length; i++) { text: c.text, }); } + +// Render a single fully-transparent blank frame the concat demuxer can use +// for gaps between subtitles. One file, reused for every gap entry. +{ + const blankPath = `${SUB_DIR}/blank.png`; + const page = await browser.newPage(); + await page.setViewport({ width: FRAME_W, height: FRAME_H, deviceScaleFactor: 1 }); + await page.setContent(``, { waitUntil: "domcontentloaded" }); + const blank = await page.screenshot({ type: "png", omitBackground: true }); + writeFileSync(blankPath, blank); + await page.close(); +} await browser.close(); writeFileSync(`${ROOT}/out/subs.json`, JSON.stringify(out, null, 2)); -console.log(`✓ ${out.length} subtitle chunks → ${SUB_DIR}/`); +console.log(`✓ ${out.length} subtitle chunks → ${SUB_DIR}/ (full-frame ${FRAME_W}×${FRAME_H})`); for (const s of out.slice(0, 5)) console.log(` ${s.startSec.toFixed(2)}-${s.endSec.toFixed(2)} "${s.text}"`); if (out.length > 5) console.log(` ... (+${out.length - 5} more)`); diff --git a/recap/bin/transcribe.mjs b/recap/bin/transcribe.mjs index c0eca49c68..58b5e14ab6 100755 --- a/recap/bin/transcribe.mjs +++ b/recap/bin/transcribe.mjs @@ -1,17 +1,26 @@ #!/usr/bin/env node // transcribe.mjs — run whisper-cli on out/recap.mp3 and emit out/words.json // in a flat shape: [{text, fromMs, toMs}, ...]. -// Usage: node bin/transcribe.mjs +// +// Caching: keyed on a content hash of recap.mp3. If `out/words.json` exists +// AND `out/words.json.hash` matches, skip the whisper run (~90s on oven CPU). +// Pass `--force` to bypass. +// +// Usage: +// node bin/transcribe.mjs +// node bin/transcribe.mjs --force import { execFileSync } from "node:child_process"; import { readFileSync, writeFileSync, existsSync } from "node:fs"; import { resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; +import { createHash } from "node:crypto"; const HERE = dirname(fileURLToPath(import.meta.url)); const ROOT = resolve(HERE, ".."); const MP3 = `${ROOT}/out/recap.mp3`; const MODEL = `${ROOT}/models/ggml-base.en.bin`; +const force = process.argv.includes("--force"); if (!existsSync(MP3)) { console.error(`✗ missing ${MP3} — run bin/tts.mjs first`); @@ -22,6 +31,26 @@ if (!existsSync(MODEL)) { process.exit(1); } +// Hash recap.mp3 contents (whisper output depends entirely on the audio). +const inputHash = createHash("sha256") + .update(readFileSync(MP3)) + .digest("hex") + .slice(0, 16); + +const wordsPath = `${ROOT}/out/words.json`; +const hashFile = `${wordsPath}.hash`; + +if (!force && existsSync(wordsPath) && existsSync(hashFile)) { + const cached = readFileSync(hashFile, "utf8").trim(); + if (cached === inputHash) { + const words = JSON.parse(readFileSync(wordsPath, "utf8")); + const last = words[words.length - 1]; + console.log(`✓ ${wordsPath} cached · ${words.length} words · hash ${inputHash} — skipping whisper`); + if (last) console.log(` audio ends at ${(last.toMs / 1000).toFixed(2)}s`); + process.exit(0); + } +} + console.log(`→ whisper-cli · ${MP3}`); execFileSync( "whisper-cli", @@ -34,5 +63,6 @@ const words = raw.transcription .map((s) => ({ text: s.text.trim(), fromMs: s.offsets.from, toMs: s.offsets.to })) .filter((w) => w.text.length > 0); -writeFileSync(`${ROOT}/out/words.json`, JSON.stringify(words, null, 2)); -console.log(`✓ ${ROOT}/out/words.json · ${words.length} words · ${(words[words.length - 1].toMs / 1000).toFixed(2)}s`); +writeFileSync(wordsPath, JSON.stringify(words, null, 2)); +writeFileSync(hashFile, inputHash + "\n"); +console.log(`✓ ${wordsPath} · ${words.length} words · ${(words[words.length - 1].toMs / 1000).toFixed(2)}s · hash ${inputHash}`); diff --git a/recap/pipeline.fish b/recap/pipeline.fish index f896327610..b0cfc5e5d6 100755 --- a/recap/pipeline.fish +++ b/recap/pipeline.fish @@ -14,29 +14,42 @@ cd $ROOT echo "━━━ recap pipeline · audience=$AUDIENCE ━━━" if test $SKIP_TTS -eq 0 - echo "▸ 1/6 tts" + echo "▸ 1/8 tts" node bin/tts.mjs $AUDIENCE; or exit 1 else - echo "▸ 1/6 tts (skipped — reusing out/recap.mp3)" + echo "▸ 1/8 tts (skipped — reusing out/recap.mp3)" end -echo "▸ 2/7 transcribe + align" +echo "▸ 2/8 transcribe + align" node bin/transcribe.mjs; or exit 1 node bin/align.mjs $AUDIENCE; or exit 1 -echo "▸ 3/7 jeffrey-photos (gpt-image-2, cached per segment)" +echo "▸ 3/8 jeffrey-photos (gpt-image-2, cached per segment)" node bin/jeffrey-photos.mjs $AUDIENCE; or exit 1 -echo "▸ 4/7 scout (resolve per-slide content queries)" +echo "▸ 3.5/8 chat-fetch (laer-klokken + system snapshots)" +node bin/chat-fetch.mjs +or echo " ↳ chat-fetch step skipped or failed — chat slide will render empty" + +echo "▸ 3.7/8 screenshots (production-URL artifact insets, cached)" +node bin/screenshots.mjs $AUDIENCE +or echo " ↳ screenshots step skipped or failed — slides without cached artifacts will render without insets" + +echo "▸ 4/8 scout (resolve per-slide content queries)" node bin/scout.mjs $AUDIENCE; or exit 1 -echo "▸ 5/7 slides" +echo "▸ 5/8 slides" node bin/slides.mjs $AUDIENCE; or exit 1 -echo "▸ 6/7 subtitles" +echo "▸ 6/8 subtitles" node bin/subtitles.mjs $AUDIENCE; or exit 1 +node bin/subtitle-track.mjs; or exit 1 + +echo "▸ 7/8 waltz (piano bed; harmless if audience.waltz is absent)" +node bin/waltz.mjs $AUDIENCE +or echo " ↳ waltz step skipped or failed — compose falls back to narration-only" -echo "▸ 7/7 compose" +echo "▸ 8/8 compose" fish bin/compose.fish; or exit 1 echo "━━━ done · $ROOT/out/recap.mp4 ━━━"