Something went wrong. Try again.
Monorepo for Aesthetic.Computer aesthetic.computer
Something went wrong. Try again.
JavaScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194// av-sync.mjs — is the mouth early, or is the sound?//// Consumer cameras and old webcam captures routinely bake in an A/V offset: the// picture and the audio are written with timestamps that disagree by a frame or// four, and nobody notices until you put captions under it and the lips start// reading as dubbed.//// You can measure it. A talking mouth is a motion signal, and a talking voice is// an amplitude signal, and they are the same signal — a mouth opens when a// syllable lands. So: pull motion energy from the mouth region of every frame,// pull an RMS envelope from the audio, and slide one against the other. The lag// with the strongest correlation is the offset.//// node marketing/lib/av-sync.mjs <video> # just report// import { measureAvOffset } from "./av-sync.mjs";//// Returns { offsetMs, confidence, curve }.//// offsetMs > 0 → the PICTURE is late (visuals behind the sound). To fix, the// video must be pulled EARLIER, or the audio pushed later.// offsetMs < 0 → the SOUND is late.//// `confidence` is the peak correlation over the mean of the curve. Under about// 1.15 the peak isn't really a peak and you should not trust the number — a shot// with little mouth movement, or a noisy room, will not correlate.//// TWO THINGS THAT WILL WASTE YOUR AFTERNOON://// · Measure the RAW CLIP, never a finished reel. Music under the voice —// a 72bpm bed with a kick on 1 and 3 — pours energy into the envelope that// no mouth produced, which flattens the true peak and hands the win to a// spurious one. A reel that is genuinely in sync will measure as though it// is not.// · One pass lands within about a frame, not on the nose. The correlation is// only sampled every 33ms, so the discrete peak can sit a whole frame off// truth and the parabola then refines the wrong summit. On the YC clip a// single pass read 123ms when the answer was 95ms. If you want it exact:// apply the measurement, re-measure the corrected pair (bed-free), and add// the residual back. That converges in one step.
import { spawn } from "node:child_process";import { faceTrack } from "./reframe.mjs";import { decodeAudioMono, computeRmsEnvelope } from "../../pop/lib/preview-shared.mjs";
const FPS = 30;const PROBE_W = 320; // the mouth is a big blob at this size; we want energy, not detailconst MAX_LAG = 12; // ±400ms at 30fps — beyond this it isn't a sync error, it's an edit
// Motion energy inside the mouth region, frame by frame.async function mouthMotion(videoPath, track) { const scale = PROBE_W / track.srcW; const probeH = Math.round(track.srcH * scale);
const dec = spawn("ffmpeg", [ "-v", "error", "-i", videoPath, "-f", "rawvideo", "-pix_fmt", "gray", "-vf", `fps=${FPS},scale=${PROBE_W}:${probeH}`, "-", ], { stdio: ["ignore", "pipe", "inherit"] });
const frameBytes = PROBE_W * probeH; const energy = []; let prev = null; let pending = Buffer.alloc(0); let f = 0;
// The face track samples at its own rate; hold the nearest box for each frame. const boxAt = (t) => { let s = track.samples[0]; for (const c of track.samples) { if (c.t > t) break; s = c; } return s; };
for await (const chunk of dec.stdout) { pending = pending.length ? Buffer.concat([pending, chunk]) : chunk; while (pending.length >= frameBytes) { const g = pending.subarray(0, frameBytes); pending = pending.subarray(frameBytes);
const b = boxAt(f / FPS); // The mouth lives in the bottom third of Vision's face box, centered. Using // the whole face would fold in head-bobs and blinks, which don't correlate // with speech and only add noise. const x0 = Math.max(0, Math.round((b.x + b.w * 0.22) * scale)); const x1 = Math.min(PROBE_W, Math.round((b.x + b.w * 0.78) * scale)); const y0 = Math.max(0, Math.round((b.y + b.h * 0.62) * scale)); const y1 = Math.min(probeH, Math.round((b.y + b.h * 1.02) * scale));
let sum = 0; let n = 0; if (prev) { for (let y = y0; y < y1; y += 1) { const row = y * PROBE_W; for (let x = x0; x < x1; x += 1) { sum += Math.abs(g[row + x] - prev[row + x]); n += 1; } } } energy.push(n > 0 ? sum / n : 0); prev = Buffer.from(g); // subarray aliases the pool; copy or it mutates f += 1; } } return energy;}
const normalize = (xs) => { const mean = xs.reduce((a, b) => a + b, 0) / (xs.length || 1); const sd = Math.sqrt(xs.reduce((a, b) => a + (b - mean) ** 2, 0) / (xs.length || 1)) || 1; return xs.map((x) => (x - mean) / sd);};
export async function measureAvOffset(videoPath, { track = null } = {}) { const tr = track || faceTrack(videoPath, { fps: FPS });
const motion = normalize(await mouthMotion(videoPath, tr)); const { audio, sr } = decodeAudioMono(videoPath); const env = normalize(Array.from(computeRmsEnvelope(audio, sr, FPS, tr.duration)));
const n = Math.min(motion.length, env.length); const curve = [];
// Slide the motion signal against the audio. `lag` is how many frames the // picture would need to move to line up: at lag k we compare motion[i + k] // with audio[i], so a positive k that wins means the mouth is moving LATER // than the sound — the picture is behind. for (let lag = -MAX_LAG; lag <= MAX_LAG; lag += 1) { let sum = 0; let count = 0; for (let i = 0; i < n; i += 1) { const j = i + lag; if (j < 0 || j >= n) continue; sum += motion[j] * env[i]; count += 1; } curve.push({ lag, r: count > 0 ? sum / count : 0 }); }
const bestI = curve.reduce((bi, c, i) => (c.r > curve[bi].r ? i : bi), 0); const mean = curve.reduce((a, b) => a + Math.abs(b.r), 0) / curve.length;
// The correlation is only sampled at whole frames, but the true offset is not // a whole number of frames — it's whatever the camera's encoder did. Fitting a // parabola through the peak and its two neighbours recovers the sub-frame // position of the real maximum, which matters because we correct on the audio // side, where we have sample precision to spend. let lag = curve[bestI].lag; const l = curve[bestI - 1]; const c0 = curve[bestI]; const r = curve[bestI + 1]; if (l && r) { const denom = l.r - 2 * c0.r + r.r; if (denom !== 0) { const delta = (0.5 * (l.r - r.r)) / denom; if (Math.abs(delta) <= 1) lag += delta; } }
return { offsetMs: Math.round((lag / FPS) * 1000), confidence: mean > 0 ? c0.r / mean : 0, curve, };}
if (import.meta.url === `file://${process.argv[1]}`) { const video = process.argv[2]; if (!video) { console.error("usage: av-sync.mjs <video>"); process.exit(2); } const { offsetMs, confidence, curve } = await measureAvOffset(video);
console.log("\nlag(ms) correlation"); for (const c of curve) { const ms = Math.round((c.lag / FPS) * 1000); const bar = "█".repeat(Math.max(0, Math.round(c.r * 60))); console.log(`${String(ms).padStart(5)} ${bar} ${c.r.toFixed(3)}`); } console.log(`\noffset: ${offsetMs}ms confidence: ${confidence.toFixed(2)}`); console.log( offsetMs > 0 ? `the PICTURE is ${offsetMs}ms late — visuals behind the sound` : offsetMs < 0 ? `the SOUND is ${-offsetMs}ms late` : "in sync", ); if (confidence < 1.15) console.log("⚠ weak peak — don't trust this; too little mouth motion or too much noise");}