/** * Captions v3 — Full pre-computation with sentence splitting. * * 1. Fetches all HLS audio segments for the video * 2. Decodes to PCM in large chunks (30s) * 3. Sends each chunk to Whisper for transcription * 4. Collects all sentence-level results with timestamps * 5. Post-processes: merges fragments, splits on sentence boundaries * 6. Displays captions based on video playback time * * No real-time audio capture — everything is pre-computed. */ const PLAYBACK_BASE = 'https://vod-beta.stream.place/xrpc/place.stream.playback'; const TARGET_SAMPLE_RATE = 16000; const CHUNK_SECONDS = 30; // Large chunks for best Whisper context const MODEL_DOWNLOADED_KEY = 'vodfrog-whisper-downloaded'; type ModelStatus = 'idle' | 'loading' | 'ready' | 'error'; interface Caption { text: string; start: number; end: number; } interface AudioSegment { url: string; byteRange?: { start: number; length: number }; duration: number; startTime: number; } // ---- Reactive state ---- let modelStatus = $state('idle'); let modelProgress = $state(0); let modelError = $state(''); let worker: Worker | null = null; let captionsEnabled = $state(false); let currentCaption = $state(''); let captions = $state([]); let lastVideoTime = $state(0); let isProcessing = $state(false); let processProgress = $state(0); let pendingAtUri = ''; let abortCtrl: AbortController | null = null; let rawCaptionBuffer: Caption[] = []; let processedRanges = $state<{ start: number; end: number }[]>([]); // ---- Exports ---- export function getModelStatus() { return modelStatus; } export function getModelProgress() { return modelProgress; } export function getModelError() { return modelError; } export function getCaptionsEnabled() { return captionsEnabled; } export function getCurrentCaption() { return currentCaption; } export function getCaptionCount() { return captions.length; } export function getIsProcessing() { return isProcessing; } export function getProcessProgress() { return processProgress; } export function getAllCaptions() { return captions; } /** Replace captions with user-edited versions. Stops further processing to prevent overwrites. */ export function setCaptions(newCaptions: Caption[]) { abortCtrl?.abort(); // Stop processing so it doesn't overwrite edits captions = newCaptions; rawCaptionBuffer = newCaptions; // Keep in sync console.log('[CC] Captions updated by user:', newCaptions.length); } export function getProcessedRanges() { return processedRanges; } export function getVideoTime() { return lastVideoTime; } /** Load captions from an existing AT Proto record (from Constellation lookup) */ export function loadExistingCaptions(entries: { timestamp: string; text: string; speaker?: string }[]) { const loaded: Caption[] = []; for (let i = 0; i < entries.length; i++) { const start = parseFloat(entries[i].timestamp) || 0; const nextStart = i < entries.length - 1 ? parseFloat(entries[i + 1].timestamp) || 0 : start + estimateDuration(entries[i].text); if (entries[i].text?.trim()) { loaded.push({ text: entries[i].text.trim(), start, end: nextStart }); } } captions = loaded; processedRanges = loaded.length > 0 ? [{ start: loaded[0].start, end: loaded[loaded.length - 1].end }] : []; console.log(`[CC] Loaded ${loaded.length} captions from existing record`); } function estimateDuration(text: string): number { return Math.max(2, (text?.split(/\s+/).length || 1) * 0.35); } export function getDebugState(videoTime: number) { return { modelStatus, captionsEnabled, isProcessing, processProgress: processProgress.toFixed(0), totalCaptions: captions.length, currentCaption, videoTime, processedRanges, captions: captions.map(c => ({ text: c.text.substring(0, 50), start: c.start, end: c.end })) }; } // ---- Model ---- export function initCaptions() { if (typeof localStorage === 'undefined') return; if (localStorage.getItem(MODEL_DOWNLOADED_KEY) === 'true') loadModel(); } export function loadModel() { if (modelStatus === 'loading' || modelStatus === 'ready') return; modelStatus = 'loading'; modelProgress = 0; modelError = ''; worker = new Worker(new URL('./whisper-worker.ts', import.meta.url), { type: 'module' }); worker.onmessage = onWorkerMessage; worker.postMessage({ type: 'load' }); } let resolveTranscription: ((caps: Caption[]) => void) | null = null; function onWorkerMessage(e: MessageEvent) { const { type } = e.data; if (type === 'status') { if (e.data.status === 'ready') { modelStatus = 'ready'; modelProgress = 100; try { localStorage.setItem(MODEL_DOWNLOADED_KEY, 'true'); } catch {} if (pendingAtUri) { const uri = pendingAtUri; pendingAtUri = ''; precomputeCaptions(uri); } } else if (e.data.status === 'error') { modelStatus = 'error'; modelError = e.data.error; } } if (type === 'progress') modelProgress = e.data.progress; if (type === 'result') { const results: Caption[] = []; if (e.data.chunks?.length) { for (const c of e.data.chunks) { const text = c.text?.trim(); if (text && text.length > 1) { results.push({ text, start: c.start ?? 0, end: c.end ?? 0 }); } } } else if (e.data.text?.trim()) { results.push({ text: e.data.text.trim(), start: 0, end: 0 }); } if (resolveTranscription) { const r = resolveTranscription; resolveTranscription = null; r(results); } } } function transcribe(audio: Float32Array): Promise { return new Promise(resolve => { if (!worker) { resolve([]); return; } resolveTranscription = resolve; worker.postMessage({ type: 'transcribe', audio, sampleRate: TARGET_SAMPLE_RATE }); }); } // ---- HLS parsing ---- function parseM3u8(text: string, baseUrl: string) { const lines = text.split('\n'); let initUrl: string | null = null; let initBR: { start: number; length: number } | null = null; const segs: AudioSegment[] = []; let dur = 0, cum = 0; let pBR: { start: number; length: number } | null = null; for (const l of lines) { const t = l.trim(); if (t.startsWith('#EXT-X-MAP:')) { const m = t.match(/URI="([^"]+)"/); if (m) initUrl = new URL(m[1], baseUrl).href; const b = t.match(/BYTERANGE="(\d+)@(\d+)"/); if (b) initBR = { length: +b[1], start: +b[2] }; } if (t.startsWith('#EXTINF:')) dur = parseFloat(t.split(':')[1]); if (t.startsWith('#EXT-X-BYTERANGE:')) { const [l, o] = t.split(':')[1].split('@'); pBR = { length: +l, start: +o }; } if (!t.startsWith('#') && t.length > 0 && dur > 0) { segs.push({ url: new URL(t, baseUrl).href, byteRange: pBR || undefined, duration: dur, startTime: cum }); cum += dur; dur = 0; pBR = null; } } return { initUrl, initBR, segs }; } function findAudioUrl(master: string, base: string): string | null { for (const l of master.split('\n')) { if (l.includes('EXT-X-MEDIA') && l.includes('TYPE=AUDIO') && l.includes('mp4a')) { const m = l.match(/URI="([^"]+)"/); if (m) return new URL(m[1], base).href; } } for (const l of master.split('\n')) { if (l.includes('EXT-X-MEDIA') && l.includes('TYPE=AUDIO')) { const m = l.match(/URI="([^"]+)"/); if (m) return new URL(m[1], base).href; } } return null; } async function fetchRange(url: string, br: { start: number; length: number }, signal?: AbortSignal) { return (await fetch(url, { headers: { Range: `bytes=${br.start}-${br.start + br.length - 1}` }, signal })).arrayBuffer(); } async function decodeToMono16k(data: ArrayBuffer): Promise { try { const ctx = new OfflineAudioContext(1, 1, TARGET_SAMPLE_RATE); const decoded = await ctx.decodeAudioData(data.slice(0)); const n = Math.ceil(decoded.duration * TARGET_SAMPLE_RATE); const off = new OfflineAudioContext(1, n, TARGET_SAMPLE_RATE); const s = off.createBufferSource(); s.buffer = decoded; s.connect(off.destination); s.start(0); return (await off.startRendering()).getChannelData(0); } catch { return new Float32Array(0); } } // ---- Sentence splitting ---- function splitIntoSentences(rawCaptions: Caption[]): Caption[] { if (rawCaptions.length === 0) return []; // Merge all raw captions into one stream, preserving timestamps const merged: { text: string; start: number; end: number }[] = []; for (const c of rawCaptions) { merged.push({ text: c.text, start: c.start, end: c.end }); } // Now split on sentence boundaries const result: Caption[] = []; let accText = ''; let accStart = merged[0].start; for (const m of merged) { accText += (accText ? ' ' : '') + m.text; // Check if accumulated text ends with sentence-ending punctuation if (/[.!?][\s]*$/.test(accText.trim())) { result.push({ text: accText.trim(), start: accStart, end: m.end > 0 ? m.end : m.start + estimateDuration(accText) }); accText = ''; accStart = m.end > 0 ? m.end : m.start + estimateDuration(accText); } } // Flush remaining if (accText.trim()) { const lastEnd = merged[merged.length - 1].end; result.push({ text: accText.trim(), start: accStart, end: lastEnd > 0 ? lastEnd : accStart + estimateDuration(accText) }); } return result; } // ---- Pre-computation ---- export async function precomputeCaptions(atUri: string) { if (modelStatus !== 'ready' || !worker) { pendingAtUri = atUri; console.log('[CC] Model not ready, queuing:', atUri); return; } abortCtrl?.abort(); abortCtrl = new AbortController(); const signal = abortCtrl.signal; isProcessing = true; processProgress = 0; captions = []; rawCaptionBuffer = []; processedRanges = []; currentCaption = ''; try { const url = `${PLAYBACK_BASE}.getVideoPlaylist?uri=${encodeURIComponent(atUri)}`; console.log('[CC] Fetching manifest...'); const masterM3u8 = await (await fetch(url, { signal })).text(); const audioUrl = findAudioUrl(masterM3u8, url); if (!audioUrl) { console.warn('[CC] No audio track'); isProcessing = false; return; } const audioM3u8 = await (await fetch(audioUrl, { signal })).text(); const { initUrl, initBR, segs } = parseM3u8(audioM3u8, audioUrl); if (segs.length === 0) { console.warn('[CC] No segments'); isProcessing = false; return; } let initData: ArrayBuffer | null = null; if (initUrl) { initData = initBR ? await fetchRange(initUrl, initBR, signal) : await (await fetch(initUrl, { signal })).arrayBuffer(); } const totalDuration = segs[segs.length - 1].startTime + segs[segs.length - 1].duration; console.log(`[CC] ${segs.length} segments, ${totalDuration.toFixed(0)}s total`); // Group segments into ~30s chunks let i = 0; while (i < segs.length) { if (signal.aborted) break; const chunkStart = segs[i].startTime; const groupSegs: AudioSegment[] = []; let groupDur = 0; while (i < segs.length && groupDur < CHUNK_SECONDS) { groupSegs.push(segs[i]); groupDur += segs[i].duration; i++; } const chunkEnd = groupSegs[groupSegs.length - 1].startTime + groupSegs[groupSegs.length - 1].duration; // Fetch segments const bufs: ArrayBuffer[] = []; for (const seg of groupSegs) { if (signal.aborted) break; bufs.push(seg.byteRange ? await fetchRange(seg.url, seg.byteRange, signal) : await (await fetch(seg.url, { signal })).arrayBuffer()); } if (signal.aborted) break; // Concat init + segments const totalSize = (initData?.byteLength ?? 0) + bufs.reduce((a, b) => a + b.byteLength, 0); const combined = new Uint8Array(totalSize); let off = 0; if (initData) { combined.set(new Uint8Array(initData), 0); off = initData.byteLength; } for (const b of bufs) { combined.set(new Uint8Array(b), off); off += b.byteLength; } // Decode + transcribe const pcm = await decodeToMono16k(combined.buffer); if (pcm.length === 0 || signal.aborted) continue; console.log(`[CC] Transcribing ${chunkStart.toFixed(0)}s-${chunkEnd.toFixed(0)}s...`); const results = await transcribe(pcm); if (signal.aborted) break; // Offset timestamps to absolute video time const chunkCaptions: Caption[] = []; for (const c of results) { chunkCaptions.push({ text: c.text, start: chunkStart + c.start, end: chunkStart + (c.end > 0 ? c.end : chunkEnd - chunkStart) }); } // Incrementally coalesce into sentence captions and update live if (chunkCaptions.length > 0) { rawCaptionBuffer = [...rawCaptionBuffer, ...chunkCaptions]; captions = splitIntoSentences(rawCaptionBuffer); processedRanges = [...processedRanges, { start: chunkStart, end: chunkEnd }]; } processProgress = (i / segs.length) * 100; console.log(`[CC] Progress: ${processProgress.toFixed(0)}% — ${captions.length} captions ready`); } if (!signal.aborted) { console.log(`[CC] Done! ${captions.length} sentence captions`); } } catch (err: any) { if (err.name !== 'AbortError') console.error('[CC] Failed:', err); } isProcessing = false; processProgress = 100; } // ---- Display ---- export function toggleCaptionsDisplay() { captionsEnabled = !captionsEnabled; if (!captionsEnabled) currentCaption = ''; } export function updateCaptionForTime(time: number) { lastVideoTime = time; if (!captionsEnabled || captions.length === 0) { currentCaption = ''; return; } for (const c of captions) { if (time >= c.start && time <= c.end) { currentCaption = c.text; return; } } currentCaption = ''; } export function destroyCaptions() { abortCtrl?.abort(); captionsEnabled = false; currentCaption = ''; captions = []; isProcessing = false; }