/** * Sprite sheet and WebVTT generation for video thumbnails * * Optimized approach: Instead of processing the entire video stream, * we parse the HLS playlist, identify which segments contain our target * timestamps, and only fetch those specific segments (~95% bandwidth reduction). */ import { ffmpeg } from "./ffmpeg" import { Parser } from "m3u8-parser" interface ThumbnailData { start: number end: number x: number y: number w: number h: number } interface Segment { uri: string duration: number startTime: number endTime: number byteRange?: { length: number; offset: number } } // High quality sprite settings const FRAME_COUNT = 8 const THUMB_W = 320 // Higher quality (was 160) const THUMB_H = 180 // 16:9 aspect ratio (was 90) const COLS = 4 const ROWS = 2 /** * Parse an HLS playlist and extract segment information with timing */ async function parsePlaylist(playlistPath: string): Promise<{ segments: Segment[]; totalDuration: number; initSegmentUrl?: string }> { console.log(`[sprites] Parsing playlist: ${playlistPath}`) // Handle both local files and remote URLs let content: string if (playlistPath.startsWith('http')) { const res = await fetch(playlistPath) if (!res.ok) { throw new Error(`Failed to fetch playlist: ${res.status} ${res.statusText}`) } content = await res.text() } else { content = await Deno.readTextFile(playlistPath) } const parser = new Parser() parser.push(content) parser.end() const manifest = parser.manifest // Handle master playlist - need to get the actual media playlist if (manifest.playlists && manifest.playlists.length > 0) { // Get the highest quality variant const variant = manifest.playlists.sort((a: { attributes?: { BANDWIDTH?: number } }, b: { attributes?: { BANDWIDTH?: number } }) => (b.attributes?.BANDWIDTH || 0) - (a.attributes?.BANDWIDTH || 0) )[0] // Fetch the media playlist const variantUri = variant.uri console.log(`[sprites] Variant URI from manifest: ${variantUri}`) // For URLs, extract base properly; for local files, use directory let baseUrl: string if (playlistPath.startsWith('http')) { // For http URLs, get base up to last / const url = new URL(playlistPath) baseUrl = url.origin + url.pathname.substring(0, url.pathname.lastIndexOf('/') + 1) } else { baseUrl = playlistPath.includes('/') ? playlistPath.substring(0, playlistPath.lastIndexOf('/') + 1) : '' } // If variant URI is already absolute, use it directly const mediaPlaylistUrl = variantUri.startsWith('http') ? variantUri : baseUrl + variantUri console.log(`[sprites] Fetching variant playlist: ${mediaPlaylistUrl}`) // For remote URLs, fetch; for local files, read let mediaContent: string if (mediaPlaylistUrl.startsWith('http')) { const res = await fetch(mediaPlaylistUrl) mediaContent = await res.text() } else { mediaContent = await Deno.readTextFile(mediaPlaylistUrl) } const mediaParser = new Parser() mediaParser.push(mediaContent) mediaParser.end() // Use the actual URL the variant came from to resolve segment URLs // This ensures segments resolve correctly against the CDN, not XRPC proxy return buildSegmentList(mediaParser.manifest.segments || [], mediaPlaylistUrl, mediaParser.manifest) } // Already a media playlist return buildSegmentList(manifest.segments || [], playlistPath, manifest) } function buildSegmentList( rawSegments: Array<{ uri: string; duration: number; byterange?: { length: number; offset: number }; map?: { uri: string } }>, playlistUri: string, manifest?: { segments?: Array<{ map?: { uri: string } }> } ): { segments: Segment[]; totalDuration: number; initSegmentUrl?: string } { const baseUrl = playlistUri.includes('/') ? playlistUri.substring(0, playlistUri.lastIndexOf('/') + 1) : '' console.log(`[sprites] Building segment list from: ${playlistUri}`) console.log(`[sprites] Base URL: ${baseUrl}`) console.log(`[sprites] Raw segments count: ${rawSegments.length}`) // Extract init segment URL from the first segment's map if available let initSegmentUrl: string | undefined const firstSegmentMap = rawSegments[0]?.map || manifest?.segments?.[0]?.map if (firstSegmentMap?.uri) { initSegmentUrl = firstSegmentMap.uri.startsWith('http') ? firstSegmentMap.uri : baseUrl + firstSegmentMap.uri console.log(`[sprites] Init segment URL: ${initSegmentUrl}`) } if (rawSegments.length > 0) { console.log(`[sprites] First raw segment URI: ${rawSegments[0].uri}`) const firstResolved = rawSegments[0].uri.startsWith('http') ? rawSegments[0].uri : baseUrl + rawSegments[0].uri console.log(`[sprites] First resolved segment URL: ${firstResolved}`) if (rawSegments[0].byterange) { console.log(`[sprites] First segment byteRange: ${rawSegments[0].byterange.length}@${rawSegments[0].byterange.offset}`) } } const segments: Segment[] = [] let currentTime = 0 for (const seg of rawSegments) { let uri = seg.uri.startsWith('http') ? seg.uri : baseUrl + seg.uri // Check for problematic XRPC proxy URLs - segments should be on CDN, not proxied if (uri.includes('/xrpc/') && !uri.endsWith('.ts')) { console.warn(`[sprites] WARNING: Segment URL looks like an XRPC proxy, not a direct segment: ${uri}`) // Try to extract actual segment path and use CDN // Skip this segment if we can't fix it } segments.push({ uri, duration: seg.duration, startTime: currentTime, endTime: currentTime + seg.duration, byteRange: seg.byterange, }) currentTime += seg.duration } return { segments, totalDuration: currentTime, initSegmentUrl } } /** * Find the segment containing a specific timestamp */ function findSegmentForTime(segments: Segment[], targetTime: number): { segment: Segment; offset: number } | null { for (const segment of segments) { if (targetTime >= segment.startTime && targetTime < segment.endTime) { return { segment, offset: targetTime - segment.startTime, } } } // Return last segment if time is at the very end if (segments.length > 0 && targetTime >= segments[segments.length - 1].endTime) { const lastSeg = segments[segments.length - 1] return { segment: lastSeg, offset: lastSeg.duration - 0.1, // Slightly before end } } return null } // Max segment size: 50MB (HLS segments are typically 2-10 seconds, ~1-5MB each) const MAX_SEGMENT_SIZE = 50 * 1024 * 1024 /** * Download data from a URL with optional byte-range * Returns the raw data as Uint8Array */ async function downloadData( url: string, byteRange?: { length: number; offset: number } ): Promise { const headers: Record = {} if (byteRange) { const start = byteRange.offset const end = byteRange.offset + byteRange.length - 1 headers['Range'] = `bytes=${start}-${end}` } const response = await fetch(url, { headers }) // Accept 200 (full response) or 206 (partial content for byte-range) if (!response.ok && response.status !== 206) { throw new Error(`Failed to download: ${response.status} ${response.statusText}`) } // Check content length before downloading const contentLength = response.headers.get('content-length') if (contentLength) { const size = parseInt(contentLength, 10) if (size > MAX_SEGMENT_SIZE) { throw new Error(`Download too large: ${(size / 1024 / 1024).toFixed(1)}MB (max ${MAX_SEGMENT_SIZE / 1024 / 1024}MB)`) } } return new Uint8Array(await response.arrayBuffer()) } /** * Download a segment to a local temp file * This is faster than letting ffmpeg handle HTTP directly * Supports byte-range requests for HLS streams that use byte ranges on a single file * For fragmented MP4, can prepend init segment data */ async function downloadSegment( segmentUri: string, localPath: string, byteRange?: { length: number; offset: number }, initData?: Uint8Array ): Promise { const segmentName = segmentUri.split('/').pop() || 'segment' console.log(`[sprites] Downloading: ${segmentName}`) console.log(`[sprites] Full URL: ${segmentUri}`) if (byteRange) { console.log(`[sprites] Byte range: ${byteRange.offset}-${byteRange.offset + byteRange.length - 1} (${(byteRange.length / 1024).toFixed(0)}KB)`) } // Download the segment data const segmentData = await downloadData(segmentUri, byteRange) console.log(`[sprites] Downloaded segment: ${(segmentData.length / 1024).toFixed(0)}KB`) // Write to file, prepending init data if provided const file = await Deno.open(localPath, { write: true, create: true, truncate: true }) try { if (initData) { await file.write(initData) console.log(`[sprites] Prepended init segment: ${(initData.length / 1024).toFixed(0)}KB`) } await file.write(segmentData) } finally { file.close() } const totalSize = (initData?.length || 0) + segmentData.length console.log(`[sprites] Total file size: ${(totalSize / 1024).toFixed(0)}KB`) } /** * Extract a single frame from a specific segment at a given offset */ async function extractFrame( segmentUri: string, offset: number, outputPath: string, width: number, height: number, tempDir: string, byteRange?: { length: number; offset: number }, initData?: Uint8Array ): Promise { // Download segment locally first - much faster than letting ffmpeg handle HTTP // Use .mp4 extension for fragmented MP4 segments (when init data is present) const ext = initData ? '.mp4' : '.ts' const segmentFileName = `segment_${Date.now()}${ext}` const localSegmentPath = `${tempDir}/${segmentFileName}` try { await downloadSegment(segmentUri, localSegmentPath, byteRange, initData) // Now extract frame from local file - ffmpeg is much faster with local files // -ss BEFORE -i = fast seek (doesn't decode everything) const result = await ffmpeg([ "-ss", offset.toFixed(3), "-i", localSegmentPath, "-vframes", "1", "-vf", `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2`, "-q:v", "2", // High quality "-y", outputPath, ]) if (!result.success) { throw new Error(`Failed to extract frame: ${result.stderr}`) } } finally { // Clean up downloaded segment try { await Deno.remove(localSegmentPath) } catch { // Ignore cleanup errors } } } /** * Composite individual frames into a sprite sheet */ async function createSpriteSheet( framePaths: string[], outputPath: string, cols: number, rows: number ): Promise { // Build input args for all frames const inputs: string[] = [] for (const path of framePaths) { inputs.push("-i", path) } // Create tile filter const result = await ffmpeg([ ...inputs, "-filter_complex", `xstack=inputs=${framePaths.length}:layout=${generateLayout(cols, rows, THUMB_W, THUMB_H)}`, "-q:v", "2", "-y", outputPath, ]) if (!result.success) { // Fallback to tile filter if xstack fails const tileResult = await ffmpeg([ ...inputs, "-filter_complex", `concat=n=${framePaths.length}:v=1:a=0,tile=${cols}x${rows}`, "-q:v", "2", "-y", outputPath, ]) if (!tileResult.success) { throw new Error(`Failed to create sprite sheet: ${tileResult.stderr}`) } } } /** * Generate xstack layout string for grid arrangement */ function generateLayout(cols: number, rows: number, w: number, h: number): string { const positions: string[] = [] for (let row = 0; row < rows; row++) { for (let col = 0; col < cols; col++) { const x = col * w const y = row * h positions.push(`${x}_${y}`) } } return positions.join("|") } /** * Generate sprite sheet and VTT using optimized segment-based extraction * * This fetches only ~8 segments instead of the entire video stream, * reducing bandwidth by ~95% for long videos. */ export async function generateSpriteWithVtt( input: string, basePath: string ): Promise { console.log(`[sprites] Parsing playlist: ${input}`) // Parse the HLS playlist to get segment information const { segments, totalDuration, initSegmentUrl } = await parsePlaylist(input) if (segments.length === 0) { throw new Error("No segments found in playlist") } console.log(`[sprites] Found ${segments.length} segments, total duration: ${totalDuration.toFixed(1)}s`) // Download init segment if present (needed for fragmented MP4) let initData: Uint8Array | undefined if (initSegmentUrl) { console.log(`[sprites] Downloading init segment: ${initSegmentUrl}`) initData = await downloadData(initSegmentUrl) console.log(`[sprites] Init segment size: ${(initData.length / 1024).toFixed(0)}KB`) } const interval = totalDuration / FRAME_COUNT const thumbs: ThumbnailData[] = [] const framePaths: string[] = [] // Extract frames from targeted segments only for (let i = 0; i < FRAME_COUNT; i++) { const targetTime = i * interval + (interval / 2) // Middle of each interval const framePath = `${basePath}/frame_${i}.jpg` framePaths.push(framePath) const segmentInfo = findSegmentForTime(segments, targetTime) if (!segmentInfo) { console.warn(`[sprites] No segment found for time ${targetTime}, using first segment`) await extractFrame(segments[0].uri, 0, framePath, THUMB_W, THUMB_H, basePath, segments[0].byteRange, initData) } else { console.log(`[sprites] Frame ${i}: t=${targetTime.toFixed(1)}s -> segment offset=${segmentInfo.offset.toFixed(2)}s`) await extractFrame(segmentInfo.segment.uri, segmentInfo.offset, framePath, THUMB_W, THUMB_H, basePath, segmentInfo.segment.byteRange, initData) } // Build VTT data const start = i * interval const end = Math.min((i + 1) * interval, totalDuration) const col = i % COLS const row = Math.floor(i / COLS) thumbs.push({ start, end, x: col * THUMB_W, y: row * THUMB_H, w: THUMB_W, h: THUMB_H, }) } // Composite frames into sprite sheet console.log(`[sprites] Creating sprite sheet...`) const spriteOutput = `${basePath}/sprites.jpg` await createSpriteSheet(framePaths, spriteOutput, COLS, ROWS) // Clean up individual frame files for (const path of framePaths) { try { await Deno.remove(path) } catch { // Ignore cleanup errors } } // Write WebVTT file const vtt = buildWebVtt(thumbs, "sprites.jpg") await Deno.writeTextFile(`${basePath}/sprites.vtt`, vtt) console.log(`[sprites] Done! Sprite sheet: ${COLS * THUMB_W}x${ROWS * THUMB_H}px`) } function formatTime(seconds: number): string { const hrs = Math.floor(seconds / 3600) const mins = Math.floor((seconds % 3600) / 60) const secs = Math.floor(seconds % 60) const ms = Math.floor((seconds % 1) * 1000) return `${String(hrs).padStart(2, "0")}:${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}.${String(ms).padStart(3, "0")}` } function buildWebVtt(thumbs: ThumbnailData[], spriteUrl: string): string { const lines = ["WEBVTT", ""] for (const t of thumbs) { lines.push(`${formatTime(t.start)} --> ${formatTime(t.end)}`) lines.push(`${spriteUrl}#xywh=${t.x},${t.y},${t.w},${t.h}`) lines.push("") } return lines.join("\n") }