Something went wrong. Try again.
An entry for the streamplace vod showcase
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479/** * Sprite sheet and WebVTT generation for video thumbnails * * Optimized approach: Instead of processing the entire video stream, * we parse the HLS playlist, identify which segments contain our target * timestamps, and only fetch those specific segments (~95% bandwidth reduction). */
import { ffmpeg } from "./ffmpeg"import { Parser } from "m3u8-parser"
interface ThumbnailData { start: number end: number x: number y: number w: number h: number}
interface Segment { uri: string duration: number startTime: number endTime: number byteRange?: { length: number; offset: number }}
// High quality sprite settingsconst FRAME_COUNT = 8const THUMB_W = 320 // Higher quality (was 160)const THUMB_H = 180 // 16:9 aspect ratio (was 90)const COLS = 4const ROWS = 2
/** * Parse an HLS playlist and extract segment information with timing */async function parsePlaylist(playlistPath: string): Promise<{ segments: Segment[]; totalDuration: number; initSegmentUrl?: string }> { console.log(`[sprites] Parsing playlist: ${playlistPath}`)
// Handle both local files and remote URLs let content: string if (playlistPath.startsWith('http')) { const res = await fetch(playlistPath) if (!res.ok) { throw new Error(`Failed to fetch playlist: ${res.status} ${res.statusText}`) } content = await res.text() } else { content = await Deno.readTextFile(playlistPath) }
const parser = new Parser() parser.push(content) parser.end()
const manifest = parser.manifest
// Handle master playlist - need to get the actual media playlist if (manifest.playlists && manifest.playlists.length > 0) { // Get the highest quality variant const variant = manifest.playlists.sort((a: { attributes?: { BANDWIDTH?: number } }, b: { attributes?: { BANDWIDTH?: number } }) => (b.attributes?.BANDWIDTH || 0) - (a.attributes?.BANDWIDTH || 0) )[0]
// Fetch the media playlist const variantUri = variant.uri console.log(`[sprites] Variant URI from manifest: ${variantUri}`)
// For URLs, extract base properly; for local files, use directory let baseUrl: string if (playlistPath.startsWith('http')) { // For http URLs, get base up to last / const url = new URL(playlistPath) baseUrl = url.origin + url.pathname.substring(0, url.pathname.lastIndexOf('/') + 1) } else { baseUrl = playlistPath.includes('/') ? playlistPath.substring(0, playlistPath.lastIndexOf('/') + 1) : '' }
// If variant URI is already absolute, use it directly const mediaPlaylistUrl = variantUri.startsWith('http') ? variantUri : baseUrl + variantUri
console.log(`[sprites] Fetching variant playlist: ${mediaPlaylistUrl}`)
// For remote URLs, fetch; for local files, read let mediaContent: string if (mediaPlaylistUrl.startsWith('http')) { const res = await fetch(mediaPlaylistUrl) mediaContent = await res.text() } else { mediaContent = await Deno.readTextFile(mediaPlaylistUrl) }
const mediaParser = new Parser() mediaParser.push(mediaContent) mediaParser.end()
// Use the actual URL the variant came from to resolve segment URLs // This ensures segments resolve correctly against the CDN, not XRPC proxy return buildSegmentList(mediaParser.manifest.segments || [], mediaPlaylistUrl, mediaParser.manifest) }
// Already a media playlist return buildSegmentList(manifest.segments || [], playlistPath, manifest)}
function buildSegmentList( rawSegments: Array<{ uri: string; duration: number; byterange?: { length: number; offset: number }; map?: { uri: string } }>, playlistUri: string, manifest?: { segments?: Array<{ map?: { uri: string } }> }): { segments: Segment[]; totalDuration: number; initSegmentUrl?: string } { const baseUrl = playlistUri.includes('/') ? playlistUri.substring(0, playlistUri.lastIndexOf('/') + 1) : ''
console.log(`[sprites] Building segment list from: ${playlistUri}`) console.log(`[sprites] Base URL: ${baseUrl}`) console.log(`[sprites] Raw segments count: ${rawSegments.length}`)
// Extract init segment URL from the first segment's map if available let initSegmentUrl: string | undefined const firstSegmentMap = rawSegments[0]?.map || manifest?.segments?.[0]?.map if (firstSegmentMap?.uri) { initSegmentUrl = firstSegmentMap.uri.startsWith('http') ? firstSegmentMap.uri : baseUrl + firstSegmentMap.uri console.log(`[sprites] Init segment URL: ${initSegmentUrl}`) }
if (rawSegments.length > 0) { console.log(`[sprites] First raw segment URI: ${rawSegments[0].uri}`) const firstResolved = rawSegments[0].uri.startsWith('http') ? rawSegments[0].uri : baseUrl + rawSegments[0].uri console.log(`[sprites] First resolved segment URL: ${firstResolved}`) if (rawSegments[0].byterange) { console.log(`[sprites] First segment byteRange: ${rawSegments[0].byterange.length}@${rawSegments[0].byterange.offset}`) } }
const segments: Segment[] = [] let currentTime = 0
for (const seg of rawSegments) { let uri = seg.uri.startsWith('http') ? seg.uri : baseUrl + seg.uri
// Check for problematic XRPC proxy URLs - segments should be on CDN, not proxied if (uri.includes('/xrpc/') && !uri.endsWith('.ts')) { console.warn(`[sprites] WARNING: Segment URL looks like an XRPC proxy, not a direct segment: ${uri}`) // Try to extract actual segment path and use CDN // Skip this segment if we can't fix it }
segments.push({ uri, duration: seg.duration, startTime: currentTime, endTime: currentTime + seg.duration, byteRange: seg.byterange, }) currentTime += seg.duration }
return { segments, totalDuration: currentTime, initSegmentUrl }}
/** * Find the segment containing a specific timestamp */function findSegmentForTime(segments: Segment[], targetTime: number): { segment: Segment; offset: number } | null { for (const segment of segments) { if (targetTime >= segment.startTime && targetTime < segment.endTime) { return { segment, offset: targetTime - segment.startTime, } } } // Return last segment if time is at the very end if (segments.length > 0 && targetTime >= segments[segments.length - 1].endTime) { const lastSeg = segments[segments.length - 1] return { segment: lastSeg, offset: lastSeg.duration - 0.1, // Slightly before end } } return null}
// Max segment size: 50MB (HLS segments are typically 2-10 seconds, ~1-5MB each)const MAX_SEGMENT_SIZE = 50 * 1024 * 1024
/** * Download data from a URL with optional byte-range * Returns the raw data as Uint8Array */async function downloadData( url: string, byteRange?: { length: number; offset: number }): Promise<Uint8Array> { const headers: Record<string, string> = {} if (byteRange) { const start = byteRange.offset const end = byteRange.offset + byteRange.length - 1 headers['Range'] = `bytes=${start}-${end}` }
const response = await fetch(url, { headers })
// Accept 200 (full response) or 206 (partial content for byte-range) if (!response.ok && response.status !== 206) { throw new Error(`Failed to download: ${response.status} ${response.statusText}`) }
// Check content length before downloading const contentLength = response.headers.get('content-length') if (contentLength) { const size = parseInt(contentLength, 10) if (size > MAX_SEGMENT_SIZE) { throw new Error(`Download too large: ${(size / 1024 / 1024).toFixed(1)}MB (max ${MAX_SEGMENT_SIZE / 1024 / 1024}MB)`) } }
return new Uint8Array(await response.arrayBuffer())}
/** * Download a segment to a local temp file * This is faster than letting ffmpeg handle HTTP directly * Supports byte-range requests for HLS streams that use byte ranges on a single file * For fragmented MP4, can prepend init segment data */async function downloadSegment( segmentUri: string, localPath: string, byteRange?: { length: number; offset: number }, initData?: Uint8Array): Promise<void> { const segmentName = segmentUri.split('/').pop() || 'segment' console.log(`[sprites] Downloading: ${segmentName}`) console.log(`[sprites] Full URL: ${segmentUri}`) if (byteRange) { console.log(`[sprites] Byte range: ${byteRange.offset}-${byteRange.offset + byteRange.length - 1} (${(byteRange.length / 1024).toFixed(0)}KB)`) }
// Download the segment data const segmentData = await downloadData(segmentUri, byteRange) console.log(`[sprites] Downloaded segment: ${(segmentData.length / 1024).toFixed(0)}KB`)
// Write to file, prepending init data if provided const file = await Deno.open(localPath, { write: true, create: true, truncate: true }) try { if (initData) { await file.write(initData) console.log(`[sprites] Prepended init segment: ${(initData.length / 1024).toFixed(0)}KB`) } await file.write(segmentData) } finally { file.close() }
const totalSize = (initData?.length || 0) + segmentData.length console.log(`[sprites] Total file size: ${(totalSize / 1024).toFixed(0)}KB`)}
/** * Extract a single frame from a specific segment at a given offset */async function extractFrame( segmentUri: string, offset: number, outputPath: string, width: number, height: number, tempDir: string, byteRange?: { length: number; offset: number }, initData?: Uint8Array): Promise<void> { // Download segment locally first - much faster than letting ffmpeg handle HTTP // Use .mp4 extension for fragmented MP4 segments (when init data is present) const ext = initData ? '.mp4' : '.ts' const segmentFileName = `segment_${Date.now()}${ext}` const localSegmentPath = `${tempDir}/${segmentFileName}`
try { await downloadSegment(segmentUri, localSegmentPath, byteRange, initData)
// Now extract frame from local file - ffmpeg is much faster with local files // -ss BEFORE -i = fast seek (doesn't decode everything) const result = await ffmpeg([ "-ss", offset.toFixed(3), "-i", localSegmentPath, "-vframes", "1", "-vf", `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2`, "-q:v", "2", // High quality "-y", outputPath, ])
if (!result.success) { throw new Error(`Failed to extract frame: ${result.stderr}`) } } finally { // Clean up downloaded segment try { await Deno.remove(localSegmentPath) } catch { // Ignore cleanup errors } }}
/** * Composite individual frames into a sprite sheet */async function createSpriteSheet( framePaths: string[], outputPath: string, cols: number, rows: number): Promise<void> { // Build input args for all frames const inputs: string[] = [] for (const path of framePaths) { inputs.push("-i", path) }
// Create tile filter const result = await ffmpeg([ ...inputs, "-filter_complex", `xstack=inputs=${framePaths.length}:layout=${generateLayout(cols, rows, THUMB_W, THUMB_H)}`, "-q:v", "2", "-y", outputPath, ])
if (!result.success) { // Fallback to tile filter if xstack fails const tileResult = await ffmpeg([ ...inputs, "-filter_complex", `concat=n=${framePaths.length}:v=1:a=0,tile=${cols}x${rows}`, "-q:v", "2", "-y", outputPath, ])
if (!tileResult.success) { throw new Error(`Failed to create sprite sheet: ${tileResult.stderr}`) } }}
/** * Generate xstack layout string for grid arrangement */function generateLayout(cols: number, rows: number, w: number, h: number): string { const positions: string[] = [] for (let row = 0; row < rows; row++) { for (let col = 0; col < cols; col++) { const x = col * w const y = row * h positions.push(`${x}_${y}`) } } return positions.join("|")}
/** * Generate sprite sheet and VTT using optimized segment-based extraction * * This fetches only ~8 segments instead of the entire video stream, * reducing bandwidth by ~95% for long videos. */export async function generateSpriteWithVtt( input: string, basePath: string): Promise<void> { console.log(`[sprites] Parsing playlist: ${input}`)
// Parse the HLS playlist to get segment information const { segments, totalDuration, initSegmentUrl } = await parsePlaylist(input)
if (segments.length === 0) { throw new Error("No segments found in playlist") }
console.log(`[sprites] Found ${segments.length} segments, total duration: ${totalDuration.toFixed(1)}s`)
// Download init segment if present (needed for fragmented MP4) let initData: Uint8Array | undefined if (initSegmentUrl) { console.log(`[sprites] Downloading init segment: ${initSegmentUrl}`) initData = await downloadData(initSegmentUrl) console.log(`[sprites] Init segment size: ${(initData.length / 1024).toFixed(0)}KB`) }
const interval = totalDuration / FRAME_COUNT const thumbs: ThumbnailData[] = [] const framePaths: string[] = []
// Extract frames from targeted segments only for (let i = 0; i < FRAME_COUNT; i++) { const targetTime = i * interval + (interval / 2) // Middle of each interval const framePath = `${basePath}/frame_${i}.jpg` framePaths.push(framePath)
const segmentInfo = findSegmentForTime(segments, targetTime)
if (!segmentInfo) { console.warn(`[sprites] No segment found for time ${targetTime}, using first segment`) await extractFrame(segments[0].uri, 0, framePath, THUMB_W, THUMB_H, basePath, segments[0].byteRange, initData) } else { console.log(`[sprites] Frame ${i}: t=${targetTime.toFixed(1)}s -> segment offset=${segmentInfo.offset.toFixed(2)}s`) await extractFrame(segmentInfo.segment.uri, segmentInfo.offset, framePath, THUMB_W, THUMB_H, basePath, segmentInfo.segment.byteRange, initData) }
// Build VTT data const start = i * interval const end = Math.min((i + 1) * interval, totalDuration) const col = i % COLS const row = Math.floor(i / COLS)
thumbs.push({ start, end, x: col * THUMB_W, y: row * THUMB_H, w: THUMB_W, h: THUMB_H, }) }
// Composite frames into sprite sheet console.log(`[sprites] Creating sprite sheet...`) const spriteOutput = `${basePath}/sprites.jpg` await createSpriteSheet(framePaths, spriteOutput, COLS, ROWS)
// Clean up individual frame files for (const path of framePaths) { try { await Deno.remove(path) } catch { // Ignore cleanup errors } }
// Write WebVTT file const vtt = buildWebVtt(thumbs, "sprites.jpg") await Deno.writeTextFile(`${basePath}/sprites.vtt`, vtt)
console.log(`[sprites] Done! Sprite sheet: ${COLS * THUMB_W}x${ROWS * THUMB_H}px`)}
function formatTime(seconds: number): string { const hrs = Math.floor(seconds / 3600) const mins = Math.floor((seconds % 3600) / 60) const secs = Math.floor(seconds % 60) const ms = Math.floor((seconds % 1) * 1000)
return `${String(hrs).padStart(2, "0")}:${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}.${String(ms).padStart(3, "0")}`}
function buildWebVtt(thumbs: ThumbnailData[], spriteUrl: string): string { const lines = ["WEBVTT", ""]
for (const t of thumbs) { lines.push(`${formatTime(t.start)} --> ${formatTime(t.end)}`) lines.push(`${spriteUrl}#xywh=${t.x},${t.y},${t.w},${t.h}`) lines.push("") }
return lines.join("\n")}