From e9bbf65ec295dda23636048575637be38480f242 Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Sat, 11 Apr 2026 20:59:08 -0700 Subject: [PATCH] native/tapes: render tape video frames during playback --- fedac/native/pieces/tapes.mjs | 48 +++++++- fedac/native/src/audio-decode.c | 204 ++++++++++++++++++++++++++++++++ fedac/native/src/audio-decode.h | 23 ++++ fedac/native/src/js-bindings.c | 93 +++++++++++++++ 4 files changed, 362 insertions(+), 6 deletions(-) diff --git a/fedac/native/pieces/tapes.mjs b/fedac/native/pieces/tapes.mjs index b418fb9300..c347bff2ab 100644 --- a/fedac/native/pieces/tapes.mjs +++ b/fedac/native/pieces/tapes.mjs @@ -28,6 +28,7 @@ let messageFrame = 0; let frame = 0; let cloudFetching = false; let cloudLoadedOnce = false; +let nowPlaying = ""; function fmtBytes(n) { if (!Number.isFinite(n)) return "?"; @@ -150,8 +151,10 @@ function act({ event: e, system, sound }) { // Play via deck 0 — same mechanism the dj piece uses. const ok = sound?.deck?.load?.(0, t.path); if (ok) { + const videoOk = sound?.deck?.prepareVideo?.(0, 120, 68, 12); sound.deck.play(0); - msg("playing local " + t.name); + nowPlaying = t.name; + msg((videoOk ? "playing local " : "playing audio-only local ") + t.name); sound?.speak?.("playing tape"); } else { msg("failed to play " + t.name); @@ -162,8 +165,10 @@ function act({ event: e, system, sound }) { // URL loading, otherwise instruct the user. const ok = sound?.deck?.load?.(0, t.mp4Url); if (ok) { + const videoOk = sound?.deck?.prepareVideo?.(0, 120, 68, 12); sound.deck.play(0); - msg("streaming " + t.slug); + nowPlaying = t.slug + ".mp4"; + msg((videoOk ? "streaming " : "streaming audio-only ") + t.slug); sound?.speak?.("streaming cloud tape"); } else { msg("cloud streaming not supported yet — see " + t.mp4Url); @@ -216,10 +221,14 @@ function act({ event: e, system, sound }) { } } -function paint({ wipe, ink, box, line, write, screen }) { +function paint({ wipe, ink, box, line, write, screen, sound }) { const w = screen.width; const h = screen.height; wipe(16, 14, 20); + const deck0 = sound?.deck?.decks?.[0] || null; + const showPreview = !!deck0?.loaded; + const previewW = showPreview ? Math.min(132, Math.max(92, Math.floor(w * 0.36))) : 0; + const listRight = showPreview ? (w - previewW - 12) : (w - 8); // Title ink(200, 220, 255); @@ -235,6 +244,31 @@ function paint({ wipe, ink, box, line, write, screen }) { const scrollStart = Math.max(0, selection - Math.floor(visibleRows / 2)); const scrollEnd = Math.min(tapes.length, scrollStart + visibleRows); + if (showPreview) { + const px = listRight + 4; + const py = 40; + const ph = Math.min(h - 60, Math.floor(previewW * 0.62)); + ink(36, 44, 58, 220); + box(px - 2, py - 2, previewW + 4, ph + 18, true); + if (deck0?.videoReady) { + sound?.deck?.videoBlit?.(0, px, py, previewW, ph); + } else { + ink(120, 140, 170); + write("video buffering...", { x: px + 8, y: py + Math.floor(ph / 2), size: 1, font: "font_1" }); + } + ink(220, 230, 240); + write("now playing", { x: px, y: py + ph + 4, size: 1, font: "font_1" }); + ink(150, 180, 210); + write((nowPlaying || deck0?.title || "deck 0").replace(/\.mp4$/, ""), { x: px, y: py + ph + 14, size: 1, font: "font_1" }); + if (deck0?.duration > 0) { + const prog = Math.max(0, Math.min(1, (deck0?.position || 0) / deck0.duration)); + ink(50, 60, 72); + box(px, py + ph + 22, previewW, 5, true); + ink(120, 220, 180); + box(px, py + ph + 22, Math.max(1, Math.floor(previewW * prog)), 5, true); + } + } + if (tapes.length === 0) { ink(180, 180, 200); write("No tapes recorded yet.", { x: 20, y: listY + 8, size: 1, font: "font_1" }); @@ -247,7 +281,7 @@ function paint({ wipe, ink, box, line, write, screen }) { const y = listY + (i - scrollStart) * rowH; if (i === selection) { ink(40, 60, 100, 200); - box(4, y - 2, w - 8, rowH, true); + box(4, y - 2, listRight - 4, rowH, true); } // Source badge (☁ cloud / 📁 local — shown as small ascii tag) const badge = t.source === "cloud" ? "C" : "L"; @@ -257,11 +291,13 @@ function paint({ wipe, ink, box, line, write, screen }) { // Name if (i === selection) ink(255, 255, 255); else ink(200, 220, 240); - write(t.name.replace(/\.mp4$/, ""), { x: 24, y: y + 2, size: 1, font: "font_1" }); + let label = t.name.replace(/\.mp4$/, ""); + if (showPreview && label.length > 20) label = label.slice(0, 20) + "..."; + write(label, { x: 24, y: y + 2, size: 1, font: "font_1" }); // Right meta: local shows size, cloud shows code ink(140, 160, 190); const meta = t.source === "local" ? fmtBytes(t.size) : ("!" + (t.code || "?")); - const metaX = w - meta.length * 6 - 10; + const metaX = listRight - meta.length * 6 - 6; write(meta, { x: metaX, y: y + 2, size: 1, font: "font_1" }); } } diff --git a/fedac/native/src/audio-decode.c b/fedac/native/src/audio-decode.c index d1d0193c35..ead4acafe7 100644 --- a/fedac/native/src/audio-decode.c +++ b/fedac/native/src/audio-decode.c @@ -13,7 +13,9 @@ #include #include #include +#include #include +#include #include extern void ac_log(const char *fmt, ...); @@ -36,6 +38,19 @@ static int ring_free(ACDeckDecoder *d) { return d->ring_size - ring_available(d); } +static void free_video_preview(ACDeckDecoder *d) { + if (!d) return; + if (d->video_frames) { + free(d->video_frames); + d->video_frames = NULL; + } + d->video_frame_count = 0; + d->video_width = 0; + d->video_height = 0; + d->video_fps = 0.0; + d->video_ready = 0; +} + // --- Decoder thread --- static void *decoder_thread_fn(void *arg) { @@ -217,6 +232,7 @@ int deck_decoder_load(ACDeckDecoder *d, const char *path) { d->seek_requested = 0; d->speed = 1.0; snprintf(d->path, sizeof(d->path), "%s", path); + free_video_preview(d); // Open file AVFormatContext *fmt_ctx = NULL; @@ -402,6 +418,7 @@ void deck_decoder_unload(ACDeckDecoder *d) { d->peaks = NULL; d->peak_count = 0; } + free_video_preview(d); d->loaded = 0; d->finished = 0; @@ -515,6 +532,193 @@ int deck_decoder_generate_peaks(ACDeckDecoder *d, int target_count) { return target_count; } +int deck_decoder_generate_video_preview(ACDeckDecoder *d, int width, int height, int fps) { + if (!d || !d->loaded || !d->path[0]) return -1; + if (width <= 0) width = 96; + if (height <= 0) height = 54; + if (fps <= 0) fps = 12; + if (fps > 24) fps = 24; + + // Reuse cached preview when the request matches the existing buffer. + if (d->video_ready && + d->video_frames && + d->video_width == width && + d->video_height == height && + fabs(d->video_fps - (double)fps) < 0.001) { + return d->video_frame_count; + } + + free_video_preview(d); + + AVFormatContext *fmt = NULL; + if (avformat_open_input(&fmt, d->path, NULL, NULL) < 0) return -1; + if (avformat_find_stream_info(fmt, NULL) < 0) { + avformat_close_input(&fmt); + return -1; + } + + int sidx = av_find_best_stream(fmt, AVMEDIA_TYPE_VIDEO, -1, -1, NULL, 0); + if (sidx < 0) { + avformat_close_input(&fmt); + return -1; + } + + AVStream *st = fmt->streams[sidx]; + const AVCodec *codec = avcodec_find_decoder(st->codecpar->codec_id); + if (!codec) { + avformat_close_input(&fmt); + return -1; + } + + AVCodecContext *cctx = avcodec_alloc_context3(codec); + if (!cctx) { + avformat_close_input(&fmt); + return -1; + } + avcodec_parameters_to_context(cctx, st->codecpar); + if (avcodec_open2(cctx, codec, NULL) < 0) { + avcodec_free_context(&cctx); + avformat_close_input(&fmt); + return -1; + } + + double duration = d->duration; + if (duration <= 0.0 && fmt->duration > 0) + duration = (double)fmt->duration / AV_TIME_BASE; + if (duration <= 0.0 && st->duration > 0) + duration = st->duration * av_q2d(st->time_base); + if (duration <= 0.0) + duration = 30.0; + + int target_count = (int)ceil(duration * (double)fps) + 1; + if (target_count < 1) target_count = 1; + if (target_count > 900) target_count = 900; + + size_t pixels_per_frame = (size_t)width * (size_t)height; + size_t total_pixels = pixels_per_frame * (size_t)target_count; + uint32_t *frames = (uint32_t *)calloc(total_pixels, sizeof(uint32_t)); + if (!frames) { + avcodec_free_context(&cctx); + avformat_close_input(&fmt); + return -1; + } + + struct SwsContext *sws = sws_getContext( + cctx->width, cctx->height, cctx->pix_fmt, + width, height, AV_PIX_FMT_BGRA, + SWS_BILINEAR, NULL, NULL, NULL); + if (!sws) { + free(frames); + avcodec_free_context(&cctx); + avformat_close_input(&fmt); + return -1; + } + + AVPacket *pkt = av_packet_alloc(); + AVFrame *frame = av_frame_alloc(); + if (!pkt || !frame) { + av_packet_free(&pkt); + av_frame_free(&frame); + sws_freeContext(sws); + free(frames); + avcodec_free_context(&cctx); + avformat_close_input(&fmt); + return -1; + } + + uint8_t *last_frame = NULL; + int frame_idx = 0; + double next_time = 0.0; + + while (av_read_frame(fmt, pkt) >= 0 && frame_idx < target_count) { + if (pkt->stream_index != sidx) { + av_packet_unref(pkt); + continue; + } + if (avcodec_send_packet(cctx, pkt) < 0) { + av_packet_unref(pkt); + continue; + } + av_packet_unref(pkt); + + while (avcodec_receive_frame(cctx, frame) >= 0 && frame_idx < target_count) { + double frame_time = 0.0; + if (frame->best_effort_timestamp != AV_NOPTS_VALUE) + frame_time = frame->best_effort_timestamp * av_q2d(st->time_base); + else if (frame->pts != AV_NOPTS_VALUE) + frame_time = frame->pts * av_q2d(st->time_base); + + uint8_t *dst_data[4] = {0}; + int dst_linesize[4] = {0}; + uint32_t *dst = frames + ((size_t)frame_idx * pixels_per_frame); + av_image_fill_arrays(dst_data, dst_linesize, (uint8_t *)dst, + AV_PIX_FMT_BGRA, width, height, 1); + + // Fill every preview sample point that this decoded frame covers. + while (frame_idx < target_count && + (frame_time >= next_time || frame_idx == 0)) { + dst = frames + ((size_t)frame_idx * pixels_per_frame); + av_image_fill_arrays(dst_data, dst_linesize, (uint8_t *)dst, + AV_PIX_FMT_BGRA, width, height, 1); + sws_scale(sws, (const uint8_t * const *)frame->data, frame->linesize, + 0, cctx->height, dst_data, dst_linesize); + last_frame = (uint8_t *)dst; + frame_idx++; + next_time = (double)frame_idx / (double)fps; + } + + av_frame_unref(frame); + } + } + + // Flush decoder for the tail of the file. + avcodec_send_packet(cctx, NULL); + while (avcodec_receive_frame(cctx, frame) >= 0 && frame_idx < target_count) { + uint8_t *dst_data[4] = {0}; + int dst_linesize[4] = {0}; + uint32_t *dst = frames + ((size_t)frame_idx * pixels_per_frame); + av_image_fill_arrays(dst_data, dst_linesize, (uint8_t *)dst, + AV_PIX_FMT_BGRA, width, height, 1); + sws_scale(sws, (const uint8_t * const *)frame->data, frame->linesize, + 0, cctx->height, dst_data, dst_linesize); + last_frame = (uint8_t *)dst; + frame_idx++; + av_frame_unref(frame); + } + + // Pad remaining preview slots with the final decoded frame so playback + // keeps showing a still image once audio reaches the tail. + if (last_frame) { + while (frame_idx < target_count) { + uint32_t *dst = frames + ((size_t)frame_idx * pixels_per_frame); + memcpy(dst, last_frame, pixels_per_frame * sizeof(uint32_t)); + frame_idx++; + } + } + + av_packet_free(&pkt); + av_frame_free(&frame); + sws_freeContext(sws); + avcodec_free_context(&cctx); + avformat_close_input(&fmt); + + if (frame_idx <= 0) { + free(frames); + return -1; + } + + d->video_frames = frames; + d->video_frame_count = frame_idx; + d->video_width = width; + d->video_height = height; + d->video_fps = (double)fps; + d->video_ready = 1; + + ac_log("[deck] video preview ready: %s (%d frames @ %dx%d %.1ffps)\n", + d->path, d->video_frame_count, d->video_width, d->video_height, d->video_fps); + return d->video_frame_count; +} + void deck_decoder_destroy(ACDeckDecoder *d) { if (!d) return; deck_decoder_unload(d); diff --git a/fedac/native/src/audio-decode.h b/fedac/native/src/audio-decode.h index c7240dad3e..66afbeeec8 100644 --- a/fedac/native/src/audio-decode.h +++ b/fedac/native/src/audio-decode.h @@ -61,6 +61,16 @@ typedef struct ACDeckDecoder { // Generated once on load by scanning the entire file via separate pass. float *peaks; // [0..1] amplitude peaks int peak_count; // number of peaks (typically 1024) + + // Optional downscaled video preview frames for pieces like tapes.mjs. + // Frames are stored as opaque ARGB32 pixels (0xAARRGGBB) in a flat array: + // [frame0 pixels][frame1 pixels]... + uint32_t *video_frames; + int video_frame_count; + int video_width; + int video_height; + double video_fps; + volatile int video_ready; } ACDeckDecoder; // Create a decoder instance for the given output sample rate @@ -83,6 +93,10 @@ void deck_decoder_unload(ACDeckDecoder *d); // Safe to call from main thread; takes a few hundred ms for typical tracks. int deck_decoder_generate_peaks(ACDeckDecoder *d, int target_count); +// Generate a lightweight downscaled video preview strip for the loaded file. +// Safe to call from main thread; intended for UI playback rather than full-res video. +int deck_decoder_generate_video_preview(ACDeckDecoder *d, int width, int height, int fps); + // Destroy decoder and free all resources void deck_decoder_destroy(ACDeckDecoder *d); @@ -105,6 +119,12 @@ typedef struct ACDeckDecoder { int ring_size; volatile int ring_write; volatile int ring_read; + uint32_t *video_frames; + int video_frame_count; + int video_width; + int video_height; + double video_fps; + volatile int video_ready; } ACDeckDecoder; static inline ACDeckDecoder *deck_decoder_create(unsigned int output_rate) { @@ -119,6 +139,9 @@ static inline void deck_decoder_seek(ACDeckDecoder *d, double s) { (void)d; (voi static inline void deck_decoder_set_speed(ACDeckDecoder *d, double s) { (void)d; (void)s; } static inline void deck_decoder_unload(ACDeckDecoder *d) { (void)d; } static inline void deck_decoder_destroy(ACDeckDecoder *d) { (void)d; } +static inline int deck_decoder_generate_video_preview(ACDeckDecoder *d, int width, int height, int fps) { + (void)d; (void)width; (void)height; (void)fps; return -1; +} #endif // HAVE_AVCODEC #endif // AC_AUDIO_DECODE_H diff --git a/fedac/native/src/js-bindings.c b/fedac/native/src/js-bindings.c index 55434d15bf..1e2d0990d4 100644 --- a/fedac/native/src/js-bindings.c +++ b/fedac/native/src/js-bindings.c @@ -2299,6 +2299,35 @@ static JSValue js_sound_bpm(JSContext *ctx, JSValueConst this_val, int argc, JSV // --- DJ deck bindings --- +static void blit_argb_nearest(ACFramebuffer *fb, const uint32_t *src, + int sw, int sh, int dx, int dy, int dw, int dh) { + if (!fb || !src || sw <= 0 || sh <= 0 || dw <= 0 || dh <= 0) return; + + int x0 = dx < 0 ? 0 : dx; + int y0 = dy < 0 ? 0 : dy; + int x1 = dx + dw; + int y1 = dy + dh; + if (x1 > fb->width) x1 = fb->width; + if (y1 > fb->height) y1 = fb->height; + if (x0 >= x1 || y0 >= y1) return; + + for (int y = y0; y < y1; y++) { + int sy = (int)(((int64_t)(y - dy) * sh) / dh); + if (sy < 0) sy = 0; + if (sy >= sh) sy = sh - 1; + const uint32_t *src_row = src + sy * sw; + uint32_t *dst_row = fb->pixels + y * fb->stride; + for (int x = x0; x < x1; x++) { + int sx = (int)(((int64_t)(x - dx) * sw) / dw); + if (sx < 0) sx = 0; + if (sx >= sw) sx = sw - 1; + uint32_t px = src_row[sx]; + if ((px >> 24) == 0) continue; + dst_row[x] = px; + } + } +} + // sound.deck.load(deck, path) -> true/false static JSValue js_deck_load(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { (void)this_val; @@ -2367,6 +2396,58 @@ static JSValue js_deck_get_peaks(JSContext *ctx, JSValueConst this_val, int argc return arr; } +// sound.deck.prepareVideo(deck[, width, height, fps]) — decode a low-res preview strip +static JSValue js_deck_prepare_video(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { + (void)this_val; + if (argc < 1 || !current_rt || !current_rt->audio) return JS_FALSE; + int deck = 0, width = 96, height = 54, fps = 12; + JS_ToInt32(ctx, &deck, argv[0]); + if (argc > 1) JS_ToInt32(ctx, &width, argv[1]); + if (argc > 2) JS_ToInt32(ctx, &height, argv[2]); + if (argc > 3) JS_ToInt32(ctx, &fps, argv[3]); + if (deck < 0 || deck >= AUDIO_MAX_DECKS) return JS_FALSE; + ACDeck *dk = ¤t_rt->audio->decks[deck]; + if (!dk->decoder) return JS_FALSE; + int ret = deck_decoder_generate_video_preview(dk->decoder, width, height, fps); + return JS_NewBool(ctx, ret > 0); +} + +// sound.deck.videoBlit(deck, x, y, w, h) — draw current preview frame into the framebuffer +static JSValue js_deck_video_blit(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { + (void)this_val; + if (argc < 5 || !current_rt || !current_rt->audio || !current_rt->graph || !current_rt->graph->fb) + return JS_FALSE; + + int deck = 0, x = 0, y = 0, w = 0, h = 0; + JS_ToInt32(ctx, &deck, argv[0]); + JS_ToInt32(ctx, &x, argv[1]); + JS_ToInt32(ctx, &y, argv[2]); + JS_ToInt32(ctx, &w, argv[3]); + JS_ToInt32(ctx, &h, argv[4]); + if (deck < 0 || deck >= AUDIO_MAX_DECKS) return JS_FALSE; + + ACDeck *dk = ¤t_rt->audio->decks[deck]; + if (!dk->decoder || !dk->decoder->video_ready || !dk->decoder->video_frames || + dk->decoder->video_frame_count <= 0 || dk->decoder->video_width <= 0 || + dk->decoder->video_height <= 0 || dk->decoder->video_fps <= 0.0) { + return JS_FALSE; + } + + if (w <= 0) w = dk->decoder->video_width; + if (h <= 0) h = dk->decoder->video_height; + + int idx = (int)floor(dk->decoder->position * dk->decoder->video_fps + 0.0001); + if (idx < 0) idx = 0; + if (idx >= dk->decoder->video_frame_count) idx = dk->decoder->video_frame_count - 1; + + size_t pixels_per_frame = (size_t)dk->decoder->video_width * (size_t)dk->decoder->video_height; + const uint32_t *src = dk->decoder->video_frames + ((size_t)idx * pixels_per_frame); + blit_argb_nearest(current_rt->graph->fb, src, + dk->decoder->video_width, dk->decoder->video_height, + x, y, w, h); + return JS_TRUE; +} + // sound.deck.setVolume(deck, vol) static JSValue js_deck_set_volume(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { (void)this_val; @@ -2541,6 +2622,8 @@ static JSValue build_sound_obj(JSContext *ctx, ACRuntime *rt) { JS_SetPropertyStr(ctx, deck_obj, "setCrossfader", JS_NewCFunction(ctx, js_deck_set_crossfader, "setCrossfader", 1)); JS_SetPropertyStr(ctx, deck_obj, "setMasterVolume", JS_NewCFunction(ctx, js_deck_set_master_vol, "setMasterVolume", 1)); JS_SetPropertyStr(ctx, deck_obj, "getPeaks", JS_NewCFunction(ctx, js_deck_get_peaks, "getPeaks", 1)); + JS_SetPropertyStr(ctx, deck_obj, "prepareVideo", JS_NewCFunction(ctx, js_deck_prepare_video, "prepareVideo", 4)); + JS_SetPropertyStr(ctx, deck_obj, "videoBlit", JS_NewCFunction(ctx, js_deck_video_blit, "videoBlit", 5)); // Deck state (read-only, rebuilt each frame) JSValue decks_arr = JS_NewArray(ctx); @@ -2560,6 +2643,11 @@ static JSValue build_sound_obj(JSContext *ctx, ACRuntime *rt) { JS_SetPropertyStr(ctx, di, "artist", JS_NewString(ctx, dk->decoder->artist)); JS_SetPropertyStr(ctx, di, "finished", JS_NewBool(ctx, dk->decoder->finished)); JS_SetPropertyStr(ctx, di, "error", JS_NewString(ctx, dk->decoder->error ? dk->decoder->error_msg : "")); + JS_SetPropertyStr(ctx, di, "videoReady", JS_NewBool(ctx, dk->decoder->video_ready)); + JS_SetPropertyStr(ctx, di, "videoWidth", JS_NewInt32(ctx, dk->decoder->video_width)); + JS_SetPropertyStr(ctx, di, "videoHeight", JS_NewInt32(ctx, dk->decoder->video_height)); + JS_SetPropertyStr(ctx, di, "videoFps", JS_NewFloat64(ctx, dk->decoder->video_fps)); + JS_SetPropertyStr(ctx, di, "videoFrames", JS_NewInt32(ctx, dk->decoder->video_frame_count)); } else { JS_SetPropertyStr(ctx, di, "position", JS_NewFloat64(ctx, 0)); JS_SetPropertyStr(ctx, di, "duration", JS_NewFloat64(ctx, 0)); @@ -2568,6 +2656,11 @@ static JSValue build_sound_obj(JSContext *ctx, ACRuntime *rt) { JS_SetPropertyStr(ctx, di, "artist", JS_NewString(ctx, "")); JS_SetPropertyStr(ctx, di, "finished", JS_FALSE); JS_SetPropertyStr(ctx, di, "error", JS_NewString(ctx, "")); + JS_SetPropertyStr(ctx, di, "videoReady", JS_FALSE); + JS_SetPropertyStr(ctx, di, "videoWidth", JS_NewInt32(ctx, 0)); + JS_SetPropertyStr(ctx, di, "videoHeight", JS_NewInt32(ctx, 0)); + JS_SetPropertyStr(ctx, di, "videoFps", JS_NewFloat64(ctx, 0)); + JS_SetPropertyStr(ctx, di, "videoFrames", JS_NewInt32(ctx, 0)); } } JS_SetPropertyUint32(ctx, decks_arr, d, di); -- 2.51.2