From 53b5ce98bf1e7eb23842a5f7aca537971f9ff722 Mon Sep 17 00:00:00 2001 From: Jeffrey Alan Scudder Date: Mon, 30 Mar 2026 22:13:05 -0700 Subject: [PATCH] =?UTF-8?q?kidlisp-wasm:=20add=20audio=20synthesis=20?= =?UTF-8?q?=E2=80=94=20pixel=20buffer=20becomes=20sound?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WASM module now exports a `sound(sampleRate, fps, frame)` function that reads the middle row of the pixel buffer, interleaves R/G/B channels as sub-samples, and writes f32 audio to memory. Visual state directly drives the audio — vortex patterns create evolving timbral changes. New sonify.mjs runner renders video + WAV + muxed MP4 (h264 + MP3 for VS Code compatibility on Fedora). Co-Authored-By: Claude Opus 4.6 (1M context) --- kidlisp-wasm/compiler.mjs | 62 +++++++++++++- kidlisp-wasm/sonify.mjs | 164 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 kidlisp-wasm/sonify.mjs diff --git a/kidlisp-wasm/compiler.mjs b/kidlisp-wasm/compiler.mjs index 7c8e94fe5..cd6175d72 100644 --- a/kidlisp-wasm/compiler.mjs +++ b/kidlisp-wasm/compiler.mjs @@ -115,6 +115,8 @@ class E { // Memory st8(off = 0) { this.b.push(0x3a, 0x00, ...uleb128(off)); return this; } ld8u(off = 0){ this.b.push(0x2d, 0x00, ...uleb128(off)); return this; } + stf32(off = 0){ this.b.push(0x38, 0x02, ...uleb128(off)); return this; } // f32.store + ldf32(off = 0){ this.b.push(0x2a, 0x02, ...uleb128(off)); return this; } // f32.load // Control if_() { this.b.push(0x04, 0x40); return this; } else_(){ this.b.push(0x05); return this; } @@ -244,7 +246,8 @@ const F_SCROLL = 11; // (f32, f32) → () const F_SPIN = 12; // (f32) → () const F_ZOOM = 13; // (f32) → () const F_CONTRAST = 14; // (f32) → () -const F_PAINT = 15; // (f32, f32, f32) → () +const F_SOUND = 15; // (f32, f32, f32) → () [sampleRate, fps, frame] +const F_PAINT = 16; // (f32, f32, f32) → () // ─── Runtime Function Emitters ────────────────────────────────────── @@ -853,6 +856,60 @@ function emitContrast() { return { locals: [[4, I32], [1, F32]], code: e.bytes() }; } +// $sound(sampleRate: f32, fps: f32, frame: f32) — generate audio from pixel buffer +// Scans the middle row horizontally as a waveform: pixel RGB → sample amplitude. +// The visual state directly becomes the sound — vortex patterns create timbral evolution. +function emitSound() { + const e = new E(); + // params: 0=sampleRate, 1=fps, 2=frame + // f32 locals: 3=sample + // i32 locals: 4=spf, 5=i, 6=row, 7=col, 8=pixOff, 9=audioOff, 10=globalSample, 11=chanSum + + // spf = floor(sampleRate / fps) + e.lg(0).lg(1).fdiv().ffloor().f2i().ls(4); + // row = h / 2 (middle row) + e.gg(G_H).i32c(2).idiv().ls(6); + // audioOff = w * h * 8 (after pixel + temp buffers) + e.gg(G_W).gg(G_H).imul().i32c(8).imul().ls(9); + + // for i = 0; i < spf; i++ + e.i32c(0).ls(5); + e.block().loop(); + e.lg(5).lg(4).ige().brif(1); + + // globalSample = frame * spf + i + e.lg(2).f2i().lg(4).imul().lg(5).iadd().ls(10); + + // col = globalSample % w (scan horizontally across the row) + e.lg(10).gg(G_W).irem().ls(7); + e.lg(7).i32c(0).ilt().if_(); + e.lg(7).gg(G_W).iadd().ls(7); + e.end(); + + // Interleave R, G, B: each channel is a separate sub-sample + // pixelIdx = (globalSample / 3) % w, channel = globalSample % 3 + e.lg(10).i32c(3).idiv().gg(G_W).irem().ls(7); // col = pixelIdx + e.lg(7).i32c(0).ilt().if_(); e.lg(7).gg(G_W).iadd().ls(7); e.end(); + e.lg(10).i32c(3).irem().ls(11); // channel index (0=R, 1=G, 2=B) + + // pixOff = (row * w + col) * 4 + channel + e.lg(6).gg(G_W).imul().lg(7).iadd().i32c(4).imul().lg(11).iadd().ls(8); + + // sample = (pixelValue - 128) / 128.0 * 0.8 + e.lg(8).ld8u().i2f().f32c(128).fsub().f32c(128).fdiv().f32c(0.8).fmul().ls(3); + + // f32.store at audioOff + i * 4 + e.lg(9).lg(5).i32c(4).imul().iadd(); + e.lg(3); + e.stf32(); + + e.lg(5).i32c(1).iadd().ls(5); + e.br(0); + e.end().end(); + e.end(); + return { locals: [[1, F32], [8, I32]], code: e.bytes() }; +} + // ─── Compiler ─────────────────────────────────────────────────────── export class Compiler { @@ -1296,6 +1353,7 @@ export class Compiler { T_F32_VOID, // spin T_F32_VOID, // zoom T_F32_VOID, // contrast + T_F32_F32_F32, // sound T_F32_F32_F32, // paint ]; out.push(...section(3, vecOf(funcTypes.map(t => [...uleb128(t)])))); @@ -1319,6 +1377,7 @@ export class Compiler { // ── Exports ── const exports = [ [...encodeString("paint"), 0x00, ...uleb128(F_PAINT)], + [...encodeString("sound"), 0x00, ...uleb128(F_SOUND)], [...encodeString("memory"), 0x02, ...uleb128(0)], ]; out.push(...section(7, vecOf(exports))); @@ -1337,6 +1396,7 @@ export class Compiler { emitSpin(), emitZoom(), emitContrast(), + emitSound(), ]; const paintBody = { locals: this.paintLocals, code: [...this.code.bytes(), 0x0b] }; diff --git a/kidlisp-wasm/sonify.mjs b/kidlisp-wasm/sonify.mjs new file mode 100644 index 000000000..af2f0dcd2 --- /dev/null +++ b/kidlisp-wasm/sonify.mjs @@ -0,0 +1,164 @@ +#!/usr/bin/env node +// Render KidLisp piece to animated WebP + WAV audio via self-contained WASM. +// Usage: node sonify.mjs [frames] [fps] [size] [sampleRate] + +import { readFileSync, writeFileSync, mkdirSync } from "fs"; +import { basename } from "path"; +import { execSync } from "child_process"; +import sharp from "sharp"; +import { Compiler } from "./compiler.mjs"; + +const OUT_DIR = new URL("./output/", import.meta.url).pathname; +mkdirSync(OUT_DIR, { recursive: true }); + +const input = process.argv[2] || "roz.lisp"; +const FRAMES = parseInt(process.argv[3]) || 300; +const FPS = parseInt(process.argv[4]) || 30; +const SIZE = parseInt(process.argv[5]) || 256; +const SAMPLE_RATE = parseInt(process.argv[6]) || 44100; +const SAMPLES_PER_FRAME = Math.floor(SAMPLE_RATE / FPS); + +const path = new URL(input, import.meta.url).pathname; +const source = readFileSync(path, "utf-8"); +const name = basename(input, ".lisp"); + +console.log(`Compiling ${input}...`); +const compiler = new Compiler(); +const wasmBytes = compiler.compile(source); +const mathImports = { + math: { + sin: (x) => Math.fround(Math.sin(x)), + cos: (x) => Math.fround(Math.cos(x)), + random: () => Math.fround(Math.random()), + }, +}; +const { instance } = await WebAssembly.instantiate(wasmBytes, mathImports); +console.log( + `WASM: ${wasmBytes.length} bytes | ${FRAMES} frames @ ${FPS}fps | ${SIZE}x${SIZE} | ${SAMPLE_RATE}Hz`, +); + +// Audio buffer offset: w * h * 8 (after pixel + temp buffers) +const audioOffset = SIZE * SIZE * 8; +const totalSamples = FRAMES * SAMPLES_PER_FRAME; +const allAudio = new Float32Array(totalSamples); + +// Render frames +const framePngs = []; +for (let f = 0; f < FRAMES; f++) { + instance.exports.paint(SIZE, SIZE, f); + instance.exports.sound(SAMPLE_RATE, FPS, f); + + // Read pixels + const mem = new Uint8Array(instance.exports.memory.buffer); + const pixels = Buffer.from(mem.slice(0, SIZE * SIZE * 4)); + const png = await sharp(pixels, { + raw: { width: SIZE, height: SIZE, channels: 4 }, + }) + .png() + .toBuffer(); + framePngs.push(png); + + // Read audio samples (f32 at audioOffset) + const audioView = new Float32Array( + instance.exports.memory.buffer, + audioOffset, + SAMPLES_PER_FRAME, + ); + allAudio.set(audioView, f * SAMPLES_PER_FRAME); + + if ((f + 1) % 30 === 0 || f === FRAMES - 1) { + process.stdout.write(`\r Rendered ${f + 1}/${FRAMES} frames`); + } +} +console.log(); + +// ─── Write WAV ─────────────────────────────────────────────────────── +function writeWav(filename, samples, sampleRate) { + const numSamples = samples.length; + const bytesPerSample = 2; // 16-bit PCM + const dataSize = numSamples * bytesPerSample; + const buffer = Buffer.alloc(44 + dataSize); + + // RIFF header + buffer.write("RIFF", 0); + buffer.writeUInt32LE(36 + dataSize, 4); + buffer.write("WAVE", 8); + + // fmt chunk + buffer.write("fmt ", 12); + buffer.writeUInt32LE(16, 16); // chunk size + buffer.writeUInt16LE(1, 20); // PCM + buffer.writeUInt16LE(1, 22); // mono + buffer.writeUInt32LE(sampleRate, 24); + buffer.writeUInt32LE(sampleRate * bytesPerSample, 28); // byte rate + buffer.writeUInt16LE(bytesPerSample, 32); // block align + buffer.writeUInt16LE(16, 34); // bits per sample + + // data chunk + buffer.write("data", 36); + buffer.writeUInt32LE(dataSize, 40); + + // Convert f32 samples to 16-bit PCM + for (let i = 0; i < numSamples; i++) { + const s = Math.max(-1, Math.min(1, samples[i])); + const pcm = Math.round(s * 32767); + buffer.writeInt16LE(pcm, 44 + i * 2); + } + + writeFileSync(filename, buffer); +} + +const wavPath = `${OUT_DIR}${name}.wav`; +writeWav(wavPath, allAudio, SAMPLE_RATE); +const wavSize = (allAudio.length * 2) / 1024; +console.log(`${name}.wav (${SAMPLE_RATE}Hz mono, ${(totalSamples / SAMPLE_RATE).toFixed(1)}s, ${wavSize.toFixed(0)}KB)`); + +// ─── Encode WebP ───────────────────────────────────────────────────── +console.log("Encoding animated WebP..."); +const tmpDir = `${OUT_DIR}.frames-${name}`; +mkdirSync(tmpDir, { recursive: true }); + +for (let f = 0; f < framePngs.length; f++) { + const framePath = `${tmpDir}/frame-${String(f).padStart(5, "0")}.png`; + await sharp(framePngs[f]).toFile(framePath); +} + +const webpPath = `${OUT_DIR}${name}.webp`; +execSync( + `ffmpeg -y -framerate ${FPS} -i "${tmpDir}/frame-%05d.png" -loop 0 -lossless 1 "${webpPath}" 2>/dev/null`, +); +execSync(`rm -rf "${tmpDir}"`); + +const { statSync } = await import("fs"); +const webpSize = statSync(webpPath).size; +console.log(`${name}.webp (${SIZE}x${SIZE}, ${FRAMES} frames, ${(webpSize / 1024).toFixed(1)}KB)`); + +// ─── Mux video + audio to mp4 ──────────────────────────────────────── +try { + const mp4Path = `${OUT_DIR}${name}.mp4`; + const mp4Video = `${OUT_DIR}${name}_video.mp4`; + + // Step 1: encode video-only mp4 from frames + const frameDir2 = `${OUT_DIR}.frames2-${name}`; + mkdirSync(frameDir2, { recursive: true }); + for (let f = 0; f < framePngs.length; f++) { + await sharp(framePngs[f]).toFile( + `${frameDir2}/frame-${String(f).padStart(5, "0")}.png`, + ); + } + execSync( + `ffmpeg -y -framerate ${FPS} -i "${frameDir2}/frame-%05d.png" -c:v libx264 -pix_fmt yuv420p "${mp4Video}" 2>/dev/null`, + ); + execSync(`rm -rf "${frameDir2}"`); + + // Step 2: mux video + audio + execSync( + `ffmpeg -y -i "${mp4Video}" -i "${wavPath}" -map 0:v -map 1:a -c:v copy -c:a libmp3lame -ar 24000 -ac 2 -b:a 160k -movflags +faststart -shortest "${mp4Path}" 2>/dev/null`, + ); + execSync(`rm -f "${mp4Video}"`); + + const mp4Size = statSync(mp4Path).size; + console.log(`${name}.mp4 (video+audio, ${(mp4Size / 1024).toFixed(1)}KB)`); +} catch { + console.log("(ffmpeg mp4 mux not available, WAV + WebP saved separately)"); +} -- 2.51.2