From 6417d65aaadd011080695eaf72abe6315248427c Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Tue, 5 May 2026 01:56:00 -0400 Subject: [PATCH] notepat/gm: wire General MIDI as default sound + digit-buffered patch picker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GM bank served from assets.aesthetic.computer/gm//.mp3, baked on lith from a FOSS SoundFont (GeneralUser GS). The disk-side wiring falls back to a sine oscillator while GM is loading or unavailable, so nothing breaks before the packs land on Spaces. - lith/scripts/gm-bake.mjs: SoundFont→per-instrument MP3 bake (128 melodic patches + GM standard drum kit, manifest.json + LICENSE.txt) - system/public/aesthetic.computer/lib/gm.mjs: lazy AudioBuffer player — loadManifest / loadPatch / loadDrumKit / prefetchPatch - system/public/aesthetic.computer/lib/midi.mjs: forward 0xC0 program-change through acSEND so notepat can react to it - notepat.mjs: "gm" prepended to wavetypes (default); top-row 0-9 picks GM programs live like menuband, with 700ms auto-clear; wave button shows GM:078 while typing; legacy oscillators stay reachable via Tab; deferred GM init waits for window.audioContext after first user gesture - plans/notepat-gm-integration.md: full integration plan + decision log Co-Authored-By: Claude Opus 4.7 (1M context) --- lith/README.md | 17 + lith/scripts/gm-bake.mjs | 528 ++++++++++++++++++ plans/notepat-gm-integration.md | 225 ++++++++ .../aesthetic.computer/disks/notepat.mjs | 189 ++++++- system/public/aesthetic.computer/lib/gm.mjs | 490 ++++++++++++++++ system/public/aesthetic.computer/lib/midi.mjs | 8 +- 6 files changed, 1445 insertions(+), 12 deletions(-) create mode 100755 lith/scripts/gm-bake.mjs create mode 100644 plans/notepat-gm-integration.md create mode 100644 system/public/aesthetic.computer/lib/gm.mjs diff --git a/lith/README.md b/lith/README.md index edc033768e..bfd120e613 100644 --- a/lith/README.md +++ b/lith/README.md @@ -22,3 +22,20 @@ Recommended workflow: 2. Fill in the real production values 3. Re-run `fish vault-tool.fish status` to confirm `lith/.env` is tracked 4. Deploy with `fish /workspaces/aesthetic-computer/lith/deploy.fish` + +## GM SoundFont bake + +`lith/scripts/gm-bake.mjs` is a one-shot baker that renders a FOSS SoundFont +(default: [GeneralUser GS v1.471](https://schristiancollins.com/generaluser.php)) +into per-instrument MP3 packs covering the full GM Level 1 bank — analogous to +how BDF fonts get hosted on `assets.aesthetic.computer`. Requires `fluidsynth` +and `ffmpeg` on PATH (`brew install fluid-synth ffmpeg` on macOS, +`apt install fluidsynth ffmpeg` on Linux). Run it with +`node lith/scripts/gm-bake.mjs` — the SF2 is downloaded into `lith/cache/` and +output lands at `lith/scripts/out/gm//.mp3` plus +`drum-000/.mp3` for the Standard Kit, with a `manifest.json` and +`LICENSE.txt` alongside. The script is re-runnable (skips existing MP3s), +defaults to every-third-semitone over A0..C8 (override with `--note-step=N`, +`--only=0,1,24`, `--skip-drums`, `--out=...`, or `--dry-run`). When the bake +finishes, publish with `npm run assets:sync:up` to push the tree to +`assets.aesthetic.computer/gm/`. diff --git a/lith/scripts/gm-bake.mjs b/lith/scripts/gm-bake.mjs new file mode 100755 index 0000000000..533ecf160b --- /dev/null +++ b/lith/scripts/gm-bake.mjs @@ -0,0 +1,528 @@ +#!/usr/bin/env node +// gm-bake.mjs — One-shot bake of a FOSS SoundFont into per-instrument MP3 packs +// for the General MIDI bank. Output is structured for upload to +// assets.aesthetic.computer/gm/* (run `npm run assets:sync:up` after). +// +// Default SoundFont: GeneralUser GS v1.471 by S. Christian Collins +// https://schristiancollins.com/generaluser.php +// Override with $GM_SF2=/path/to/file.sf2 if the upstream URL drifts. +// +// Required tools (both expected on lith; check + install hints printed below): +// fluidsynth brew install fluid-synth | apt install fluidsynth +// ffmpeg brew install ffmpeg | apt install ffmpeg +// +// Usage: +// node lith/scripts/gm-bake.mjs # melodic + drums, step=3 +// node lith/scripts/gm-bake.mjs --note-step=1 # render every semitone +// node lith/scripts/gm-bake.mjs --note-step=6 # coarse, smaller bundle +// node lith/scripts/gm-bake.mjs --only=0,1,24 # subset of program IDs +// node lith/scripts/gm-bake.mjs --skip-drums +// node lith/scripts/gm-bake.mjs --out=/tmp/gm # alt output dir +// node lith/scripts/gm-bake.mjs --dry-run # plan only, no render +// +// Re-runnable: skips MP3s that already exist on disk. +// Output tree (default): +// lith/cache/ (downloaded SF2) +// lith/scripts/out/gm/manifest.json +// lith/scripts/out/gm/LICENSE.txt +// lith/scripts/out/gm/000/A4.mp3 (program 0 = Acoustic Grand, A4) +// lith/scripts/out/gm/000/Cs4.mp3 (C#4 — sharps spelled with 's') +// ... +// lith/scripts/out/gm/drum-000/35.mp3 (Standard Kit, MIDI note 35) +// ... + +import { spawnSync, execFileSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + writeFileSync, + rmSync, + statSync, +} from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { tmpdir } from "node:os"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const LITH_DIR = resolve(__dirname, ".."); +const CACHE_DIR = join(LITH_DIR, "cache"); + +// ─── ANSI ─────────────────────────────────────────────────────────────────── +const C = { + red: "\x1b[0;31m", + green: "\x1b[0;32m", + yellow: "\x1b[1;33m", + dim: "\x1b[2m", + reset: "\x1b[0m", +}; +const log = (msg) => console.log(`${C.green}->${C.reset} ${msg}`); +const warn = (msg) => console.warn(`${C.yellow}!${C.reset} ${msg}`); +const die = (msg) => { + console.error(`${C.red}x${C.reset} ${msg}`); + process.exit(1); +}; + +// ─── CLI args ─────────────────────────────────────────────────────────────── +const argv = process.argv.slice(2); +const flag = (name, fallback = undefined) => { + for (const a of argv) { + if (a === `--${name}`) return true; + if (a.startsWith(`--${name}=`)) return a.slice(name.length + 3); + } + return fallback; +}; +const NOTE_STEP = parseInt(flag("note-step", "3"), 10); +const ONLY = flag("only"); +const SKIP_DRUMS = flag("skip-drums", false) === true; +const SKIP_MELODIC = flag("skip-melodic", false) === true; +const DRY_RUN = flag("dry-run", false) === true; +const OUT_DIR = resolve(flag("out", join(__dirname, "out", "gm"))); + +if (!Number.isInteger(NOTE_STEP) || NOTE_STEP < 1 || NOTE_STEP > 12) { + die(`--note-step must be an integer 1..12 (got ${NOTE_STEP})`); +} + +// ─── GM patch names (program 0..127) ──────────────────────────────────────── +// Standard General MIDI Level 1 melodic bank. +const GM_PATCHES = [ + "Acoustic Grand Piano", "Bright Acoustic Piano", "Electric Grand Piano", + "Honky-tonk Piano", "Electric Piano 1", "Electric Piano 2", "Harpsichord", + "Clavi", + "Celesta", "Glockenspiel", "Music Box", "Vibraphone", "Marimba", "Xylophone", + "Tubular Bells", "Dulcimer", + "Drawbar Organ", "Percussive Organ", "Rock Organ", "Church Organ", + "Reed Organ", "Accordion", "Harmonica", "Tango Accordion", + "Acoustic Guitar (nylon)", "Acoustic Guitar (steel)", + "Electric Guitar (jazz)", "Electric Guitar (clean)", + "Electric Guitar (muted)", "Overdriven Guitar", "Distortion Guitar", + "Guitar harmonics", + "Acoustic Bass", "Electric Bass (finger)", "Electric Bass (pick)", + "Fretless Bass", "Slap Bass 1", "Slap Bass 2", "Synth Bass 1", "Synth Bass 2", + "Violin", "Viola", "Cello", "Contrabass", "Tremolo Strings", + "Pizzicato Strings", "Orchestral Harp", "Timpani", + "String Ensemble 1", "String Ensemble 2", "SynthStrings 1", "SynthStrings 2", + "Choir Aahs", "Voice Oohs", "Synth Voice", "Orchestra Hit", + "Trumpet", "Trombone", "Tuba", "Muted Trumpet", "French Horn", + "Brass Section", "SynthBrass 1", "SynthBrass 2", + "Soprano Sax", "Alto Sax", "Tenor Sax", "Baritone Sax", + "Oboe", "English Horn", "Bassoon", "Clarinet", + "Piccolo", "Flute", "Recorder", "Pan Flute", + "Blown Bottle", "Shakuhachi", "Whistle", "Ocarina", + "Lead 1 (square)", "Lead 2 (sawtooth)", "Lead 3 (calliope)", + "Lead 4 (chiff)", "Lead 5 (charang)", "Lead 6 (voice)", + "Lead 7 (fifths)", "Lead 8 (bass + lead)", + "Pad 1 (new age)", "Pad 2 (warm)", "Pad 3 (polysynth)", "Pad 4 (choir)", + "Pad 5 (bowed)", "Pad 6 (metallic)", "Pad 7 (halo)", "Pad 8 (sweep)", + "FX 1 (rain)", "FX 2 (soundtrack)", "FX 3 (crystal)", "FX 4 (atmosphere)", + "FX 5 (brightness)", "FX 6 (goblins)", "FX 7 (echoes)", "FX 8 (sci-fi)", + "Sitar", "Banjo", "Shamisen", "Koto", "Kalimba", "Bag pipe", "Fiddle", + "Shanai", + "Tinkle Bell", "Agogo", "Steel Drums", "Woodblock", "Taiko Drum", + "Melodic Tom", "Synth Drum", "Reverse Cymbal", + "Guitar Fret Noise", "Breath Noise", "Seashore", "Bird Tweet", + "Telephone Ring", "Helicopter", "Applause", "Gunshot", +]; +if (GM_PATCHES.length !== 128) { + die(`internal error: GM_PATCHES length = ${GM_PATCHES.length}, expected 128`); +} + +// GM Level 1 standard kit note names (MIDI notes 35..81). +const GM_DRUM_KIT_NOTE_NAMES = { + 35: "Acoustic Bass Drum", 36: "Bass Drum 1", 37: "Side Stick", + 38: "Acoustic Snare", 39: "Hand Clap", 40: "Electric Snare", + 41: "Low Floor Tom", 42: "Closed Hi Hat", 43: "High Floor Tom", + 44: "Pedal Hi-Hat", 45: "Low Tom", 46: "Open Hi-Hat", 47: "Low-Mid Tom", + 48: "Hi-Mid Tom", 49: "Crash Cymbal 1", 50: "High Tom", 51: "Ride Cymbal 1", + 52: "Chinese Cymbal", 53: "Ride Bell", 54: "Tambourine", 55: "Splash Cymbal", + 56: "Cowbell", 57: "Crash Cymbal 2", 58: "Vibraslap", 59: "Ride Cymbal 2", + 60: "Hi Bongo", 61: "Low Bongo", 62: "Mute Hi Conga", 63: "Open Hi Conga", + 64: "Low Conga", 65: "High Timbale", 66: "Low Timbale", 67: "High Agogo", + 68: "Low Agogo", 69: "Cabasa", 70: "Maracas", 71: "Short Whistle", + 72: "Long Whistle", 73: "Short Guiro", 74: "Long Guiro", 75: "Claves", + 76: "Hi Wood Block", 77: "Low Wood Block", 78: "Mute Cuica", 79: "Open Cuica", + 80: "Mute Triangle", 81: "Open Triangle", +}; + +// ─── note helpers ─────────────────────────────────────────────────────────── +const NOTE_NAMES = [ + "C", "Cs", "D", "Ds", "E", "F", "Fs", "G", "Gs", "A", "As", "B", +]; +function midiToName(m) { + const pc = NOTE_NAMES[m % 12]; + const oct = Math.floor(m / 12) - 1; // MIDI 0 = C-1, 60 = C4 + return `${pc}${oct}`; +} + +// ─── tool checks ──────────────────────────────────────────────────────────── +function which(bin) { + const r = spawnSync("command", ["-v", bin], { shell: true }); + return r.status === 0; +} +function requireTool(bin, hint) { + if (!which(bin)) { + console.error(`${C.red}x${C.reset} missing required tool: ${bin}`); + console.error(` install: ${hint}`); + process.exit(1); + } +} +if (!DRY_RUN) { + requireTool( + "fluidsynth", + "macOS: brew install fluid-synth\n Linux: apt install fluidsynth", + ); + requireTool( + "ffmpeg", + "macOS: brew install ffmpeg\n Linux: apt install ffmpeg", + ); +} + +// ─── SoundFont resolution ─────────────────────────────────────────────────── +const GENERALUSER_URL = + "https://schristiancollins.com/soundfonts/GeneralUser_GS_1.471.zip"; +const GENERALUSER_NAME = "GeneralUser GS v1.471"; +const GENERALUSER_LICENSE = `${GENERALUSER_NAME} +by S. Christian Collins (https://schristiancollins.com/generaluser.php) + +GeneralUser GS is freely usable for any purpose, commercial or otherwise, +and may be redistributed under the same conditions, with the following +notes from the author: + + "GeneralUser GS is a GM and GS compatible SoundFont bank for composing, + playing MIDI files, and use in any sound module that supports the + SoundFont 2.01 standard. It is free to use and freely distributable, as + long as you do not charge specifically for it or modify the included + documentation. You are free to use samples from this SoundFont in your + own work, including commercial work, without any further permission." + +See the upstream page for the full README and the most current license +text: + https://schristiancollins.com/generaluser.php +`; + +function resolveSoundFont() { + const env = process.env.GM_SF2; + if (env) { + if (!existsSync(env)) die(`GM_SF2 set but file missing: ${env}`); + log(`using $GM_SF2 = ${env}`); + return { path: env, name: GENERALUSER_NAME }; + } + mkdirSync(CACHE_DIR, { recursive: true }); + const cachedSf2 = join(CACHE_DIR, "GeneralUser_GS.sf2"); + if (existsSync(cachedSf2)) { + log(`using cached SF2: ${cachedSf2}`); + return { path: cachedSf2, name: GENERALUSER_NAME }; + } + if (DRY_RUN) { + warn(`SF2 not present; would download from ${GENERALUSER_URL}`); + return { path: cachedSf2, name: GENERALUSER_NAME }; + } + log(`downloading ${GENERALUSER_NAME} → ${cachedSf2}`); + const zip = join(CACHE_DIR, "GeneralUser_GS.zip"); + const curl = spawnSync( + "curl", + ["-fsSL", "-o", zip, GENERALUSER_URL], + { stdio: "inherit" }, + ); + if (curl.status !== 0) { + die( + `download failed. Set $GM_SF2=/path/to/file.sf2 to bypass.\n` + + ` Upstream page: https://schristiancollins.com/generaluser.php`, + ); + } + // Try both unzip and bsdtar; locate the .sf2 inside. + const extractDir = join(CACHE_DIR, "extract"); + rmSync(extractDir, { recursive: true, force: true }); + mkdirSync(extractDir, { recursive: true }); + const unzip = spawnSync("unzip", ["-q", "-o", zip, "-d", extractDir], { + stdio: "inherit", + }); + if (unzip.status !== 0) die("unzip failed; install `unzip` or extract manually"); + // Find the SF2 file recursively. + const found = execFileSync("find", [extractDir, "-name", "*.sf2"]) + .toString() + .trim() + .split("\n") + .filter(Boolean); + if (found.length === 0) die("no .sf2 found inside the archive"); + execFileSync("cp", [found[0], cachedSf2]); + log(`extracted: ${cachedSf2}`); + return { path: cachedSf2, name: GENERALUSER_NAME }; +} + +// ─── render core ──────────────────────────────────────────────────────────── +const HOLD_SEC = 3; +const RELEASE_SEC = 1; +const TOTAL_SEC = HOLD_SEC + RELEASE_SEC; +const BITRATE = "96k"; +const SAMPLE_RATE = 44100; +const VELOCITY = 100; + +// Build a tiny MIDI file in memory: program change + note on + note off. +// Format 0, single track, 480 PPQ, tempo 120 → 1 quarter = 0.5s. +// Hold = HOLD_SEC. Track tail extends RELEASE_SEC for the release sample. +function buildMidi({ program, note, isDrum }) { + const PPQ = 480; + const TEMPO_US_PER_QN = 500000; // 120 BPM + const secondsToTicks = (s) => Math.round((s * 1_000_000 * PPQ) / TEMPO_US_PER_QN); + + const writeVarLen = (n) => { + const bytes = []; + bytes.push(n & 0x7f); + n >>= 7; + while (n > 0) { + bytes.unshift((n & 0x7f) | 0x80); + n >>= 7; + } + return bytes; + }; + + const channel = isDrum ? 9 : 0; // GM channel 10 = index 9 + const events = []; + + // Tempo meta event @ tick 0. + events.push(0); // delta + events.push(0xff, 0x51, 0x03, + (TEMPO_US_PER_QN >> 16) & 0xff, + (TEMPO_US_PER_QN >> 8) & 0xff, + TEMPO_US_PER_QN & 0xff); + + // Program change (skip for drum channel — kit selection is via bank, + // but for GM standard kit on chan 10 we still emit a PC=0). + events.push(...writeVarLen(0)); + events.push(0xc0 | channel, program & 0x7f); + + // Note on @ delta=0 + events.push(...writeVarLen(0)); + events.push(0x90 | channel, note & 0x7f, VELOCITY); + + // Note off @ delta=HOLD + events.push(...writeVarLen(secondsToTicks(HOLD_SEC))); + events.push(0x80 | channel, note & 0x7f, 0x40); + + // End of track @ delta=RELEASE (lets the SF release tail render fully) + events.push(...writeVarLen(secondsToTicks(RELEASE_SEC))); + events.push(0xff, 0x2f, 0x00); + + const trackBody = Buffer.from(events); + const header = Buffer.alloc(14); + header.write("MThd", 0, "ascii"); + header.writeUInt32BE(6, 4); + header.writeUInt16BE(0, 8); // format 0 + header.writeUInt16BE(1, 10); // 1 track + header.writeUInt16BE(PPQ, 12); + const trkHdr = Buffer.alloc(8); + trkHdr.write("MTrk", 0, "ascii"); + trkHdr.writeUInt32BE(trackBody.length, 4); + return Buffer.concat([header, trkHdr, trackBody]); +} + +function renderOne({ sf2, midiFile, wavFile, mp3File }) { + // fluidsynth headless render → wav + const fs = spawnSync( + "fluidsynth", + [ + "-ni", + "-g", "0.7", + "-r", String(SAMPLE_RATE), + "-F", wavFile, + "--fast-render", wavFile, + sf2, + midiFile, + ], + { stdio: ["ignore", "ignore", "pipe"] }, + ); + // Note: some fluidsynth builds use `--fast-render=path` form — try fallback. + if (fs.status !== 0 || !existsSync(wavFile)) { + const fs2 = spawnSync( + "fluidsynth", + [ + "-ni", + "-g", "0.7", + "-r", String(SAMPLE_RATE), + `--fast-render=${wavFile}`, + sf2, + midiFile, + ], + { stdio: ["ignore", "ignore", "pipe"] }, + ); + if (fs2.status !== 0 || !existsSync(wavFile)) { + const stderr = (fs.stderr?.toString() || "") + (fs2.stderr?.toString() || ""); + throw new Error(`fluidsynth failed:\n${stderr}`); + } + } + // ffmpeg → mp3 mono 96k + const ff = spawnSync( + "ffmpeg", + [ + "-y", "-loglevel", "error", + "-i", wavFile, + "-t", String(TOTAL_SEC), + "-codec:a", "libmp3lame", + "-b:a", BITRATE, + "-ac", "1", + "-ar", String(SAMPLE_RATE), + mp3File, + ], + { stdio: ["ignore", "ignore", "pipe"] }, + ); + if (ff.status !== 0 || !existsSync(mp3File)) { + throw new Error(`ffmpeg failed:\n${ff.stderr?.toString() || ""}`); + } +} + +// ─── plan & execute ───────────────────────────────────────────────────────── +function planMelodicNotes(step) { + const notes = []; + for (let m = 21; m <= 108; m += step) notes.push(m); + // Always include the boundary if step skipped it. + if (notes[notes.length - 1] !== 108) notes.push(108); + return notes; +} +function planDrumNotes() { + return Object.keys(GM_DRUM_KIT_NOTE_NAMES) + .map(Number) + .sort((a, b) => a - b); +} + +function selectedPrograms() { + if (!ONLY) return [...Array(128).keys()]; + return ONLY.split(",") + .map((s) => parseInt(s.trim(), 10)) + .filter((n) => Number.isInteger(n) && n >= 0 && n <= 127); +} + +async function main() { + log(`output: ${OUT_DIR}`); + log(`note step: ${NOTE_STEP} (range A0..C8 = 21..108)`); + if (DRY_RUN) warn("dry-run: planning only, no audio rendered"); + + const { path: sf2Path, name: sf2Name } = resolveSoundFont(); + mkdirSync(OUT_DIR, { recursive: true }); + + const programs = selectedPrograms(); + const melodicNotes = planMelodicNotes(NOTE_STEP); + const drumNotes = planDrumNotes(); + + const manifest = { + soundfont: sf2Name, + license: "https://schristiancollins.com/generaluser.php", + noteStep: NOTE_STEP, + format: "mp3", + sampleRate: SAMPLE_RATE, + bitrate: BITRATE, + channels: 1, + durationSec: TOTAL_SEC, + holdSec: HOLD_SEC, + releaseSec: RELEASE_SEC, + patches: [], + drumKits: [], + }; + + const tmp = tmpdir(); + let rendered = 0; + let skipped = 0; + const failures = []; + + // Melodic programs. + if (!SKIP_MELODIC) { + for (const program of programs) { + const dirName = String(program).padStart(3, "0"); + const progDir = join(OUT_DIR, dirName); + mkdirSync(progDir, { recursive: true }); + manifest.patches.push({ + id: program, + name: GM_PATCHES[program], + notes: [...melodicNotes], + }); + for (const note of melodicNotes) { + const noteName = midiToName(note); + const mp3File = join(progDir, `${noteName}.mp3`); + if (existsSync(mp3File) && statSync(mp3File).size > 0) { + skipped++; + continue; + } + if (DRY_RUN) { + rendered++; + continue; + } + const midiFile = join(tmp, `gm-${program}-${note}.mid`); + const wavFile = join(tmp, `gm-${program}-${note}.wav`); + try { + writeFileSync(midiFile, buildMidi({ program, note, isDrum: false })); + renderOne({ sf2: sf2Path, midiFile, wavFile, mp3File }); + rendered++; + if (rendered % 25 === 0) { + process.stdout.write( + `${C.dim} rendered ${rendered} (skipped ${skipped})${C.reset}\n`, + ); + } + } catch (err) { + failures.push({ program, note, error: err.message }); + warn(`program ${program} note ${note}: ${err.message.split("\n")[0]}`); + } finally { + rmSync(midiFile, { force: true }); + rmSync(wavFile, { force: true }); + } + } + } + } + + // Drum kit (GM Standard Kit on channel 10, program 0). + if (!SKIP_DRUMS) { + const drumDir = join(OUT_DIR, "drum-000"); + mkdirSync(drumDir, { recursive: true }); + manifest.drumKits.push({ + id: 0, + name: "Standard Kit", + notes: drumNotes, + noteNames: GM_DRUM_KIT_NOTE_NAMES, + }); + for (const note of drumNotes) { + const mp3File = join(drumDir, `${note}.mp3`); + if (existsSync(mp3File) && statSync(mp3File).size > 0) { + skipped++; + continue; + } + if (DRY_RUN) { + rendered++; + continue; + } + const midiFile = join(tmp, `gm-drum-${note}.mid`); + const wavFile = join(tmp, `gm-drum-${note}.wav`); + try { + writeFileSync(midiFile, buildMidi({ program: 0, note, isDrum: true })); + renderOne({ sf2: sf2Path, midiFile, wavFile, mp3File }); + rendered++; + } catch (err) { + failures.push({ kit: 0, note, error: err.message }); + warn(`drum note ${note}: ${err.message.split("\n")[0]}`); + } finally { + rmSync(midiFile, { force: true }); + rmSync(wavFile, { force: true }); + } + } + } + + // Manifest + license. + writeFileSync( + join(OUT_DIR, "manifest.json"), + JSON.stringify(manifest, null, 2) + "\n", + ); + writeFileSync(join(OUT_DIR, "LICENSE.txt"), GENERALUSER_LICENSE); + + log(`done. rendered=${rendered} skipped=${skipped} failures=${failures.length}`); + if (failures.length > 0) { + warn(`${failures.length} render failure(s); see warnings above`); + } + console.log(""); + console.log(` manifest: ${join(OUT_DIR, "manifest.json")}`); + console.log(` license: ${join(OUT_DIR, "LICENSE.txt")}`); + console.log(""); + console.log(` publish: npm run assets:sync:up # uploads to assets.aesthetic.computer/gm/`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/plans/notepat-gm-integration.md b/plans/notepat-gm-integration.md new file mode 100644 index 0000000000..61bdef534f --- /dev/null +++ b/plans/notepat-gm-integration.md @@ -0,0 +1,225 @@ +# Notepat GM Integration Plan + +## 1. Conflict audit — top-row digits 0–9 in notepat.mjs + +I searched all keyboard handlers for digit bindings. + +**Result: there are NO existing keyboard handlers for digit keys 0–9 in notepat.mjs.** Confirmed via: +- `grep -nE 'keyboard:down:[0-9]'` — zero hits. +- `grep -nE '"[0-9]"'` — only finds glyph-resolution lookups (typeface lookups for the character "0"), the `buttonOctaves` array (line 1058 — listing octave *numbers* as strings used as octBtn labels, not key bindings), DAW query parsing (line 1468), and HID scancode→key tables (lines 30–31, used by NuPhy WebHID, not the keyboard event bus). + +**Adjacent bindings that are nearby on the keyboard but not on digit row:** +- `,` (comma) and `.` (period) — `upperOctaveShift -=/+= 1` (lines 6535, 6539). Keep — not on digit row. +- `-` and `=` — paint overlay toggles (lines 6626–6627). Keep. +- `tab` — cycles `waveIndex` (line 6613–6620). **Repurpose / keep:** still cycles wave but now wave list begins with `"gm"`. +- `space` — metronome toggle (line 7281). Keep. +- `arrowup/down/left/right` — drum pads (lines 7312–7334). Keep. +- `alt` — crash/ride drums (lines 7300–7310). Keep. +- `shift`, `enter`, `backspace`, `escape`, `/`, `\`, `` ` `` — various toggles. Keep. + +**Conclusion: digit row is fully free.** No conflicts. All 10 digits (0–9) can be claimed for the GM digit-buffered picker without breaking anything. + +The NuPhy HID table at lines 30–31 maps HID scancodes to digit *strings* — that is hardware-side translation, not an `act` handler, and the resulting `keyboard:down:0` … `keyboard:down:9` events flow through the same dispatcher that currently ignores digits. Once we add a digit handler, NuPhy digit presses will pick GM patches just like an OS keyboard. (Worth a one-line comment in the code.) + +## 2. State changes + +New module-level state to add near line 393 (next to `wave`): + +```js +// GM (General MIDI) sound backend — see lib/gm.mjs +let gmReady = false; // True once gm.loadManifest resolves. +let gmManifest = null; // Cached manifest from gm.mjs. +let gmProgram = 0; // Current GM program 0..127 (0 = Acoustic Grand Piano). +let gmPatch = null; // Currently loaded patch handle from gm.loadPatch. +let gmPatchLoading = false; // In-flight guard. +let gmDrumKit = null; // Optional GM drum kit handle. +let gmDigitBuffer = ""; // "0".."999" while user types. +let gmDigitBufferDeadline = 0; // performance.now() when buffer auto-commits. +const GM_DIGIT_TIMEOUT_MS = 700; // Auto-commit window (matches user spec). +const GM_PROGRAM_NAMES = [/* 128 GM names */]; // For the HUD label. +// Map of active GM voices: voiceId -> noteHandle returned by patch.play(midi). +const gmVoices = new Map(); +``` + +Existing variables changed: + +- `wavetypes` (line 380) becomes `["gm", ...legacyWavetypes]`. Keep the legacy names exactly so saved settings still work. +- `STARTING_WAVE` (line 392) → `"gm"`. +- `waveIndex` (line 391) → `0` (still index zero of new array). +- `shortWaveNames` (line 8504) — add `gm: "GM"`. +- `displayWave` formatting (line 8514) — when `wave !== "gm"`, optionally prefix with `"L:"` to make it visually clear the user is in legacy mode (decision needed — see §8). + +## 3. Sound dispatch changes + +The unified branch point lives in `makeNoteSound(tone, velocity, pan)` at **line 5967**. The current `if/else if/else` ladder distinguishes `stample`/`sample` → `play()`, `composite` → multi-synth, default → single `synth({ type: synthType })`. + +Plan: add `wave === "gm"` as the **first** branch. This branch must: + +1. Convert notepat's `tone` (a string like `"4C"`, `"5G#"`) to a MIDI note number. Use `soundContext.freq(tone)` to get Hz, then `12 * log2(hz/440) + 69`. Or expose a helper from `lib/note-colors.mjs` if one already does this. (Decision: add a tiny inline `toneToMidi(tone)` helper at module scope; see §8.) +2. Bail to oscillator path (`return synth({ type: "sine", … })`) if `!gmReady || !gmPatch` — graceful fallback while GM is still loading or assets missing. +3. Call `const noteHandle = gmPatch.play(midi, { velocity, pan });` +4. Wrap in a uniform return object with `kill(fade)` → `noteHandle.release()` and a no-op `update()` (or a real `update()` if the gm.mjs API lands one — currently it does not, per the agent's prompt). +5. Track in `gmVoices` keyed by the `voiceId` so panic / clearHeldVoices can reach them. + +Drum case: at **line 6146** the existing `wave === "drum"` branch calls `playPercussion(...)`. Add a parallel branch *above* it: `if (wave === "gm" && /* user mapped drum kit */) { gmDrumKit?.play(midiDrumNote); return true; }`. **Decision needed (§8):** simplest cut is to keep `wave === "drum"` doing the existing percussion lib, and let GM stay melodic-only for v1. Drums via GM kit can be a follow-up — the bake agent is producing per-instrument packs anyway and a GM drum kit is patch 128 conceptually. + +Pitch bend in `applyPitchBendToNotes` (line 6082) needs a GM branch too — for v1 we can skip live pitch bend on GM voices (just return early when `wave === "gm"`) since the gm.mjs API doesn't expose a per-voice frequency setter. Document this limitation. + +## 4. Tab / legacy UX + +**`wavetypes` ordering (line 380):** + +```js +const wavetypes = [ + "gm", // 0 — General MIDI (default) + "sine", // 1 + "triangle", // 2 + "sawtooth", // 3 + "square", + "harp", + "whistle", + "composite", + "stample", + "drum", +]; +``` + +Tab key (line 6613) and waveBtn `push` (line 7383) already cycle `waveIndex = (waveIndex + 1) % wavetypes.length`. No change to that mechanism — they automatically include "gm" as position 0. + +**Display label (`buildWaveButton` ~line 8500):** when `wave === "gm"` the button label is `GM:078` (current program shown so the user always knows what's loaded). When in any legacy wave, the label keeps the wave name (`sine`, `tri`, etc.). Optional polish: dim the legacy labels or add a `[L]` prefix to signal they're not the default. + +The drum special case at line 1133 comment stays — `wave === "drum"` still routes through `lib/percussion.mjs`. + +## 5. Digit-buffered picker — pseudocode + +Insertion point: in `act` near the existing tab handler (line 6613), well above the percussion arrow handlers. + +```js +// GM patch picker — type a 1-3 digit decimal program number. +// Mirrors menubands behavior with one addition: a 700ms timeout +// auto-commits and clears the buffer if the user pauses, matching +// the spec. Buffer also clears on any non-digit key press, on +// reaching 3 digits, or on a note key being struck. +{ + const digitMatch = e.is("keyboard:down") && /^[0-9]$/.test(e.key) && !e.repeat; + const now = performance.now(); + + // Auto-commit on timeout before processing this event. + if (gmDigitBuffer && now > gmDigitBufferDeadline) { + gmDigitBuffer = ""; + } + + if (digitMatch) { + if (gmDigitBuffer.length >= 3) gmDigitBuffer = ""; + gmDigitBuffer += e.key; + gmDigitBufferDeadline = now + GM_DIGIT_TIMEOUT_MS; + + const v = parseInt(gmDigitBuffer, 10); + // Decision: notepat does not have menuband's MIDI-passthrough + // mode, so "0" / "00" / "000" maps to GM program 0 (Acoustic + // Grand Piano) instead of toggling a passthrough. Document this + // divergence from menuband. See §8. + const program = v === 0 ? 0 : Math.max(0, Math.min(127, v - 1)); + + // Switch to GM if the user is currently on a legacy wave, so + // typing a digit always produces a GM voice. (Optional — could + // also just queue the program for next time wave=="gm". Pick + // the auto-switch behavior; it's friendlier.) + if (wave !== "gm") { + waveIndex = wavetypes.indexOf("gm"); + wave = "gm"; + buildWaveButton(api); + } + + setGmProgram(program); // async: loads patch, swaps gmPatch on resolve. + api.beep(); // Subtle audio confirmation. + return; + } + + // Non-digit key: clear the buffer (don't auto-commit anything new + // — the program was already applied live on each digit). + if (e.is("keyboard:down") && gmDigitBuffer) { + gmDigitBuffer = ""; + } +} +``` + +Helper: + +```js +async function setGmProgram(program) { + gmProgram = program; + if (gmPatchLoading) return; // Latest call wins via re-check after await. + gmPatchLoading = true; + try { + const next = await gm.loadPatch(program, window.audioContext); + if (program === gmProgram) { + gmPatch = next; // Apply only if user hasn't typed past us. + } + buildWaveButton(api); // Refresh "GM:078" label. + } catch (err) { + console.warn("🎼 GM load failed", program, err); + } finally { + gmPatchLoading = false; + } +} +``` + +Debug HUD: `gmDigitBuffer` is rendered as `GM:078` while typing in the existing top-bar status row (next to the wave button) so the user sees the partial buffer live. After commit, the label shows the full program name (e.g., `78 Whistle`). + +Note key resets buffer: in `startButtonNote` (line 6123) and `startRelayButtonNote` (line ~5780), add `gmDigitBuffer = ""` at the top — mirrors menuband line 1176. + +## 6. MIDI input parity + +**Recommend yes** — incoming MIDI program-change messages should switch the GM patch. + +In `lib/midi.mjs` line 13, the filter `if (command !== NOTE_ON && command !== NOTE_OFF && command !== PITCH_BEND) return;` drops program-change. Add `0xC0` (PROGRAM_CHANGE) to the allow-list and forward it via `acSEND`. + +In notepat.mjs `act`'s `midi:keyboard` block (line 7233), after the existing `MIDI_PITCH_BEND` branch, add: + +```js +const MIDI_PROGRAM_CHANGE = 0xC0; +if (command === MIDI_PROGRAM_CHANGE) { + const program = e.data?.[1] ?? 0; + setGmProgram(Math.max(0, Math.min(127, program))); + return; +} +``` + +This keeps the relay/MIDI plumbing in the existing single `midi:keyboard` event channel; no new event type needed. + +## 7. Step-by-step implementation order + +1. **Add `gm.mjs` import + state (~30 lines).** Top of notepat.mjs near other imports. Add module-level state from §2. *Blocks on:* parallel agent's gm.mjs landing — but the import will simply 404 until then; we can guard with a try/catch around `await import(...)`. For development, a stubbed local gm.mjs that no-ops is sufficient. **Can start immediately.** +2. **Extend `wavetypes` array + `displayWave` map (~10 lines).** Lines 380, 8504. Independent — can land first. +3. **Boot-time GM init (~20 lines).** In `boot()` around line 1457, after `setSoundContext`, call `await gm.loadManifest(window.audioContext)` and `setGmProgram(0)`. Set `gmReady = true`. *Blocks on:* gm.mjs API. +4. **`makeNoteSound` GM branch (~25 lines).** Line 5967. Add the early-exit gm path with fallback. *Blocks on:* gm.mjs play API. +5. **Stop-path / panic / pitch-bend GM handling (~20 lines).** `stopButtonNote`, `clearHeldVoices`, escape panic block (line 6640), `applyPitchBendToNotes` early-return for gm. Independent of gm.mjs once the play path is shaped — uses the returned handle's `release()`. +6. **Digit picker handler (~35 lines).** Insert in `act()` around line 6620 (just after the tab/wave handler so digits can't clobber tab cycling). Add `gmDigitBuffer = ""` resets in `startButtonNote` and `startRelayButtonNote`. Independent. +7. **Wave-button label refresh (~5 lines).** `buildWaveButton` line 8500 — when `wave === "gm"`, label is `GM:NNN`. +8. **MIDI program-change forwarding (~10 lines).** `lib/midi.mjs` (allow 0xC0) + notepat.mjs midi:keyboard branch. Coordinate with the midi.mjs owner — minor edit, low risk. +9. **Bake-output verification (manual).** Once the bake agent publishes to `assets.aesthetic.computer/gm/`, smoke-test by switching to a few patches (1, 25, 78, 128) and a drum kit. *Blocks on:* bake landing. +10. **Optional polish.** GM drum kit routing, sustain/pan parity for GM voices, persisted last-program in `store`. + +Total disk-side change ≈ 150–180 net lines added, ~10 lines edited. + +## 8. Risks / open questions — decisions needed + +1. **0 / 00 / 000 mapping.** Menuband uses these for "MIDI passthrough." Notepat has no equivalent backend slot. **Recommended decision:** map `0`/`00`/`000` to GM program 0 (Acoustic Grand). Document the divergence in a code comment. Alternative: use `0` to *toggle MIDI passthrough* — i.e., temporarily disable GM and let the user's external synth handle voicing. Probably overkill for v1. +2. **Auto-switch wave on digit press?** Plan above auto-switches `wave` to `"gm"` when the user types a digit while in legacy mode. Alternative: queue the program but stay in legacy mode. **Recommended:** auto-switch — matches user intent ("I'm picking an instrument"). +3. **gm.mjs API timing.** If notepat ships before `gm.mjs` does, the import will 404 and break boot. Wrap in `try { gmModule = await import("../lib/gm.mjs"); } catch { gmModule = null; }` and treat `gmModule == null` as "fall back to oscillator forever." This lets disk-side ship independently. (Recommended.) +4. **Bake assets missing at runtime.** `gm.loadManifest` will fail on first run if `assets.aesthetic.computer/gm/` is empty. The plan's fallback (`!gmReady || !gmPatch` → oscillator path) handles this, but the user will silently get sine instead of a GM voice. **Recommended:** when GM fails to load, surface a one-time HUD note (`"GM unavailable, using oscillator"`) and keep the wave label as `gm` but parenthesized: `(gm)`. +5. **Pitch bend on GM voices.** gm.mjs API as documented does not expose per-voice frequency setters. v1 plan: skip pitch bend for GM voices (early-return). **Decision needed:** acceptable for v1, or block on gm.mjs adding `noteHandle.setDetune(cents)`? +6. **`wave === "gm" && wave === "drum"` collision.** If the user wants a GM drum kit, today's `wavetypes` entry `"drum"` means percussion-lib drums, not GM kit. v1 plan keeps `"drum"` as legacy percussion-lib and leaves "GM drum kit" as a follow-up. **Decision needed:** add a separate `"gm-drum"` wavetypes entry, or extend program > 127 to mean drum kit? +7. **`tone` → MIDI note conversion.** notepat's `tone` is a string. `soundContext.freq(tone)` returns Hz. `Math.round(12 * Math.log2(hz/440) + 69)` produces the MIDI note. **Decision needed:** put the helper in notepat.mjs locally, or add to `lib/note-colors.mjs` for reuse? +8. **Velocity scaling.** gm.mjs `play(note, { velocity })` semantics not documented (0–127? 0–1?). Assume 0–127 to match menuband. **Coordinate with the gm.mjs author.** +9. **Voice stealing.** If the user holds a chord on GM and the patch's polyphony cap is exceeded, who wins? Probably gm.mjs handles internally; notepat just calls `play` and trusts the player. Document. +10. **`gmDigitBuffer` and NuPhy.** NuPhy delivers digit keys as `keyboard:down:0`–`9` through the same bus, so the picker works for hardware presses too — desirable. + +## Critical Files for Implementation + +- /Users/jas/aesthetic-computer/system/public/aesthetic.computer/disks/notepat.mjs +- /Users/jas/aesthetic-computer/system/public/aesthetic.computer/lib/gm.mjs +- /Users/jas/aesthetic-computer/system/public/aesthetic.computer/lib/midi.mjs +- /Users/jas/aesthetic-computer/slab/menuband/Sources/MenuBand/MenuBandController.swift +- /Users/jas/aesthetic-computer/system/public/aesthetic.computer/lib/note-colors.mjs diff --git a/system/public/aesthetic.computer/disks/notepat.mjs b/system/public/aesthetic.computer/disks/notepat.mjs index e7e04d0036..31b66a3ea6 100644 --- a/system/public/aesthetic.computer/disks/notepat.mjs +++ b/system/public/aesthetic.computer/disks/notepat.mjs @@ -13,6 +13,7 @@ import { loadPaintingAsAudio, } from "../lib/pixel-sample.mjs"; import { playPercussion } from "../lib/percussion.mjs"; +import * as gm from "../lib/gm.mjs"; // 🎹 NuPhy Air60 HE — WebHID analog pressure support // Sends activation commands then reads 0xA0 analog reports with per-key pressure. @@ -378,15 +379,16 @@ TODO: 💮 Daisy let STARTING_OCTAVE = "4"; const wavetypes = [ - "sine", // 0 - "triangle", // 1 - "sawtooth", // 2 - "square", // 3 - "harp", // 4 - Karplus-Strong plucked string (Karplus & Strong 1983) - "whistle", // 5 - digital waveguide flute (Cook/STK) - "composite", // 6 - "stample", // 7 - "drum", // 8 - shared 12-drum kit (lib/percussion.mjs), both octaves + "gm", // 0 - General MIDI (lib/gm.mjs); number-row digits pick programs 0-127 + "sine", // 1 + "triangle", // 2 + "sawtooth", // 3 + "square", // 4 + "harp", // 5 - Karplus-Strong plucked string (Karplus & Strong 1983) + "whistle", // 6 - digital waveguide flute (Cook/STK) + "composite", // 7 + "stample", // 8 + "drum", // 9 - shared 12-drum kit (lib/percussion.mjs), both octaves ]; let waveIndex = 0; // 0; const STARTING_WAVE = wavetypes[waveIndex]; //"sine"; @@ -398,6 +400,43 @@ let roomMode = false; // 🏠 Global reverb toggle let roomAmount = 0.5; // 🎚️ Room/reverb amount (0-1) let glitchMode = false; // 🧩 Global glitch toggle let octave = STARTING_OCTAVE; + +// 🎼 GM (General MIDI) state — see lib/gm.mjs +// Number-row digits 0-9 type into a 1-3 digit decimal buffer that picks GM +// program 0-127 live (mirrors menuband). 700ms pause auto-clears the buffer. +// "0" / "00" / "000" map to program 0 (Acoustic Grand Piano) — notepat has +// no MIDI-passthrough slot like menuband does. +let gmReady = false; +let gmProgram = 0; +let gmPatch = null; +let gmPatchLoading = false; +let gmInitStarted = false; +let gmDigitBuffer = ""; +let gmDigitBufferDeadline = 0; +const GM_DIGIT_TIMEOUT_MS = 700; + +// Fire the GM bootstrap exactly once, as soon as window.audioContext is +// available. Browsers gate audio creation behind the first user gesture, +// so this can't run at boot — it has to wait for setSoundContext to see +// a real context. +function maybeInitGM() { + if (gmInitStarted) return; + if (typeof window === "undefined" || !(/** @type {any} */ (window).audioContext)) return; + gmInitStarted = true; + (async () => { + try { + await gm.loadManifest(); + const patch = await gm.loadPatch(gmProgram); + if (patch) { + gmPatch = patch; + gmReady = true; + } + } catch (err) { + console.warn("🎼 GM init failed — staying on oscillator fallback:", err); + } + })(); +} + let keys = ""; let tap = false; let tapIndex = 0; @@ -1662,6 +1701,7 @@ async function boot({ // } const wavetypes = [ + "gm", "square", "sine", "triangle", @@ -5683,6 +5723,7 @@ let soundContext = null; function setSoundContext(ctx) { soundContext = ctx; + maybeInitGM(); } function lowerBaseOctave() { @@ -5964,6 +6005,28 @@ function flushRelayMidiQueue() { } } +// 🎼 Switch the live GM patch. Awaitable; safely no-ops if a newer call +// supersedes this one mid-load. Triggered by the digit-buffered picker +// in `act()` and by incoming MIDI program-change messages. +async function setGmProgram(program, apiRef) { + const next = Math.max(0, Math.min(127, program | 0)); + gmProgram = next; + if (gmPatchLoading) return; + gmPatchLoading = true; + try { + const patch = await gm.loadPatch(next); + if (gmProgram === next) { + gmPatch = patch; + gmReady = true; + } + if (apiRef) buildWaveButton(apiRef); + } catch (err) { + console.warn("🎼 GM: loadPatch failed for", next, err); + } finally { + gmPatchLoading = false; + } +} + function makeNoteSound(tone, velocity = 127, pan = 0) { const synth = soundContext?.synth; const play = soundContext?.play; @@ -5991,7 +6054,38 @@ function makeNoteSound(tone, velocity = 127, pan = 0) { const minVelocityVolume = 0.05; // Keep a subtle floor so very light taps still play. const volumeScale = minVelocityVolume + (1 - minVelocityVolume) * velocityRatio; - if (wave === "stample" || wave === "sample") { + if (wave === "gm") { + // 🎼 GM playback via lib/gm.mjs. Falls back to a sine oscillator if the + // patch isn't ready yet (manifest still loading, or asset host down). + // The shim mimics the synth handle shape — { startedAt, kill, update } — + // so the rest of notepat (panic, sustain, pitch-bend) doesn't have to + // care which backend played the note. Pitch-bend update is a no-op for + // GM voices in v1; gm.mjs doesn't expose a per-voice frequency setter. + if (gmReady && gmPatch) { + const hz = freq(tone); + const midi = Math.round(12 * Math.log2(hz / 440) + 69); + const ctx = + typeof window !== "undefined" ? /** @type {any} */ (window).audioContext : null; + const startedAt = ctx?.currentTime ?? performance.now() / 1000; + const noteHandle = gmPatch.play(midi, { + velocity: Math.round(velocityRatio * 127), + }); + return { + startedAt, + kill: (_fade) => noteHandle.release(), + update: () => {}, + }; + } + // Fallback while GM is still loading or unavailable. + return synth({ + type: "sine", + attack: quickFade ? 0.0015 : attack, + tone, + duration: "🔁", + volume: toneVolume * volumeScale, + pan, + }); + } else if (wave === "stample" || wave === "sample") { const sampleId = stampleSampleId || startupSfx; return play(sampleId, { volume: volumeScale, @@ -6619,6 +6713,54 @@ function act({ buildOsButton(api); } + // 🎼 GM digit-buffered patch picker — top-row 0-9 like menuband. Each + // digit press updates the live GM program; the buffer auto-clears after + // GM_DIGIT_TIMEOUT_MS of inactivity or when a non-digit key arrives. + // NuPhy WebHID also delivers digits through this channel, so hardware + // presses pick patches too. + { + const isKbdDown = e.is("keyboard:down") && !e.repeat; + const digit = isKbdDown && /^[0-9]$/.test(e.key) ? e.key : null; + const now = performance.now(); + + if (gmDigitBuffer && now > gmDigitBufferDeadline) { + gmDigitBuffer = ""; + } + + if (digit) { + if (gmDigitBuffer.length >= 3) gmDigitBuffer = ""; + gmDigitBuffer += digit; + gmDigitBufferDeadline = now + GM_DIGIT_TIMEOUT_MS; + + // "0" / "00" / "000" → program 0; otherwise typed value clamped 0-127. + const v = parseInt(gmDigitBuffer, 10); + const program = Math.max(0, Math.min(127, v)); + + // Friendly auto-switch: typing a digit while in a legacy wave snaps + // the wave switcher to "gm" so the user actually hears the program + // they just picked. + if (wave !== "gm") { + waveIndex = wavetypes.indexOf("gm"); + if (waveIndex >= 0) { + wave = "gm"; + buildAbletonButton(api); + buildOsButton(api); + } + } + + setGmProgram(program, api); + buildWaveButton(api); // Immediate label refresh — shows the buffer live. + api.beep(); + return; + } + + // Non-digit keypress clears any pending buffer (the live program was + // already applied per-digit, so nothing to commit). + if (isKbdDown && gmDigitBuffer) { + gmDigitBuffer = ""; + } + } + // if (e.is("keyboard:down:shift") && !e.repeat) { // lowerOctaveShift -= 1; // } @@ -7016,6 +7158,7 @@ function act({ const MIDI_NOTE_ON = 0x90; const MIDI_NOTE_OFF = 0x80; const MIDI_PITCH_BEND = 0xe0; + const MIDI_PROGRAM_CHANGE = 0xc0; const midiNoteToButton = (noteNumber) => { if (!midiUtil?.note || typeof noteNumber !== "number") return null; @@ -7255,6 +7398,21 @@ function act({ return; } + // 🎼 Incoming GM program-change → switch the active GM patch. Auto- + // snaps the wave switcher to "gm" so the user hears the new voice. + if (command === MIDI_PROGRAM_CHANGE) { + const program = noteNumber ?? 0; + if (wave !== "gm") { + const idx = wavetypes.indexOf("gm"); + if (idx >= 0) { + waveIndex = idx; + wave = "gm"; + } + } + setGmProgram(program, api); + return; + } + if (typeof noteNumber === "number") { if (command === MIDI_NOTE_ON && velocity > 0) { // 📊 Track MIDI key press time for latency measurement @@ -8510,8 +8668,17 @@ function buildWaveButton({ screen, ui, typeface }) { composite: "cmp", stample: "stp", drum: "drm", + gm: "gm", }; - const displayWave = isNarrow ? (shortWaveNames[wave] || wave.slice(0, 3)) : wave; + // GM wave shows the live program/buffer so the user always knows what's + // loaded and what they're partway through typing. + let displayWave; + if (wave === "gm") { + const shown = gmDigitBuffer || String(gmProgram).padStart(3, "0"); + displayWave = isNarrow ? `g${shown}` : `GM:${shown}`; + } else { + displayWave = isNarrow ? (shortWaveNames[wave] || wave.slice(0, 3)) : wave; + } const waveWidth = displayWave.length * glyphWidth; const margin = isNarrow ? 2 : 4; waveBtn = new ui.Button( diff --git a/system/public/aesthetic.computer/lib/gm.mjs b/system/public/aesthetic.computer/lib/gm.mjs new file mode 100644 index 0000000000..c7764b5eb6 --- /dev/null +++ b/system/public/aesthetic.computer/lib/gm.mjs @@ -0,0 +1,490 @@ +// GM (General MIDI) sample-based player. +// +// Loads pitched melodic patches and the GM standard drum kit from the +// AC asset CDN and plays them through the existing browser AudioContext +// (`window.audioContext`, set up by bios.mjs). Each patch is sparsely +// rendered (every 3rd semitone) so this player pitch-shifts the nearest +// rendered sample via Web Audio's `playbackRate` to fill in missing +// notes, applies a snappy ADSR-ish gain envelope, and exposes per-note +// stop()/release() handles. +// +// Asset layout (produced by the lith bake script — see lith/): +// https://assets.aesthetic.computer/gm/manifest.json +// https://assets.aesthetic.computer/gm//.mp3 (NNN = 000..127) +// https://assets.aesthetic.computer/gm/drum-000/.mp3 +// +// Note files use scientific-pitch names with `s` for sharps: +// C4.mp3, Cs4.mp3, D4.mp3, Ds4.mp3, ... A4.mp3, As4.mp3, B4.mp3 +// +// This module has NO build step — it is a plain ES module loaded by the +// runtime. It does not modify disk.mjs, midi.mjs, or any disk file; the +// program-change wiring will be hooked up elsewhere. + +const ASSET_BASE = "https://assets.aesthetic.computer/gm"; +const MANIFEST_URL = `${ASSET_BASE}/manifest.json`; + +// ─── AudioContext access ────────────────────────────────────────────── +// AC's bios.mjs creates and stores a single AudioContext at +// `window.audioContext`. All other lib helpers either receive a `sound` +// API or read the global. We accept an explicit context for testability +// and fall back to the window global. +function resolveAudioContext(audioContext) { + if (audioContext) return audioContext; + const w = typeof window !== "undefined" ? /** @type {any} */ (window) : null; + if (w && w.audioContext) return w.audioContext; + return null; +} + +// ─── Note name <-> MIDI helpers ─────────────────────────────────────── +// Sample filenames use `s` (e.g. "Cs4") instead of "#" because URL +// safety. Octave numbering is scientific-pitch: C4 = MIDI 60. +const NOTE_TO_PC = { + C: 0, Cs: 1, D: 2, Ds: 3, E: 4, F: 5, + Fs: 6, G: 7, Gs: 8, A: 9, As: 10, B: 11, +}; +const PC_TO_NOTE = ["C", "Cs", "D", "Ds", "E", "F", "Fs", "G", "Gs", "A", "As", "B"]; + +export function midiToNoteName(midi) { + const pc = ((midi % 12) + 12) % 12; + const octave = Math.floor(midi / 12) - 1; + return `${PC_TO_NOTE[pc]}${octave}`; +} + +export function noteNameToMidi(name) { + const m = /^([A-G])(s?)(-?\d+)$/.exec(name); + if (!m) return null; + const letter = m[1] + (m[2] || ""); + const pc = NOTE_TO_PC[letter]; + if (pc === undefined) return null; + const octave = parseInt(m[3], 10); + return (octave + 1) * 12 + pc; +} + +// ─── Manifest ───────────────────────────────────────────────────────── +// The bake script writes a manifest.json describing which programs are +// available, which notes were rendered per program, sample rate, etc. +// We tolerate a missing/incomplete manifest by falling back to a +// default note grid (every 3rd semitone over MIDI 24..96). +let _manifest = null; +let _manifestPromise = null; + +const DEFAULT_NOTE_STEP = 3; +const DEFAULT_LOW = 24; // C1 +const DEFAULT_HIGH = 96; // C7 + +function defaultNoteList() { + const list = []; + for (let m = DEFAULT_LOW; m <= DEFAULT_HIGH; m += DEFAULT_NOTE_STEP) { + list.push(m); + } + return list; +} + +export async function loadManifest(audioContext) { + // audioContext is accepted for API symmetry with loadPatch / loadDrumKit + // (they need it to decode); manifest loading itself is pure JSON. + void audioContext; + if (_manifest) return _manifest; + if (_manifestPromise) return _manifestPromise; + + _manifestPromise = (async () => { + try { + const res = await fetch(MANIFEST_URL, { cache: "force-cache" }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const json = await res.json(); + _manifest = normalizeManifest(json); + } catch (err) { + console.warn("🎼 GM: manifest unavailable, using defaults:", err.message); + _manifest = { + patches: {}, + drumKits: { 0: { notes: defaultNoteList() } }, + noteStep: DEFAULT_NOTE_STEP, + defaultNotes: defaultNoteList(), + _fallback: true, + }; + } + return _manifest; + })(); + + return _manifestPromise; +} + +function normalizeManifest(raw) { + const out = { + patches: {}, + drumKits: {}, + noteStep: raw.noteStep || raw.step || DEFAULT_NOTE_STEP, + defaultNotes: null, + _raw: raw, + }; + // Patches may be an array, an object keyed by program number, or a + // top-level `patches` field. Normalize to { [program]: { notes:[...] } }. + const patchSrc = raw.patches || raw.programs || raw; + if (Array.isArray(patchSrc)) { + for (const entry of patchSrc) { + if (entry?.program != null) { + out.patches[entry.program] = { notes: entry.notes || null, name: entry.name }; + } + } + } else if (patchSrc && typeof patchSrc === "object") { + for (const [k, v] of Object.entries(patchSrc)) { + const pn = parseInt(k, 10); + if (!Number.isFinite(pn) || pn < 0 || pn > 127) continue; + if (v && typeof v === "object") { + out.patches[pn] = { notes: v.notes || null, name: v.name }; + } + } + } + const drumSrc = raw.drumKits || raw.drums || {}; + for (const [k, v] of Object.entries(drumSrc)) { + const id = parseInt(k, 10); + if (!Number.isFinite(id)) continue; + out.drumKits[id] = { notes: v?.notes || null, name: v?.name }; + } + if (!out.drumKits[0]) out.drumKits[0] = { notes: null }; + out.defaultNotes = defaultNoteList(); + return out; +} + +// ─── Buffer fetch + decode (cached per URL) ─────────────────────────── +const _bufferCache = new Map(); // url -> Promise + +function fetchAndDecode(url, ctx) { + if (_bufferCache.has(url)) return _bufferCache.get(url); + const p = (async () => { + try { + const res = await fetch(url, { cache: "force-cache" }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const arrayBuf = await res.arrayBuffer(); + // decodeAudioData has both promise and callback forms. Use the + // promise form; old Safari is wrapped by the AC bios so it should + // be fine here. + const audioBuf = await ctx.decodeAudioData(arrayBuf); + return audioBuf; + } catch (err) { + console.warn(`🎼 GM: failed to load ${url}:`, err.message); + return null; + } + })(); + _bufferCache.set(url, p); + return p; +} + +// ─── Patch / kit handles ────────────────────────────────────────────── +// A patch handle keeps its set of rendered MIDI notes and lazily loads +// any individual sample on demand. `play()` finds the nearest rendered +// note, pitch-shifts it, and schedules with an ADSR-ish envelope. + +const _patchCache = new Map(); // programNumber -> Promise +const _drumCache = new Map(); // kitId -> Promise + +function programDir(programNumber) { + return `${ASSET_BASE}/${String(programNumber).padStart(3, "0")}`; +} + +function drumDir(kitId) { + return `${ASSET_BASE}/drum-${String(kitId).padStart(3, "0")}`; +} + +// Build the URL for a single rendered note (e.g. midi 60 -> ".../C4.mp3"). +function noteUrl(dir, midi) { + return `${dir}/${midiToNoteName(midi)}.mp3`; +} + +// Pick the closest rendered MIDI note to a target. +function nearestRenderedNote(noteList, target) { + if (!noteList || noteList.length === 0) return null; + let best = noteList[0]; + let bestDist = Math.abs(best - target); + for (let i = 1; i < noteList.length; i++) { + const d = Math.abs(noteList[i] - target); + if (d < bestDist) { + best = noteList[i]; + bestDist = d; + } + } + return best; +} + +// No-op note handle returned when something fails — keeps callers safe. +function noopNoteHandle(reason) { + return { + stop() {}, + release() {}, + _failed: true, + _reason: reason, + }; +} + +// Schedule a single sample with envelope. Returns a noteHandle. +function scheduleNote(ctx, audioBuffer, opts) { + const { + playbackRate = 1, + velocity = 100, + when = 0, + duration, // optional; if omitted note sustains until release() + attack = 0.005, // 5ms snappy + decay = 0.04, + sustain = 0.85, // fraction of peak + release = 0.15, // 150ms + destination, + } = opts; + + const startTime = when || ctx.currentTime; + const peak = Math.max(0, Math.min(1, velocity / 127)); + + let source, gain; + try { + source = ctx.createBufferSource(); + source.buffer = audioBuffer; + source.playbackRate.value = playbackRate; + + gain = ctx.createGain(); + gain.gain.setValueAtTime(0, startTime); + gain.gain.linearRampToValueAtTime(peak, startTime + attack); + gain.gain.linearRampToValueAtTime( + peak * sustain, + startTime + attack + decay, + ); + + source.connect(gain); + gain.connect(destination || ctx.destination); + + source.start(startTime); + } catch (err) { + console.warn("🎼 GM: scheduleNote failed:", err); + return noopNoteHandle("schedule-failed"); + } + + let stopped = false; + let scheduledStop = null; + + const releaseAt = (t) => { + if (stopped) return; + stopped = true; + const time = Math.max(t, ctx.currentTime); + try { + gain.gain.cancelScheduledValues(time); + // Hold current value, then ramp to 0 over `release` seconds. + const current = gain.gain.value; + gain.gain.setValueAtTime(current, time); + gain.gain.linearRampToValueAtTime(0.0001, time + release); + // Schedule actual stop slightly after the release fade. + scheduledStop = time + release + 0.02; + source.stop(scheduledStop); + } catch (err) { + // Already stopped or invalid state — ignore. + } + }; + + // If a fixed duration was supplied, schedule the release at start+dur. + if (typeof duration === "number" && duration > 0) { + releaseAt(startTime + duration); + } + + return { + stop(when = 0) { + releaseAt(when || ctx.currentTime); + }, + release() { + releaseAt(ctx.currentTime); + }, + get _node() { return source; }, + get _gain() { return gain; }, + }; +} + +// ─── Patch loader ───────────────────────────────────────────────────── +export async function loadPatch(programNumber, audioContext) { + const program = programNumber | 0; + if (program < 0 || program > 127) { + console.warn("🎼 GM: program out of range:", programNumber); + return null; + } + + if (_patchCache.has(program)) return _patchCache.get(program); + + const p = (async () => { + const ctx = resolveAudioContext(audioContext); + if (!ctx) { + console.warn("🎼 GM: no AudioContext available"); + return null; + } + + const manifest = await loadManifest(ctx); + const patchEntry = manifest.patches[program]; + const noteList = + (patchEntry && patchEntry.notes) || manifest.defaultNotes || defaultNoteList(); + const dir = programDir(program); + + // Per-note buffer cache scoped to this patch — the inner URL cache + // dedupes across patches if any happen to share notes. + const samples = new Map(); // midi -> Promise + + function getSample(midi) { + if (samples.has(midi)) return samples.get(midi); + const promise = fetchAndDecode(noteUrl(dir, midi), ctx); + samples.set(midi, promise); + return promise; + } + + return { + program, + name: patchEntry?.name || `program-${program}`, + notes: noteList, + // Warm the cache for a single rendered note (or all of them). + async prefetch(midi) { + if (typeof midi === "number") return getSample(midi); + return Promise.all(noteList.map((m) => getSample(m))); + }, + play(midiNote, opts = {}) { + const target = midiNote | 0; + const sampleNote = nearestRenderedNote(noteList, target); + if (sampleNote == null) return noopNoteHandle("no-samples"); + const playbackRate = Math.pow(2, (target - sampleNote) / 12); + + let live = noopNoteHandle("pending"); + let externalReleased = false; + + getSample(sampleNote).then((buf) => { + if (!buf) { + live = noopNoteHandle("decode-failed"); + return; + } + if (externalReleased) { + // Caller already released before the buffer arrived; skip. + return; + } + live = scheduleNote(ctx, buf, { + ...opts, + playbackRate, + }); + }); + + return { + stop(when = 0) { + externalReleased = true; + live?.stop?.(when); + }, + release() { + externalReleased = true; + live?.release?.(); + }, + }; + }, + }; + })(); + + _patchCache.set(program, p); + return p; +} + +// Warm a patch's samples without playing anything. +export async function prefetchPatch(programNumber, audioContext) { + const patch = await loadPatch(programNumber, audioContext); + if (!patch) return null; + await patch.prefetch(); + return patch; +} + +// ─── Drum kit loader ────────────────────────────────────────────────── +// GM percussion uses MIDI channel 10 with one-shot samples per MIDI +// note number (35..81 in the standard kit). Drum hits don't pitch-shift +// — we play whichever exact note was requested if rendered, otherwise +// fall back to the nearest rendered note WITHOUT changing playbackRate +// (changing rate would distort the timbre). A future option flag could +// re-enable pitched drums. +export async function loadDrumKit(kitId = 0, audioContext) { + const id = kitId | 0; + if (_drumCache.has(id)) return _drumCache.get(id); + + const p = (async () => { + const ctx = resolveAudioContext(audioContext); + if (!ctx) return null; + const manifest = await loadManifest(ctx); + const kitEntry = manifest.drumKits[id] || manifest.drumKits[0]; + const noteList = (kitEntry && kitEntry.notes) || manifest.defaultNotes || defaultNoteList(); + const dir = drumDir(id); + const samples = new Map(); + + function getSample(midi) { + if (samples.has(midi)) return samples.get(midi); + const promise = fetchAndDecode(noteUrl(dir, midi), ctx); + samples.set(midi, promise); + return promise; + } + + return { + kitId: id, + name: kitEntry?.name || `drum-${id}`, + notes: noteList, + async prefetch(midi) { + if (typeof midi === "number") return getSample(midi); + return Promise.all(noteList.map((m) => getSample(m))); + }, + play(midiNote, opts = {}) { + const target = midiNote | 0; + const sampleNote = nearestRenderedNote(noteList, target); + if (sampleNote == null) return noopNoteHandle("no-samples"); + + let live = noopNoteHandle("pending"); + let externalReleased = false; + + getSample(sampleNote).then((buf) => { + if (!buf) { live = noopNoteHandle("decode-failed"); return; } + if (externalReleased) return; + // Drums: no pitch shift, very short release, let the sample's + // own decay carry the tail. + live = scheduleNote(ctx, buf, { + ...opts, + playbackRate: 1, + attack: 0.001, + decay: 0.01, + sustain: 1.0, + release: 0.03, + }); + }); + + return { + stop(when = 0) { externalReleased = true; live?.stop?.(when); }, + release() { externalReleased = true; live?.release?.(); }, + }; + }, + }; + })(); + + _drumCache.set(id, p); + return p; +} + +// ─── Cache controls (mostly for tests / hot reload) ─────────────────── +export function _resetGMCaches() { + _bufferCache.clear(); + _patchCache.clear(); + _drumCache.clear(); + _manifest = null; + _manifestPromise = null; +} + +// ────────────────────────────────────────────────────────────────────── +// Self-test snippet — paste into a piece's `boot` to sanity-check. +// (Requires the bake script to have populated assets.aesthetic.computer.) +// +// import { loadPatch, loadDrumKit, prefetchPatch } from "/aesthetic.computer/lib/gm.mjs"; +// +// async function boot({ sound }) { +// // 1. Acoustic Grand Piano (GM program 0): +// const piano = await loadPatch(0); +// const note = piano.play(60, { velocity: 110 }); // middle C +// setTimeout(() => note.release(), 800); +// +// // 2. Pitch-shifted note that wasn't directly rendered: +// piano.play(61, { velocity: 90, duration: 0.5 }); // C# via nearest sample +// +// // 3. Standard drum kit — kick (MIDI 36) and snare (MIDI 38): +// const kit = await loadDrumKit(0); +// kit.play(36); setTimeout(() => kit.play(38), 250); +// +// // 4. Warm a patch ahead of time: +// prefetchPatch(24); // nylon guitar +// } diff --git a/system/public/aesthetic.computer/lib/midi.mjs b/system/public/aesthetic.computer/lib/midi.mjs index 20fc2fa841..53efd7185d 100644 --- a/system/public/aesthetic.computer/lib/midi.mjs +++ b/system/public/aesthetic.computer/lib/midi.mjs @@ -4,13 +4,19 @@ const activeInputs = new Map(); const NOTE_ON = 0x90; const NOTE_OFF = 0x80; const PITCH_BEND = 0xe0; +const PROGRAM_CHANGE = 0xc0; function handleMidiMessage(message) { const [status, note, velocity] = message.data || []; if (status === undefined) return; const command = status & 0xf0; - if (command !== NOTE_ON && command !== NOTE_OFF && command !== PITCH_BEND) { + if ( + command !== NOTE_ON && + command !== NOTE_OFF && + command !== PITCH_BEND && + command !== PROGRAM_CHANGE + ) { return; } -- 2.51.2