diff --git a/ac-m4l/README.md b/ac-m4l/README.md index 72989e5f56..ba495c4ed7 100644 --- a/ac-m4l/README.md +++ b/ac-m4l/README.md @@ -42,6 +42,24 @@ Key settings in the JSON source: } ``` +### Remote Relay Playback + +`notepat` now understands relay subscription parameters for live note playback from `ac-native` sources: + +```text +https://localhost:8888/notepat?daw=1&relayHandle=jeffrey +https://localhost:8888/notepat?daw=1&relayHandle=jeffrey&relayMachine=ac-1234abcd +https://localhost:8888/notepat?daw=1&relayAll=1 +``` + +You can also retarget a running `jweb~` instance with `postMessage`: + +```js +window.postMessage({ type: "notepat:midi:set-source", handle: "jeffrey" }, "*"); +window.postMessage({ type: "notepat:midi:set-source", handle: "jeffrey", machineId: "ac-1234abcd" }, "*"); +window.postMessage({ type: "notepat:midi:sources" }, "*"); +``` + ### Requirements - Ableton Live with Max for Live diff --git a/fedac/native/pieces/list.mjs b/fedac/native/pieces/list.mjs index 42aef71649..2090d98a27 100644 --- a/fedac/native/pieces/list.mjs +++ b/fedac/native/pieces/list.mjs @@ -29,7 +29,7 @@ const COMMANDS = [ { name: "bye", desc: "log out" }, { name: "version", desc: "OS hash" }, { name: "ssh", desc: "SSH server" }, - { name: "midi", desc: "usb midi gadget" }, + { name: "midi", desc: "usb midi + udp relay" }, { name: "clear", desc: "clear history" }, { name: "help", desc: "quick help" }, ]; diff --git a/fedac/native/pieces/notepat.mjs b/fedac/native/pieces/notepat.mjs index 999679d715..a6958395c9 100644 --- a/fedac/native/pieces/notepat.mjs +++ b/fedac/native/pieces/notepat.mjs @@ -110,6 +110,8 @@ let lastTabletMode = null; let usbMidiRecent = []; let usbMidiNextRefreshFrame = 0; let usbMidiTypecSignature = ""; +let udpMidiBroadcast = false; +let udpMidiNextHeartbeatFrame = 0; // OS update panel state machine @@ -286,12 +288,46 @@ function usbMidiStatusText(status) { return "USB MIDI OFF"; } +function loadUdpMidiConfig(system) { + try { + const raw = system?.readFile?.("/mnt/config.json"); + if (!raw) { + udpMidiBroadcast = false; + return; + } + const cfg = JSON.parse(raw); + udpMidiBroadcast = cfg.udpMidiBroadcast === true || cfg.udpMidiBroadcast === "true"; + } catch (_) { + udpMidiBroadcast = false; + } +} + +function sendUdpMidiEvent(system, event, midiNote, velocity, channel = 0) { + if (!udpMidiBroadcast || !system?.udp?.connected) return; + system?.udp?.sendMidi?.(event, midiNote, velocity, channel, "notepat"); +} + +function maybeSendUdpMidiHeartbeat(system) { + if (!udpMidiBroadcast || !system?.udp?.connected) return; + if (frame < udpMidiNextHeartbeatFrame) return; + system?.udp?.sendMidiHeartbeat?.("notepat"); + udpMidiNextHeartbeatFrame = frame + 300; +} + +function udpMidiRelayStatusText(system) { + if (!udpMidiBroadcast) return ""; + const handle = system?.udp?.handle || system?.config?.handle || ""; + if (system?.udp?.connected) return handle ? "relay @" + handle : "relay on"; + return handle ? "relay ...@" + handle : "relay ..."; +} + function rememberSound(key, entry, system, velocity = 1) { if (!entry) return; entry.midiNote = noteToMidiNumber(entry.note, entry.octave); entry.midiChannel = 0; sounds[key] = entry; system?.usbMidi?.noteOn?.(entry.midiNote, velocityToMidi(velocity), entry.midiChannel); + sendUdpMidiEvent(system, "note_on", entry.midiNote, velocityToMidi(velocity), entry.midiChannel); pushUsbMidiRecent(">", entry.note, entry.octave); } @@ -305,6 +341,7 @@ function stopSoundKey(key, sound, system, fade = 0.08) { } if (entry.midiNote !== undefined) { system?.usbMidi?.noteOff?.(entry.midiNote, 0, entry.midiChannel || 0); + sendUdpMidiEvent(system, "note_off", entry.midiNote, 0, entry.midiChannel || 0); pushUsbMidiRecent("<", entry.note, entry.octave); } delete sounds[key]; @@ -441,6 +478,8 @@ function boot({ wipe, system, sound }) { wipe(0); soundAPI = sound; systemAPI = system; + loadUdpMidiConfig(system); + udpMidiNextHeartbeatFrame = 0; const mic = sound?.microphone || null; if (mic && (mic.sampleLength || 0) > 0) { sampleLoaded = true; @@ -1266,6 +1305,17 @@ function paint({ wipe, ink, box, line, write, screen, sound, system, trackpad, p statusWrite(usbMidiText, FG_DIM, FG_DIM, FG_DIM, 200); } + const relayText = udpMidiRelayStatusText(system); + if (relayText) { + statusWrite( + relayText, + system?.udp?.connected ? 80 : 255, + system?.udp?.connected ? 180 : 180, + system?.udp?.connected ? 255 : 90, + 210 + ); + } + // Metronome indicator (pendulum) in status bar β€” shown when enabled if (metronomeEnabled) { const bpmLabel = metronomeBPM + "b"; @@ -1437,6 +1487,7 @@ function paint({ wipe, ink, box, line, write, screen, sound, system, trackpad, p system.ws?.connect("wss://chat-system.aesthetic.computer/"); // Connect raw UDP for fairy co-presence system.udp?.connect("session-server.aesthetic.computer", 10010); + udpMidiNextHeartbeatFrame = frame + 30; if (system?.writeFile && savedCreds.length > 0) { system.writeFile(CREDS_PATH, JSON.stringify(savedCreds)); } @@ -2508,6 +2559,7 @@ function paint({ wipe, ink, box, line, write, screen, sound, system, trackpad, p // 🧚 Fairy co-presence β€” send cursor + paint received fireflies if (system.udp?.connected) { + maybeSendUdpMidiHeartbeat(system); // Send current cursor/touch position as fairy point (~60Hz, throttled by UDP thread) if (hoverX >= 0 && hoverY >= 0) { system.udp.sendFairy(hoverX / w, hoverY / h); diff --git a/fedac/native/pieces/prompt.mjs b/fedac/native/pieces/prompt.mjs index da569b2795..343b7bd86e 100644 --- a/fedac/native/pieces/prompt.mjs +++ b/fedac/native/pieces/prompt.mjs @@ -56,7 +56,7 @@ const PIECE_DESC = { "theme": "prompt theme", "voice": "system voice", "login": "switch identity", - "midi": "usb midi gadget", + "midi": "usb midi + udp relay", "dark": "dark mode", "light": "light mode", "auto": "auto dark/light", @@ -140,6 +140,28 @@ function formatUsbMidiStatus(status) { return "usb midi off"; } +function readConfig(system) { + try { + const raw = system?.readFile?.("/mnt/config.json"); + return raw ? JSON.parse(raw) : {}; + } catch (_) { + return {}; + } +} + +function udpMidiRelayEnabled(system) { + const cfg = readConfig(system); + return cfg.udpMidiBroadcast === true || cfg.udpMidiBroadcast === "true"; +} + +function formatUdpMidiRelayStatus(system) { + const cfg = readConfig(system); + const enabled = cfg.udpMidiBroadcast === true || cfg.udpMidiBroadcast === "true"; + if (!enabled) return "udp midi relay off"; + if (cfg.handle) return "udp midi relay on @" + cfg.handle; + return "udp midi relay on"; +} + function boot({ system }) { message = ""; // Load saved theme from config (persists across reboots) @@ -331,8 +353,31 @@ function execute(cmd, system) { } if (baseWord === "midi") { const arg = spaceIdx >= 0 ? lower.slice(spaceIdx + 1).trim() : "status"; + if (arg === "relay" || arg.startsWith("relay ")) { + const relayArg = arg === "relay" ? "status" : arg.slice("relay ".length).trim(); + if (!relayArg || relayArg === "status") { + message = formatUdpMidiRelayStatus(system); + messageFrame = 0; + return; + } + if (relayArg === "on" || relayArg === "enable") { + system?.saveConfig?.("udpMidiBroadcast", "true"); + message = formatUdpMidiRelayStatus(system); + messageFrame = 0; + return; + } + if (relayArg === "off" || relayArg === "disable") { + system?.saveConfig?.("udpMidiBroadcast", "false"); + message = formatUdpMidiRelayStatus(system); + messageFrame = 0; + return; + } + message = "usage: midi relay [status|on|off]"; + messageFrame = 0; + return; + } if (!arg || arg === "status") { - message = formatUsbMidiStatus(readUsbMidiStatus(system)); + message = formatUsbMidiStatus(readUsbMidiStatus(system)) + " | " + formatUdpMidiRelayStatus(system); messageFrame = 0; return; } @@ -356,7 +401,7 @@ function execute(cmd, system) { messageFrame = 0; return; } - message = "usage: midi [status|on|off|refresh]"; + message = "usage: midi [status|on|off|refresh|relay]"; messageFrame = 0; return; } diff --git a/fedac/native/src/js-bindings.c b/fedac/native/src/js-bindings.c index d7cfccb050..cc91be7e94 100644 --- a/fedac/native/src/js-bindings.c +++ b/fedac/native/src/js-bindings.c @@ -2567,6 +2567,17 @@ static JSValue js_camera_blit(JSContext *ctx, JSValueConst this_val, int argc, J // system.udp β€” Raw UDP fairy point co-presence // --------------------------------------------------------------------------- +static void sync_udp_identity(void) { + extern char g_machine_id[64]; + + if (!current_rt || !current_rt->udp) return; + udp_set_identity( + current_rt->udp, + current_rt->handle[0] ? current_rt->handle : "", + g_machine_id[0] ? g_machine_id : "unknown" + ); +} + static JSValue js_udp_connect(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { (void)this_val; if (!current_rt || !current_rt->udp) return JS_UNDEFINED; @@ -2574,13 +2585,7 @@ static JSValue js_udp_connect(JSContext *ctx, JSValueConst this_val, int argc, J if (!host) host = "session-server.aesthetic.computer"; int port = UDP_FAIRY_PORT; if (argc > 1) JS_ToInt32(ctx, &port, argv[1]); - // Set handle for identity - if (current_rt->handle[0]) { - pthread_mutex_lock(¤t_rt->udp->mu); - strncpy(current_rt->udp->handle, current_rt->handle, 63); - current_rt->udp->handle[63] = 0; - pthread_mutex_unlock(¤t_rt->udp->mu); - } + sync_udp_identity(); udp_connect(current_rt->udp, host, port); if (argc > 0) JS_FreeCString(ctx, host); return JS_UNDEFINED; @@ -2596,14 +2601,64 @@ static JSValue js_udp_send_fairy(JSContext *ctx, JSValueConst this_val, int argc return JS_UNDEFINED; } +static JSValue js_udp_send_midi(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { + (void)this_val; + if (!current_rt || !current_rt->udp || argc < 4) return JS_UNDEFINED; + + const char *event = JS_ToCString(ctx, argv[0]); + int32_t note = 0; + int32_t velocity = 0; + int32_t channel = 0; + const char *piece = argc > 4 ? JS_ToCString(ctx, argv[4]) : NULL; + + JS_ToInt32(ctx, ¬e, argv[1]); + JS_ToInt32(ctx, &velocity, argv[2]); + JS_ToInt32(ctx, &channel, argv[3]); + + if (!event) { + if (piece) JS_FreeCString(ctx, piece); + return JS_UNDEFINED; + } + + sync_udp_identity(); + udp_send_midi( + current_rt->udp, + event, + (int)note, + (int)velocity, + (int)channel, + piece ? piece : "notepat" + ); + + JS_FreeCString(ctx, event); + if (piece) JS_FreeCString(ctx, piece); + return JS_UNDEFINED; +} + +static JSValue js_udp_send_midi_heartbeat(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { + (void)this_val; + if (!current_rt || !current_rt->udp) return JS_UNDEFINED; + + const char *piece = argc > 0 ? JS_ToCString(ctx, argv[0]) : NULL; + sync_udp_identity(); + udp_send_midi_heartbeat(current_rt->udp, piece ? piece : "notepat"); + if (piece) JS_FreeCString(ctx, piece); + return JS_UNDEFINED; +} + static JSValue build_udp_obj(JSContext *ctx, const char *phase) { JSValue obj = JS_NewObject(ctx); JS_SetPropertyStr(ctx, obj, "connect", JS_NewCFunction(ctx, js_udp_connect, "connect", 2)); JS_SetPropertyStr(ctx, obj, "sendFairy", JS_NewCFunction(ctx, js_udp_send_fairy, "sendFairy", 2)); + JS_SetPropertyStr(ctx, obj, "sendMidi", JS_NewCFunction(ctx, js_udp_send_midi, "sendMidi", 5)); + JS_SetPropertyStr(ctx, obj, "sendMidiHeartbeat", JS_NewCFunction(ctx, js_udp_send_midi_heartbeat, "sendMidiHeartbeat", 1)); ACUdp *udp = current_rt ? current_rt->udp : NULL; if (udp) { + sync_udp_identity(); JS_SetPropertyStr(ctx, obj, "connected", JS_NewBool(ctx, udp->connected)); + JS_SetPropertyStr(ctx, obj, "handle", + JS_NewString(ctx, current_rt && current_rt->handle[0] ? current_rt->handle : "")); // Only deliver fairies during paint phase if (strcmp(phase, "paint") == 0) { diff --git a/fedac/native/src/udp-client.c b/fedac/native/src/udp-client.c index 19e4ba17f2..efc257068e 100644 --- a/fedac/native/src/udp-client.c +++ b/fedac/native/src/udp-client.c @@ -1,5 +1,6 @@ #include "udp-client.h" +#include #include #include #include @@ -10,6 +11,7 @@ #include #include #include +#include extern void ac_log(const char *fmt, ...); @@ -23,6 +25,85 @@ extern void ac_log(const char *fmt, ...); #define PKT_FAIRY_SEND 0x01 #define PKT_FAIRY_RECV 0x02 +static uint64_t udp_now_ms(void) { + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + return (uint64_t)ts.tv_sec * 1000ULL + (uint64_t)ts.tv_nsec / 1000000ULL; +} + +static void json_escape_copy(char *dst, size_t dst_size, const char *src) { + size_t out = 0; + + if (!dst || dst_size == 0) return; + if (!src) src = ""; + + while (*src && out + 1 < dst_size) { + unsigned char c = (unsigned char)*src++; + if (c == '"' || c == '\\') { + if (out + 2 >= dst_size) break; + dst[out++] = '\\'; + dst[out++] = (char)c; + } else if (c >= 32 && c < 127) { + dst[out++] = (char)c; + } + } + + dst[out] = '\0'; +} + +static int udp_snapshot_identity( + ACUdp *udp, + int *sock_out, + struct sockaddr_in *addr_out, + char *handle_out, + size_t handle_out_size, + char *machine_out, + size_t machine_out_size +) { + int sock = -1; + int connected = 0; + + if (!udp) return 0; + + pthread_mutex_lock(&udp->mu); + sock = udp->sock; + connected = udp->connected; + if (addr_out) *addr_out = udp->server_addr; + if (handle_out && handle_out_size > 0) { + strncpy(handle_out, udp->handle, handle_out_size - 1); + handle_out[handle_out_size - 1] = 0; + } + if (machine_out && machine_out_size > 0) { + strncpy(machine_out, udp->machine_id, machine_out_size - 1); + machine_out[machine_out_size - 1] = 0; + } + pthread_mutex_unlock(&udp->mu); + + if (sock_out) *sock_out = sock; + return connected && sock >= 0; +} + +static void udp_send_json(ACUdp *udp, const char *json) { + int sock = -1; + struct sockaddr_in server_addr; + + if (!udp || !json || !json[0]) return; + if (!udp_snapshot_identity(udp, &sock, &server_addr, NULL, 0, NULL, 0)) return; + + ssize_t sent = sendto( + sock, + json, + strlen(json), + 0, + (struct sockaddr *)&server_addr, + sizeof(server_addr) + ); + + if (sent < 0 && errno != EAGAIN && errno != EWOULDBLOCK) { + ac_log("[udp] json send failed: %s\n", strerror(errno)); + } +} + static void *udp_thread(void *arg) { ACUdp *udp = (ACUdp *)arg; unsigned char buf[256]; @@ -141,6 +222,84 @@ void udp_send_fairy(ACUdp *udp, float x, float y) { pthread_mutex_unlock(&udp->mu); } +void udp_set_identity(ACUdp *udp, const char *handle, const char *machine_id) { + if (!udp) return; + pthread_mutex_lock(&udp->mu); + if (handle) { + strncpy(udp->handle, handle, sizeof(udp->handle) - 1); + udp->handle[sizeof(udp->handle) - 1] = 0; + } + if (machine_id) { + strncpy(udp->machine_id, machine_id, sizeof(udp->machine_id) - 1); + udp->machine_id[sizeof(udp->machine_id) - 1] = 0; + } + pthread_mutex_unlock(&udp->mu); +} + +void udp_send_midi(ACUdp *udp, const char *event, int note, int velocity, int channel, const char *piece) { + char handle[64] = ""; + char machine_id[64] = ""; + char handle_json[128]; + char machine_json[128]; + char piece_json[64]; + char event_json[32]; + char json[512]; + + if (!udp || !event || !piece) return; + if (!udp_snapshot_identity(udp, NULL, NULL, handle, sizeof(handle), machine_id, sizeof(machine_id))) return; + + json_escape_copy(handle_json, sizeof(handle_json), handle); + json_escape_copy(machine_json, sizeof(machine_json), machine_id[0] ? machine_id : "unknown"); + json_escape_copy(piece_json, sizeof(piece_json), piece); + json_escape_copy(event_json, sizeof(event_json), event); + + snprintf( + json, + sizeof(json), + "{\"type\":\"notepat:midi\",\"event\":\"%s\",\"note\":%d,\"velocity\":%d," + "\"channel\":%d,\"handle\":\"%s\",\"machineId\":\"%s\",\"piece\":\"%s\",\"ts\":%llu}", + event_json, + note, + velocity, + channel, + handle_json, + machine_json, + piece_json, + (unsigned long long)udp_now_ms() + ); + + udp_send_json(udp, json); +} + +void udp_send_midi_heartbeat(ACUdp *udp, const char *piece) { + char handle[64] = ""; + char machine_id[64] = ""; + char handle_json[128]; + char machine_json[128]; + char piece_json[64]; + char json[512]; + + if (!udp || !piece) return; + if (!udp_snapshot_identity(udp, NULL, NULL, handle, sizeof(handle), machine_id, sizeof(machine_id))) return; + + json_escape_copy(handle_json, sizeof(handle_json), handle); + json_escape_copy(machine_json, sizeof(machine_json), machine_id[0] ? machine_id : "unknown"); + json_escape_copy(piece_json, sizeof(piece_json), piece); + + snprintf( + json, + sizeof(json), + "{\"type\":\"notepat:midi:heartbeat\",\"handle\":\"%s\",\"machineId\":\"%s\"," + "\"piece\":\"%s\",\"broadcast\":true,\"ts\":%llu}", + handle_json, + machine_json, + piece_json, + (unsigned long long)udp_now_ms() + ); + + udp_send_json(udp, json); +} + int udp_poll_fairies(ACUdp *udp, UDPFairy *out, int max) { if (!udp) return 0; pthread_mutex_lock(&udp->mu); diff --git a/fedac/native/src/udp-client.h b/fedac/native/src/udp-client.h index 9b4c05937c..8d9eb157be 100644 --- a/fedac/native/src/udp-client.h +++ b/fedac/native/src/udp-client.h @@ -34,6 +34,7 @@ typedef struct { // Identity char handle[64]; + char machine_id[64]; pthread_mutex_t mu; } ACUdp; @@ -47,6 +48,13 @@ void udp_connect(ACUdp *udp, const char *host, int port); // Queue a fairy point to send (non-blocking, main thread) void udp_send_fairy(ACUdp *udp, float x, float y); +// Update identifying metadata used by outgoing packets. +void udp_set_identity(ACUdp *udp, const char *handle, const char *machine_id); + +// Send note relay events to the session server over UDP. +void udp_send_midi(ACUdp *udp, const char *event, int note, int velocity, int channel, const char *piece); +void udp_send_midi_heartbeat(ACUdp *udp, const char *piece); + // Poll received fairies (returns count, fills out[] up to max) // Clears the buffer after reading. int udp_poll_fairies(ACUdp *udp, UDPFairy *out, int max); diff --git a/plans/notepat-udp-midi-relay.md b/plans/notepat-udp-midi-relay.md new file mode 100644 index 0000000000..629cd2d7af --- /dev/null +++ b/plans/notepat-udp-midi-relay.md @@ -0,0 +1,512 @@ +# Notepat UDP MIDI Relay Plan + +## Overview + +Let running `ac-native` `notepat` sessions automatically publish note events to the Aesthetic Computer network, so a Max for Live device or `notepat.com`-based device can subscribe to those events and play them back live. + +The important twist is identity: + +- every note stream should be attributable to a `@handle` +- we should also preserve `machineId` so multiple devices owned by the same person can be distinguished +- subscribers should be able to filter by `@handle`, by `machineId`, or by "everyone" + +This would make an `ac-native` machine feel like a network instrument source, while Ableton / Max acts like a remote listener, mirror, recorder, or re-player. + +## Why This Shape + +We already have the right footholds: + +- `fedac/native/src/udp-client.c` already sends compact UDP packets from `ac-native` +- `fedac/native/src/js-bindings.c` already injects the current `handle` into the UDP client +- `system.machineId` already exists in native +- `notepat.mjs` already has a clean note-on / note-off seam for outgoing MIDI events +- `ac-m4l/AC Notepat.amxd` and related M4L assets already exist + +The part that does not exist yet is a note-event relay path and a subscription model on the receiver side. + +## Recommended Architecture + +Use a hybrid model: + +1. `ac-native` sends note events to a UDP relay endpoint for low-latency ingest. +2. The relay normalizes those events and republishes them to WebSocket subscribers. +3. Max for Live / `notepat.com` devices subscribe over WebSocket by `@handle` or `machineId`. + +This is better than raw UDP end-to-end because: + +- browser-based receivers do not want to deal with raw UDP directly +- WebSocket subscription/filtering is much easier to debug in M4L and web UIs +- the relay can enrich events with metadata, timestamps, and presence +- we can keep the native sender lightweight + +## Architecture + +```text +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ ac-native / notepat β”‚ +β”‚ on Yoga / AC machine β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”‚ UDP note events + β”‚ handle + machineId + midi + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ session-server MIDI relay β”‚ +β”‚ - UDP ingest β”‚ +β”‚ - presence / last-seen map β”‚ +β”‚ - WS fanout β”‚ +β”‚ - handle + machine filters β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”‚ WebSocket subscribe: + β”‚ notepat:midi:subscribe + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Max for Live / AC Notepat β”‚ +β”‚ or notepat.com device β”‚ +β”‚ - choose source handle β”‚ +β”‚ - optional machine filter β”‚ +β”‚ - playback / monitor UI β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +## Current Repo Footholds + +### Native sender + +- `fedac/native/pieces/notepat.mjs` + - already computes note-on / note-off centrally + - already knows `system.usbMidi`, `system.udp`, and `system.machineId` +- `fedac/native/src/js-bindings.c` + - already exposes `system.udp` + - already exposes `system.machineId` + - already sets UDP identity from the config `handle` +- `fedac/native/src/udp-client.c` + - currently supports fairy packets only + - already has background send / recv thread + +### Receiver side + +- `ac-m4l/AC Notepat.amxd` +- `ac-m4l/AC-Notepat.maxpat` +- `ac-m4l/ac-ws-server.js` +- `plans/kidlisp-m4l-integration.md` + +These give us a plausible place for a handle-selectable network note receiver. + +## Event Model + +### Minimum event payload + +Each note event should carry: + +- `type`: `note_on` or `note_off` +- `note`: MIDI note number +- `velocity`: 0-127 +- `channel`: MIDI channel +- `handle`: performer handle without `@`, if known +- `machineId`: native machine identity +- `piece`: usually `notepat` +- `ts`: sender timestamp in ms + +### Suggested normalized relay JSON + +```json +{ + "type": "notepat:midi", + "event": "note_on", + "note": 60, + "velocity": 109, + "channel": 0, + "handle": "jeffrey", + "machineId": "ac-1234abcd", + "piece": "notepat", + "ts": 1710000000000 +} +``` + +### Optional later fields + +- `source`: `ac-native` +- `wave`: current notepat wave mode +- `usbMidi`: whether it was also forwarded to a local USB MIDI host +- `session`: optional room / channel +- `pressure`: for aftertouch-like experiments + +## UDP Protocol Plan + +### Phase 1 + +Add new packet types to `fedac/native/src/udp-client.c`: + +- `0x10` = note on +- `0x11` = note off +- `0x12` = heartbeat / source status + +Suggested binary shape: + +```text +[1 type] +[1 note] +[1 velocity] +[1 channel] +[1 handle_len][N handle_bytes] +[1 machine_len][N machine_bytes] +[4 timestamp_ms_u32] +``` + +This keeps the packet compact while still carrying identity. + +### Alternative + +If we want to move faster and accept a slightly fatter payload, we could send short JSON strings over UDP first, then tighten to binary later. That would reduce implementation friction in the relay. + +Recommendation: + +- binary if we want to keep it native-first and stable long-term +- JSON if we want the fastest prototype + +My recommendation is: + +- prototype with JSON-over-UDP +- switch to binary once the event schema settles + +## Relay / Server Plan + +The relay should do four jobs: + +1. accept note events from native senders +2. maintain a live source index +3. expose source discovery to listeners +4. fan events out to subscribers + +### Source index + +Maintain a map keyed by `handle + machineId`: + +```js +{ + "jeffrey:ac-1234abcd": { + handle: "jeffrey", + machineId: "ac-1234abcd", + lastSeen: 1710000000000, + lastEvent: "note_on", + piece: "notepat" + } +} +``` + +### WebSocket API + +Add a small family of messages: + +```json +{ "type": "notepat:midi:sources" } +``` + +Response: + +```json +{ + "type": "notepat:midi:sources", + "sources": [ + { "handle": "jeffrey", "machineId": "ac-1234abcd", "piece": "notepat", "lastSeen": 1710000000000 } + ] +} +``` + +Subscribe: + +```json +{ "type": "notepat:midi:subscribe", "handle": "jeffrey" } +``` + +Or: + +```json +{ "type": "notepat:midi:subscribe", "handle": "jeffrey", "machineId": "ac-1234abcd" } +``` + +Or: + +```json +{ "type": "notepat:midi:subscribe", "all": true } +``` + +Broadcast event: + +```json +{ + "type": "notepat:midi", + "event": "note_on", + "handle": "jeffrey", + "machineId": "ac-1234abcd", + "note": 60, + "velocity": 96, + "channel": 0, + "ts": 1710000000000 +} +``` + +## Native Sender Changes + +### `notepat.mjs` + +Extend the current note send helpers so note events can also go to the network: + +- note on -> UDP relay send +- note off -> UDP relay send +- all notes off -> optional cleanup event + +Suggested behavior: + +- default off until user opts in +- per-boot toggle in prompt or `notepat` +- persist in `/mnt/config.json` + +Suggested config key: + +```json +{ "udpMidiBroadcast": true } +``` + +### `js-bindings.c` + +Expose one small higher-level helper on `system.udp`, for example: + +- `system.udp.sendMidi(event, note, velocity, channel, machineId, piece)` + +That keeps the piece code clean and avoids reimplementing packet shape in JS. + +### `udp-client.c` + +Add: + +- note packet encoding +- light send queue / coalescing +- heartbeat support + +The heartbeat should announce: + +- `handle` +- `machineId` +- `piece=notepat` +- `broadcast=true` + +every few seconds while the piece is active. + +## Max for Live / notepat.com Receiver Plan + +### UX + +The receiver device should have: + +- source mode: `all`, `handle`, or `handle + machine` +- handle selector +- machine selector +- monitor list showing live note events +- armed / muted toggle +- optional MIDI passthrough to Live track + +### Playback behavior + +For the first version: + +- incoming `note_on` -> emit MIDI note on in Max +- incoming `note_off` -> emit MIDI note off in Max + +Later: + +- optional quantization +- optional octave remap +- optional channel remap +- optional monitor-only mode + +### Good first target + +Build this into `AC Notepat.amxd` or a sibling device such as: + +- `AC Notepat Relay.amxd` + +That keeps the "instrument" device separate from the "remote listener" device until the UI settles. + +## Filtering By Handle + +This is the core product behavior. + +### Rules + +- if `handle` is selected, accept events from all machines owned by that handle +- if `handle + machineId` is selected, accept only that machine +- if `all` is selected, accept all public broadcasts + +### Why both handle and machineId matter + +`@handle` alone is not enough if: + +- someone has two AC machines running at once +- one machine is a practice keyboard and one is a stage performer +- we want to mirror a specific machine into a specific Ableton set + +So `handle` should be the human-facing grouping key, while `machineId` is the precise routing key. + +## Presence and Discovery + +The receiver should be able to see active sources without typing blindly. + +Recommended behavior: + +- native sender heartbeats every 5-10 seconds +- relay expires sources after 15-20 seconds of silence +- M4L receiver requests source list on load and periodically refreshes + +Displayed label: + +```text +@jeffrey ac-1234abcd notepat seen 2s ago +``` + +## Logging and Observability + +We should keep a paper trail because this will be hard to debug live. + +### Native + +Log to `/mnt/ac-native.log`: + +- UDP relay connected +- UDP note-on / note-off send errors +- broadcast enabled / disabled + +### Relay + +Log: + +- source registration +- malformed packets +- subscriber counts +- fanout counts by handle + +### M4L / receiver + +Expose a small event monitor: + +- last event time +- last source handle +- last note +- dropped note count if any + +## Privacy / Safety + +Because note streams can become ambient network presence, this should be opt-in. + +### Recommendation + +- sender broadcasting is off by default +- enabling it is explicit in prompt / config +- receiver subscribes explicitly; nothing auto-plays by surprise + +Optional later: + +- `public` vs `friends` vs `private` +- signed session tokens +- authenticated per-handle subscriptions + +## Phased Implementation + +### Phase 1: Native -> relay prototype + +- [ ] Add `udpMidiBroadcast` config flag +- [ ] Add note send hook in `fedac/native/pieces/notepat.mjs` +- [ ] Add UDP note packet support in `fedac/native/src/udp-client.c` +- [ ] Add relay listener endpoint +- [ ] Log received events server-side + +Success condition: + +- notes from one `ac-native` machine appear in relay logs with `handle` and `machineId` + +### Phase 2: Relay -> WebSocket subscribers + +- [ ] Add `notepat:midi:sources` +- [ ] Add `notepat:midi:subscribe` +- [ ] Add source expiry / heartbeat tracking +- [ ] Broadcast normalized note events over WebSocket + +Success condition: + +- a browser console can subscribe to `@handle` and print incoming notes + +### Phase 3: Max for Live receiver + +- [ ] Create or fork `AC Notepat` receiver device +- [ ] Subscribe by `handle` +- [ ] Emit Live MIDI notes +- [ ] Show source / last-note monitor + +Success condition: + +- notes played on `ac-native` trigger a Live instrument from the selected handle + +### Phase 4: Refinement + +- [ ] machine-level filtering +- [ ] monitor-only mode +- [ ] quantize / transform options +- [ ] optional recording into Live clips +- [ ] note-pressure / CC extension + +## Open Questions + +### 1. Should the relay live inside the existing session server? + +Recommendation: + +- yes for the first version + +Reason: + +- it already handles realtime identity-oriented traffic +- it already has WebSocket clients +- it is the easiest place to hang `handle`-filtered subscriptions + +### 2. Should the receiver use raw UDP too? + +Recommendation: + +- no for the first version + +Reason: + +- browser / M4L environments are easier over WebSocket +- UDP ingest + WS fanout already gets most of the benefit + +### 3. Should we mirror every note or only notes that go to USB MIDI? + +Recommendation: + +- every note played in `notepat`, independent of USB MIDI state + +Reason: + +- network relay and local USB MIDI solve different problems +- it is more musically useful if network relay does not depend on a cable + +### 4. Should this be public performance presence? + +Recommendation: + +- start opt-in and semi-private +- add public browsing later if it feels magical + +## First Concrete Build Slice + +If we want the smallest useful first slice: + +1. add `udpMidiBroadcast` toggle in native prompt +2. send JSON note events from `notepat` over UDP +3. add relay logging + source list in session server +4. build a tiny browser subscriber page that filters by `@handle` +5. only then move into Max for Live playback + +That gets us from idea to audible proof fast, without overcommitting to a packet format or Max UI too early. diff --git a/session-server/session.mjs b/session-server/session.mjs index 2e27f3cd41..2adbb1e56a 100644 --- a/session-server/session.mjs +++ b/session-server/session.mjs @@ -73,9 +73,12 @@ function getModuleHash(modulePath) { const fairyThrottle = new Map(); // channelId -> last publish timestamp const FAIRY_THROTTLE_MS = 100; // 10Hz max per connection -// Raw UDP fairy relay (for native bare-metal clients) -const udpRelay = dgram.createSocket("udp4"); -const udpClients = new Map(); // key "ip:port" β†’ { address, port, handle, lastSeen } +// Raw UDP fairy relay (for native bare-metal clients) +const udpRelay = dgram.createSocket("udp4"); +const udpClients = new Map(); // key "ip:port" β†’ { address, port, handle, lastSeen } +const UDP_MIDI_SOURCE_TTL_MS = 20000; +const notepatMidiSources = new Map(); // key "@handle:machine" -> source metadata +const notepatMidiSubscribers = new Map(); // connection id -> { ws, all, handle, machineId } // Error logging ring buffer (for dashboard display) const errorLog = []; @@ -2325,11 +2328,11 @@ wss.on("connection", async (ws, req) => { return; } - if (msg.type === "daw:code") { - // IDE sending code to all connected devices - log(`🎹 DAW code broadcast from ${id} to ${dawDevices.size} devices`); - const codeMsg = JSON.stringify({ - type: "daw:code", + if (msg.type === "daw:code") { + // IDE sending code to all connected devices + log(`🎹 DAW code broadcast from ${id} to ${dawDevices.size} devices`); + const codeMsg = JSON.stringify({ + type: "daw:code", content: msg.content, from: id }); @@ -2341,11 +2344,30 @@ wss.on("connection", async (ws, req) => { deviceWs.send(codeMsg); log(`🎹 Sent code to device ${deviceId}`); } - } - return; - } - - msg.id = id; // TODO: When sending a server generated message, use a special id. + } + return; + } + + if (msg.type === "notepat:midi:sources") { + sendNotepatMidiSources(ws); + return; + } + + if (msg.type === "notepat:midi:subscribe") { + const filter = msg.content || {}; + addNotepatMidiSubscriber(id, ws, filter); + return; + } + + if (msg.type === "notepat:midi:unsubscribe") { + removeNotepatMidiSubscriber(id); + if (ws.readyState === WebSocket.OPEN) { + ws.send(pack("notepat:midi:unsubscribed", true, "midi-relay")); + } + return; + } + + msg.id = id; // TODO: When sending a server generated message, use a special id. // Extract user identity and handle from ANY message that contains it if (msg.content?.user?.sub) { @@ -2821,12 +2843,13 @@ wss.on("connection", async (ws, req) => { }); // More info: https://stackoverflow.com/a/49791634/8146077 - ws.on("close", () => { - log("πŸšͺ Someone left:", id, "Online:", wss.clients.size, "πŸ«‚"); - const departingHandle = normalizeProfileHandle(clients?.[id]?.handle); - - // Remove from VSCode clients if present - vscodeClients.delete(ws); + ws.on("close", () => { + log("πŸšͺ Someone left:", id, "Online:", wss.clients.size, "πŸ«‚"); + const departingHandle = normalizeProfileHandle(clients?.[id]?.handle); + removeNotepatMidiSubscriber(id); + + // Remove from VSCode clients if present + vscodeClients.delete(ws); // Remove from DAW devices if present if (dawDevices.has(id)) { @@ -3001,12 +3024,141 @@ const profileLastSeen = new Map(); // Track VSCode extension clients for direct jump message routing const vscodeClients = new Set(); -function normalizeProfileHandle(handle) { - if (!handle) return null; - const raw = `${handle}`.trim(); - if (!raw) return null; - return `@${raw.replace(/^@+/, "").toLowerCase()}`; -} +function normalizeProfileHandle(handle) { + if (!handle) return null; + const raw = `${handle}`.trim(); + if (!raw) return null; + return `@${raw.replace(/^@+/, "").toLowerCase()}`; +} + +function normalizeMidiHandle(handle) { + const normalized = normalizeProfileHandle(handle); + return normalized ? normalized.slice(1) : ""; +} + +function notepatMidiSourceKey(handle, machineId) { + const handleKey = normalizeProfileHandle(handle) || "@unknown"; + const machineKey = `${machineId || "unknown"}`.trim() || "unknown"; + return `${handleKey}:${machineKey}`; +} + +function listNotepatMidiSources() { + return [...notepatMidiSources.values()] + .sort((a, b) => (b.lastSeen || 0) - (a.lastSeen || 0)) + .map((source) => ({ + handle: source.handle || null, + machineId: source.machineId, + piece: source.piece || "notepat", + lastSeen: source.lastSeen || 0, + lastEvent: source.lastEvent || null, + })); +} + +function sendNotepatMidiSources(ws) { + if (!ws || ws.readyState !== WebSocket.OPEN) return; + try { + ws.send(pack("notepat:midi:sources", { sources: listNotepatMidiSources() }, "midi-relay")); + } catch (err) { + error("🎹 Failed to send notepat midi sources:", err); + } +} + +function removeNotepatMidiSubscriber(id) { + if (id === undefined || id === null) return; + notepatMidiSubscribers.delete(id); +} + +function addNotepatMidiSubscriber(id, ws, filter = {}) { + if (id === undefined || id === null || !ws) return; + + notepatMidiSubscribers.set(id, { + ws, + all: filter.all === true, + handle: normalizeMidiHandle(filter.handle), + machineId: filter.machineId ? `${filter.machineId}`.trim() : "", + }); + + if (ws.readyState === WebSocket.OPEN) { + ws.send(pack("notepat:midi:subscribed", { + all: filter.all === true, + handle: normalizeMidiHandle(filter.handle) || null, + machineId: filter.machineId ? `${filter.machineId}`.trim() : null, + }, "midi-relay")); + } + + sendNotepatMidiSources(ws); +} + +function broadcastNotepatMidiSources() { + for (const [id, sub] of notepatMidiSubscribers) { + if (!sub?.ws || sub.ws.readyState !== WebSocket.OPEN) { + notepatMidiSubscribers.delete(id); + continue; + } + sendNotepatMidiSources(sub.ws); + } +} + +function notepatMidiSubscriberMatches(sub, event) { + if (!sub) return false; + if (sub.all) return true; + + const eventHandle = normalizeMidiHandle(event?.handle); + const eventMachine = event?.machineId ? `${event.machineId}`.trim() : ""; + + if (sub.handle && sub.handle !== eventHandle) return false; + if (sub.machineId && sub.machineId !== eventMachine) return false; + + return !!(sub.handle || sub.machineId); +} + +function broadcastNotepatMidiEvent(event) { + for (const [id, sub] of notepatMidiSubscribers) { + if (!sub?.ws || sub.ws.readyState !== WebSocket.OPEN) { + notepatMidiSubscribers.delete(id); + continue; + } + if (!notepatMidiSubscriberMatches(sub, event)) continue; + try { + sub.ws.send(pack("notepat:midi", event, "midi-relay")); + } catch (err) { + error("🎹 Failed to fan out notepat midi event:", err); + } + } +} + +function upsertNotepatMidiSource({ handle, machineId, piece, lastEvent, ts, address, port }) { + const cleanHandle = normalizeMidiHandle(handle); + const cleanMachineId = `${machineId || "unknown"}`.trim() || "unknown"; + const key = notepatMidiSourceKey(cleanHandle, cleanMachineId); + const previous = notepatMidiSources.get(key); + const next = { + handle: cleanHandle || null, + machineId: cleanMachineId, + piece: piece || "notepat", + lastSeen: ts || Date.now(), + lastEvent: lastEvent || previous?.lastEvent || null, + address: address || previous?.address || null, + port: port || previous?.port || null, + }; + + notepatMidiSources.set(key, next); + + if (!previous) { + log(`🎹 Notepat MIDI source online: ${next.handle ? "@" + next.handle : "@unknown"} ${next.machineId}`); + } + + if ( + !previous || + previous.handle !== next.handle || + previous.machineId !== next.machineId || + previous.piece !== next.piece + ) { + broadcastNotepatMidiSources(); + } + + return next; +} function compactProfileText(value) { return `${value || ""}`.replace(/\s+/g, " ").trim(); @@ -3430,60 +3582,138 @@ io.onConnection((channel) => { // #endregion -// --------------------------------------------------------------------------- -// 🧚 Raw UDP fairy relay (port 10010) β€” for native bare-metal clients -// Binary packet format: -// [1 byte type] [4 float x LE] [4 float y LE] [1 handle_len] [N handle] -// Type 0x01 = clientβ†’server, 0x02 = serverβ†’client broadcast -// --------------------------------------------------------------------------- -const UDP_FAIRY_PORT = 10010; - -udpRelay.on("message", (msg, rinfo) => { - if (msg.length < 10 || msg[0] !== 0x01) return; - - const key = `${rinfo.address}:${rinfo.port}`; - const x = msg.readFloatLE(1); - const y = msg.readFloatLE(5); - const hlen = msg[9]; - const handle = msg.slice(10, 10 + hlen).toString("utf8"); - - udpClients.set(key, { address: rinfo.address, port: rinfo.port, handle, lastSeen: Date.now() }); - - // Build broadcast packet (type 0x02) - const bcast = Buffer.alloc(msg.length); - msg.copy(bcast); - bcast[0] = 0x02; - - // Broadcast to all other UDP clients - for (const [k, client] of udpClients) { - if (k !== key) { - udpRelay.send(bcast, client.port, client.address); - } - } - - // Also broadcast to Geckos.io WebRTC clients as fairy:point - const fairyData = JSON.stringify({ x, y, handle }); - try { - // Emit to all geckos channels - io.room().emit("fairy:point", fairyData); - } catch (e) { /* ignore */ } - - // Publish to Redis for silo firehose (throttled) - const now = Date.now(); - const lastFairy = fairyThrottle.get(key) || 0; - if (now - lastFairy >= FAIRY_THROTTLE_MS) { - fairyThrottle.set(key, now); - pub.publish("fairy:point", fairyData).catch(() => {}); - } -}); - -// Clean up stale UDP clients every 30s -setInterval(() => { - const now = Date.now(); - for (const [key, client] of udpClients) { - if (now - client.lastSeen > 30000) udpClients.delete(key); - } -}, 30000); +// --------------------------------------------------------------------------- +// 🧚 Raw UDP fairy relay (port 10010) β€” for native bare-metal clients +// Binary packet format: +// [1 byte type] [4 float x LE] [4 float y LE] [1 handle_len] [N handle] +// Type 0x01 = clientβ†’server, 0x02 = serverβ†’client broadcast +// --------------------------------------------------------------------------- +const UDP_FAIRY_PORT = 10010; + +function handleNotepatMidiUdpPacket(payload, rinfo) { + if (!payload || (payload.type !== "notepat:midi" && payload.type !== "notepat:midi:heartbeat")) { + return false; + } + + const now = Date.now(); + const source = upsertNotepatMidiSource({ + handle: payload.handle, + machineId: payload.machineId, + piece: payload.piece || "notepat", + lastEvent: payload.type === "notepat:midi" ? payload.event : "heartbeat", + ts: now, + address: rinfo.address, + port: rinfo.port, + }); + + if (!source.handle && !source.machineId) { + return true; + } + + if (payload.type === "notepat:midi:heartbeat") { + return true; + } + + const rawNote = Number(payload.note); + const rawVelocity = Number(payload.velocity); + const rawChannel = Number(payload.channel); + if (!Number.isFinite(rawNote) || !Number.isFinite(rawVelocity) || !Number.isFinite(rawChannel)) { + log("🎹 Invalid notepat midi UDP payload:", payload); + return true; + } + + let event = payload.event === "note_off" ? "note_off" : "note_on"; + const note = Math.max(0, Math.min(127, Math.round(rawNote))); + const velocity = Math.max(0, Math.min(127, Math.round(rawVelocity))); + const channel = Math.max(0, Math.min(15, Math.round(rawChannel))); + if (event === "note_on" && velocity === 0) event = "note_off"; + + broadcastNotepatMidiEvent({ + type: "notepat:midi", + event, + note, + velocity, + channel, + handle: source.handle, + machineId: source.machineId, + piece: source.piece || "notepat", + ts: Number.isFinite(Number(payload.ts)) ? Number(payload.ts) : now, + }); + + return true; +} + +function pruneNotepatMidiSources() { + const now = Date.now(); + let changed = false; + + for (const [key, source] of notepatMidiSources) { + if (now - (source.lastSeen || 0) > UDP_MIDI_SOURCE_TTL_MS) { + notepatMidiSources.delete(key); + changed = true; + } + } + + if (changed) broadcastNotepatMidiSources(); +} + +udpRelay.on("message", (msg, rinfo) => { + if (msg.length > 0 && msg[0] === 0x01 && msg.length >= 10) { + const key = `${rinfo.address}:${rinfo.port}`; + const x = msg.readFloatLE(1); + const y = msg.readFloatLE(5); + const hlen = msg[9]; + const handle = msg.slice(10, 10 + hlen).toString("utf8"); + + udpClients.set(key, { address: rinfo.address, port: rinfo.port, handle, lastSeen: Date.now() }); + + // Build broadcast packet (type 0x02) + const bcast = Buffer.alloc(msg.length); + msg.copy(bcast); + bcast[0] = 0x02; + + // Broadcast to all other UDP clients + for (const [k, client] of udpClients) { + if (k !== key) { + udpRelay.send(bcast, client.port, client.address); + } + } + + // Also broadcast to Geckos.io WebRTC clients as fairy:point + const fairyData = JSON.stringify({ x, y, handle }); + try { + // Emit to all geckos channels + io.room().emit("fairy:point", fairyData); + } catch (e) { /* ignore */ } + + // Publish to Redis for silo firehose (throttled) + const now = Date.now(); + const lastFairy = fairyThrottle.get(key) || 0; + if (now - lastFairy >= FAIRY_THROTTLE_MS) { + fairyThrottle.set(key, now); + pub.publish("fairy:point", fairyData).catch(() => {}); + } + return; + } + + if (msg.length > 0 && msg[0] === 0x7b) { + try { + const payload = JSON.parse(msg.toString("utf8")); + if (handleNotepatMidiUdpPacket(payload, rinfo)) return; + } catch (err) { + log("🎹 Failed to parse UDP JSON packet:", err?.message || err); + } + } +}); + +// Clean up stale UDP clients every 30s +setInterval(() => { + const now = Date.now(); + for (const [key, client] of udpClients) { + if (now - client.lastSeen > 30000) udpClients.delete(key); + } + pruneNotepatMidiSources(); +}, 30000); udpRelay.bind(UDP_FAIRY_PORT, () => { console.log(`🧚 Raw UDP fairy relay listening on port ${UDP_FAIRY_PORT}`); diff --git a/system/public/aesthetic.computer/disks/notepat.mjs b/system/public/aesthetic.computer/disks/notepat.mjs index 4d47f0047f..2180c1961b 100644 --- a/system/public/aesthetic.computer/disks/notepat.mjs +++ b/system/public/aesthetic.computer/disks/notepat.mjs @@ -894,9 +894,10 @@ const buttonNotes = [ "+b", ]; -const buttonNoteLookup = new Set(buttonNotes); - -const midiActiveNotes = new Map(); +const buttonNoteLookup = new Set(buttonNotes); +const MIDI_NOTE_NAMES = ["c", "c#", "d", "d#", "e", "f", "f#", "g", "g#", "a", "a#", "b"]; + +const midiActiveNotes = new Map(); const MIDI_PITCH_BEND_RANGE = 2; // Semitones up/down for pitch wheel. let midiPitchBendValue = 0; // Normalized -1..1 position of the wheel. @@ -1250,24 +1251,149 @@ function parseSong(raw) { // song = parseSong(rawSong); -let startupSfx; -let udpServer; - -let stampleSampleId = null; -let stampleSampleData = null; -let stampleSampleRate = null; +let startupSfx; +let udpServer; +let relaySocket = null; +let relaySourceHandle = ""; +let relaySourceMachineId = ""; +let relaySubscribeAll = false; +let relaySources = []; +let relayStatusText = "relay off"; +let relayLastEventText = ""; +let relayLastEventAt = 0; +let relayNeedsPanic = false; +let relayMessageHandler = null; +const relayMidiQueue = []; +const relayActiveNotes = new Map(); + +let stampleSampleId = null; +let stampleSampleData = null; +let stampleSampleRate = null; let stampleNeedleProgress = 0; let stampleNeedleNote = null; let stampleProgressTick = 0; let kidlispSampleRefreshTick = 0; -let autopatHud = null; -let autopatTypeface = null; - -let picture; -let matrixFont; // MatrixChunky8 font for note letters - -async function boot({ +let autopatHud = null; +let autopatTypeface = null; + +let picture; +let matrixFont; // MatrixChunky8 font for note letters + +function normalizeRelayHandle(handle) { + return `${handle || ""}`.trim().replace(/^@+/, "").toLowerCase(); +} + +function relayTargetLabel() { + if (relaySubscribeAll) return "all"; + if (relaySourceHandle && relaySourceMachineId) return `@${relaySourceHandle} ${relaySourceMachineId}`; + if (relaySourceHandle) return `@${relaySourceHandle}`; + if (relaySourceMachineId) return relaySourceMachineId; + return "off"; +} + +function relaySubscriptionPayload() { + if (relaySubscribeAll) return { all: true }; + if (relaySourceHandle || relaySourceMachineId) { + return { + handle: relaySourceHandle || undefined, + machineId: relaySourceMachineId || undefined, + }; + } + return null; +} + +function updateRelayBridgeState() { + if (typeof window === "undefined") return; + window.acNotepatRelay = { + status: relayStatusText, + target: relayTargetLabel(), + sources: relaySources, + lastEvent: relayLastEventText, + setSource(next = {}) { + setRelaySource(next); + }, + }; +} + +function requestRelaySources() { + relaySocket?.send?.("notepat:midi:sources"); +} + +function sendRelaySubscription() { + const payload = relaySubscriptionPayload(); + if (!relaySocket?.send) return; + if (!payload) { + relaySocket.send("notepat:midi:unsubscribe", true); + relayStatusText = "relay off"; + updateRelayBridgeState(); + return; + } + relaySocket.send("notepat:midi:subscribe", payload); + relayStatusText = `relay ${relayTargetLabel()}`; + updateRelayBridgeState(); +} + +function setRelaySource(next = {}) { + const hasHandle = Object.prototype.hasOwnProperty.call(next, "handle"); + const hasMachineId = Object.prototype.hasOwnProperty.call(next, "machineId"); + const hasAll = Object.prototype.hasOwnProperty.call(next, "all"); + + if (hasHandle) { + relaySourceHandle = normalizeRelayHandle(next.handle); + } + if (hasMachineId) { + relaySourceMachineId = `${next.machineId || ""}`.trim(); + } else if (hasHandle) { + relaySourceMachineId = ""; + } + if (hasAll) { + relaySubscribeAll = next.all === true; + } + if (relaySubscribeAll) { + relaySourceHandle = ""; + relaySourceMachineId = ""; + } + relayNeedsPanic = true; + requestRelaySources(); + sendRelaySubscription(); +} + +function handleRelaySocketMessage(id, type, content) { + if (type === "connected" || type === "connected:already") { + requestRelaySources(); + sendRelaySubscription(); + return; + } + + if (type === "notepat:midi:sources") { + relaySources = Array.isArray(content?.sources) ? content.sources : []; + updateRelayBridgeState(); + return; + } + + if (type === "notepat:midi:subscribed") { + relayStatusText = `relay ${relayTargetLabel()}`; + updateRelayBridgeState(); + return; + } + + if (type === "notepat:midi:unsubscribed") { + relayStatusText = "relay off"; + relayNeedsPanic = true; + updateRelayBridgeState(); + return; + } + + if (type === "notepat:midi") { + relayMidiQueue.push(content); + relayLastEventText = `${content?.handle ? "@" + content.handle + " " : ""}${content?.event || "event"} ${content?.note ?? ""}`; + relayLastEventAt = Date.now(); + updateRelayBridgeState(); + } +} + +async function boot({ params, api, colon, @@ -1282,18 +1408,23 @@ async function boot({ sound, clock, query, -}) { - autopatApi = api; - autopatHud = hud; - autopatTypeface = typeface; +}) { + autopatApi = api; + autopatHud = hud; + autopatTypeface = typeface; + setSoundContext({ + synth: sound?.synth, + play: sound?.play, + freq: sound?.freq, + }); // ✨ Show ".com" superscript in the HUD corner label (notepat.com branding) hud.superscript(".com"); dotComMode = true; // 🎹 Check if we're in DAW mode (loaded from Ableton M4L) - dawMode = query?.daw === "1" || query?.daw === 1 || query?.daw === true; - console.log("🎹 Notepat: dawMode =", dawMode, "query.daw =", query?.daw, typeof query?.daw); + dawMode = query?.daw === "1" || query?.daw === 1 || query?.daw === true; + console.log("🎹 Notepat: dawMode =", dawMode, "query.daw =", query?.daw, typeof query?.daw); // Also check if we already have DAW data (survives hot reload) if (!dawMode && sound.daw?.bpm) { @@ -1349,11 +1480,38 @@ async function boot({ // Disabled: dynamic audio reinit was breaking audio - now using 48kHz globally pendingAudioReinit = false; - // fps(4); - udpServer = net.udp(); // For sending messages to `tv`. - - // Create picture buffer at quarter resolution (quarter width, quarter height) - const pictureWidth = Math.max(1, Math.floor(screen.width / 4)); + // fps(4); + udpServer = net.udp(); // For sending messages to `tv`. + relaySourceHandle = normalizeRelayHandle(query?.relayHandle); + relaySourceMachineId = `${query?.relayMachine || ""}`.trim(); + relaySubscribeAll = query?.relayAll === "1" || query?.relay === "all"; + relayStatusText = relaySubscriptionPayload() ? `relay ${relayTargetLabel()}` : "relay off"; + relaySocket = net.socket(handleRelaySocketMessage); + requestRelaySources(); + sendRelaySubscription(); + if (typeof window !== "undefined") { + if (relayMessageHandler) window.removeEventListener("message", relayMessageHandler); + relayMessageHandler = (event) => { + const data = event?.data; + if (!data || typeof data !== "object") return; + if (data.type === "notepat:midi:set-source" || data.type === "notepat:midi:subscribe") { + setRelaySource({ + handle: data.handle, + machineId: data.machineId, + all: data.all === true, + }); + } else if (data.type === "notepat:midi:unsubscribe") { + setRelaySource({ handle: "", machineId: "", all: false }); + } else if (data.type === "notepat:midi:sources") { + requestRelaySources(); + } + }; + window.addEventListener("message", relayMessageHandler); + updateRelayBridgeState(); + } + + // Create picture buffer at quarter resolution (quarter width, quarter height) + const pictureWidth = Math.max(1, Math.floor(screen.width / 4)); const pictureHeight = Math.max(1, Math.floor(screen.height / 4)); picture = painting(pictureWidth, pictureHeight, ({ wipe }) => { wipe("gray"); @@ -1526,10 +1684,17 @@ async function boot({ setupButtons(api); } -function sim({ sound, simCount, num, clock, painting }) { - const simTick = typeof simCount === "bigint" ? Number(simCount) : simCount; - - if (lowLatencyMode) { +function sim({ sound, simCount, num, clock, painting }) { + const simTick = typeof simCount === "bigint" ? Number(simCount) : simCount; + setSoundContext({ + synth: sound?.synth, + play: sound?.play, + freq: sound?.freq, + num, + }); + flushRelayMidiQueue(); + + if (lowLatencyMode) { if (simTick % 3 === 0) sound.speaker?.poll(); } else { sound.speaker?.poll(); @@ -5320,15 +5485,228 @@ const percDowns = {}; const connectedGamepads = {}; let miniMapActiveNote = null; let miniMapActiveKey = null; -let topBarPianoActiveNote = null; // Track active note from top bar piano -let soundContext = null; - -function setSoundContext(ctx) { - soundContext = ctx; -} - -function makeNoteSound(tone, velocity = 127, pan = 0) { - const synth = soundContext?.synth; +let topBarPianoActiveNote = null; // Track active note from top bar piano +let soundContext = null; + +function setSoundContext(ctx) { + soundContext = ctx; +} + +function lowerBaseOctave() { + return parseInt(octave) + lowerOctaveShift; +} + +function upperBaseOctave() { + return parseInt(octave) + 1 + upperOctaveShift; +} + +function midiNoteToRelayButton(noteNumber) { + if (typeof noteNumber !== "number") return null; + + const normalizedNote = ((noteNumber % 12) + 12) % 12; + const noteName = MIDI_NOTE_NAMES[normalizedNote]; + const noteOctave = Math.floor(noteNumber / 12) - 1; + const lowerOct = lowerBaseOctave(); + const upperOct = upperBaseOctave(); + + if (noteOctave === lowerOct && buttonNoteLookup.has(noteName)) { + return noteName; + } + + if (noteOctave === upperOct) { + const upperNote = `+${noteName}`; + if (buttonNoteLookup.has(upperNote)) return upperNote; + } + + if (noteOctave < lowerOct) { + const baseNote = noteOctave < lowerOct - 1 ? noteName : `+${noteName}`; + if (buttonNoteLookup.has(baseNote)) return baseNote; + } + + if (noteOctave > upperOct) { + const baseNote = noteOctave > upperOct + 1 ? noteName : `+${noteName}`; + if (buttonNoteLookup.has(baseNote)) return baseNote; + } + + if (buttonNoteLookup.has(noteName)) return noteName; + if (buttonNoteLookup.has(`+${noteName}`)) return `+${noteName}`; + return null; +} + +function startRelayButtonNote(buttonNote, velocity = 127) { + if (!buttonNote) return false; + + const num = soundContext?.num; + const synth = soundContext?.synth; + const freq = soundContext?.freq; + + if (song && buttonNote.toUpperCase() !== song?.[songIndex]?.[0]) { + synth?.({ + type: "noise-white", + tone: 1000, + duration: 0.05, + volume: 0.3, + attack: 0, + }); + return false; + } + + anyDown = true; + noteShake[buttonNote] = 3; + + let noteName = buttonNote; + let targetOctave = lowerBaseOctave(); + if (buttonNote.startsWith("+")) { + noteName = buttonNote.slice(1); + targetOctave = upperBaseOctave(); + } + + const tone = `${targetOctave}${noteName.toUpperCase()}`; + const active = orderedByCount(sounds); + + if (slide && active.length > 0) { + sounds[active[0]]?.sound?.update({ tone, duration: 0.1 }); + tonestack[buttonNote] = { + count: Object.keys(tonestack).length, + tone, + }; + sounds[buttonNote] = sounds[active[0]]; + if (sounds[buttonNote]) sounds[buttonNote].note = buttonNote; + delete sounds[active[0]]; + applyPitchBendToNotes([buttonNote], { immediate: true }); + } else { + tonestack[buttonNote] = { + count: Object.keys(tonestack).length, + tone, + }; + + const pan = getPanForButtonNote(buttonNote); + let soundHandle = makeNoteSound(tone, velocity, pan); + + if (!soundHandle || typeof soundHandle.kill !== "function") { + const velocityRatio = velocity === undefined ? 1 : velocity / 127; + const clampedRatio = num?.clamp + ? num.clamp(velocityRatio, 0, 1) + : Math.max(0, Math.min(1, velocityRatio)); + const minVelocityVolume = 0.05; + const fallbackVolume = + toneVolume * (minVelocityVolume + (1 - minVelocityVolume) * clampedRatio); + soundHandle = synth?.({ + type: "sine", + tone: freq?.(tone), + attack: quickFade ? 0.0015 : attack, + decay: 0.9, + duration: 0.4, + volume: fallbackVolume, + pan, + }); + } + + sounds[buttonNote] = { + note: buttonNote, + count: active.length + 1, + sound: soundHandle, + }; + + applyPitchBendToNotes([buttonNote], { immediate: true }); + + if (buttonNote.toUpperCase() === song?.[songIndex]?.[0]) { + songNoteDown = true; + } + + delete trail[buttonNote]; + + if (autopatApi) pictureAdd(autopatApi, tone); + } + + if (buttons[buttonNote]) { + buttons[buttonNote].down = true; + buttons[buttonNote].over = true; + } + + return true; +} + +function stopRelayButtonNote(buttonNote) { + if (!buttonNote) return; + + const orderedTones = orderedByCount(tonestack); + + if (slide && orderedTones.length > 1 && sounds[buttonNote]) { + const previousKey = orderedTones[orderedTones.length - 2]; + const previousTone = tonestack[previousKey]?.tone; + if (previousTone) { + sounds[buttonNote]?.sound?.update({ tone: previousTone, duration: 0.1 }); + sounds[previousKey] = sounds[buttonNote]; + if (sounds[previousKey]) sounds[previousKey].note = previousKey; + applyPitchBendToNotes([previousKey], { immediate: true }); + } + } else if (sounds[buttonNote]?.sound) { + const soundEntry = sounds[buttonNote]; + const lifespan = soundEntry.sound?.startedAt + ? performance.now() / 1000 - soundEntry.sound.startedAt + : 0.1; + const fade = max(0.075, min(lifespan, 0.15)); + soundEntry.sound.kill(quickFade ? fastFade : fade); + } + + if (buttonNote.toUpperCase() === song?.[songIndex]?.[0]) { + songIndex = (songIndex + 1) % song.length; + songNoteDown = false; + songShifting = true; + } + + delete tonestack[buttonNote]; + delete sounds[buttonNote]; + trail[buttonNote] = 1; + + if (buttons[buttonNote]) { + buttons[buttonNote].down = false; + buttons[buttonNote].over = false; + } +} + +function flushRelayMidiQueue() { + if (relayNeedsPanic) { + for (const buttonNote of relayActiveNotes.values()) { + stopRelayButtonNote(buttonNote); + } + relayActiveNotes.clear(); + relayNeedsPanic = false; + } + + while (relayMidiQueue.length > 0) { + const event = relayMidiQueue.shift(); + const noteNumber = Number(event?.note); + if (!Number.isFinite(noteNumber)) continue; + + const key = [ + normalizeRelayHandle(event?.handle), + `${event?.machineId || "unknown"}`, + `${event?.channel ?? 0}`, + `${Math.round(noteNumber)}`, + ].join(":"); + + if (event?.event === "note_off" || Number(event?.velocity) === 0) { + const activeButtonNote = relayActiveNotes.get(key) || midiNoteToRelayButton(Math.round(noteNumber)); + if (activeButtonNote) stopRelayButtonNote(activeButtonNote); + relayActiveNotes.delete(key); + continue; + } + + const buttonNote = midiNoteToRelayButton(Math.round(noteNumber)); + if (!buttonNote) continue; + if (relayActiveNotes.has(key)) { + stopRelayButtonNote(relayActiveNotes.get(key)); + } + if (startRelayButtonNote(buttonNote, Number(event?.velocity) || 127)) { + relayActiveNotes.set(key, buttonNote); + } + } +} + +function makeNoteSound(tone, velocity = 127, pan = 0) { + const synth = soundContext?.synth; const play = soundContext?.play; const freq = soundContext?.freq; const num = soundContext?.num;