diff --git a/pop/bin/mbtape.py b/pop/bin/mbtape.py new file mode 100644 index 0000000000..c9dfbc8fa4 --- /dev/null +++ b/pop/bin/mbtape.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Verify and unpack Menu Band performances for /pop; never execute their events.""" +import argparse +import hashlib +import json +from pathlib import Path, PurePosixPath +import shutil +import tempfile +import zipfile + + +def inspect(path): + with zipfile.ZipFile(path) as tape: + members = tape.infolist() + names = [m.filename for m in members] + if len(names) != len(set(names)): + raise ValueError('duplicate archive paths') + for member in members: + p = PurePosixPath(member.filename) + if p.is_absolute() or '..' in p.parts or '\\' in member.filename or member.is_dir(): + raise ValueError('invalid file path: ' + member.filename) + if member.file_size > 2_000_000_000 or (member.external_attr >> 16) & 0o170000 == 0o120000: + raise ValueError('oversized file or symlink') + if sum(m.file_size for m in members) > 4_000_000_000: + raise ValueError('archive exceeds 4 GB') + manifest = json.loads(tape.read('manifest.json')) + if manifest.get('format') != 'mbtape' or manifest.get('version') != 1: + raise ValueError('unsupported mbtape format/version') + entries = manifest['files'] + paths = [f['path'] for f in entries] + if len(paths) != len(set(paths)) or set(paths) != set(names) - {'manifest.json'}: + raise ValueError('manifest and archive members disagree') + required = {'master.wav', 'score.mbscore', 'events.json', 'state.json', 'stems/mix.json'} + if not required.issubset(paths): + raise ValueError('incomplete recording') + for entry in entries: + digest = hashlib.sha256() + size = 0 + with tape.open(entry['path']) as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + size += len(chunk) + digest.update(chunk) + if size != entry['bytes'] or digest.hexdigest() != entry['sha256']: + raise ValueError('checksum mismatch: ' + entry['path']) + score = json.loads(tape.read('score.mbscore')) + events = json.loads(tape.read('events.json')) + return {'path': str(path), 'id': manifest['id'], 'duration': manifest['duration'], + 'verifiedFiles': len(entries), 'notes': len(score.get('notes', [])), + 'actions': len(events.get('events', [])), + 'replayLimits': manifest.get('replayLimits', [])} + + +def extract(path, destination): + report = inspect(path) + if destination.exists(): + raise FileExistsError(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(prefix='.mbtape-', dir=destination.parent)) + try: + with zipfile.ZipFile(path) as tape: + tape.extractall(work) + # Recheck after reading; never quietly overwrite an existing lane. + if destination.exists(): + raise FileExistsError(destination) + work.rename(destination) + finally: + if work.exists(): + shutil.rmtree(work) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('tape', type=Path) + parser.add_argument('--extract', type=Path, help='new working directory; originals stay in the tape') + args = parser.parse_args() + report = extract(args.tape, args.extract) if args.extract else inspect(args.tape) + print(json.dumps(report, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/pop/tests/test_mbtape.py b/pop/tests/test_mbtape.py new file mode 100644 index 0000000000..1cb9984735 --- /dev/null +++ b/pop/tests/test_mbtape.py @@ -0,0 +1,66 @@ +import hashlib +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest +import zipfile + +spec = importlib.util.spec_from_file_location('mbtape', Path(__file__).parents[1] / 'bin/mbtape.py') +mbtape = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mbtape) + + +class TapeImportTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + + def tape(self, corrupt=False, unsafe=False): + files = {'master.wav': b'audio fixture', 'score.mbscore': b'{"notes": []}', + 'events.json': b'{"events": []}', 'state.json': b'{}', 'stems/mix.json': b'{}'} + if unsafe: + files['../outside'] = b'no' + manifest = {'format': 'mbtape', 'version': 1, 'id': 'fixture', 'duration': 1, + 'files': [{'path': p, 'bytes': len(d), 'sha256': hashlib.sha256(d).hexdigest()} + for p, d in files.items()]} + if corrupt: + files['master.wav'] = b'changed fixture' + path = self.root / 'test.mbtape' + with zipfile.ZipFile(path, 'w') as archive: + for p, d in files.items(): + archive.writestr(p, d) + archive.writestr('manifest.json', json.dumps(manifest)) + return path + + def test_verified_extraction_preserves_source_and_rejects_overwrite(self): + path = self.tape() + original = path.read_bytes() + destination = self.root / 'source' + self.assertEqual(mbtape.extract(path, destination)['verifiedFiles'], 5) + self.assertEqual((destination / 'master.wav').read_bytes(), b'audio fixture') + self.assertEqual(path.read_bytes(), original) + with self.assertRaises(FileExistsError): + mbtape.extract(path, destination) + + def test_corrupt_recording_is_not_extracted(self): + destination = self.root / 'source' + with self.assertRaisesRegex(ValueError, 'checksum'): + mbtape.extract(self.tape(corrupt=True), destination) + self.assertFalse(destination.exists()) + + def test_path_escape_is_rejected(self): + with self.assertRaisesRegex(ValueError, 'invalid file path'): + mbtape.inspect(self.tape(unsafe=True)) + + def test_unlisted_member_is_rejected(self): + path = self.tape() + with zipfile.ZipFile(path, 'a') as archive: + archive.writestr('extra', b'unlisted') + with self.assertRaisesRegex(ValueError, 'disagree'): + mbtape.inspect(path) + + +if __name__ == '__main__': + unittest.main() diff --git a/slab/menuband/Info.plist b/slab/menuband/Info.plist index 1ea892a9ba..48e8e6a2e9 100644 --- a/slab/menuband/Info.plist +++ b/slab/menuband/Info.plist @@ -70,6 +70,14 @@ Menu Band transcribes your voice on-device so you can dictate text straight into whatever app is in front, hands on the keyboard. UTExportedTypeDeclarations + + UTTypeConformsTopublic.zip-archive + UTTypeDescriptionMenu Band Tape + UTTypeIdentifiercomputer.aestheticcomputer.menuband.tape + UTTypeTagSpecification + public.filename-extensionmbtape + + UTTypeConformsTo diff --git a/slab/menuband/MBTAPE.md b/slab/menuband/MBTAPE.md new file mode 100644 index 0000000000..78d23da526 --- /dev/null +++ b/slab/menuband/MBTAPE.md @@ -0,0 +1,57 @@ +# Menu Band Tape v1 + +A `.mbtape` is a stored ZIP of one recording. The direct-download app saves it +beside the listening MP3 and includes it when the cassette is dragged out. +Every stopped take is also retained under +`~/Library/Application Support/MenuBand/Tapes//` after background export. +The App Store export remains unchanged. + +| File | Meaning | +| --- | --- | +| `manifest.json` | Version, take UUID, date, duration, per-file SHA-256 hashes and replay limits | +| `master.wav` | Captured four-channel export: heard mix L/R, dry mic duplicated | +| `stems/*.wav` | Raw tones, percussion and voice; new takes retain float32 headroom | +| `stems/mix.json` | Initial faders, automation, routing and mix normalization | +| `score.mbscore` | Editable note gates in seconds, velocities, channels and pan; links to full events | +| `notes.mid` | DAW-compatible note projection, when the take contains notes | +| `events.json` | Ordered semantic commands, arguments and monotonic time from REC | +| `state.json` | Initial instrument, backend, controls, pan and Fluoddity state | +| `assets/*.wav` | Immutable versions of microphone samples used during the take | +| `preview.mp3` | Listening copy, when an encoder is available | + +`events.json` is the performance source. The score and MIDI are projections; +pedal messages, bends, effects, percussion and surface gestures remain events. +Captured actions use the corresponding synth method names. `noteOn`/`noteOff` +carry note, velocity, channel and pan; percussion on/off uses a string group ID; +surface events carry points, anchors, direction, speed and energy. `sampleBank` +links sample slots to packaged audio and reference pitch. `fluoddity.trigger` +includes the note's exact seed and 80-parameter rule. No raw typing, filesystem +paths, device IDs or account credentials are recorded. + +All event timestamps retain the original REC origin. For alignment with exported +audio, subtract `audioOffsetSeconds` (35 ms record-key suppression plus leading +silence trim). Keep negative-time events: they may establish controls or start a +note that carries into the audible interval. The score clips note intervals to +the exported audio; MIDI no longer rebases the first note to zero independently. +Sequence numbers preserve order when timestamps coincide. These are input-side +monotonic timestamps, not promises of sample-accurate render scheduling. + +The journal and sample assets are frozen with the stopped take. Export happens on +the serial utility queue; a new recording gets fresh storage. The package is staged +and renamed only when complete. Existing destinations are never overwritten. +The original audio remains authoritative. External plugins/patches, live streams, +reverse history from before REC, already-sounding voices and stochastic percussion +can prevent exact synthesis replay. This version captures data for reconstruction; +it does not include a universal action-timeline playback engine or cloud sync. + +For `/pop`, verify first, then unpack into a new working directory: + +```sh +python3 pop/bin/mbtape.py /path/to/take.mbtape +python3 pop/bin/mbtape.py /path/to/take.mbtape --extract pop/out/my-track/source +``` + +Keep the original tape. Put new scores, transformations and rendered tracks in the +track's workspace and refer back to the manifest take UUID and file hashes. +Older WAV/MIDI exports do not contain the full action history; packaging them +cannot recover missing controls, gestures, seeds or original pre-trim timing. diff --git a/slab/menuband/Sources/MenuBand/AppDelegate.swift b/slab/menuband/Sources/MenuBand/AppDelegate.swift index 55b4d9dd0a..bd09dcc842 100644 --- a/slab/menuband/Sources/MenuBand/AppDelegate.swift +++ b/slab/menuband/Sources/MenuBand/AppDelegate.swift @@ -99,6 +99,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate { private struct ExportedTape { let file: URL let stems: URL? + let package: URL? let cover: NSImage? } // Main-thread state; encoding and file IO run on the serial utility queue. @@ -4217,7 +4218,11 @@ final class AppDelegate: NSObject, NSApplicationDelegate { do { try FileManager.default.copyItem(at: result.file, to: dest) reveal = dest - if let stems = result.stems { + if let package = result.package { + let target = self.uniqueDestinationOnDesktop(in: desktop, preferredName: package.lastPathComponent) + try FileManager.default.copyItem(at: package, to: target) + reveal = target + } else if let stems = result.stems { let target = self.uniqueDestinationOnDesktop(in: desktop, preferredName: stems.lastPathComponent) try FileManager.default.copyItem(at: stems, to: target) @@ -4290,7 +4295,18 @@ final class AppDelegate: NSObject, NSApplicationDelegate { NSWorkspace.shared.setIcon(cover, forFile: file.path, options: []) } #endif - result = ExportedTape(file: file, stems: take.stems, cover: take.cover) + var package: URL? + var complete = true +#if !MAC_APP_STORE + do { + package = try snapshot.package(preview: file.pathExtension == "mp3" ? file : nil) + guard package != nil else { throw CocoaError(.fileWriteUnknown) } + if let package, let cover = take.cover { + NSWorkspace.shared.setIcon(cover, forFile: package.path, options: []) + } + } catch { complete = false; NSLog("MenuBand: mbtape packaging failed: \(error)") } +#endif + if complete { result = ExportedTape(file: file, stems: take.stems, package: package, cover: take.cover) } } NSLog("MenuBand: tape export finished in \(ProcessInfo.processInfo.systemUptime - started)s off main") let exported = result @@ -4508,7 +4524,14 @@ final class AppDelegate: NSObject, NSApplicationDelegate { ?? NSImage(named: NSImage.multipleDocumentsName) let audioItem = NSDraggingItem(pasteboardWriter: audio.provider(fileType: type)) audioItem.setDraggingFrame(frame, contents: preview) +#if MAC_APP_STORE let stemsItem = NSDraggingItem(pasteboardWriter: stems.provider(fileType: "public.folder")) +#else + let package = TapeFilePromise(name: snapshot.exportName + ".mbtape") { [self] done in + exportTapeMP3(snapshot: snapshot) { done($0?.package) } + } + let stemsItem = NSDraggingItem(pasteboardWriter: package.provider(fileType: "computer.aestheticcomputer.menuband.tape")) +#endif stemsItem.setDraggingFrame(frame.offsetBy(dx: 18, dy: -18), contents: NSImage(named: NSImage.folderName)) tapeDragSource = source button.beginDraggingSession(with: [audioItem, stemsItem], event: event, source: source) @@ -4880,10 +4903,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate { DispatchQueue.main.asyncAfter(deadline: .now() + stopAfter) { [weak self] in guard let self else { return } self.menuBand.stopTapeNow() - DispatchQueue.global(qos: .userInitiated).async { - let saved = self.menuBand.saveStoppedTakeToDesktop() - NSLog("MenuBand mix-test take saved → \(saved?.path ?? "none")") - } + self.dropTapeOnDesktop() } } #endif @@ -6367,10 +6387,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate { updateIcon() // keys/chip back to normal color // The WAV render + album-art icon + Desktop copy are heavy — do them // off the main thread so Escape never blocks. - DispatchQueue.global(qos: .userInitiated).async { [weak self] in - let saved = self?.menuBand.saveStoppedTakeToDesktop() - NSLog("MenuBand: recording dumped → \(saved?.path ?? "nothing captured")") - } + dropTapeOnDesktop() } #endif diff --git a/slab/menuband/Sources/MenuBand/MBTape.swift b/slab/menuband/Sources/MenuBand/MBTape.swift new file mode 100644 index 0000000000..c359a0a972 --- /dev/null +++ b/slab/menuband/Sources/MenuBand/MBTape.swift @@ -0,0 +1,162 @@ +import Foundation +import CryptoKit + +/// Performance commands, timestamped on one monotonic clock. Never called +/// from a render callback: the audio buffers remain the sound authority. +final class MBTapeJournal { + private let lock = NSLock() + private var rolling = false + private var epoch: Double = 0 + private var initial: [String: Any] = [:] + private var events: [[String: Any]] = [] + + func begin(state: [String: Any], at time: Double = ProcessInfo.processInfo.systemUptime) { + lock.lock(); defer { lock.unlock() } + initial = state; events = []; epoch = time; rolling = true + } + + func record(_ action: String, _ arguments: [String: Any] = [:], + at time: Double = ProcessInfo.processInfo.systemUptime) { + lock.lock(); defer { lock.unlock() } + guard rolling, time.isFinite, JSONSerialization.isValidJSONObject(arguments) else { return } + events.append(["sequence": events.count, "time": max(0, time - epoch), + "action": action, "arguments": arguments]) + } + + func end(at time: Double = ProcessInfo.processInfo.systemUptime) { + lock.lock(); defer { lock.unlock() } + guard rolling else { return } + events.append(["sequence": events.count, "time": max(0, time - epoch), + "action": "record.stop", "arguments": [:] as [String: Any]]) + rolling = false + } + + func snapshot() -> [String: Any] { + lock.lock(); defer { lock.unlock() } + return ["version": 1, "timeUnit": "seconds", "clock": "monotonic-record-start", + "initialState": initial, "events": events] + } +} + +enum MBTape { + /// A note projection, not a claim that notes alone reproduce every sound. + /// Preserve channel, velocity and pan; pair repeated pitches FIFO. + static func score(journal: [String: Any], name: String, + audioOffset: Double, duration: Double) -> [String: Any] { + var held: [String: [[String: Any]]] = [:] + var notes: [[String: Any]] = [] + func close(_ event: [String: Any], at end: Double) { + guard let a = event["arguments"] as? [String: Any], + let raw = event["time"] as? Double else { return } + let start = max(0, raw - audioOffset) + let stop = min(duration, end - audioOffset) + guard stop > start else { return } + notes.append(["midi": a["note"] ?? 0, "start": start, "dur": stop - start, + "velocity": a["velocity"] ?? 100, "channel": a["channel"] ?? 0, + "pan": a["pan"] ?? 64]) + } + for event in journal["events"] as? [[String: Any]] ?? [] { + guard let action = event["action"] as? String, + let a = event["arguments"] as? [String: Any], + let t = event["time"] as? Double else { continue } + let key = "\(a["channel"] ?? 0):\(a["note"] ?? 0)" + if action == "noteOn", (a["velocity"] as? Int ?? 0) > 0 { + held[key, default: []].append(event) + } else if action == "noteOff" || action == "noteOn" { + if var queue = held[key], !queue.isEmpty { + close(queue.removeFirst(), at: t); held[key] = queue + } + } else if action == "panic" || action == "record.stop" { + for queue in held.values { for on in queue { close(on, at: t) } } + held.removeAll() + } + } + for queue in held.values { for on in queue { close(on, at: audioOffset + duration) } } + notes.sort { ($0["start"] as? Double ?? 0) < ($1["start"] as? Double ?? 0) } + return ["version": 1, "name": name, "notes": notes, "tailSeconds": 0, + "duration": duration, "events": "events.json", "state": "state.json", + "audioOffsetSeconds": audioOffset, + "projection": "captured-note-gates; expression and non-note triggers remain in events.json"] + } + + static func writeJSON(_ object: [String: Any], to url: URL) throws { + try JSONSerialization.data(withJSONObject: object, options: [.prettyPrinted, .sortedKeys]) + .write(to: url, options: .atomic) + } + + /// Stores files without flattening directories, with a manifest of hashes. + /// Stage and archive on the utility export queue; publish by atomic rename. + static func package(master: URL, stems: URL, midi: URL?, preview: URL?, + journal: [String: Any], id: UUID, date: Date, + audioOffset: Double, duration: Double, destination: URL, assets: [URL] = []) throws -> URL { + let fm = FileManager.default + let work = destination.deletingLastPathComponent().appendingPathComponent(".mbtape-\(UUID())") + let stage = work.appendingPathComponent("content") + try fm.createDirectory(at: stage, withIntermediateDirectories: true) + defer { try? fm.removeItem(at: work) } + func copy(_ from: URL, _ name: String) throws { + try fm.copyItem(at: from, to: stage.appendingPathComponent(name)) + } + if !assets.isEmpty { + try fm.createDirectory(at: stage.appendingPathComponent("assets"), withIntermediateDirectories: true) + for asset in assets { try copy(asset, "assets/" + asset.lastPathComponent) } + } + try copy(master, "master.wav") + try copy(stems, "stems") + if let midi { try copy(midi, "notes.mid") } + if let preview { try copy(preview, "preview.\(preview.pathExtension)") } + var timeline = journal + timeline["audioOffsetSeconds"] = audioOffset + try writeJSON(timeline, to: stage.appendingPathComponent("events.json")) + try writeJSON(journal["initialState"] as? [String: Any] ?? [:], + to: stage.appendingPathComponent("state.json")) + try writeJSON(score(journal: journal, name: master.deletingPathExtension().lastPathComponent, + audioOffset: audioOffset, duration: duration), + to: stage.appendingPathComponent("score.mbscore")) + // mix.json keeps relative references valid inside this package. + let mixURL = stage.appendingPathComponent("stems/mix.json") + guard var mix = try JSONSerialization.jsonObject(with: Data(contentsOf: mixURL)) as? [String: Any] else { + throw CocoaError(.fileReadCorruptFile) + } + var mixFiles = mix["files"] as? [String: String] ?? [:] + mixFiles["mix"] = "../master.wav"; mix["files"] = mixFiles + try writeJSON(mix, to: mixURL) + + var files: [[String: Any]] = [] + let enumerator = fm.enumerator(at: stage, includingPropertiesForKeys: [.isRegularFileKey])! + var paths: [String] = [] + for case let url as URL in enumerator { + guard try url.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile == true else { continue } + let relative = String(url.resolvingSymlinksInPath().path.dropFirst(stage.resolvingSymlinksInPath().path.count + 1)) + // Finder resource files are decoration, not recording data. + if url.lastPathComponent == "Icon\r" || url.lastPathComponent.hasPrefix("._") { + try fm.removeItem(at: url); continue + } + let data = try Data(contentsOf: url, options: .mappedIfSafe) + files.append(["path": relative, "bytes": data.count, + "sha256": SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined()]) + paths.append(relative) + } + try writeJSON(["format": "mbtape", "version": 1, "id": id.uuidString, + "createdAt": ISO8601DateFormatter().string(from: date), + "duration": duration, "audioOffsetSeconds": audioOffset, + "sampleRate": 44100, "files": files.sorted { ($0["path"] as! String) < ($1["path"] as! String) }, + "capture": "semantic-synth-commands", + "replayLimits": ["timestamps are input-side, not audio-render sample timestamps", + "external plugins, live streams, pre-record reverse history and stochastic percussion require audio stems for exact sound", + "raw keyboard events and OS input are intentionally not recorded"]], + to: stage.appendingPathComponent("manifest.json")) + paths.append("manifest.json") + let archive = work.appendingPathComponent("take.mbtape") + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/zip") + process.currentDirectoryURL = stage + process.arguments = ["-0", "-q", archive.path, "--"] + paths.sorted() + process.qualityOfService = .utility + try process.run(); process.waitUntilExit() + guard process.terminationStatus == 0 else { throw CocoaError(.fileWriteUnknown) } + // No replacement: another take must never be overwritten. + try fm.moveItem(at: archive, to: destination) + return destination + } +} diff --git a/slab/menuband/Sources/MenuBand/MenuBandController.swift b/slab/menuband/Sources/MenuBand/MenuBandController.swift index 29c672aea3..9cc496ca8b 100644 --- a/slab/menuband/Sources/MenuBand/MenuBandController.swift +++ b/slab/menuband/Sources/MenuBand/MenuBandController.swift @@ -2173,7 +2173,8 @@ final class MenuBandController { // already lands in the buffer. synth.addWaveformTapPin(tapeWaveformPinReason) synth.pinHotMic(reason: tapeMicPinReason) - tape.record(micAlreadyInMix: inputMonitoringEnabled, mix: currentMixSnapshot()) + tape.record(micAlreadyInMix: inputMonitoringEnabled, mix: currentMixSnapshot(), performanceState: synth.performanceSnapshot()) + tape.captureSamples(synth.performanceSamples()) FocusCueBeep.shared.recordStart() } @@ -2209,11 +2210,12 @@ final class MenuBandController { func startSynthOnlyRecording() { guard tape.state != .recording else { return } synth.addWaveformTapPin(tapeWaveformPinReason) - tape.record() + tape.record(mix: currentMixSnapshot(), performanceState: synth.performanceSnapshot()) + tape.captureSamples(synth.performanceSamples()) } /// Instant transport stop — just flips the tape state. The heavy WAV - /// render happens later in `saveStoppedTakeToDesktop()` off the main + /// render happens later on the serial export queue off the main /// thread, so Escape feels immediate. func stopTapeNow() { if tape.state == .recording { tape.stop() } @@ -2229,31 +2231,6 @@ final class MenuBandController { } } -#if !MAC_APP_STORE - /// Render the stopped take and wrap it into a per-take **DMG "record - /// release"** on the Desktop: our logo as the volume icon, the take's WAV - /// (with its generative album-art icon) inside, and that album art on the - /// .dmg file itself. HEAVY (audio conversion + drawing + hdiutil) — call - /// OFF the main thread. Returns the .dmg URL, or nil if empty. - @discardableResult - func saveStoppedTakeToDesktop() -> URL? { - guard let take = tape.eject() else { return nil } - let src = take.file - // The generative album art eject() made → the artifact icon. Use the - // in-memory image directly; `icon(forFile:)` races the async icon write - // and hands back the stale generic .wav type icon (no cover art at all). - let cover = take.cover ?? NSWorkspace.shared.icon(forFile: src.path) - let name = src.deletingPathExtension().lastPathComponent - // The sidecar .mid (editable notes for Ableton) rides inside the DMG. - let extras = [take.midi].compactMap { $0 } - // DMG only — the .mbtape (zip) alternative is dropped. `buildMbtape` - // stays in TakeDMG if we ever want to flag it back on. - let dmg = TakeDMG.build(wav: src, extras: extras, name: name, coverIcon: cover) - if dmg != nil { ReadyChime.shared.play() } // cool "release" chime when the artifact lands - return dmg - } -#endif - /// State-change observer. Drops the pins whenever the tape goes /// back to idle so the synth engine can suspend on inactivity. private func handleTapeChange() { @@ -2322,7 +2299,14 @@ final class MenuBandController { } // Capture the note performance as MIDI alongside the audio, so a take // carries editable notes (a .mid drops onto an Ableton MIDI track). + synth.fluodVoice.onPerformanceEvent = { [weak self] action, arguments in + self?.tape.performance.record(action, arguments) + } + synth.onPerformanceEvent = { [weak self] action, arguments in + self?.tape.performance.record(action, arguments) + } synth.onNoteEvent = { [weak self] note, vel, on, ch, pan in + if on, let self, self.tape.state == .recording { self.tape.captureSamples(self.synth.performanceSamples()) } self?.tape.ingestNote(note, velocity: vel, on: on, channel: ch, pan: pan) if on { self?.mixAnalysis.mark("melody") } } diff --git a/slab/menuband/Sources/MenuBand/MenuBandFluoddityVoice.swift b/slab/menuband/Sources/MenuBand/MenuBandFluoddityVoice.swift index 54acceda84..6a68b3a8f9 100644 --- a/slab/menuband/Sources/MenuBand/MenuBandFluoddityVoice.swift +++ b/slab/menuband/Sources/MenuBand/MenuBandFluoddityVoice.swift @@ -256,12 +256,25 @@ final class MenuBandFluoddityVoice { return out } + var onPerformanceEvent: ((String, [String: Any]) -> Void)? + func performanceState() -> [String: Any] { + lock.lock(); defer { lock.unlock() } + var flat = [Float](repeating: 0, count: 80) + flat.withUnsafeMutableBufferPointer { fluod_rule_to_floats(&genome, $0.baseAddress!) } + return ["rule": flat, "seedCounter": seedCounter] + } + func noteOn(_ midi: UInt8, velocity: UInt8, channel: UInt8) { lock.lock() seedCounter = seedCounter &* 1_664_525 &+ 1_013_904_223 pending.append(.noteOn(midi: midi, channel: channel, velocity: velocity, rule: genome, seed: seedCounter)) + let seed = seedCounter + var flat = [Float](repeating: 0, count: 80) + flat.withUnsafeMutableBufferPointer { fluod_rule_to_floats(&genome, $0.baseAddress!) } lock.unlock() + onPerformanceEvent?("fluoddity.trigger", ["note": Int(midi), "velocity": Int(velocity), + "channel": Int(channel), "seed": seed, "rule": flat]) } func noteOff(_ midi: UInt8, channel: UInt8) { diff --git a/slab/menuband/Sources/MenuBand/MenuBandSampleVoice.swift b/slab/menuband/Sources/MenuBand/MenuBandSampleVoice.swift index bd01be0db0..9bee4594f0 100644 --- a/slab/menuband/Sources/MenuBand/MenuBandSampleVoice.swift +++ b/slab/menuband/Sources/MenuBand/MenuBandSampleVoice.swift @@ -61,6 +61,19 @@ final class MenuBandSampleVoice { private var recordedBuffer: AVAudioPCMBuffer? private let bufferLock = NSLock() + /// Published buffers are immutable; retaining them is cheap and safe while + /// a later sample capture replaces the live bank. Writing happens off-main. + func performanceSamples() -> [(String, AVAudioPCMBuffer, Double)] { + bufferLock.lock(); defer { bufferLock.unlock() } + var result: [(String, AVAudioPCMBuffer, Double)] = [] + if let recordedBuffer { result.append(("global", recordedBuffer, detectedFundamental)) } + for key in perKeyBuffers.keys.sorted() { + let anchor = perKeyAnchorMidi[key] ?? key + result.append(("key-\(key)", perKeyBuffers[key]!, 440 * pow(2, (Double(anchor) - 69) / 12))) + } + return result + } + /// Auto-detected fundamental pitch (Hz) of the current recording. Used /// as the chromatic playback reference so each key sounds at its true /// note frequency rather than assuming the sample was middle C. Falls diff --git a/slab/menuband/Sources/MenuBand/MenuBandSynth.swift b/slab/menuband/Sources/MenuBand/MenuBandSynth.swift index afe0228f31..9973ea39fa 100644 --- a/slab/menuband/Sources/MenuBand/MenuBandSynth.swift +++ b/slab/menuband/Sources/MenuBand/MenuBandSynth.swift @@ -420,6 +420,8 @@ final class MenuBandSynth { /// (via notification). Silences any in-flight GM voices on disable so a /// held note doesn't hang on the now-bypassed node. func setUseACMIDI(_ on: Bool) { + capturePerformance("setUseACMIDI", ["enabled": on], stateful: true) + useACMIDI = on if gmSynthEnabled, !on { gmSynth.panic() } } @@ -430,6 +432,8 @@ final class MenuBandSynth { private(set) var usingFluoddityVoice = false func setFluoddityVoice(_ on: Bool) { + capturePerformance("setFluoddityVoice", ["enabled": on], stateful: true) + usingFluoddityVoice = on if !on { fluodVoice.panic() } } @@ -464,6 +468,8 @@ final class MenuBandSynth { } func setInputMonitoringEnabled(_ enabled: Bool) { + capturePerformance("setInputMonitoringEnabled", ["enabled": enabled], stateful: true) + inputMonitoringWanted = enabled let needsAttach = enabled && !inputMonitor.isAttached && duplexMonitorEligible() // Never detach on a plain toggle: closing and reopening the mic @@ -1225,6 +1231,8 @@ final class MenuBandSynth { /// convex curve preserves a precise dry area around center, then lets the /// room bloom as the finger travels outward. func setSpace(_ amount: Float) { + capturePerformance("setSpace", ["amount": amount], stateful: true) + let p = max(0, min(1, amount)) guard abs(p - spaceAmount) >= 0.0005 else { return } spaceAmount = p @@ -1235,6 +1243,8 @@ final class MenuBandSynth { /// only the rampable send gain moves, eliminating zipper pops while /// retaining an immediate response to the trackpad. func setEcho(_ amount: Float) { + capturePerformance("setEcho", ["amount": amount], stateful: true) + let a = max(0, min(1, amount)) guard abs(a - echoAmount) >= 0.0005 else { return } echoAmount = a @@ -1260,16 +1270,22 @@ final class MenuBandSynth { /// Speak a short phrase through the fx bus (About-window language /// easter egg). The voice rides whatever bend/space/echo is engaged. func speak(_ text: String, languageCode: String) { + capturePerformance("speak", ["text": text, "languageCode": languageCode]) + speechVoice.say(text, languageCode: languageCode) } /// Speak one key-cap letter without replacing the active instrument. func speakLetter(_ letter: String) { + capturePerformance("speakLetter", ["letter": letter]) + speechVoice.spell(letter) } /// Debounced number-row feedback from the bundled Jeffrey voice bank. func playSpokenNumber(_ number: Int) { + capturePerformance("playSpokenNumber", ["number": number]) + speechVoice.playNumber(number) } @@ -1279,6 +1295,7 @@ final class MenuBandSynth { /// wasn't enough captured audio to rewind (caller can cue a fallback). @discardableResult func playReverse() -> Bool { + capturePerformance("playReverse", ["preRecordHistoryMayBeRequired": true]) return rewindVoice.playReverse() } @@ -1287,6 +1304,8 @@ final class MenuBandSynth { /// note resets that anchor to the live head /// (see the `resetReverseAnchor` call in `noteOn`). func releaseReverse() { + capturePerformance("releaseReverse", [:]) + rewindVoice.release() } @@ -1304,13 +1323,19 @@ final class MenuBandSynth { /// Route the trackpad pitch-bend / echo onto the spacebar tape playback /// (its own DRY inserts — see MenuBandRewindVoice), so moving the mouse /// while space is held warps the reverse audio just like a live note. - func setRewindBend(amount: Float) { rewindVoice.setBend(amount: amount) } - func setRewindEcho(amount: Float) { rewindVoice.setEcho(amount: amount) } + func setRewindBend(amount: Float) { + capturePerformance("setRewindBend", ["amount": amount], stateful: true) + rewindVoice.setBend(amount: amount) } + func setRewindEcho(amount: Float) { + capturePerformance("setRewindEcho", ["amount": amount], stateful: true) + rewindVoice.setEcho(amount: amount) } /// Play one random drum hit — the About-window card-flip easter egg /// pairs this with the sparkle chord so repeated flips build a little /// beat. Uses the live percussion kit (already attached to the bus). func playEasterEggDrum() { + capturePerformance("playEasterEggDrum", [:]) + let kit: [MenuBandPercussion.Drum] = [.kick, .snare, .hatClosed, .clap, .tambo, .block, .cowbell, .snap, .ride] let drum = kit.randomElement() ?? .kick @@ -1322,6 +1347,8 @@ final class MenuBandSynth { /// down → left) so a fast swipe reads as a little rising/falling clave run. /// Dry (percussion bypasses the fx bus) so it stays crisp under any bend. func playOctaveTick(up: Bool) { + capturePerformance("playOctaveTick", ["up": up]) + percussion.play(.block, velocity: 88, pan: up ? 0.28 : -0.28) } @@ -1329,6 +1356,8 @@ final class MenuBandSynth { /// voice (AVAudioUnitTimePitch — ignores MIDI bend), so the TTS slides /// in pitch alongside every other voice. func setSpeechPitchBend(amount: Float) { + capturePerformance("setSpeechPitchBend", ["amount": amount], stateful: true) + speechVoice.setBend(amount: amount) singerVoice.setBend(amount: amount) simSingers.values.forEach { $0.setBend(amount: amount) } @@ -1337,21 +1366,29 @@ final class MenuBandSynth { /// Keep the surface cues speaking at the sounding pitch — fed from the /// controller's single bend funnel alongside the other bus voices. func setSurfaceCueBend(amount: Float) { + capturePerformance("setSurfaceCueBend", ["amount": amount], stateful: true) + surfaceCue.setBend(amount: amount) airVoice.setBend(amount: amount) } /// ⇧Space down: woosh air in around these pitches. func pushAir(pitches: [UInt8]) { + capturePerformance("pushAir", ["pitches": pitches.map { Int($0) }]) + airVoice.push(pitches: pitches) } /// ⇧Space up: let the air drift out. func releaseAir() { + capturePerformance("releaseAir", [:]) + airVoice.release() } func stopAir() { + capturePerformance("stopAir", [:]) + airVoice.stop() } @@ -1364,10 +1401,14 @@ final class MenuBandSynth { /// One movement's worth of idle-page friction, 0…1. func rubSurface(intensity: Float) { + capturePerformance("rubSurface", ["intensity": intensity]) + surfaceCue.rub(intensity: intensity) } func stopSurfaceRub() { + capturePerformance("stopSurfaceRub", [:]) + surfaceCue.stopRub() } @@ -2235,6 +2276,8 @@ final class MenuBandSynth { /// so every backend scales together. Idempotent — safe to call /// before or after `start()`. Clamped to [0, 1]. func setMasterVolume(_ value: Float) { + capturePerformance("setMasterVolume", ["value": value], stateful: true) + let clamped = max(0, min(1, value)) masterVolume = clamped // outputVolume is realtime-safe — AVAudioEngine ramps it to @@ -2247,11 +2290,15 @@ final class MenuBandSynth { /// `percFader`, AFTER the raw stem tap, so the tape's percussion stem /// is unity while the mix follows the slider. func setPercussionVolume(_ value: Float) { + capturePerformance("setPercussionVolume", ["value": value], stateful: true) + percFader.outputVolume = max(0, min(1, value)) } /// Tones trim (every instrument) — same pre-tap/post-fader split. func setTonesVolume(_ value: Float) { + capturePerformance("setTonesVolume", ["value": value], stateful: true) + tonesFader.outputVolume = max(0, min(1, value)) } @@ -2260,6 +2307,8 @@ final class MenuBandSynth { /// back. The graph stays wired and the tape keeps its dry stem. private var monitorFocusMuted = false // audible at launch; Esc (defocus) mutes, focus unmutes func setMonitorFocusMute(_ muted: Bool) { + capturePerformance("setMonitorFocusMute", ["muted": muted], stateful: true) + monitorFocusMuted = muted inputMonitor.setEnabled(inputMonitoringWanted && !muted) NSLog("MenuBand monitor: focus \(muted ? "lost → muted" : "gained → audible") (wanted=\(inputMonitoringWanted) attached=\(inputMonitor.isAttached))") @@ -2268,6 +2317,8 @@ final class MenuBandSynth { /// Mic monitor gain, 0…2 (unity = 1). Only what the duplex monitor /// feeds into the mix; the tape's voice stem stays the dry input. func setMonitorGain(_ value: Float) { + capturePerformance("setMonitorGain", ["value": value], stateful: true) + inputMonitor.setGain(max(0, min(2, value))) } @@ -2326,6 +2377,8 @@ final class MenuBandSynth { /// (sub-millisecond MIDI Program Change); blocks ~100 ms otherwise /// (sampler bank reload). func setMelodicProgram(_ program: UInt8) { + capturePerformance("setMelodicProgram", ["program": Int(program)], stateful: true) + currentMelodicProgram = program // Keep the AC GM node's program in sync so a note-on routed there // synthesizes the selected instrument. (Sounding voices keep their @@ -2376,6 +2429,8 @@ final class MenuBandSynth { /// set) so we don't bother with a delayed-playback dance. @discardableResult func setGarageBandPatch(at url: URL) -> Bool { + capturePerformance("setGarageBandPatch", ["assetName": url.lastPathComponent, "externalAssetRequired": true], stateful: true) + guard started else { return false } do { try melodic.loadInstrument(at: url) @@ -2413,6 +2468,8 @@ final class MenuBandSynth { } func stopCDJInternetRadio() { + capturePerformance("stopCDJInternetRadio", [:]) + radio.setOutputEnabled(false) radio.stopStreaming() } @@ -2427,8 +2484,12 @@ final class MenuBandSynth { spotifyDeck.start(processID: processID) } - func stopCDJSpotify() { spotifyDeck.stop() } - func silenceCDJSpotify() { spotifyDeck.setOutputEnabled(false) } + func stopCDJSpotify() { + capturePerformance("stopCDJSpotify", [:]) + spotifyDeck.stop() } + func silenceCDJSpotify() { + capturePerformance("silenceCDJSpotify", [:]) + spotifyDeck.setOutputEnabled(false) } /// Lift the live deck off the bus and put the gesture grain voice on its /// rolling audio. Returns false until enough current-CD audio exists. @@ -2452,6 +2513,8 @@ final class MenuBandSynth { } func scratchCDJ(deltaPoints: Double, velocity: Double) { + capturePerformance("scratchCDJ", ["deltaPoints": deltaPoints, "velocity": velocity]) + cdjScratch.scrub(deltaPoints: deltaPoints, velocity: velocity) } @@ -2526,6 +2589,8 @@ final class MenuBandSynth { /// Compatibility shim for older controller builds. Radio is now a /// continuous CDJ deck and never becomes the melodic key backend. func setRadioBackend(_ enabled: Bool) { + capturePerformance("setRadioBackend", ["enabled": enabled], stateful: true) + if enabled { if started { _ = resumeAudioEngineIfNeeded() @@ -2553,6 +2618,8 @@ final class MenuBandSynth { /// Mutually exclusive with radio/sample/GB — picking a plugin exits /// those, identical to `setMelodicProgram`'s mutex semantics. func setPluginInstrument(_ avUnit: AVAudioUnit?) { + capturePerformance("setPluginInstrument", ["name": avUnit?.name ?? "none", "externalAssetRequired": avUnit != nil], stateful: true) + // Tear down any previously-loaded plugin first. if let prev = pluginUnit { if pluginConnected { @@ -2615,7 +2682,9 @@ final class MenuBandSynth { } /// Clear all per-key custom samples (the ` "Home" gesture). - func clearPerKeySamples() { sampleVoice.clearPerKeySamples() } + func clearPerKeySamples() { + capturePerformance("clearPerKeySamples", [:]) + sampleVoice.clearPerKeySamples() } /// Stop recording. Returns true iff a usable buffer (≥100 ms) was /// captured; the caller flips the active backend to `.sample` only @@ -2675,6 +2744,8 @@ final class MenuBandSynth { /// MIDISynth / sampler notes so we don't double-trigger; disabling /// closes the master gate immediately. func setSampleBackend(_ enabled: Bool) { + capturePerformance("setSampleBackend", ["enabled": enabled], stateful: true) + if enabled { usingSampleBackend = true usingGarageBandPatch = false @@ -2751,6 +2822,8 @@ final class MenuBandSynth { /// no-op the sampler fallback rather than yanking the entire /// melodic mix. func setPan(_ pan: UInt8, channel: UInt8 = 0) { + capturePerformance("setPan", ["pan": Int(pan), "channel": Int(channel)], stateful: true) + // Mirror the pan into per-channel state regardless of the audible path, // so the tape's note hook can stamp CC10 onto captured MIDI note-ons. channelPan[Int(channel & 0x0F)] = pan & 0x7F @@ -2763,6 +2836,8 @@ final class MenuBandSynth { /// keyboard uses (64 = center); converted to the −1…1 the percussion /// node wants. func playPercussion(_ drum: MenuBandPercussion.Drum, velocity: UInt8, pan: UInt8) { + capturePerformance("playPercussion", ["drum": drum.rawValue, "velocity": Int(velocity), "pan": Int(pan)]) + guard started else { return } // Drums don't register in `activeNotes`, so the idle-suspend logic // would otherwise leave the engine paused and the hit silent. Wake @@ -2793,6 +2868,8 @@ final class MenuBandSynth { /// A rising, suction-like bass-drum envelope for the trackpad's /// two-finger upper-half gesture. func playReverseKick(velocity: UInt8, pan: UInt8) { + capturePerformance("playReverseKick", ["velocity": Int(velocity), "pan": Int(pan)]) + guard started else { return } _ = resumeAudioEngineIfNeeded() rewindVoice.resetReverseAnchor() @@ -2803,6 +2880,8 @@ final class MenuBandSynth { } func playDrumSkin(strike: CGPoint, anchors: [CGPoint], velocity: UInt8) { + capturePerformance("playDrumSkin", ["point": [strike.x, strike.y], "anchors": anchors.map { [$0.x, $0.y] }, "velocity": Int(velocity)]) + guard started else { return } _ = resumeAudioEngineIfNeeded() rewindVoice.resetReverseAnchor() @@ -2816,6 +2895,8 @@ final class MenuBandSynth { } func playSynthSurface(strike: CGPoint, anchors: [CGPoint], velocity: UInt8) { + capturePerformance("playSynthSurface", ["point": [strike.x, strike.y], "anchors": anchors.map { [$0.x, $0.y] }, "velocity": Int(velocity)]) + guard started else { return } _ = resumeAudioEngineIfNeeded() rewindVoice.resetReverseAnchor() @@ -2828,6 +2909,8 @@ final class MenuBandSynth { func playSurfaceLift(at point: CGPoint, anchors: [CGPoint], velocity: UInt8, synthetic: Bool) { + capturePerformance("playSurfaceLift", ["point": [point.x, point.y], "anchors": anchors.map { [$0.x, $0.y] }, "velocity": Int(velocity), "synthetic": synthetic]) + guard started else { return } _ = resumeAudioEngineIfNeeded() rewindVoice.resetReverseAnchor() @@ -2842,6 +2925,8 @@ final class MenuBandSynth { direction: CGVector = .zero, surfaceEnergy: Double = 0, synthetic: Bool = false) { + capturePerformance("setDrumSkinScratch", ["point": [point.x, point.y], "speed": speed, "anchors": anchors.map { [$0.x, $0.y] }, "direction": [direction.dx, direction.dy], "energy": surfaceEnergy, "synthetic": synthetic]) + guard started else { return } percussion.setDrumSkinScratch(at: point, speed: speed, anchors: anchors, @@ -2851,6 +2936,8 @@ final class MenuBandSynth { } func stopDrumSkinScratch() { + capturePerformance("stopDrumSkinScratch", [:]) + percussion.stopDrumSkinScratch() } @@ -2875,6 +2962,7 @@ final class MenuBandSynth { triggerMelodicDuck(depth: depth, velocity: velocity, hold: drum == .kick ? 0.105 : 0.055) let g = percussion.noteOn(drum, velocity: velocity, pan: p, accent: accent) + capturePerformance("percussionNoteOn", ["drum": drum.rawValue, "velocity": Int(velocity), "pan": Int(pan), "accent": accent, "group": String(g)]) // Roomward: a TrackDrum hit flashes its own pars (no-op without a // widget) — routed apart from the keys, see DMXOut.fixtures. DMXOut.shared.drum(drum, velocity: Int(velocity)) @@ -2903,6 +2991,8 @@ final class MenuBandSynth { /// Key-up for a split drum — damps held voices (open hat) and fires the /// release burst (closed hat). Harmless for one-shot drums. func percussionNoteOff(_ group: UInt64) { + capturePerformance("percussionNoteOff", ["group": String(group)]) + guard started, group != 0 else { return } percussion.noteOff(group) if !keepEngineWarm { scheduleIdleSuspendAfterPercussion() } @@ -2929,6 +3019,33 @@ final class MenuBandSynth { /// Fires on every note-on/off (note, velocity, on, channel, pan) — the tape /// subscribes to capture a MIDI performance (incl. CC10 pan) alongside the /// audio. `pan` is the channel's current CC10 value (64 = center). + var onPerformanceEvent: ((String, [String: Any]) -> Void)? + private let performanceLock = NSLock() + private var performanceControls: [String: [String: Any]] = [:] + + /// State and semantic commands, never raw keystrokes or device identifiers. + private func capturePerformance(_ action: String, _ arguments: [String: Any] = [:], stateful: Bool = false) { + if stateful { + let key = action + ":" + String(describing: arguments["channel"] ?? "global") + + ":" + String(describing: arguments["controller"] ?? "") + performanceLock.lock(); performanceControls[key] = arguments; performanceLock.unlock() + } + onPerformanceEvent?(action, arguments) + } + + func performanceSamples() -> [(String, AVAudioPCMBuffer, Double)] { sampleVoice.performanceSamples() } + + func performanceSnapshot() -> [String: Any] { + performanceLock.lock(); let controls = performanceControls; performanceLock.unlock() + return ["program": Int(currentMelodicProgram), "useACMIDI": useACMIDI, + "sampleBackend": usingSampleBackend, "radioBackend": usingRadioBackend, + "fluoddityBackend": usingFluoddityVoice, "garageBandPatch": usingGarageBandPatch, + "pan": channelPan.map { Int($0) }, "controls": controls, + "echo": echoAmount, "space": spaceAmount, "master": masterVolume, + "fluoddity": fluodVoice.performanceState(), + "appVersion": Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "development"] + } + var onNoteEvent: ((UInt8, UInt8, Bool, UInt8, UInt8) -> Void)? /// Per-channel CC10 pan (0–127, 64 = center), mirrored from `setPan` so the @@ -3018,6 +3135,8 @@ final class MenuBandSynth { /// pitch-bend natively; the AVAudioUnitSampler fallback is a /// no-op (and the trackpad gesture caller checks midiMode). func sendPitchBend(value: Int16, channel: UInt8 = 0) { + capturePerformance("sendPitchBend", ["value": Int(value), "channel": Int(channel)], stateful: true) + guard started else { return } let v = max(-8192, min(8191, Int(value))) + 8192 // 0…16383 let lsb = UInt8(v & 0x7F) @@ -3037,6 +3156,8 @@ final class MenuBandSynth { /// implemented in MenuBandController so it also works for sample/radio/GM /// backends that do not consume CC64 themselves. func sendControlChange(_ controller: UInt8, value: UInt8, channel: UInt8 = 0) { + capturePerformance("sendControlChange", ["controller": Int(controller), "value": Int(value), "channel": Int(channel)], stateful: true) + guard started else { return } let status: UInt8 = 0xB0 | (channel & 0x0F) if usingPluginInstrument && channel != 9, let au = pluginUnit?.audioUnit { @@ -3057,6 +3178,8 @@ final class MenuBandSynth { /// `.pitch` (cents). AVAudio's time-pitch doesn't respond to MIDI /// pitch-bend so we have to route this signal in-process. func setSamplePitchBend(amount: Float) { + capturePerformance("setSamplePitchBend", ["amount": amount], stateful: true) + sampleVoice.setBend(amount: amount) } @@ -3065,6 +3188,8 @@ final class MenuBandSynth { /// sample, so (like the varispeed backends) it doesn't see MIDI /// pitch-bend — we route the signed amount in-process instead. func setGMPitchBend(amount: Float) { + capturePerformance("setGMPitchBend", ["amount": amount], stateful: true) + if gmSynthEnabled { gmSynth.setPitchBend(amount: amount) } // The Fluoddity scanner re-derives phase from the live per-sample // frequency exactly like gm_synth, so it rides the same in-process @@ -3077,6 +3202,8 @@ final class MenuBandSynth { /// ignores MIDI pitch-bend, so the signed amount is routed in-process /// — the live stream slides in pitch alongside every other voice. func setRadioPitchBend(amount: Float) { + capturePerformance("setRadioPitchBend", ["amount": amount], stateful: true) + radio.setBend(amount: amount) spotifyDeck.setPitch(semitones: amount * 12) } @@ -3096,6 +3223,8 @@ final class MenuBandSynth { } func setMonitorPitchBend(amount: Float) { + capturePerformance("setMonitorPitchBend", ["amount": amount], stateful: true) + inputMonitor.setPitchBend(amount: amount) let now = ProcessInfo.processInfo.systemUptime if now - lastMonitorBendLog > 0.5 { @@ -3109,6 +3238,8 @@ final class MenuBandSynth { /// audibly over the tail instead of holding at full volume and /// then snapping silent at the cleanup noteOff. func sendExpression(value: UInt8, channel: UInt8 = 0) { + capturePerformance("sendExpression", ["value": Int(value), "channel": Int(channel)], stateful: true) + guard started, midiSynthReady, let au = midiSynth?.audioUnit else { return } sendMIDIEvent(au, status: 0xB0 | (channel & 0x0F), data1: 11, data2: value & 0x7F) @@ -3118,12 +3249,16 @@ final class MenuBandSynth { /// linger fade — CC 11 doesn't reach AVAudioPlayerNode, so we /// ramp the per-note player volume directly. func setSampleNoteVolume(midi: UInt8, channel: UInt8, value: UInt8) { + capturePerformance("setSampleNoteVolume", ["note": Int(midi), "channel": Int(channel), "value": Int(value)]) + sampleVoice.setNoteVolume(midi: midi, channel: channel, value: value) } /// Snap every sample voice on `channel` back to full volume so a /// new noteOn isn't dragged under by a leftover linger fade. func resetSampleChannelVolumes(channel: UInt8) { + capturePerformance("resetSampleChannelVolumes", ["channel": Int(channel)]) + sampleVoice.resetChannelVolumes(channel: channel) } @@ -3175,6 +3310,8 @@ final class MenuBandSynth { } func panic() { + capturePerformance("panic", [:]) + guard started else { return } DMXOut.shared.blackout() activeNotes.removeAll() diff --git a/slab/menuband/Sources/MenuBand/MenuBandTape.swift b/slab/menuband/Sources/MenuBand/MenuBandTape.swift index e1862ddfbd..cd4c1369cd 100644 --- a/slab/menuband/Sources/MenuBand/MenuBandTape.swift +++ b/slab/menuband/Sources/MenuBand/MenuBandTape.swift @@ -220,6 +220,10 @@ final class MenuBandTape { recordStartDate = source.recordStartDate takeID = source.takeID source.midiLock.lock() + source.sampleLock.lock() + sampleAssets = source.sampleAssets + source.sampleLock.unlock() + frozenPerformance = source.performance.snapshot() midiEvents = source.midiEvents source.midiLock.unlock() isExportSnapshot = true @@ -262,7 +266,7 @@ final class MenuBandTape { // MARK: - Transport - func record(micAlreadyInMix: Bool = false, mix: [String: Float] = [:]) { + func record(micAlreadyInMix: Bool = false, mix: [String: Float] = [:], performanceState: [String: Any] = [:]) { switch state { case .recording: return case .playing, .paused: stop() @@ -312,6 +316,11 @@ final class MenuBandTape { midiLock.lock() midiEvents.removeAll() midiStart = ProcessInfo.processInfo.systemUptime + performance.begin(state: performanceState, at: midiStart) + frozenPerformance = nil + sampleLock.lock() + sampleAssets = [:]; sampleAssetIDs = [:]; sampleBindings = [:] + sampleLock.unlock() midiLock.unlock() state = .recording } @@ -319,6 +328,7 @@ final class MenuBandTape { func stop() { switch state { case .recording: + performance.end() state = .idle postChange() case .playing, .paused: @@ -416,6 +426,13 @@ final class MenuBandTape { // MIDI performance capture — note events (relative to record start) so a // take carries editable notes for Ableton alongside the rendered audio. + private let sampleLock = NSLock() + private var sampleAssets: [String: AVAudioPCMBuffer] = [:] + private var sampleAssetIDs: [ObjectIdentifier: String] = [:] + private var sampleBindings: [String: String] = [:] + let performance = MBTapeJournal() + private var frozenPerformance: [String: Any]? + private var audioOffsetSeconds: Double = 0 private var midiEvents: [MidiFile.Event] = [] private var midiStart: TimeInterval = 0 private var midiLock = NSLock() @@ -427,6 +444,7 @@ final class MenuBandTape { bufferLock.lock() let t = Double(synthWriteFrame) / Self.sampleRate mixEvents.append(MixEvent(time: t, param: param, value: value)) + performance.record("mix", ["parameter": param, "value": value]) bufferLock.unlock() } @@ -434,13 +452,59 @@ final class MenuBandTape { /// note-on/off. Records only while the tape is rolling. func ingestNote(_ note: UInt8, velocity: UInt8, on: Bool, channel: UInt8, pan: UInt8) { guard state == .recording else { return } - let t = ProcessInfo.processInfo.systemUptime - midiStart + let now = ProcessInfo.processInfo.systemUptime + let t = now - midiStart - Double(Self.leadTrimFrames) / Self.sampleRate + performance.record(on ? "noteOn" : "noteOff", + ["note": Int(note), "velocity": Int(velocity), "channel": Int(channel), "pan": Int(pan)], at: now) midiLock.lock() midiEvents.append(MidiFile.Event(time: t, note: note, velocity: velocity, on: on, channel: channel, pan: pan)) midiLock.unlock() } + func captureSamples(_ samples: [(String, AVAudioPCMBuffer, Double)]) { + guard state == .recording else { return } + sampleLock.lock(); defer { sampleLock.unlock() } + var bindings: [String: String] = [:] + var entries: [[String: Any]] = [] + for (slot, buffer, fundamental) in samples { + let identity = ObjectIdentifier(buffer) + let name = sampleAssetIDs[identity] ?? "sample-\(UUID().uuidString).wav" + sampleAssetIDs[identity] = name; sampleAssets[name] = buffer + bindings[slot] = name + entries.append(["slot": slot, "file": "assets/" + name, "fundamentalHz": fundamental]) + } + if bindings != sampleBindings { + performance.record("sampleBank", ["samples": entries]) + sampleBindings = bindings + } + } + + func package(preview: URL?) throws -> URL? { + guard let take = eject(), let stems = take.stems else { return nil } + let fm = FileManager.default + let library = (fm.urls(for: .applicationSupportDirectory, in: .userDomainMask).first + ?? fm.homeDirectoryForCurrentUser.appendingPathComponent("Library/Application Support")) + .appendingPathComponent("MenuBand/Tapes/\(takeID.uuidString)", isDirectory: true) + try fm.createDirectory(at: library, withIntermediateDirectories: true) + let url = library.appendingPathComponent(take.file.deletingPathExtension().lastPathComponent + ".mbtape") + if FileManager.default.fileExists(atPath: url.path) { return url } + let assetsDir = take.file.deletingLastPathComponent().appendingPathComponent("mbassets-\(takeID)") + try FileManager.default.createDirectory(at: assetsDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: assetsDir) } + var assets: [URL] = [] + for (name, buffer) in sampleAssets { + let path = assetsDir.appendingPathComponent(name) + let file = try AVAudioFile(forWriting: path, settings: buffer.format.settings) + try file.write(from: buffer) + assets.append(path) + } + return try MBTape.package(master: take.file, stems: stems, midi: take.midi, + preview: preview, journal: frozenPerformance ?? performance.snapshot(), + id: takeID, date: take.date, audioOffset: audioOffsetSeconds, + duration: take.duration, destination: url, assets: assets) + } + /// Render the recording into a single 4-channel WAV: /// /// Channel 1 — synth L @@ -538,6 +602,7 @@ final class MenuBandTape { NSLog("MenuBandTape: eject raw=\(rawFrames) offset=\(offset) end=\(endFrame) frames=\(frames)") guard frames > 0 else { return nil } let duration = Double(frames) / Self.sampleRate + audioOffsetSeconds = Double(offset + Self.leadTrimFrames) / Self.sampleRate let date = recordStartDate // No mastering on the way out: the take leaves at the level it was // heard at (the limiter already holds it under 0 dBFS). The old @@ -694,7 +759,7 @@ final class MenuBandTape { pan: $0.pan) } let mURL = url.deletingPathExtension().appendingPathExtension("mid") - if (try? MidiFile.data(events: shifted, trackName: baseName).write(to: mURL)) != nil { + if (try? MidiFile.data(events: shifted, trackName: baseName, normalizeStart: false).write(to: mURL)) != nil { midiURL = mURL } } @@ -836,45 +901,25 @@ final class MenuBandTape { catch { NSLog("MenuBandTape: stems dir failed: \(error)"); return nil } func writeWav(_ name: String, channels: [UnsafePointer]) -> Bool { - let url = dir.appendingPathComponent(name) - guard let out = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: Self.sampleRate, - channels: AVAudioChannelCount(channels.count), interleaved: true), - let flt = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: Self.sampleRate, - channels: AVAudioChannelCount(channels.count), interleaved: false), - let converter = AVAudioConverter(from: flt, to: out) else { return false } - var file: AVAudioFile? + guard let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: Self.sampleRate, + channels: AVAudioChannelCount(channels.count), interleaved: false) else { return false } do { - file = try AVAudioFile(forWriting: url, settings: out.settings, - commonFormat: .pcmFormatInt16, interleaved: true) - } catch { NSLog("MenuBandTape: stem \(name) open failed: \(error)"); return false } - let chunk = 8192 - var cursor = 0 - while cursor < frames { - let take = min(chunk, frames - cursor) - guard let src = AVAudioPCMBuffer(pcmFormat: flt, frameCapacity: AVAudioFrameCount(take)), - let dst = AVAudioPCMBuffer(pcmFormat: out, frameCapacity: AVAudioFrameCount(take)) - else { return false } - src.frameLength = AVAudioFrameCount(take) - bufferLock.lock() - for (i, ch) in channels.enumerated() { - if let d = src.floatChannelData?[i] { - memcpy(d, ch.advanced(by: offset + cursor), take * MemoryLayout.size) + let file = try AVAudioFile(forWriting: dir.appendingPathComponent(name), settings: format.settings) + var cursor = 0 + while cursor < frames { + let count = min(8192, frames - cursor) + guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(count)) else { return false } + buffer.frameLength = AVAudioFrameCount(count) + bufferLock.lock() + for (i, source) in channels.enumerated() { + memcpy(buffer.floatChannelData![i], source.advanced(by: offset + cursor), count * MemoryLayout.size) } + bufferLock.unlock() + try file.write(from: buffer) + cursor += count } - bufferLock.unlock() - var supplied = false - var error: NSError? - let status = converter.convert(to: dst, error: &error) { _, outStatus in - if supplied { outStatus.pointee = .endOfStream; return nil } - supplied = true; outStatus.pointee = .haveData - return src - } - if status == .error { return false } - do { try file?.write(from: dst) } catch { return false } - cursor += take - } - file = nil - return true + return true + } catch { NSLog("MenuBandTape: stem write failed: \(error)"); return false } } guard let tl = tonesStem.buffer.floatChannelData?[0], let tr = tonesStem.buffer.floatChannelData?[1], diff --git a/slab/menuband/Sources/MenuBand/MidiFile.swift b/slab/menuband/Sources/MenuBand/MidiFile.swift index 90c131732b..b5682b844a 100644 --- a/slab/menuband/Sources/MenuBand/MidiFile.swift +++ b/slab/menuband/Sources/MenuBand/MidiFile.swift @@ -17,11 +17,12 @@ enum MidiFile { private static let ppq = 480 private static let ticksPerSecond = Double(ppq) * 2.0 // 120 BPM → 0.5 s/qn - /// Build the `.mid` bytes. Times are normalized so the first event is 0. + /// Build the `.mid` bytes. Standalone scores normalize their first event; + /// tape exports pass normalizeStart=false to retain audio alignment. /// `trackName` (if given) is written as an FF 03 track-name meta event so /// the take is labeled on import. Each note-on emits a CC10 (pan) first when /// its channel's pan changed, reproducing Menu Band's keyboard-position pan. - static func data(events: [Event], trackName: String? = nil) -> Data { + static func data(events: [Event], trackName: String? = nil, normalizeStart: Bool = true) -> Data { var track = Data() // Tempo meta (500000 µs/quarter = 120 BPM) at delta 0. @@ -36,7 +37,7 @@ enum MidiFile { } let sorted = events.sorted { $0.time < $1.time } - let base = sorted.first?.time ?? 0 + let base = normalizeStart ? (sorted.first?.time ?? 0) : 0 var lastTick = 0 var lastPan = [UInt8](repeating: 255, count: 16) // force first CC10 per channel for e in sorted { diff --git a/slab/menuband/Sources/MenuBand/TakeDMG.swift b/slab/menuband/Sources/MenuBand/TakeDMG.swift index 1f4676ab57..223d1bc6a2 100644 --- a/slab/menuband/Sources/MenuBand/TakeDMG.swift +++ b/slab/menuband/Sources/MenuBand/TakeDMG.swift @@ -78,26 +78,6 @@ enum TakeDMG { } } - /// Instant alternative to the DMG: a `.mbtape` — a stored (uncompressed) - /// zip of the take's files, written in milliseconds (no filesystem image, - /// no mount). Drags into Messages the same way; just not mountable. The - /// album-art cover goes on the .mbtape file. - @discardableResult - static func buildMbtape(files: [URL], name: String, coverIcon: NSImage?) -> URL? { - let fm = FileManager.default - let desktop = fm.urls(for: .desktopDirectory, in: .userDomainMask).first - ?? fm.homeDirectoryForCurrentUser.appendingPathComponent("Desktop") - let out = uniqueURL(desktop.appendingPathComponent("\(name).mbtape")) - try? fm.removeItem(at: out) - // zip -0 store (audio won't compress anyway), -j flat, -q quiet. - guard run("/usr/bin/zip", ["-0", "-j", "-q", out.path] + files.map { $0.path }) else { return nil } - if let cover = coverIcon { - NSWorkspace.shared.setIcon(cover, forFile: out.path, options: []) - } - NSLog("MenuBand mbtape: \(out.path)") - return out - } - /// Block (off the main thread) until a custom-icon resource fork of at least /// `minBytes` has been written — `NSWorkspace.setIcon` flushes it /// asynchronously, so this prevents moving a file whose icon is half-written. diff --git a/slab/menuband/Tests/MenuBandTests/MBTapeTests.swift b/slab/menuband/Tests/MenuBandTests/MBTapeTests.swift new file mode 100644 index 0000000000..6f08009ce0 --- /dev/null +++ b/slab/menuband/Tests/MenuBandTests/MBTapeTests.swift @@ -0,0 +1,86 @@ +import XCTest +@testable import MenuBand + +final class MBTapeTests: XCTestCase { + func testSnapshotSurvivesNextRecordingAndExcludesStoppedActions() throws { + let log = MBTapeJournal() + log.begin(state: ["program": 7], at: 100) + log.record("setSpace", ["amount": 0.4], at: 100.25) + log.end(at: 101) + log.record("noteOn", ["note": 60], at: 102) + let first = log.snapshot() + log.begin(state: ["program": 9], at: 200) + XCTAssertEqual((first["events"] as? [[String: Any]])?.count, 2) + XCTAssertEqual((first["initialState"] as? [String: Int])?["program"], 7) + XCTAssertEqual((log.snapshot()["events"] as? [[String: Any]])?.count, 0) + } + + func testScoreKeepsDelayChannelsAndClosesHeldNotes() throws { + let log = MBTapeJournal() + log.begin(state: [:], at: 100) + log.record("noteOn", ["note": 60, "velocity": 83, "channel": 2, "pan": 20], at: 101) + log.record("noteOn", ["note": 60, "velocity": 77, "channel": 3, "pan": 90], at: 101.5) + log.record("noteOff", ["note": 60, "channel": 2], at: 102) + log.end(at: 103) + let score = MBTape.score(journal: log.snapshot(), name: "take", audioOffset: 0.5, duration: 2.5) + let notes = try XCTUnwrap(score["notes"] as? [[String: Any]]) + XCTAssertEqual(notes.count, 2) + XCTAssertEqual(notes[0]["start"] as? Double, 0.5) + XCTAssertEqual(notes[0]["dur"] as? Double, 1) + XCTAssertEqual(notes[1]["channel"] as? Int, 3) + XCTAssertEqual(notes[1]["dur"] as? Double, 1.5) + XCTAssertEqual(notes[1]["velocity"] as? Int, 77) + let data = try JSONSerialization.data(withJSONObject: score) + XCTAssertEqual(try JSONDecoder().decode(MBScore.self, from: data).notes.count, 2) + } + + func testRepeatedPitchAndTrimmedNote() throws { + let log = MBTapeJournal() + log.begin(state: [:], at: 0) + log.record("noteOn", ["note": 64, "velocity": 90, "channel": 0], at: 0.1) + log.record("noteOn", ["note": 64, "velocity": 80, "channel": 0], at: 0.8) + log.record("noteOff", ["note": 64, "channel": 0], at: 1) + log.record("panic", at: 1.2) + let score = MBTape.score(journal: log.snapshot(), name: "take", audioOffset: 0.5, duration: 2) + let notes = try XCTUnwrap(score["notes"] as? [[String: Any]]) + XCTAssertEqual(notes.count, 2) + XCTAssertEqual(notes[0]["start"] as? Double, 0) + XCTAssertEqual(notes[0]["dur"] as? Double, 0.5) + XCTAssertEqual(notes[1]["dur"] as! Double, 0.4, accuracy: 0.00001) + } + + func testArchiveRoundTripAndNoOverwrite() throws { + let fm = FileManager.default + let root = fm.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try fm.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? fm.removeItem(at: root) } + let master = root.appendingPathComponent("take.wav"), stems = root.appendingPathComponent("stems") + try Data("master fixture".utf8).write(to: master) + try fm.createDirectory(at: stems, withIntermediateDirectories: true) + try MBTape.writeJSON(["files": ["mix": "../take.wav"]], to: stems.appendingPathComponent("mix.json")) + try Data("stem fixture".utf8).write(to: stems.appendingPathComponent("tones.wav")) + let out = root.appendingPathComponent("take.mbtape") + let log = MBTapeJournal(); log.begin(state: [:], at: 0); log.end(at: 1) + func pack() throws -> URL { + try MBTape.package(master: master, stems: stems, midi: nil, preview: nil, + journal: log.snapshot(), id: UUID(), date: Date(), audioOffset: 0, + duration: 1, destination: out) + } + XCTAssertEqual(try pack(), out) + let original = try Data(contentsOf: out) + XCTAssertThrowsError(try pack()) + XCTAssertEqual(try Data(contentsOf: out), original) + let pipe = Pipe(), process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/unzip") + process.arguments = ["-p", out.path, "manifest.json"] + process.standardOutput = pipe + try process.run() + let data = pipe.fileHandleForReading.readDataToEndOfFile(); process.waitUntilExit() + XCTAssertEqual(process.terminationStatus, 0) + let manifest = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) + XCTAssertEqual(manifest["format"] as? String, "mbtape") + let files = try XCTUnwrap(manifest["files"] as? [[String: Any]]) + XCTAssertTrue(files.contains { $0["path"] as? String == "stems/tones.wav" }) + XCTAssertTrue(files.allSatisfy { ($0["sha256"] as? String)?.count == 64 }) + } +} diff --git a/slab/menuband/Tests/MenuBandTests/TapeExportSnapshotTests.swift b/slab/menuband/Tests/MenuBandTests/TapeExportSnapshotTests.swift index 2914aedaf4..0ab53e5da4 100644 --- a/slab/menuband/Tests/MenuBandTests/TapeExportSnapshotTests.swift +++ b/slab/menuband/Tests/MenuBandTests/TapeExportSnapshotTests.swift @@ -13,9 +13,9 @@ final class TapeExportSnapshotTests: XCTestCase { func testNextRecordingCannotChangeAnExportingTake() throws { let tape = MenuBandTape() - tape.record(micAlreadyInMix: true, mix: ["mic": 0.5]) + tape.record(micAlreadyInMix: true, mix: ["mic": 0.5], performanceState: ["program": 42]) tape.ingestSynth(signal(0.25)) - tape.ingestTones(signal(0.125)) + tape.ingestTones(signal(1.25)) tape.stop() let snapshot = try XCTUnwrap(tape.snapshotForExport()) XCTAssertTrue(snapshot === tape.snapshotForExport()) @@ -45,9 +45,25 @@ final class TapeExportSnapshotTests: XCTestCase { let stems = try XCTUnwrap(take.stems) let mix = try JSONSerialization.jsonObject(with: Data(contentsOf: stems.appendingPathComponent("mix.json"))) as! [String: Any] XCTAssertEqual((mix["start"] as? [String: Double])?["mic"], 0.5) + let toneFile = try AVAudioFile(forReading: stems.appendingPathComponent("tones.wav")) + let toneAudio = AVAudioPCMBuffer(pcmFormat: toneFile.processingFormat, + frameCapacity: AVAudioFrameCount(toneFile.length))! + try toneFile.read(into: toneAudio) + XCTAssertEqual(toneAudio.floatChannelData![0][100], 1.25, accuracy: 0.00001) + let archive = try XCTUnwrap(snapshot.package(preview: nil)) + defer { try? FileManager.default.removeItem(at: archive.deletingLastPathComponent()) } + let process = Process(), pipe = Pipe() + process.executableURL = URL(fileURLWithPath: "/usr/bin/unzip") + process.arguments = ["-p", archive.path, "state.json"] + process.standardOutput = pipe + try process.run() + let data = pipe.fileHandleForReading.readDataToEndOfFile(); process.waitUntilExit() + let state = try JSONSerialization.jsonObject(with: data) as! [String: Any] + XCTAssertEqual(state["program"] as? Int, 42) + } catch { XCTFail("Export failed: \(error)") } } - wait(for: [finished], timeout: 20) + wait(for: [finished], timeout: 30) } func testLiveRecordingCannotBeSharedWithAnExporter() {