From cd42cda7cbde9beeb34ce28de769fc56d32069f2 Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Sun, 28 Jun 2026 09:51:43 -0800 Subject: [PATCH] menuband: absolute trackpad fx pad via private MultitouchSupport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NSTouch never reaches Menu Band's non-activating menubar panel (it isn't the frontmost app), so the trackpad bend was historically unreliable. Tap the private MultitouchSupport.framework instead (dlopen'd, gated #if !MAC_APP_STORE — App Store forbids private APIs + the sandbox blocks the multitouch HID), which delivers focus-independent normalized finger positions. - MultitouchTrackpad: enumerates trackpads, registers a frame callback, reports all fingers (0…1, origin bottom-left) on the main thread. - Default delta pitch-bend unchanged. Pressing Tab mid-bend toggles an absolute mode: the finger's raw trackpad position maps 1:1 onto the fx grid (grid geometry mirrors the trackpad). Frame glows accent in that mode; auto-reverts when the gesture ends. - Overlay paints live touch dots in both modes ("show the touches"). - The tap also feeds setTrackpadTouchActive() — the reliable finger-lift signal the dead NSTouch TouchSensorView never delivered. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Sources/MenuBand/AppDelegate.swift | 89 +++++++++- .../Sources/MenuBand/MultitouchTrackpad.swift | 168 ++++++++++++++++++ .../Sources/MenuBand/PitchBendCursor.swift | 51 ++++-- 3 files changed, 296 insertions(+), 12 deletions(-) create mode 100644 slab/menuband/Sources/MenuBand/MultitouchTrackpad.swift diff --git a/slab/menuband/Sources/MenuBand/AppDelegate.swift b/slab/menuband/Sources/MenuBand/AppDelegate.swift index c2fe4563cf..848db1e4de 100644 --- a/slab/menuband/Sources/MenuBand/AppDelegate.swift +++ b/slab/menuband/Sources/MenuBand/AppDelegate.swift @@ -312,6 +312,23 @@ final class AppDelegate: NSObject, NSApplicationDelegate { /// note changes / legato — for as long as this is true; it only /// springs back once the last finger lifts. private var trackpadTouchActive = false + /// Absolute-position fx mode. Toggled with Tab MID-bend: instead of the + /// default relative-delta gesture, the finger's raw trackpad position maps + /// 1:1 onto the fx grid (grid geometry == trackpad geometry). Driven by the + /// private MultitouchSupport tap (see `MultitouchTrackpad`), so it only does + /// anything on the non-sandboxed direct-download build. + private var pitchBendAbsoluteMode = false + /// Latest set of fingers on the trackpad (normalized 0…1, origin + /// bottom-left) from the MultitouchSupport tap — drives the absolute fx + /// mapping and the touch dots painted on the overlay. Empty on the App + /// Store build (no tap). + private var mtTouches: [CGPoint] = [] + /// True only where the private MultitouchSupport tap is compiled in. + #if !MAC_APP_STORE + private let trackpadFxAvailable = true + #else + private let trackpadFxAvailable = false + #endif /// Tracks the last lit-note count we observed in `onLitChanged` /// so we can detect the all-notes-released edge and trigger /// both the bend rubber-band and the cursor pop. @@ -428,6 +445,16 @@ final class AppDelegate: NSObject, NSApplicationDelegate { reason: "Menu Band live instrument audio" ) Self.registerBundledFonts() + #if !MAC_APP_STORE + // Global trackpad tap via private MultitouchSupport — the + // focus-independent input source for the absolute fx pad (NSTouch never + // reaches this non-activating menubar panel). Frames arrive on the main + // thread; route them through the bend/fx + overlay handler. + MultitouchTrackpad.shared.onFrame = { [weak self] touches in + self?.handleTrackpadFrame(touches) + } + MultitouchTrackpad.shared.start() + #endif // Apply forceLayout *before* statusItem creation so the initial // status-item length matches the pinned layout's imageSize. // Otherwise the icon flashes the default `.full` width until @@ -943,6 +970,15 @@ final class AppDelegate: NSObject, NSApplicationDelegate { // user sees the layout dynamically appear while typing. localCapture.onKey = { [weak self] keyCode, isDown, isRepeat, flags in guard let self = self else { return false } + // Tab toggles the absolute trackpad fx mode — but only mid-bend + // (the overlay is up) and only where the MultitouchSupport tap + // exists. Consume it so focus traversal doesn't fire; otherwise + // leave Tab alone for normal use. + if keyCode == 48 /* kVK_Tab */, self.trackpadFxAvailable, + self.pitchBendCursorPushed { + if isDown && !isRepeat { self.toggleAbsoluteFxMode() } + return true + } // Escape disarms capture explicitly. Useful when the user // wants to release focus without clicking another app. Also // the explicit exit for latched pitch-bend mode. @@ -3525,6 +3561,49 @@ final class AppDelegate: NSObject, NSApplicationDelegate { /// window). Either source flipping the flag lets the trackpad /// pitch-bend gesture engage; whichever currently owns the key /// window's responder chain is the one delivering indirect touches. + /// Main-thread sink for every MultitouchSupport frame (direct-download + /// build only). Maintains the live touch set, drives the reliable + /// finger-down/up gate the dead NSTouch sensor never could, feeds the + /// absolute fx mapping when that mode is latched, and keeps the overlay's + /// touch dots fresh. + private func handleTrackpadFrame(_ touches: [CGPoint]) { + mtTouches = touches + let active = !touches.isEmpty + if active != trackpadTouchActive { setTrackpadTouchActive(active) } + if pitchBendAbsoluteMode, pitchBendCursorPushed, let p = touches.first { + applyAbsoluteFx(point: p) + } + if pitchBendCursorPushed { updatePitchBendOverlayImage() } + } + + /// Map a finger's absolute trackpad position (0…1) straight onto the fx + /// grid: X = bipolar space/echo axis, Y = pitch-bend. The grid the user + /// sees now mirrors the trackpad they're touching, 1:1. + private func applyAbsoluteFx(point p: CGPoint) { + let fxMax: Float = Self.fxEchoEnabled ? 1 : 0 + fxX = max(Float(-1), min(fxMax, Float(p.x - 0.5) * 2)) + echoAmount = Self.fxEchoEnabled ? max(Float(0), fxX) : 0 + spaceAmount = max(Float(0), -fxX) + cancelFxRelease() + bendGestureTarget = max(-Self.bendRange, + min(Self.bendRange, Float(p.y - 0.5) * 2 * Self.bendRange)) + startBendEase() + menuBand.setSpace(amount: spaceAmount) + menuBand.setEcho(amount: echoAmount) + pushStaffPitchShift() + } + + /// Toggle the absolute trackpad fx mode (Tab, mid-bend). Snaps onto the + /// finger's current position when entering so there's no jump. + private func toggleAbsoluteFxMode() { + pitchBendAbsoluteMode.toggle() + if pitchBendAbsoluteMode, let p = mtTouches.first { + applyAbsoluteFx(point: p) + } + updatePitchBendOverlayImage() + debugLog("absolute fx mode = \(pitchBendAbsoluteMode)") + } + func setTrackpadTouchActive(_ active: Bool) { trackpadTouchActive = active guard !active else { return } @@ -3583,6 +3662,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate { // so engage the gesture even with no key held. guard menuBand.keyboardNotesHeld || shift || inReleaseGrace || menuBand.isRewinding else { return } + // In absolute mode the finger's raw trackpad position drives the grid + // (see handleTrackpadFrame); ignore relative mouse deltas so the two + // sources don't fight over bend/fxX. + if pitchBendAbsoluteMode { return } let dy = Float(event.deltaY) let dx = Float(event.deltaX) guard dy != 0 || dx != 0 else { return } @@ -3721,7 +3804,8 @@ final class AppDelegate: NSObject, NSApplicationDelegate { // the puck reaches the grid edge exactly at the ±bendRange cap // (PitchBendCursor clamps the normalized value to ±1 internally). PitchBendCursor.image(forBend: bendAmount / Self.bendRange, echo: fxX, - keyDown: menuBand.keyboardNotesHeld) + keyDown: menuBand.keyboardNotesHeld, + touches: mtTouches, absolute: pitchBendAbsoluteMode) } private func showPitchBendOverlay() { @@ -3886,6 +3970,9 @@ final class AppDelegate: NSObject, NSApplicationDelegate { // Esc / focus-loss always fully exits, even if the lock flag was // somehow already cleared. pitchBendModeLatched = false + // Absolute fx mode is per-gesture; always revert to the default + // relative bend for the next session. + pitchBendAbsoluteMode = false guard pitchBendCursorLocked else { // Mode was latched but cursor not currently locked — still // make sure the overlay is gone and fx spring back. diff --git a/slab/menuband/Sources/MenuBand/MultitouchTrackpad.swift b/slab/menuband/Sources/MenuBand/MultitouchTrackpad.swift new file mode 100644 index 0000000000..97ade7fc32 --- /dev/null +++ b/slab/menuband/Sources/MenuBand/MultitouchTrackpad.swift @@ -0,0 +1,168 @@ +import Foundation + +// MultitouchTrackpad — global trackpad-finger tap via Apple's PRIVATE +// MultitouchSupport.framework. Unlike NSTouch (which only reaches the +// frontmost app's first responder), this receives every finger on every +// trackpad regardless of which app is active — exactly what the pitch-bend +// needs, since Menu Band is a non-activating menubar panel that never owns +// activation at touch time. +// +// PRIVATE API — App Store forbidden (Review Guideline 2.5.1 + the App +// Sandbox blocks the multitouch HID device). Hence the whole file is gated +// out of the MAS build; the Developer-ID / direct-download build (no +// sandbox) is the only place it compiles or runs. We dlopen the framework +// rather than link it so there's no symbol dependency unless this path runs. +#if !MAC_APP_STORE + +/// One finger as MultitouchSupport reports it. Layout MUST match the +/// framework's `MTTouch`/`Finger` struct byte-for-byte — only `normalized` +/// (0…1 position, origin bottom-left) is consumed here but every field is +/// declared so the offsets line up. +struct MTPoint { var x: Float; var y: Float } +struct MTReadout { var position: MTPoint; var velocity: MTPoint } +struct MTTouch { + var frame: Int32 + var timestamp: Double + var identifier: Int32 + var state: Int32 // 1 not-touching … 4 touching … 7 leaving + var fingerID: Int32 + var handID: Int32 + var normalized: MTReadout + var size: Float + var zero1: Int32 + var angle: Float + var majorAxis: Float + var minorAxis: Float + var absolute: MTReadout // millimetres + var zero2a: Int32 + var zero2b: Int32 + var zDensity: Float +} + +/// C callback shape: `int (*)(MTDeviceRef, MTTouch*, int, double, int)`. +/// The contacts pointer crosses as a raw pointer (a typed Swift-struct +/// pointer isn't C-representable) and is rebound to MTTouch inside. +typealias MTContactCallback = @convention(c) ( + UnsafeMutableRawPointer?, UnsafeMutableRawPointer?, Int32, Double, Int32 +) -> Int32 + +/// Free function so it's a valid C function pointer (no captured context); +/// forwards into the singleton, which the framework can't reach directly. +private func mtFrameCallback( + _ device: UnsafeMutableRawPointer?, + _ contacts: UnsafeMutableRawPointer?, + _ numContacts: Int32, + _ timestamp: Double, + _ frame: Int32 +) -> Int32 { + let typed = contacts?.assumingMemoryBound(to: MTTouch.self) + MultitouchTrackpad.shared.handle(contacts: typed, + count: Int(numContacts), + timestamp: timestamp) + return 0 +} + +final class MultitouchTrackpad { + static let shared = MultitouchTrackpad() + + /// Every finger currently on the trackpad, as absolute normalized points + /// (0…1 each axis, origin bottom-left). Empty when no finger is down. + /// Delivered on the MAIN thread (the framework calls back on its own + /// thread; AppKit/audio state must be touched on main). This is the clean, + /// focus-independent signal the pitch-bend / fx pad consumes — no pointer + /// acceleration, unlike the dead NSTouch `TouchSensorView` path. + var onFrame: (([CGPoint]) -> Void)? + + private var handle: UnsafeMutableRawPointer? + private var devices: [UnsafeMutableRawPointer] = [] + private var started = false + private var loggedCount = 0 + private var lastLogStamp: Double = 0 + + private typealias CreateListFn = @convention(c) () -> Unmanaged? + private typealias RegisterFn = @convention(c) + (UnsafeMutableRawPointer, MTContactCallback) -> Void + private typealias StartFn = @convention(c) + (UnsafeMutableRawPointer, Int32) -> Void + private typealias StopFn = @convention(c) (UnsafeMutableRawPointer) -> Void + private typealias UnregisterFn = @convention(c) (UnsafeMutableRawPointer) -> Void + + /// Open the private framework, enumerate trackpads, register + start the + /// frame callback on each. Idempotent. Returns false (and logs) if the + /// framework or any required symbol is unavailable. + @discardableResult + func start() -> Bool { + guard !started else { return true } + let path = "/System/Library/PrivateFrameworks/" + + "MultitouchSupport.framework/MultitouchSupport" + guard let h = dlopen(path, RTLD_NOW) else { + NSLog("MenuBand MTouch: dlopen failed — %s", + dlerror().map { String(cString: $0) } ?? "unknown") + return false + } + handle = h + func sym(_ name: String) -> UnsafeMutableRawPointer? { dlsym(h, name) } + guard let createSym = sym("MTDeviceCreateList"), + let registerSym = sym("MTRegisterContactFrameCallback"), + let startSym = sym("MTDeviceStart") else { + NSLog("MenuBand MTouch: missing symbol(s) in MultitouchSupport") + return false + } + let createList = unsafeBitCast(createSym, to: CreateListFn.self) + let register = unsafeBitCast(registerSym, to: RegisterFn.self) + let startDevice = unsafeBitCast(startSym, to: StartFn.self) + + guard let list = createList()?.takeRetainedValue() else { + NSLog("MenuBand MTouch: MTDeviceCreateList returned nil") + return false + } + let count = CFArrayGetCount(list) + for i in 0..?, count: Int, timestamp: Double) { + var points: [CGPoint] = [] + if let contacts { + for i in 0.. 0.1 { + lastLogStamp = timestamp + loggedCount += 1 + let p = points[0] + NSLog("MenuBand MTouch: x=%.3f y=%.3f fingers=%d", p.x, p.y, points.count) + } + DispatchQueue.main.async { [weak self] in self?.onFrame?(points) } + } + + func stop() { + guard let h = handle else { return } + func sym(_ name: String) -> UnsafeMutableRawPointer? { dlsym(h, name) } + if let stopSym = sym("MTDeviceStop") { + let stopDevice = unsafeBitCast(stopSym, to: StopFn.self) + devices.forEach { stopDevice($0) } + } + if let unregSym = sym("MTUnregisterContactFrameCallback") { + let unregister = unsafeBitCast(unregSym, to: UnregisterFn.self) + devices.forEach { unregister($0) } + } + devices.removeAll() + started = false + } +} + +#endif diff --git a/slab/menuband/Sources/MenuBand/PitchBendCursor.swift b/slab/menuband/Sources/MenuBand/PitchBendCursor.swift index 781a57c540..337476f293 100644 --- a/slab/menuband/Sources/MenuBand/PitchBendCursor.swift +++ b/slab/menuband/Sources/MenuBand/PitchBendCursor.swift @@ -22,8 +22,10 @@ enum PitchBendCursor { buildImage(bend: CGFloat(amount), echo: 0, keyDown: false) } - static func image(forBend bend: Float, echo: Float, keyDown: Bool = false) -> NSImage { - buildImage(bend: CGFloat(bend), echo: CGFloat(echo), keyDown: keyDown) + static func image(forBend bend: Float, echo: Float, keyDown: Bool = false, + touches: [CGPoint] = [], absolute: Bool = false) -> NSImage { + buildImage(bend: CGFloat(bend), echo: CGFloat(echo), keyDown: keyDown, + touches: touches, absolute: absolute) } static func cursor(forBend amount: Float) -> NSCursor { @@ -34,7 +36,8 @@ enum PitchBendCursor { NSCursor(image: image(forBend: bend, echo: echo), hotSpot: hotSpot) } - private static func buildImage(bend: CGFloat, echo: CGFloat, keyDown: Bool) -> NSImage { + private static func buildImage(bend: CGFloat, echo: CGFloat, keyDown: Bool, + touches: [CGPoint] = [], absolute: Bool = false) -> NSImage { let bendC = max(-1, min(1, bend)) // `echo` is the bipolar fx-X driver in [-1, +1]: positive // (right) is echo, negative (left) is space/reverb. We keep @@ -51,17 +54,20 @@ enum PitchBendCursor { return NSImage(size: size, flipped: false) { rect in if #available(macOS 11.0, *) { appearance.performAsCurrentDrawingAppearance { - drawChart(in: rect, bend: bendC, echo: xC, isDark: isDark, keyDown: keyDown) + drawChart(in: rect, bend: bendC, echo: xC, isDark: isDark, + keyDown: keyDown, touches: touches, absolute: absolute) } } else { - drawChart(in: rect, bend: bendC, echo: xC, isDark: isDark, keyDown: keyDown) + drawChart(in: rect, bend: bendC, echo: xC, isDark: isDark, + keyDown: keyDown, touches: touches, absolute: absolute) } return true } } private static func drawChart(in rect: NSRect, bend: CGFloat, echo: CGFloat, - isDark: Bool, keyDown: Bool) { + isDark: Bool, keyDown: Bool, + touches: [CGPoint] = [], absolute: Bool = false) { // A Menu Band keycap plate, cleanly divided into four quadrants by a // single thin cross (no axis labels, no arrowheads, no end-caps). The // puck carries the live bend (Y) / echo (X) and lights up with the @@ -138,11 +144,34 @@ enum PitchBendCursor { knob.lineWidth = keyDown ? 1.0 : 0.8 knob.stroke() - // Keycap outline last so the whole pad is framed like a key. - let edge = isDark - ? NSColor.black.withAlphaComponent(0.85) - : NSColor(srgbRed: 0.34, green: 0.28, blue: 0.18, alpha: 0.85) - edge.setStroke(); body.lineWidth = 1.3; body.stroke() + // Live trackpad touches (private MultitouchSupport tap). Each finger's + // absolute normalized position is mapped straight into the chart, so + // the pad reads as a tiny mirror of the trackpad. In absolute mode the + // dots ARE the control (puck == primary finger); in the default mode + // they're an ambient read-out of where the hand is. + if !touches.isEmpty { + let dotR: CGFloat = absolute ? 4 : 3 + for t in touches { + let px = chart.minX + max(0, min(1, CGFloat(t.x))) * chart.width + let py = chart.minY + max(0, min(1, CGFloat(t.y))) * chart.height + let dot = NSBezierPath(ovalIn: NSRect(x: px - dotR, y: py - dotR, + width: dotR * 2, height: dotR * 2)) + accent.withAlphaComponent(absolute ? 0.55 : 0.30).setFill() + dot.fill() + accent.withAlphaComponent(absolute ? 0.95 : 0.6).setStroke() + dot.lineWidth = 0.8 + dot.stroke() + } + } + + // Keycap outline last so the whole pad is framed like a key. In + // absolute mode the frame glows accent so the mode switch is obvious. + let edge = absolute + ? accent.withAlphaComponent(0.95) + : (isDark + ? NSColor.black.withAlphaComponent(0.85) + : NSColor(srgbRed: 0.34, green: 0.28, blue: 0.18, alpha: 0.85)) + edge.setStroke(); body.lineWidth = absolute ? 2 : 1.3; body.stroke() } } -- 2.51.2