From f9abe63d063c2491565b48e2bebb114954a8dd78 Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Wed, 24 Jun 2026 10:21:41 -0700 Subject: [PATCH] =?UTF-8?q?notepat/menuband:=20chromatic=20sample=20mode?= =?UTF-8?q?=20=E2=80=94=20pitch=20from=20recorded=20f0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sample playback now resamples from the sample's actual detected fundamental instead of assuming the recording is C4/A4, so each key sounds at its true note frequency. - native notepat: windowed-autocorrelation detectFundamental(); store as per-sample base (global + per-key banks); play/pitch-bend use it, with a C4 fallback for untuned/quiet captures. - menuband: mirror detector in Swift; cents = 1200*log2(targetHz/f0) into AVAudioUnitTimePitch; detect in stopRecording under bufferLock. --- fedac/native/pieces/notepat.mjs | 84 +++++++++++++++-- .../MenuBand/MenuBandSampleVoice.swift | 93 ++++++++++++++++++- 2 files changed, 164 insertions(+), 13 deletions(-) diff --git a/fedac/native/pieces/notepat.mjs b/fedac/native/pieces/notepat.mjs index 9770bf8737..d280fabead 100644 --- a/fedac/native/pieces/notepat.mjs +++ b/fedac/native/pieces/notepat.mjs @@ -55,7 +55,69 @@ let recording = false; // true while holding REC let recPointerId = null; // touch pointer currently holding REC button let recStartTime = 0; // Date.now() when recording started const MAX_REC_SECS = 10; // matches AUDIO_MAX_SAMPLE_SECS -const SAMPLE_BASE_FREQ = 261.63; // C4 — base pitch for sample playback +const SAMPLE_BASE_FREQ = 261.63; // C4 — fallback base when a sample isn't tonal + +// Estimate the fundamental frequency (Hz) of a recorded mono sample via +// windowed autocorrelation. Sample playback resamples by `freq / base`, so +// feeding the note the user *actually* recorded as `base` makes every key +// sound at its true note frequency — a chromatic sampler — instead of +// assuming the recording was C4. Returns null when the sample is too quiet +// or untuned (a drum hit, noise) so callers fall back to SAMPLE_BASE_FREQ. +function detectFundamental(data, rate, fMin = 50, fMax = 1500) { + if (!data || !rate || data.length < 2048) return null; + // Window around the loudest region so silent lead-in doesn't poison the + // estimate. Subsample the scan — we only need the rough peak location. + const N = Math.min(4096, data.length); + const scanStep = Math.max(1, Math.floor(data.length / 2048)); + let peakIdx = 0, peakAmp = 0; + for (let i = 0; i < data.length; i += scanStep) { + const a = data[i] < 0 ? -data[i] : data[i]; + if (a > peakAmp) { peakAmp = a; peakIdx = i; } + } + if (peakAmp < 0.01) return null; // effectively silent + let start = peakIdx - (N >> 1); + if (start < 0) start = 0; + if (start + N > data.length) start = data.length - N; + // De-mean + Hann window into a working buffer. + const buf = new Float32Array(N); + let mean = 0; + for (let i = 0; i < N; i++) mean += data[start + i]; + mean /= N; + let energy0 = 0; + for (let i = 0; i < N; i++) { + const w = 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / (N - 1)); + const v = (data[start + i] - mean) * w; + buf[i] = v; + energy0 += v * v; + } + if (energy0 <= 1e-9) return null; + const minLag = Math.max(2, Math.floor(rate / fMax)); + const maxLag = Math.min(N - 1, Math.floor(rate / fMin)); + if (maxLag <= minLag) return null; + // Normalized autocorrelation; take the first strong local maximum so we + // lock onto the fundamental rather than a louder higher harmonic. + const nac = new Float32Array(maxLag + 1); + for (let lag = minLag; lag <= maxLag; lag++) { + let corr = 0; + for (let i = 0; i < N - lag; i++) corr += buf[i] * buf[i + lag]; + nac[lag] = corr / energy0; + } + let bestLag = -1, bestVal = 0; + for (let lag = minLag + 1; lag < maxLag; lag++) { + if (nac[lag] > nac[lag - 1] && nac[lag] >= nac[lag + 1]) { + if (nac[lag] > bestVal) { bestVal = nac[lag]; bestLag = lag; } + if (bestVal > 0.6) break; // confident enough — first solid peak wins + } + } + if (bestLag < 0 || bestVal < 0.3) return null; // not tonal enough to trust + // Parabolic interpolation for sub-sample lag precision. + const a = nac[bestLag - 1], b = nac[bestLag], c = nac[bestLag + 1]; + const denom = a - 2 * b + c; + const shift = denom !== 0 ? (0.5 * (a - c)) / denom : 0; + const refinedLag = bestLag + shift; + const f0 = rate / refinedLag; + return f0 >= fMin && f0 <= fMax ? f0 : null; +} // Per-key sample bank: End key arms, tone key records to that key only let sampleBank = {}; // key -> { data: Float32Array, len: number, rate: number } @@ -245,7 +307,7 @@ function applyPitchShiftToActiveSounds(force = false) { for (const k of Object.keys(sounds)) { const s = sounds[k]; if (s && s.synth && s.baseFreq) { - if (s.isSample) s.synth.update({ tone: s.baseFreq * factor, base: SAMPLE_BASE_FREQ }); + if (s.isSample) s.synth.update({ tone: s.baseFreq * factor, base: s.sampleBase || SAMPLE_BASE_FREQ }); else s.synth.update({ tone: s.baseFreq * factor }); } } @@ -3124,11 +3186,12 @@ function act({ event: e, sound, wifi, system }) { sound.sample.loadData(targetSample.data, targetSample.rate); lastLoadedSample = targetSample; } + const sampleBase = targetSample?.base || SAMPLE_BASE_FREQ; const smp = sound.sample.play({ - tone: playFreq, base: SAMPLE_BASE_FREQ, volume: vol, pan, loop: true, + tone: playFreq, base: sampleBase, volume: vol, pan, loop: true, }); if (smp) { - rememberSound(key, { synth: smp, note: letter, octave: noteOctave, baseFreq: freq, isSample: true, gridOffset: offset, baseVol }, system, velocity); + rememberSound(key, { synth: smp, note: letter, octave: noteOctave, baseFreq: freq, isSample: true, sampleBase, gridOffset: offset, baseVol }, system, velocity); } else { const synth = sound.synth({ type: "sine", tone: playFreq, duration: Infinity, @@ -3189,9 +3252,12 @@ function act({ event: e, sound, wifi, system }) { // Save global sample data for bank restore const data = sound.sample.getData?.(); if (data && data.length > 0) { - globalSample = { data: new Float32Array(data), len: data.length, rate: sound.microphone?.sampleRate || 48000 }; + const buf = new Float32Array(data); + const rate = sound.microphone?.sampleRate || 48000; + const base = detectFundamental(buf, rate) || SAMPLE_BASE_FREQ; + globalSample = { data: buf, len: data.length, rate, base }; lastLoadedSample = null; // force reload on next key press - console.log(`[sample-bank] global sample saved (${data.length} samples)`); + console.log(`[sample-bank] global sample saved (${data.length} samples, base ${base.toFixed(1)}Hz)`); } return; } @@ -3217,8 +3283,10 @@ function act({ event: e, sound, wifi, system }) { percussionSampleBank[recDrum] = { data: new Float32Array(data), len: data.length, rate }; console.log(`[perc-bank] saved ${data.length} samples to drum '${recDrum}'`); } else { - sampleBank[key] = { data: new Float32Array(data), len: data.length, rate }; - console.log(`[sample-bank] saved ${data.length} samples to key '${key}'`); + const buf = new Float32Array(data); + const base = detectFundamental(buf, rate) || SAMPLE_BASE_FREQ; + sampleBank[key] = { data: buf, len: data.length, rate, base }; + console.log(`[sample-bank] saved ${data.length} samples to key '${key}' (base ${base.toFixed(1)}Hz)`); } sampleLoaded = true; // Confirmation beep diff --git a/slab/menuband/Sources/MenuBand/MenuBandSampleVoice.swift b/slab/menuband/Sources/MenuBand/MenuBandSampleVoice.swift index a150552302..0ae02a3ef9 100644 --- a/slab/menuband/Sources/MenuBand/MenuBandSampleVoice.swift +++ b/slab/menuband/Sources/MenuBand/MenuBandSampleVoice.swift @@ -59,6 +59,13 @@ final class MenuBandSampleVoice { private var recordedBuffer: AVAudioPCMBuffer? private let bufferLock = NSLock() + /// Auto-detected fundamental pitch (Hz) of the current recording. Used + /// as the chromatic playback reference so each key sounds at its true + /// note frequency rather than assuming the sample was middle C. Falls + /// back to C4 when the capture isn't tonal enough to trust. Written + /// under `bufferLock` alongside `recordedBuffer`. + private var detectedFundamental: Double = 261.63 + /// Active recording state. We tap the engine's input node into a /// scratch buffer; on `stopRecording` we trim to actual length and /// promote to `recordedBuffer`. @@ -502,14 +509,17 @@ final class MenuBandSampleVoice { return false } out.frameLength = AVAudioFrameCount(frames) + var f0: Double? = nil if let src = scratch.floatChannelData?[0], let dst = out.floatChannelData?[0] { memcpy(dst, src.advanced(by: startFrame), frames * MemoryLayout.size) let stats = shapeCapturedSample(dst, frames: frames) - NSLog("MenuBand SampleVoice: sample shaped peak \(stats.peakBefore) -> \(stats.peakAfter), rms \(stats.rmsBefore) -> \(stats.rmsAfter), gain=\(stats.gain)") + f0 = detectFundamental(dst, frames: frames, rate: sampleRate) + NSLog("MenuBand SampleVoice: sample shaped peak \(stats.peakBefore) -> \(stats.peakAfter), rms \(stats.rmsBefore) -> \(stats.rmsAfter), gain=\(stats.gain), f0=\(f0.map { String(format: "%.1fHz", $0) } ?? "untuned→C4")") } bufferLock.lock() recordedBuffer = out + detectedFundamental = f0 ?? 261.63 bufferLock.unlock() NSLog("MenuBand SampleVoice: recording captured \(frames) frames (\(Double(frames) / sampleRate) s), trimmed \(startFrame) leading frames") return true @@ -562,6 +572,74 @@ final class MenuBandSampleVoice { return (peakBefore, peakAfter, rmsBefore, rmsAfter, gain) } + /// Estimate the fundamental frequency (Hz) of the shaped mono sample via + /// windowed autocorrelation, so chromatic playback can pitch from the + /// note actually recorded instead of a fixed C4. Mirrors the native + /// notepat detector. Returns nil when the capture is too quiet or + /// untuned (a drum hit, noise) to trust — callers fall back to C4. + private func detectFundamental(_ data: UnsafeMutablePointer, frames: Int, + rate: Double, fMin: Double = 50, fMax: Double = 1500) -> Double? { + guard frames >= 2048, rate > 0 else { return nil } + let n = min(4096, frames) + // Window around the loudest region so silent lead-in/out doesn't + // poison the estimate. + let scanStep = max(1, frames / 2048) + var peakIdx = 0 + var peakAmp: Float = 0 + var i = 0 + while i < frames { + let a = abs(data[i]) + if a > peakAmp { peakAmp = a; peakIdx = i } + i += scanStep + } + if peakAmp < 0.01 { return nil } // effectively silent + var start = peakIdx - (n / 2) + if start < 0 { start = 0 } + if start + n > frames { start = frames - n } + // De-mean + Hann window into a working buffer. + var buf = [Float](repeating: 0, count: n) + var mean: Float = 0 + for j in 0.. nac[lag - 1] && nac[lag] >= nac[lag + 1] { + if nac[lag] > bestVal { bestVal = nac[lag]; bestLag = lag } + if bestVal > 0.6 { break } // confident enough — first solid peak + } + lag += 1 + } + if bestLag < 0 || bestVal < 0.3 { return nil } // not tonal enough + // Parabolic interpolation for sub-sample lag precision. + let a = nac[bestLag - 1], b = nac[bestLag], c = nac[bestLag + 1] + let denom = a - 2 * b + c + let shift: Float = denom != 0 ? 0.5 * (a - c) / denom : 0 + let refinedLag = Double(bestLag) + Double(shift) + let f0 = rate / refinedLag + return (f0 >= fMin && f0 <= fMax) ? f0 : nil + } + private func trimmedStartFrame(scratch: AVAudioPCMBuffer, frames: Int) -> Int { guard let data = scratch.floatChannelData?[0], frames > 0 else { return 0 } let window = 256 @@ -842,12 +920,17 @@ final class MenuBandSampleVoice { return v } - /// Pitch shift in CENTS for `midi` relative to middle C (60). 60 → 0. - /// 100 cents per semitone. Drives `AVAudioUnitTimePitch.pitch`, which - /// shifts pitch without touching duration/speed. + /// Pitch shift in CENTS for `midi`, chromatic from the sample's actual + /// recorded fundamental: shift = 1200·log2(targetHz / f0). So each key + /// sounds at its true note frequency regardless of what pitch was hummed + /// into the mic. Falls back to C4 (261.63 Hz) when detection failed, which + /// reduces to the old `(midi-60)·100` behavior for a C4-pitched sample. + /// Drives `AVAudioUnitTimePitch.pitch` (shifts pitch, not duration/speed). @inline(__always) private func cents(forNote midi: UInt8) -> Float { - Float(Int(midi) - 60) * 100.0 + let targetHz = 440.0 * pow(2.0, (Double(midi) - 69.0) / 12.0) + let base = detectedFundamental > 0 ? detectedFundamental : 261.63 + return Float(1200.0 * log2(targetHz / base)) } /// Combined note + trackpad-bend pitch in cents, clamped to the -- 2.51.2