Something went wrong. Try again.
twitter voice plus plus
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400use std::f32::consts::PI;
use crate::MeterFrame;
const HIGH_LIFT: f32 = 0.65;const STANDARD_RISING_RESPONSE: f32 = 0.72;const STANDARD_FALLING_RESPONSE: f32 = 0.22;const BASS_RISING_RESPONSE: f32 = STANDARD_RISING_RESPONSE / 2.0;const BASS_FALLING_RESPONSE: f32 = STANDARD_FALLING_RESPONSE / 2.0;
#[derive(Clone, Debug, Default, PartialEq)]pub struct AudioAnalysis { pub frames: Vec<MeterFrame>, pub average_hz: f32, pub min_hz: u32, pub max_hz: u32,}
pub fn normalize_amplitude(amplitude: f32) -> f32 { if amplitude <= 0.0 { return 0.0; } let decibels = 20.0 * amplitude.min(1.0).log10(); let normalized = ((decibels + 60.0) / 60.0).clamp(0.0, 1.0); normalized.powf(0.8)}
pub fn overall_energy(meter: MeterFrame) -> f32 { (meter.bass + meter.mid + meter.high) / 3.0}
pub fn high_energy_for_mapping(meter: MeterFrame) -> f32 { let low_mid_reference = (meter.bass + meter.mid) / 2.0; (meter.high + (low_mid_reference - meter.high).max(0.0) * HIGH_LIFT).min(1.0)}
pub fn low_energy_for_mapping(meter: MeterFrame) -> f32 { let mid_high_reference = (meter.mid + meter.high) / 2.0; meter.bass - (meter.bass - mid_high_reference).max(0.0) * 0.25}
#[derive(Clone, Copy)]enum MeterBand { Rms, Bass, Mid, High,}
fn smooth_meter_value(next: f32, previous: f32, band: MeterBand) -> f32 { let (rising_response, falling_response) = match band { MeterBand::Bass => (BASS_RISING_RESPONSE, BASS_FALLING_RESPONSE), MeterBand::Rms | MeterBand::Mid | MeterBand::High => { (STANDARD_RISING_RESPONSE, STANDARD_FALLING_RESPONSE) } }; let response = if next > previous { rising_response } else { falling_response }; previous + (next - previous) * response}
fn goertzel(samples: &[f32], start: usize, end: usize, sample_rate: f32, frequency: f32) -> f32 { let coefficient = 2.0 * ((2.0 * PI * frequency) / sample_rate).cos(); let mut previous = 0.0; let mut previous_previous = 0.0;
for sample in samples.get(start..end).unwrap_or_default() { let current = *sample + coefficient * previous - previous_previous; previous_previous = previous; previous = current; }
let power = previous_previous * previous_previous + previous * previous - coefficient * previous * previous_previous; power.max(0.0).sqrt() / (end.saturating_sub(start) as f32).max(1.0)}
fn band_energy( channels: &[Vec<f32>], start: usize, end: usize, sample_rate: f32, frequencies: &[f32],) -> f32 { let mut peak: f32 = 0.0; for channel in channels { for &frequency in frequencies { if frequency > 0.0 && frequency <= sample_rate / 2.0 { peak = peak.max(goertzel(channel, start, end, sample_rate, frequency)); } } } normalize_amplitude(peak * 2.5)}
fn fft_magnitudes(samples: &[f32]) -> Vec<f32> { let mut real = samples.to_vec(); let mut imaginary = vec![0.0; samples.len()]; let mut reversed = 0;
for index in 1..samples.len() { let mut bit = samples.len() >> 1; while reversed & bit != 0 { reversed ^= bit; bit >>= 1; } reversed ^= bit; if index < reversed { real.swap(index, reversed); } }
let mut length = 2; while length <= samples.len() { let angle = (-2.0 * PI) / length as f32; let phase_real = angle.cos(); let phase_imaginary = angle.sin(); for start in (0..samples.len()).step_by(length) { let mut current_real = 1.0; let mut current_imaginary = 0.0; for offset in 0..length / 2 { let even = start + offset; let odd = even + length / 2; let odd_real = real[odd] * current_real - imaginary[odd] * current_imaginary; let odd_imaginary = real[odd] * current_imaginary + imaginary[odd] * current_real; real[odd] = real[even] - odd_real; imaginary[odd] = imaginary[even] - odd_imaginary; real[even] += odd_real; imaginary[even] += odd_imaginary; let next_real = current_real * phase_real - current_imaginary * phase_imaginary; current_imaginary = current_real * phase_imaginary + current_imaginary * phase_real; current_real = next_real; } } length <<= 1; }
real.into_iter() .zip(imaginary) .take(samples.len() / 2) .map(|(real, imaginary)| { (real * real + imaginary * imaginary).sqrt() / samples.len() as f32 }) .collect()}
struct SpectralProfile { profile: Vec<f32>, fft_size: usize,}
fn spectral_profile( channels: &[Vec<f32>], sample_rate: f32, sample_count: usize,) -> SpectralProfile { let target_size = 256.max(2048.min((sample_rate / 20.0).floor() as usize)); let fft_size = target_size.next_power_of_two(); let windows = 120.min(sample_count.div_ceil(fft_size)); let mut profile = vec![0.0; fft_size / 2];
for window in 0..windows { let start = (window * sample_count.saturating_sub(fft_size)) / windows.saturating_sub(1).max(1); let mut samples = vec![0.0; fft_size]; for (offset, sample) in samples.iter_mut().enumerate() { let index = start + offset; let mut value = 0.0; for channel in channels { value += channel.get(index).copied().unwrap_or(0.0); } *sample = (value / channels.len().max(1) as f32) * (0.5 - 0.5 * ((2.0 * PI * offset as f32) / (fft_size - 1) as f32).cos()); } for (index, magnitude) in fft_magnitudes(&samples).into_iter().enumerate() { profile[index] += magnitude / windows.max(1) as f32; } }
SpectralProfile { profile, fft_size }}
fn average_frequency(profile: &SpectralProfile, sample_rate: f32) -> f32 { let mut weighted = 0.0; let mut total = 0.0; for (index, energy) in profile.profile.iter().copied().enumerate() { let frequency = index as f32 * sample_rate / profile.fft_size as f32; if !(60.0..=12_000.0).contains(&frequency) { continue; } weighted += frequency * energy; total += energy; } if total > 0.0 { weighted / total } else { 0.0 }}
fn frequency_range(profile: &SpectralProfile, sample_rate: f32) -> (u32, u32) { let peak = profile.profile.iter().skip(2).copied().fold(0.0, f32::max); if peak == 0.0 { return (0, 0); } let threshold = peak * 0.02; let mut min_frequency = 0.0; let mut max_frequency = 0.0; for (index, energy) in profile.profile.iter().copied().enumerate() { let frequency = index as f32 * sample_rate / profile.fft_size as f32; if frequency >= 40.0 && energy >= threshold { if min_frequency == 0.0 { min_frequency = frequency; } max_frequency = frequency; } } (min_frequency.round() as u32, max_frequency.round() as u32)}
fn rms_energy(channels: &[Vec<f32>], start: usize, end: usize) -> f32 { let (sum_squares, sample_count) = channels.iter().fold((0.0, 0), |(sum, count), channel| { let samples = channel.get(start..end).unwrap_or_default(); ( sum + samples.iter().map(|sample| sample * sample).sum::<f32>(), count + samples.len(), ) }); normalize_amplitude((sum_squares / sample_count.max(1) as f32).sqrt())}
fn meter_for_frame( channels: &[Vec<f32>], start: usize, end: usize, sample_rate: f32, center_frequency: f32,) -> MeterFrame { MeterFrame { rms: rms_energy(channels, start, end), bass: band_energy( channels, start, end, sample_rate, &[ 70.0, 100.0, 140.0, 180.0, center_frequency * 0.45, center_frequency * 0.65, ], ), mid: band_energy( channels, start, end, sample_rate, &[ 250.0, 400.0, 700.0, (center_frequency * 0.8).max(250.0), center_frequency.max(300.0), (center_frequency * 1.2).max(500.0), 1_500.0, 2_000.0, ], ), high: band_energy( channels, start, end, sample_rate, &[ 2_500.0, 4_000.0, 6_000.0, 8_000.0, (center_frequency * 1.6).max(2_500.0), (center_frequency * 2.2).max(3_000.0), ], ), }}
fn frame_peak(frames: &[MeterFrame], value: impl Fn(MeterFrame) -> f32) -> f32 { frames.iter().copied().map(value).fold(1e-6, f32::max)}
fn normalize_frame(frame: MeterFrame, peaks: MeterFrame) -> MeterFrame { MeterFrame { rms: (frame.rms / peaks.rms).powf(1.35), bass: (frame.bass / peaks.bass).powf(1.35), mid: (frame.mid / peaks.mid).powf(1.35), high: (frame.high / peaks.high).powf(1.35), }}
fn smooth_frame(next: MeterFrame, previous: MeterFrame) -> MeterFrame { MeterFrame { rms: smooth_meter_value(next.rms, previous.rms, MeterBand::Rms), bass: smooth_meter_value(next.bass, previous.bass, MeterBand::Bass), mid: smooth_meter_value(next.mid, previous.mid, MeterBand::Mid), high: smooth_meter_value(next.high, previous.high, MeterBand::High), }}
fn normalize_and_smooth(frames: &mut [MeterFrame]) { let peaks = MeterFrame { rms: frame_peak(frames, |frame| frame.rms), bass: frame_peak(frames, |frame| frame.bass), mid: frame_peak(frames, |frame| frame.mid), high: frame_peak(frames, |frame| frame.high), }; let mut previous = MeterFrame::default(); for frame in frames { previous = smooth_frame(normalize_frame(*frame, peaks), previous); *frame = previous; }}
pub fn analyse_audio(channels: &[Vec<f32>], sample_rate: f32, fps: f32) -> AudioAnalysis { let sample_count = channels.iter().map(Vec::len).max().unwrap_or(0); if sample_count == 0 || sample_rate <= 0.0 || fps <= 0.0 { return AudioAnalysis::default(); }
let samples_per_frame = sample_rate / fps; let frame_count = (sample_count as f32 / samples_per_frame).ceil() as usize; let profile = spectral_profile(channels, sample_rate, sample_count); let clip_average_hz = average_frequency(&profile, sample_rate); let (min_hz, max_hz) = frequency_range(&profile, sample_rate); let center_frequency = if clip_average_hz > 0.0 { clip_average_hz } else { 500.0 }; let mut frames: Vec<_> = (0..frame_count) .map(|frame| { let start = (frame as f32 * samples_per_frame).floor() as usize; let end = sample_count.min(((frame + 1) as f32 * samples_per_frame).floor() as usize); meter_for_frame(channels, start, end, sample_rate, center_frequency) }) .collect(); normalize_and_smooth(&mut frames);
AudioAnalysis { frames, average_hz: clip_average_hz, min_hz, max_hz, }}
#[cfg(test)]mod tests { use super::*;
fn sine(frequency: f32, sample_rate: f32, seconds: f32) -> Vec<f32> { let length = (sample_rate * seconds) as usize; (0..length) .map(|index| (2.0 * PI * frequency * index as f32 / sample_rate).sin() * 0.5) .collect() }
#[test] fn normalizes_amplitude_to_the_expected_range() { assert_eq!(normalize_amplitude(0.0), 0.0); assert!(normalize_amplitude(0.5) > normalize_amplitude(0.1)); assert!(normalize_amplitude(2.0) <= 1.0); }
#[test] fn smooths_bass_with_half_the_response_of_other_bands() { assert!((smooth_meter_value(1.0, 0.0, MeterBand::Bass) - 0.36).abs() < 1e-6); assert!((smooth_meter_value(1.0, 0.0, MeterBand::Mid) - 0.72).abs() < 1e-6); assert!((smooth_meter_value(0.0, 1.0, MeterBand::Bass) - 0.89).abs() < 1e-6); assert!((smooth_meter_value(0.0, 1.0, MeterBand::High) - 0.78).abs() < 1e-6); }
#[test] fn detects_a_high_frequency_peak() { let result = analyse_audio(&[sine(16_000.0, 48_000.0, 0.2)], 48_000.0, 30.0); assert!(result.average_hz <= 12_000.0); assert!(result.max_hz > 12_000); }
#[test] fn silence_has_no_frequency_range() { let result = analyse_audio(&[vec![0.0; 4_800]], 48_000.0, 30.0); assert_eq!((result.min_hz, result.max_hz), (0, 0)); assert!(result.frames.iter().all(|frame| frame.rms == 0.0)); }}