From 5e29ab7e4ce667821c444f8296d477b7149c4a72 Mon Sep 17 00:00:00 2001 From: Steven Vandevelde Date: Mon, 3 Aug 2026 17:50:45 +0200 Subject: [PATCH] feat: spectogram analysis --- deno.jsonc | 5 + lexicons/output/track.json | 5 + .../metadata/spectrogram/types.d.ts | 26 + src/_data/elements.js | 12 + src/_data/facets.json | 8 + src/common/foundation.js | 39 ++ .../metadata/spectrogram/element.js | 68 +++ src/components/metadata/spectrogram/fft.js | 458 ++++++++++++++++++ src/components/metadata/spectrogram/worker.js | 179 +++++++ .../orchestrator/spectrogram-audio/element.js | 195 ++++++++ src/facets/data/spectrogram/index.html | 1 + src/facets/data/spectrogram/index.inline.js | 14 + 12 files changed, 1010 insertions(+) create mode 100644 specs/components/metadata/spectrogram/types.d.ts create mode 100644 src/components/metadata/spectrogram/element.js create mode 100644 src/components/metadata/spectrogram/fft.js create mode 100644 src/components/metadata/spectrogram/worker.js create mode 100644 src/components/orchestrator/spectrogram-audio/element.js create mode 100644 src/facets/data/spectrogram/index.html create mode 100644 src/facets/data/spectrogram/index.inline.js diff --git a/deno.jsonc b/deno.jsonc index f23adac9..773a6b55 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -154,6 +154,9 @@ "./components/metadata/audio-file/element.js": "./src/components/metadata/audio-file/element.js", "./components/metadata/audio-file/worker.js": "./src/components/metadata/audio-file/worker.js", "./components/metadata/common.js": "./src/components/metadata/common.js", + "./components/metadata/spectrogram/element.js": "./src/components/metadata/spectrogram/element.js", + "./components/metadata/spectrogram/fft.js": "./src/components/metadata/spectrogram/fft.js", + "./components/metadata/spectrogram/worker.js": "./src/components/metadata/spectrogram/worker.js", "./components/orchestrator/artwork/element.js": "./src/components/orchestrator/artwork/element.js", "./components/orchestrator/artwork/worker.js": "./src/components/orchestrator/artwork/worker.js", "./components/orchestrator/auto-queue/element.js": "./src/components/orchestrator/auto-queue/element.js", @@ -170,6 +173,7 @@ "./components/orchestrator/scoped-tracks/worker.js": "./src/components/orchestrator/scoped-tracks/worker.js", "./components/orchestrator/scrobble-audio/element.js": "./src/components/orchestrator/scrobble-audio/element.js", "./components/orchestrator/sources/element.js": "./src/components/orchestrator/sources/element.js", + "./components/orchestrator/spectrogram-audio/element.js": "./src/components/orchestrator/spectrogram-audio/element.js", "./components/output/bytes/s3/constants.js": "./src/components/output/bytes/s3/constants.js", "./components/output/bytes/s3/element.js": "./src/components/output/bytes/s3/element.js", "./components/output/bytes/s3/worker.js": "./src/components/output/bytes/s3/worker.js", @@ -220,6 +224,7 @@ "./components/input/types.d.ts": "./specs/components/input/types.d.ts", "./components/input/webdav/types.d.ts": "./specs/components/input/webdav/types.d.ts", "./components/metadata/audio-file/types.d.ts": "./specs/components/metadata/audio-file/types.d.ts", + "./components/metadata/spectrogram/types.d.ts": "./specs/components/metadata/spectrogram/types.d.ts", "./components/metadata/types.d.ts": "./specs/components/metadata/types.d.ts", "./components/orchestrator/artwork/types.d.ts": "./specs/components/orchestrator/artwork/types.d.ts", "./components/orchestrator/favourites/types.d.ts": "./specs/components/orchestrator/favourites/types.d.ts", diff --git a/lexicons/output/track.json b/lexicons/output/track.json index 8222c660..f0c04ca0 100644 --- a/lexicons/output/track.json +++ b/lexicons/output/track.json @@ -51,6 +51,11 @@ "lossless": { "type": "boolean", "description": "Is track lossless" }, "numberOfChannels": { "type": "integer", "description": "Number of audio channels" }, "sampleRate": { "type": "integer", "description": "Samples per second" }, + "spectralCentroid": { "type": "integer", "description": "Average spectral centroid in Hz, the perceptual brightness of the track derived from its spectrogram" }, + "spectralFlatness": { "type": "integer", "description": "Average spectral flatness on a 0-1000 scale (0 = fully tonal, 1000 = fully noisy), derived from the track's spectrogram" }, + "spectralFlux": { "type": "integer", "description": "Average spectral flux scaled by 1000, measuring how quickly the spectrum changes over time in the track's spectrogram" }, + "spectralRolloff": { "type": "integer", "description": "Average spectral rolloff in Hz, the frequency below which 85% of the spectral energy is contained, derived from the track's spectrogram" }, + "spectralSpread": { "type": "integer", "description": "Average spectral spread in Hz, the standard deviation of the spectrum around its centroid, derived from the track's spectrogram" }, "trackGain": { "type": "integer", "description": "Track gain in dB" } } }, diff --git a/specs/components/metadata/spectrogram/types.d.ts b/specs/components/metadata/spectrogram/types.d.ts new file mode 100644 index 00000000..4c3f31bc --- /dev/null +++ b/specs/components/metadata/spectrogram/types.d.ts @@ -0,0 +1,26 @@ +import type { TrackStats } from "~/definitions/types.d.ts"; + +/** + * The spectral descriptors derived from a track's spectrogram. These are the + * spectrogram-analysis fields stored on `TrackStats`. + */ +export type SpectralStats = { + spectralCentroid: number; + spectralRolloff: number; + spectralSpread: number; + spectralFlatness: number; + spectralFlux: number; +}; + +/** + * The spectrogram component re-uses the standard metadata `patch` action, + * merging its spectral stats into the track's existing `stats`. + */ +export type SpectralTrackStats = Pick< + TrackStats, + | "spectralCentroid" + | "spectralRolloff" + | "spectralSpread" + | "spectralFlatness" + | "spectralFlux" +>; diff --git a/src/_data/elements.js b/src/_data/elements.js index 52a5147c..7a42dea0 100644 --- a/src/_data/elements.js +++ b/src/_data/elements.js @@ -163,6 +163,12 @@ export default { desc: "Extracts tags and audio stats from audio files using the music-metadata library.", }, + { + url: "components/metadata/spectrogram/element.js", + title: "Spectrogram", + desc: + "Decodes a track's audio, computes a spectrogram and stores the derived spectral descriptors (centroid, rolloff, spread, flatness, flux) on the track's stats.", + }, ], orchestrators: [ @@ -242,6 +248,12 @@ export default { desc: "Monitor tracks from the given output to form a list of sources based on the input's sources return value.", }, + { + url: "components/orchestrator/spectrogram-audio/element.js", + title: "Spectrogram ⭤ Audio", + desc: + "When a track starts playing and is missing spectrogram-derived stats, analyses it via the spectrogram metadata element and persists the result to the output. Only the leader tab performs the analysis.", + }, ], output: [ diff --git a/src/_data/facets.json b/src/_data/facets.json index aae87b72..99916097 100644 --- a/src/_data/facets.json +++ b/src/_data/facets.json @@ -258,6 +258,14 @@ "featured": true, "desc": "An overview of all audio inputs. Disable a source to hide its tracks without removing them." }, + { + "url": "facets/data/spectrogram/index.html", + "title": "Spectrogram Analysis", + "kind": "prelude", + "category": "Data", + "featured": true, + "desc": "Analyse the spectrogram of tracks as they play and store the data on their stats. Only runs for tracks that don't already have spectral stats." + }, { "url": "facets/misc/split-view/index.html", "title": "Split View", diff --git a/src/common/foundation.js b/src/common/foundation.js index b3515c85..5c95dd2c 100644 --- a/src/common/foundation.js +++ b/src/common/foundation.js @@ -86,6 +86,9 @@ const signals = { sources: signal( /** @type {import("~/components/orchestrator/sources/element.js").CLASS | null} */ (null), ), + spectrogramAudio: signal( + /** @type {import("~/components/orchestrator/spectrogram-audio/element.js").CLASS | null} */ (null), + ), }, }; @@ -125,6 +128,7 @@ export const config = { scopedTracks, scrobbleAudio, sources, + spectrogramAudio, }, /** @@ -160,6 +164,7 @@ export const config = { scopedTracks: signals.orchestrator.scopedTracks.get, scrobbleAudio: signals.orchestrator.scrobbleAudio.get, sources: signals.orchestrator.sources.get, + spectrogramAudio: signals.orchestrator.spectrogramAudio.get, }, }, @@ -509,6 +514,40 @@ async function scrobbleAudio() { return findExistingOrAdd(sao, signals.orchestrator.scrobbleAudio); } +async function spectrogramAudio() { + const [ + { CLASS: SpectrogramAudioOrchestrator }, + { CLASS: SpectrogramMetadata }, + a, + i, + o, + ] = await Promise.all([ + import("~/components/orchestrator/spectrogram-audio/element.js"), + import("~/components/metadata/spectrogram/element.js"), + audio(), + input(), + output(), + ]); + + // A standalone spectrogram metadata element the orchestrator can call + // directly. It's given the input configurator so its worker can fetch audio. + const sm = new SpectrogramMetadata(); + sm.setAttribute("group", GROUP); + sm.setAttribute("input-selector", i.selector); + + if (!document.body.querySelector(sm.selector)) { + document.body.append(sm); + } + + const sao = new SpectrogramAudioOrchestrator(); + sao.setAttribute("group", GROUP); + sao.setAttribute("audio-engine-selector", a.selector); + sao.setAttribute("spectrogram-selector", sm.selector); + sao.setAttribute("output-selector", o.selector); + + return findExistingOrAdd(sao, signals.orchestrator.spectrogramAudio); +} + async function pathCollections() { const [{ CLASS: PathCollectionsOrchestrator }, o] = await Promise.all([ import("~/components/orchestrator/path-collections/element.js"), diff --git a/src/components/metadata/spectrogram/element.js b/src/components/metadata/spectrogram/element.js new file mode 100644 index 00000000..23a36b46 --- /dev/null +++ b/src/components/metadata/spectrogram/element.js @@ -0,0 +1,68 @@ +import { defineElement, DiffuseElement, query } from "~/common/element.js"; + +/** + * @import {ProxiedActions} from "~/common/worker.d.ts" + * @import {InputElement} from "@specs/components/input/types.d.ts" + * @import {Actions} from "@specs/components/metadata/types.d.ts" + */ + +//////////////////////////////////////////// +// ELEMENT +//////////////////////////////////////////// + +/** + * Computes a spectrogram for a track and stores the derived spectral + * descriptors on the track's `stats`. + * + * Like the audio-file metadata element, it exposes a `patch(track)` action and + * is designed to be a child of the `` configurator, where it runs + * alongside `` during track processing. + * + * @implements {ProxiedActions} + */ +class SpectrogramMetadata extends DiffuseElement { + static NAME = "diffuse/metadata/spectrogram"; + static WORKER_URL = "components/metadata/spectrogram/worker.js"; + + constructor() { + super(); + + /** @type {ProxiedActions} */ + const p = this.workerProxy(); + + this.patch = p.patch; + } + + // LIFECYCLE + + /** @override */ + async connectedCallback() { + super.connectedCallback(); + + /** @type {InputElement} */ + this.input = query(this, "input-selector"); + + await customElements.whenDefined(this.input.localName); + } + + // WORKERS + + /** + * @override + */ + dependencies() { + if (!this.input) throw new Error("Input element not defined yet"); + return { input: this.input }; + } +} + +export default SpectrogramMetadata; + +//////////////////////////////////////////// +// REGISTER +//////////////////////////////////////////// + +export const CLASS = SpectrogramMetadata; +export const NAME = "dm-spectrogram"; + +defineElement(NAME, SpectrogramMetadata); diff --git a/src/components/metadata/spectrogram/fft.js b/src/components/metadata/spectrogram/fft.js new file mode 100644 index 00000000..e349f893 --- /dev/null +++ b/src/components/metadata/spectrogram/fft.js @@ -0,0 +1,458 @@ +/** + * Radix-2 iterative Cooley-Tukey FFT and the spectral descriptors used by the + * spectrogram metadata component. + * + * Everything here is pure (no DOM / worker APIs) so it can be unit-tested with + * documentation tests, matching the rest of the codebase. + */ + +/** + * @import { SpectralStats } from "@specs/components/metadata/spectrogram/types.d.ts" + */ + +//////////////////////////////////////////// +// FFT +//////////////////////////////////////////// + +/** + * In-place radix-2 Cooley-Tukey FFT. Mutates the provided real/imag arrays. + * + * `re` and `im` must be the same length, and the length must be a power of two. + * Pass a negative `inverse` scale for the inverse transform. + * + * @param {Float32Array} re + * @param {Float32Array} im + * @param {number} n + * @param {number} direction `1` for forward, `-1` for inverse. + * + * @example A pure DC signal transforms to a single non-zero bin at index 0 + * ```js + * import { fft } from "~/components/metadata/spectrogram/fft.js"; + * + * const n = 8; + * const re = new Float32Array(n); + * const im = new Float32Array(n); + * for (let i = 0; i < n; i++) re[i] = 1; // constant signal + * fft(re, im, n, 1); + * + * // DC bin carries the average, all other bins are ~0 + * if (Math.abs(re[0] - 1) > 1e-6) throw new Error("DC bin should be 1"); + * for (let i = 1; i < n; i++) { + * if (Math.abs(re[i]) > 1e-6 || Math.abs(im[i]) > 1e-6) { + * throw new Error("non-DC bins should be ~0"); + * } + * } + * ``` + * + * @example A sinusoid at bin k produces peaks at bins k and n-k + * ```js + * import { fft } from "~/components/metadata/spectrogram/fft.js"; + * + * const n = 16; + * const k = 3; + * const re = new Float32Array(n); + * const im = new Float32Array(n); + * for (let i = 0; i < n; i++) re[i] = Math.cos((2 * Math.PI * k * i) / n); + * fft(re, im, n, 1); + * + * // magnitude peak at k and n-k + * if (Math.hypot(re[k], im[k]) < 0.4) throw new Error("expected peak at bin " + k); + * if (Math.hypot(re[n - k], im[n - k]) < 0.4) throw new Error("expected peak at bin " + (n - k)); + * ``` + */ +export function fft(re, im, n, direction) { + // Bit-reversal permutation + for (let i = 1, j = 0; i < n; i++) { + let bit = n >> 1; + for (; j & bit; bit >>= 1) j ^= bit; + j ^= bit; + + if (i < j) { + const tr = re[i]; + re[i] = re[j]; + re[j] = tr; + const ti = im[i]; + im[i] = im[j]; + im[j] = ti; + } + } + + // Butterflies + for (let len = 2; len <= n; len <<= 1) { + const half = len >> 1; + const angle = (direction * 2 * Math.PI) / len; + const wRe = Math.cos(angle); + const wIm = Math.sin(angle); + + for (let i = 0; i < n; i += len) { + let curRe = 1; + let curIm = 0; + + for (let k = 0; k < half; k++) { + const aRe = re[i + k]; + const aIm = im[i + k]; + const bRe = re[i + k + half]; + const bIm = im[i + k + half]; + const tRe = curRe * bRe - curIm * bIm; + const tIm = curRe * bIm + curIm * bRe; + + re[i + k] = aRe + tRe; + im[i + k] = aIm + tIm; + re[i + k + half] = aRe - tRe; + im[i + k + half] = aIm - tIm; + + const nextRe = curRe * wRe - curIm * wIm; + curIm = curRe * wIm + curIm * wRe; + curRe = nextRe; + } + } + } + + // Forward transform: divide by n to keep magnitudes in a comparable range. + if (direction === 1) { + for (let i = 0; i < n; i++) { + re[i] /= n; + im[i] /= n; + } + } +} + +//////////////////////////////////////////// +// WINDOW +//////////////////////////////////////////// + +/** + * A Hann window of length `n`. Used to reduce spectral leakage at frame edges. + * + * @param {number} n + * @returns {Float32Array} + * + * @example Endpoint values are zero, the midpoint (odd length) is 1 + * ```js + * import { hannWindow } from "~/components/metadata/spectrogram/fft.js"; + * + * const w = hannWindow(9); // odd length -> sample lands on the peak + * if (w[0] > 1e-6 || w[8] > 1e-6) throw new Error("Hann endpoints should be ~0"); + * if (Math.abs(w[4] - 1) > 1e-6) throw new Error("Hann midpoint should be 1"); + * ``` + */ +export function hannWindow(n) { + const w = new Float32Array(n); + for (let i = 0; i < n; i++) { + w[i] = 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / (n - 1)); + } + return w; +} + +//////////////////////////////////////////// +// SPECTRAL DESCRIPTORS +//////////////////////////////////////////// + +/** + * Frequency in Hz of FFT bin `i` for a given sample rate and FFT size. + * + * @param {number} i + * @param {number} sampleRate + * @param {number} fftSize + * @returns {number} + */ +function binFrequency(i, sampleRate, fftSize) { + return (i * sampleRate) / fftSize; +} + +/** + * Spectral centroid: the magnitude-weighted mean frequency, a measure of + * perceptual brightness. + * + * @param {Float32Array} magnitude + * @param {number} sampleRate + * @param {number} fftSize + * @returns {number} + * + * @example A single-bin spectrum has its centroid at that bin's frequency + * ```js + * import { spectralCentroid } from "~/components/metadata/spectrogram/fft.js"; + * + * const fftSize = 8; + * const sampleRate = 8; + * const mag = new Float32Array(fftSize / 2 + 1); + * mag[2] = 1; // all energy at bin 2 -> 2 Hz + * if (Math.abs(spectralCentroid(mag, sampleRate, fftSize) - 2) > 1e-6) { + * throw new Error("centroid should equal the single bin's frequency"); + * } + * ``` + */ +export function spectralCentroid(magnitude, sampleRate, fftSize) { + let weighted = 0; + let total = 0; + + for (let i = 0; i < magnitude.length; i++) { + const m = magnitude[i]; + if (m <= 0) continue; + weighted += binFrequency(i, sampleRate, fftSize) * m; + total += m; + } + + return total > 0 ? weighted / total : 0; +} + +/** + * Spectral spread: the standard deviation of the spectrum around its centroid, + * describing how concentrated or dispersed the spectral energy is. + * + * @param {Float32Array} magnitude + * @param {number} sampleRate + * @param {number} fftSize + * @returns {number} + * + * @example A single-bin spectrum has zero spread + * ```js + * import { spectralSpread } from "~/components/metadata/spectrogram/fft.js"; + * + * const fftSize = 8; + * const sampleRate = 8; + * const mag = new Float32Array(fftSize / 2 + 1); + * mag[2] = 1; + * if (spectralSpread(mag, sampleRate, fftSize) > 1e-6) { + * throw new Error("a single-bin spectrum should have zero spread"); + * } + * ``` + */ +export function spectralSpread(magnitude, sampleRate, fftSize) { + const centroid = spectralCentroid(magnitude, sampleRate, fftSize); + + let weighted = 0; + let total = 0; + + for (let i = 0; i < magnitude.length; i++) { + const m = magnitude[i]; + if (m <= 0) continue; + const f = binFrequency(i, sampleRate, fftSize); + weighted += m * (f - centroid) ** 2; + total += m; + } + + return total > 0 ? Math.sqrt(weighted / total) : 0; +} + +/** + * Spectral rolloff: the lowest frequency below which `threshold` (0-1) of the + * total spectral energy is contained. + * + * @param {Float32Array} magnitude + * @param {number} sampleRate + * @param {number} fftSize + * @param {number} threshold Fraction of cumulative energy, e.g. 0.85. + * @returns {number} + * + * @example All energy in the first bin -> rolloff at bin 0's frequency + * ```js + * import { spectralRolloff } from "~/components/metadata/spectrogram/fft.js"; + * + * const fftSize = 8; + * const sampleRate = 8; + * const mag = new Float32Array(fftSize / 2 + 1); + * mag[0] = 1; + * if (spectralRolloff(mag, sampleRate, fftSize, 0.85) > 1e-6) { + * throw new Error("rolloff should be 0 when all energy is at bin 0"); + * } + * ``` + */ +export function spectralRolloff(magnitude, sampleRate, fftSize, threshold) { + let total = 0; + for (let i = 0; i < magnitude.length; i++) total += magnitude[i]; + if (total <= 0) return 0; + + const target = total * threshold; + let cumulative = 0; + + for (let i = 0; i < magnitude.length; i++) { + cumulative += magnitude[i]; + if (cumulative >= target) { + return binFrequency(i, sampleRate, fftSize); + } + } + + return binFrequency(magnitude.length - 1, sampleRate, fftSize); +} + +/** + * Spectral flatness: geometric mean / arithmetic mean of the (positive) + * magnitudes. Ranges from 0 (completely tonal) to 1 (completely noisy). + * + * @param {Float32Array} magnitude + * @returns {number} + * + * @example A uniform spectrum is maximally flat (1) + * ```js + * import { spectralFlatness } from "~/components/metadata/spectrogram/fft.js"; + * + * const mag = new Float32Array(5).fill(2); + * if (Math.abs(spectralFlatness(mag) - 1) > 1e-6) { + * throw new Error("a uniform spectrum should have flatness 1"); + * ``` + * + * @example Concentrated (tonal) energy yields a flatness near 0 + * ```js + * import { spectralFlatness } from "~/components/metadata/spectrogram/fft.js"; + * + * const mag = new Float32Array([1000, 1, 1, 1, 1]); // one dominant bin + * if (spectralFlatness(mag) > 0.1) { + * throw new Error("a tonal spectrum should have flatness near 0"); + * } + * ``` + */ +export function spectralFlatness(magnitude) { + let count = 0; + let logSum = 0; + let sum = 0; + + for (let i = 0; i < magnitude.length; i++) { + const m = magnitude[i]; + if (m <= 0) continue; + count++; + logSum += Math.log(m); + sum += m; + } + + if (count === 0 || sum === 0) return 0; + const geometricMean = Math.exp(logSum / count); + const arithmeticMean = sum / count; + return arithmeticMean > 0 ? geometricMean / arithmeticMean : 0; +} + +/** + * Per-frame spectral flux: the sum of *positive* magnitude differences between + * consecutive frames, measuring how much new spectral energy appears. + * + * @param {Float32Array} prev Previous frame magnitudes. + * @param {Float32Array} curr Current frame magnitudes. + * @returns {number} + * + * @example Identical frames produce zero flux + * ```js + * import { spectralFlux } from "~/components/metadata/spectrogram/fft.js"; + * + * const m = new Float32Array(4).fill(1); + * if (spectralFlux(m, m) > 1e-6) { + * throw new Error("identical frames should have zero flux"); + * } + * ``` + * + * @example Only positive differences count (decays are ignored) + * ```js + * import { spectralFlux } from "~/components/metadata/spectrogram/fft.js"; + * + * const prev = new Float32Array([1, 1, 1, 1]); + * const curr = new Float32Array([2, 0, 0, 0]); // +1 at bin 0, rest decrease + * if (Math.abs(spectralFlux(prev, curr) - 1) > 1e-6) { + * throw new Error("only positive differences should be counted"); + * } + * ``` + */ +export function spectralFlux(prev, curr) { + let sum = 0; + const len = Math.min(prev.length, curr.length); + for (let i = 0; i < len; i++) { + const diff = curr[i] - prev[i]; + if (diff > 0) sum += diff * diff; + } + return sum; +} + +//////////////////////////////////////////// +// SPECTROGRAM ANALYSIS +//////////////////////////////////////////// + +/** + * Computes a magnitude spectrogram from mono PCM samples via the STFT, then + * reduces it to a compact set of spectral descriptors suitable for storing on + * `TrackStats`. + * + * @param {Float32Array} samples Mono PCM samples. + * @param {number} sampleRate Sample rate of `samples` (Hz). + * @returns {SpectralStats} + * + * @example Silence yields all-zero descriptors + * ```js + * import { analyseSpectrogram } from "~/components/metadata/spectrogram/fft.js"; + * + * const stats = analyseSpectrogram(new Float32Array(4096), 22050); + * if (stats.spectralCentroid !== 0) throw new Error("silence: centroid should be 0"); + * if (stats.spectralFlatness !== 0) throw new Error("silence: flatness should be 0"); + * if (stats.spectralFlux !== 0) throw new Error("silence: flux should be 0"); + * ``` + */ +export function analyseSpectrogram(samples, sampleRate) { + const fftSize = 2048; + const hopSize = 1024; + const window = hannWindow(fftSize); + const bins = fftSize / 2 + 1; + + const re = new Float32Array(fftSize); + const im = new Float32Array(fftSize); + + /** @type {Float32Array[]} */ + const frames = []; + /** @type {number[]} */ + const centroids = []; + /** @type {number[]} */ + const spreads = []; + /** @type {number[]} */ + const rolloffs = []; + /** @type {number[]} */ + const flatnesses = []; + /** @type {number[]} */ + const fluxes = []; + + /** @type {Float32Array | null} */ + let prevMag = null; + + for (let start = 0; start + fftSize <= samples.length; start += hopSize) { + for (let i = 0; i < fftSize; i++) { + re[i] = samples[start + i] * window[i]; + im[i] = 0; + } + + fft(re, im, fftSize, 1); + + const mag = new Float32Array(bins); + for (let i = 0; i < bins; i++) { + mag[i] = Math.hypot(re[i], im[i]); + } + frames.push(mag); + + centroids.push(spectralCentroid(mag, sampleRate, fftSize)); + spreads.push(spectralSpread(mag, sampleRate, fftSize)); + rolloffs.push(spectralRolloff(mag, sampleRate, fftSize, 0.85)); + flatnesses.push(spectralFlatness(mag)); + + if (prevMag) { + fluxes.push(spectralFlux(prevMag, mag)); + } + prevMag = mag; + } + + if (frames.length === 0) { + return { + spectralCentroid: 0, + spectralRolloff: 0, + spectralSpread: 0, + spectralFlatness: 0, + spectralFlux: 0, + }; + } + + const mean = (/** @type {number[]} */ xs) => + xs.reduce((a, b) => a + b, 0) / xs.length; + + return { + spectralCentroid: Math.round(mean(centroids)), + spectralRolloff: Math.round(mean(rolloffs)), + spectralSpread: Math.round(mean(spreads)), + // Flatness is 0-1; store as a 0-1000 integer (per-mille). + spectralFlatness: Math.round(mean(flatnesses) * 1000), + // Flux is a small float; scale by 1000 for integer storage. + spectralFlux: Math.round(mean(fluxes.length ? fluxes : [0]) * 1000), + }; +} diff --git a/src/components/metadata/spectrogram/worker.js b/src/components/metadata/spectrogram/worker.js new file mode 100644 index 00000000..15fc9b26 --- /dev/null +++ b/src/components/metadata/spectrogram/worker.js @@ -0,0 +1,179 @@ +import { ostiary, rpc, workerProxy } from "~/common/worker.js"; +import { removeUndefinedValuesFromRecord } from "~/common/utils.js"; +import { analyseSpectrogram } from "./fft.js"; + +/** + * @import {Track, TrackStats} from "~/definitions/types.d.ts" + * @import {ActionsWithTunnel, ProxiedActions} from "~/common/worker.d.ts" + * @import {InputActions} from "@specs/components/input/types.d.ts" + * @import {Actions} from "@specs/components/metadata/types.d.ts" + */ + +// Spectrogram analysis is a frequency-domain operation, so the analysis sample +// rate doesn't need to exceed the Nyquist frequency of the highest band we care +// about. 22.05 kHz covers the full audible range and keeps FFT frames cheap. +const ANALYSIS_SAMPLE_RATE = 22050; + +// Skip analysis for tracks longer than this (milliseconds). Spectrogram +// analysis decodes the *entire* file, so very long tracks are prohibitively +// expensive. The existing audio-file metadata already captured their other +// stats; we just skip the spectral descriptors. +const MAX_DURATION_MS = 4 * 60 * 60 * 1000; // 4 hours + +const SPECTRAL_KEYS = [ + "spectralCentroid", + "spectralRolloff", + "spectralSpread", + "spectralFlatness", + "spectralFlux", +]; + +/** + * @param {TrackStats | undefined} stats + * @returns {boolean} + */ +function hasSpectralStats(stats) { + if (!stats) return false; + return SPECTRAL_KEYS.every((k) => stats[/** @type {keyof TrackStats} */ (k)] != null); +} + +/** + * Reads a `ReadableStream` into a single `ArrayBuffer`. + * + * @param {ReadableStream} stream + * @returns {Promise} + */ +async function readStreamToArrayBuffer(stream) { + const reader = stream.getReader(); + /** @type {Uint8Array[]} */ + const chunks = []; + let total = 0; + + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (value) { + chunks.push(value); + total += value.byteLength; + } + } + + const result = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + return result.buffer; +} + +/** + * Decodes an encoded audio `ArrayBuffer` into mono PCM samples at a fixed + * analysis sample rate, using an `OfflineAudioContext`. + * + * Returns `null` when decoding is unavailable (e.g. older browsers that don't + * expose `OfflineAudioContext` inside a worker) so the caller can gracefully + * skip spectral analysis without losing the rest of the track's stats. + * + * @param {ArrayBuffer} arrayBuffer + * @returns {Promise<{ samples: Float32Array; sampleRate: number } | null>} + */ +async function decodeToMono(arrayBuffer) { + /** @type {typeof OfflineAudioContext | undefined} */ + const Ctx = globalThis.OfflineAudioContext ?? + /** @type {any} */ (globalThis).webkitOfflineAudioContext; + if (!Ctx) return null; + + // A throwaway context is enough — `decodeAudioData` resamples to the + // context's sample rate, giving us a consistent rate to analyse at. + /** @type {OfflineAudioContext} */ + const ctx = new Ctx(1, 1, ANALYSIS_SAMPLE_RATE); + + let audioBuffer; + try { + audioBuffer = await ctx.decodeAudioData(arrayBuffer.slice(0)); + } catch (err) { + console.warn("spectrogram: failed to decode audio", err); + return null; + } + + const channels = audioBuffer.numberOfChannels; + if (channels === 0) return null; + + // Mix down to mono by averaging all channels. + const length = audioBuffer.length; + const mono = new Float32Array(length); + for (let c = 0; c < channels; c++) { + const data = audioBuffer.getChannelData(c); + for (let i = 0; i < length; i++) mono[i] += data[i] / channels; + } + + return { samples: mono, sampleRate: audioBuffer.sampleRate }; +} + +/** + * @type {ActionsWithTunnel['patch']} + */ +export async function patch({ data: track, ports }) { + // Skip if we already analysed this track (caching). + if (hasSpectralStats(track.stats)) return track; + + // Skip streams and very long tracks — decoding them is too costly. + if (track.kind === "stream") return track; + if (track.stats?.duration && track.stats.duration > MAX_DURATION_MS) { + return track; + } + + /** @type {ProxiedActions} */ + const input = workerProxy(() => { + ports.input.start(); + return ports.input; + }); + + const resGet = await input.resolve({ method: "GET", uri: track.uri }); + if (!resGet) return track; + + // Turn whatever `resolve` gave us into an ArrayBuffer. + let arrayBuffer; + try { + if ("stream" in resGet) { + arrayBuffer = await readStreamToArrayBuffer( + /** @type {ReadableStream} */ (resGet.stream), + ); + } else if ("url" in resGet) { + const res = await fetch(resGet.url); + if (!res.ok) return track; + arrayBuffer = await res.arrayBuffer(); + } else { + return track; + } + } catch (err) { + console.warn("spectrogram: failed to fetch audio data", err); + return track; + } + + const decoded = await decodeToMono(arrayBuffer); + if (!decoded) return track; + + const spectral = analyseSpectrogram(decoded.samples, decoded.sampleRate); + + /** @type {TrackStats} */ + const stats = removeUndefinedValuesFromRecord({ + ...track.stats, + ...spectral, + }); + + return { + ...track, + stats, + updatedAt: new Date().toISOString(), + }; +} + +//////////////////////////////////////////// +// ⚡️ +//////////////////////////////////////////// + +ostiary((context) => { + rpc(context, { patch }); +}); diff --git a/src/components/orchestrator/spectrogram-audio/element.js b/src/components/orchestrator/spectrogram-audio/element.js new file mode 100644 index 00000000..137b5148 --- /dev/null +++ b/src/components/orchestrator/spectrogram-audio/element.js @@ -0,0 +1,195 @@ +import { + BroadcastableDiffuseElement, + defineElement, + query, +} from "~/common/element.js"; +import { data, mergeById } from "~/common/output.js"; + +/** + * @import {OutputElement} from "@specs/components/output/types.d.ts" + * @import AudioEngine from "~/components/engine/audio/element.js" + * @import {MetadataElement} from "@specs/components/metadata/types.d.ts" + * @import {Track} from "~/definitions/types.d.ts" + */ + +//////////////////////////////////////////// +// ELEMENT +//////////////////////////////////////////// + +/** + * Fetches spectrogram data for a track when it starts playing, if it doesn't + * already have spectral stats. + * + * Monitors the audio engine for the active (non-preload) track. When playback + * begins, it checks whether the track is missing spectrogram-derived stats and, + * if so, asks the spectrogram metadata element to analyse it, then persists the + * patched track back to the output. + * + * Only the leader tab performs the analysis (broadcasting), so multi-tab setups + * don't duplicate the (expensive) decoding work. + */ +class SpectrogramAudioOrchestrator extends BroadcastableDiffuseElement { + static NAME = "diffuse/orchestrator/spectrogram-audio"; + + // LIFECYCLE + + /** @override */ + async connectedCallback() { + if (this.hasAttribute("group")) { + this.broadcast(this.identifier, {}); + } + + super.connectedCallback(); + + /** @type {AudioEngine} */ + this.audio = query(this, "audio-engine-selector"); + + /** @type {MetadataElement} */ + this.spectrogram = query(this, "spectrogram-selector"); + + /** @type {OutputElement} */ + this.output = query(this, "output-selector"); + + await customElements.whenDefined(this.audio.localName); + await customElements.whenDefined(this.spectrogram.localName); + await customElements.whenDefined(this.output.localName); + + this.effect(() => this.#monitorAudio()); + } + + // TRACK STATE + // Resets whenever the active (non-preload) audio item changes. + + /** @type {string | null} */ + #trackId = null; + + /** Whether analysis has already been kicked off for the current track. */ + #analysed = false; + + /** + * Track ids with an in-flight analysis, so a rapidly toggling play/pause or + * a track change doesn't trigger duplicate concurrent analyses. + * + * @type {Set} + */ + #inFlight = new Set(); + + // EFFECT + + /** + * Reacts to audio item changes and playback state. When a track starts + * playing and is missing spectral stats, triggers the spectrogram analysis. + */ + #monitorAudio() { + if (!this.audio) return; + + const active = this.audio.items().find((item) => !item.isPreload); + const id = active?.id ?? null; + + // Detect track change + if (id !== this.#trackId) { + this.#trackId = id; + this.#analysed = false; + } + + if (!id || !active) return; + + const state = this.audio.state(id); + const loadingState = state?.loadingState() ?? "loading"; + const hasError = typeof loadingState === "object" && "error" in loadingState; + // Wait until `canplay` (loadingState "loaded") before fetching, so the + // audio element gets first priority for bandwidth during its initial load + // and the spectrogram's full-file fetch doesn't compete with it. + const canPlay = !hasError && loadingState === "loaded"; + + if (!canPlay) return; + if (this.#analysed) return; + + this.#analysed = true; + this.#analyse(id, active.track); + } + + // ANALYSIS + + /** + * @param {string} id + * @param {Track} track + */ + async #analyse(id, track) { + // Already has spectral stats — nothing to do. + if (hasSpectralStats(track.stats)) return; + // Skip streams and very long tracks, matching the spectrogram worker's + // own guardrails. Checking here avoids queueing work the worker would drop. + if (track.kind === "stream") return; + if (track.stats?.duration && track.stats.duration > MAX_DURATION_MS) return; + // A previous play is still analysing this same track. + if (this.#inFlight.has(id)) return; + + if (!(await this.isLeader())) return; + if (this.#trackId !== id) return; // track changed while we awaited + + this.#inFlight.add(id); + + try { + const patched = await this.spectrogram?.patch(track); + if (!patched || patched === track) return; + if (this.#trackId !== id) return; // track changed during analysis + + await this.#save(patched); + } catch (err) { + console.warn("spectrogram-audio: analysis failed", err); + } finally { + this.#inFlight.delete(id); + } + } + + /** + * Merges the patched track back into the output's track list and persists it. + * + * @param {Track} track + */ + async #save(track) { + if (!this.output) return; + + const existing = await data(this.output.tracks); + const merged = mergeById(existing, [track]); + await this.output.tracks.save(merged); + } +} + +//////////////////////////////////////////// +// STATS HELPERS +//////////////////////////////////////////// + +const SPECTRAL_KEYS = [ + "spectralCentroid", + "spectralRolloff", + "spectralSpread", + "spectralFlatness", + "spectralFlux", +]; + +/** + * @param {import("~/definitions/types.d.ts").TrackStats | undefined} stats + * @returns {boolean} + */ +function hasSpectralStats(stats) { + if (!stats) return false; + return SPECTRAL_KEYS.every((k) => + stats[/** @type {keyof import("~/definitions/types.d.ts").TrackStats} */ (k)] != null + ); +} + +// Keep this in sync with the spectrogram worker's `MAX_DURATION_MS`. +const MAX_DURATION_MS = 4 * 60 * 60 * 1000; // 4 hours + +export default SpectrogramAudioOrchestrator; + +//////////////////////////////////////////// +// REGISTER +//////////////////////////////////////////// + +export const CLASS = SpectrogramAudioOrchestrator; +export const NAME = "do-spectrogram-audio"; + +defineElement(NAME, SpectrogramAudioOrchestrator); diff --git a/src/facets/data/spectrogram/index.html b/src/facets/data/spectrogram/index.html new file mode 100644 index 00000000..6590b0e7 --- /dev/null +++ b/src/facets/data/spectrogram/index.html @@ -0,0 +1 @@ + diff --git a/src/facets/data/spectrogram/index.inline.js b/src/facets/data/spectrogram/index.inline.js new file mode 100644 index 00000000..1bda4183 --- /dev/null +++ b/src/facets/data/spectrogram/index.inline.js @@ -0,0 +1,14 @@ +import foundation from "~/common/foundation.js"; +import { effect } from "~/common/signal.js"; + +let initialised = false; + +effect(() => { + const audio = foundation.signals.engine.audio(); + if (!audio) return; + + if (!initialised) { + foundation.orchestrator.spectrogramAudio(); + initialised = true; + } +}); -- 2.51.2