From bd27d7a34fa1718d27aa8f69b4059b51d27555ba Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Sat, 30 May 2026 13:20:34 -0700 Subject: [PATCH] pop/samples: add BBC Sound Effects (bbcrewind) as a sample source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLI fetcher mirroring freesound-fetch.mjs: search the public BBC Sound Effects archive API and download samples into pop/samples//. - pop/lib/bbc-rewind.mjs — search + download + manifest helper, no creds (uses the same public search/media endpoints the site's app calls). Tries the full /zip/.wav first, falls back to the MP3 preview decoded to 44.1k WAV via ffmpeg (the direct WAV path is gated behind the site's WebSocket batch-zip API; preview is the always-available tier). - pop/bin/bbc-fetch.mjs — CLI (--query/--list/--count/--slug/--id). - pop/samples/README.md — documents the source + licence guardrail. Licence: RemArc, NON-COMMERCIAL only. Every fetched sample is tagged license:"remarc-noncommercial", commercialUse:false so it can be used for sketches/research and non-commercial live performance (the AC `dj` piece) but never baked into a DistroKid release master. Co-Authored-By: Claude Opus 4.8 (1M context) --- pop/bin/bbc-fetch.mjs | 120 ++++++++++++++++++++++++ pop/lib/bbc-rewind.mjs | 202 +++++++++++++++++++++++++++++++++++++++++ pop/samples/README.md | 32 +++++++ 3 files changed, 354 insertions(+) create mode 100755 pop/bin/bbc-fetch.mjs create mode 100644 pop/lib/bbc-rewind.mjs diff --git a/pop/bin/bbc-fetch.mjs b/pop/bin/bbc-fetch.mjs new file mode 100755 index 000000000..80f5eecb6 --- /dev/null +++ b/pop/bin/bbc-fetch.mjs @@ -0,0 +1,120 @@ +#!/usr/bin/env node +// bbc-fetch.mjs — pull samples from the BBC Sound Effects archive +// (sound-effects.bbcrewind.co.uk) into pop/samples//. +// +// ⚠ RemArc licence: NON-COMMERCIAL only (personal/educational/research + +// non-commercial live performance). Fetched samples are tagged +// commercialUse:false — never bake them into a DistroKid release master. +// See pop/lib/bbc-rewind.mjs for the full note. +// +// Each sample downloads as full-quality WAV when the CDN allows it, else +// falls back to the MP3 preview decoded to WAV (the manifest records which +// tier — "wav" or "preview" — landed per id). +// +// Usage: +// # search only — print results so you can pick ids: +// node pop/bin/bbc-fetch.mjs --query "pachinko" --list +// +// # search + download the top N into pop/samples//: +// node pop/bin/bbc-fetch.mjs --query "pachinko" --count 6 +// node pop/bin/bbc-fetch.mjs --query "pachinko" --count 6 --slug pachinko-bbc +// +// # download specific ids directly (skips search): +// node pop/bin/bbc-fetch.mjs --slug pachinko-bbc --id 07022449,07032210 +// +// Flags: +// --query search text (required unless --id given) +// --count N number of top results to download (default 8) +// --from N search offset for paging (default 0) +// --slug NAME output dir under pop/samples/ (default: -bbc) +// --id A,B,C download these exact BBC ids (slug required) +// --list search and print results only, download nothing + +import { search, downloadSample, writeManifest, slugify, SAMPLES_DIR } from "../lib/bbc-rewind.mjs"; +import { resolve } from "node:path"; + +const flags = {}; +for (let i = 2; i < process.argv.length; i++) { + const a = process.argv[i]; + if (!a.startsWith("--")) continue; + const next = process.argv[i + 1]; + if (next !== undefined && !next.startsWith("--")) { flags[a.slice(2)] = next; i++; } + else flags[a.slice(2)] = true; +} + +const fmtDur = (s) => (typeof s === "number" ? `${s.toFixed(1)}s`.padStart(7) : " ? "); + +function die(msg) { console.error(msg); process.exit(1); } + +const ids = flags.id ? String(flags.id).split(",").map((s) => s.trim()).filter(Boolean) : null; + +if (!flags.query && !ids) { + die( + "usage: bbc-fetch.mjs --query [--count N] [--slug NAME] [--list]\n" + + " bbc-fetch.mjs --slug NAME --id \n" + + "⚠ RemArc licence — non-commercial use only (see pop/lib/bbc-rewind.mjs)" + ); +} + +// ── direct id download (no search) ─────────────────────────────────── +if (ids) { + const slug = flags.slug ? slugify(flags.slug) : null; + if (!slug) die("--id requires --slug NAME"); + const dir = resolve(SAMPLES_DIR, slug); + const got = []; + for (const id of ids) { + process.stderr.write(`↓ ${id} … `); + try { + const r = await downloadSample(id, dir); + console.error(r.skipped ? "cached" : `${r.quality} ${(r.bytes / 1e6).toFixed(1)} MB`); + got.push({ id, description: "", category: "", durationSec: null, tags: [], quality: r.quality }); + } catch (e) { console.error(`FAILED — ${e.message}`); } + } + if (got.length) { + const { manifest } = writeManifest({ slug, query: flags.query || slug, samples: got }); + console.error(`✓ ${got.length} sample(s) → ${dir}`); + console.error(` manifest: ${manifest} (license: remarc-noncommercial)`); + } + process.exit(0); +} + +// ── search ─────────────────────────────────────────────────────────── +const count = Number(flags.count || 8); +const from = Number(flags.from || 0); +const { total, results } = await search({ query: flags.query, from, size: Math.max(count, flags.list ? 20 : count) }); + +console.error(`“${flags.query}” — ${total} result(s) in the archive:`); +for (const r of results) { + console.error(` ${String(r.id).padEnd(12)} ${fmtDur(r.durationSec)} ${r.category ? `[${r.category}] ` : ""}${r.description}`); +} + +if (flags.list || total === 0) { + if (!total) console.error("(no results)"); + else console.error(`\n↳ download with: --count N or --slug NAME --id ${results.slice(0, 3).map((r) => r.id).join(",")}`); + process.exit(0); +} + +// ── download top `count` ───────────────────────────────────────────── +const slug = flags.slug ? slugify(flags.slug) : `${slugify(flags.query)}-bbc`; +const dir = resolve(SAMPLES_DIR, slug); +const picks = results.slice(0, count); +const got = []; +console.error(`\n→ ${slug}/`); +for (const r of picks) { + process.stderr.write(`↓ ${r.id} … `); + try { + const d = await downloadSample(r.id, dir); + console.error(d.skipped ? "cached" : `${d.quality} ${(d.bytes / 1e6).toFixed(1)} MB`); + got.push({ ...r, quality: d.quality }); + } catch (e) { console.error(`FAILED — ${e.message}`); } +} + +if (got.length) { + const { manifest } = writeManifest({ slug, query: flags.query, samples: got }); + const previews = got.filter((g) => g.quality === "preview").length; + console.error(`\n✓ ${got.length}/${picks.length} sample(s) → ${dir}`); + if (previews) console.error(` (${previews} fell back to MP3 preview — rerun from a residential IP for full WAV)`); + console.error(` manifest: ${manifest} (license: remarc-noncommercial — non-commercial use only)`); +} else { + die("\n✗ nothing downloaded"); +} diff --git a/pop/lib/bbc-rewind.mjs b/pop/lib/bbc-rewind.mjs new file mode 100644 index 000000000..4ee9ac745 --- /dev/null +++ b/pop/lib/bbc-rewind.mjs @@ -0,0 +1,202 @@ +// bbc-rewind.mjs — BBC Sound Effects (sound-effects.bbcrewind.co.uk) +// search + download helper for /pop. +// +// ⚠ LICENCE — RemArc (https://sound-effects.bbcrewind.co.uk/licensing) +// The archive is free for PERSONAL, EDUCATIONAL, RESEARCH and other +// NON-COMMERCIAL use only. That covers sketching, research baking, and +// non-commercial live performance (e.g. the AC-native `dj` piece at a +// free/art set). It does NOT cover commercial release — a BBC sample +// must never be baked into a DistroKid master. For commercial use the +// BBC licenses the same library via Pro Sound Effects. +// +// Every fetched sample is tagged `license: "remarc-noncommercial"` and +// `commercialUse: false` in its manifest so downstream tools can guard. +// +// No credentials needed — the search + media endpoints are the public +// ones the website's React app calls. Dependency-free except ffmpeg +// (only used to decode the MP3 fallback). Node 18+ for global fetch. +// +// Two media tiers: +// • full WAV https://sound-effects-media.bbcrewind.co.uk/zip/.wav +// • preview https://sound-effects-media.bbcrewind.co.uk/mp3/.mp3 +// The /zip/ WAV is served by S3+CloudFront and returns 403 to some +// datacenter IPs (it works fine from a normal residential connection — +// that's how pachinko-bbc/ was pulled). downloadSample() tries the WAV +// first and falls back to the MP3 preview (decoded to WAV via ffmpeg) +// so it always lands *something* usable. readWavMono in ../lib/wav.mjs +// downmixes at read time. + +import { execSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, writeFileSync, statSync } from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +export const SAMPLES_DIR = resolve(HERE, "../samples"); +const INDEX_PATH = resolve(SAMPLES_DIR, "INDEX.json"); + +const SEARCH_URL = "https://sound-effects-api.bbcrewind.co.uk/api/sfx/search"; +const MEDIA = "https://sound-effects-media.bbcrewind.co.uk"; +const MEDIA_WAV = (id) => `${MEDIA}/zip/${id}.wav`; +const MEDIA_MP3 = (id) => `${MEDIA}/mp3/${id}.mp3`; +export const LICENSE_URL = "https://sound-effects.bbcrewind.co.uk/licensing"; + +const SITE = "https://sound-effects.bbcrewind.co.uk"; +const UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15"; +// Send the site's Referer/Origin on every request — the API and media +// CDN both expect them. +const SITE_HEADERS = { "User-Agent": UA, Referer: SITE + "/", Origin: SITE }; + +export function slugify(s) { + return String(s).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 48); +} + +/** + * Search the BBC Sound Effects archive. + * @returns {Promise<{total:number, results:Array}>} + * Each result: { id, description, category, durationSec, source, tags, + * sampleRate, channels, bits, raw } + */ +export async function search({ query = "", from = 0, size = 20 } = {}) { + const body = { + criteria: { + from, size, + tags: null, categories: null, durations: null, continents: null, + sortBy: null, source: null, recordists: null, habitats: null, + query, + }, + }; + const res = await fetch(SEARCH_URL, { + method: "POST", + headers: { "Content-Type": "application/json", ...SITE_HEADERS }, + body: JSON.stringify(body), + }); + if (!res.ok) { + throw new Error(`bbc search ${res.status}: ${(await res.text()).slice(0, 200)}`); + } + const data = await res.json(); + const results = (data.results || []).map((r) => { + const tm = r.technicalMetadata || {}; + return { + id: r.id, + description: r.description ?? "", + category: r.categories?.[0]?.className ?? "", + // BBC reports duration in milliseconds. + durationSec: typeof r.duration === "number" ? +(r.duration / 1000).toFixed(2) : null, + source: r.source ?? "", + tags: r.tags ?? [], + sampleRate: tm.sample_rate ? Number(tm.sample_rate) : null, + channels: tm.channels ?? null, + bits: tm.bits_per_sample ?? null, + raw: r, + }; + }); + return { total: data.total ?? results.length, results }; +} + +/** + * Download one sample by id into `dir`. Tries the full WAV, falls back to + * the MP3 preview decoded to WAV via ffmpeg. + * @returns {Promise<{id, path, bytes, quality:"wav"|"preview", skipped}>} + */ +export async function downloadSample(id, dir) { + mkdirSync(dir, { recursive: true }); + const wavPath = resolve(dir, `${id}.wav`); + if (existsSync(wavPath)) { + return { id, path: wavPath, bytes: statSync(wavPath).size, quality: "existing", skipped: true }; + } + + // 1) full-quality WAV + try { + const r = await fetch(MEDIA_WAV(id), { headers: SITE_HEADERS }); + if (r.ok) { + const buf = Buffer.from(await r.arrayBuffer()); + // Guard against an S3 XML body being saved as ".wav". + if (buf.length > 64 && buf.subarray(0, 4).toString("latin1") === "RIFF") { + writeFileSync(wavPath, buf); + return { id, path: wavPath, bytes: buf.length, quality: "wav", skipped: false }; + } + } + } catch { /* fall through to preview */ } + + // 2) MP3 preview → decode to WAV + const mp3Path = resolve(dir, `${id}.preview.mp3`); + const r = await fetch(MEDIA_MP3(id), { headers: SITE_HEADERS }); + if (!r.ok) { + throw new Error(`bbc ${id}: WAV 403 and preview ${r.status} (${MEDIA_MP3(id)})`); + } + const mp3 = Buffer.from(await r.arrayBuffer()); + writeFileSync(mp3Path, mp3); + try { + execSync( + `ffmpeg -hide_banner -loglevel error -y -i ${JSON.stringify(mp3Path)} ` + + `-ar 44100 -ac 1 ${JSON.stringify(wavPath)}`, + { stdio: ["ignore", "ignore", "pipe"] }, + ); + } catch (e) { + throw new Error(`bbc ${id}: ffmpeg decode of preview failed — ${e.message}`); + } + return { id, path: wavPath, bytes: statSync(wavPath).size, quality: "preview", skipped: false }; +} + +function readJson(p, fallback) { + if (!existsSync(p)) return fallback; + try { return JSON.parse(readFileSync(p, "utf8")); } catch { return fallback; } +} + +/** + * Write/refresh the per-slug manifest with the RemArc licence stamp, drop a + * .gitignore so the audio stays local, and upsert the global INDEX.json entry. + */ +export function writeManifest({ slug, query, samples }) { + const dir = resolve(SAMPLES_DIR, slug); + mkdirSync(dir, { recursive: true }); + // Keep third-party audio out of git (same posture as the rest of samples/). + writeFileSync(resolve(dir, ".gitignore"), "*.wav\n*.mp3\n"); + + const now = new Date().toISOString(); + const manifest = { + source: "bbc-rewind", + license: "remarc-noncommercial", + licenseUrl: LICENSE_URL, + commercialUse: false, + usageNote: + "RemArc licence — personal/educational/research + non-commercial live " + + "performance only. Do NOT bake into a DistroKid release master.", + query, + slug, + fetchedAt: now, + samples: samples.map((s) => ({ + id: s.id, + description: s.description, + category: s.category, + durationSec: s.durationSec ?? null, + tags: s.tags ?? [], + quality: s.quality ?? null, // "wav" (full) or "preview" (mp3-decoded) + wav: `${s.id}.wav`, + wavUrl: MEDIA_WAV(s.id), + previewUrl: MEDIA_MP3(s.id), + })), + }; + writeFileSync(resolve(dir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n"); + + // Upsert into the global index (replace any existing entry for this slug). + const index = readJson(INDEX_PATH, { version: 1, sources: [] }); + index.sources = (index.sources || []).filter((s) => s.slug !== slug); + index.sources.push({ + slug, + title: query, + source: "bbc-rewind", + license: "remarc-noncommercial", + commercialUse: false, + url: `${SITE}/search?q=${encodeURIComponent(query)}`, + samples: samples.length, + manifest: `${slug}/manifest.json`, + addedAt: now, + note: "non-commercial / live + sketch only", + }); + index.generatedAt = now; + writeFileSync(INDEX_PATH, JSON.stringify(index, null, 2) + "\n"); + + return { dir, manifest: resolve(dir, "manifest.json") }; +} diff --git a/pop/samples/README.md b/pop/samples/README.md index fa3015fdf..83d6cd714 100644 --- a/pop/samples/README.md +++ b/pop/samples/README.md @@ -58,6 +58,38 @@ librosa if you only want to retune `--min-ms` / `--max-ms` (the cutter re-runs onsets every time today, but the data is committed for audit and downstream tools). +## BBC Sound Effects (sound-effects.bbcrewind.co.uk) + +A second source: the BBC Sound Effects archive (30k+ effects). Fetched +via `pop/bin/bbc-fetch.mjs` (lib: `pop/lib/bbc-rewind.mjs`) — no +credentials needed, it hits the same public search + media endpoints the +website's app uses, and writes full-quality WAVs named by BBC id +(`.wav`, e.g. `pachinko-bbc/07022449.wav`). + +```bash +# search only — print results so you can pick ids: +node pop/bin/bbc-fetch.mjs --query "pachinko" --list + +# search + download the top N into pop/samples//: +node pop/bin/bbc-fetch.mjs --query "pachinko" --count 6 --slug pachinko-bbc + +# download specific ids directly: +node pop/bin/bbc-fetch.mjs --slug pachinko-bbc --id 07022449,07032210 +``` + +Each fetch writes a tracked `manifest.json` (id + description + the +RemArc licence stamp) and drops a `.gitignore` so the WAV audio stays +local, then upserts the global `INDEX.json`. + +> ⚠ **Licence — RemArc, non-commercial only.** The BBC archive is free +> for personal / educational / research use and **non-commercial live +> performance** (e.g. the AC-native `dj` piece at a free/art set). It is +> **not** cleared for commercial release — a BBC sample must never be +> baked into a DistroKid release master. Every fetched sample is tagged +> `license: "remarc-noncommercial"`, `commercialUse: false`. For +> commercial use the BBC licenses the same library via Pro Sound Effects. +> Terms: + ## why third-party audio stays out of git Same posture as `pop/references/README.md`: third-party copyrighted -- 2.51.2