From a154c0ff7d89a97cf84c25c1ab0adc3512914df3 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:35:19 +0200 Subject: [PATCH] Add native SQLite benchmark backend --- apps/benchmark/README.md | 10 ++- .../baselines/calendar-sqlite-validated.json | 67 ++++++++++++++++ apps/benchmark/src/bench.ts | 76 +++++++++++++------ 3 files changed, 127 insertions(+), 26 deletions(-) create mode 100644 apps/benchmark/baselines/calendar-sqlite-validated.json diff --git a/apps/benchmark/README.md b/apps/benchmark/README.md index a32244d..e744cca 100644 --- a/apps/benchmark/README.md +++ b/apps/benchmark/README.md @@ -8,6 +8,12 @@ From the repository root: pnpm bench --config calendar.config.json ``` +D1 is the default backend. Use `--backend sqlite` for a fresh in-memory native SQLite run; the result records Node and SQLite versions and gains `-sqlite` in its filename: + +```bash +pnpm bench --backend sqlite --config calendar-records-only.config.json +``` + `calendar.config.json` mirrors the retained indexing shape from `11-atproto/02-atmo-rsvp`: calendar events, RSVPs, profiles, follows/feeds, query indexes, relation counts, and event full-text search. Removed product modules and read-only pipeline handlers are intentionally absent because they do not participate in indexing. External sinks are also omitted so the benchmark measures Contrail and D1 rather than Meilisearch latency. To run the same full workload with strict runtime Lexicon validation and canonical DAG-CBOR CID verification enabled: @@ -43,7 +49,7 @@ The current defaults are 100 concurrent identity resolutions, 20 active PDS host Before every run the harness recursively deletes its config/concurrency-specific `.cache` directory. It disposes and deletes the local D1 afterward as well; pass `--keep-cache` only for debugging. -Results are written to ignored JSON files under `results/`. Selected reference runs live in `baselines/`: [`calendar-default.json`](baselines/calendar-default.json) is the original 774.49-second global-concurrency run, [`calendar-host-aware.json`](baselines/calendar-host-aware.json) is the 219.74-second host-aware result with set-based derived projection rebuilds, and [`calendar-pipelined.json`](baselines/calendar-pipelined.json) is the comparable 134.98-second result after streaming identity resolution and atomically checkpointing projected pages. [`calendar-runtime-comparison.json`](baselines/calendar-runtime-comparison.json) isolates the local workerd regression from Contrail changes, while [`calendar-validation-comparison.json`](baselines/calendar-validation-comparison.json) retains the final adjacent validation-disabled/enabled pair. +Results are written to ignored JSON files under `results/`. Selected reference runs live in `baselines/`: [`calendar-default.json`](baselines/calendar-default.json) is the original 774.49-second global-concurrency run, [`calendar-host-aware.json`](baselines/calendar-host-aware.json) is the 219.74-second host-aware result with set-based derived projection rebuilds, and [`calendar-pipelined.json`](baselines/calendar-pipelined.json) is the comparable 134.98-second result after streaming identity resolution and atomically checkpointing projected pages. [`calendar-runtime-comparison.json`](baselines/calendar-runtime-comparison.json) isolates the local workerd regression from Contrail changes, [`calendar-validation-comparison.json`](baselines/calendar-validation-comparison.json) retains the final adjacent validation-disabled/enabled D1 pair, and [`calendar-sqlite-validated.json`](baselines/calendar-sqlite-validated.json) records the validated native-SQLite records-only run. Each result includes: @@ -51,7 +57,7 @@ Each result includes: - accepted and indexed records; - records per second; - peak RSS; -- exact Node, Wrangler, and workerd versions; +- exact Node plus Wrangler/workerd or native SQLite versions; - account and per-collection backfill state; - bounded aggregate ingest rejection diagnostics; - validation/CID settings when enabled; and diff --git a/apps/benchmark/baselines/calendar-sqlite-validated.json b/apps/benchmark/baselines/calendar-sqlite-validated.json new file mode 100644 index 0000000..4567718 --- /dev/null +++ b/apps/benchmark/baselines/calendar-sqlite-validated.json @@ -0,0 +1,67 @@ +{ + "format": "contrail.validation-benchmark", + "version": 1, + "observed_at": "2026-08-05T13:33:26Z", + "backend": "native-sqlite", + "runtime": { + "node": "v26.5.0", + "sqlite": "3.53.3" + }, + "config": "configs/calendar-records-only.config.json", + "result": "results/calendar-records-only-sqlite-validated-r100-h20-d3-2026-08-05T13-33-25-881Z.json", + "options": { + "identity_concurrency": 100, + "pds_concurrency": 20, + "dids_per_pds": 3, + "max_immediate_attempts": 1, + "runtime_lexicons": 10, + "strict": true, + "verify_cid": true + }, + "timings_ms": { + "binding": 3.5, + "init": 2.04, + "discovery": 987.28, + "backfill": 46148.69, + "total": 47162.98, + "projection_and_checkpoint": 860.66 + }, + "source": { + "requests": 3692, + "fetched_records": 20763, + "records": { + "community.lexicon.calendar.event": { + "fetched": 14613, + "accepted": 4834, + "indexed": 4834 + }, + "community.lexicon.calendar.rsvp": { + "fetched": 6150, + "accepted": 3697, + "indexed": 3697 + } + } + }, + "accepted_records": 8531, + "indexed_records": 8531, + "accepted_records_per_second": 184.86, + "peak_rss_kib": 494640, + "rejections": { + "lexicon_validation": 12232, + "cid_mismatch": 0, + "cid_encoding": 0, + "missing_cid": 0 + }, + "accounts": { + "total": 1633, + "complete": 1553, + "retrying": 80, + "failed": 0 + }, + "notes": [ + "This fixture contains only calendar events and RSVPs. Profiles, follows, feeds, FTS, relation counts, field-query indexes, and Constellation are disabled.", + "The run used a fresh in-memory native SQLite database. Projection plus atomic checkpoint work took 0.86 seconds cumulatively; wall time was dominated by live identity/PDS requests.", + "Transient network failures left 80 accounts retrying and fetched roughly 0.6% fewer records than the retained validation-disabled records-only run. Report wall time, record totals, and completion state together; do not use this single live run as isolated validation overhead.", + "Strict validation rejected 12,232 records and CID verification found no mismatches, encoding failures, or missing authoritative CIDs." + ] +} diff --git a/apps/benchmark/src/bench.ts b/apps/benchmark/src/bench.ts index b9debaf..d971a86 100644 --- a/apps/benchmark/src/bench.ts +++ b/apps/benchmark/src/bench.ts @@ -9,6 +9,7 @@ import { type Database, type LexiconDoc, } from "@atmo-dev/contrail"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; import { getPlatformProxy } from "wrangler"; const APP_DIR = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -18,8 +19,11 @@ const RESULTS_DIR = resolve(APP_DIR, "results"); const WRANGLER_CONFIG = resolve(APP_DIR, "wrangler.jsonc"); const require = createRequire(import.meta.url); +type BenchmarkBackend = "d1" | "sqlite"; + interface Options { config: string; + backend: BenchmarkBackend; concurrency: number; pdsConcurrency: number; didsPerPds: number; @@ -40,6 +44,7 @@ function positiveInteger(raw: string | undefined, name: string, fallback: number function parseArgs(argv: string[]): Options { let config: string | undefined; + let backend: BenchmarkBackend = "d1"; let concurrencyRaw: string | undefined; let pdsConcurrencyRaw: string | undefined; let didsPerPdsRaw: string | undefined; @@ -52,7 +57,13 @@ function parseArgs(argv: string[]): Options { const arg = argv[index]; if (arg === "--config") config = argv[++index]; else if (arg.startsWith("--config=")) config = arg.slice("--config=".length); - else if (arg === "--concurrency") concurrencyRaw = argv[++index]; + else if (arg === "--backend" || arg.startsWith("--backend=")) { + const value = arg === "--backend" ? argv[++index] : arg.slice("--backend=".length); + if (value !== "d1" && value !== "sqlite") { + throw new Error(`--backend must be d1 or sqlite (got ${value})`); + } + backend = value; + } else if (arg === "--concurrency") concurrencyRaw = argv[++index]; else if (arg.startsWith("--concurrency=")) { concurrencyRaw = arg.slice("--concurrency=".length); } else if (arg === "--pds-concurrency") pdsConcurrencyRaw = argv[++index]; @@ -77,6 +88,7 @@ function parseArgs(argv: string[]): Options { Options: --config JSON config path or filename under configs/ (required) + --backend d1 (default) or native sqlite --concurrency Concurrent identity resolutions (default: 100) --pds-concurrency Concurrent PDS hosts (default: 20) --dids-per-pds Concurrent accounts per PDS (default: 3) @@ -95,6 +107,7 @@ Options: if (!config) throw new Error("--config is required"); return { config, + backend, concurrency: positiveInteger(concurrencyRaw, "--concurrency", 100), pdsConcurrency: positiveInteger(pdsConcurrencyRaw, "--pds-concurrency", 20), didsPerPds: positiveInteger(didsPerPdsRaw, "--dids-per-pds", 3), @@ -245,30 +258,41 @@ async function main(): Promise { verifyCid: true, }; } - const wranglerPackagePath = require.resolve("wrangler/package.json"); + const wranglerPackagePath = + options.backend === "d1" ? require.resolve("wrangler/package.json") : null; const runtime = { node: process.version, - wrangler: await installedPackageVersion("wrangler"), - workerd: await installedPackageVersion( - "workerd", - createRequire(wranglerPackagePath), - ), + wrangler: + options.backend === "d1" ? await installedPackageVersion("wrangler") : null, + workerd: wranglerPackagePath + ? await installedPackageVersion("workerd", createRequire(wranglerPackagePath)) + : null, + sqlite: options.backend === "sqlite" ? (process.versions.sqlite ?? null) : null, }; - const name = `${safeName(configPath)}${validation ? "-validated" : ""}`; + const name = `${safeName(configPath)}${ + options.backend === "sqlite" ? "-sqlite" : "" + }${validation ? "-validated" : ""}`; const cachePath = resolve( CACHE_DIR, `${name}-r${options.concurrency}-h${options.pdsConcurrency}-d${options.didsPerPds}`, ); - await rm(cachePath, { recursive: true, force: true }); - await mkdir(cachePath, { recursive: true }); + if (options.backend === "d1") { + await rm(cachePath, { recursive: true, force: true }); + await mkdir(cachePath, { recursive: true }); + } await mkdir(RESULTS_DIR, { recursive: true }); console.log(`config: ${configPath}`); - console.log(`backend: fresh local D1`); console.log( - `runtime: Wrangler ${runtime.wrangler ?? "unknown"}, ` + - `workerd ${runtime.workerd ?? "unknown"}`, + `backend: ${ + options.backend === "d1" ? "fresh local D1" : "fresh in-memory native SQLite" + }`, + ); + console.log( + options.backend === "d1" + ? `runtime: Wrangler ${runtime.wrangler ?? "unknown"}, workerd ${runtime.workerd ?? "unknown"}` + : `runtime: Node ${runtime.node}, SQLite ${runtime.sqlite ?? "unknown"}`, ); console.log(`resolution: ${options.concurrency}`); console.log(`PDS hosts: ${options.pdsConcurrency}`); @@ -276,7 +300,7 @@ async function main(): Promise { console.log(`max attempts: ${options.maxAttempts}`); console.log(`excluded: ${options.excludeDids.length} actors`); console.log(`validation: ${validation ? `${validation.documents.length} Lexicons + CID` : "disabled"}`); - console.log(`cache: reset ${cachePath}`); + if (options.backend === "d1") console.log(`cache: reset ${cachePath}`); const startedAt = new Date(); const totalStart = performance.now(); @@ -294,16 +318,20 @@ async function main(): Promise { try { let phaseStart = performance.now(); - proxy = await getPlatformProxy({ - configPath: WRANGLER_CONFIG, - persist: { path: cachePath }, - remoteBindings: false, - envFiles: [], - }); + let db: Database; + if (options.backend === "sqlite") { + db = createSqliteDatabase(":memory:"); + } else { + proxy = await getPlatformProxy({ + configPath: WRANGLER_CONFIG, + persist: { path: cachePath }, + remoteBindings: false, + envFiles: [], + }); + db = (proxy.env as Record).DB as Database; + } bindingMs = elapsed(phaseStart); fetchInstrumentation = instrumentFetch(); - - const db = (proxy.env as Record).DB as Database; const contrail = new Contrail({ ...config, db }); phaseStart = performance.now(); @@ -353,7 +381,7 @@ async function main(): Promise { } finally { fetchInstrumentation?.restore(); await proxy?.dispose(); - if (!options.keepCache) { + if (options.backend === "d1" && !options.keepCache) { await rm(cachePath, { recursive: true, force: true }); } } @@ -376,7 +404,7 @@ async function main(): Promise { format: "contrail.backfill-benchmark", version: 1, config: relativeConfig.startsWith("..") ? configPath : relativeConfig, - backend: "wrangler-local-d1", + backend: options.backend === "d1" ? "wrangler-local-d1" : "native-sqlite", runtime, options: { concurrency: options.concurrency, -- 2.51.2