From af9dc11365294bb48d0e5ee9e138f004dabba44e Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Sun, 24 May 2026 15:30:33 -0500 Subject: [PATCH] Add reference PDS benchmark probes --- bench/README.md | 151 ++++++++++++++++++++++------ bench/justfile | 24 +++++ bench/main.zig | 142 +++++++++++++++++++++++++- bench/official-pds-records.bench.ts | 90 +++++++++++++++++ bench/run-official-pds.sh | 30 ++++++ docs/benchmarking.md | 79 +++++++++++++++ src/storage/store.zig | 29 ++++++ 7 files changed, 514 insertions(+), 31 deletions(-) create mode 100644 bench/official-pds-records.bench.ts create mode 100755 bench/run-official-pds.sh create mode 100644 docs/benchmarking.md diff --git a/bench/README.md b/bench/README.md index 627545d..f6362c5 100644 --- a/bench/README.md +++ b/bench/README.md @@ -18,6 +18,12 @@ just bench repo just bench blob just bench metastore 10 1000 just bench get-cid 10 1000 +just bench get-block 10 1000 +just bench decode-record 10 1000 +just bench render-record 10 1000 +just bench get-record 10 1000 +just bench list-records 10 1000 +just bench official-pds just bench write-profile 10 500 just bench run --scenario write --records 10000 ``` @@ -32,9 +38,17 @@ just bench run --scenario write --records 10000 - `repo`: full repo CAR materialization through `writeRepoCar`. - `blob`: blob write/read against disk blobstore plus SQLite metadata. - `metastore`: Tranquil-shaped apply/get/list benchmark with caller counts and - latency percentiles. + latency percentiles. Its get/list rows are ZDS-local probes, not all direct + Tranquil comparisons. - `get-cid`: CID-only record lookup, matching Tranquil's `get_record_cid` metastore benchmark. +- `get-block`: record index lookup plus `repo_blocks` byte fetch. +- `decode-record`: DAG-CBOR decode of one seeded record block, without SQLite. +- `render-record`: DAG-CBOR decode plus JSON rendering, without SQLite. +- `get-record`: full `com.atproto.repo.getRecord` storage materialization. +- `list-records`: full `com.atproto.repo.listRecords` storage materialization. +- `official-pds`: copies the official-PDS read probe into a local + `bluesky-social/atproto` checkout and runs it with Vitest. - `write-profile`: `applyWrites` stage timing under write concurrency. Each run uses temporary SQLite and blobstore state. @@ -51,6 +65,22 @@ Each run uses temporary SQLite and blobstore state. - Do not compare paths that perform different correctness work. For example, CID-only metadata lookup is separate from full `getRecord` materialization. +## traffic model + +For a small hosted PDS, the hot paths are not exotic. A hundred active accounts +mainly means more session checks, appview-proxied feed/profile reads, record +writes from clients and apps, blob uploads, sync subscribers, crawl requests, +and occasional repo export/import. The storage benchmarks should therefore keep +these domains separate: + +- record index: current CID lookups, existence checks, and index-only lists. +- record materialization: index lookup, block fetch, DAG-CBOR decode, and JSON + response rendering. +- write path: validation, repo mutation, commit/block persistence, event + publication, and crawl notification. +- blob path: upload body ingest, disk write, metadata persistence, and readback. +- sync path: event delivery and repo CAR materialization. + ## current local baseline On this machine, with `--records 1000` unless noted: @@ -62,7 +92,7 @@ On this machine, with `--records 1000` unless noted: | write repo car | 252 ops/s, 440 MB/s | | blob put+get | 368 ops/s, 46 MB/s | -## Tranquil comparison +## reference matrix Run Tranquil's metastore benchmark: @@ -80,31 +110,69 @@ just bench get-cid 10 1000 just bench get-cid 100 200 ``` -This table only includes operation/count pairs measured in both implementations. -The operations are matched to Tranquil's metastore bench: write one record, look -up one record CID, and list records from a seeded repo. +This matrix only includes operation/count pairs measured with the same unit of +work. Tranquil's `get_record_cid` and `list_records` are record-index +operations: `list_records` returns `rkey + record_cid`, not record body JSON. +The official PDS probe measures its actor-store read paths with Vitest. That is +apples-to-apples for the internal read operations, but not for concurrency +curves: Vitest reports a single async benchmark loop, while ZDS and Tranquil +also have explicit caller-count runs. + +Summary: ZDS is fastest in the single-caller read probes, Tranquil is fastest in +the 10/100 caller CID lookup probes, and the official PDS is slower in these +microbenchmarks because the probe goes through its actor-store/test-network +stack rather than a bare SQLite reader. For writes, official PDS is faster than +ZDS in the single-caller row, Tranquil is faster than ZDS at 10 callers, and ZDS +is faster than Tranquil at 100 callers but with a worse p99/max tail. + +| operation | callers | zds | tranquil | official PDS | +|---|---:|---:|---:|---:| +| apply one record commit | 1 | 642 ops/s, p95 2.9 ms | 287 ops/s, p95 4.1 ms | 486 ops/s, p99 3.9 ms | +| current record CID lookup | 1 | 414k ops/s, p95 4 us | 96.8k ops/s, p95 15 us | 3.2k ops/s, p99 477 us | -Summary: Tranquil is faster on the read rows after ZDS moved record materialize -paths to decode DAG-CBOR from `repo_blocks`: about 1.5-1.8x faster for CID -lookup and about 1.6x faster for list records. Tranquil is also faster on -10-caller writes, by about 2.8x. ZDS is faster on the 100-caller write row, by -about 1.3x, which is the main sign that the sharded write lane is helping under -contention. +The current caller-count concurrency matrix is only ZDS and Tranquil until the +official-PDS probe grows explicit concurrent workers. | operation | callers | ops | zds | tranquil | |---|---:|---:|---:|---:| -| apply/write | 10 | 10000 | 282 ops/s, p95 93.8 ms | 780 ops/s, p95 14.1 ms | -| apply/write | 100 | 20000 | 1357 ops/s, p95 66.7 ms | 1007 ops/s, p95 109 ms | -| get record CID | 10 | 10000 | 296k ops/s, p95 6 us | 455k ops/s, p95 43 us | -| get record CID | 100 | 20000 | 294k ops/s, p95 405 us | 537k ops/s, p95 233 us | -| list records | 10 | 10000 | 22k ops/s, p95 52 us | 36k ops/s, p95 310 us | -| list records | 100 | 20000 | 23k ops/s, p95 49 us | 36k ops/s, p95 2.9 ms | - -Tranquil's read row is `get_record_cid`, so ZDS reports `get-cid` for that -comparison. Full `com.atproto.repo.getRecord` is a separate local probe because -it joins the record index to `repo_blocks`, decodes DAG-CBOR, and renders the -XRPC JSON response shape. Tranquil's metastore bench intentionally measures a -metadata lookup for the current CID, not record body materialization. +| apply one record commit | 10 | 10000 | 259 ops/s, p95 98.9 ms | 780 ops/s, p95 14.1 ms | +| apply one record commit | 100 | 20000 | 1223 ops/s, p95 67.5 ms | 1007 ops/s, p95 109 ms | +| current record CID lookup | 10 | 10000 | 222k ops/s, p95 225 us | 455k ops/s, p95 43 us | +| current record CID lookup | 100 | 20000 | 211k ops/s, p95 887 us | 537k ops/s, p95 233 us | + +Tranquil's metastore `list_records` benchmark is not in this table because ZDS +does not currently expose an equivalent "list rkeys and CIDs only" benchmark. +ZDS's `listRecords` follows the official Bluesky PDS read shape: join the record +index to `repo_blocks`, decode DAG-CBOR, and return record values. + +## official PDS probe + +The official-PDS probe lives at `bench/official-pds-records.bench.ts` and is +run with: + +```sh +ATPROTO_REPO=/tmp/bluesky-atproto just bench official-pds +``` + +It seeds 1000 `app.bsky.feed.post` records through the official test network, +then measures actor-store reads directly: + +| operation | official PDS | +|---|---:| +| apply one record commit | 486 ops/s, p99 3.9 ms | +| current record CID lookup | 3.2k ops/s, p99 477 us | +| full getRecord materialization | 3.1k ops/s, p99 469 us | +| full listRecords, limit 50 | 639 ops/s, p99 2.3 ms | + +These are useful for the official-PDS column, but they should not be mixed with +ZDS's caller-count rows. The matching ZDS single-caller rows are: + +| operation | zds | +|---|---:| +| apply one record commit | 642 ops/s, p95 2.9 ms | +| current record CID lookup | 414k ops/s, p95 4 us | +| full getRecord materialization | 48.6k ops/s, p95 28 us | +| full listRecords, limit 50 | 21.6k ops/s, p95 60 us | ## local probes @@ -112,8 +180,21 @@ These rows are useful for ZDS tuning but are not direct Tranquil comparisons. | operation | callers | ops | zds | |---|---:|---:|---:| -| full getRecord | 10 | 10000 | 40k ops/s, p95 352 us | -| full getRecord | 100 | 20000 | 39k ops/s, p95 8.2 ms | +| get record block | 10 | 10000 | 128k ops/s, p95 11 us | +| get record block | 100 | 20000 | 121k ops/s, p95 15 us | +| decode DAG-CBOR | 10 | 10000 | 40.6M ops/s, p95 below timer resolution | +| decode DAG-CBOR | 100 | 20000 | 12.6M ops/s, p95 below timer resolution | +| render record JSON | 10 | 10000 | 24.8M ops/s, p95 below timer resolution | +| render record JSON | 100 | 20000 | 12.4M ops/s, p95 1 us | +| full getRecord | 10 | 10000 | 38.4k ops/s, p95 173 us | +| full getRecord | 100 | 20000 | 38.1k ops/s, p95 11.7 ms | +| full listRecords, limit 50 | 10 | 10000 | 21.1k ops/s, p95 57 us | +| full listRecords, limit 50 | 100 | 20000 | 19.5k ops/s, p95 78 us | + +The read-path split says SQLite serialization and row materialization dominate +single-record reads. Decode/render is effectively invisible for tiny benchmark +records; it needs a larger-record probe before we can extrapolate image-heavy or +rich-record traffic. ## write profile @@ -139,14 +220,24 @@ Tranquil benchmark source: - [`crates/tranquil-store/benches/blockstore.rs`](https://tangled.org/tranquil.farm/tranquil-pds/tree/main/crates/tranquil-store/benches/blockstore.rs) - [`crates/tranquil-store/benches/eventlog.rs`](https://tangled.org/tranquil.farm/tranquil-pds/tree/main/crates/tranquil-store/benches/eventlog.rs) -The first apples-to-apples comparison should cover: +Official PDS source for matching probes: + +- [`packages/pds/src/actor-store/record/reader.ts`](https://github.com/bluesky-social/atproto/blob/main/packages/pds/src/actor-store/record/reader.ts) +- [`packages/pds/src/api/com/atproto/repo/applyWrites.ts`](https://github.com/bluesky-social/atproto/blob/main/packages/pds/src/api/com/atproto/repo/applyWrites.ts) +- [`packages/pds/src/actor-store/repo/sql-repo-reader.ts`](https://github.com/bluesky-social/atproto/blob/main/packages/pds/src/actor-store/repo/sql-repo-reader.ts) + +The next apples-to-apples comparison work should cover: - single-repo write curve as record count grows - multi-repo concurrent writes -- listRecords latency under read concurrency +- record-index list latency without record body materialization, across ZDS and + Tranquil +- official-PDS concurrency probe for the same read/write boundaries - full repo export cost - blob/block store throughput -ZDS and Tranquil both report latency percentiles for the metastore-shaped -benchmark. The remaining comparison gap is broader coverage of blockstore, -eventlog, and full repo export behavior. +ZDS and Tranquil both report latency percentiles for the metastore-shaped write +and CID lookup rows. The remaining comparison gap is broader coverage of +blockstore, eventlog, full repo export behavior, an official-PDS benchmark +harness, and an explicit ZDS index-only list probe that matches Tranquil's +`list_records`. diff --git a/bench/justfile b/bench/justfile index 6a84469..27ebb4e 100644 --- a/bench/justfile +++ b/bench/justfile @@ -33,6 +33,30 @@ metastore callers="1" ops_per_caller="100": get-cid callers="10" ops_per_caller="1000": {{zig}} build bench -Doptimize=ReleaseFast -- --scenario get-cid --callers {{callers}} --ops-per-caller {{ops_per_caller}} +# benchmark record index lookup plus repo_blocks byte fetch +get-block callers="10" ops_per_caller="1000": + {{zig}} build bench -Doptimize=ReleaseFast -- --scenario get-block --callers {{callers}} --ops-per-caller {{ops_per_caller}} + +# benchmark DAG-CBOR decode of a seeded record block without SQLite +decode-record callers="10" ops_per_caller="1000": + {{zig}} build bench -Doptimize=ReleaseFast -- --scenario decode-record --callers {{callers}} --ops-per-caller {{ops_per_caller}} + +# benchmark DAG-CBOR decode plus JSON response rendering without SQLite +render-record callers="10" ops_per_caller="1000": + {{zig}} build bench -Doptimize=ReleaseFast -- --scenario render-record --callers {{callers}} --ops-per-caller {{ops_per_caller}} + +# benchmark full com.atproto.repo.getRecord storage materialization +get-record callers="10" ops_per_caller="1000": + {{zig}} build bench -Doptimize=ReleaseFast -- --scenario get-record --callers {{callers}} --ops-per-caller {{ops_per_caller}} + +# benchmark full listRecords storage materialization +list-records callers="10" ops_per_caller="1000": + {{zig}} build bench -Doptimize=ReleaseFast -- --scenario list-records --callers {{callers}} --ops-per-caller {{ops_per_caller}} + +# run the official Bluesky PDS read probe from a local atproto checkout +official-pds: + ./bench/run-official-pds.sh + # profile applyWrites stage timing under write concurrency write-profile callers="10" ops_per_caller="500": {{zig}} build bench -Doptimize=ReleaseFast -- --scenario write-profile --callers {{callers}} --ops-per-caller {{ops_per_caller}} diff --git a/bench/main.zig b/bench/main.zig index 44ec050..85aa8e0 100644 --- a/bench/main.zig +++ b/bench/main.zig @@ -9,6 +9,11 @@ const Scenario = enum { repo, metastore, get_cid, + get_block, + decode_record, + render_record, + get_record, + list_records, write_profile, }; @@ -96,6 +101,11 @@ pub fn main(init: std.process.Init) !void { .blob => (try benchBlob(allocator, state.account, options.blobs, options.blob_size)).print(), .metastore => try benchMetastore(allocator, options), .get_cid => try benchGetCid(allocator, options), + .get_block => try benchGetBlock(allocator, options), + .decode_record => try benchDecodeRecord(allocator, options), + .render_record => try benchRenderRecord(allocator, options), + .get_record => try benchGetRecord(allocator, options), + .list_records => try benchListRecords(allocator, options), .write_profile => try benchWriteProfile(allocator, options), } } @@ -400,6 +410,60 @@ fn benchGetCid(allocator: std.mem.Allocator, options: Options) !void { concurrentResult("getRecordCid", level, total_ops, nowNs() - start, latencies).print(); } +fn benchGetBlock(allocator: std.mem.Allocator, options: Options) !void { + const level: ConcurrencyLevel = .{ + .callers = options.callers orelse 10, + .ops_per_caller = options.ops_per_caller orelse 1000, + }; + var state = try initBench(allocator); + defer state.deinit(); + try seedIndexedRecords(allocator, state.account, 1000); + + const total_ops = level.callers * level.ops_per_caller; + const latencies = try allocator.alloc(u64, total_ops); + defer allocator.free(latencies); + const threads = try allocator.alloc(std.Thread, level.callers); + defer allocator.free(threads); + + const start = nowNs(); + for (threads, 0..) |*thread, i| { + thread.* = try std.Thread.spawn(.{}, getBlockWorker, .{ + state.account.did, + i, + latencies[i * level.ops_per_caller ..][0..level.ops_per_caller], + }); + } + for (threads) |thread| thread.join(); + + concurrentResult("getRecordBlock", level, total_ops, nowNs() - start, latencies).print(); +} + +fn benchGetRecord(allocator: std.mem.Allocator, options: Options) !void { + const level: ConcurrencyLevel = .{ + .callers = options.callers orelse 10, + .ops_per_caller = options.ops_per_caller orelse 1000, + }; + const result = try benchConcurrentGets(allocator, level); + result.print(); +} + +fn benchListRecords(allocator: std.mem.Allocator, options: Options) !void { + const level: ConcurrencyLevel = .{ + .callers = options.callers orelse 10, + .ops_per_caller = options.ops_per_caller orelse 1000, + }; + const result = try benchConcurrentLists(allocator, level); + result.print(); +} + +fn benchDecodeRecord(allocator: std.mem.Allocator, options: Options) !void { + try benchRecordBlockCpu(allocator, options, .decode); +} + +fn benchRenderRecord(allocator: std.mem.Allocator, options: Options) !void { + try benchRecordBlockCpu(allocator, options, .render); +} + fn benchConcurrentLists(allocator: std.mem.Allocator, level: ConcurrencyLevel) !ConcurrentResult { var state = try initBench(allocator); defer state.deinit(); @@ -523,6 +587,21 @@ fn getCidWorker(did: []const u8, task_id: usize, latencies: []u64) !void { } } +fn getBlockWorker(did: []const u8, task_id: usize, latencies: []u64) !void { + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena.deinit(); + for (latencies, 0..) |*latency, i| { + _ = arena.reset(.retain_capacity); + const rec_idx = (task_id * 7 + i * 13) % 1000; + var rkey_buf: [32]u8 = undefined; + const rkey = try std.fmt.bufPrint(&rkey_buf, "rec{d:0>8}", .{rec_idx}); + const start = nowNs(); + const block = zds.storage.store.getRecordBlock(arena.allocator(), did, bench_collection, rkey) orelse return error.MissingRecord; + if (block.cid.len == 0 or block.data.len == 0) return error.MissingRecord; + latency.* = nowNs() - start; + } +} + fn listWorker(did: []const u8, latencies: []u64) !void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); @@ -535,6 +614,62 @@ fn listWorker(did: []const u8, latencies: []u64) !void { } } +const CpuPath = enum { decode, render }; + +fn benchRecordBlockCpu(allocator: std.mem.Allocator, options: Options, path: CpuPath) !void { + const level: ConcurrencyLevel = .{ + .callers = options.callers orelse 10, + .ops_per_caller = options.ops_per_caller orelse 1000, + }; + var state = try initBench(allocator); + defer state.deinit(); + try seedIndexedRecords(allocator, state.account, 1); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const block = zds.storage.store.getRecordBlock(arena.allocator(), state.account.did, bench_collection, "rec00000000") orelse return error.MissingRecord; + + const total_ops = level.callers * level.ops_per_caller; + const latencies = try allocator.alloc(u64, total_ops); + defer allocator.free(latencies); + const threads = try allocator.alloc(std.Thread, level.callers); + defer allocator.free(threads); + + const start = nowNs(); + for (threads, 0..) |*thread, i| { + thread.* = try std.Thread.spawn(.{}, recordBlockCpuWorker, .{ + block.data, + path, + latencies[i * level.ops_per_caller ..][0..level.ops_per_caller], + }); + } + for (threads) |thread| thread.join(); + + concurrentResult(switch (path) { + .decode => "decodeDagCbor", + .render => "renderRecordJson", + }, level, total_ops, nowNs() - start, latencies).print(); +} + +fn recordBlockCpuWorker(data: []const u8, path: CpuPath, latencies: []u64) !void { + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena.deinit(); + for (latencies) |*latency| { + _ = arena.reset(.retain_capacity); + const start = nowNs(); + switch (path) { + .decode => { + _ = try zds.zat.cbor.decodeAll(arena.allocator(), data); + }, + .render => { + const json = try zds.storage.store.recordJsonFromDagCbor(arena.allocator(), data); + if (json.len == 0) return error.EmptyRecordJson; + }, + } + latency.* = nowNs() - start; + } +} + fn seedIndexedRecords(allocator: std.mem.Allocator, account: zds.auth.tokens.Account, target: usize) !void { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); @@ -649,13 +784,18 @@ fn parseScenario(value: []const u8) !Scenario { if (std.mem.eql(u8, value, "repo")) return .repo; if (std.mem.eql(u8, value, "metastore")) return .metastore; if (std.mem.eql(u8, value, "get-cid")) return .get_cid; + if (std.mem.eql(u8, value, "get-block")) return .get_block; + if (std.mem.eql(u8, value, "decode-record")) return .decode_record; + if (std.mem.eql(u8, value, "render-record")) return .render_record; + if (std.mem.eql(u8, value, "get-record")) return .get_record; + if (std.mem.eql(u8, value, "list-records")) return .list_records; if (std.mem.eql(u8, value, "write-profile")) return .write_profile; return error.UnknownScenario; } fn usage() void { std.debug.print( - \\usage: zds-bench [--scenario all|write|read|blob|repo|metastore|get-cid|write-profile] [--records N] [--blobs N] [--blob-size BYTES] [--callers N --ops-per-caller N] + \\usage: zds-bench [--scenario all|write|read|blob|repo|metastore|get-cid|get-block|decode-record|render-record|get-record|list-records|write-profile] [--records N] [--blobs N] [--blob-size BYTES] [--callers N --ops-per-caller N] \\ , .{}); } diff --git a/bench/official-pds-records.bench.ts b/bench/official-pds-records.bench.ts new file mode 100644 index 0000000..c5f1177 --- /dev/null +++ b/bench/official-pds-records.bench.ts @@ -0,0 +1,90 @@ +import { bench, beforeAll, afterAll } from 'vitest' +import { AtUri } from '@atproto/syntax' +import { TestNetworkNoAppView } from '@atproto/dev-env' +import { prepareCreate } from './src/repo/index.js' + +const records: { uri: string; cid: string; rkey: string }[] = [] +let network: TestNetworkNoAppView +let did: string +let writeIdx = 0 + +beforeAll(async () => { + network = await TestNetworkNoAppView.create({ + dbPostgresSchema: 'zds_bench', + }) + const agent = network.pds.getAgent() + await agent.createAccount({ + email: 'bench@test.com', + handle: 'bench.test', + password: 'bench-pass', + }) + did = agent.assertDid + + for (let i = 0; i < 1000; i++) { + const res = await agent.com.atproto.repo.createRecord({ + repo: did, + collection: 'app.bsky.feed.post', + record: { + $type: 'app.bsky.feed.post', + text: `official pds bench record ${i}`, + createdAt: `2026-05-23T00:${(i % 60).toString().padStart(2, '0')}:00.000Z`, + }, + }) + records.push({ + uri: res.data.uri, + cid: res.data.cid, + rkey: new AtUri(res.data.uri).rkey, + }) + } +}, 120_000) + +afterAll(async () => { + await network.close() +}) + +bench('current record CID lookup', async () => { + const rec = records[Math.floor(Math.random() * records.length)] + const cid = await network.pds.ctx.actorStore.read(did, (store) => + store.record.getCurrentRecordCid(new AtUri(rec.uri)), + ) + if (!cid) throw new Error('missing cid') +}) + +bench('apply one record commit', async () => { + const i = writeIdx++ + const write = await prepareCreate({ + did, + collection: 'app.bsky.feed.post', + validate: false, + record: { + $type: 'app.bsky.feed.post', + text: `official pds write bench ${i}`, + createdAt: `2026-05-23T01:${(i % 60).toString().padStart(2, '0')}:00.000Z`, + }, + }) + const commit = await network.pds.ctx.actorStore.transact(did, async (actorTxn) => { + const next = await actorTxn.repo.processWrites([write]) + await network.pds.ctx.sequencer.sequenceCommit(did, next) + return next + }) + await network.pds.ctx.accountManager.updateRepoRoot(did, commit.cid, commit.rev) +}) + +bench('full getRecord materialization', async () => { + const rec = records[Math.floor(Math.random() * records.length)] + const record = await network.pds.ctx.actorStore.read(did, (store) => + store.record.getRecord(new AtUri(rec.uri), rec.cid), + ) + if (!record) throw new Error('missing record') +}) + +bench('full listRecords materialization, limit 50', async () => { + const rows = await network.pds.ctx.actorStore.read(did, (store) => + store.record.listRecordsForCollection({ + collection: 'app.bsky.feed.post', + limit: 50, + reverse: false, + }), + ) + if (rows.length === 0) throw new Error('missing records') +}) diff --git a/bench/run-official-pds.sh b/bench/run-official-pds.sh new file mode 100755 index 0000000..abba84e --- /dev/null +++ b/bench/run-official-pds.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env sh +set -eu + +ATPROTO_REPO="${ATPROTO_REPO:-/tmp/bluesky-atproto}" +BENCH_SOURCE="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)/official-pds-records.bench.ts" +BENCH_TARGET="$ATPROTO_REPO/packages/pds/zds-records.bench.ts" +CONFIG_TARGET="$ATPROTO_REPO/zds-vitest.config.ts" + +if [ ! -d "$ATPROTO_REPO/packages/pds" ]; then + echo "missing official atproto checkout: $ATPROTO_REPO" >&2 + exit 1 +fi + +cp "$BENCH_SOURCE" "$BENCH_TARGET" +cat > "$CONFIG_TARGET" <<'EOF' +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + include: ['packages/pds/zds-records.bench.ts'], + benchmark: { + include: ['packages/pds/zds-records.bench.ts'], + }, + }, +}) +EOF +trap 'rm -f "$BENCH_TARGET" "$CONFIG_TARGET"' EXIT INT TERM + +cd "$ATPROTO_REPO" +corepack pnpm exec vitest bench --run --config zds-vitest.config.ts diff --git a/docs/benchmarking.md b/docs/benchmarking.md new file mode 100644 index 0000000..4af725c --- /dev/null +++ b/docs/benchmarking.md @@ -0,0 +1,79 @@ +# benchmarking + +Benchmark notes should separate behavior domains and only compare equivalent +work. If a measurement does not exist for the same unit of work, keep it out of +the comparison table instead of filling the cell with a nearby but different +operation. + +## references + +- ZDS: `bench/main.zig` +- Tranquil: `crates/tranquil-store/benches/metastore.rs` +- Official PDS: `packages/pds/src/actor-store/record/reader.ts` + +## current matrix + +The active three-way matrix is: + +- apply one record commit +- current record CID lookup +- record-index list +- full record materialization +- blob/block put and get +- repo export +- sync event delivery + +ZDS, Tranquil, and the official PDS now have matching single-caller numbers for +apply-one-record and current-CID lookup. ZDS and Tranquil also have explicit +10/100 caller concurrency curves; the official probe should grow those workers +before it is included in the concurrency matrix. + +## read path + +ZDS follows the official PDS read shape for `getRecord` and `listRecords`: read +the current record index row, fetch matching `repo_blocks` bytes, decode +DAG-CBOR, and render JSON for the XRPC response. Tranquil's metastore +`list_records` is a different unit of work because it returns only rkeys and +CIDs. Keep those rows separate. + +Current ZDS read breakdown on this machine: + +| operation | callers | ops | result | +|---|---:|---:|---:| +| current CID lookup | 10 | 10000 | 222k ops/s, p95 225 us | +| current CID lookup | 100 | 20000 | 211k ops/s, p95 887 us | +| index + block fetch | 10 | 10000 | 128k ops/s, p95 11 us | +| index + block fetch | 100 | 20000 | 121k ops/s, p95 15 us | +| full getRecord | 10 | 10000 | 38.4k ops/s, p95 173 us | +| full getRecord | 100 | 20000 | 38.1k ops/s, p95 11.7 ms | +| full listRecords, limit 50 | 10 | 10000 | 21.1k ops/s, p95 57 us | +| full listRecords, limit 50 | 100 | 20000 | 19.5k ops/s, p95 78 us | + +The official PDS probe on the same machine reports: + +| operation | result | +|---|---:| +| apply one record commit | 486 ops/s, p99 3.9 ms | +| current CID lookup | 3.2k ops/s, p99 477 us | +| full getRecord | 3.1k ops/s, p99 469 us | +| full listRecords, limit 50 | 639 ops/s, p99 2.3 ms | + +For tiny records, DAG-CBOR decode and JSON rendering are below useful timer +resolution. The important next read benchmark is larger records plus an +index-only list probe matching Tranquil. + +## traffic model + +For roughly 100 hosted accounts, useful benchmarks should mirror real traffic: + +- clients call `getSession`, preferences, profiles, timelines, and proxied + appview routes repeatedly while open. +- apps call `createRecord`, `applyWrites`, `getRecord`, and `listRecords`. +- media posts call `uploadBlob` before record creation. +- relays and tools call sync endpoints, repo export, and blob reads. +- the landing page reads residents, record counts, recent records, and profile + blobs. + +This points to two instrumentation tasks: per-route latency counters in ZDS +logs, and benchmark scenarios that replay observed route mixes instead of only +isolated microbenchmarks. diff --git a/src/storage/store.zig b/src/storage/store.zig index 0aa8dfd..ad38f71 100644 --- a/src/storage/store.zig +++ b/src/storage/store.zig @@ -972,6 +972,31 @@ pub fn getCid(allocator: std.mem.Allocator, did: []const u8, collection: []const return allocator.dupe(u8, row.?.text(0)) catch null; } +pub const RecordBlock = struct { + cid: []const u8, + data: []const u8, +}; + +pub fn getRecordBlock(allocator: std.mem.Allocator, did: []const u8, collection: []const u8, rkey: []const u8) ?RecordBlock { + db_mutex.lockUncancelable(store_io); + defer db_mutex.unlock(store_io); + requireInitialized() catch return null; + + const row = conn.row( + \\SELECT r.cid, rb.data + \\FROM records r + \\JOIN repo_blocks rb ON rb.did = r.did AND rb.cid = r.cid + \\WHERE r.did = ? AND r.collection = ? AND r.rkey = ? + , .{ did, collection, rkey }) catch return null; + if (row == null) return null; + defer row.?.deinit(); + + return .{ + .cid = allocator.dupe(u8, row.?.text(0)) catch return null, + .data = allocator.dupe(u8, row.?.nullableBlob(1) orelse return null) catch return null, + }; +} + pub fn getByUri(uri: []const u8) ?Record { db_mutex.lockUncancelable(store_io); defer db_mutex.unlock(store_io); @@ -2415,6 +2440,10 @@ fn recordJsonFromBlock(allocator: std.mem.Allocator, data: []const u8) ![]const return cbor_json.writeAlloc(allocator, value); } +pub fn recordJsonFromDagCbor(allocator: std.mem.Allocator, data: []const u8) ![]const u8 { + return recordJsonFromBlock(allocator, data); +} + fn oauthRequestFromRow(row: zqlite.Row, allocator: std.mem.Allocator) !OAuthRequest { return .{ .request_id = try allocator.dupe(u8, row.text(0)), -- 2.51.2