From a0a1efb7e8b3009578dbcfdb37ae927e922d908c Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Fri, 31 Jul 2026 11:54:18 -0500 Subject: [PATCH] Benchmark concurrent publisher pressure --- bench/README.md | 13 +- bench/fly/loadgen.mjs | 223 +++++++++++++++++++++++++++++++++ bench/fly/run.sh | 103 +++++++++++++++ bench/fly/target-entrypoint.sh | 59 +++++++++ bench/justfile | 8 ++ bench/main.zig | 152 +++++++++++++++++++++- 6 files changed, 556 insertions(+), 2 deletions(-) create mode 100644 bench/fly/loadgen.mjs create mode 100644 bench/fly/run.sh create mode 100644 bench/fly/target-entrypoint.sh diff --git a/bench/README.md b/bench/README.md index 1491598..c2747bd 100644 --- a/bench/README.md +++ b/bench/README.md @@ -26,6 +26,8 @@ just bench list-records 10 1000 just bench repo-size 5000 just bench get-repo 5000 4 5 just bench firehose 1000 32 100 +just bench publish-sync 100 3 3 3145728 +just bench fly-upload just bench repo-large 100000 just bench space 1000 just bench http @@ -65,6 +67,15 @@ just bench run --scenario write --records 10000 measuring resident account-read latency. Event-history reads must remain ordinary indexed range reads; migration or repair work in this path causes global database-lock contention that stalls login, writes, and blob reads. +- `publish-sync`: mixed workload modeled on continuous media publishers. It + runs concurrent multi-megabyte blob uploads and same-repo writes alongside + blob/repo sync reads, while measuring a small metadata lookup for unrelated + tail-latency stalls. +- `fly-upload`: end-to-end upload/write/read pressure on an isolated, + production-shaped Fly Machine. A separate load-generator Machine logs in 100 + synthetic accounts, publishes blobs and records at staged concurrency, and + records collateral authenticated-read latency. The temporary Fly app and all + of its Machines are deleted after the run. - `space`: experimental permissioned-data storage probes for space discovery, private record writes, private record reads/lists, repo oplog catch-up, and blob readback. @@ -221,7 +232,7 @@ High-level ZDS surface area represented in `bench/`: | repo reads | `get-cid`, `get-block`, `get-record`, `list-records` | | repo export/sync snapshots | `repo`, `repo-size`, `get-repo` | | sync event storage | `metastore` write path plus smoke tests | -| blob storage | `blob`, `space` blob readback | +| blob storage | `blob`, `publish-sync`, `space` blob readback | | HTTP routing/serialization/auth overhead | `http` | | OAuth/session/account routes | `http` route matrix | | permissioned spaces | `space` | diff --git a/bench/fly/loadgen.mjs b/bench/fly/loadgen.mjs new file mode 100644 index 0000000..4e875a1 --- /dev/null +++ b/bench/fly/loadgen.mjs @@ -0,0 +1,223 @@ +import { performance } from "node:perf_hooks"; + +const target = process.env.ZDS_BENCH_TARGET; +const accountCount = Number(process.env.ZDS_BENCH_ACCOUNTS ?? "100"); +const matrix = parseMatrix( + process.env.ZDS_BENCH_MATRIX ?? + "65536:20,100;3145728:1,3,5,10,20,40,60,100;20971520:1,3,5,10", +); +const timeoutMs = Number(process.env.ZDS_BENCH_TIMEOUT_MS ?? "60000"); + +if (!target) throw new Error("ZDS_BENCH_TARGET is required"); + +const sessions = await loginAccounts(); +const results = []; + +for (const { bytes, concurrencies } of matrix) { + const payload = Buffer.alloc(bytes, 0x5a); + for (const concurrency of concurrencies) { + const result = await runStage(payload, concurrency); + results.push(result); + console.log(JSON.stringify(result)); + } +} + +console.log("\nsize\tusers\tok\tfail\tpublish p50\tp95\tp99\tmax\tprobe p95"); +for (const row of results) { + console.log( + [ + formatBytes(row.bytes), + row.concurrency, + row.ok, + row.failed, + formatMs(row.publish.p50), + formatMs(row.publish.p95), + formatMs(row.publish.p99), + formatMs(row.publish.max), + formatMs(row.probe.p95), + ].join("\t"), + ); +} + +if (results.some((row) => row.failed > 0)) process.exitCode = 1; + +async function loginAccounts() { + const output = new Array(accountCount); + await parallel(accountCount, 20, async (index) => { + const number = index + 1; + const handle = `bench${String(number).padStart(3, "0")}.test`; + const response = await request("/xrpc/com.atproto.server.createSession", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ identifier: handle, password: "password" }), + }); + output[index] = { + did: response.did, + token: response.accessJwt, + }; + }); + return output; +} + +async function runStage(payload, concurrency) { + const publish = []; + const upload = []; + const write = []; + const verify = []; + const probe = []; + const errors = []; + let probing = true; + let probeIndex = 0; + + const probeTask = (async () => { + while (probing) { + const session = sessions[probeIndex++ % sessions.length]; + const started = performance.now(); + try { + await request("/xrpc/com.atproto.server.getSession", { + headers: { authorization: `Bearer ${session.token}` }, + }); + probe.push(performance.now() - started); + } catch (error) { + probe.push(performance.now() - started); + errors.push(`probe: ${error.message}`); + } + await sleep(25); + } + })(); + + const started = performance.now(); + await Promise.all( + sessions.slice(0, concurrency).map(async (session, index) => { + const operationStarted = performance.now(); + try { + const uploadStarted = performance.now(); + const uploaded = await request("/xrpc/com.atproto.repo.uploadBlob", { + method: "POST", + headers: { + authorization: `Bearer ${session.token}`, + "content-type": "audio/mpeg", + }, + body: payload, + }); + upload.push(performance.now() - uploadStarted); + + const rkey = `bench-${Date.now()}-${index}`; + const writeStarted = performance.now(); + await request("/xrpc/com.atproto.repo.createRecord", { + method: "POST", + headers: { + authorization: `Bearer ${session.token}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + repo: session.did, + collection: "com.example.bench.record", + rkey, + validate: false, + record: { + $type: "com.example.bench.record", + blob: uploaded.blob, + }, + }), + }); + write.push(performance.now() - writeStarted); + + const verifyStarted = performance.now(); + await request( + `/xrpc/com.atproto.repo.getRecord?repo=${encodeURIComponent(session.did)}&collection=com.example.bench.record&rkey=${encodeURIComponent(rkey)}`, + { headers: { authorization: `Bearer ${session.token}` } }, + ); + verify.push(performance.now() - verifyStarted); + publish.push(performance.now() - operationStarted); + } catch (error) { + publish.push(performance.now() - operationStarted); + errors.push(`user ${index + 1}: ${error.message}`); + } + }), + ); + const elapsedMs = performance.now() - started; + probing = false; + await probeTask; + + return { + bytes: payload.length, + concurrency, + elapsedMs, + ok: concurrency - errors.filter((error) => error.startsWith("user ")).length, + failed: errors.filter((error) => error.startsWith("user ")).length, + publish: distribution(publish), + upload: distribution(upload), + write: distribution(write), + verify: distribution(verify), + probe: distribution(probe), + errors: errors.slice(0, 10), + }; +} + +async function request(path, init = {}) { + const response = await fetch(`${target}${path}`, { + ...init, + signal: AbortSignal.timeout(timeoutMs), + }); + const text = await response.text(); + if (!response.ok) { + throw new Error(`${response.status} ${text.slice(0, 200)}`); + } + return text ? JSON.parse(text) : {}; +} + +async function parallel(count, concurrency, operation) { + let next = 0; + await Promise.all( + Array.from({ length: Math.min(count, concurrency) }, async () => { + while (true) { + const index = next++; + if (index >= count) return; + await operation(index); + } + }), + ); +} + +function distribution(values) { + if (values.length === 0) { + return { count: 0, p50: 0, p95: 0, p99: 0, max: 0 }; + } + const sorted = [...values].sort((a, b) => a - b); + return { + count: sorted.length, + p50: percentile(sorted, 0.5), + p95: percentile(sorted, 0.95), + p99: percentile(sorted, 0.99), + max: sorted.at(-1), + }; +} + +function percentile(sorted, quantile) { + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)]; +} + +function parseMatrix(value) { + return value.split(";").map((stage) => { + const [bytes, concurrencies] = stage.split(":"); + return { + bytes: Number(bytes), + concurrencies: concurrencies.split(",").map(Number), + }; + }); +} + +function formatBytes(value) { + return value >= 1024 * 1024 + ? `${(value / 1024 / 1024).toFixed(0)}MiB` + : `${(value / 1024).toFixed(0)}KiB`; +} + +function formatMs(value) { + return `${value.toFixed(1)}ms`; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/bench/fly/run.sh b/bench/fly/run.sh new file mode 100644 index 0000000..de89e41 --- /dev/null +++ b/bench/fly/run.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +region="${ZDS_BENCH_REGION:-ord}" +org="${ZDS_BENCH_ORG:-personal}" +account_count="${ZDS_BENCH_ACCOUNTS:-100}" +app="${ZDS_BENCH_APP:-zds-bench-$(date +%s)}" +label="bench-$(git -C "$repo_root" rev-parse --short=12 HEAD)-$(date +%s)" +image="registry.fly.io/${app}:${label}" +results_dir="${ZDS_BENCH_RESULTS_DIR:-$repo_root/bench/results}" +result_prefix="$results_dir/$(date -u +%Y%m%dT%H%M%SZ)-$(git -C "$repo_root" rev-parse --short=12 HEAD)" +target_id="" + +mkdir -p "$results_dir" + +cleanup() { + if [[ "${ZDS_BENCH_KEEP_APP:-false}" != "true" ]]; then + fly apps destroy "$app" --yes >/dev/null 2>&1 || true + else + printf 'kept Fly app %s\n' "$app" + fi +} +trap cleanup EXIT + +fly apps create "$app" --org "$org" --yes +fly deploy \ + --app "$app" \ + --config "$repo_root/fly.toml" \ + --build-only \ + --push \ + --image-label "$label" \ + --remote-only + +fly machine run "$image" \ + --app "$app" \ + --name zds-target \ + --region "$region" \ + --vm-cpu-kind shared \ + --vm-cpus 1 \ + --vm-memory 1024 \ + --rootfs-size 8gb \ + --restart no \ + --autostop off \ + --file-local "/app/zds-bench-target.sh=$repo_root/bench/fly/target-entrypoint.sh" \ + --entrypoint /bin/bash \ + --env ZDS_HOST=0.0.0.0 \ + --env ZDS_PORT=2583 \ + --env ZDS_DB=/data/zds.sqlite3 \ + --env ZDS_BLOBSTORE_PATH=/data/blobs \ + --env ZDS_PUBLIC_URL="https://${app}.fly.dev" \ + --env ZDS_SERVER_DID="did:web:${app}.fly.dev" \ + --env ZDS_HANDLE_DOMAINS=.test \ + --env ZDS_INVITE_REQUIRED=false \ + --env ZDS_PERMISSIONED_DATA=false \ + --env ZDS_BLOB_UPLOAD_LIMIT=100000000 \ + --env ZDS_BENCH_ACCOUNTS="$account_count" \ + --detach \ + /app/zds-bench-target.sh + +target_id="$(fly machines list --app "$app" --json | jq -r '.[] | select(.name == "zds-target") | .id')" +test -n "$target_id" + +for _ in $(seq 1 120); do + if fly ssh console --app "$app" --machine "$target_id" --command \ + "bash -lc 'exec 3<>/dev/tcp/127.0.0.1/2583'" >/dev/null 2>&1; then + break + fi + sleep 1 +done +fly ssh console --app "$app" --machine "$target_id" --command \ + "bash -lc 'exec 3<>/dev/tcp/127.0.0.1/2583'" + +set +e +fly machine run node:22-bookworm-slim \ + --app "$app" \ + --name zds-loadgen \ + --region "$region" \ + --vm-cpu-kind performance \ + --vm-cpus 4 \ + --vm-memory 4096 \ + --restart no \ + --rm \ + --file-local "/bench/loadgen.mjs=$repo_root/bench/fly/loadgen.mjs" \ + --entrypoint node \ + --env ZDS_BENCH_TARGET="http://${app}.internal:2583" \ + --env ZDS_BENCH_ACCOUNTS="$account_count" \ + --env ZDS_BENCH_MATRIX="${ZDS_BENCH_MATRIX:-65536:20,100;3145728:1,3,5,10,20,40,60,100;20971520:1,3,5,10}" \ + /bench/loadgen.mjs | + tee "${result_prefix}-loadgen.txt" +loadgen_status=${PIPESTATUS[0]} +set -e + +fly ssh console --app "$app" --machine "$target_id" --command \ + "cat /tmp/zds-bench-resources.csv" >"${result_prefix}-resources.csv" +fly machine status --app "$app" "$target_id" --display-config >"${result_prefix}-machine.txt" + +printf 'load generator: %s\nresources: %s\nmachine: %s\n' \ + "${result_prefix}-loadgen.txt" \ + "${result_prefix}-resources.csv" \ + "${result_prefix}-machine.txt" + +exit "$loadgen_status" diff --git a/bench/fly/target-entrypoint.sh b/bench/fly/target-entrypoint.sh new file mode 100644 index 0000000..ebb18fe --- /dev/null +++ b/bench/fly/target-entrypoint.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +db="${ZDS_DB:-/data/zds.sqlite3}" +account_count="${ZDS_BENCH_ACCOUNTS:-100}" + +zds & +bootstrap_pid=$! +for _ in $(seq 1 120); do + if sqlite3 "$db" "SELECT 1 FROM accounts LIMIT 1" >/dev/null 2>&1; then + break + fi + sleep 0.25 +done +sqlite3 "$db" "SELECT 1 FROM accounts LIMIT 1" >/dev/null +kill "$bootstrap_pid" +wait "$bootstrap_pid" || true + +sqlite3 "$db" </tmp/zds-bench-resources.csv +( + while true; do + timestamp="$(date +%s)" + memory_bytes="$(cat /sys/fs/cgroup/memory.current 2>/dev/null || printf 0)" + cpu_usage_usec="$(awk '/usage_usec/ { print $2; exit }' /sys/fs/cgroup/cpu.stat 2>/dev/null || printf 0)" + db_bytes="$(stat -c %s "$db" 2>/dev/null || printf 0)" + blob_bytes="$(du -sb "${ZDS_BLOBSTORE_PATH:-/data/blobs}" 2>/dev/null | awk '{ print $1 }' || printf 0)" + printf '%s,%s,%s,%s,%s\n' "$timestamp" "$memory_bytes" "$cpu_usage_usec" "$db_bytes" "$blob_bytes" + sleep 1 + done +) >>/tmp/zds-bench-resources.csv & + +exec zds diff --git a/bench/justfile b/bench/justfile index dcdfd08..b27e9e4 100644 --- a/bench/justfile +++ b/bench/justfile @@ -69,6 +69,14 @@ get-repo records="5000" callers="4" ops_per_caller="5": firehose records="1000" subscribers="32" polls_per_subscriber="100": {{zig}} build bench -Doptimize=ReleaseFast -- --scenario firehose --records {{records}} --callers {{subscribers}} --ops-per-caller {{polls_per_subscriber}} +# benchmark the production publisher shape: blob IO, repo writes, and sync reads together +publish-sync records="100" callers="3" ops_per_caller="3" blob_size="3145728": + {{zig}} build bench -Doptimize=ReleaseFast -- --scenario publish-sync --records {{records}} --callers {{callers}} --ops-per-caller {{ops_per_caller}} --blob-size {{blob_size}} + +# run the upload/write/read mix on disposable Fly machines +fly-upload: + ./fly/run.sh + # deliberately slow large-repo lane for Paul/Jerry-shaped synthetic repositories repo-large records="100000": {{zig}} build bench -Doptimize=ReleaseFast -- --scenario repo-size --records {{records}} diff --git a/bench/main.zig b/bench/main.zig index e78478b..e017dd0 100644 --- a/bench/main.zig +++ b/bench/main.zig @@ -17,6 +17,7 @@ const Scenario = enum { repo_size, get_repo, firehose, + publish_sync, space, write_profile, }; @@ -127,6 +128,7 @@ pub fn main(init: std.process.Init) !void { .repo_size => try benchRepoSizes(allocator, options.records), .get_repo => try benchGetRepo(allocator, options), .firehose => try benchFirehoseContention(allocator, options), + .publish_sync => try benchPublishSync(allocator, options), .space => try benchSpace(allocator, options), .write_profile => try benchWriteProfile(allocator, options), } @@ -621,6 +623,74 @@ fn firehosePollWorker(iterations: usize, start: *std.atomic.Value(bool)) void { } } +fn benchPublishSync(allocator: std.mem.Allocator, options: Options) !void { + const level: ConcurrencyLevel = .{ + .callers = options.callers orelse 3, + .ops_per_caller = options.ops_per_caller orelse 3, + }; + var state = try initBench(allocator); + defer state.deinit(); + try seedIndexedRecords(allocator, state.account, @max(options.records, 1)); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const a = arena.allocator(); + const blob_size = @max(options.blob_size, 1); + const seed_blob = try a.alloc(u8, blob_size); + @memset(seed_blob, 0x5a); + const seed_cid = try zds.storage.store.putBlob( + a, + std.Options.debug_io, + state.account, + seed_blob, + "audio/mpeg", + ); + + const total_ops = level.callers * level.ops_per_caller; + const publish_latencies = try allocator.alloc(u64, total_ops); + defer allocator.free(publish_latencies); + const sync_latencies = try allocator.alloc(u64, total_ops); + defer allocator.free(sync_latencies); + const probe_latencies = try allocator.alloc(u64, total_ops); + defer allocator.free(probe_latencies); + const threads = try allocator.alloc(std.Thread, level.callers); + defer allocator.free(threads); + var start: std.atomic.Value(bool) = .init(false); + + for (threads, 0..) |*thread, i| { + thread.* = try std.Thread.spawn(.{}, publishWorker, .{ + state.account, + blob_size, + i * level.ops_per_caller, + publish_latencies[i * level.ops_per_caller ..][0..level.ops_per_caller], + &start, + }); + } + const sync_thread = try std.Thread.spawn(.{}, publishSyncWorker, .{ + state.account.did, + seed_cid, + sync_latencies, + &start, + }); + const probe_thread = try std.Thread.spawn(.{}, publishProbeWorker, .{ + state.account.did, + probe_latencies, + &start, + }); + + const bench_start = nowNs(); + start.store(true, .release); + for (threads) |thread| thread.join(); + sync_thread.join(); + probe_thread.join(); + const elapsed = nowNs() - bench_start; + + std.debug.print("\n=== mixed publisher/sync benchmark ===\n", .{}); + concurrentResult("blob+repo publish", level, total_ops, elapsed, publish_latencies).print(); + concurrentResult("blob+repo sync", .{ .callers = 1, .ops_per_caller = total_ops }, total_ops, elapsed, sync_latencies).print(); + concurrentResult("metadata probe", .{ .callers = 1, .ops_per_caller = total_ops }, total_ops, elapsed, probe_latencies).print(); +} + fn benchRepoCarSince( allocator: std.mem.Allocator, account: zds.auth.tokens.Account, @@ -1083,6 +1153,76 @@ fn repoCarWorker(did: []const u8, latencies: []u64) !void { } } +fn publishWorker( + account: zds.auth.tokens.Account, + blob_size: usize, + offset: usize, + latencies: []u64, + start: *std.atomic.Value(bool), +) !void { + waitForBenchStart(start); + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena.deinit(); + for (latencies, 0..) |*latency, i| { + _ = arena.reset(.retain_capacity); + const a = arena.allocator(); + const payload = try a.alloc(u8, blob_size); + @memset(payload, @intCast((offset + i) % 251)); + if (payload.len > 0) payload[0] = @intCast((offset + i + 1) % 251); + const started = nowNs(); + const cid = try zds.storage.store.putBlob(a, std.Options.debug_io, account, payload, "audio/mpeg"); + const value = try blobRecordValue(a, offset + i, cid, blob_size); + const result = try zds.storage.store.applyWrites(a, account, &.{.{ .create = .{ + .collection = bench_collection, + .rkey = try std.fmt.allocPrint(a, "publish{d:0>8}", .{offset + i}), + .value = value, + } }}); + if (result.records.len != 1) return error.UnexpectedWriteResult; + latency.* = nowNs() - started; + } +} + +fn publishSyncWorker( + did: []const u8, + cid: []const u8, + latencies: []u64, + start: *std.atomic.Value(bool), +) !void { + waitForBenchStart(start); + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena.deinit(); + for (latencies) |*latency| { + _ = arena.reset(.retain_capacity); + const started = nowNs(); + const blob = zds.storage.store.getBlob(arena.allocator(), did, cid) orelse return error.MissingBlob; + if (blob.data.len == 0) return error.MissingBlob; + const car = try zds.storage.store.writeRepoCar(arena.allocator(), did); + if (car.len == 0) return error.EmptyRepoCar; + latency.* = nowNs() - started; + } +} + +fn publishProbeWorker( + did: []const u8, + latencies: []u64, + start: *std.atomic.Value(bool), +) !void { + waitForBenchStart(start); + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena.deinit(); + for (latencies) |*latency| { + _ = arena.reset(.retain_capacity); + const started = nowNs(); + const cid = zds.storage.store.getCid(arena.allocator(), did, bench_collection, "rec00000000") orelse return error.MissingRecord; + if (cid.len == 0) return error.MissingRecord; + latency.* = nowNs() - started; + } +} + +fn waitForBenchStart(start: *std.atomic.Value(bool)) void { + while (!start.load(.acquire)) std.Thread.yield() catch {}; +} + fn repoCarWorkerStarting( did: []const u8, latencies: []u64, @@ -1276,6 +1416,15 @@ fn benchRecordValue(allocator: std.mem.Allocator, index: usize) !std.json.Value return try std.json.parseFromSliceLeaky(std.json.Value, allocator, json, .{}); } +fn blobRecordValue(allocator: std.mem.Allocator, index: usize, cid: []const u8, size: usize) !std.json.Value { + const json = try std.fmt.allocPrint( + allocator, + "{{\"$type\":\"{s}\",\"text\":\"publish {d}\",\"audio\":{{\"$type\":\"blob\",\"ref\":{{\"$link\":{f}}},\"mimeType\":\"audio/mpeg\",\"size\":{d}}}}}", + .{ bench_collection, index, std.json.fmt(cid, .{}), size }, + ); + return try std.json.parseFromSliceLeaky(std.json.Value, allocator, json, .{}); +} + fn spaceRecordValue(allocator: std.mem.Allocator, index: usize) !std.json.Value { const json = try std.fmt.allocPrint( allocator, @@ -1340,6 +1489,7 @@ fn parseScenario(value: []const u8) !Scenario { if (std.mem.eql(u8, value, "repo-size")) return .repo_size; if (std.mem.eql(u8, value, "get-repo")) return .get_repo; if (std.mem.eql(u8, value, "firehose")) return .firehose; + if (std.mem.eql(u8, value, "publish-sync")) return .publish_sync; if (std.mem.eql(u8, value, "space")) return .space; if (std.mem.eql(u8, value, "write-profile")) return .write_profile; return error.UnknownScenario; @@ -1347,7 +1497,7 @@ fn parseScenario(value: []const u8) !Scenario { fn usage() void { std.debug.print( - \\usage: zds-bench [--scenario all|write|read|blob|repo|metastore|get-cid|get-block|decode-record|render-record|get-record|list-records|repo-size|get-repo|firehose|space|write-profile] [--records N] [--blobs N] [--blob-size BYTES] [--callers N --ops-per-caller N] + \\usage: zds-bench [--scenario all|write|read|blob|repo|metastore|get-cid|get-block|decode-record|render-record|get-record|list-records|repo-size|get-repo|firehose|publish-sync|space|write-profile] [--records N] [--blobs N] [--blob-size BYTES] [--callers N --ops-per-caller N] \\ , .{}); } -- 2.51.2