diff --git a/bench/README.md b/bench/README.md index 0e85340..31a14d0 100644 --- a/bench/README.md +++ b/bench/README.md @@ -47,9 +47,10 @@ just bench run --scenario write --records 10000 - `render-record`: DAG-CBOR decode plus JSON rendering, without SQLite. - `get-record`: full `com.atproto.repo.getRecord` storage materialization. - `list-records`: full `com.atproto.repo.listRecords` storage materialization. -- `http`: starts a local ReleaseFast ZDS server and measures selected HTTP - routes with ApacheBench. This is the route-level check for the `httpz` server - boundary and should be tracked separately from storage microbenchmarks. +- `http`: starts a local ReleaseFast ZDS server and measures a seeded HTTP + route matrix with ApacheBench. This is the route-level check for the `httpz` + server boundary and should be tracked separately from storage + microbenchmarks. - `official-pds`: copies the official-PDS read probe into a local `bluesky-social/atproto` checkout and runs it with Vitest. - `write-profile`: `applyWrites` stage timing under write concurrency. @@ -99,8 +100,27 @@ On this machine, with `--records 1000` unless noted: | blob put+get | 368 ops/s, 46 MB/s | Current local HTTP route baseline after adopting `httpz`, measured with -`just bench http` using 5000 requests at concurrency 50. The pre-httpz column -is the same route set run against commit `4841aec`. +`just bench http` using 5000 requests at concurrency 50 for normal routes and +500 requests at concurrency 10 for blob/CAR routes. The harness covers public +metadata, OAuth metadata, session/account reads, preference reads, repo reads, +sync reads, blob reads, and repo CAR reads. Stateful write endpoints and +WebSocket `subscribeRepos` need separate harnesses. + +| route group | httpz | +|---|---:| +| static/landing routes | 11.6k-40.2k req/s | +| public identity/server metadata | 31.9k-38.9k req/s | +| OAuth metadata/JWKS | 35.6k-37.0k req/s | +| OAuth authorize JSON | 4.4k req/s, p95 28 ms | +| session/account reads | 20.5k-34.9k req/s | +| `createSession` | 7.4k req/s, p95 20 ms | +| repo reads | 26.2k-39.0k req/s | +| `listRecords`, limit 50 | 26.2k req/s, p95 6 ms | +| sync JSON reads | 24.1k-39.4k req/s | +| blob/CAR reads | 34.5k-37.9k req/s | + +The pre-httpz column below is the original narrow route set run against commit +`4841aec`; it should not be treated as coverage for the expanded route matrix. | route | pre-httpz | httpz | |---|---:|---:| diff --git a/docs/benchmarking.md b/docs/benchmarking.md index 0b40fc7..2c93361 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -74,14 +74,52 @@ comparison gaps instead of being folded into the current table. ## HTTP route path -`just bench http` starts a local ReleaseFast ZDS server and measures selected -routes with ApacheBench. These numbers include HTTP request parsing, routing, -response serialization, `httpz` connection handling, and whatever endpoint work -the route performs. Keep them separate from the storage microbenchmarks above. +`just bench http` starts a local ReleaseFast ZDS server and measures a seeded +route matrix with ApacheBench. These numbers include HTTP request parsing, +routing, response serialization, auth/session work, `httpz` connection +handling, and whatever endpoint work the route performs. Keep them separate +from the storage microbenchmarks above. + +The current harness covers public metadata, OAuth metadata, session/account +reads, preference reads, repo reads, sync reads, blob reads, and repo CAR reads. +It deliberately does not loop over stateful write endpoints such as +`createRecord`, `applyWrites`, `uploadBlob`, or passkey registration because a +static ApacheBench body would mostly measure duplicate/error paths after the +first request. WebSocket `subscribeRepos` also needs a separate harness. Current local route baseline after adopting `httpz`, with 5000 requests at -concurrency 50. The pre-httpz column is the same route set run against commit -`4841aec`. +concurrency 50 for normal routes and 500 requests at concurrency 10 for the +larger blob/CAR routes: + +| route | httpz | +|---|---:| +| `/` | 15.0k req/s, p95 10 ms | +| `/favicon.svg` | 40.2k req/s, p95 1 ms | +| `/og-image` | 11.6k req/s, p95 5 ms | +| `/xrpc/_health` | 31.9k req/s, p95 1 ms | +| `/.well-known/did.json` | 38.9k req/s, p95 1 ms | +| `/.well-known/atproto-did` | 38.5k req/s, p95 1 ms | +| OAuth metadata/JWKS | 35.6k-37.0k req/s, p95 2 ms | +| `oauth/authorize` JSON | 4.4k req/s, p95 28 ms | +| `describeServer` | 38.7k req/s, p95 1 ms | +| `createSession` | 7.4k req/s, p95 20 ms | +| `getSession` | 23.8k req/s, p95 6 ms | +| `getServiceAuth` | 24.5k req/s, p95 5 ms | +| account/security reads | 20.5k-34.9k req/s, p95 4-7 ms | +| `getPreferences` | 32.0k req/s, p95 5 ms | +| `resolveHandle` | 23.9k req/s, p95 2 ms | +| `describeRepo` | 39.0k req/s, p95 1 ms | +| `getRecord` | 30.0k req/s, p95 5 ms | +| `listRecords`, limit 50 | 26.2k req/s, p95 6 ms | +| `listMissingBlobs` | 27.6k req/s, p95 6 ms | +| sync JSON reads | 24.1k-39.4k req/s, p95 1-4 ms | +| `sync.getBlob` | 34.5k req/s, p95 1 ms | +| `sync.getBlob` HEAD | 34.8k req/s, p95 1 ms | +| `sync.getRepo` | 37.9k req/s, p95 below 1 ms | +| `sync.getRepo` HEAD | 36.6k req/s, p95 below 1 ms | + +The only pre-httpz route numbers currently available are the original narrow +route set, run against commit `4841aec` with the same four probes: | route | pre-httpz | httpz | |---|---:|---:| diff --git a/tools/http_bench.sh b/tools/http_bench.sh index 650e381..4de769f 100755 --- a/tools/http_bench.sh +++ b/tools/http_bench.sh @@ -5,9 +5,13 @@ root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) port="${ZDS_HTTP_BENCH_PORT:-2586}" requests="${ZDS_HTTP_BENCH_REQUESTS:-5000}" concurrency="${ZDS_HTTP_BENCH_CONCURRENCY:-50}" +heavy_requests="${ZDS_HTTP_BENCH_HEAVY_REQUESTS:-500}" +heavy_concurrency="${ZDS_HTTP_BENCH_HEAVY_CONCURRENCY:-10}" db="${TMPDIR:-/tmp}/zds-http-bench.sqlite3" blob_root="${TMPDIR:-/tmp}/zds-http-bench-blobs" log="${TMPDIR:-/tmp}/zds-http-bench.log" +session_body="${TMPDIR:-/tmp}/zds-http-bench-create-session.json" +blob_payload="${TMPDIR:-/tmp}/zds-http-bench-blob.jpg" base="http://127.0.0.1:${port}" cleanup() { @@ -31,7 +35,30 @@ bench() { shift 2 echo echo "== $name ==" - ab -n "$requests" -c "$concurrency" "$@" "$url" | + ab -l -n "$requests" -c "$concurrency" "$@" "$url" | + awk ' + /Complete requests:/ || + /Failed requests:/ || + /Requests per second:/ || + /Time per request:/ || + /Transfer rate:/ || + /^ 50%/ || + /^ 95%/ || + /^ 99%/ { + print + } + ' +} + +bench_sized() { + name="$1" + url="$2" + route_requests="$3" + route_concurrency="$4" + shift 4 + echo + echo "== $name ==" + ab -l -n "$route_requests" -c "$route_concurrency" "$@" "$url" | awk ' /Complete requests:/ || /Failed requests:/ || @@ -52,7 +79,7 @@ need sqlite3 need zig cd "$root" -rm -f "$db" "$db-wal" "$db-shm" "$log" +rm -f "$db" "$db-wal" "$db-shm" "$log" "$session_body" "$blob_payload" rm -rf "$blob_root" mkdir -p "$blob_root" @@ -80,25 +107,74 @@ done curl -fsS "$base/xrpc/_health" >/dev/null sqlite3 "$db" "insert into accounts (did, handle, email, password_hash, activated_at, email_confirmed_at) values ('did:plc:httpbench', 'http-bench.test', 'http-bench@test.com', 'password', unixepoch(), unixepoch())" +sqlite3 "$db" "insert into oauth_requests (request_id, client_id, redirect_uri, scope, state, code_challenge, code_challenge_method, login_hint, expires_at) values ('http-bench-oauth', 'https://client.example/oauth-client.json', 'https://client.example/callback', 'repo:*?action=create blob:*/*', 'state', 'challenge', 'S256', 'http-bench.test', unixepoch() + 600)" +printf '%s' '{"identifier":"http-bench.test","password":"password"}' > "$session_body" session=$(curl -fsS -X POST "$base/xrpc/com.atproto.server.createSession" \ -H 'content-type: application/json' \ - --data '{"identifier":"http-bench.test","password":"password"}') + --data @"$session_body") token=$(printf '%s' "$session" | sed -n 's/.*"accessJwt":"\([^"]*\)".*/\1/p') test -n "$token" +curl -fsS -X POST "$base/xrpc/com.atproto.server.createInviteCode" \ + -H "authorization: Bearer http-bench-admin-token" \ + -H 'content-type: application/json' \ + --data '{"useCount":1,"forAccount":"did:plc:httpbench"}' >/dev/null +curl -fsS -X POST "$base/xrpc/app.bsky.actor.putPreferences" \ + -H "authorization: Bearer $token" \ + -H 'content-type: application/json' \ + --data '{"preferences":[{"$type":"app.bsky.actor.defs#contentLabelPref","label":"cats","visibility":"warn"}]}' >/dev/null + create=$(curl -fsS -X POST "$base/xrpc/com.atproto.repo.createRecord" \ -H "authorization: Bearer $token" \ -H 'content-type: application/json' \ --data '{"repo":"did:plc:httpbench","collection":"app.bsky.feed.post","record":{"$type":"app.bsky.feed.post","text":"http bench","createdAt":"2026-05-31T00:00:00.000Z"}}') printf '%s' "$create" | grep -q '"validationStatus":"valid"' +post_rkey=$(printf '%s' "$create" | sed -n 's#.*"uri":"at://did:plc:httpbench/app.bsky.feed.post/\([^"]*\)".*#\1#p') +test -n "$post_rkey" + +printf '\377\330\377\340zds-http-bench' > "$blob_payload" +blob=$(curl -fsS -X POST "$base/xrpc/com.atproto.repo.uploadBlob" \ + -H "authorization: Bearer $token" \ + -H 'content-type: image/jpeg' \ + --data-binary @"$blob_payload") +blob_cid=$(printf '%s' "$blob" | sed -n 's/.*"\$link":"\([^"]*\)".*/\1/p') +test -n "$blob_cid" echo "zds HTTP benchmarks" echo "requests=$requests concurrency=$concurrency" +echo "heavy_requests=$heavy_requests heavy_concurrency=$heavy_concurrency" echo "server_log=$log" +bench "root" "$base/" +bench "favicon" "$base/favicon.svg" +bench "ogImage" "$base/og-image" bench "health" "$base/xrpc/_health" +bench "didJson" "$base/.well-known/did.json" +bench "atprotoDid" "$base/.well-known/atproto-did" +bench "oauthProtectedResource" "$base/.well-known/oauth-protected-resource" +bench "oauthAuthorizationServer" "$base/.well-known/oauth-authorization-server" +bench "oauthJwks" "$base/oauth/jwks" +bench "oauthAuthorizeJson" "$base/oauth/authorize?request_uri=urn%3Aietf%3Aparams%3Aoauth%3Arequest_uri%3Ahttp-bench-oauth" -H "accept: application/json" bench "describeServer" "$base/xrpc/com.atproto.server.describeServer" +bench "createSession" "$base/xrpc/com.atproto.server.createSession" -p "$session_body" -T "application/json" bench "getSession" "$base/xrpc/com.atproto.server.getSession" -H "authorization: Bearer $token" +bench "getServiceAuth" "$base/xrpc/com.atproto.server.getServiceAuth?aud=did%3Aplc%3Aappview&lxm=app.bsky.feed.getTimeline" -H "authorization: Bearer $token" +bench "checkAccountStatus" "$base/xrpc/com.atproto.server.checkAccountStatus" -H "authorization: Bearer $token" +bench "getAccountInviteCodes" "$base/xrpc/com.atproto.server.getAccountInviteCodes" -H "authorization: Bearer $token" +bench "listAppPasswords" "$base/xrpc/com.atproto.server.listAppPasswords" -H "authorization: Bearer $token" +bench "listPasskeys" "$base/xrpc/com.atproto.server.listPasskeys" -H "authorization: Bearer $token" +bench "getPreferences" "$base/xrpc/app.bsky.actor.getPreferences" -H "authorization: Bearer $token" +bench "resolveHandle" "$base/xrpc/com.atproto.identity.resolveHandle?handle=http-bench.test" +bench "describeRepo" "$base/xrpc/com.atproto.repo.describeRepo?repo=did:plc:httpbench" +bench "getRecord" "$base/xrpc/com.atproto.repo.getRecord?repo=did:plc:httpbench&collection=app.bsky.feed.post&rkey=$post_rkey" bench "listRecords" "$base/xrpc/com.atproto.repo.listRecords?repo=did:plc:httpbench&collection=app.bsky.feed.post&limit=50" -H "authorization: Bearer $token" - +bench "listMissingBlobs" "$base/xrpc/com.atproto.repo.listMissingBlobs?repo=did:plc:httpbench" -H "authorization: Bearer $token" +bench "syncListRepos" "$base/xrpc/com.atproto.sync.listRepos?limit=50" +bench "syncListBlobs" "$base/xrpc/com.atproto.sync.listBlobs?did=did:plc:httpbench&limit=50" +bench "syncGetLatestCommit" "$base/xrpc/com.atproto.sync.getLatestCommit?did=did:plc:httpbench" +bench "syncGetRepoStatus" "$base/xrpc/com.atproto.sync.getRepoStatus?did=did:plc:httpbench" +bench_sized "syncGetBlob" "$base/xrpc/com.atproto.sync.getBlob?did=did:plc:httpbench&cid=$blob_cid" "$heavy_requests" "$heavy_concurrency" +bench_sized "syncGetBlobHead" "$base/xrpc/com.atproto.sync.getBlob?did=did:plc:httpbench&cid=$blob_cid" "$heavy_requests" "$heavy_concurrency" -m HEAD +bench_sized "syncGetRepo" "$base/xrpc/com.atproto.sync.getRepo?did=did:plc:httpbench" "$heavy_requests" "$heavy_concurrency" +bench_sized "syncGetRepoHead" "$base/xrpc/com.atproto.sync.getRepo?did=did:plc:httpbench" "$heavy_requests" "$heavy_concurrency" -m HEAD