From 2f774da07530b060b0050961f24d5a58915e09e7 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:54:48 +0200 Subject: [PATCH 1/2] add benchmark ci --- .claude/settings.local.json | 18 -- .github/workflows/benchmark.yml | 95 +++++++++ .../calendar-happyview-2.12-comparison.json | 188 ++++++++++++++++++ 3 files changed, 283 insertions(+), 18 deletions(-) delete mode 100644 .claude/settings.local.json create mode 100644 .github/workflows/benchmark.yml create mode 100644 apps/benchmark/baselines/calendar-happyview-2.12-comparison.json diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index a756c0a..0000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(pnpm generate:*)", - "Bash(pnpm typecheck:*)", - "Bash(npx tsx:*)", - "Bash(npx tsc:*)", - "Bash(pnpm test:*)", - "Bash(npx vitest:*)", - "Bash(npm test:*)", - "Bash(npx lex-cli:*)", - "Bash(pnpm vitest:*)", - "WebFetch(domain:flo-bit.dev)", - "WebFetch(domain:atproto.com)", - "WebFetch(domain:docs.bsky.app)" - ] - } -} diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..e9ef21e --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,95 @@ +name: Pull request benchmark + +on: + pull_request: + +permissions: + contents: read + +concurrency: + group: pr-benchmark-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + benchmark: + name: Native SQLite backfill + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out code + uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + with: + version: 10.30.1 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22.14 + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build Contrail + run: pnpm --filter @atmo-dev/contrail build + + - name: Run benchmark + run: >- + pnpm bench + --backend sqlite + --config calendar-records-only.config.json + --concurrency 100 + --pds-concurrency 20 + --dids-per-pds 3 + --max-attempts 1 + + - name: Publish benchmark summary + if: always() + shell: bash + run: | + result="$(find apps/benchmark/results -type f -name 'calendar-records-only-sqlite-*.json' -print -quit)" + if [[ -z "$result" ]]; then + printf '## Native SQLite benchmark\n\nThe benchmark did not produce a result.\n' >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + RESULT_PATH="$result" node --input-type=module <<'NODE' + import { appendFileSync, readFileSync } from "node:fs"; + import { basename } from "node:path"; + + const result = JSON.parse(readFileSync(process.env.RESULT_PATH, "utf8")); + const seconds = (milliseconds) => `${(Number(milliseconds) / 1000).toFixed(2)} s`; + const integer = (value) => Number(value).toLocaleString("en-US"); + const decimal = (value) => Number(value).toFixed(2); + const accounts = result.backfill?.accounts ?? {}; + const lines = [ + "## Native SQLite benchmark", + "", + "| Metric | Result |", + "| --- | ---: |", + `| Total time | ${seconds(result.timings_ms.total)} |`, + `| Backfill time | ${seconds(result.timings_ms.backfill)} |`, + `| Indexed records | ${integer(result.indexed_records)} |`, + `| Accepted records/s | ${decimal(result.throughput.accepted_records_per_second)} |`, + `| Indexed records/s | ${decimal(result.throughput.indexed_records_per_second)} |`, + `| Complete accounts | ${integer(accounts.complete ?? 0)} |`, + `| Retrying accounts | ${integer(accounts.retrying ?? 0)} |`, + "", + `Runtime: Node ${result.runtime.node}, SQLite ${result.runtime.sqlite ?? "unknown"}`, + "", + `Artifact source: \`${basename(process.env.RESULT_PATH)}\``, + ]; + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${lines.join("\n")}\n`); + NODE + + - name: Upload benchmark result + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-pr-${{ github.event.pull_request.number }}-${{ github.run_attempt }} + path: apps/benchmark/results/*.json + if-no-files-found: ignore + retention-days: 14 diff --git a/apps/benchmark/baselines/calendar-happyview-2.12-comparison.json b/apps/benchmark/baselines/calendar-happyview-2.12-comparison.json new file mode 100644 index 0000000..10ff688 --- /dev/null +++ b/apps/benchmark/baselines/calendar-happyview-2.12-comparison.json @@ -0,0 +1,188 @@ +{ + "format": "contrail.comparison-benchmark", + "version": 1, + "observed_at": "2026-08-05T14:50:24Z", + "workload": { + "collections": [ + "community.lexicon.calendar.event", + "community.lexicon.calendar.rsvp" + ], + "profiles": false, + "follows": false, + "feeds": false, + "full_text_search": false, + "relation_counts": false, + "field_query_indexes": false, + "live_ingestion_during_measurement": false, + "discovered_accounts": 1633, + "backend": "native-sqlite", + "identity_concurrency": 100, + "pds_concurrency": 10, + "dids_per_pds": 3 + }, + "happyview": { + "version": "2.12.0", + "commit": "a5caf5b879d4d5a3b1c25b79b815d7df6d7026de", + "source_patch": null, + "runtime": { + "rustc": "rustc 1.96.1 (31fca3adb 2026-06-26)", + "target": "aarch64-apple-darwin", + "os": "26.4", + "hardware": "MacBookPro18,3" + }, + "default_retry_run": { + "max_attempts": 3, + "job_ms": 205539.548, + "harness_wall_ms": 205687.39, + "indexed_records": 20751, + "records": { + "community.lexicon.calendar.event": 14612, + "community.lexicon.calendar.rsvp": 6139 + }, + "indexed_records_per_second": 100.9, + "repos": { + "discovered": 1633, + "resolved": 1630, + "processed_reported": 1630, + "completed_rows": 1630, + "pending_rows": 3 + }, + "completion": { + "status": "completed", + "durable_error_rows": 84, + "error_counts": { + "connection_failed": 8, + "did_doc_not_found": 1, + "dns_failure": 2, + "other": 2, + "pds_server_error": 1, + "repo_not_found": 33, + "timeout": 37 + } + }, + "database_bytes": 53055488, + "peak_rss_bytes": null, + "raw_result": "../results/happyview-2.12-records-only-2026-08-05T14-42-43-323014Z.json" + }, + "matched_retry_run": { + "max_attempts": 1, + "job_ms": 73590.165, + "harness_wall_ms": 73632.29, + "indexed_records": 20738, + "records": { + "community.lexicon.calendar.event": 14607, + "community.lexicon.calendar.rsvp": 6131 + }, + "indexed_records_per_second": 281.6, + "repos": { + "discovered": 1633, + "resolved": 1630, + "processed_reported": 1630, + "completed_rows": 1630, + "pending_rows": 3 + }, + "completion": { + "status": "completed", + "durable_error_rows": 91, + "error_counts": { + "connection_failed": 8, + "did_doc_not_found": 1, + "dns_failure": 2, + "other": 9, + "pds_server_error": 1, + "repo_not_found": 33, + "timeout": 37 + } + }, + "database_bytes": 52924416, + "peak_rss_bytes": null, + "successful_request_count": null, + "cumulative_request_ms": null, + "raw_result": "../results/happyview-2.12-records-only-2026-08-05T14-47-56-569071Z.json" + } + }, + "contrail_bracketing_runs": [ + { + "position": "before_happyview_matched", + "commit": "012c4d744b7878533b00ec79fc01636ebd1f8772", + "runtime": { + "node": "v26.5.0", + "sqlite": "3.53.3" + }, + "max_attempts": 1, + "discovery_and_backfill_ms": 63886.26, + "full_harness_ms": 63905.13, + "indexed_records": 20760, + "records": { + "community.lexicon.calendar.event": 14610, + "community.lexicon.calendar.rsvp": 6150 + }, + "accounts": { + "complete": 1552, + "pending": 0, + "retrying": 81, + "failed": 0 + }, + "network": { + "requests": 5327, + "errors": 52, + "cumulative_request_ms": 1474297.32, + "max_concurrent": 127 + }, + "peak_rss_kib": 335168, + "raw_result": "../results/calendar-records-only-sqlite-r100-h10-d3-2026-08-05T14-47-32-558Z.json" + }, + { + "position": "after_happyview_matched", + "commit": "012c4d744b7878533b00ec79fc01636ebd1f8772", + "runtime": { + "node": "v26.5.0", + "sqlite": "3.53.3" + }, + "max_attempts": 1, + "discovery_and_backfill_ms": 61565.76, + "full_harness_ms": 61577.62, + "indexed_records": 20765, + "records": { + "community.lexicon.calendar.event": 14614, + "community.lexicon.calendar.rsvp": 6151 + }, + "accounts": { + "complete": 1554, + "pending": 0, + "retrying": 79, + "failed": 0 + }, + "network": { + "requests": 5329, + "errors": 50, + "cumulative_request_ms": 1451698.97, + "max_concurrent": 129 + }, + "peak_rss_kib": 407072, + "raw_result": "../results/calendar-records-only-sqlite-r100-h10-d3-2026-08-05T14-50-23-917Z.json" + } + ], + "observations": { + "matched_happyview_wall_ms": 73632.29, + "contrail_bracketing_wall_ms": { + "minimum": 61577.62, + "maximum": 63905.13, + "mean": 62741.375 + }, + "happyview_slower_than_bracketing_mean_percent": 17.36, + "record_count_difference_from_bracketing_runs": { + "minimum": 22, + "maximum": 27, + "maximum_percent": 0.13 + } + }, + "notes": [ + "The matched comparison is the middle Contrail-HappyView-Contrail sequence with one immediate attempt in every run. HappyView's three-attempt default is retained separately as a retry-behavior observation, not used for the matched speed comparison.", + "The live network was materially less healthy than the earlier 2.11.8 comparison: Contrail retained 79-81 retrying accounts and HappyView recorded 91 durable errors in its matched run. The matched record totals nevertheless differed by at most 27 records (0.13%).", + "This single bracketing sequence observed HappyView 17.36% slower than the mean of the adjacent Contrail runs. It is not enough to claim a stable 2.12 regression or a general Contrail speedup; repeated alternating runs are still required.", + "HappyView reported status=completed while retaining three unresolved repos and durable fetch/resolution errors. Contrail reported overall state=complete separately from 79-81 account-level retrying states.", + "HappyView 2.12 does not expose successful upstream request counts or cumulative request time through this benchmark path. Peak RSS was not retained because the first harness monitor failed to finalize its sample before result serialization.", + "The prior HappyView 2.11.8 result used an uncommitted scheduler fix. HappyView 2.12.0 contains the upstream deadlock fix at commit 5d840f3 and was benchmarked without a source patch." + ] +} -- 2.51.2 From 492b796158d06c766df6c3b5631521d4344e5e92 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:54:42 +0200 Subject: [PATCH 2/2] comment with benchmark results --- .github/workflows/benchmark.yml | 57 ++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e9ef21e..4ebb3f4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -5,6 +5,7 @@ on: permissions: contents: read + pull-requests: write concurrency: group: pr-benchmark-${{ github.event.pull_request.number }} @@ -52,13 +53,14 @@ jobs: run: | result="$(find apps/benchmark/results -type f -name 'calendar-records-only-sqlite-*.json' -print -quit)" if [[ -z "$result" ]]; then - printf '## Native SQLite benchmark\n\nThe benchmark did not produce a result.\n' >> "$GITHUB_STEP_SUMMARY" + printf '## Native SQLite benchmark\n\nThe benchmark did not produce a result.\n' > "$RUNNER_TEMP/benchmark-comment.md" + cat "$RUNNER_TEMP/benchmark-comment.md" >> "$GITHUB_STEP_SUMMARY" exit 0 fi RESULT_PATH="$result" node --input-type=module <<'NODE' - import { appendFileSync, readFileSync } from "node:fs"; - import { basename } from "node:path"; + import { appendFileSync, readFileSync, writeFileSync } from "node:fs"; + import { basename, join } from "node:path"; const result = JSON.parse(readFileSync(process.env.RESULT_PATH, "utf8")); const seconds = (milliseconds) => `${(Number(milliseconds) / 1000).toFixed(2)} s`; @@ -82,9 +84,56 @@ jobs: "", `Artifact source: \`${basename(process.env.RESULT_PATH)}\``, ]; - appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${lines.join("\n")}\n`); + const report = `${lines.join("\n")}\n`; + appendFileSync(process.env.GITHUB_STEP_SUMMARY, report); + writeFileSync(join(process.env.RUNNER_TEMP, "benchmark-comment.md"), report); NODE + - name: Comment on pull request + if: always() && github.event.pull_request.head.repo.full_name == github.repository + uses: actions/github-script@v7 + with: + script: | + const { readFileSync } = require("node:fs"); + const { join } = require("node:path"); + const marker = ""; + const report = readFileSync( + join(process.env.RUNNER_TEMP, "benchmark-comment.md"), + "utf8", + ); + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = `${marker}\n${report}\n[View workflow run](${runUrl})`; + const comments = await github.paginate( + github.rest.issues.listComments, + { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }, + ); + const existing = comments.find( + (comment) => + comment.user?.login === "github-actions[bot]" && + comment.body?.includes(marker), + ); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + - name: Upload benchmark result if: always() uses: actions/upload-artifact@v4 -- 2.51.2