From 27894ecdd9cf8a9b6de7d6be1ab23893af6c421f Mon Sep 17 00:00:00 2001 From: Vaclav Vancura Date: Sun, 29 Mar 2026 09:52:08 +0200 Subject: [PATCH] feat(ci): add Tier 2 GPU performance benchmarks to workflow - Introduced a new `perf` job for Tier 2 GPU performance benchmarks in `ci.yml`: - Runs on `push` or PR events labeled with `perf-tier-2`. - Includes steps for running GPU performance tests, uploading results, and comparing to baselines. - Posts PR comments with performance comparisons and fails on regressions exceeding 50%. - Updated `compare-benchmarks.mjs` to `compare-tier-1-benchmarks.mjs` for naming consistency. - Added `scripts/compare-tier-2-perf-results.mjs` for Tier 2 performance result comparison. Co-Authored-By: Claude Signed-off-by: Vaclav Vancura --- .github/workflows/ci.yml | 202 +++++++- ...arks.mjs => compare-tier-1-benchmarks.mjs} | 0 scripts/compare-tier-2-perf-results.mjs | 471 ++++++++++++++++++ 3 files changed, 668 insertions(+), 5 deletions(-) rename scripts/{compare-benchmarks.mjs => compare-tier-1-benchmarks.mjs} (100%) create mode 100644 scripts/compare-tier-2-perf-results.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7046ba7..d4f6c58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest if: github.event_name != 'pull_request' || !contains(fromJSON('["labeled","unlabeled"]'), github.event.action) || - github.event.label.name == 'perf-tier-1' + (github.event.action == 'labeled' && contains(fromJSON('["perf-tier-1","perf-tier-2"]'), github.event.label.name)) steps: - name: Checkout code @@ -49,9 +49,7 @@ jobs: name: Build Library runs-on: ubuntu-latest needs: quality - if: - github.event_name != 'pull_request' || !contains(fromJSON('["labeled","unlabeled"]'), github.event.action) || - github.event.label.name == 'perf-tier-1' + if: github.event_name != 'pull_request' || !contains(fromJSON('["labeled","unlabeled"]'), github.event.action) steps: - name: Checkout code @@ -196,7 +194,7 @@ jobs: - name: Compare benchmark results if: github.event_name == 'pull_request' run: > - node scripts/compare-benchmarks.mjs --current benchmark-results.json --baseline + node scripts/compare-tier-1-benchmarks.mjs --current benchmark-results.json --baseline baseline-artifact/benchmark-results.json --json-out benchmark-comparison.json --markdown-out benchmark-comment.md --threshold 10 @@ -274,6 +272,200 @@ jobs: } EOF + perf: + name: Tier 2 GPU Perf + runs-on: ubuntu-latest + needs: quality + if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'perf-tier-2') + concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-perf + cancel-in-progress: true + permissions: + actions: read + contents: read + pull-requests: write + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.26.2 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: '20' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Install Chrome for Playwright perf tests + run: pnpm exec playwright install --with-deps chrome + + - name: Run Tier 2 GPU perf tests + run: pnpm test:perf + + - name: Upload current perf results + uses: actions/upload-artifact@v7 + with: + name: perf-results-current + path: test-results/perf/perf-results.json + if-no-files-found: error + retention-days: 14 + + - name: Upload main perf baseline + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@v7 + with: + name: perf-baseline + path: test-results/perf/perf-results.json + if-no-files-found: error + retention-days: 90 + + - name: Find latest main perf baseline + id: find-perf-baseline + if: github.event_name == 'pull_request' + uses: actions/github-script@v8 + with: + script: | + const workflowId = 'ci.yml'; + const runs = await github.paginate(github.rest.actions.listWorkflowRuns, { + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: workflowId, + branch: 'main', + event: 'push', + status: 'completed', + per_page: 100, + }); + + for (const run of runs) { + if (run.conclusion !== 'success') { + continue; + } + + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id, + per_page: 100, + }); + const baselineArtifact = artifacts.find( + (artifact) => artifact.name === 'perf-baseline' && artifact.expired === false, + ); + + if (!baselineArtifact) { + continue; + } + + core.setOutput('found', 'true'); + core.setOutput('download-url', baselineArtifact.archive_download_url); + core.setOutput('artifact-id', String(baselineArtifact.id)); + core.setOutput('run-id', String(run.id)); + return; + } + + core.setOutput('found', 'false'); + + - name: Download main perf baseline + if: github.event_name == 'pull_request' && steps.find-perf-baseline.outputs.found == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + BASELINE_URL: ${{ steps.find-perf-baseline.outputs.download-url }} + run: | + mkdir -p perf-baseline-artifact + curl --fail --location \ + --header "Authorization: Bearer ${GITHUB_TOKEN}" \ + --header "Accept: application/vnd.github+json" \ + "${BASELINE_URL}" \ + --output perf-baseline-artifact.zip + unzip -o perf-baseline-artifact.zip -d perf-baseline-artifact + + - name: Compare perf results + if: github.event_name == 'pull_request' + run: > + node scripts/compare-tier-2-perf-results.mjs --current test-results/perf/perf-results.json --baseline + perf-baseline-artifact/perf-results.json --json-out perf-comparison.json --markdown-out perf-comment.md + --threshold 50 + + - name: Upload perf comparison artifacts + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v7 + with: + name: perf-comparison + path: | + perf-comparison.json + perf-comment.md + test-results/perf/perf-results.json + if-no-files-found: ignore + retention-days: 14 + + - name: Post perf PR comment + if: + github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false && github.actor != + 'dependabot[bot]' + uses: actions/github-script@v8 + env: + PERF_COMMENT_PATH: perf-comment.md + with: + script: | + const fs = require('node:fs'); + const marker = ''; + const body = fs.readFileSync(process.env.PERF_COMMENT_PATH, 'utf8'); + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + const existingComment = comments.find((comment) => + comment.user?.type === 'Bot' && comment.body?.includes(marker), + ); + + if (existingComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existingComment.id, + body, + }); + return; + } + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + + - name: Fail on perf regressions + if: github.event_name == 'pull_request' + run: | + node --input-type=module <<'EOF' + import fs from 'node:fs'; + + if (!fs.existsSync('perf-comparison.json')) { + process.exit(0); + } + + const report = JSON.parse(fs.readFileSync('perf-comparison.json', 'utf8')); + + if (!report.hasBaseline) { + process.exit(0); + } + + const hasFailures = report.summary.regressions > 0 || report.summary.missingScenarios > 0; + + if (hasFailures) { + process.exit(1); + } + EOF + # Placeholder for tests - will run when tests are added test: name: Run Tests diff --git a/scripts/compare-benchmarks.mjs b/scripts/compare-tier-1-benchmarks.mjs similarity index 100% rename from scripts/compare-benchmarks.mjs rename to scripts/compare-tier-1-benchmarks.mjs diff --git a/scripts/compare-tier-2-perf-results.mjs b/scripts/compare-tier-2-perf-results.mjs new file mode 100644 index 0000000..303660c --- /dev/null +++ b/scripts/compare-tier-2-perf-results.mjs @@ -0,0 +1,471 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +const COMMENT_MARKER = ''; +const DEFAULT_THRESHOLD = 50; + +/** + * Parses CLI arguments for the perf comparison command. + * + * @param {string[]} argv Raw CLI arguments after the node/script prefix. + * @returns {{ + * baseline: string | null, + * current: string | null, + * jsonOut: string, + * markdownOut: string, + * threshold: number, + * }} Parsed command options. + */ +function parseArgs(argv) { + const args = { + baseline: null, + current: null, + jsonOut: 'perf-comparison.json', + markdownOut: 'perf-comment.md', + threshold: DEFAULT_THRESHOLD, + }; + + for (let index = 0; index < argv.length; index += 1) { + // eslint-disable-next-line security/detect-object-injection -- CLI flags are parsed from a fixed argv array shape. + const value = argv[index]; + const nextValue = argv[index + 1]; + + if (!value.startsWith('--')) { + continue; + } + + if (nextValue === undefined) { + throw new Error(`Missing value for argument: ${value}`); + } + + switch (value) { + case '--baseline': + args.baseline = nextValue; + index += 1; + break; + case '--current': + args.current = nextValue; + index += 1; + break; + case '--json-out': + args.jsonOut = nextValue; + index += 1; + break; + case '--markdown-out': + args.markdownOut = nextValue; + index += 1; + break; + case '--threshold': + args.threshold = Number(nextValue); + index += 1; + break; + default: + throw new Error(`Unknown argument: ${value}`); + } + } + + if (!args.current) { + throw new Error('The --current argument is required'); + } + + if (!Number.isFinite(args.threshold) || args.threshold < 0) { + throw new Error(`Invalid --threshold value: ${String(args.threshold)}`); + } + + return args; +} + +/** + * Ensures that the parent directory for an output file exists. + * + * @param {string} filePath Output file path. + * @returns {void} + */ +function ensureParentDirectory(filePath) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); +} + +/** + * Reads and parses a JSON file from disk. + * + * @param {string} filePath JSON file path. + * @returns {unknown} Parsed JSON value. + */ +function readJsonFile(filePath) { + return JSON.parse(fs.readFileSync(filePath, 'utf8')); +} + +/** + * Throws when a required condition is not met. + * + * @param {unknown} condition Condition to evaluate. + * @param {string} message Error message for failed assertions. + * @returns {void} + */ +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +/** + * Validates the top-level shape of a perf benchmark report. + * + * @param {unknown} report Parsed perf report JSON. + * @param {string} sourceLabel Human-readable source label for error messages. + * @returns {void} + */ +function validatePerfReport(report, sourceLabel) { + assert(report !== null && typeof report === 'object', `Invalid perf report in ${sourceLabel}: expected object`); + assert(Array.isArray(report.scenarios), `Invalid perf report in ${sourceLabel}: report.scenarios must be an array`); +} + +/** + * Flattens the perf report into scenario entries suitable for comparison. + * + * @param {{ scenarios: unknown[], __sourceLabel?: string }} report Parsed perf report. + * @returns {Array<{ + * fixture: string, + * label: string, + * matchKey: string, + * name: string, + * stats: { + * frames: number, + * max: number, + * median: number, + * min: number, + * p95: number, + * p99: number, + * }, + * }>} Flattened perf scenario entries. + */ +function flattenPerfScenarios(report) { + const entries = []; + const reportLabel = report.__sourceLabel ?? 'perf report'; + + validatePerfReport(report, reportLabel); + + for (const [scenarioIndex, scenario] of report.scenarios.entries()) { + assert( + scenario !== null && typeof scenario === 'object', + `Invalid scenario entry at index ${scenarioIndex} in ${reportLabel}`, + ); + assert( + typeof scenario.fixture === 'string', + `Invalid scenario.fixture at index ${scenarioIndex} in ${reportLabel}`, + ); + assert(typeof scenario.name === 'string', `Invalid scenario.name at index ${scenarioIndex} in ${reportLabel}`); + assert( + scenario.stats !== null && typeof scenario.stats === 'object', + `Invalid scenario.stats for ${scenario.name} in ${reportLabel}`, + ); + assert(Number.isFinite(scenario.stats.frames), `Invalid stats.frames for ${scenario.name} in ${reportLabel}`); + assert(Number.isFinite(scenario.stats.median), `Invalid stats.median for ${scenario.name} in ${reportLabel}`); + assert(Number.isFinite(scenario.stats.p95), `Invalid stats.p95 for ${scenario.name} in ${reportLabel}`); + assert(Number.isFinite(scenario.stats.p99), `Invalid stats.p99 for ${scenario.name} in ${reportLabel}`); + assert(Number.isFinite(scenario.stats.min), `Invalid stats.min for ${scenario.name} in ${reportLabel}`); + assert(Number.isFinite(scenario.stats.max), `Invalid stats.max for ${scenario.name} in ${reportLabel}`); + + const matchKey = `${scenario.fixture}::${scenario.name}`; + + entries.push({ + fixture: scenario.fixture, + label: scenario.name, + matchKey, + name: scenario.name, + stats: scenario.stats, + }); + } + + return entries; +} + +/** + * Calculates the percentage change for a frame-time metric. + * + * @param {number} baselineValue Baseline frame-time value. + * @param {number} currentValue Current frame-time value. + * @returns {number} Percentage change where positive means slower. + */ +function calculateRegressionPct(baselineValue, currentValue) { + return ((currentValue - baselineValue) / baselineValue) * 100; +} + +/** + * Compares current perf results against an optional baseline report. + * + * @param {{ scenarios: unknown[], __sourceLabel?: string }} currentReport Current perf report. + * @param {{ scenarios: unknown[], __sourceLabel?: string } | null} baselineReport Baseline perf report, if available. + * @param {number} thresholdPct Maximum allowed slowdown percentage before failure. + * @returns {{ + * generatedAt: string, + * hasBaseline: boolean, + * thresholdPct: number, + * summary: { + * total: number, + * compared: number, + * regressions: number, + * improvements: number, + * pass: number, + * newScenarios: number, + * missingScenarios: number, + * }, + * scenarios: Array<{ + * fixture: string, + * name: string, + * baselineStats: { + * frames: number, + * max: number, + * median: number, + * min: number, + * p95: number, + * p99: number, + * } | null, + * currentStats: { + * frames: number, + * max: number, + * median: number, + * min: number, + * p95: number, + * p99: number, + * } | null, + * deltas: { + * median: number | null, + * p95: number | null, + * p99: number | null, + * }, + * status: string, + * }>, + * }} Comparison report used by CI and PR comments. + */ +function comparePerfReports(currentReport, baselineReport, thresholdPct) { + const currentEntries = flattenPerfScenarios(currentReport); + const baselineEntries = baselineReport ? flattenPerfScenarios(baselineReport) : []; + const currentByKey = new Map(currentEntries.map((entry) => [entry.matchKey, entry])); + const baselineByKey = new Map(baselineEntries.map((entry) => [entry.matchKey, entry])); + const keys = [...new Set([...baselineByKey.keys(), ...currentByKey.keys()])].sort((left, right) => + left.localeCompare(right), + ); + + const scenarios = keys.map((key) => { + const baselineEntry = baselineByKey.get(key) ?? null; + const currentEntry = currentByKey.get(key) ?? null; + + if (!baselineEntry) { + return { + fixture: currentEntry?.fixture ?? 'unknown', + name: currentEntry?.label ?? key, + baselineStats: null, + currentStats: currentEntry?.stats ?? null, + deltas: { median: null, p95: null, p99: null }, + status: 'new', + }; + } + + if (!currentEntry) { + return { + fixture: baselineEntry.fixture, + name: baselineEntry.label, + baselineStats: baselineEntry.stats, + currentStats: null, + deltas: { median: null, p95: null, p99: null }, + status: 'missing', + }; + } + + const deltas = { + median: calculateRegressionPct(baselineEntry.stats.median, currentEntry.stats.median), + p95: calculateRegressionPct(baselineEntry.stats.p95, currentEntry.stats.p95), + p99: calculateRegressionPct(baselineEntry.stats.p99, currentEntry.stats.p99), + }; + let status = 'pass'; + + if (deltas.median > thresholdPct) { + status = 'fail'; + } else if (deltas.median < 0) { + status = 'improved'; + } + + return { + fixture: currentEntry.fixture, + name: currentEntry.label, + baselineStats: baselineEntry.stats, + currentStats: currentEntry.stats, + deltas, + status, + }; + }); + + const summary = { + total: scenarios.length, + compared: scenarios.filter((scenario) => scenario.deltas.median !== null).length, + regressions: scenarios.filter((scenario) => scenario.status === 'fail').length, + improvements: scenarios.filter((scenario) => scenario.status === 'improved').length, + pass: scenarios.filter((scenario) => scenario.status === 'pass').length, + newScenarios: scenarios.filter((scenario) => scenario.status === 'new').length, + missingScenarios: scenarios.filter((scenario) => scenario.status === 'missing').length, + }; + + return { + generatedAt: new Date().toISOString(), + hasBaseline: baselineReport !== null, + thresholdPct, + summary, + scenarios, + }; +} + +/** + * Formats a frame-time value for markdown output. + * + * @param {number | null} value Frame-time value in milliseconds. + * @returns {string} Formatted frame-time string. + */ +function formatFrameTime(value) { + if (value === null) { + return 'n/a'; + } + + return `${value.toFixed(2)}ms`; +} + +/** + * Formats a regression delta percentage for markdown output. + * + * @param {number | null} value Percentage delta where positive means slower. + * @returns {string} Formatted delta string. + */ +function formatDelta(value) { + if (value === null) { + return 'n/a'; + } + + const sign = value > 0 ? '+' : ''; + + return `${sign}${value.toFixed(2)}%`; +} + +/** + * Converts an internal perf scenario status into the PR comment label. + * + * @param {string} status Internal scenario status. + * @returns {string} Human-readable status label. + */ +function formatStatus(status) { + switch (status) { + case 'fail': + return 'FAIL'; + case 'improved': + return 'IMPROVED'; + case 'new': + return 'NEW'; + case 'missing': + return 'MISSING'; + default: + return 'PASS'; + } +} + +/** + * Escapes a markdown table cell value. + * + * @param {string} value Raw markdown cell content. + * @returns {string} Escaped markdown-safe value. + */ +function escapeMarkdownCell(value) { + return value.replaceAll('|', '\\|'); +} + +/** + * Builds the markdown body for the perf PR comment. + * + * @param {{ + * hasBaseline: boolean, + * thresholdPct: number, + * summary: { + * compared: number, + * regressions: number, + * improvements: number, + * newScenarios: number, + * missingScenarios: number, + * }, + * scenarios: Array<{ + * name: string, + * baselineStats: { median: number } | null, + * currentStats: { median: number } | null, + * deltas: { median: number | null, p95: number | null, p99: number | null }, + * status: string, + * }>, + * }} report Comparison report. + * @returns {string} Markdown comment body. + */ +function buildMarkdown(report) { + const lines = [COMMENT_MARKER, '## Tier 2 GPU Perf Comparison', '']; + + if (!report.hasBaseline) { + lines.push( + 'No `main` branch GPU perf baseline artifact is available yet. This run produced fresh perf results and uploaded them as artifacts.', + '', + `Configured regression threshold: ${report.thresholdPct}% slower median frame time.`, + ); + + return `${lines.join('\n')}\n`; + } + + lines.push( + `Compared ${report.summary.compared} perf scenarios against the latest \`main\` baseline. Regression threshold: ${report.thresholdPct}% slower median frame time.`, + '', + `Regressions: ${report.summary.regressions} | Improvements: ${report.summary.improvements} | New: ${report.summary.newScenarios} | Missing: ${report.summary.missingScenarios}`, + '', + '
', + 'Perf table', + '', + '| Scenario | Baseline median | Current median | Median delta | P95 delta | P99 delta | Status |', + '| --- | ---: | ---: | ---: | ---: | ---: | --- |', + ); + + for (const scenario of report.scenarios) { + lines.push( + `| ${escapeMarkdownCell(scenario.name)} | ${formatFrameTime(scenario.baselineStats?.median ?? null)} | ${formatFrameTime(scenario.currentStats?.median ?? null)} | ${formatDelta(scenario.deltas.median)} | ${formatDelta(scenario.deltas.p95)} | ${formatDelta(scenario.deltas.p99)} | ${formatStatus(scenario.status)} |`, + ); + } + + lines.push('', '
'); + + return `${lines.join('\n')}\n`; +} + +/** + * Writes a file to disk, creating parent directories when needed. + * + * @param {string} filePath Output file path. + * @param {string} content File contents. + * @returns {void} + */ +function writeFile(filePath, content) { + ensureParentDirectory(filePath); + fs.writeFileSync(filePath, content); +} + +/** + * Runs the perf comparison CLI. + * + * @returns {void} + */ +function main() { + const args = parseArgs(process.argv.slice(2)); + const currentReport = readJsonFile(args.current); + currentReport.__sourceLabel = args.current; + const baselineReport = args.baseline && fs.existsSync(args.baseline) ? readJsonFile(args.baseline) : null; + + if (baselineReport !== null) { + baselineReport.__sourceLabel = args.baseline; + } + + const report = comparePerfReports(currentReport, baselineReport, args.threshold); + + writeFile(args.jsonOut, JSON.stringify(report, null, 2)); + writeFile(args.markdownOut, buildMarkdown(report)); +} + +main(); -- 2.51.2