Something went wrong. Try again.
fork of prime agent with changes i want but are not upstreamable probqbly (and some fixes)
Something went wrong. Try again.
4.7 kB · 121 lines
YAML
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122name: Prime Agent benchmarksrun-name: 'Prime Agent benchmarks · PR #${{ github.event.pull_request.number || inputs.pr }}'
on: pull_request_target: types: [opened, synchronize, reopened, ready_for_review] workflow_dispatch: inputs: pr: description: Open PR number to benchmark against current main required: true type: number
concurrency: group: prime-agent-benchmark-${{ github.event.pull_request.number || inputs.pr }} cancel-in-progress: true
permissions: {}
jobs: request: if: github.event_name == 'pull_request_target' || github.ref == 'refs/heads/main' runs-on: ubuntu-24.04 timeout-minutes: 5 concurrency: group: 'prime-agent-benchmark-comment-Prime Agent benchmarks · PR #${{ github.event.pull_request.number || inputs.pr }}' cancel-in-progress: false permissions: contents: read actions: read pull-requests: write outputs: allowed: ${{ steps.resolve.outputs.needed == 'true' && steps.vouch.outputs.vouched == 'true' }} harness: ${{ steps.resolve.outputs.harness }} steps: # pull_request_target supplies the default-branch SHA, including for stacked PRs. - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.sha }} persist-credentials: false - name: Install benchmark tooling run: | python3 -m venv .benchmark-bootstrap .benchmark-bootstrap/bin/pip install uv==0.12.9 .benchmark-bootstrap/bin/uv sync --locked --project scripts/benchmarks echo "$PWD/.benchmark-bootstrap/bin" >> "$GITHUB_PATH" - name: Resolve exact revisions id: resolve working-directory: scripts/benchmarks run: uv run --locked cli.py resolve --results request env: GITHUB_TOKEN: ${{ github.token }} - name: Check contributor trust id: vouch if: steps.resolve.outputs.needed == 'true' uses: mitchellh/vouch/action/check-user@d66fa29a64600490892131ad87597c30c91fcac4 # v1 with: user: ${{ steps.resolve.outputs.author }} allow-fail: true env: GITHUB_TOKEN: ${{ github.token }} - name: Update benchmark comment if: steps.resolve.outputs.needed == 'true' working-directory: scripts/benchmarks run: | if [ "$VOUCHED" = true ]; then uv run --locked cli.py publish --results request else uv run --locked cli.py publish --results request --pending-trust fi env: GITHUB_TOKEN: ${{ github.token }} VOUCHED: ${{ steps.vouch.outputs.vouched }} - name: Save benchmark request if: steps.resolve.outputs.needed == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 with: name: benchmark-request-${{ github.run_attempt }} path: scripts/benchmarks/request/report.json retention-days: 14
benchmark: needs: request if: needs.request.outputs.allowed == 'true' runs-on: ubuntu-24.04 timeout-minutes: 25 permissions: contents: read actions: read pull-requests: read steps: # The controller is trusted main code. PR code executes only in Prime sandboxes. - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ needs.request.outputs.harness }} persist-credentials: false - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.0 with: name: benchmark-request-${{ github.run_attempt }} path: scripts/benchmarks/request - name: Install benchmark tooling run: | python3 -m venv .benchmark-bootstrap .benchmark-bootstrap/bin/pip install uv==0.12.9 .benchmark-bootstrap/bin/uv sync --locked --project scripts/benchmarks echo "$PWD/.benchmark-bootstrap/bin" >> "$GITHUB_PATH" - name: Benchmark main and PR in Prime sandboxes working-directory: scripts/benchmarks run: uv run --locked cli.py run env: GITHUB_TOKEN: ${{ github.token }} PRIME_SANDBOX_API_KEY: ${{ secrets.PRIME_SANDBOX_API_KEY }} PRIME_TEAM_ID: ${{ vars.PRIME_BENCHMARK_TEAM_ID }} - name: Save benchmark results if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 with: name: benchmark-results-${{ github.run_attempt }} path: scripts/benchmarks/results/ if-no-files-found: warn retention-days: 14