From 792db48bd0e3f595bc88db7e3bdbe095836eaac7 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 19 May 2026 10:00:32 +0200 Subject: [PATCH] chore: use agentscan to automatically flag prs (#10378) --- .../actions/send-ai-bot-comment/action.yml | 33 +++++++++ .github/workflows/issue-labeled.yml | 20 ++---- .github/workflows/pr-labeled-automated.yml | 70 +++++++++++++++---- 3 files changed, 93 insertions(+), 30 deletions(-) create mode 100644 .github/actions/send-ai-bot-comment/action.yml diff --git a/.github/actions/send-ai-bot-comment/action.yml b/.github/actions/send-ai-bot-comment/action.yml new file mode 100644 index 000000000..95ea9bdec --- /dev/null +++ b/.github/actions/send-ai-bot-comment/action.yml @@ -0,0 +1,33 @@ +name: Send Automated PR Comment +description: Sends the comment to PRs that were automatically generated +inputs: + token: + required: true + description: GitHub Token + pr-number: + required: true + description: PR number + login: + required: true + description: The login of the user + +runs: + using: composite + steps: + - name: maybe automated + uses: actions-cool/issues-helper@71b62d7da76e59ff7b193904feb6e77d4dbb2777 # v3.7.6 + with: + actions: create-comment + token: ${{ inputs.token }} + issue-number: ${{ inputs.pr-number }} + body: | + Hello @${{ inputs.login }}. Your PR has been labeled `maybe automated` because it appears to have been fully generated by AI with no human involvement. It will be **closed automatically in 3 days** unless a real person responds. + + If you're a real person behind this contribution, please: + - Confirm you've personally reviewed and stand behind its content + - Make sure it follows our [contribution guidelines](https://github.com/vitest-dev/vitest/blob/main/CONTRIBUTING.md) and uses the correct [GitHub template](https://github.com/vitest-dev/vitest/blob/main/.github/PULL_REQUEST_TEMPLATE.md) + - Disclose any AI tools you used (e.g. Claude, Copilot, Codex) + + If you believe this was flagged by mistake, leave a comment. + + *These measures help us reduce maintenance burden and keep the team's work efficient. See our [AI contributions policy](https://github.com/vitest-dev/vitest/blob/main/CONTRIBUTING.md#ai-contributions) for more context.* diff --git a/.github/workflows/issue-labeled.yml b/.github/workflows/issue-labeled.yml index 60cf0483f..1853c61d0 100644 --- a/.github/workflows/issue-labeled.yml +++ b/.github/workflows/issue-labeled.yml @@ -57,27 +57,15 @@ jobs: *These measures help us reduce maintenance burden and keep the team's work efficient. See our [AI contributions policy](https://github.com/vitest-dev/vitest/blob/main/CONTRIBUTING.md#ai-contributions) for more context.* - issue-pr-comment: + pr-clanker-comment: runs-on: ubuntu-slim if: github.repository == 'vitest-dev/vitest' && github.event.label.name == 'maybe automated' && github.event_name == 'pull_request_target' name: Comment on Bot PR permissions: pull-requests: write # sending a comment steps: - - name: maybe automated - uses: actions-cool/issues-helper@71b62d7da76e59ff7b193904feb6e77d4dbb2777 # v3.7.6 + - uses: ./.github/actions/send-ai-bot-comment with: - actions: create-comment token: ${{ secrets.GITHUB_TOKEN }} - issue-number: ${{ github.event.pull_request.number }} - body: | - Hello @${{ github.event.pull_request.user.login }}. Your PR has been labeled `maybe automated` because it appears to have been fully generated by AI with no human involvement. It will be **closed automatically in 3 days** unless a real person responds. - - If you're a real person behind this contribution, please: - - Confirm you've personally reviewed and stand behind its content - - Make sure it follows our [contribution guidelines](https://github.com/vitest-dev/vitest/blob/main/CONTRIBUTING.md) and uses the correct [GitHub template](https://github.com/vitest-dev/vitest/blob/main/.github/PULL_REQUEST_TEMPLATE.md) - - Disclose any AI tools you used (e.g. Claude, Copilot, Codex) - - If you believe this was flagged by mistake, leave a comment. - - *These measures help us reduce maintenance burden and keep the team's work efficient. See our [AI contributions policy](https://github.com/vitest-dev/vitest/blob/main/CONTRIBUTING.md#ai-contributions) for more context.* + pr-number: ${{ github.event.pull_request.number }} + login: ${{ github.event.pull_request.user.login }} diff --git a/.github/workflows/pr-labeled-automated.yml b/.github/workflows/pr-labeled-automated.yml index a19232c1b..759dbc127 100644 --- a/.github/workflows/pr-labeled-automated.yml +++ b/.github/workflows/pr-labeled-automated.yml @@ -4,7 +4,7 @@ on: # zizmor: ignore[dangerous-triggers] # Information from the PR is used only inside builtin `contains` function, it's not passed down as untrusted code. pull_request_target: - types: [opened, edited] + types: [opened, reopened] permissions: {} @@ -13,8 +13,8 @@ concurrency: cancel-in-progress: true jobs: - label: - runs-on: ubuntu-latest + prompt-label: + runs-on: ubuntu-slim if: github.repository == 'vitest-dev/vitest' && contains(github.event.pull_request.body, '') name: Automatic Clanker Alert permissions: @@ -27,20 +27,62 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} issue-number: ${{ github.event.pull_request.number }} labels: maybe automated - - name: maybe automated (pr) + - uses: ./.github/actions/send-ai-bot-comment + with: + token: ${{ secrets.GITHUB_TOKEN }} + pr-number: ${{ github.event.pull_request.number }} + login: ${{ github.event.pull_request.user.login }} + + agentscan: + runs-on: ubuntu-slim + # run only for our repo and ignore PRs from origin that only maintainers can do + # also ignore known bots + if: | + github.repository == 'vitest-dev/vitest' && + github.event.pull_request.head.repo.full_name != github.repository && + !contains( + fromJSON('["dependabot[bot]", "github-actions[bot]"]'), + github.event.pull_request.user.login + ) + name: AgentScan Alert + permissions: + pull-requests: write # comment and label on PRs + steps: + - name: AgentScan + id: agentscan + uses: MatteoGabriele/agentscan-action@21f25b07e4dda43e6597ffb71c93f8e4c9fed812 # v1.7.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + agent-scan-comment: false + + # just put a label and send a comment if the account looks suspicious + - name: Label flagged PR + if: contains(fromJSON('["automation","suspicious"]'), steps.agentscan.outputs.classification) && !contains(steps.agentscan.outputs.community-flagged, 'true') uses: actions-cool/issues-helper@71b62d7da76e59ff7b193904feb6e77d4dbb2777 # v3.7.6 with: - actions: create-comment + actions: add-labels token: ${{ secrets.GITHUB_TOKEN }} issue-number: ${{ github.event.pull_request.number }} - body: | - Hello @${{ github.event.pull_request.user.login }}. Your PR has been labeled `maybe automated` because it appears to have been fully generated by AI with no human involvement. It will be **closed automatically in 3 days** unless a real person responds. - - If you're a real person behind this contribution, please: - - Confirm you've personally reviewed and stand behind its content - - Make sure it follows our [contribution guidelines](https://github.com/vitest-dev/vitest/blob/main/CONTRIBUTING.md) and uses the correct [GitHub template](https://github.com/vitest-dev/vitest/blob/main/.github/PULL_REQUEST_TEMPLATE.md) - - Disclose any AI tools you used (e.g. Claude, Copilot, Codex) + labels: maybe automated + - name: Comment flagged PR + if: contains(fromJSON('["automation","suspicious"]'), steps.agentscan.outputs.classification) && !contains(steps.agentscan.outputs.community-flagged, 'true') + uses: ./.github/actions/send-ai-bot-comment + with: + token: ${{ secrets.GITHUB_TOKEN }} + pr-number: ${{ github.event.pull_request.number }} + login: ${{ github.event.pull_request.user.login }} - If you believe this was flagged by mistake, leave a comment. + # if the account is confirmed to be a bot, just close the PR + - name: Close community flagged accounts + if: steps.agentscan.outputs.community-flagged == 'true' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const prNumber = context.payload.pull_request.number; - *These measures help us reduce maintenance burden and keep the team's work efficient. See our [AI contributions policy](https://github.com/vitest-dev/vitest/blob/main/CONTRIBUTING.md#ai-contributions) for more context.* + await github.rest.pulls.update({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + state: 'closed', + }); -- 2.51.2