diff --git a/.buildkite/pipeline.poll-issues.yml b/.buildkite/pipeline.poll-issues.yml index 65255b8..ce2d8c5 100644 --- a/.buildkite/pipeline.poll-issues.yml +++ b/.buildkite/pipeline.poll-issues.yml @@ -1,18 +1,13 @@ -# Optional scheduled pipeline: poll for new Radicle issues every 5 minutes. -# Enable in Buildkite UI → Pipeline → Settings → Schedules, or upload this file -# from a separate Buildkite pipeline pointed at the same Radicle Garden repo. -# -# Secrets / env: -# BUILDKITE_API_TOKEN — API token with build create permission -# BUILDKITE_ORG — default nandi (set in trigger-issue-build.sh) -# BUILDKITE_PIPELINE — default sleek -# -# Manual trigger for one issue: -# BUILDKITE_API_TOKEN=… ./scripts/buildkite/trigger-issue-build.sh +# Optional standalone upload target for issue polling. +# Prefer scheduling the main pipeline with env RADICLE_TRIGGER=poll instead +# (see create_pipeline_schedule / Buildkite UI → Schedules). agents: queue: auto +env: + RADICLE_TRIGGER: poll + steps: - label: ":radicle: Poll for new issues" command: bash scripts/buildkite/poll-issues.sh @@ -20,3 +15,4 @@ steps: BUILDKITE_API_TOKEN: ${BUILDKITE_API_TOKEN} BUILDKITE_ORG: ${BUILDKITE_ORG} BUILDKITE_PIPELINE: ${BUILDKITE_PIPELINE} + CURSOR_API_KEY: ${CURSOR_API_KEY} diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index fc75ea3..393565d 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -29,10 +29,12 @@ # shell vars at pipeline-upload time. # # Radicle issue → cursor-agent → patch: -# Garden adapter skips issue COB commits (no pipeline.yml there). Trigger on -# main with RADICLE_ISSUE_ID / RADICLE_TRIGGER=issue instead: -# ./scripts/buildkite/trigger-issue-build.sh -# Optional schedule: .buildkite/pipeline.poll-issues.yml +# Garden adapter skips issue COB commits (no pipeline.yml there). Auto path: +# Buildkite schedule every 5m with RADICLE_TRIGGER=poll +# → scripts/buildkite/poll-issues.sh lists issue COBs via git ls-remote +# → dispatches run-issue-agent.sh (pipeline upload or API trigger) +# Manual: ./scripts/buildkite/trigger-issue-build.sh +# Requires cluster secret CURSOR_API_KEY (and rad identity on the agent to push patches). agents: queue: auto @@ -44,8 +46,8 @@ env: steps: - label: ":rust: Check" key: check - # Issue-triggered builds only need the agent step (see below). - if: build.env("RADICLE_TRIGGER") != "issue" + # Issue/poll builds only need the agent / poll steps. + if: build.env("RADICLE_TRIGGER") != "issue" && build.env("RADICLE_TRIGGER") != "poll" timeout_in_minutes: 120 command: | set -euo pipefail @@ -109,7 +111,7 @@ steps: - label: ":android: APK (nixbuild.net)" key: apk - if: build.env("RADICLE_TRIGGER") != "issue" + if: build.env("RADICLE_TRIGGER") != "issue" && build.env("RADICLE_TRIGGER") != "poll" depends_on: check allow_dependency_failure: true timeout_in_minutes: 120 @@ -150,7 +152,7 @@ steps: - label: ":package: Flatpak (nixbuild.net)" key: flatpak - if: build.env("RADICLE_TRIGGER") != "issue" + if: build.env("RADICLE_TRIGGER") != "issue" && build.env("RADICLE_TRIGGER") != "poll" depends_on: check allow_dependency_failure: true timeout_in_minutes: 180 @@ -194,6 +196,13 @@ steps: || cp --reflink=auto "$$bundle" uk.nandi.sleek.flatpak buildkite-agent artifact upload uk.nandi.sleek.flatpak + # --- Radicle issue poll (Buildkite schedule: RADICLE_TRIGGER=poll) --- + - label: ":radicle: Poll issues → dispatch agent" + key: radicle-issue-poll + if: build.env("RADICLE_TRIGGER") == "poll" + command: bash scripts/buildkite/poll-issues.sh + timeout_in_minutes: 15 + # --- Radicle issue agent (triggered via RADICLE_ISSUE_ID on main) --- - label: ":radicle: Issue → :cursor: agent → patch" key: radicle-issue-agent diff --git a/scripts/buildkite/bootstrap.sh b/scripts/buildkite/bootstrap.sh index af983aa..1309c20 100755 --- a/scripts/buildkite/bootstrap.sh +++ b/scripts/buildkite/bootstrap.sh @@ -56,8 +56,13 @@ build_radicle_mcp() { verify_auth() { local cmd cmd="$(cursor_agent_cmd)" + if [[ -z "${CURSOR_API_KEY:-}" ]] && command -v buildkite-agent >/dev/null 2>&1; then + if key="$(buildkite-agent secret get CURSOR_API_KEY 2>/dev/null || true)"; then + [[ -n "$key" ]] && export CURSOR_API_KEY="$key" + fi + fi if [[ -z "${CURSOR_API_KEY:-}" ]]; then - bk_die "CURSOR_API_KEY is not set — add it as a Buildkite secret" + bk_die "CURSOR_API_KEY is not set — add it as a Buildkite cluster secret" fi if ! "$cmd" status >/dev/null 2>&1; then bk_die "Cursor CLI auth failed — check CURSOR_API_KEY" diff --git a/scripts/buildkite/detect-issue.sh b/scripts/buildkite/detect-issue.sh index 91b5d63..d8773b6 100755 --- a/scripts/buildkite/detect-issue.sh +++ b/scripts/buildkite/detect-issue.sh @@ -18,17 +18,21 @@ if [[ -z "$COMMIT" ]]; then exit 1 fi -bk_require_cmd rad -bk_require_rad_repo +# rad is optional when RADICLE_ISSUE_ID is provided (Garden HTTPS checkouts). +if [[ -z "${RADICLE_ISSUE_ID:-}" ]]; then + bk_require_cmd rad + bk_require_rad_repo +fi # Issue opens create a COB root commit that does not contain .buildkite/pipeline.yml, # so the Radicle Buildkite adapter skips them. Trigger builds on main instead and -# pass the issue id explicitly (see scripts/buildkite/trigger-issue-build.sh). +# pass the issue id explicitly (see scripts/buildkite/trigger-issue-build.sh / poll). if [[ -n "${RADICLE_ISSUE_ID:-}" ]]; then ISSUE_ID="$RADICLE_ISSUE_ID" - if ! rad issue show "$ISSUE_ID" --header >/dev/null 2>&1; then - echo "RADICLE_ISSUE_ID=$ISSUE_ID is not a valid issue" >&2 - exit 1 + if command -v rad >/dev/null 2>&1 && rad issue show "$ISSUE_ID" --header >/dev/null 2>&1; then + : + else + echo "warn: rad issue show unavailable — using RADICLE_ISSUE_ID only" >&2 fi elif bk_commit_is_new_issue "$COMMIT"; then ISSUE_ID="$COMMIT" @@ -36,12 +40,18 @@ else echo "commit $COMMIT is not a new issue COB root (set RADICLE_ISSUE_ID to trigger from main)" >&2 exit 2 fi -mapfile -t _details < <(bk_issue_details "$ISSUE_ID") -TITLE="${_details[0]:-}" -BODY="${_details[1]:-}" + +TITLE="" +BODY="" +if command -v rad >/dev/null 2>&1 && command -v python3 >/dev/null 2>&1; then + if mapfile -t _details < <(bk_issue_details "$ISSUE_ID" 2>/dev/null); then + TITLE="${_details[0]:-}" + BODY="${_details[1]:-}" + fi +fi if [[ -z "$TITLE" ]]; then - TITLE="(no title)" + TITLE="Radicle issue $(bk_short_id "$ISSUE_ID")" fi SHORT="$(bk_short_id "$ISSUE_ID")" diff --git a/scripts/buildkite/poll-issues.sh b/scripts/buildkite/poll-issues.sh index 46183f7..2d0188f 100755 --- a/scripts/buildkite/poll-issues.sh +++ b/scripts/buildkite/poll-issues.sh @@ -1,11 +1,16 @@ #!/usr/bin/env bash -# Poll for new open Radicle issues and trigger Buildkite agent builds. -# Run from a scheduled Buildkite pipeline on main (e.g. every 5 minutes). +# Poll for Radicle issue COBs and dispatch cursor-agent steps. # -# Requires: -# BUILDKITE_ORG, BUILDKITE_PIPELINE, BUILDKITE_API_TOKEN -# Optional: -# RADICLE_ISSUE_POLL_STATE_FILE — path to last-seen issue ids (default: .buildkite/.issue-poll-state) +# Intended for a Buildkite schedule with RADICLE_TRIGGER=poll (every few minutes). +# Garden HTTPS clones expose issue COBs as: +# refs/namespaces/*/refs/cobs/xyz.radicle.issue/ +# so this does not require a local `rad` install on agents. +# +# Dispatch strategy (first match): +# 1. If BUILDKITE_API_TOKEN is available → trigger-issue-build.sh (separate builds) +# 2. Else if buildkite-agent is available → pipeline-upload agent steps into this build +# +# Idempotency: skip issues that already have a remote branch issue/. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -15,43 +20,133 @@ cd "$REPO_ROOT" # shellcheck source=lib.sh source "$SCRIPT_DIR/lib.sh" -bk_require_cmd rad -bk_require_rad_repo - -STATE_FILE="${RADICLE_ISSUE_POLL_STATE_FILE:-$REPO_ROOT/.buildkite/.issue-poll-state}" -mkdir -p "$(dirname "$STATE_FILE")" -touch "$STATE_FILE" - -mapfile -t known < "$STATE_FILE" +bk_require_cmd git -current=() -while IFS= read -r line; do - [[ -n "$line" ]] && current+=("$line") -done < <(rad cob list --type xyz.radicle.issue 2>/dev/null || true) +# Soft-load API token from cluster secrets when running on an agent. +if [[ -z "${BUILDKITE_API_TOKEN:-}" ]] && command -v buildkite-agent >/dev/null 2>&1; then + if tok="$(buildkite-agent secret get BUILDKITE_API_TOKEN 2>/dev/null || true)"; then + [[ -n "$tok" ]] && export BUILDKITE_API_TOKEN="$tok" + fi +fi -new_ids=() -for id in "${current[@]}"; do - found=0 - for old in "${known[@]}"; do - if [[ "$id" == "$old" ]]; then - found=1 - break +# Prefer a Git remote that advertises COB refs (Garden HTTPS). Plain `rad://` +# ls-remote often omits namespaces/*/refs/cobs/*. +poll_remote() { + if [[ -n "${RADICLE_POLL_REMOTE:-}" ]]; then + echo "$RADICLE_POLL_REMOTE" + return 0 + fi + local url + for candidate in origin rad; do + url="$(git remote get-url "$candidate" 2>/dev/null || true)" + [[ -n "$url" ]] || continue + if [[ "$url" == *radicle.garden* || "$url" == https://* || "$url" == http://* ]]; then + echo "$candidate" + return 0 fi done - if [[ "$found" -eq 0 ]]; then - new_ids+=("$id") + # Derive Garden URL from rad RID when only rad:// is configured (local nodes). + if url="$(git remote get-url rad 2>/dev/null || true)"; then + if [[ "$url" =~ rad://([A-Za-z0-9]+) ]]; then + echo "https://nandi.radicle.garden/${BASH_REMATCH[1]}.git" + return 0 + fi fi -done + if git remote get-url origin >/dev/null 2>&1; then + echo origin + return 0 + fi + return 1 +} + +REMOTE="$(poll_remote)" || bk_die "no git remote to poll for issue COBs" +echo "poll: listing issue COBs via git ls-remote ($REMOTE)" -printf '%s\n' "${current[@]}" >"$STATE_FILE" +list_issue_ids() { + git ls-remote "$REMOTE" 2>/dev/null \ + | awk '{print $2}' \ + | grep -E 'refs/cobs/xyz\.radicle\.issue/[0-9a-f]{40}$' \ + | sed -E 's|.*/||' \ + | sort -u +} + +# Fallback: rad cob list when ls-remote sees no COBs (some remotes hide them). +mapfile -t issue_ids < <(list_issue_ids || true) +if [[ "${#issue_ids[@]}" -eq 0 ]] && command -v rad >/dev/null 2>&1; then + echo "poll: ls-remote found no COBs — trying rad cob list" + rid="$(git remote get-url rad 2>/dev/null || true)" + if [[ -n "$rid" ]]; then + mapfile -t issue_ids < <(rad cob list --repo "$rid" --type xyz.radicle.issue 2>/dev/null || true) + fi +fi -if [[ "${#new_ids[@]}" -eq 0 ]]; then - echo "poll: no new issues" +if [[ "${#issue_ids[@]}" -eq 0 ]]; then + echo "poll: no issue COBs found" exit 0 fi -echo "poll: ${#new_ids[@]} new issue(s)" -for id in "${new_ids[@]}"; do - echo " triggering build for $id" - "$SCRIPT_DIR/trigger-issue-build.sh" "$id" +echo "poll: ${#issue_ids[@]} issue COB(s)" + +branch_exists() { + local short=$1 + git ls-remote "$REMOTE" 2>/dev/null | grep -E "/refs/heads/issue/${short}$|refs/heads/issue/${short}$" >/dev/null +} + +pending=() +for id in "${issue_ids[@]}"; do + short="$(bk_short_id "$id")" + if branch_exists "$short"; then + echo "poll: skip $short — remote branch issue/${short} exists" + continue + fi + pending+=("$id") done + +if [[ "${#pending[@]}" -eq 0 ]]; then + echo "poll: nothing to dispatch" + exit 0 +fi + +echo "poll: ${#pending[@]} issue(s) to dispatch" + +dispatch_via_api() { + local id=$1 + BUILDKITE_ORG="${BUILDKITE_ORG:-nandi}" \ + BUILDKITE_PIPELINE="${BUILDKITE_PIPELINE:-sleek}" \ + "$SCRIPT_DIR/trigger-issue-build.sh" "$id" +} + +dispatch_via_upload() { + local id=$1 + local short + short="$(bk_short_id "$id")" + local yaml + yaml="$(cat </dev/null 2>&1; then + dispatch_via_upload "$id" + else + echo "poll: would dispatch $short (no BUILDKITE_API_TOKEN / buildkite-agent here)" + fi +done + +echo "poll: done" diff --git a/scripts/buildkite/trigger-issue-build.sh b/scripts/buildkite/trigger-issue-build.sh index 3412cbd..af77f6d 100755 --- a/scripts/buildkite/trigger-issue-build.sh +++ b/scripts/buildkite/trigger-issue-build.sh @@ -35,13 +35,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/lib.sh" bk_require_cmd git -bk_require_cmd rad bk_require_cmd curl bk_require_cmd jq -bk_require_rad_repo -if ! rad issue show "$ISSUE_ID" --header >/dev/null 2>&1; then - bk_die "not a valid issue: $ISSUE_ID" +# Prefer validating via rad when available; otherwise accept the id (poll already +# filtered to xyz.radicle.issue COB refs). +if command -v rad >/dev/null 2>&1 && git remote get-url rad >/dev/null 2>&1; then + if ! rad issue show "$ISSUE_ID" --header >/dev/null 2>&1; then + bk_die "not a valid issue: $ISSUE_ID" + fi fi COMMIT="$(git rev-parse "$BRANCH")"