From 18d9663437183c0068bec5ac3b3a4fbddc73fcdb Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 1 May 2026 15:23:37 -0700 Subject: [PATCH] better docs --- README.md | 213 ++++++------------------------------- docs/buildkite.md | 126 ++++++++++++++++++++++ provider_buildkite_test.go | 54 ++++------ 3 files changed, 174 insertions(+), 219 deletions(-) create mode 100644 docs/buildkite.md diff --git a/README.md b/README.md index 9014fff..1ab7080 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,31 @@ CI on alternate providers and reports their results back to Tangled using standard ATProto records so they show up natively in Tangled's UI. -## What it does +## Example + +A Tangled workflow that fires a Buildkite pipeline on every push to +`main` and every pull request targeting `main`: + +```yaml +# .tangled/workflows/ci.yml +when: + - event: ["push"] + branch: ["main"] + - event: ["pull_request"] + branch: ["main"] + +tack: + buildkite: + pipeline: my-app-ci +``` + +The `when:` block is the standard Tangled trigger schema; the +`tack:` block tells tack which Buildkite pipeline to fire and how. +See [docs/buildkite.md](docs/buildkite.md) for the full set of +options and end-to-end Buildkite setup. Tack can support multiple +providers. + +## How it Works Tack is a drop-in alternative to the stock `spindle` runner. You run `tack` and [register it using the standard UI](https://tangled.org/settings/spindles). @@ -60,187 +84,10 @@ When no provider is configured, tack runs an in-process fake provider that's useful for exercising the jetstream → knot → `/events` flow locally without a real CI account. -## Buildkite - -[Buildkite](https://buildkite.com) is the primary provider tack -supports today. In Buildkite mode, every Tangled pipeline trigger -fans out into one Buildkite build per workflow on the pipeline that -workflow names; build state flows back to Tangled via Buildkite's -notification webhooks. - -### How it fits together - -``` - sh.tangled.pipeline Buildkite - trigger record ──▶ tack ──▶ Create Build ─┐ - │ - /webhooks/buildkite ◀──── notification ◀─────┘ - │ - ▼ - sh.tangled.pipeline.status (broadcast on /events) -``` - -* **Spawn:** for each workflow on a pipeline trigger, tack POSTs to - `/v2/organizations//pipelines//builds`. Both `` and - `` come from the workflow's YAML body (see - [Configuring your workflows](#configuring-your-workflows)). -* **Track:** tack persists the resulting `(build_uuid → knot, rkey, - workflow)` mapping in its local SQLite store so it can later - resolve incoming webhooks back to the originating Tangled - pipeline. -* **Report:** Buildkite delivers `build.*` events to - `POST /webhooks/buildkite`. tack authenticates each request, - translates the Buildkite state into a Tangled status, and - broadcasts a `sh.tangled.pipeline.status` record on `/events`. - -### Setting up Buildkite - -These steps happen once on the Buildkite side, before tack can talk -to it. - -#### 1. Create one or more pipelines - -Each Tangled workflow targets exactly one Buildkite pipeline by -slug. There's no requirement that pipelines map 1:1 to workflows — -many users point every workflow at a single pipeline whose -`pipeline.yml` does `pipeline upload some-file-${TACK_WORKFLOW}.yml`, -keeping all the per-workflow logic in the repo rather than in -Buildkite's UI. - -In your Buildkite org, **Pipelines → New pipeline**: - -* Repository: any URL (the agent only needs to be able to clone it). -* Steps: a minimal `pipeline upload` is usually enough — tack passes - the workflow name through `$TACK_WORKFLOW` so you can branch on - it. - -Note the pipeline slug from the URL -(`https://buildkite.com//`); your workflow YAML -will reference it. - -#### 2. Create an API access token - -Tack uses a single API token to create builds, list jobs, and fetch -logs. Generate one at - with these scopes: - -| Scope | Used for | -| ------------------- | ------------------------------------------------- | -| `read_organizations`| Sanity-checking the configured org slug | -| `write_builds` | `POST .../builds` when a Tangled trigger arrives | -| `read_builds` | Resolving build → jobs for the `/logs` endpoint | -| `read_build_logs` | Streaming job logs back to the Tangled appview | - -Restrict the token to the specific organization(s) tack will spawn -into. +## Providers -#### 3. Configure a notification webhook +Provider-specific setup (Buildkite-side configuration, the +provider's tack env vars, and the workflow YAML schema) lives in +its own doc per provider: -Builds report their state back to tack through Buildkite's -notification service. - -In your Buildkite org, **Settings → Notification Services → Add → -Webhook**: - -* **Webhook URL:** `https:///webhooks/buildkite` -* **Token / Secret:** any high-entropy string. You'll set the same - value in `TACK_BUILDKITE_WEBHOOK_SECRET`. -* **Events:** `build.scheduled`, `build.running`, `build.finished` - (job-level events are ignored). -* **Pipelines:** the pipelines tack will fire builds on. - -Buildkite supports two header schemes for authenticating webhooks; -tack supports both: - -| Header scheme | `TACK_BUILDKITE_WEBHOOK_MODE` | Notes | -| ----------------------- | ----------------------------- | -------------------------------------------- | -| `X-Buildkite-Token` | `token` (default) | Secret is sent verbatim in the header | -| `X-Buildkite-Signature` | `signature` | HMAC-SHA256 of `.`; safer | - -Pick `signature` if the notification setting offers it — it doesn't -expose the secret on the wire. - -### Configuring tack - -Setting `TACK_BUILDKITE_TOKEN` is the master switch that puts tack -into Buildkite mode. The other variables in this section are then -required. - -| Env var | Description | -| ------------------------------- | ------------------------------------------------------------------------------ | -| `TACK_BUILDKITE_TOKEN` | Buildkite API token (enables Buildkite mode) | -| `TACK_BUILDKITE_ORG` | Default Buildkite organization slug (workflows may override via YAML) | -| `TACK_BUILDKITE_WEBHOOK_SECRET` | Shared secret for `/webhooks/buildkite` auth | -| `TACK_BUILDKITE_WEBHOOK_MODE` | `token` (default) or `signature` — must match the notification service | - -The pipeline a workflow runs against is **not** an environment -variable. It lives inside the workflow YAML so each repo can target -its own pipeline without an operator round-trip. - -### Configuring your workflows - -A Tangled workflow's `raw` body is parsed by tack as YAML. Only -`pipeline` is required — every other field is an optional override -or extension of what the trigger metadata already provides: - -```yaml -# Required: which Buildkite pipeline this workflow fires. -pipeline: my-pipeline-slug - -# Optional: org override. Defaults to TACK_BUILDKITE_ORG. The API -# token must have access to whichever org you target. -org: another-org - -# Optional: human-readable build message (default: "tangled: "). -message: "Custom build message" - -# Optional: pin the commit/branch tack would otherwise derive from -# the trigger. Useful for manual triggers (which carry no commit). -commit: abcdef0123 -branch: main - -# Optional: extra env + meta_data merged on top of tack's defaults -# (see "What tack injects into every build" below). -env: - CUSTOM_VAR: value -meta_data: - custom-key: value - -# Optional: forwarded verbatim to the Buildkite create-build API. -clean_checkout: true -ignore_pipeline_branch_filters: true # default: true -author: - name: "Author Name" - email: "author@example.com" -``` - -When the trigger is a pull request, tack auto-populates Buildkite's -`pull_request_base_branch` from the PR target so step-level branch -filters work without extra config. - -#### What tack injects into every build - -Regardless of what the workflow YAML adds on top, tack always -provides the following so your Buildkite pipeline can recover the -Tangled identity of the build: - -| Channel | Key | Value | -| ----------- | -------------------- | ---------------------------------------- | -| `env` | `TACK_KNOT` | knot hostname the pipeline came from | -| `env` | `TACK_PIPELINE_RKEY` | rkey of the originating pipeline record | -| `env` | `TACK_WORKFLOW` | workflow name (typically a YAML filename) | -| `env` | `TACK_WORKFLOW_RAW` | the workflow's raw YAML body | -| `meta_data` | `tack:knot` | same as `TACK_KNOT` | -| `meta_data` | `tack:pipeline_rkey` | same as `TACK_PIPELINE_RKEY` | -| `meta_data` | `tack:workflow` | same as `TACK_WORKFLOW` | - -A common pattern is for the Buildkite pipeline's root step to do a -`pipeline upload` against a workflow-specific YAML file based on -`$TACK_WORKFLOW`, e.g.: - -```yaml -# Buildkite pipeline.yml -steps: - - label: ":pipeline: dispatch ${TACK_WORKFLOW}" - command: "buildkite-agent pipeline upload .buildkite/${TACK_WORKFLOW}" -``` +* [Buildkite](docs/buildkite.md) diff --git a/docs/buildkite.md b/docs/buildkite.md new file mode 100644 index 0000000..606ee2c --- /dev/null +++ b/docs/buildkite.md @@ -0,0 +1,126 @@ +# Buildkite + +For [Buildkite](https://buildkite.com), every Tangled pipeline trigger +fans out into one Buildkite build per workflow on the pipeline that +workflow configures. Buildkite must be configured with a webhook +back to Tack to communicate status updates. + +## Setting up Buildkite + +This must happen before configuring tack within Buildkite. + +### 1. Create one or more pipelines + +In your Buildkite org, **Pipelines → New pipeline**: + +* Repository: any URL (the agent only needs to be able to clone it). +* Steps: whatever you want. + +Note the pipeline slug from the URL +(`https://buildkite.com//`); your workflow YAML +will reference it. + +### 2. Create an API access token + +Tack uses a single API token to create builds, list jobs, and fetch +logs. Generate one at + with these scopes: + +| Scope | Used for | +| ------------------- | ------------------------------------------------- | +| `read_organizations`| Sanity-checking the configured org slug | +| `write_builds` | `POST .../builds` when a Tangled trigger arrives | +| `read_builds` | Resolving build → jobs for the `/logs` endpoint | +| `read_build_logs` | Streaming job logs back to the Tangled appview | + +Restrict the token to the specific organization(s) tack will spawn +into. + +### 3. Configure a notification webhook + +Builds report their state back to tack through Buildkite's +notification service. + +In your Buildkite org, **Settings → Notification Services → Add → +Webhook**: + +* **Webhook URL:** `https:///webhooks/buildkite` +* **Token / Secret:** any high-entropy string. You'll set the same + value in `TACK_BUILDKITE_WEBHOOK_SECRET`. +* **Events:** `build.scheduled`, `build.running`, `build.finished` + (job-level events are ignored). +* **Pipelines:** the pipelines tack will fire builds on. + +Buildkite supports two header schemes for authenticating webhooks and +Tack supports both: + +| Header scheme | `TACK_BUILDKITE_WEBHOOK_MODE` | Notes | +| ----------------------- | ----------------------------- | -------------------------------------------- | +| `X-Buildkite-Token` | `token` (default) | Secret is sent verbatim in the header | +| `X-Buildkite-Signature` | `signature` | HMAC-SHA256 of `.`; safer | + +## Configure Tack + +| Env var | Description | +| ------------------------------- | ------------------------------------------------------------------------------ | +| `TACK_BUILDKITE_TOKEN` | Buildkite API token (enables Buildkite mode) | +| `TACK_BUILDKITE_ORG` | Default Buildkite organization slug (workflows may override via YAML) | +| `TACK_BUILDKITE_WEBHOOK_SECRET` | Shared secret for `/webhooks/buildkite` auth | +| `TACK_BUILDKITE_WEBHOOK_MODE` | `token` (default) or `signature` — must match the notification service | + +The pipeline a workflow runs against is **not** an environment +variable. It lives inside the workflow YAML so each repo can target +its own pipeline without an operator round-trip. + +## Configuring your Tangled workflows + +Tack's configuration lives under a `tack:` namespace so the workflow body +can grow other top-level keys without colliding. + +Only `pipeline` is required: + +```yaml +tack: + buildkite: + # Required: which Buildkite pipeline this workflow fires. + pipeline: my-pipeline-slug + + # Optional: org override. Defaults to TACK_BUILDKITE_ORG. The + # API token must have access to whichever org you target. + org: another-org + + # Optional: forwarded verbatim to the Buildkite create-build + # API. Omit to use Buildkite's default (false). + clean_checkout: true +``` + +When the trigger is a pull request, tack auto-populates Buildkite's +`pull_request_base_branch` from the PR target so step-level branch +filters work without extra config. + +### What tack injects into every build + +Regardless of what the workflow YAML adds on top, tack always +provides the following so your Buildkite pipeline can recover the +Tangled identity of the build: + +| Channel | Key | Value | +| ----------- | -------------------- | ---------------------------------------- | +| `env` | `TACK_KNOT` | knot hostname the pipeline came from | +| `env` | `TACK_PIPELINE_RKEY` | rkey of the originating pipeline record | +| `env` | `TACK_WORKFLOW` | workflow name (typically a YAML filename) | +| `env` | `TACK_WORKFLOW_RAW` | the workflow's raw YAML body | +| `meta_data` | `tack:knot` | same as `TACK_KNOT` | +| `meta_data` | `tack:pipeline_rkey` | same as `TACK_PIPELINE_RKEY` | +| `meta_data` | `tack:workflow` | same as `TACK_WORKFLOW` | + +A common pattern is for the Buildkite pipeline's root step to do a +`pipeline upload` against a workflow-specific YAML file based on +`$TACK_WORKFLOW`, e.g.: + +```yaml +# Buildkite pipeline.yml +steps: + - label: ":pipeline: dispatch ${TACK_WORKFLOW}" + command: "buildkite-agent pipeline upload .buildkite/${TACK_WORKFLOW}" +``` diff --git a/provider_buildkite_test.go b/provider_buildkite_test.go index d46607a..272c83f 100644 --- a/provider_buildkite_test.go +++ b/provider_buildkite_test.go @@ -155,10 +155,9 @@ func TestBuildkiteSpawnNoCommit(t *testing.T) { } // TestBuildkiteSpawnWorkflowConfig pins the YAML → create-build -// translation: pipeline + org from YAML pick the request URL, -// message/env/meta_data come through, and the trigger's PR target -// branch lands as `pull_request_base_branch`. Together these cover -// the "smuggle Buildkite parameters through workflow YAML" path. +// translation: `tack.buildkite.{pipeline,org}` pick the request URL, +// `clean_checkout` flows through, and the trigger's PR target +// branch lands as `pull_request_base_branch` automatically. func TestBuildkiteSpawnWorkflowConfig(t *testing.T) { type captured struct { path string @@ -175,17 +174,11 @@ func TestBuildkiteSpawnWorkflowConfig(t *testing.T) { p, _, _, _ := newBuildkiteTestProvider(t, buildkite.WebhookModeToken, "s", bk) raw := strings.Join([]string{ - "pipeline: workflow-pipe", - "org: workflow-org", - "message: smuggled message", - "env:", - " CUSTOM: value", - "meta_data:", - " custom: meta", - "clean_checkout: true", - "author:", - " name: Author", - " email: a@example.com", + "tack:", + " buildkite:", + " pipeline: workflow-pipe", + " org: workflow-org", + " clean_checkout: true", }, "\n") + "\n" trigger := &tangled.Pipeline_TriggerMetadata{ @@ -208,45 +201,34 @@ func TestBuildkiteSpawnWorkflowConfig(t *testing.T) { if got.body.Commit != "deadbeef" || got.body.Branch != "feature" { t.Fatalf("commit/branch = %q/%q", got.body.Commit, got.body.Branch) } - if got.body.Message != "smuggled message" { - t.Fatalf("message = %q", got.body.Message) - } - if got.body.Env["CUSTOM"] != "value" { - t.Fatalf("env[CUSTOM] missing: %+v", got.body.Env) - } - // tack defaults must still be present (user keys merge, - // don't replace). + // tack-managed env/meta still present. if got.body.Env["TACK_WORKFLOW"] != "ci.yml" { t.Fatalf("env[TACK_WORKFLOW] missing: %+v", got.body.Env) } - if got.body.MetaData["custom"] != "meta" || - got.body.MetaData[bkMetaWorkflow] != "ci.yml" { - t.Fatalf("meta_data wrong: %+v", got.body.MetaData) + if got.body.MetaData[bkMetaWorkflow] != "ci.yml" { + t.Fatalf("meta_data missing identity tuple: %+v", got.body.MetaData) } if !got.body.CleanCheckout { t.Fatalf("clean_checkout not set") } - // IgnorePipelineBranchFilters defaults to true (see - // workflowConfig comment). + // IgnorePipelineBranchFilters is hard-coded true; see + // buildCreateRequest comment. if !got.body.IgnorePipelineBranchFilters { - t.Fatalf("ignore_pipeline_branch_filters not defaulted to true") + t.Fatalf("ignore_pipeline_branch_filters not on") } if got.body.PullRequestBaseBranch != "main" { t.Fatalf("pr base branch = %q; want main", got.body.PullRequestBaseBranch) } - if got.body.Author == nil || got.body.Author.Email != "a@example.com" { - t.Fatalf("author = %+v", got.body.Author) - } case <-time.After(2 * time.Second): t.Fatal("CreateBuild not called") } } // TestBuildkiteSpawnInvalidYAML proves a workflow without the -// required `pipeline` field is skipped — no API call, no DB row, no -// status. A misconfigured workflow shouldn't be silently swept onto -// some default pipeline. +// required `tack.buildkite.pipeline` field is skipped — no API +// call, no DB row, no status. A misconfigured workflow shouldn't +// be silently swept onto some default pipeline. func TestBuildkiteSpawnInvalidYAML(t *testing.T) { called := false bk := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -265,7 +247,7 @@ func TestBuildkiteSpawnInvalidYAML(t *testing.T) { time.Sleep(50 * time.Millisecond) if called { - t.Fatal("CreateBuild called for workflow missing pipeline") + t.Fatal("CreateBuild called for workflow missing tack.buildkite.pipeline") } rows, _ := st.EventsAfter(context.Background(), 0) if len(rows) != 0 { -- 2.51.2