From 0e591b6fa0c80be5215b5f081fa54fc8f75a05da Mon Sep 17 00:00:00 2001 From: Bretton Date: Sat, 15 Aug 2026 22:18:51 -0700 Subject: [PATCH] =?UTF-8?q?feat(cmd):=20tidepool=20rotate-kek=20=E2=80=94?= =?UTF-8?q?=20the=20drill=20becomes=20an=20operator=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reads exactly DATABASE_URL, BRIDGE_KEK and BRIDGE_KEK_PREVIOUS (the migrate principle: a rotation must never be blocked by unrelated config), validates before dialling, blames precisely, refuses equal keys (one key in both variables would read exactly like the zero-run gate), logs the per-table inventory, and exits nonzero whenever the walk could not move every blob. Compose forwards BRIDGE_KEK_PREVIOUS (inert when unset); DEPLOY §6 carries the runbook, FOLLOWUPS closes the blocker and files the adjacent lost-escrow-row hazard. Co-Authored-By: Claude Fable 5 --- .env.prod.example | 13 ++- DEPLOY.md | 216 +++++++++++++++++++++++++++++--------- FOLLOWUPS.md | 65 +++++++++--- README.md | 6 +- cmd/tidepool/main.go | 97 +++++++++++++++-- docker-compose.prod.yml | 7 ++ internal/config/config.go | 14 ++- 7 files changed, 336 insertions(+), 82 deletions(-) diff --git a/.env.prod.example b/.env.prod.example index b034491..30d3bec 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -35,8 +35,8 @@ AP_USER_ORIGIN=https://coves.social # 32-byte key-encryption key sealing per-actor signing keys at rest # (AES-256-GCM). Generate with: openssl rand -hex 32 # Losing it means losing every repo's signing keys — back it up. -# THERE IS NO ROTATION PATH. Nothing in the codebase re-seals existing -# ciphertext under a new KEK; changing this value orphans every sealed key, +# CHANGING IT IS A PROCEDURE, NOT AN EDIT. Set BRIDGE_KEK_PREVIOUS below and +# follow the runbook; editing this value on its own orphans every sealed key, # across THREE tables, with no recovery: # - bridged_actors.signing_key every bridged (Lemmy-origin) identity's # escrowed secp256k1 atproto repo key @@ -47,9 +47,16 @@ AP_USER_ORIGIN=https://coves.social # rotation or backup plan written before # v2 existed — and it is the entire # native-user population. -# See DEPLOY.md "Not implemented" before you touch it. BRIDGE_KEK=CHANGE_ME +# The KEK being rotated AWAY from. Unset in steady state; set only for the +# length of a rotation, during which the bridge opens material under either key +# and `tidepool rotate-kek` moves every sealed blob onto BRIDGE_KEK. Unset it +# again only after the walk reports zero re-seals and zero failures — until +# then it is the only key some rows still open under. +# Full procedure: DEPLOY.md §6, "Runbook: rotating BRIDGE_KEK". +#BRIDGE_KEK_PREVIOUS= + # Bearer token for the /admin API. Generate with: openssl rand -hex 32 ADMIN_TOKEN=CHANGE_ME diff --git a/DEPLOY.md b/DEPLOY.md index 8c4d8a3..c0fcf24 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -680,23 +680,22 @@ schema-version question. ## 6. Not implemented -Everything in this section is a real operational gap. None of it has a -mechanism in the code today. It is written down because an operator who -assumes one of these exists will look for it during the exact incident where -looking costs the most. +Everything in this section is a real operational gap, with one exception kept +here on purpose: **`BRIDGE_KEK` rotation used to be the headline gap and is now +built**, so its entry has become the runbook rather than being deleted — an +operator who comes here looking for the gap finds the procedure instead of a +stale warning. Everything else below has no mechanism in the code today. It is +written down because an operator who assumes one of these exists will look for +it during the exact incident where looking costs the most. -### Key rotation — `BRIDGE_KEK` and per-actor RSA keys +### Key rotation — `BRIDGE_KEK` (BUILT) and per-actor RSA keys (not) -**No rotation path exists. Not partial, not manual, not scripted.** +**`BRIDGE_KEK` rotation exists: a dual-read custodian plus a `rotate-kek` +re-seal walk, runbook below.** Per-actor **RSA** key rotation still does not +exist, and neither does a rotation for the plaintext service-actor key; both +are at the end of this entry. -Every mention of `BRIDGE_KEK` in this repository is a warning, never a -procedure. `internal/config/config.go:49-53` documents the value; the sealing -itself is AES-256-GCM in `internal/identity/keys.go`. There is nothing that -re-seals existing ciphertext under a new key: the binary has exactly two -subcommands, `tidepool` and `tidepool migrate` -(`cmd/tidepool/main.go:69-78`). - -Blast radius of losing or changing it — **three** tables, not two: +What the KEK seals — **three** places, not two, and the walk covers all three: 1. **`bridged_actors.signing_key`** — the escrowed **secp256k1 atproto** repo signing key of every bridged (Lemmy-origin) identity @@ -704,7 +703,8 @@ Blast radius of losing or changing it — **three** tables, not two: `internal/identity/keys.go:86-96`). Approximately 950 of them. 2. **`service_keys.key_material`, row `plc-rotation`** — the PLC **escrow rotation key** (`internal/identity/keys.go:140-144`), the one thing that - could recover the DIDs, itself sealed under the key you just replaced. Note + could recover the DIDs, and itself sealed under the KEK — which is why the + boot canary in step 2 of the runbook is a real test of the rotation. Note the column is `key_material`, not `private_key_pem`; migration 013 renamed it precisely because only this row is ciphertext — the sibling `service-actor` row is **plaintext** PKCS#8 PEM and is *not* KEK-sealed @@ -717,47 +717,159 @@ Blast radius of losing or changing it — **three** tables, not two: handed to `personas.New` at `cmd/tidepool/main.go:521-527`. **(3) is the v2 one, and it is the one a rotation plan will forget**, because -it did not exist when this section was first written. A rotation built to -handle only `bridged_actors` and `service_keys` would leave every native user -unable to sign a single outbound activity — the exact population v2 exists to -serve. Change the KEK and all three sets of ciphertext become undecryptable. -There is no recovery. +it did not exist when this section was first written. A rotation that handled +only `bridged_actors` and `service_keys` would leave every native user unable +to sign a single outbound activity — the exact population v2 exists to serve. +The walk covers it; anything hand-rolled must too. *Naming trap:* `LoadOrCreateRotationKey` is **not** KEK rotation. It loads or generates the did:plc escrow/recovery key — an atproto identity concept — which is itself sealed under the KEK. Do not read that symbol as evidence that -rotation is implemented. - -What a real rotation would require: - -- **A key-version column or KEK-id alongside each sealed blob.** *Partly - present, and this is worth knowing before anyone designs it from scratch.* - `ap_actors` **already has one**: `rsa_key_version INT NOT NULL` - (`internal/db/migrations/017_ap_actors.sql:47`), and that migration's own - comment says it is there so "rotation [is] definable without a schema change" - (`:23-24`). It is stamped from `currentRSAKeyVersion = 1` - (`internal/personas/personas.go:25`, applied at `:185`) and read back, but - **nothing uses it as a selector** — no code branches on it to choose a KEK. - So on `ap_actors` the schema work is done and only the logic is missing. - `bridged_actors.signing_key` and `service_keys.key_material` genuinely have - no version column; those two need the migration as well. -- **A dual-read custodian** that tries the new KEK then the old. -- **An online re-seal pass** over all three tables above — `ap_actors` - included, which is the one a v1-era plan omits. -- **A cutover** that retires the old KEK only after the pass completes. - -The ciphertext also carries a one-byte version prefix -(`internal/identity/keys.go:127`) — but that is the *envelope format* version, -checked for equality and rejected otherwise, not a key id. Neither it nor -`rsa_key_version` selects a key today. - -Per-actor **RSA** rotation is equally undefined: rotating an actor's key means -republishing `publicKey` in its actor document and having every peer that -cached it re-fetch, with no grace-overlap mechanism in the code to publish two -keys at once. - -**Until this is built, treat `BRIDGE_KEK` as immutable, and back it up -somewhere that survives the loss of the server.** +KEK rotation lives there; it lives in `internal/identity/reseal.go`. + +#### Why there is no key-version column + +This section used to say a rotation would need "a key-version column or KEK-id +alongside each sealed blob", and a migration for the two tables that lack one. +**That claim is retired. The shipped design needs no schema change and no +stored selector**, and anyone designing an extension should know why before +re-proposing one. + +GCM authentication is a *definitive* discriminator. Opening a blob under the +wrong key does not return plausible garbage — it fails the tag check, and the +odds of a wrong key authenticating are about 2⁻¹²⁸. So the KEK a blob is +sealed under can simply be **tried**, and the answer is as trustworthy as any +column would have been — while a column can be wrong (written by a crashed +half-rotation, restored from a mismatched backup) in a way the ciphertext +cannot. `NewCustodianWithPrevious` (`internal/identity/keys.go`) is that trial +on the read path; `identity.Reseal` (`internal/identity/reseal.go`) is the same +trial used to classify: opens under current → already moved, leave the bytes +alone; opens under previous → re-seal and write back; opens under neither → +report it and **never** write. + +`ap_actors.rsa_key_version` was never the missing selector, despite looking +like one. It versions **the actor's RSA key** — a different, still-unbuilt +rotation (below) — not the KEK the key is wrapped in. Reading it as a KEK +selector is the trap this paragraph exists to close. Likewise the one-byte +prefix on every ciphertext (`internal/identity/keys.go`) is the *envelope +format* version, checked for equality and rejected otherwise; it is not a key +id either. + +#### Runbook: rotating `BRIDGE_KEK` + +The old key stays load-bearing for the whole of this procedure. Do not delete +it from anywhere until step 6. + +1. **Put both keys in `/opt/tidepool/.env`** — the new one current, the old one + previous. Generate the new one with `openssl rand -hex 32`. + + ```sh + BRIDGE_KEK= + BRIDGE_KEK_PREVIOUS= + ``` + + Both are forwarded to the `tidepool` service in + `docker-compose.prod.yml`; `BRIDGE_KEK_PREVIOUS` is empty in steady state + and the bridge treats empty as unset. + +2. **Restart the bridge.** + + ```sh + docker compose -f docker-compose.prod.yml up -d tidepool + ``` + + It must come up clean. The boot canary opens the `plc-rotation` key under + the KEK before serving traffic, and at this point it only opens under the + *previous* one — so a clean boot is proof the dual read is working, and a + failed boot means one of the two values is wrong. Fix it here, where + nothing has been rewritten yet. + +3. **Run the walk.** + + ```sh + docker compose -f docker-compose.prod.yml run --rm tidepool rotate-kek + ``` + + Not the `tidepool-migrate` one-shot: that service is deliberately wired + with `DATABASE_URL` alone, so it has no KEKs at all and `rotate-kek` there + would fail naming a variable that container is never meant to carry. + +4. **Read the inventory.** One line per table, plus one line per row that + could not be moved: + + ``` + level=INFO msg="kek re-seal table" table=bridged_actors resealed=948 already_current=0 skipped=3 failed=0 + level=INFO msg="kek re-seal table" table=ap_actors resealed=112 already_current=0 skipped=0 failed=0 + level=INFO msg="kek re-seal table" table=service_keys resealed=1 already_current=0 skipped=0 failed=0 + ``` + + A row that could not be moved gets its own line, at ERROR, naming the table, + the row, and the class: + + ``` + level=ERROR msg="kek re-seal failure" table=ap_actors id=did:plc:7iza… reason=malformed + ``` + + `skipped` is a bridged actor with no escrowed key (`signing_key IS NULL`) — + normal, not a problem. The plaintext `service-actor` row is not counted + anywhere, by design: the walk selects `plc-rotation` by name and never reads + its plaintext sibling. + +5. **Re-run until the zero-run gate opens**, i.e. every table reports + `resealed=0 failed=0` and the command exits 0. That re-run is the only + evidence that nothing is still sealed under the old key — the first run's + own counts are not, because a row written between its read and its finish + would not be in them. + +6. **Only then remove `BRIDGE_KEK_PREVIOUS`** from `/opt/tidepool/.env` and + restart: + + ```sh + docker compose -f docker-compose.prod.yml up -d tidepool + ``` + + The boot canary now runs on the new key alone. If it fails, put the old key + back immediately and go to step 3 — the material is still intact at that + point, because the walk never overwrites what it cannot open. + +**Warnings.** + +- **Never start a second rotation before the first has shown its zero run.** + This tool moves blobs from *one* previous key to *one* current key. Introduce + a third key while blobs are still under the first and those blobs open under + neither: `rotate-kek` reports them `wrong-key` and stops, and nothing in this + repository can recover them without the missing key itself. +- **`failed=... reason=wrong-key` is a key-history question.** The blob is well + formed and simply not sealed under either key you supplied. Find the KEK it + belongs to (an older `.env`, a restored volume, another deployment) — + **do not rotate again**, which only adds a key to the search. +- **`reason=malformed` is data corruption, not a key problem.** The blob is + truncated or carries an unknown version byte; it was never decrypted under + either key, so no KEK is implicated. Restore that single row from backup. + Rotating again cannot help and buries the evidence. +- **`reason=contended` means only that another writer kept touching the row.** + Nothing is wrong. Re-run the drill. +- **A failed walk exits non-zero and finishes the inventory anyway.** Both are + deliberate: the exit code is what a deploy script reads as permission to + unset the previous KEK, and the full inventory means one pass tells the + operator how much is affected rather than stopping at the first bad row. + +#### Still not implemented: per-actor RSA rotation + +Rotating an individual actor's **RSA** key is undefined. It means republishing +`publicKey` in that actor's document and having every peer that cached it +re-fetch, with no grace-overlap mechanism in the code to publish two keys at +once. `ap_actors.rsa_key_version` exists for exactly this rotation +(`internal/db/migrations/017_ap_actors.sql`) and nothing reads it yet. + +The bridge's own **service-actor** RSA key is also still stored as plaintext +PKCS#8 PEM (`service_keys`, row `service-actor`), so it is outside the KEK +entirely — rotating it is a separate, unbuilt procedure, and the KEK walk +correctly refuses to touch it. + +**Back up `BRIDGE_KEK` somewhere that survives the loss of the server.** +Rotation is no longer a reason to treat it as immutable, but losing it — with +no previous key to fall back to — is still unrecoverable. ### Backup and restore diff --git a/FOLLOWUPS.md b/FOLLOWUPS.md index db3f75f..d9212d1 100644 --- a/FOLLOWUPS.md +++ b/FOLLOWUPS.md @@ -328,25 +328,41 @@ longer load-bearing: the self-hosted relay + Jetstream in hosts with an effectively unlimited account limit. `bsky.network` remains the wider-visibility path only. Runbook: `SELF_HOSTED_RELAY.md`. +RESOLVED — **the `BRIDGE_KEK` rotation path is built.** A KEK can now be +changed without orphaning the three sets of sealed material +(`bridged_actors.signing_key`, `service_keys.key_material` row `plc-rotation`, +`ap_actors.rsa_key_sealed`). Runbook: `DEPLOY.md` §6, "Runbook: rotating +BRIDGE_KEK". + +- *Design chosen:* **dual-read by trial, plus an offline re-seal walk.** + `NewCustodianWithPrevious` opens under the current KEK then the previous one; + `identity.Reseal` (driven by `tidepool rotate-kek`) walks all three domains + and uses the same trial to classify every blob — opens under current (leave + the bytes alone), opens under previous (re-seal and write back under an + optimistic guard), opens under neither (report, never write). Re-run until it + reports zero re-seals and zero failures, then unset `BRIDGE_KEK_PREVIOUS`. +- *Rejected candidate:* **a persisted key-version selector on each sealed + blob**, which this list previously called the first requirement. It buys + nothing here: GCM authentication already answers "which key sealed this?" + definitively (a wrong key fails the tag; ~2⁻¹²⁸), and unlike a column the + ciphertext cannot be *wrong* about it — a stored version can be left stale by + a half-finished rotation or a mismatched restore, and would then send the + reader to a key that does not open the bytes. Rejecting it also removed the + migration on `bridged_actors` and `service_keys` that the earlier plan + required. **`ap_actors.rsa_key_version` was never that selector**, though it + reads like one: it versions the *actor's RSA key* — an independent, still + unbuilt rotation — not the KEK the key is wrapped in. +- *Residual limits, still open:* **per-actor RSA key rotation itself is + unbuilt** (republishing `publicKey` and getting peers to re-fetch, with no + two-key grace overlap in the code; `rsa_key_version` is stamped and never + read), and **the bridge's own service-actor RSA key is still plaintext PEM** + in `service_keys`, outside the KEK entirely — the walk deliberately refuses + to touch that row, so rotating it remains a separate unbuilt procedure. + DOCUMENTED, not resolved — the v2 deploy gaps below now have a written home in `DEPLOY.md` (§6 "Not implemented") with their blast radius. Writing them down is not building them; they stay open here: -- **No `BRIDGE_KEK` / per-actor RSA rotation path.** Nothing re-seals existing - ciphertext under a new KEK, and the binary's only subcommand is `migrate` - (no args = serve). Changing the KEK orphans sealed key material in **three** - tables, not two: `bridged_actors.signing_key` (~950 bridged identities' - escrowed secp256k1 repo keys), `service_keys.key_material` row `plc-rotation` - (the PLC escrow key, the only DID recovery path), and — added by v2, and the - one a pre-v2 plan omits — `ap_actors.rsa_key_sealed`, **every native Coves - user's AP signing key** (`internal/db/migrations/017_ap_actors.sql:46`). No - recovery for any of it. Would need a key-version selector on each sealed - blob, a dual-read custodian, an online re-seal pass over all three, and a - cutover. Partial credit on the first: `ap_actors.rsa_key_version` already - exists (`017_ap_actors.sql:47`, stamped from `currentRSAKeyVersion = 1` at - `internal/personas/personas.go:25`) and is deliberately there so rotation is - definable without a schema change — but nothing reads it as a selector, and - the other two tables have no version column at all. - **No backup or restore procedure.** `docker-compose.prod.yml` mounts `./backups` into the Postgres container and nothing writes to it. Note `BRIDGE_KEK` lives in `.env` and is not covered by any database backup at @@ -370,6 +386,25 @@ is not building them; they stay open here: one-community canary must enumerate every *other* subscribed community — and adding a community to `communities.yaml` silently escapes an existing canary. +Found while planning the KEK re-seal drill (NOT a defect of that work): + +- **A MISSING `plc-rotation` row on a populated database boots clean and + silently orphans every bridged DID.** `LoadOrCreateRotationKey` is + create-on-absence by design — that is what makes first boot and the + bootstrap race safe — but it cannot tell "first boot" from "the row is gone". + Restore a backup taken before the row existed, restore `bridged_actors` + without `service_keys`, or point `DATABASE_URL` at the wrong database, and + the bridge mints a **fresh** escrow key, seals it, stores it, and comes up + green. Nothing is logged as unusual. Every already-minted did:plc document + still names the OLD rotation key, so the bridge now holds an authority over + nothing while the only key that could recover ~950 DIDs is the one that just + went missing — and the clean boot is exactly what stops anyone from looking. + The KEK drill sharpened this: `rotate-kek` reports `service_keys` counts, so + an operator can see the row is there, but only if they run it. + *Candidate guard:* refuse to create when `bridged_actors` is non-empty — a + populated bridge with no rotation key is never a legitimate first boot — and + fail startup with a message that says which restore went wrong. + Still open, unchanged: - `ENVIRONMENT=production` has not been exercised end-to-end. The harness uses diff --git a/README.md b/README.md index a8e6f59..1044039 100644 --- a/README.md +++ b/README.md @@ -308,6 +308,7 @@ Two classes, and the difference matters at boot: | `BRIDGE_SCHEME` | `https` | scheme of the bridge's own AP URLs (actor id, inbox, activity ids). `http` is dev-only — the e2e harness federates with a debug-mode Lemmy over plain HTTP | | `PLC_DIRECTORY_URL` | `http://localhost:3002` (local, `make plc-up`) | did:plc directory; production uses `https://plc.directory` | | `BRIDGE_KEK` | fixed public dev key | 32-byte key-encryption key (64 hex chars or base64) sealing per-actor signing keys and the escrow rotation key at rest (AES-256-GCM) | +| `BRIDGE_KEK_PREVIOUS` | *(unset)* | the KEK being rotated away from, same encodings. Set only during a rotation: sealed material opens under either key while it is set, nothing new is sealed under it, and `tidepool rotate-kek` moves every blob onto `BRIDGE_KEK` so it can be unset again (runbook: `DEPLOY.md` §6) | | `BRIDGE_SERVICE_DID` | *(optional)* | pre-provisioned service DID for the bridge's own actor | | `USER_AGENT` | derived | outbound HTTP user agent | | `ALLOW_PRIVATE_FETCH` | off | dev-only: disables the SSRF egress guard (AP fetches **and** PLC directory requests) so localhost targets work | @@ -722,8 +723,9 @@ The runbook is **[`DEPLOY.md`](DEPLOY.md)**: the boot-time config gate, the v2 flag topology (`CONSUMER_ENABLED` → `OUTBOUND_WORKERS` → kill switches), the cross-repo Caddy change that puts the native-user AP surface on `coves.social`, the staged canary and its rollback order, and — explicitly — -the things that have **no** mechanism today (KEK/RSA rotation, backup/restore, -a divergence off switch, a periodic vote re-seed). +the things that have **no** mechanism today (per-actor RSA rotation, +backup/restore, a divergence off switch, a periodic vote re-seed). `BRIDGE_KEK` +rotation used to head that list and is now a runbook in the same section. [`SELF_HOSTED_RELAY.md`](SELF_HOSTED_RELAY.md) covers the relay + Jetstream ingest path. diff --git a/cmd/tidepool/main.go b/cmd/tidepool/main.go index 25db5f6..24af79e 100644 --- a/cmd/tidepool/main.go +++ b/cmd/tidepool/main.go @@ -4,6 +4,7 @@ package main import ( + "bytes" "context" "database/sql" "errors" @@ -13,6 +14,7 @@ import ( "net/url" "os" "os/signal" + "strings" "sync" "syscall" "time" @@ -75,18 +77,101 @@ func dispatch(logger *slog.Logger, args []string) error { case len(args) == 1 && args[0] == "rotate-kek": return runRotateKEK(logger) default: - return fmt.Errorf("usage: tidepool [migrate]") + return fmt.Errorf("usage: tidepool [migrate|rotate-kek]") } } // runRotateKEK re-seals every KEK-sealed blob under the current BRIDGE_KEK so -// the operator can retire BRIDGE_KEK_PREVIOUS. +// the operator can retire BRIDGE_KEK_PREVIOUS (the runbook is in DEPLOY.md). // -// STUB: the dispatch wiring exists so the command's contract — which -// variables it requires and which it must not — can be driven by tests. The -// walk itself is identity.Reseal, already implemented. +// Like runMigrations it reads a minimal environment — DATABASE_URL, +// BRIDGE_KEK, BRIDGE_KEK_PREVIOUS — and deliberately never calls config.Load. +// Here that is load-bearing rather than tidy: the operator running this is +// mid-rotation, often from a one-off container or a maintenance shell that +// carries the database URL and the two keys and nothing else, and a rotation +// that refuses to start because some unrelated HTTP or relay variable is +// unset strands every sealed blob under the key being retired. func runRotateKEK(logger *slog.Logger) error { - return fmt.Errorf("rotate-kek: not implemented") + // Every variable is checked BEFORE the database is dialled, and each is + // blamed strictly by its own name: an operator holding two 32-byte secrets + // who is sent to edit the one that is already correct will orphan the + // material still sealed under it. + databaseURL := os.Getenv("DATABASE_URL") + if databaseURL == "" { + return fmt.Errorf("rotate-kek: DATABASE_URL is required") + } + currentEncoded := strings.TrimSpace(os.Getenv("BRIDGE_KEK")) + if currentEncoded == "" { + return fmt.Errorf("rotate-kek: BRIDGE_KEK is required: it names the key every sealed blob is moved onto") + } + previousEncoded := strings.TrimSpace(os.Getenv("BRIDGE_KEK_PREVIOUS")) + if previousEncoded == "" { + return fmt.Errorf("rotate-kek: BRIDGE_KEK_PREVIOUS is required: it names the key the blobs are moved off, and without it there is nothing to re-seal") + } + + current, err := config.DecodeKEK("BRIDGE_KEK", currentEncoded) + if err != nil { + return fmt.Errorf("rotate-kek: %w", err) + } + previous, err := config.DecodeKEK("BRIDGE_KEK_PREVIOUS", previousEncoded) + if err != nil { + return fmt.Errorf("rotate-kek: %w", err) + } + // Compared on the decoded bytes for the same reason config.Load does it: + // one key pasted into both variables is not a rotation, and a walk that + // reported every blob as already-current would be read as the zero-run + // that clears the operator to retire a key. + if bytes.Equal(current, previous) { + return fmt.Errorf("rotate-kek: the two KEKs decode to the same key; a rotation needs two different keys") + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + database, err := db.Open(ctx, databaseURL) + if err != nil { + return fmt.Errorf("rotate-kek: %w", err) + } + defer func() { _ = database.Close() }() + + // The report comes back even on the error path, and it is the whole point + // of the command, so it is logged before the error is returned. + report, resealErr := identity.Reseal(ctx, database, current, previous) + if report != nil { + logResealReport(logger, report) + } + if resealErr != nil { + return fmt.Errorf("rotate-kek: %w", resealErr) + } + logger.Info("kek re-seal complete; re-run until every table reports resealed=0 and failed=0 before unsetting BRIDGE_KEK_PREVIOUS") + return nil +} + +// logResealReport writes the inventory one line per table, then one line per +// unmovable row. The operator's decision to retire a KEK is made from these +// lines, so nothing is folded into a grand total: a table is where they look +// for the zero-run, and a failure has to name its row and its class (wrong-key +// sends them to key history, malformed to backups, contended to a re-run). +func logResealReport(logger *slog.Logger, report *identity.ResealReport) { + for _, table := range []struct { + name string + counts identity.ResealCounts + }{ + {"bridged_actors", report.BridgedActors}, + {"ap_actors", report.APActors}, + {"service_keys", report.ServiceKeys}, + } { + logger.Info("kek re-seal table", + "table", table.name, + "resealed", table.counts.Resealed, + "already_current", table.counts.AlreadyCurrent, + "skipped", table.counts.Skipped, + "failed", table.counts.Failed) + } + for _, failure := range report.Failures { + logger.Error("kek re-seal failure", + "table", failure.Table, "id", failure.ID, "reason", string(failure.Reason)) + } } // runMigrations applies every pending embedded Goose migration and exits. It diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 9481667..49b9487 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -95,6 +95,13 @@ services: BRIDGE_HOSTNAME: ${BRIDGE_HOSTNAME:-tdpl.io} PLC_DIRECTORY_URL: https://plc.directory BRIDGE_KEK: ${BRIDGE_KEK} + # Empty in steady state (the bridge reads empty as unset). It is set for + # the length of a KEK rotation ONLY, and it must be forwarded here rather + # than only to a one-off container: the running bridge needs the old key + # to open pre-rotation material, and `docker compose run tidepool + # rotate-kek` inherits this same service environment. See DEPLOY.md §6, + # "Runbook: rotating BRIDGE_KEK". + BRIDGE_KEK_PREVIOUS: ${BRIDGE_KEK_PREVIOUS:-} ADMIN_TOKEN: ${ADMIN_TOKEN} # Gentler than the built-in 60/120: the public plc.directory 429s # mint bursts during community backfill (observed at launch), and a diff --git a/internal/config/config.go b/internal/config/config.go index 42c218f..ed686c6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -340,7 +340,7 @@ func Load(logger *slog.Logger) (*Config, error) { if err != nil { return nil, err } - cfg.BridgeKEK, err = decodeKEK("BRIDGE_KEK", kekEncoded) + cfg.BridgeKEK, err = DecodeKEK("BRIDGE_KEK", kekEncoded) if err != nil { return nil, err } @@ -350,7 +350,7 @@ func Load(logger *slog.Logger) (*Config, error) { // KEK would refuse to boot every bridge that has never rotated. A rotation // is a temporary state; the absence of the variable is the normal one. if previousEncoded := strings.TrimSpace(os.Getenv("BRIDGE_KEK_PREVIOUS")); previousEncoded != "" { - cfg.BridgeKEKPrevious, err = decodeKEK("BRIDGE_KEK_PREVIOUS", previousEncoded) + cfg.BridgeKEKPrevious, err = DecodeKEK("BRIDGE_KEK_PREVIOUS", previousEncoded) if err != nil { return nil, err } @@ -657,11 +657,17 @@ func (c *Config) IsDevelopment() bool { return c.Environment == EnvironmentDevelopment } -// decodeKEK parses a KEK-carrying variable: 64 hex chars or standard base64, +// DecodeKEK parses a KEK-carrying variable: 64 hex chars or standard base64, // either way decoding to exactly 32 bytes. name is the environment variable // the value came from, so an operator holding two KEKs mid-rotation is told // which one they broke rather than being sent to check the good one. -func decodeKEK(name, encoded string) ([]byte, error) { +// +// Exported for the rotate-kek one-shot, which reads BRIDGE_KEK and +// BRIDGE_KEK_PREVIOUS without going through Load (an operational command must +// not be blockable by config it does not use) and must still accept exactly +// the encodings the server does — a second decoder would eventually drift and +// reject the very key the running bridge is sealing under. +func DecodeKEK(name, encoded string) ([]byte, error) { encoded = strings.TrimSpace(encoded) if raw, err := hex.DecodeString(encoded); err == nil { if len(raw) != 32 { -- 2.51.2