From 2676629e79c432f2a16ef3ccd8fbd4cf2cafd9ca Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Thu, 3 Sep 2026 14:55:28 -0400 Subject: [PATCH] refactor(infra)!: split the PDS and the website into two roots Each root keys its own state in the did.bot account's state bucket, so applying the website cannot lock or roll back the deployment. The delegation between them is one value: `infra/pds/`'s `name_servers` output goes to `infra/site/`'s `pds_zone_name_servers`. Co-Authored-By: Claude Opus 5 (1M context) Change-Id: If25943e96e1bf3758fac64581a47854b1a322d92 --- crates/didbot-dns/src/route53.rs | 4 +- crates/didbot-pds/src/lock.rs | 2 +- crates/didbot-serve/src/bin/didbot-pds.rs | 8 ++-- crates/didbot-serve/src/lib.rs | 2 +- .../didbot-serve/tests/graceful_shutdown.rs | 2 +- docs/deployment.md | 8 ++-- docs/operations.md | 4 +- infra/{ => pds}/.terraform.lock.hcl | 0 infra/pds/backend.tf | 23 +++++++++++ infra/{ => pds}/backup.tf | 0 infra/{ => pds}/ec2.tf | 2 +- infra/{ => pds}/iam.tf | 0 infra/{ => pds}/outputs.tf | 10 ++--- infra/{ => pds}/route53.tf | 11 +++--- infra/{ => pds}/security_groups.tf | 0 infra/{ => pds}/templates/user_data.sh.tftpl | 2 +- infra/{ => pds}/variables.tf | 0 infra/{ => pds}/versions.tf | 5 ++- plan/adversarial.md | 2 +- plan/agent-accounts.md | 2 +- plan/aws-deploy.md | 38 ++++++++++--------- plan/blob-storage-tiers.md | 2 +- plan/cost.md | 22 +++++------ plan/dedupe-audit.md | 2 +- plan/deploy.md | 19 +++++++--- plan/didjson-archive.md | 2 +- plan/onboarding.md | 2 +- plan/periodic-backups.md | 10 ++--- plan/tombstone-serving.md | 2 +- 29 files changed, 111 insertions(+), 75 deletions(-) rename infra/{ => pds}/.terraform.lock.hcl (100%) create mode 100644 infra/pds/backend.tf rename infra/{ => pds}/backup.tf (100%) rename infra/{ => pds}/ec2.tf (99%) rename infra/{ => pds}/iam.tf (100%) rename infra/{ => pds}/outputs.tf (81%) rename infra/{ => pds}/route53.tf (81%) rename infra/{ => pds}/security_groups.tf (100%) rename infra/{ => pds}/templates/user_data.sh.tftpl (98%) rename infra/{ => pds}/variables.tf (100%) rename infra/{ => pds}/versions.tf (68%) diff --git a/crates/didbot-dns/src/route53.rs b/crates/didbot-dns/src/route53.rs index 1a0f7390..71172202 100644 --- a/crates/didbot-dns/src/route53.rs +++ b/crates/didbot-dns/src/route53.rs @@ -67,7 +67,7 @@ //! //! Seeing it coming is worth more here than the account arithmetic suggests. //! Certificate renewal on this deployment is ACME DNS-01 — there is no -//! HTTP-01 fallback, `infra/security_groups.tf` opens no port 80 — and every +//! HTTP-01 fallback, `infra/pds/security_groups.tf` opens no port 80 — and every //! renewal creates one record set, the `_acme-challenge.` `TXT`, and //! deletes it again when the order is done. A zone at its quota cannot //! create it, so the certificate stops being renewable before it stops being @@ -165,7 +165,7 @@ impl Route53Credentials { /// /// IMDSv2 specifically, not v1: v1's tokenless `GET` is what let the /// 2019 Capital One breach turn an unrelated request-smuggling bug into - /// stolen instance-role credentials, and `infra/`'s launch template + /// stolen instance-role credentials, and `infra/pds/`'s launch template /// requires the token hop (`HttpTokens = required`) for exactly that /// reason — see `docs/deployment.md`. This is the one credential path /// this crate offers a production deployment: there is no environment diff --git a/crates/didbot-pds/src/lock.rs b/crates/didbot-pds/src/lock.rs index d3ba1a27..b2ff1c7e 100644 --- a/crates/didbot-pds/src/lock.rs +++ b/crates/didbot-pds/src/lock.rs @@ -40,7 +40,7 @@ //! * **It is local to the host.** `flock` on NFS is not dependable. The data //! directory is an ext4 filesystem on one attached volume; an EBS volume //! cannot be attached to two instances at once in the configuration -//! `infra/ec2.tf` uses, so the host boundary and the volume boundary are the +//! `infra/pds/ec2.tf` uses, so the host boundary and the volume boundary are the //! same boundary. //! //! [`Durable::open`]: crate::Durable::open diff --git a/crates/didbot-serve/src/bin/didbot-pds.rs b/crates/didbot-serve/src/bin/didbot-pds.rs index c999ce7c..7dec8b9d 100644 --- a/crates/didbot-serve/src/bin/didbot-pds.rs +++ b/crates/didbot-serve/src/bin/didbot-pds.rs @@ -1,5 +1,5 @@ //! One command that boots the whole stack — the same binary for a laptop and -//! for the deployment described in `infra/`. +//! for the deployment described in `infra/pds/`. //! //! An account store behind the HTTP surface from [`didbot_serve`], with every //! lifecycle transition streamed to `/events` and written to the terminal. @@ -1476,7 +1476,7 @@ fn policy_gate() -> Arc { /// writes the record; /// * a real zone without one, [`WildcardDns`], which publishes *nothing*. /// That was correct while a Terraform-managed wildcard `ALIAS` record -/// covered every hostname under the zone; `infra/` no longer creates one, +/// covered every hostname under the zone; `infra/pds/` no longer creates one, /// so this is now a development stand-in and says so, loudly, once per /// zone at startup. /// @@ -1509,7 +1509,7 @@ fn build_zone_dns(args: &Args, zone: &Zone) -> Result, Stri zone = zone.host(), "no --route53-zone-id for this zone: agent hostnames will not be published \ anywhere. This is a development stand-in -- the wildcard ALIAS record it \ - relied on is not part of infra/ any more" + relied on is not part of infra/pds/ any more" ); Ok(Box::new(WildcardDns::new(zone.host().to_owned()))) } @@ -2048,7 +2048,7 @@ mod config_tests { ("RECORD_TARGET_PLACEHOLDER", "203.0.113.47"), ]; - let unit = include_str!("../../../../infra/templates/user_data.sh.tftpl"); + let unit = include_str!("../../../../infra/pds/templates/user_data.sh.tftpl"); let exec_start = unit .split_once("\nExecStart=") .expect("the unit has an ExecStart") diff --git a/crates/didbot-serve/src/lib.rs b/crates/didbot-serve/src/lib.rs index dd491122..de244455 100644 --- a/crates/didbot-serve/src/lib.rs +++ b/crates/didbot-serve/src/lib.rs @@ -400,7 +400,7 @@ pub async fn serve_tls( /// to, is the normal case rather than the edge one. /// /// The number is chosen against the deployment's own: the unit file in -/// `infra/templates/user_data.sh.tftpl` stops the container with +/// `infra/pds/templates/user_data.sh.tftpl` stops the container with /// `docker stop -t 20`, so there are twenty seconds between the `SIGTERM` /// and the `SIGKILL`. Stopping at ten leaves the other ten for what runs /// *after* serving ends — the write-ahead log is synced there, and a sync diff --git a/crates/didbot-serve/tests/graceful_shutdown.rs b/crates/didbot-serve/tests/graceful_shutdown.rs index a4e5c478..63ce9240 100644 --- a/crates/didbot-serve/tests/graceful_shutdown.rs +++ b/crates/didbot-serve/tests/graceful_shutdown.rs @@ -1,7 +1,7 @@ //! `SIGTERM` stops the server the way `SIGINT` does. //! //! This is the signal a deployment actually sends. `docker stop` sends it, -//! and so does systemd's `ExecStop` — see `infra/templates/user_data.sh.tftpl` +//! and so does systemd's `ExecStop` — see `infra/pds/templates/user_data.sh.tftpl` //! — so a server that only listens for Ctrl-C is one whose every stop runs to //! the grace period and ends in `SIGKILL`. In the container that is worse than //! slow: the process is PID 1 there, which has no default disposition for diff --git a/docs/deployment.md b/docs/deployment.md index e5462084..0b457f38 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -12,7 +12,7 @@ interface and no part of a deployment. The image is built from the repository's `Dockerfile` with `--features route53`, tagged with the release it holds, and pushed to ECR. -`infra/templates/user_data.sh.tftpl` writes `/etc/systemd/system/ +`infra/pds/templates/user_data.sh.tftpl` writes `/etc/systemd/system/ didbot-pds.service` at first boot, substituting the image reference and the deployment's zone, owner and hosted zone id directly into the unit — so `systemctl cat didbot-pds` shows exactly what is running, with no environment @@ -23,7 +23,7 @@ rather than a preference: - **The unit must be a single-writer service.** The store is one write-ahead log behind one lock, and two live writers interleave frames neither can - replay. `infra/ec2.tf` is one instance, not an autoscaling group; the unit + replay. `infra/pds/ec2.tf` is one instance, not an autoscaling group; the unit runs the container under a fixed `--name` with an `ExecStartPre` that removes a previous one, so a restart racing a slow shutdown cannot produce two. Systemd is not the only way to start a process on a host, so the @@ -325,7 +325,7 @@ Three, and the process holds all three itself: | The Route53 credential | Nowhere on disk | The instance role, over IMDSv2 | The DNS credential is the only one that is not a file, and deliberately: the -instance profile in `infra/iam.tf` grants +instance profile in `infra/pds/iam.tf` grants `ChangeResourceRecordSets`/`ListResourceRecordSets` on the one hosted zone and `GetChange` unscoped, and the AWS SDK reads short-lived credentials for that role from the metadata endpoint. Nothing is passed on the unit's command line @@ -349,7 +349,7 @@ credential](#the-dns-credential) above states both. The commands are in [operations](operations.md); this section states what a restore is and is not. -`infra/backup.tf` puts the data volume under an AWS Backup plan: daily at +`infra/pds/backup.tf` puts the data volume under an AWS Backup plan: daily at 09:00 UTC, snapshots kept thirty days. The volume is the whole of the state — the log and the blobs are both on it — so one snapshot is one consistent deployment, give or take writes in the seconds around it. diff --git a/docs/operations.md b/docs/operations.md index 741a98cf..b6c72bb8 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1,6 +1,6 @@ # Operations -Procedures for the deployment `infra/templates/user_data.sh.tftpl` builds: one +Procedures for the deployment `infra/pds/templates/user_data.sh.tftpl` builds: one instance, one systemd unit `didbot-pds`, one container of the release image, one data volume at `/data` with the data directory at `/data/pds`. @@ -54,7 +54,7 @@ taken just before the swap. ## Backups -`infra/backup.tf` puts the one EBS data volume under an AWS Backup plan: vault +`infra/pds/backup.tf` puts the one EBS data volume under an AWS Backup plan: vault `didbot-pds`, rule `daily`, schedule `cron(0 9 * * ? *)` — 09:00 UTC — and `delete_after = 30`, thirty daily recovery points. **The RPO is twenty-four hours.** `aws_backup_selection.pds` names the one resource the plan covers: diff --git a/infra/.terraform.lock.hcl b/infra/pds/.terraform.lock.hcl similarity index 100% rename from infra/.terraform.lock.hcl rename to infra/pds/.terraform.lock.hcl diff --git a/infra/pds/backend.tf b/infra/pds/backend.tf new file mode 100644 index 00000000..4ffac1d0 --- /dev/null +++ b/infra/pds/backend.tf @@ -0,0 +1,23 @@ +# State lives in the did.bot account's own state bucket, which +# permadeath-aws-org's `envs/accounts` created with the account: versioned, +# encrypted, public access blocked. `infra/site/` uses the same bucket under +# its own key -- one bucket, one prefix per root, so neither root can lock or +# overwrite the other's state. +# +# A backend block takes no variables, so the name is written out; it is +# deterministic -- -tfstate- -- and 226086920337 is the +# did.bot account. +# +# Locking is the S3 lock file, supported from 1.11 on, so this needs no second +# resource of its own. + +terraform { + backend "s3" { + bucket = "permadeath-tfstate-226086920337" + key = "pds/terraform.tfstate" + region = "us-east-1" + + encrypt = true + use_lockfile = true + } +} diff --git a/infra/backup.tf b/infra/pds/backup.tf similarity index 100% rename from infra/backup.tf rename to infra/pds/backup.tf diff --git a/infra/ec2.tf b/infra/pds/ec2.tf similarity index 99% rename from infra/ec2.tf rename to infra/pds/ec2.tf index 6c9c17c3..34f19482 100644 --- a/infra/ec2.tf +++ b/infra/pds/ec2.tf @@ -39,7 +39,7 @@ resource "aws_instance" "pds" { # Explicit rather than relying on the default. http_endpoint = "enabled" - # 1, matching this instance's run unit: infra/templates/ + # 1, matching this instance's run unit: infra/pds/templates/ # user_data.sh.tftpl's systemd unit runs the container with `--network # host`, so the server process shares the host's network namespace and # reaches 169.254.169.254 exactly as the host would -- no extra hop. A diff --git a/infra/iam.tf b/infra/pds/iam.tf similarity index 100% rename from infra/iam.tf rename to infra/pds/iam.tf diff --git a/infra/outputs.tf b/infra/pds/outputs.tf similarity index 81% rename from infra/outputs.tf rename to infra/pds/outputs.tf index 86ecd8f7..52f782b6 100644 --- a/infra/outputs.tf +++ b/infra/pds/outputs.tf @@ -6,11 +6,11 @@ output "instance_public_ip" { output "name_servers" { description = <<-EOT The four name servers Route53 assigned this stack's hosted zone for - `root_zone`. A human adds an NS record for `root_zone` at its parent - zone's registrar or DNS provider, naming exactly these four servers -- - that delegation is what makes this stack's zone authoritative for - anything. Terraform cannot make this record itself: the parent zone is - not this stack's to manage. + `root_zone`. An NS record for `root_zone` naming exactly these four is + what makes this zone authoritative for anything. When the parent is + did.bot, that record is `infra/site/`'s: pass these as its + `pds_zone_name_servers` and apply it. Under any other parent it is a + visit to that zone's own registrar or DNS provider. EOT value = aws_route53_zone.root.name_servers } diff --git a/infra/route53.tf b/infra/pds/route53.tf similarity index 81% rename from infra/route53.tf rename to infra/pds/route53.tf index 3fd98d2d..db96ef9a 100644 --- a/infra/route53.tf +++ b/infra/pds/route53.tf @@ -1,7 +1,8 @@ -# One hosted zone for root_zone. did.bot (or whatever the parent turns out -# to be) is the owner's apex and is not managed here -- see this stack's -# outputs for the NS records the owner adds there by hand to delegate into -# this zone. +# One hosted zone for root_zone. The parent -- did.bot, the apex the website +# is served from -- is `infra/site/`'s, a separate root with separate state. +# The delegation into this zone is that stack's `pds_zone_name_servers` +# variable, which takes this one's `name_servers` output; two roots and one +# value passed between them, so either can be applied without the other. # # This file creates the zone, the one record inside it the server cannot # write, and nothing else. Everything else in the zone is the PDS's job at @@ -28,7 +29,7 @@ # knows which account is about to be used and by nothing else. resource "aws_route53_zone" "root" { name = var.root_zone - comment = "didbot deploy epic: managed by infra/, delegated from the parent zone by hand" + comment = "didbot deploy epic: managed by infra/pds/, delegated from did.bot by infra/site/" } # The address the internet reaches this deployment at. `--port 443` and the diff --git a/infra/security_groups.tf b/infra/pds/security_groups.tf similarity index 100% rename from infra/security_groups.tf rename to infra/pds/security_groups.tf diff --git a/infra/templates/user_data.sh.tftpl b/infra/pds/templates/user_data.sh.tftpl similarity index 98% rename from infra/templates/user_data.sh.tftpl rename to infra/pds/templates/user_data.sh.tftpl index d2edd59f..8f823cb7 100644 --- a/infra/templates/user_data.sh.tftpl +++ b/infra/pds/templates/user_data.sh.tftpl @@ -92,7 +92,7 @@ RequiresMountsFor=/data # this stack's PR body exists to prevent. ExecStartPre=-/usr/bin/docker rm -f didbot-pds # Port 443, not 3000: there is no load balancer terminating TLS in front of -# this instance any more (see infra/ec2.tf and infra/security_groups.tf), so +# this instance any more (see infra/pds/ec2.tf and infra/pds/security_groups.tf), so # the server itself must be the thing bound to the port the internet reaches. # `--network host` (above) is what lets a container process bind 443 without # a published-port mapping, and is also what lets the container's IMDSv2 diff --git a/infra/variables.tf b/infra/pds/variables.tf similarity index 100% rename from infra/variables.tf rename to infra/pds/variables.tf diff --git a/infra/versions.tf b/infra/pds/versions.tf similarity index 68% rename from infra/versions.tf rename to infra/pds/versions.tf index 9f272e16..5db69c5e 100644 --- a/infra/versions.tf +++ b/infra/pds/versions.tf @@ -3,8 +3,11 @@ # init` picks up whatever provider version happens to be cached, and a plan # that looked safe on one machine can differ on another. +# The floor is 1.11 for `backend.tf`'s `use_lockfile`: state locking is the S3 +# lock file from that release on, which is why this root needs no DynamoDB +# table of its own. terraform { - required_version = ">= 1.9.0" + required_version = ">= 1.11" required_providers { aws = { diff --git a/plan/adversarial.md b/plan/adversarial.md index 7a000ff6..3d514e80 100644 --- a/plan/adversarial.md +++ b/plan/adversarial.md @@ -365,7 +365,7 @@ Left open, in rough order of how much a deployment should care: operator, opening `deleteAgent`, `setAgentPinned`, every dashboard panel and e-stop release. Reachable, not theoretical: `--operator-secret-file` reads whatever is at a path, and - `infra/templates/user_data.sh.tftpl` redirected an SSM fetch straight + `infra/pds/templates/user_data.sh.tftpl` redirected an SSM fetch straight at that path, so a failed fetch left a zero-byte file behind — a redirection creates its target before the command that fills it runs. Closed at three layers: the comparison refuses an empty configured diff --git a/plan/agent-accounts.md b/plan/agent-accounts.md index 9dc319c7..ce0ca98d 100644 --- a/plan/agent-accounts.md +++ b/plan/agent-accounts.md @@ -328,7 +328,7 @@ Two hostnames per account: the DID's, and the handle's, which needs `didbot-pds --tls acme` wires it end to end: `--route53-zone-id` names the hosted zone, `Route53Credentials::from_instance_metadata` reads the instance's IAM role over IMDSv2, and - `infra/templates/user_data.sh.tftpl` passes both to the deployed + `infra/pds/templates/user_data.sh.tftpl` passes both to the deployed container. **Unverified**: there is no AWS account or reachable ACME directory in diff --git a/plan/aws-deploy.md b/plan/aws-deploy.md index 9a1d6701..0fe550bc 100644 --- a/plan/aws-deploy.md +++ b/plan/aws-deploy.md @@ -13,7 +13,7 @@ exitCriterion: > # aws-deploy -`infra/` lives inside this repository today: a checkout, a `terraform apply`, +`infra/pds/` lives inside this repository today: a checkout, a `terraform apply`, and knowledge of which files matter. The owner asked for a one-shot Terraform module, published as its own repository, that stands up the whole thing for an operator who has not cloned this source tree and should not need to. @@ -23,7 +23,7 @@ This project already has the precedent for splitting a component out: their own repository (see the `build!` commit that removed `crates/vibescrobble-index` and the rest, and left [index](index.md) and [canvas](canvas.md) in `plan/` naming no crates, "for now", until that -repository is published). This epic is the same move applied to `infra/`: +repository is published). This epic is the same move applied to `infra/pds/`: extract it, version it, and leave a pointer here rather than the Terraform itself. Whatever conventions that split settled — how the new repository is named, how it is linked back from here — this epic should follow rather than @@ -32,7 +32,7 @@ invent a second way of doing the same thing. ## The design problems, which matter more than the extraction itself **The module and the binary must stay in step.** -`infra/templates/user_data.sh.tftpl` writes a systemd unit whose `ExecStart` +`infra/pds/templates/user_data.sh.tftpl` writes a systemd unit whose `ExecStart` passes specific flags — `--zone`, `--owner`, `--port`, `--data`, `--secret-file` today, confirmed by reading that template. A module version and a server (container image) version are therefore coupled: a flag renamed, dropped, or given new meaning between two @@ -62,7 +62,7 @@ direction. is built, and the CLI-flag surface if it has not — either way, name which one a given module version speaks. -**Secrets and Terraform state.** `infra/variables.tf`'s `ssm_prefix` comment +**Secrets and Terraform state.** `infra/pds/variables.tf`'s `ssm_prefix` comment already states the current discipline: secret *values* are written to SSM out of band, by a human or a separate process this configuration does not run, so `terraform.tfstate` never carries key material. A one-shot module that @@ -92,16 +92,18 @@ people who never meant it to hold credentials. but a state file that later grows a generated value because someone relaxed the rule above is a slow, easy mistake. -**What it must output.** Above everything else: the **NS records the -operator delegates at their parent zone**. `infra/outputs.tf`'s +**What it must output.** Above everything else: the **NS records that +delegate the new zone at its parent**. `infra/pds/outputs.tf`'s `name_servers` output already says why in its own comment — Route53 assigns -these when the hosted zone is created, Terraform cannot make the delegation -record itself because the parent zone is not this stack's to manage, and -nothing under the new zone resolves until a human adds that record by hand. -A one-shot module that applies cleanly and prints nothing actionable has -shipped half a deployment. - -- [ ] **Carry every output `infra/outputs.tf` has today** — the Elastic IP, +these when the hosted zone is created, and nothing under the new zone +resolves until an NS record naming them exists at the parent. Under did.bot +that record is `infra/site/`'s `pds_zone_name_servers`, so the delegation is +an apply; under any other parent it is a visit to that zone's own provider. +Either way the module has to hand the operator the four names: a one-shot +module that applies cleanly and prints nothing actionable has shipped half a +deployment. + +- [ ] **Carry every output `infra/pds/outputs.tf` has today** — the Elastic IP, the hosted zone id, the instance id, the data volume id — since each exists for an operational reason (pointing external monitoring at the IP before DNS resolves, `aws ssm start-session`, the restore drill) @@ -111,7 +113,7 @@ shipped half a deployment. be loud, and the module's own top-level documentation should lead with it rather than bury it under variable reference tables. -**What it must not assume.** `infra/variables.tf` today requires a `vpc_id` +**What it must not assume.** `infra/pds/variables.tf` today requires a `vpc_id` and an `instance_subnet_id` with no default, on purpose — this project has never assumed an existing VPC. A one-shot module aimed at an operator with "an AWS account and nothing else" raises the question of whether it should @@ -123,11 +125,11 @@ exists at all. Offering a minimal-VPC path as one option is reasonable; making it the only path is not. - [ ] **A region is a variable with no default**, matching `root_zone`'s own - treatment in `infra/variables.tf` today (a value this stack should not + treatment in `infra/pds/variables.tf` today (a value this stack should not silently choose for someone). - [ ] **State explicitly what a fresh apply does *not* set up**, alongside what it does: `plan/deploy.md`'s open items on restore drills, log-format - upgrades, and rollback do not become solved by turning `infra/` into a + upgrades, and rollback do not become solved by turning `infra/pds/` into a module. A module that stands up the instance is not a module that has answered any of those; say so in its own documentation so packaging is never mistaken for having closed them. @@ -143,13 +145,13 @@ rather than a script that assumes it is the only deployment on the account. This epic does not need fleet to exist first, but the module's variables (anything that would collide across two instantiations in the same account — IAM role names, KMS alias names, the backup vault name, all of which -`infra/`'s current `.tf` files hardcode as `didbot-pds*` today) need to be +`infra/pds/`'s current `.tf` files hardcode as `didbot-pds*` today) need to be parameterised with that in mind, or a second deployment in the same account will fail to apply rather than coexist. - [ ] **Namespace every account-global resource name** — the IAM role, instance profile, KMS alias, backup vault and plan currently hardcoded - as `didbot-pds*` in `infra/iam.tf` and `infra/backup.tf` — by a variable + as `didbot-pds*` in `infra/pds/iam.tf` and `infra/pds/backup.tf` — by a variable (a deployment name or the zone itself), so two instantiations of this module in one AWS account do not collide on a role name neither chose. diff --git a/plan/blob-storage-tiers.md b/plan/blob-storage-tiers.md index 91d67519..ceade0b8 100644 --- a/plan/blob-storage-tiers.md +++ b/plan/blob-storage-tiers.md @@ -66,7 +66,7 @@ an account holder asking for their data to be gone does not want. durable copy exists, at the cost of a network round trip in every upload. Proposal: disk-authoritative, with the bucket as replication, because the disk copy is already covered by - `periodic-backups` and by `infra/backup.tf` for the window before + `periodic-backups` and by `infra/pds/backup.tf` for the window before replication catches up. Either way the acknowledged meaning is written down, because it is what a client is entitled to assume. - [ ] **An upload interrupted by a dying process.** The local half has this: diff --git a/plan/cost.md b/plan/cost.md index db11b527..86e9e6c2 100644 --- a/plan/cost.md +++ b/plan/cost.md @@ -6,7 +6,7 @@ crates: [] dependsOn: [deploy] exitCriterion: > An operator can name, for one deployment, the estimated monthly cost of - every AWS service `infra/` provisions, the marginal cost of one more + every AWS service `infra/pds/` provisions, the marginal cost of one more account, and the marginal cost of one account that goes viral — each figure labelled measured or estimated, and cited to its source. --- @@ -14,23 +14,23 @@ exitCriterion: > # cost The owner asked for a cost model, broken out by service, benchmarked against -AWS. Nothing has run on real AWS yet — `infra/` has never been applied — so +AWS. Nothing has run on real AWS yet — `infra/pds/` has never been applied — so **every figure in this epic starts as an estimate**, sourced from AWS's public pricing pages rather than a bill, and stays labelled that way until a real deployment's Cost Explorer says otherwise. An estimate presented as measured is worse than a gap, because it gets planned against. -## What `infra/` provisions +## What `infra/pds/` provisions Confirmed against the tree as of this writing, not guessed: one EC2 instance -(`infra/ec2.tf`, `t3.small` by default) with two `gp3` EBS volumes — a 20 GB +(`infra/pds/ec2.tf`, `t3.small` by default) with two `gp3` EBS volumes — a 20 GB root volume and a 20 GB data volume holding the write-ahead log and blob -store — an Elastic IP, a Route53 hosted zone (`infra/route53.tf`) and nothing +store — an Elastic IP, a Route53 hosted zone (`infra/pds/route53.tf`) and nothing inside it (the server populates records itself at runtime), an IAM role and -instance profile (`infra/iam.tf`) with SSM read, ECR pull and Route53 write +instance profile (`infra/pds/iam.tf`) with SSM read, ECR pull and Route53 write grants, a dedicated KMS key for the SSM SecureString parameters, a security group open only on 443, an AWS Backup vault and plan -(`infra/backup.tf`) covering the data volume, and an ECR repository for the +(`infra/pds/backup.tf`) covering the data volume, and an ECR repository for the container image. There is no ALB and no ACM certificate: `plan/deploy.md` records that the server terminates TLS itself over ACME DNS-01, so certificate issuance is free and the load balancer's cost — and its data @@ -47,7 +47,7 @@ has been confirmed against an actual invoice. |---|---|---|---| | EC2 instance hours | `t3.small` on-demand, ~$0.0208/hr ([AWS EC2 pricing](https://aws.amazon.com/ec2/pricing/on-demand/); rate itself not readable from that page's rendered text, cross-checked against third-party trackers that quote it) | 730 hrs/month | ~$15.18 | | EBS, `gp3` | $0.08/GB-month ([AWS EBS pricing](https://aws.amazon.com/ebs/pricing/); exact figure from third-party aggregation, AWS's own page states the formula but not this region's number in fetchable text) | 20 GB root + 20 GB data = 40 GB | ~$3.20 | -| Public IPv4 address (the Elastic IP) | $0.005/hr per address, **charged since February 2024 even while attached to a running instance** — this is not the pre-2024 "only when idle" rule `infra/ec2.tf`'s own comments do not mention | 1 address, attached | ~$3.65 | +| Public IPv4 address (the Elastic IP) | $0.005/hr per address, **charged since February 2024 even while attached to a running instance** — this is not the pre-2024 "only when idle" rule `infra/pds/ec2.tf`'s own comments do not mention | 1 address, attached | ~$3.65 | | Route53 hosted zone | $0.50/month for each of the first 25 zones ([AWS Route53 pricing](https://aws.amazon.com/route53/pricing/)) | 1 zone | $0.50 | | Route53 queries | $0.40 per million standard queries (first 1B/month) | negligible at rest | ~$0.00 | | KMS key (`aws_kms_key.ssm`) | $1/month per key, plus $0.03 per 10,000 requests | 1 key, few requests | ~$1.00 | @@ -70,7 +70,7 @@ absorbing that load and this method has none, so read volume scales with how interesting the accounts are rather than with how many there are."* That is a capacity statement and a cost statement at once, and it is the most important thing this epic has to say: **an operator cannot cache this away.** There is -no CDN or directory in front of the instance (see "what `infra/` provisions" +no CDN or directory in front of the instance (see "what `infra/pds/` provisions" above — no ALB, nothing else in the request path), so every `did.json` fetch is billed data transfer out, and every DNS lookup for an agent's hostname that a client's resolver did not already cache is a billed Route53 query. @@ -104,7 +104,7 @@ a real deployment, which does not exist yet. ## What is measured versus estimated -Nothing here is measured. `infra/` has never been applied against an AWS +Nothing here is measured. `infra/pds/` has never been applied against an AWS account (see `plan/deploy.md`'s open backup-restore item, which says the same thing about restore: wired up, never run). Every rate above is AWS's public, current list price; none of it accounts for savings plans, reserved @@ -144,7 +144,7 @@ to whichever of those two epics picks it up. "times N" rather than re-deriving it, and note where it is *not* linear (a shared operator credential, if fleet ever needs one, is not a tenth of anything). -- [ ] **Revisit the EIP line** if `infra/ec2.tf` ever changes: the $0.005/hr +- [ ] **Revisit the EIP line** if `infra/pds/ec2.tf` ever changes: the $0.005/hr public-IPv4 charge applies to the address regardless of whether it is attached, which was not true before February 2024 and is easy to re-assume away. diff --git a/plan/dedupe-audit.md b/plan/dedupe-audit.md index a5f7f0c1..cc24582d 100644 --- a/plan/dedupe-audit.md +++ b/plan/dedupe-audit.md @@ -52,7 +52,7 @@ operator or a maintainer, fixed in this branch: - **`crates/didbot-serve/src/onboarding.rs`** told an operator in `ServerState::Provisioning` to run `aws ssm put-parameter` for an `attestation-secret` and an `operator-secret`. Neither parameter is read - by anything (`infra/templates/user_data.sh.tftpl` no longer touches + by anything (`infra/pds/templates/user_data.sh.tftpl` no longer touches either name), and `ServerState::Provisioning` — per `didbot_pds::server_state` — is the state in which the server mints its own bootstrap keypair unattended; no operator action unblocks it at all. diff --git a/plan/deploy.md b/plan/deploy.md index 244024bc..608c75ac 100644 --- a/plan/deploy.md +++ b/plan/deploy.md @@ -42,7 +42,7 @@ own bucket and distribution to provision — see that epic's open items. ## Staying up -- [ ] **Backups and restore.** `infra/backup.tf` puts the data volume — the +- [ ] **Backups and restore.** `infra/pds/backup.tf` puts the data volume — the write-ahead log and the blob store both live on it — under an AWS Backup plan, daily, kept thirty days. [Deployment](../docs/deployment.md) states what a restore recovers @@ -129,21 +129,28 @@ The project owner corrected the original design here: ALB-terminated TLS put plaintext on the wire between the load balancer and the instance, and a Terraform-managed wildcard DNS record meant the server never wrote DNS at all — removing a capability [agent-accounts](agent-accounts.md) required -from the start. `infra/` has been reworked to match (the ALB and its ACM -certificates are gone; `infra/route53.tf` now creates only the hosted zone; -`infra/iam.tf` grants the instance write on it; the instance takes an +from the start. `infra/pds/` has been reworked to match (the ALB and its ACM +certificates are gone; `infra/pds/route53.tf` now creates only the hosted zone; +`infra/pds/iam.tf` grants the instance write on it; the instance takes an Elastic IP and is reached on 443 directly). The three items below are what is left, and all three depend on work sibling agents are doing in parallel that had not landed as this revision was written. ## Done +- [x] **Two roots, one state bucket.** `infra/pds/` is this deployment and + `infra/site/` is the did.bot website; each has its own key in the + did.bot account's state bucket, so applying one cannot lock or roll + back the other. The wire between them is one value: `infra/pds/`'s + `name_servers` output goes into `infra/site/`'s + `pds_zone_name_servers`, which writes the `NS` record at `pds.did.bot` + that makes this zone answer. - [x] **The unit is a container image under one systemd unit.** Not binaries and unit files, not an orchestrator; `scripts/` is a developer's interface and no part of a deployment. [Deployment](../docs/deployment.md)'s "what runs" states it and the three process properties that decide it. The deciding one is that the - store is a single-writer log: `infra/ec2.tf` is one instance rather + store is a single-writer log: `infra/pds/ec2.tf` is one instance rather than an autoscaling group, and the unit runs the container under a fixed `--name` behind an `ExecStartPre` that removes a previous one, so a restart racing a slow shutdown cannot produce two. That is the unit @@ -211,7 +218,7 @@ that had not landed as this revision was written. zones it may write, the posture `dns.may_create_zones` describes. - [x] **Certificates: issuance and renewal, in the instance.** The server terminates TLS itself over a certificate it obtains and renews - in-process, and `infra/templates/user_data.sh.tftpl`'s `ExecStart` + in-process, and `infra/pds/templates/user_data.sh.tftpl`'s `ExecStart` passes `--tls acme`, `--acme-environment` and the hosted zone id the DNS-01 challenge is published into; no credential is on that command line, since `Route53Dns` reads the instance role over IMDSv2. What a diff --git a/plan/didjson-archive.md b/plan/didjson-archive.md index 2592a1df..b7082e5f 100644 --- a/plan/didjson-archive.md +++ b/plan/didjson-archive.md @@ -72,7 +72,7 @@ another is still live. scheduling is `plan/periodic-backups.md`'s: this epic contributes the job, that one owns when jobs run and what happens when one overruns the next. Proposal for the interval is daily, matching - [backup.tf](../infra/backup.tf)'s existing rule so an operator has one + [backup.tf](../infra/pds/backup.tf)'s existing rule so an operator has one cadence to reason about rather than two. - [ ] **Enumeration on the backend, which two epics want.** [`ObjectBackend`](../crates/didbot-pds/src/object_blobs.rs) is diff --git a/plan/onboarding.md b/plan/onboarding.md index c40d0778..0ad079a7 100644 --- a/plan/onboarding.md +++ b/plan/onboarding.md @@ -196,7 +196,7 @@ default above, restated as what an operator actually sees when it happens. - [ ] **A certificate that has not issued yet.** `--tls acme` is wired now — `didbot-pds` refuses at startup without `--data`, a real zone and `--route53-zone-id` (`crates/didbot-serve/src/bin/didbot-pds.rs`), and - `infra/templates/user_data.sh.tftpl`'s `ExecStart` passes it. What is + `infra/pds/templates/user_data.sh.tftpl`'s `ExecStart` passes it. What is still unverified, per [deploy](deploy.md)'s own Done entry, is the protocol exchange itself against a real ACME directory — nobody has run it from an environment with the network access to try. An operator's diff --git a/plan/periodic-backups.md b/plan/periodic-backups.md index cfe15d0e..9e319afb 100644 --- a/plan/periodic-backups.md +++ b/plan/periodic-backups.md @@ -15,8 +15,8 @@ exitCriterion: > There are two backups here with two different guarantees, and the whole point of this epic is that they are not substitutes. -[`infra/backup.tf`](../infra/backup.tf) is the first. It puts the one EBS -volume from [`infra/ec2.tf`](../infra/ec2.tf) — the write-ahead log, the blob +[`infra/pds/backup.tf`](../infra/pds/backup.tf) is the first. It puts the one EBS +volume from [`infra/pds/ec2.tf`](../infra/pds/ec2.tf) — the write-ahead log, the blob store, `tls/` and the `pds.layout` stamp, all of it — under an AWS Backup plan, daily at 09:00 UTC, kept thirty days. Its unit is the volume, so one recovery point is one whole deployment at one moment, and it recovers the signing keys @@ -92,7 +92,7 @@ twenty-four hours, and a copy that survives the account the vault is in. same failure handling; whichever lands first owns the plumbing and the other two consume it. Proposal: one prefix per deployment, server-side encryption with a key the deployment names, and the instance profile in - [`infra/iam.tf`](../infra/iam.tf) gaining write on that prefix and + [`infra/pds/iam.tf`](../infra/pds/iam.tf) gaining write on that prefix and nothing else. - [ ] **Cadence, as two mechanisms rather than one.** The log is append-only between compactions, so the cheap pass is shipping the bytes @@ -105,11 +105,11 @@ twenty-four hours, and a copy that survives the account the vault is in. from — see `COMPACT_RATIO` — so a full after a compaction is the ordering constraint on any tail scheme. - [ ] **Retention is one number an operator can hold in their head.** - Proposal: mirror `delete_after = 30` from `infra/backup.tf`, so both + Proposal: mirror `delete_after = 30` from `infra/pds/backup.tf`, so both backups age out together and an operator reasoning about "how far back can I go" has one answer. A longer-kept monthly is the obvious second tier and is worth having only once somebody has wanted a snapshot older - than thirty days, which is the argument `infra/backup.tf` already makes + than thirty days, which is the argument `infra/pds/backup.tf` already makes about widening its own window. Retention is also a deletion deadline: an account holder's deletion has to reach these artefacts, and a stated short window is the mechanism — see `blob-storage-tiers`, which has the diff --git a/plan/tombstone-serving.md b/plan/tombstone-serving.md index 153efb75..047173b7 100644 --- a/plan/tombstone-serving.md +++ b/plan/tombstone-serving.md @@ -145,7 +145,7 @@ rewriting a path rather than by running a query planner. references is a different operation, with different failure modes, from one that publishes a manifest of keys. - [ ] **A drill, because a snapshot is not a restore.** - [backup.tf](../infra/backup.tf) already says this about EBS snapshots + [backup.tf](../infra/pds/backup.tf) already says this about EBS snapshots and the same sentence applies harder here: the check is a resolver and a relay reading a frozen account with the PDS stopped, which is this epic's exit criterion and wants a test in -- 2.51.2