From b9df0a0e89cc0fd35b8dbfd0921b7d09afffd2b2 Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Thu, 3 Sep 2026 18:37:26 -0400 Subject: [PATCH] feat(infra): name a relay on the unit and reach the socket that announces to it The systemd unit now carries `--relay-hostname` from a new `relay_hostname` Terraform variable, and `--estop-socket` on the mounted data volume so an operator on the instance can open it. Boot dials nothing; the unit test renders both settings of the variable and checks every flag against the binary's own parser. Change-Id: Ie7a6ae6959acc35a9537f4751474678b2e22b99f --- crates/didbot-serve/src/bin/didbot-pds.rs | 168 ++++++++++++++++------ docs/deployment.md | 39 +++++ infra/pds/ec2.tf | 1 + infra/pds/templates/user_data.sh.tftpl | 23 ++- infra/pds/variables.tf | 18 +++ plan/federation.md | 19 +++ 6 files changed, 226 insertions(+), 42 deletions(-) diff --git a/crates/didbot-serve/src/bin/didbot-pds.rs b/crates/didbot-serve/src/bin/didbot-pds.rs index 4b67d312..7c3aae31 100644 --- a/crates/didbot-serve/src/bin/didbot-pds.rs +++ b/crates/didbot-serve/src/bin/didbot-pds.rs @@ -2062,8 +2062,41 @@ mod config_tests { ); } + /// The one Terraform `%{ if }..%{ endif }` block in the + /// deployment's unit template, rendered for `taken` or dropped for + /// `!taken`. + /// + /// A hand-rolled two-case renderer rather than a template engine: the + /// unit has exactly one conditional and this asserts that, so the day it + /// grows a second one this stops silently rendering only the first. + #[cfg(test)] + fn render_unit_conditional(unit: &str, taken: bool) -> String { + const OPEN: &str = "%{ if "; + const CLOSE: &str = " }"; + const END: &str = "%{ endif }"; + let Some((before, rest)) = unit.split_once(OPEN) else { + panic!("the unit template lost its conditional relay flag"); + }; + let (_condition, rest) = rest + .split_once(CLOSE) + .expect("an `%{ if` directive closes with ` }`"); + let (inner, after) = rest + .split_once(END) + .expect("an `%{ if` block has an `%{ endif }`"); + assert!( + !after.contains(OPEN), + "the unit template grew a second conditional; this renderer only handles one" + ); + let mut out = String::from(before); + if taken { + out.push_str(inner); + } + out.push_str(after); + out + } + /// Every flag the deployment's systemd unit passes is a flag this binary - /// accepts. + /// accepts, whether or not the deployment names a relay. /// /// The two drift silently and in the direction that hurts: the unit is /// substituted, written and enabled by cloud-init on a first boot, so a @@ -2071,60 +2104,113 @@ mod config_tests { /// on an instance nobody is watching yet. Nothing else compares them -- /// `tofu validate` type-checks the template's variables and has no idea /// what a didbot flag is. + /// + /// Both renderings are driven because `relay_hostname = ""` is a + /// supported deployment: the `%{ if }` block that drops `--relay-hostname` + /// also carries the line continuation before it, so getting that wrong + /// produces an `ExecStart` that is well-formed in one case and a dangling + /// backslash in the other. #[test] fn the_deployments_systemd_unit_passes_flags_this_binary_accepts() { // Terraform's `templatefile` substitutes `${...}` in the template; // these are the shell-side placeholders `sed` then replaces, and a // new one has to be given a value here or this test fails rather // than skipping it. - const PLACEHOLDERS: [(&str, &str); 5] = [ + const PLACEHOLDERS: [(&str, &str); 6] = [ ("ZONE_PLACEHOLDER", "agents.quernstone.example"), ("OWNER_PLACEHOLDER", "did:web:owner.example"), ("ACME_ENVIRONMENT_PLACEHOLDER", "staging"), ("ROUTE53_ZONE_ID_PLACEHOLDER", "Z0QUERNSTONE"), ("RECORD_TARGET_PLACEHOLDER", "203.0.113.47"), + ("RELAY_HOSTNAME_PLACEHOLDER", "relay.quernstone.example"), ]; - let unit = include_str!("../../../../infra/pds/templates/user_data.sh.tftpl"); - let exec_start = unit - .split_once("\nExecStart=") - .expect("the unit has an ExecStart") - .1 - .split_once("\n\n") - .expect("ExecStart's continued lines end at a blank line") - .0; - // Everything before the image is `docker run`'s, not this binary's. - let flags = exec_start - .split_once("IMAGE_PLACEHOLDER") - .expect("the unit runs the image by placeholder") - .1; - - let args: Vec = flags - .split_whitespace() - .filter(|token| *token != "\\") - .map(|token| { - let mut token = token.to_owned(); - for (placeholder, value) in PLACEHOLDERS { - token = token.replace(placeholder, value); - } - assert!( - !token.contains("PLACEHOLDER"), - "the unit gained the placeholder `{token}`; give it a value in this test" - ); - token - }) - .collect(); - assert!( - args.iter().any(|arg| arg == "--record-target"), - "the unit stopped passing --record-target, so this deployment would publish \ - loopback into its hosted zone: {args:?}" - ); - - if let Err(err) = parse_args(args.clone().into_iter()) { - panic!( - "the deployment's unit runs `didbot-pds {}`, which this binary refuses: {err}", - args.join(" ") + let template = include_str!("../../../../infra/pds/templates/user_data.sh.tftpl"); + for names_a_relay in [true, false] { + let unit = render_unit_conditional(template, names_a_relay); + let exec_start = unit + .split_once("\nExecStart=") + .expect("the unit has an ExecStart") + .1 + .split_once("\n\n") + .expect("ExecStart's continued lines end at a blank line") + .0; + assert!( + !exec_start.trim_end().ends_with('\\'), + "ExecStart ends on a line continuation with nothing after it \ + (relay named: {names_a_relay}): {exec_start}" + ); + // Everything before the image is `docker run`'s, not this binary's. + let flags = exec_start + .split_once("IMAGE_PLACEHOLDER") + .expect("the unit runs the image by placeholder") + .1; + + let args: Vec = flags + .split_whitespace() + .filter(|token| *token != "\\") + .map(|token| { + let mut token = token.to_owned(); + for (placeholder, value) in PLACEHOLDERS { + token = token.replace(placeholder, value); + } + assert!( + !token.contains("PLACEHOLDER"), + "the unit gained the placeholder `{token}`; give it a value in this test" + ); + token + }) + .collect(); + assert!( + args.iter().any(|arg| arg == "--record-target"), + "the unit stopped passing --record-target, so this deployment would publish \ + loopback into its hosted zone: {args:?}" + ); + // The admin socket is what `ANNOUNCE` is reached over, and the + // container's default path for it is inside the container's own + // filesystem, where an operator on the instance cannot open it. + // On the mounted data volume it is `/data/pds/estop.sock` on the + // host. See docs/deployment.md's "Announcing to a relay". + let socket = args + .iter() + .position(|arg| arg == "--estop-socket") + .and_then(|at| args.get(at + 1)) + .unwrap_or_else(|| { + panic!( + "the unit stopped passing --estop-socket, so ANNOUNCE is reachable \ + only from inside the container: {args:?}" + ) + }); + assert!( + socket.starts_with("/data/"), + "--estop-socket is {socket}, which is not on the volume the host mounts, \ + so an operator on the instance cannot open it" ); + + let relay = args + .iter() + .position(|arg| arg == "--relay-hostname") + .and_then(|at| args.get(at + 1)); + match (names_a_relay, relay) { + (true, Some(host)) => assert_eq!(host, "relay.quernstone.example"), + (true, None) => panic!( + "a deployment that names a relay renders no --relay-hostname, so its \ + ANNOUNCE command would have no relay to reach: {args:?}" + ), + (false, Some(host)) => panic!( + "a deployment naming no relay still renders --relay-hostname {host}: \ + {args:?}" + ), + (false, None) => {} + } + + if let Err(err) = parse_args(args.clone().into_iter()) { + panic!( + "the deployment's unit runs `didbot-pds {}`, which this binary refuses: \ + {err}", + args.join(" ") + ); + } } } } diff --git a/docs/deployment.md b/docs/deployment.md index 83354abd..f6c4fe89 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -301,6 +301,45 @@ one publishes nothing and says so at startup. Creating a hosted zone, or delegating a new one with `NS` records in the zone above, is a separate posture — `dns.may_create_zones` in `didbot-config` — and is off by default. +## Announcing to a relay + +A relay learns a personal data server exists by being told. This deployment +names one in the `relay_hostname` Terraform variable — `bsky.network` by +default, `""` to run naming none — and `infra/pds/templates/user_data.sh.tftpl` +puts it on the unit's `ExecStart` as `--relay-hostname`. That flag decides +which relay `com.atproto.sync.requestCrawl` and the deprecated +`notifyOfUpdate` are sent to; sending either is an operator action. + +**Boot does not dial the relay.** The server builds a client for the named +host and hands it to the e-stop admin socket, and that is all it does with it +until somebody asks. A relay that is down while this instance comes up +therefore costs the instance nothing — there is no retry loop to wedge in and +no startup step to fail. A relay that goes away later, or one that answers +`HostBanned`, is felt by the operator who runs the command: the reply +distinguishes `accepted` from `refused` (with the relay's status and body) +from `unreachable` (with the transport error), and a call that gets no answer +gives up after ten seconds rather than holding the socket open. + +**The command is reached over the data volume.** The unit passes +`--estop-socket /data/estop.sock`, which is `/data/pds/estop.sock` on the +instance, so an operator on a break-glass SSH or SSM session reaches it +without entering the container. Opening it is the whole authorization: the +server creates the directory it sits in, re-tightens that directory to `0700` +on every start and the socket to `0600`, so the callers are the server's own +uid and root. `ANNOUNCE` sends `requestCrawl`; `NOTIFY` sends +`notifyOfUpdate`, which upstream deprecates in its favor but still names for +resuming a relay's subscription after a gap. + +```console +$ printf 'ANNOUNCE\n' | sudo socat - UNIX-CONNECT:/data/pds/estop.sock +OK {"result":"accepted"} +``` + +Under a `.localhost` zone the flag is read and the command is left unwired, +and the server says so at startup: no relay resolves a loopback name, so the +only thing wiring it up there could produce is a failure an operator asked +for. + ## The data directory One directory, `--data`, on a volume separate from the root volume so an AMI diff --git a/infra/pds/ec2.tf b/infra/pds/ec2.tf index 34f19482..c20bc38f 100644 --- a/infra/pds/ec2.tf +++ b/infra/pds/ec2.tf @@ -63,6 +63,7 @@ resource "aws_instance" "pds" { hosted_zone_id = aws_route53_zone.root.zone_id acme_environment = var.acme_environment record_target = aws_eip.pds.public_ip + relay_hostname = var.relay_hostname }) # user_data changes (a new image tag, a config change) should not silently diff --git a/infra/pds/templates/user_data.sh.tftpl b/infra/pds/templates/user_data.sh.tftpl index 80198e42..a6041b56 100644 --- a/infra/pds/templates/user_data.sh.tftpl +++ b/infra/pds/templates/user_data.sh.tftpl @@ -134,6 +134,24 @@ ExecStartPre=-/usr/bin/docker rm -f didbot-pds # enough to leave a `dnf update` and a second image tag room to land and large # enough to hold on the order of a million request lines at the ~250 bytes # json-file's envelope makes of one. +# +# `--estop-socket` puts the e-stop admin socket on the mounted data volume, +# where it is `/data/pds/estop.sock` on this host and reachable from a break- +# glass SSH or SSM session without entering the container. That socket is the +# only surface `ANNOUNCE` and `NOTIFY` are reached over, so the relay flag +# below is only usable to an operator because of this one. The server creates +# the directory it sits in and re-tightens it to 0700 on every start, and the +# socket itself to 0600 -- reaching it is the whole authorization, see +# crates/didbot-serve/src/estop_admin.rs. +# +# `--relay-hostname` (present only when the `relay_hostname` variable is +# non-empty) names the relay those two commands call +# `com.atproto.sync.requestCrawl` against. It is inert at boot: the server +# builds a client for it and hands that to the admin socket without dialing +# anything, so a relay that is down while this instance comes up costs +# nothing and a relay that goes away later is felt only by an operator who +# runs ANNOUNCE, whose reply distinguishes a refusal from an unreachable +# relay. See docs/deployment.md's "Announcing to a relay". ExecStart=/usr/bin/docker run --rm --name didbot-pds \ --network host \ --log-opt max-size=100m \ @@ -147,7 +165,9 @@ ExecStart=/usr/bin/docker run --rm --name didbot-pds \ --tls acme \ --acme-environment ACME_ENVIRONMENT_PLACEHOLDER \ --route53-zone-id ROUTE53_ZONE_ID_PLACEHOLDER \ - --record-target RECORD_TARGET_PLACEHOLDER + --record-target RECORD_TARGET_PLACEHOLDER \ + --estop-socket /data/estop.sock%{ if relay_hostname != "" } \ + --relay-hostname RELAY_HOSTNAME_PLACEHOLDER%{ endif } ExecStop=/usr/bin/docker stop -t 20 didbot-pds Restart=on-failure @@ -168,6 +188,7 @@ sed -i \ -e "s|ROUTE53_ZONE_ID_PLACEHOLDER|${hosted_zone_id}|" \ -e "s|RECORD_TARGET_PLACEHOLDER|${record_target}|" \ -e "s|ACME_ENVIRONMENT_PLACEHOLDER|${acme_environment}|" \ + -e "s|RELAY_HOSTNAME_PLACEHOLDER|${relay_hostname}|" \ /etc/systemd/system/didbot-pds.service systemctl daemon-reload diff --git a/infra/pds/variables.tf b/infra/pds/variables.tf index 8c884d83..706823b4 100644 --- a/infra/pds/variables.tf +++ b/infra/pds/variables.tf @@ -100,6 +100,24 @@ variable "owner_did" { type = string } +# Which relay this deployment announces itself to. The server has no default +# relay of its own and never dials one on its own -- naming one here only +# gives the e-stop admin socket's ANNOUNCE and NOTIFY commands somewhere to +# send `com.atproto.sync.requestCrawl`, which an operator runs deliberately. +# See docs/deployment.md's "Announcing to a relay". +variable "relay_hostname" { + description = <<-EOT + Hostname of the relay the ANNOUNCE and NOTIFY admin commands reach, + without a scheme -- the server always dials it over https. Defaults to + the public atproto relay, which is the one an alpha wants an account to + be visible through. Set to "" to run with the announce commands holding + no relay, in which case they answer that there is none to reach rather + than calling anything. + EOT + type = string + default = "bsky.network" +} + variable "acme_environment" { description = <<-EOT Which ACME directory the server's own --tls acme requests a certificate diff --git a/plan/federation.md b/plan/federation.md index 85af634d..1dd485b7 100644 --- a/plan/federation.md +++ b/plan/federation.md @@ -72,8 +72,27 @@ sync half — `subscribeRepos`, and whatever a relay does with the stream once it can open one — is the part with no working implementation to point at yet, and is where this epic's actual risk sits. +- [ ] **Point `relay_hostname` at a relay that will accept this deployment, + and run `ANNOUNCE` against it.** The wiring below is checked against a + loopback stand-in. What an alpha still needs is a real relay's reply, + and a record of which relay gave it. + ## Done +- [x] **The deployment names a relay, and the socket that announces to it is + reachable from the instance.** `infra/pds/`'s `relay_hostname` variable + puts `--relay-hostname` on the systemd unit's `ExecStart`, and + `--estop-socket` puts the e-stop admin socket on the mounted data + volume — `/data/pds/estop.sock` on the host — so an operator's SSH or + SSM session opens it without entering the container. Boot dials + nothing: the server builds a client for the named relay and hands it to + the socket, so a relay that is down while an instance comes up costs + the instance nothing, and one that refuses or has gone away is reported + to the operator who ran `ANNOUNCE` as `refused` or `unreachable` + rather than to a log nobody reads. A test renders the unit for both + settings of the variable and checks every flag against this binary's + own parser. See `docs/deployment.md`'s "Announcing to a relay". + - [x] **Announce a deletion on the stream.** `deleteRecord` puts a `#commit` on `subscribeRepos` carrying a `delete` op — null `cid`, the removed record's CID as `prev` — over a covering proof taken the same way a -- 2.51.2