From 6be8f0d448b8a09f1ddef75ffeeaf423bb0bf303 Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Thu, 3 Sep 2026 22:55:01 -0400 Subject: [PATCH] test(didbot-pds): pin what a restore finds in its hosted zone A restore puts an old account store beside a current zone. One test drives a restore older than its zone and holds the pair: the sweep withdraws the name it brought back, and the name it did not is reported unattributed and left standing. The other reads the startup zone hydration, which is what lets a restored account be retired at all. Change-Id: I2e517517770fa228273edece0e70bc13b8313336 --- crates/didbot-pds/tests/restore_zone.rs | 300 ++++++++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 crates/didbot-pds/tests/restore_zone.rs diff --git a/crates/didbot-pds/tests/restore_zone.rs b/crates/didbot-pds/tests/restore_zone.rs new file mode 100644 index 00000000..fff8ecaf --- /dev/null +++ b/crates/didbot-pds/tests/restore_zone.rs @@ -0,0 +1,300 @@ +//! What a restore does to the hosted zone, which is not on the volume. +//! +//! `crates/didbot-pds/tests/restore.rs` drills the volume: everything a +//! snapshot carries. The zone is the other half of a deployment's state and +//! it is somewhere else entirely — Route53 outlives the instance, so a +//! restore puts an old account store next to a current zone and the two +//! disagree about every account provisioned after the snapshot. +//! +//! `route53_seam.rs` asks the same question of a *crash* between the zone +//! write and the store insert. A restore differs in what it can do about it: +//! the accounts come back off the log rather than out of memory, and the two +//! machines that could act — the stale sweep and the reconciler's survey — +//! reach exactly one of the two kinds of name this leaves behind. Which one +//! is what this file pins. +//! +//! The provider is the real `Route53Dns` over `didbot_dns::aws_fake`, so the +//! zone is read back over the listing wire format rather than out of the +//! provider's own bookkeeping. + +use std::collections::BTreeMap; +use std::net::Ipv4Addr; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use didbot_dns::aws_fake::{route53_provider, FakeAws}; +use didbot_dns::route53::Route53Dns; +use didbot_dns::RecordTarget; +use didbot_identity::Zone; +use didbot_pds::{ + AccountStore, Durable, FileAccountStore, ProvisionRequest, Provisioner, Registry, +}; +use didbot_reconcile::{Drift, ProviderView, Survey, ZoneView}; +use time::Duration; + +const ZONE_HOST: &str = "agents.example.com"; +const PDS_ENDPOINT: &str = "https://pds.example.com"; +const HOSTED_ZONE: &str = "Z0FAKEZONE"; + +type Pds = Provisioner>; + +fn scratch(name: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("didbot-restore-zone-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + dir +} + +fn zone() -> Zone { + Zone::delegated("example.com", ZONE_HOST).expect("test zone should be constructible") +} + +fn target() -> RecordTarget { + RecordTarget::Ipv4(Ipv4Addr::new(203, 0, 113, 7)) +} + +fn host_of(agent_id: &str) -> String { + format!("{agent_id}.{ZONE_HOST}") +} + +/// A deployment over `dir`, publishing into `aws`. +/// +/// The store is the file-backed one the log replays, because that is what a +/// restore brings back — a memory store would be handed its contents by the +/// test rather than by the volume. +fn boot(dir: &Path, aws: &Arc) -> (Durable, Pds) { + let durable = Durable::open(dir, Duration::days(30)).expect("the log should open"); + let pds = Provisioner::new( + "did:web:owner.example", + zone(), + PDS_ENDPOINT.to_string(), + route53_provider(ZONE_HOST, aws), + durable.accounts(), + ) + .with_record_store(durable.records()) + .with_record_target(target()); + (durable, pds) +} + +/// A block snapshot: every byte of the volume, and nothing off it. +fn snapshot(from: &Path, into: &Path) { + let _ = std::fs::remove_dir_all(into); + std::fs::create_dir_all(into).expect("the restore target"); + for entry in std::fs::read_dir(from) + .expect("the source directory") + .flatten() + { + let path = entry.path(); + let target = into.join(entry.file_name()); + if path.is_dir() { + snapshot(&path, &target); + } else { + std::fs::copy(&path, &target).expect("copy a file"); + } + } +} + +/// What this deployment means the zone to hold: one address record per +/// account it has, pointed at this instance. +fn intent(pds: &Pds) -> BTreeMap { + pds.store() + .list() + .into_iter() + .map(|account| (account.did.host().to_owned(), target())) + .collect() +} + +/// The zone as it reads, over the provider's own listing. +fn observed(aws: &Arc) -> BTreeMap { + let dns = Arc::new(route53_provider(ZONE_HOST, aws)); + ProviderView::new(dns, ZONE_HOST.to_string()) + .refreshed_by(|dns: &Route53Dns| dns.resync().map_err(|error| error.to_string())) + .observe() + .expect("the zone reads") +} + +/// Hydrates the provider from the zone, the way the server binary does at +/// startup. +/// +/// `Route53Dns` keeps its own map of what it published, and a restored +/// process has published nothing. The binary calls this before serving; the +/// test below that omits it says what that call is worth. +fn resync(pds: &Pds) { + pds.dns().resync().expect("the zone reads"); +} + +/// A restore is older than the zone beside it, and of the two kinds of name +/// that leaves, the deployment can act on exactly one. +/// +/// `marmot` was provisioned after the snapshot, so its address record is live +/// and the restored deployment has never heard of it. `kestrel` was in the +/// snapshot, so the restored deployment owns it. +/// +/// The survey separates them, and only one is anything's to act on. +/// `Drift::Unattributed` yields no `Repair` — the reconciler has two repair +/// variants and neither deletes. The stale sweep does not reach it either: a +/// sweep walks the account store and withdraws the hostname of an account it +/// finds there, and a hostname with no account behind it is not on the walk. +/// So a sweep set to take *every* account the restore did bring back +/// withdraws `kestrel` and leaves `marmot` standing — an address record +/// pointed at an instance that answers `404` for it, which is what a restore +/// to an older snapshot hands an operator. +#[test] +fn a_restore_older_than_its_zone_leaves_a_name_the_deployment_never_withdraws() { + let aws = FakeAws::new(HOSTED_ZONE); + let live = scratch("older-live"); + let copy = scratch("older-copy"); + + let (durable, pds) = boot(&live, &aws); + pds.provision(ProvisionRequest::new("kestrel", None)) + .expect("the zone is empty and healthy"); + durable.wal().sync().expect("flush"); + + // The snapshot, and then an account only the zone and the volume it was + // not copied from will ever know about. + snapshot(&live, ©); + pds.provision(ProvisionRequest::new("marmot", None)) + .expect("provisioning should succeed"); + drop(pds); + drop(durable); + + assert!( + aws.holds(&host_of("kestrel"), "A") && aws.holds(&host_of("marmot"), "A"), + "the zone should hold both names before anything is restored" + ); + + // The restore: the older volume, the current zone. + let (restored, pds) = boot(©, &aws); + resync(&pds); + let hosts: Vec = pds + .accounts() + .into_iter() + .map(|account| account.did.host().to_owned()) + .collect(); + assert_eq!( + hosts, + vec![host_of("kestrel")], + "the restore brought back an account the snapshot did not carry" + ); + assert!( + pds.did_document(&host_of("marmot")).is_none(), + "the restored deployment answers for a name it has no account for" + ); + + // The survey, which is where the store and the zone meet. + let survey = Survey::compare(&intent(&pds), &observed(&aws)); + assert!( + survey.is_clean(), + "every name the restore did bring back should still be live: {survey:?}" + ); + let unattributed: Vec<&str> = survey.unattributed.iter().map(Drift::host).collect(); + assert_eq!( + unattributed, + vec![host_of("marmot").as_str()], + "the name the restore lost is the one the survey cannot attribute" + ); + assert!( + survey + .unattributed + .iter() + .all(|drift| drift.repair().is_none()), + "the reconciler offered to act on a record it cannot prove stale" + ); + + // The sweep, set to take everything it can see. A negative window puts + // every account it walks past the cutoff, so a name that survives this + // call survived by being unreachable rather than by being young. + let swept = pds.sweep_stale(Duration::seconds(-1)); + assert_eq!( + swept.len(), + 1, + "the sweep should have taken the one account the restore holds: {swept:?}" + ); + assert!( + !aws.holds(&host_of("kestrel"), "A"), + "the sweep took the account and left its hostname resolving" + ); + assert!( + aws.holds(&host_of("marmot"), "A"), + "the fixture is not exercising anything if the sweep reached this name" + ); + + // Still the only thing standing: one address record pointed here that + // this deployment did not publish and will not withdraw. + let after = Survey::compare(&intent(&pds), &observed(&aws)); + let left: Vec<&str> = after.unattributed.iter().map(Drift::host).collect(); + assert_eq!(left, vec![host_of("marmot").as_str()], "{after:?}"); + + drop(restored); + let _ = std::fs::remove_dir_all(&live); + let _ = std::fs::remove_dir_all(©); +} + +/// What the startup zone read is worth, measured by a restore that skips it. +/// +/// `Route53Dns` withdraws a name out of its own record of having published +/// it, and a restored process has published nothing: every account came off +/// the log. So until the provider is hydrated from the zone, a deletion of a +/// restored account is refused at the DNS step and the account stays — the +/// sweep takes nothing, and `bot.did.deleteAgent` answers the same way. +/// +/// The binary reads the zone at startup and logs a warning if it cannot, so +/// this is the state a restore comes up in when the read failed: serving, and +/// unable to retire a single account it restored. The second half is the same +/// deployment after the read succeeds, which is what makes this a reading of +/// the resync rather than of the sweep. +#[test] +fn a_restored_deployment_retires_its_accounts_once_it_has_read_the_zone() { + let aws = FakeAws::new(HOSTED_ZONE); + let live = scratch("resync-live"); + let copy = scratch("resync-copy"); + + let (durable, pds) = boot(&live, &aws); + pds.provision(ProvisionRequest::new("brambleflit", None)) + .expect("the zone is empty and healthy"); + durable.wal().sync().expect("flush"); + snapshot(&live, ©); + drop(pds); + drop(durable); + + let (restored, pds) = boot(©, &aws); + let host = host_of("brambleflit"); + + let swept = pds.sweep_stale(Duration::seconds(-1)); + assert!( + swept.is_empty(), + "a provider that has not read the zone withdrew a name it never published: {swept:?}" + ); + assert_eq!( + pds.accounts().len(), + 1, + "the account went even though its hostname stayed, which is the pair \ + the deletion holds together" + ); + assert!( + aws.holds(&host, "A"), + "the hostname went without the account" + ); + + // The read the binary performs at startup. + resync(&pds); + + let swept = pds.sweep_stale(Duration::seconds(-1)); + assert_eq!( + swept.len(), + 1, + "the sweep still cannot retire an account it restored: {swept:?}" + ); + assert!( + pds.accounts().is_empty(), + "the account survived a sweep that reported taking it" + ); + assert!( + !aws.holds(&host, "A"), + "the account went and its hostname stayed resolving" + ); + + drop(restored); + let _ = std::fs::remove_dir_all(&live); + let _ = std::fs::remove_dir_all(©); +} -- 2.51.2