diff --git a/crates/didbot-pds/tests/restore_zone.rs b/crates/didbot-pds/tests/restore_zone.rs new file mode 100644 index 00000000..fff8ecaf --- /dev/null +++ b/crates/didbot-pds/tests/restore_zone.rs @@ -0,0 +1,300 @@ +//! What a restore does to the hosted zone, which is not on the volume. +//! +//! `crates/didbot-pds/tests/restore.rs` drills the volume: everything a +//! snapshot carries. The zone is the other half of a deployment's state and +//! it is somewhere else entirely — Route53 outlives the instance, so a +//! restore puts an old account store next to a current zone and the two +//! disagree about every account provisioned after the snapshot. +//! +//! `route53_seam.rs` asks the same question of a *crash* between the zone +//! write and the store insert. A restore differs in what it can do about it: +//! the accounts come back off the log rather than out of memory, and the two +//! machines that could act — the stale sweep and the reconciler's survey — +//! reach exactly one of the two kinds of name this leaves behind. Which one +//! is what this file pins. +//! +//! The provider is the real `Route53Dns` over `didbot_dns::aws_fake`, so the +//! zone is read back over the listing wire format rather than out of the +//! provider's own bookkeeping. + +use std::collections::BTreeMap; +use std::net::Ipv4Addr; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use didbot_dns::aws_fake::{route53_provider, FakeAws}; +use didbot_dns::route53::Route53Dns; +use didbot_dns::RecordTarget; +use didbot_identity::Zone; +use didbot_pds::{ + AccountStore, Durable, FileAccountStore, ProvisionRequest, Provisioner, Registry, +}; +use didbot_reconcile::{Drift, ProviderView, Survey, ZoneView}; +use time::Duration; + +const ZONE_HOST: &str = "agents.example.com"; +const PDS_ENDPOINT: &str = "https://pds.example.com"; +const HOSTED_ZONE: &str = "Z0FAKEZONE"; + +type Pds = Provisioner>; + +fn scratch(name: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("didbot-restore-zone-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + dir +} + +fn zone() -> Zone { + Zone::delegated("example.com", ZONE_HOST).expect("test zone should be constructible") +} + +fn target() -> RecordTarget { + RecordTarget::Ipv4(Ipv4Addr::new(203, 0, 113, 7)) +} + +fn host_of(agent_id: &str) -> String { + format!("{agent_id}.{ZONE_HOST}") +} + +/// A deployment over `dir`, publishing into `aws`. +/// +/// The store is the file-backed one the log replays, because that is what a +/// restore brings back — a memory store would be handed its contents by the +/// test rather than by the volume. +fn boot(dir: &Path, aws: &Arc) -> (Durable, Pds) { + let durable = Durable::open(dir, Duration::days(30)).expect("the log should open"); + let pds = Provisioner::new( + "did:web:owner.example", + zone(), + PDS_ENDPOINT.to_string(), + route53_provider(ZONE_HOST, aws), + durable.accounts(), + ) + .with_record_store(durable.records()) + .with_record_target(target()); + (durable, pds) +} + +/// A block snapshot: every byte of the volume, and nothing off it. +fn snapshot(from: &Path, into: &Path) { + let _ = std::fs::remove_dir_all(into); + std::fs::create_dir_all(into).expect("the restore target"); + for entry in std::fs::read_dir(from) + .expect("the source directory") + .flatten() + { + let path = entry.path(); + let target = into.join(entry.file_name()); + if path.is_dir() { + snapshot(&path, &target); + } else { + std::fs::copy(&path, &target).expect("copy a file"); + } + } +} + +/// What this deployment means the zone to hold: one address record per +/// account it has, pointed at this instance. +fn intent(pds: &Pds) -> BTreeMap { + pds.store() + .list() + .into_iter() + .map(|account| (account.did.host().to_owned(), target())) + .collect() +} + +/// The zone as it reads, over the provider's own listing. +fn observed(aws: &Arc) -> BTreeMap { + let dns = Arc::new(route53_provider(ZONE_HOST, aws)); + ProviderView::new(dns, ZONE_HOST.to_string()) + .refreshed_by(|dns: &Route53Dns| dns.resync().map_err(|error| error.to_string())) + .observe() + .expect("the zone reads") +} + +/// Hydrates the provider from the zone, the way the server binary does at +/// startup. +/// +/// `Route53Dns` keeps its own map of what it published, and a restored +/// process has published nothing. The binary calls this before serving; the +/// test below that omits it says what that call is worth. +fn resync(pds: &Pds) { + pds.dns().resync().expect("the zone reads"); +} + +/// A restore is older than the zone beside it, and of the two kinds of name +/// that leaves, the deployment can act on exactly one. +/// +/// `marmot` was provisioned after the snapshot, so its address record is live +/// and the restored deployment has never heard of it. `kestrel` was in the +/// snapshot, so the restored deployment owns it. +/// +/// The survey separates them, and only one is anything's to act on. +/// `Drift::Unattributed` yields no `Repair` — the reconciler has two repair +/// variants and neither deletes. The stale sweep does not reach it either: a +/// sweep walks the account store and withdraws the hostname of an account it +/// finds there, and a hostname with no account behind it is not on the walk. +/// So a sweep set to take *every* account the restore did bring back +/// withdraws `kestrel` and leaves `marmot` standing — an address record +/// pointed at an instance that answers `404` for it, which is what a restore +/// to an older snapshot hands an operator. +#[test] +fn a_restore_older_than_its_zone_leaves_a_name_the_deployment_never_withdraws() { + let aws = FakeAws::new(HOSTED_ZONE); + let live = scratch("older-live"); + let copy = scratch("older-copy"); + + let (durable, pds) = boot(&live, &aws); + pds.provision(ProvisionRequest::new("kestrel", None)) + .expect("the zone is empty and healthy"); + durable.wal().sync().expect("flush"); + + // The snapshot, and then an account only the zone and the volume it was + // not copied from will ever know about. + snapshot(&live, ©); + pds.provision(ProvisionRequest::new("marmot", None)) + .expect("provisioning should succeed"); + drop(pds); + drop(durable); + + assert!( + aws.holds(&host_of("kestrel"), "A") && aws.holds(&host_of("marmot"), "A"), + "the zone should hold both names before anything is restored" + ); + + // The restore: the older volume, the current zone. + let (restored, pds) = boot(©, &aws); + resync(&pds); + let hosts: Vec = pds + .accounts() + .into_iter() + .map(|account| account.did.host().to_owned()) + .collect(); + assert_eq!( + hosts, + vec![host_of("kestrel")], + "the restore brought back an account the snapshot did not carry" + ); + assert!( + pds.did_document(&host_of("marmot")).is_none(), + "the restored deployment answers for a name it has no account for" + ); + + // The survey, which is where the store and the zone meet. + let survey = Survey::compare(&intent(&pds), &observed(&aws)); + assert!( + survey.is_clean(), + "every name the restore did bring back should still be live: {survey:?}" + ); + let unattributed: Vec<&str> = survey.unattributed.iter().map(Drift::host).collect(); + assert_eq!( + unattributed, + vec![host_of("marmot").as_str()], + "the name the restore lost is the one the survey cannot attribute" + ); + assert!( + survey + .unattributed + .iter() + .all(|drift| drift.repair().is_none()), + "the reconciler offered to act on a record it cannot prove stale" + ); + + // The sweep, set to take everything it can see. A negative window puts + // every account it walks past the cutoff, so a name that survives this + // call survived by being unreachable rather than by being young. + let swept = pds.sweep_stale(Duration::seconds(-1)); + assert_eq!( + swept.len(), + 1, + "the sweep should have taken the one account the restore holds: {swept:?}" + ); + assert!( + !aws.holds(&host_of("kestrel"), "A"), + "the sweep took the account and left its hostname resolving" + ); + assert!( + aws.holds(&host_of("marmot"), "A"), + "the fixture is not exercising anything if the sweep reached this name" + ); + + // Still the only thing standing: one address record pointed here that + // this deployment did not publish and will not withdraw. + let after = Survey::compare(&intent(&pds), &observed(&aws)); + let left: Vec<&str> = after.unattributed.iter().map(Drift::host).collect(); + assert_eq!(left, vec![host_of("marmot").as_str()], "{after:?}"); + + drop(restored); + let _ = std::fs::remove_dir_all(&live); + let _ = std::fs::remove_dir_all(©); +} + +/// What the startup zone read is worth, measured by a restore that skips it. +/// +/// `Route53Dns` withdraws a name out of its own record of having published +/// it, and a restored process has published nothing: every account came off +/// the log. So until the provider is hydrated from the zone, a deletion of a +/// restored account is refused at the DNS step and the account stays — the +/// sweep takes nothing, and `bot.did.deleteAgent` answers the same way. +/// +/// The binary reads the zone at startup and logs a warning if it cannot, so +/// this is the state a restore comes up in when the read failed: serving, and +/// unable to retire a single account it restored. The second half is the same +/// deployment after the read succeeds, which is what makes this a reading of +/// the resync rather than of the sweep. +#[test] +fn a_restored_deployment_retires_its_accounts_once_it_has_read_the_zone() { + let aws = FakeAws::new(HOSTED_ZONE); + let live = scratch("resync-live"); + let copy = scratch("resync-copy"); + + let (durable, pds) = boot(&live, &aws); + pds.provision(ProvisionRequest::new("brambleflit", None)) + .expect("the zone is empty and healthy"); + durable.wal().sync().expect("flush"); + snapshot(&live, ©); + drop(pds); + drop(durable); + + let (restored, pds) = boot(©, &aws); + let host = host_of("brambleflit"); + + let swept = pds.sweep_stale(Duration::seconds(-1)); + assert!( + swept.is_empty(), + "a provider that has not read the zone withdrew a name it never published: {swept:?}" + ); + assert_eq!( + pds.accounts().len(), + 1, + "the account went even though its hostname stayed, which is the pair \ + the deletion holds together" + ); + assert!( + aws.holds(&host, "A"), + "the hostname went without the account" + ); + + // The read the binary performs at startup. + resync(&pds); + + let swept = pds.sweep_stale(Duration::seconds(-1)); + assert_eq!( + swept.len(), + 1, + "the sweep still cannot retire an account it restored: {swept:?}" + ); + assert!( + pds.accounts().is_empty(), + "the account survived a sweep that reported taking it" + ); + assert!( + !aws.holds(&host, "A"), + "the account went and its hostname stayed resolving" + ); + + drop(restored); + let _ = std::fs::remove_dir_all(&live); + let _ = std::fs::remove_dir_all(©); +}