diff --git a/knot2/crates/knot-sim/tests/lease_chaos.rs b/knot2/crates/knot-sim/tests/lease_chaos.rs new file mode 100644 index 000000000..980819a46 --- /dev/null +++ b/knot2/crates/knot-sim/tests/lease_chaos.rs @@ -0,0 +1,221 @@ +mod common; + +use std::collections::BTreeMap; +use std::path::Path; +use std::time::Instant; + +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Registration, RegistryChange}; +use knot_git::{Layout, Repo}; +use knot_index::{Index, Resolved}; +use knot_secrets::MasterKey; +use knot_types::{ + AccountDid, KnotId, OwnerDid, RefName, RepoDid, RepoName, RepoRkey, UnixSeconds, Visibility, +}; + +const KNOT_HOST: &str = "knot.nel.pet"; +const ACCOUNTS: usize = 8; +const PASSES: usize = 5; +const FIRST_RENEWAL: i64 = 1_000_000; +const HOUR_AND_A_BIT: i64 = 3_700; +const TTL_SECS: i64 = 3 * 86_400; +const MASTER_KEY: [u8; 32] = [7u8; 32]; + +fn knot() -> KnotId { + KnotId::new(format!("did:web:{KNOT_HOST}")).unwrap() +} + +fn account(index: usize) -> AccountDid { + AccountDid::new(format!("did:plc:lessee{index}")).unwrap() +} + +fn repo(index: usize) -> RepoDid { + RepoDid::new(format!("did:plc:leasedrepo{index}")).unwrap() +} + +fn renewed_at(pass: usize, index: usize) -> i64 { + FIRST_RENEWAL + (pass * ACCOUNTS + index) as i64 * HOUR_AND_A_BIT +} + +fn build_target(root: &Path) { + let (layout, signer) = + common::bootstrap_target(root, &knot(), &MasterKey::new(MASTER_KEY).unwrap()); + let meta = Repo::open(layout.meta_path(&knot()).unwrap()).unwrap(); + let store = CobStore::new(&meta); + let home = CobHome::from(&knot()); + let registration = |index: usize| { + layout.create(&repo(index)).unwrap(); + RegistryChange::Register(Registration { + owner: OwnerDid::new(account(index).as_str()).unwrap(), + rkey: RepoRkey::new(format!("repo{index}")).unwrap(), + name: RepoName::new(format!("repo{index}")).unwrap(), + repo: repo(index), + created_at: UnixSeconds::new(1), + policy: knot_types::ContributionPolicy::default(), + visibility: Visibility::Private, + spindle: None, + }) + }; + let registry = store + .create(&home, ®istration(0), &signer, UnixSeconds::new(1)) + .unwrap() + .object; + (1..ACCOUNTS).for_each(|index| { + store + .update( + &home, + registry, + ®istration(index), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + }); +} + +fn ready_index(target: &Path) -> (Index, knot_acl::Repos) { + let layout = Layout::new(target.join("repos")); + let meta = layout.meta_path(&knot()).unwrap(); + let index = Index::new(meta, layout.clone()); + index.rebuild().unwrap(); + index.warm_collaborators(); + (index, knot_acl::Repos::new(layout)) +} + +#[test] +fn lease_chaos_worker() { + if std::env::var("KNOT_CHAOS_ROLE").as_deref() != Ok("lease") { + return; + } + let target = std::env::var("KNOT_CHAOS_TARGET").unwrap(); + let (index, repos) = ready_index(Path::new(&target)); + (0..PASSES).for_each(|pass| { + (0..ACCOUNTS).for_each(|who| { + let now = UnixSeconds::new(renewed_at(pass, who)); + index.renew_lease(&account(who), now); + knot_xrpc::persist_leases(&index, &repos, &knot(), now); + }); + }); +} + +fn spawn_worker(target: &Path) -> std::process::Child { + common::spawn_worker( + "lease_chaos_worker", + &[ + ("KNOT_CHAOS_ROLE", "lease"), + ("KNOT_CHAOS_TARGET", target.to_str().unwrap()), + ], + ) +} + +fn stored_leases(target: &Path, trial: &str) -> BTreeMap { + let (index, repos) = ready_index(target); + let meta = Repo::open( + Layout::new(target.join("repos")) + .meta_path(&knot()) + .unwrap(), + ) + .unwrap(); + let ref_exists = meta + .find_ref(&RefName::new(knot_xrpc::LEASES_REF).unwrap()) + .unwrap() + .is_some(); + let boot = UnixSeconds::new(FIRST_RENEWAL); + knot_xrpc::load_leases(&index, &repos, &knot(), boot); + let Resolved::Ready(covered) = index.covered_private_repos(boot) else { + panic!("{trial}: the rebooted index must be ready"); + }; + let held: BTreeMap = covered + .live + .iter() + .map(|(covered_repo, expiry)| { + let who = (0..ACCOUNTS) + .find(|who| repo(*who) == *covered_repo) + .unwrap_or_else(|| panic!("{trial}: {covered_repo} is not a repo of ours")); + (who, expiry.get()) + }) + .collect(); + assert!( + held.len() == covered.live.len(), + "{trial}: two covered repos for one account" + ); + assert!( + !ref_exists || !held.is_empty(), + "{trial}: the stored leases ref exists but loads no lease, so it no longer parses" + ); + held +} + +fn expected_after(steps: usize) -> BTreeMap { + (0..ACCOUNTS) + .filter_map(|who| { + let renewed = (0..PASSES) + .filter(|pass| pass * ACCOUNTS + who < steps) + .count(); + renewed + .checked_sub(1) + .map(|last| (who, renewed_at(last, who) + TTL_SECS)) + }) + .collect() +} + +fn stored_steps(held: &BTreeMap, trial: &str) -> usize { + let renewals: usize = held + .iter() + .map(|(who, expiry)| { + (0..PASSES) + .find(|pass| renewed_at(*pass, *who) + TTL_SECS == *expiry) + .unwrap_or_else(|| { + panic!("{trial}: account {who} holds expiry {expiry}, which is no renewal plus the ttl") + }) + + 1 + }) + .sum(); + assert_eq!( + held, + &expected_after(renewals), + "{trial}: the stored leases are no state the worker passed through" + ); + renewals +} + +#[test] +fn lease_chaos() { + let scratch = tempfile::tempdir().unwrap(); + + let warm = scratch.path().join("warm"); + build_target(&warm); + let started = Instant::now(); + spawn_worker(&warm).wait().unwrap(); + let full = started.elapsed(); + let full_steps = stored_steps(&stored_leases(&warm, "warm"), "warm"); + assert_eq!( + full_steps, + ACCOUNTS * PASSES, + "an uninterrupted worker stores every renewal" + ); + + let outcomes: Vec = common::kill_delays(full) + .iter() + .enumerate() + .map(|(trial, delay)| { + let target = scratch.path().join(format!("trial-{trial}")); + build_target(&target); + let mut child = spawn_worker(&target); + std::thread::sleep(*delay); + let _ = child.kill(); + child.wait().unwrap(); + let name = format!("trial {trial}"); + stored_steps(&stored_leases(&target, &name), &name) + }) + .collect(); + + assert!( + outcomes.iter().any(|steps| *steps < full_steps), + "some trial must be killed before every renewal is stored, or the kill delays are all too long" + ); + assert!( + outcomes.contains(&full_steps), + "some trial must reach every renewal, or the kill delays are all too short" + ); +}