diff --git a/crates/didbot/tests/scenarios.rs b/crates/didbot/tests/scenarios.rs new file mode 100644 index 00000000..2e99625b --- /dev/null +++ b/crates/didbot/tests/scenarios.rs @@ -0,0 +1,5804 @@ +//! The scenarios `plan/ownership.md` names, run with the real binaries. +//! +//! `didbot-pds` serves a `.localhost` zone on a port nobody chose; `didbot` +//! dispatches to `didbot-operator`, `didbot-register` and `didbot-oauth` +//! from the build directory; `didbot-agentd` holds a host's key and creates +//! agents over its socket. Two parties are stood in for, each a loopback +//! router answering the requests the stack makes of it: the human's own PDS +//! — their `did:web` document, their repository over `com.atproto.repo.*`, +//! and `com.atproto.server.getServiceAuth` signing with the key that +//! document names — and an OpenID Connect issuer with a discovery document, +//! a JWKS, and ID tokens signed by the key in it. +//! +//! `didbot operate` signs in to the human's PDS through OAuth, which is a +//! browser. The session it would keep afterwards is written here through +//! the same store and types the command reads it back with, so every run +//! resumes it; the consent screen is the one exchange not exercised. The +//! record writes, the service-auth mint and every call at the server are +//! the command's own. +//! +//! Every assertion is what a stranger could check: public reads of records +//! and documents, plus each command's exit status and named refusal. The +//! zone is `.localhost`, where every binary speaks plain HTTP; nothing here +//! terminates TLS. + +mod support; + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::{Duration, Instant}; + +use axum::extract::{Query, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use base64::prelude::*; +use didbot::data::{dag_cbor, Value as Dag}; +use didbot_authstore::FileAuthStore; +use didbot_key::SigningKey; +use didbot_operator::operate::scope::admit_scope; +use jacquard_oauth::authstore::ClientAuthStore; +use p256::ecdsa::signature::Signer; +use serde_json::{json, Value}; +use time::format_description::well_known::Rfc3339; +use time::OffsetDateTime; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; + +/// How long anything here waits for a server, a poll or a process before +/// naming what never happened. The operator poll's floor is ten seconds, +/// so a nudged claim lands inside this with room. +const PATIENCE: Duration = Duration::from_secs(45); + +/// How often a wait re-reads. +const BEAT: Duration = Duration::from_millis(200); + +fn b64(bytes: &[u8]) -> String { + BASE64_URL_SAFE_NO_PAD.encode(bytes) +} + +fn now_rfc3339() -> String { + OffsetDateTime::now_utc() + .format(&Rfc3339) + .expect("a clock formats") +} + +/// Re-reads until `read` answers, or fails naming what never happened. +async fn wait_for(what: &str, mut read: F) -> T +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let started = Instant::now(); + loop { + if let Some(value) = read().await { + return value; + } + assert!( + started.elapsed() < PATIENCE, + "waited {PATIENCE:?} for {what}, which never happened" + ); + tokio::time::sleep(BEAT).await; + } +} + +// --------------------------------------------------------------------------- +// The binaries +// --------------------------------------------------------------------------- + +/// The build directory the binaries live in, built once per process. +/// +/// `cargo test -p didbot` builds this crate's tests and nothing else's +/// binaries, so the first test to ask builds them; a build that is already +/// current returns at once. +fn bins() -> &'static Path { + static BUILT: OnceLock = OnceLock::new(); + BUILT.get_or_init(|| { + let exe = std::env::current_exe().expect("this test has a path"); + // …/target//deps/scenarios- + let dir = exe + .parent() + .and_then(Path::parent) + .expect("the test binary sits under target//deps") + .to_path_buf(); + let mut args = vec![ + "build", + "--quiet", + "--bins", + "-p", + "didbot-serve", + "-p", + "didbot-operator", + "-p", + "didbot-agentd", + "-p", + "didbot-dispatch", + ]; + if dir.file_name().is_some_and(|name| name == "release") { + args.push("--release"); + } + let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + let status = std::process::Command::new(cargo) + .args(&args) + .env( + "CARGO_TARGET_DIR", + dir.parent().expect("target/ has a parent"), + ) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .status() + .expect("cargo runs"); + assert!(status.success(), "the binaries build"); + for name in [ + "didbot", + "didbot-pds", + "didbot-operator", + "didbot-register", + "didbot-oauth", + "didbot-agentd", + ] { + assert!(dir.join(name).is_file(), "{name} is in {}", dir.display()); + } + dir + }) +} + +/// A directory under the system's temporary directory, short enough for a +/// unix socket path to fit, removed when the test passes and named when it +/// does not. +struct Work { + root: PathBuf, +} + +impl Work { + /// One label names one directory for the whole run, so two scenarios + /// sharing a label share the directory and wipe each other's files. + fn new(label: &str) -> Self { + let root = + std::env::temp_dir().join(format!("didbot-scenarios-{}-{label}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("a working directory"); + Self { root } + } + + fn dir(&self, name: &str) -> PathBuf { + let path = self.root.join(name); + std::fs::create_dir_all(&path).expect("a directory"); + path + } + + fn file(&self, name: &str, contents: &str) -> PathBuf { + let path = self.root.join(name); + std::fs::write(&path, contents).expect("a file writes"); + path + } +} + +impl Drop for Work { + fn drop(&mut self) { + if std::thread::panicking() { + eprintln!("scenarios: kept {} for inspection", self.root.display()); + } else { + let _ = std::fs::remove_dir_all(&self.root); + } + } +} + +/// A free port on this host, both families. +fn free_port() -> u16 { + std::net::TcpListener::bind("[::]:0") + .expect("a port") + .local_addr() + .expect("an address") + .port() +} + +// --------------------------------------------------------------------------- +// The human's own PDS +// --------------------------------------------------------------------------- + +#[derive(Default)] +struct Repository { + /// Records by collection, then record key. + records: BTreeMap>, + /// The `Authorization` scheme every write arrived with. + write_auth: Vec, +} + +/// How the human's PDS answers: as itself, or as an outage would have it. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Outage { + None, + /// Every request is a 500. + Failing, + /// Every request is accepted and never answered. + Hanging, + /// Reads answer, and every listing is empty. + EmptyListings, +} + +struct HumanState { + did: String, + origin: String, + key: SigningKey, + repository: Mutex, + outage: Mutex, +} + +/// A stand-in for the human's own PDS, on `human.localhost`. +struct Human { + state: Arc, +} + +impl Human { + async fn start() -> Self { + Self::start_on(0).await + } + + /// The same stand-in on a port somebody chose, so a server started by + /// hand can be pointed at it. + async fn start_on(port: u16) -> Self { + let listener = tokio::net::TcpListener::bind(("::", port)) + .await + .expect("bind"); + let port = listener.local_addr().expect("local addr").port(); + let state = Arc::new(HumanState { + did: format!("did:web:human.localhost%3A{port}"), + origin: format!("http://human.localhost:{port}"), + key: SigningKey::generate(), + repository: Mutex::new(Repository::default()), + outage: Mutex::new(Outage::None), + }); + let router = Router::new() + .route("/.well-known/did.json", get(Self::did_json)) + .route("/xrpc/com.atproto.repo.getRecord", get(Self::get_record)) + .route( + "/xrpc/com.atproto.repo.listRecords", + get(Self::list_records), + ) + .route("/xrpc/com.atproto.repo.putRecord", post(Self::put_record)) + .route( + "/xrpc/com.atproto.repo.deleteRecord", + post(Self::delete_record), + ) + .route( + "/xrpc/com.atproto.server.getServiceAuth", + get(Self::get_service_auth), + ) + .with_state(state.clone()); + tokio::spawn(async move { + axum::serve(listener, router).await.ok(); + }); + Self { state } + } + + fn did(&self) -> &str { + &self.state.did + } + + fn origin(&self) -> &str { + &self.state.origin + } + + /// Puts the human's PDS into `outage`, or back. + fn set_outage(&self, outage: Outage) { + *self.state.outage.lock().expect("outage") = outage; + } + + /// The refusal an outage answers with, if one is on. + async fn outage(state: &HumanState) -> Option { + let outage = *state.outage.lock().expect("outage"); + match outage { + Outage::None | Outage::EmptyListings => None, + Outage::Failing => { + Some((StatusCode::INTERNAL_SERVER_ERROR, "the human's PDS is down").into_response()) + } + Outage::Hanging => { + tokio::time::sleep(Duration::from_secs(120)).await; + Some(StatusCode::GATEWAY_TIMEOUT.into_response()) + } + } + } + + async fn did_json(State(state): State>) -> axum::response::Response { + if let Some(refusal) = Self::outage(&state).await { + return refusal; + } + Json(json!({ + "@context": ["https://www.w3.org/ns/did/v1"], + "id": state.did, + "verificationMethod": [{ + "id": format!("{}#atproto", state.did), + "type": "Multikey", + "controller": state.did, + "publicKeyMultibase": state.key.verifying_key().to_multibase(), + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": state.origin, + }], + })) + .into_response() + } + + fn refuse(status: StatusCode, error: &str, message: &str) -> axum::response::Response { + (status, Json(json!({ "error": error, "message": message }))).into_response() + } + + fn uri(state: &HumanState, collection: &str, rkey: &str) -> String { + format!("at://{}/{collection}/{rkey}", state.did) + } + + async fn get_record( + State(state): State>, + Query(query): Query>, + ) -> axum::response::Response { + if let Some(refusal) = Self::outage(&state).await { + return refusal; + } + let (Some(repo), Some(collection), Some(rkey)) = ( + query.get("repo"), + query.get("collection"), + query.get("rkey"), + ) else { + return Self::refuse( + StatusCode::BAD_REQUEST, + "InvalidRequest", + "repo, collection and rkey", + ); + }; + if repo != &state.did { + return Self::refuse(StatusCode::BAD_REQUEST, "RepoNotFound", repo); + } + let held = state.repository.lock().expect("repository"); + match held.records.get(collection).and_then(|c| c.get(rkey)) { + Some(value) => Json(json!({ + "uri": Self::uri(&state, collection, rkey), + "cid": "bafyfake", + "value": value, + })) + .into_response(), + None => Self::refuse(StatusCode::BAD_REQUEST, "RecordNotFound", rkey), + } + } + + async fn list_records( + State(state): State>, + Query(query): Query>, + ) -> axum::response::Response { + if let Some(refusal) = Self::outage(&state).await { + return refusal; + } + let (Some(repo), Some(collection)) = (query.get("repo"), query.get("collection")) else { + return Self::refuse( + StatusCode::BAD_REQUEST, + "InvalidRequest", + "repo and collection", + ); + }; + if repo != &state.did { + return Self::refuse(StatusCode::BAD_REQUEST, "RepoNotFound", repo); + } + if *state.outage.lock().expect("outage") == Outage::EmptyListings { + return Json(json!({ "records": [] })).into_response(); + } + let held = state.repository.lock().expect("repository"); + let records: Vec = held + .records + .get(collection) + .map(|c| { + c.iter() + .map(|(rkey, value)| { + json!({ + "uri": Self::uri(&state, collection, rkey), + "cid": "bafyfake", + "value": value, + }) + }) + .collect() + }) + .unwrap_or_default(); + Json(json!({ "records": records })).into_response() + } + + fn scheme_of(headers: &HeaderMap) -> String { + headers + .get("authorization") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(' ').next()) + .unwrap_or("none") + .to_owned() + } + + async fn put_record( + State(state): State>, + headers: HeaderMap, + Json(body): Json, + ) -> axum::response::Response { + let (Some(collection), Some(rkey)) = (body["collection"].as_str(), body["rkey"].as_str()) + else { + return Self::refuse( + StatusCode::BAD_REQUEST, + "InvalidRequest", + "collection and rkey", + ); + }; + let mut held = state.repository.lock().expect("repository"); + held.write_auth.push(Self::scheme_of(&headers)); + held.records + .entry(collection.to_owned()) + .or_default() + .insert(rkey.to_owned(), body["record"].clone()); + Json(json!({ "uri": Self::uri(&state, collection, rkey), "cid": "bafyfake" })) + .into_response() + } + + async fn delete_record( + State(state): State>, + headers: HeaderMap, + Json(body): Json, + ) -> axum::response::Response { + let (Some(collection), Some(rkey)) = (body["collection"].as_str(), body["rkey"].as_str()) + else { + return Self::refuse( + StatusCode::BAD_REQUEST, + "InvalidRequest", + "collection and rkey", + ); + }; + let mut held = state.repository.lock().expect("repository"); + held.write_auth.push(Self::scheme_of(&headers)); + if let Some(c) = held.records.get_mut(collection) { + c.remove(rkey); + } + Json(json!({})).into_response() + } + + /// The service-auth token the human's PDS mints for one call: signed + /// ES256K with the key the human's document names as `#atproto`. + fn mint_service_auth(state: &HumanState, aud: &str, lxm: &str) -> String { + let now = OffsetDateTime::now_utc(); + let payload = json!({ + "iss": state.did, + "aud": aud, + "lxm": lxm, + "iat": now.unix_timestamp(), + "exp": (now + time::Duration::minutes(5)).unix_timestamp(), + "jti": format!("{}", now.unix_timestamp_nanos()), + }); + let input = format!( + "{}.{}", + b64(br#"{"typ":"JWT","alg":"ES256K"}"#), + b64(payload.to_string().as_bytes()) + ); + let signature = state.key.sign(input.as_bytes()).to_bytes(); + format!("{input}.{}", b64(&signature)) + } + + async fn get_service_auth( + State(state): State>, + Query(query): Query>, + ) -> axum::response::Response { + if let Some(refusal) = Self::outage(&state).await { + return refusal; + } + let (Some(aud), Some(lxm)) = (query.get("aud"), query.get("lxm")) else { + return Self::refuse(StatusCode::BAD_REQUEST, "InvalidRequest", "aud and lxm"); + }; + Json(json!({ "token": Self::mint_service_auth(&state, aud, lxm) })).into_response() + } + + /// A service-auth token for one call at `aud`, as the human's PDS + /// mints it, held here so it can be presented more than once. + fn service_auth(&self, aud: &str, lxm: &str) -> String { + Self::mint_service_auth(&self.state, aud, lxm) + } + + /// The record keys in one of the human's collections, as anyone reads + /// them. + async fn record_keys(&self, collection: &str) -> Vec { + list_record_keys(self.origin(), self.did(), collection).await + } + + /// Writes a record the way something other than `didbot operate` + /// would: a policy from the policy page, say. + fn write(&self, collection: &str, rkey: &str, value: Value) { + self.state + .repository + .lock() + .expect("repository") + .records + .entry(collection.to_owned()) + .or_default() + .insert(rkey.to_owned(), value); + } + + /// One of the human's records, as their repository holds it. + fn record(&self, collection: &str, rkey: &str) -> Option { + self.state + .repository + .lock() + .expect("repository") + .records + .get(collection) + .and_then(|c| c.get(rkey)) + .cloned() + } + + /// Deletes a record, as the human would from any client of their PDS. + fn delete(&self, collection: &str, rkey: &str) { + if let Some(c) = self + .state + .repository + .lock() + .expect("repository") + .records + .get_mut(collection) + { + c.remove(rkey); + } + } + + fn write_auth(&self) -> Vec { + self.state + .repository + .lock() + .expect("repository") + .write_auth + .clone() + } + + /// The session `didbot operate` resumes: written through the store the + /// command opens, granted `scope`, bound to a DPoP key made here, and + /// good for long enough that no refresh is ever asked for. + async fn seed_session(&self, config_home: &Path, scope: &str) { + use jacquard_common::deps::fluent_uri::Uri; + use jacquard_common::session::SessionKey; + use jacquard_common::types::datetime::Datetime; + use jacquard_common::types::did::Did; + use jacquard_oauth::scopes::Scopes; + use jacquard_oauth::session::{ClientSessionData, DpopClientData}; + use jacquard_oauth::types::{OAuthTokenType, TokenSet}; + use smol_str::SmolStr; + + use p256::elliptic_curve::sec1::ToEncodedPoint; + let secret = p256::SecretKey::random(&mut rand_core::OsRng); + let point = secret.public_key().to_encoded_point(false); + let dpop_key: jose_jwk::Key = serde_json::from_value(json!({ + "kty": "EC", + "crv": "P-256", + "x": b64(point.x().expect("x")), + "y": b64(point.y().expect("y")), + "d": b64(&secret.to_bytes()), + })) + .expect("a P-256 JWK"); + let did: Did = Did::new(SmolStr::new(self.did())).expect("a did"); + let session_id = SmolStr::new("scenario-session"); + let data = ClientSessionData:: { + account_did: did.clone(), + session_id: session_id.clone(), + host_url: Uri::parse(self.origin().to_owned()).expect("a url"), + authserver_url: SmolStr::new(self.origin()), + authserver_token_endpoint: SmolStr::new(format!("{}/oauth/token", self.origin())), + authserver_revocation_endpoint: None, + scopes: Scopes::new(SmolStr::new(scope)).expect("the scope parses"), + dpop_data: DpopClientData { + dpop_key, + dpop_authserver_nonce: SmolStr::default(), + dpop_host_nonce: SmolStr::default(), + }, + token_set: TokenSet { + iss: SmolStr::new(self.origin()), + sub: did.clone(), + aud: SmolStr::new(self.origin()), + scope: Some(SmolStr::new(scope)), + refresh_token: None, + access_token: SmolStr::new("a-session-the-human-signed-in-for"), + token_type: OAuthTokenType::DPoP, + expires_at: Some( + "2100-01-01T00:00:00Z" + .parse::() + .expect("a datetime"), + ), + }, + resolved_scopes: None, + }; + let path = config_home.join("didbot-operator").join("sessions.json"); + let store = FileAuthStore::open(&path).expect("the store opens"); + store + .upsert_session(data) + .await + .expect("the session is written"); + store + .remember(self.did(), SessionKey::new(did, session_id)) + .expect("the handle is remembered"); + } +} + +// --------------------------------------------------------------------------- +// An OpenID Connect issuer +// --------------------------------------------------------------------------- + +/// A platform's issuer: discovery, a JWKS with one P-256 key, and ID tokens +/// signed with it. +struct Issuer { + base: String, + key: p256::ecdsa::SigningKey, +} + +impl Issuer { + async fn start() -> Self { + Self::start_on(0).await + } + + /// The same issuer on a port somebody chose. + async fn start_on(port: u16) -> Self { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", port)) + .await + .expect("bind"); + let addr = listener.local_addr().expect("local addr"); + let base = format!("http://{addr}"); + let key = p256::ecdsa::SigningKey::random(&mut rand_core::OsRng); + let point = key.verifying_key().to_encoded_point(false); + let jwks = json!({ + "keys": [{ + "kty": "EC", + "crv": "P-256", + "kid": "k1", + "alg": "ES256", + "use": "sig", + "x": b64(point.x().expect("x")), + "y": b64(point.y().expect("y")), + }] + }); + let discovery = json!({ "issuer": base, "jwks_uri": format!("{base}/jwks") }); + let router = Router::new() + .route( + "/.well-known/openid-configuration", + get(move || { + let discovery = discovery.clone(); + async move { Json(discovery) } + }), + ) + .route( + "/jwks", + get(move || { + let jwks = jwks.clone(); + async move { Json(jwks) } + }), + ); + tokio::spawn(async move { + axum::serve(listener, router).await.ok(); + }); + Self { base, key } + } + + /// An ID token for `aud` carrying `claims` beside the standard ones, + /// the way a platform hands one to a run or an instance. + fn id_token(&self, aud: &str, claims: Value) -> String { + let now = OffsetDateTime::now_utc(); + let mut payload = json!({ + "iss": self.base, + "aud": aud, + "sub": "subject-1", + "iat": now.unix_timestamp(), + "exp": (now + time::Duration::minutes(5)).unix_timestamp(), + "jti": format!("{}", now.unix_timestamp_nanos()), + }); + for (name, value) in claims.as_object().expect("claims are an object") { + // A platform that mints no `jti` is spelled by setting it null. + if value.is_null() { + payload.as_object_mut().expect("an object").remove(name); + } else { + payload[name] = value.clone(); + } + } + let input = format!( + "{}.{}", + b64(br#"{"alg":"ES256","kid":"k1","typ":"JWT"}"#), + b64(payload.to_string().as_bytes()) + ); + let signature: p256::ecdsa::Signature = self.key.sign(input.as_bytes()); + format!("{input}.{}", b64(&signature.to_bytes())) + } +} + +// --------------------------------------------------------------------------- +// The server under test +// --------------------------------------------------------------------------- + +/// A `didbot-pds` process on its own zone, operated by the human. +struct Server { + /// `.localhost:`, which is the server's own hostname and the + /// suffix every account under it carries. + host: String, + origin: String, + did: String, + child: std::process::Child, + log: PathBuf, + /// The command line the process was started with, so a restart is the + /// same process again: the same port, zone, operator, configuration + /// and, when it has one, data directory. + args: Vec, + /// The `--data` directory, for a server that keeps one. + data: Option, + /// The `--config` file, rewritten by a scenario that restarts the + /// server under other settings. + config: PathBuf, + /// The port the process listens on, which is part of its DID and so is + /// the same port on every restart. + port: u16, +} + +impl Server { + /// The configuration every server here runs with: no grace once the + /// human's record is gone, so a deletion is acted on at the very next + /// poll rather than hours later. + const CONFIG: &'static str = "[operator]\ngrace_window_hours = 0\n[park]\nrate_limit = 1000\n"; + + /// A server on `config`, keeping a `--data` directory when it has one, + /// on a chosen port, so a restart is the same server again. + async fn start_with( + work: &Work, + label: &str, + human: &Human, + config: &str, + port: u16, + data: Option, + ) -> Self { + let zone = format!("{label}.localhost"); + let host = format!("{zone}:{port}"); + let config = work.file(&format!("{label}.toml"), config); + let log = work.root.join(format!("{label}.log")); + let log_file = std::fs::File::options() + .create(true) + .append(true) + .open(&log) + .expect("a log file"); + let mut args = vec![ + "--port".to_owned(), + port.to_string(), + "--zone".to_owned(), + zone.clone(), + "--operator".to_owned(), + human.did().to_owned(), + "--config".to_owned(), + config.to_str().expect("utf-8").to_owned(), + ]; + if let Some(data) = &data { + args.push("--data".to_owned()); + args.push(data.to_str().expect("utf-8").to_owned()); + } + let child = std::process::Command::new(bins().join("didbot-pds")) + .args(&args) + .env("NO_COLOR", "1") + .stdin(Stdio::null()) + .stdout(Stdio::from(log_file.try_clone().expect("a handle"))) + .stderr(Stdio::from(log_file)) + .spawn() + .expect("didbot-pds starts"); + let server = Self { + origin: format!("http://{host}"), + did: format!("did:web:{zone}%3A{port}"), + host, + child, + log, + args, + data, + config, + port, + }; + let mut server = server; + server.take_port().await; + server + } + + fn spawn(args: &[String], log: &Path) -> std::process::Child { + let log_file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(log) + .expect("a log file"); + std::process::Command::new(bins().join("didbot-pds")) + .args(args) + .env("NO_COLOR", "1") + .stdin(Stdio::null()) + .stdout(Stdio::from(log_file.try_clone().expect("a handle"))) + .stderr(Stdio::from(log_file)) + .spawn() + .expect("didbot-pds starts") + } + + /// Whether the process answers `/health` right now. + async fn healthy(&self) -> bool { + http() + .get(format!("{}/health", self.origin)) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + } + + /// Kills the process outright, the way a crash or a supervisor does, + /// and leaves it down. + fn kill(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } + + /// Waits until the process serves. The port is part of the server's + /// DID, so a server takes that port or nothing: a process that exited + /// found the port briefly held by another scenario's `free_port`, and + /// is started again until it gets it. + async fn take_port(&mut self) { + let started = Instant::now(); + while !self.healthy().await { + assert!( + started.elapsed() < PATIENCE, + "the server never served on port {}", + self.port + ); + if matches!(self.child.try_wait(), Ok(Some(_))) { + self.child = Self::spawn(&self.args, &self.log); + } + tokio::time::sleep(BEAT).await; + } + } + + /// Starts the same process again on the same port and data directory, + /// and waits until it answers. Returns how long it was unreachable. + async fn start_again(&mut self) -> Duration { + let down = Instant::now(); + self.child = Self::spawn(&self.args, &self.log); + self.take_port().await; + down.elapsed() + } + + /// A restart: killed and started again on the same state. + async fn restart(&mut self) -> Duration { + self.kill(); + self.start_again().await + } + + /// Starts the same process again and waits for it to exit on its own, + /// for a start that is expected to be refused. Returns the exit code + /// and the lines it logged. + fn start_again_and_wait(&mut self) -> (i32, String) { + let before = std::fs::read_to_string(&self.log).unwrap_or_default().len(); + self.child = Self::spawn(&self.args, &self.log); + let started = Instant::now(); + loop { + if let Ok(Some(status)) = self.child.try_wait() { + let log = std::fs::read_to_string(&self.log).unwrap_or_default(); + return (status.code().unwrap_or(-1), log[before..].to_owned()); + } + assert!( + started.elapsed() < PATIENCE, + "the server kept running on a directory it should have refused" + ); + std::thread::sleep(BEAT); + } + } + + /// The data directory, for a scenario that reaches into it. + fn data_dir(&self) -> &Path { + self.data.as_deref().expect("a durable server") + } + + /// Where a raw socket reaches this server. + fn addr(&self) -> String { + format!("127.0.0.1:{}", self.port) + } + + /// What the server has logged so far. + fn log(&self) -> String { + std::fs::read_to_string(&self.log).unwrap_or_default() + } + + /// Stops the process, keeping everything on disk. + fn stop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } + + /// `bot.did.createAccount` with no name and no credential: the error + /// name the server refuses it by, which says whether creates are open. + async fn create_probe(&self) -> String { + let (_, body) = post_json( + &format!("{}/xrpc/bot.did.createAccount", self.origin), + None, + json!({ "kind": "agent" }), + ) + .await; + body["error"].as_str().unwrap_or_default().to_owned() + } + + /// Asks the server to read the human's repository now. + async fn nudge(&self) { + let status = http() + .post(format!("{}/xrpc/bot.did.pollOperatorClaim", self.origin)) + .send() + .await + .expect("the nudge is sent") + .status(); + assert_eq!(status.as_u16(), 202, "a nudge is accepted"); + } + + /// Waits until the server serves creates, which it does once its poll + /// has read the human's record naming it. The probe names no account + /// and carries no credential, so it is refused by name either way: + /// `ServerNotReady` before the claim stands, `NameRequired` after. + async fn wait_claimed(&self) { + let origin = self.origin.clone(); + wait_for("the server to find the human's claim", || { + let origin = origin.clone(); + async move { + let (_, body) = post_json( + &format!("{origin}/xrpc/bot.did.createAccount"), + None, + json!({ "kind": "agent" }), + ) + .await; + (body["error"] == "NameRequired").then_some(()) + } + }) + .await; + } + + /// Asks the server to read the human's repository now, and waits until + /// it has installed the allowances it found there. + async fn read_allowances(&self) { + let mark = self.log().len(); + self.nudge().await; + wait_for("the server to install the human's allowances", || async { + self.log()[mark..] + .contains("installed the operator's allowances") + .then_some(()) + }) + .await; + } +} + +impl Drop for Server { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + if std::thread::panicking() { + if let Ok(log) = std::fs::read_to_string(&self.log) { + let tail: Vec<&str> = log.lines().rev().take(30).collect(); + eprintln!("--- {} (last lines) ---", self.log.display()); + for line in tail.into_iter().rev() { + eprintln!("{line}"); + } + } + } + } +} + +// --------------------------------------------------------------------------- +// Public reads +// --------------------------------------------------------------------------- + +fn http() -> reqwest::Client { + didbot_http::client() +} + +async fn get_json(url: &str) -> Value { + http() + .get(url) + .send() + .await + .unwrap_or_else(|err| panic!("GET {url}: {err}")) + .json() + .await + .unwrap_or_else(|err| panic!("reading {url}: {err}")) +} + +async fn post_json(url: &str, bearer: Option<&str>, body: Value) -> (u16, Value) { + let mut request = http().post(url).json(&body); + if let Some(token) = bearer { + request = request.bearer_auth(token); + } + let response = request + .send() + .await + .unwrap_or_else(|err| panic!("POST {url}: {err}")); + let status = response.status().as_u16(); + let body = response.json().await.unwrap_or(Value::Null); + (status, body) +} + +/// `com.atproto.repo.getRecord` at `origin`, as anyone reads it. +async fn get_record(origin: &str, repo: &str, collection: &str, rkey: &str) -> (u16, Value) { + let response = http() + .get(format!("{origin}/xrpc/com.atproto.repo.getRecord")) + .query(&[("repo", repo), ("collection", collection), ("rkey", rkey)]) + .send() + .await + .expect("getRecord answers"); + let status = response.status().as_u16(); + (status, response.json().await.unwrap_or(Value::Null)) +} + +/// The record keys `repo` holds in `collection` at `origin`. +async fn list_record_keys(origin: &str, repo: &str, collection: &str) -> Vec { + let response = http() + .get(format!("{origin}/xrpc/com.atproto.repo.listRecords")) + .query(&[("repo", repo), ("collection", collection), ("limit", "100")]) + .send() + .await + .expect("listRecords answers"); + let body: Value = response.json().await.unwrap_or(Value::Null); + let mut keys: Vec = body["records"] + .as_array() + .map(|records| { + records + .iter() + .filter_map(|record| record["uri"].as_str()) + .filter_map(|uri| uri.rsplit('/').next()) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default(); + keys.sort(); + keys +} + +/// The `bot.did.registration/self` of a hosted account. +async fn registration(server: &Server, did: &str) -> Value { + let (status, body) = get_record(&server.origin, did, "bot.did.registration", "self").await; + assert_eq!(status, 200, "{did}'s registration reads: {body}"); + body["value"].clone() +} + +/// The `credential` of every `bot.did.credential` record a hosted account's +/// repository holds, each checked to name the account. +async fn credentials(server: &Server, did: &str) -> Vec { + let response = http() + .get(format!( + "{}/xrpc/com.atproto.repo.listRecords", + server.origin + )) + .query(&[ + ("repo", did), + ("collection", "bot.did.credential"), + ("limit", "100"), + ]) + .send() + .await + .expect("listRecords answers"); + assert_eq!(response.status().as_u16(), 200, "{did}'s credentials list"); + let body: Value = response.json().await.unwrap_or(Value::Null); + body["records"] + .as_array() + .map(|records| { + records + .iter() + .map(|record| { + assert_eq!(record["value"]["did"], did, "{record}"); + record["value"]["credential"].clone() + }) + .collect() + }) + .unwrap_or_default() +} + +/// Whether a credential is a key, parsed by this deployment's own readers. +fn is_key_credential(credential: &Value) -> bool { + credential["$type"] == "bot.did.credential#publicKey" + && credential["publicKeyMultibase"] + .as_str() + .is_some_and(|multibase| didbot_key::VerifyingKey::from_multibase(multibase).is_ok()) +} + +/// A hosted account's document, fetched at its own hostname the way a +/// resolver would. +async fn document(name: &str) -> Value { + get_json(&format!("http://{name}/.well-known/did.json")).await +} + +/// The record key `bot.did.operator` is written under for `name`: the +/// account's authority as its own DID spells it, port and all. +fn rkey_of(name: &str) -> String { + name.to_owned() +} + +fn did_of(name: &str) -> String { + format!("did:web:{}", name.replace(':', "%3A")) +} + +// --------------------------------------------------------------------------- +// Machines and the commands run on them +// --------------------------------------------------------------------------- + +/// One machine's worth of state: where its sessions, keys and account +/// tokens go. The human has one; so does every host, instance and run. +struct Machine { + home: PathBuf, + config: PathBuf, + state: PathBuf, +} + +impl Machine { + fn new(work: &Work, name: &str) -> Self { + let root = work.dir(name); + Self { + home: root.join("home"), + config: root.join("config"), + state: root.join("state"), + } + } + + /// Where `didbot operate` and `didbot register` keep an account's + /// session. + fn token_file(&self, name: &str) -> PathBuf { + self.state.join("accounts").join(format!("{name}.token")) + } + + /// A credential handed to this machine, in a file only it reads. + fn secret_file(&self, name: &str, contents: &str) -> PathBuf { + std::fs::create_dir_all(&self.home).expect("a home"); + let path = self.home.join(name); + std::fs::write(&path, contents).expect("the secret writes"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).expect("mode"); + } + path + } + + /// `didbot ` on this machine, as a person would type it. + fn command(&self, args: &[&str], env: &[(&str, &str)]) -> tokio::process::Command { + let path = format!( + "{}:{}", + bins().display(), + std::env::var("PATH").unwrap_or_default() + ); + let mut command = tokio::process::Command::new(bins().join("didbot")); + command + .args(args) + .env_remove("DIDBOT_PDS") + .env_remove("DIDBOT_SOCK") + .env_remove("DIDBOT_ACCOUNT_TOKEN") + .env_remove("DIDBOT_ACCOUNT_TOKEN_FILE") + .env("PATH", path) + .env("HOME", &self.home) + .env("XDG_CONFIG_HOME", &self.config) + .env("XDG_STATE_HOME", self.state.join("xdg")) + .env("DIDBOT_STATE", &self.state) + .env("NO_COLOR", "1") + .env("RUST_LOG", "warn") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + for (name, value) in env { + command.env(name, value); + } + command + } + + async fn didbot(&self, args: &[&str], env: &[(&str, &str)]) -> Ran { + let output = self.command(args, env).output().await.expect("didbot runs"); + Ran { + command: format!("didbot {}", args.join(" ")), + status: output.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + } + } +} + +/// What a command did. +struct Ran { + command: String, + status: i32, + stdout: String, + stderr: String, +} + +impl Ran { + /// Exit 0, and the `--json` line it printed. + fn ok(self) -> Value { + assert_eq!( + self.status, 0, + "`{}` exited {}\nstdout:\n{}\nstderr:\n{}", + self.command, self.status, self.stdout, self.stderr + ); + self.stdout + .lines() + .rev() + .find_map(|line| serde_json::from_str::(line).ok()) + .unwrap_or(Value::Null) + } + + /// Exit 1: understood, and refused. The refusal line is returned. + fn refused(self) -> String { + assert_eq!( + self.status, 1, + "`{}` exited {}\nstdout:\n{}\nstderr:\n{}", + self.command, self.status, self.stdout, self.stderr + ); + self.stderr + } +} + +// --------------------------------------------------------------------------- +// The daemon on a host +// --------------------------------------------------------------------------- + +/// A `didbot-agentd` holding one host's key, reached over its socket. +struct Daemon { + sock: PathBuf, + child: tokio::process::Child, + log: PathBuf, +} + +impl Daemon { + async fn start(machine: &Machine, server: &Server) -> Self { + let sock = machine.state.join("agent.sock"); + let log = machine.home.join("agentd.log"); + std::fs::create_dir_all(&machine.home).expect("a home"); + let log_file = std::fs::File::create(&log).expect("a log file"); + let child = tokio::process::Command::new(bins().join("didbot-agentd")) + .env("DIDBOT_PDS", &server.origin) + .env("DIDBOT_SOCK", &sock) + .env("DIDBOT_STATE", &machine.state) + .env("HOME", &machine.home) + .env("NO_COLOR", "1") + .stdin(Stdio::null()) + .stdout(Stdio::from(log_file.try_clone().expect("a handle"))) + .stderr(Stdio::from(log_file)) + .kill_on_drop(true) + .spawn() + .expect("didbot-agentd starts"); + let path = sock.clone(); + wait_for("the daemon's socket", || { + let path = path.clone(); + async move { path.exists().then_some(()) } + }) + .await; + Self { sock, child, log } + } + + /// One exchange over the socket: a line of JSON in, one out. + async fn ask(&self, message: Value) -> Value { + let mut stream = tokio::net::UnixStream::connect(&self.sock) + .await + .expect("the socket connects"); + let mut line = message.to_string(); + line.push('\n'); + stream + .write_all(line.as_bytes()) + .await + .expect("the request writes"); + let mut reader = BufReader::new(stream); + let mut answer = String::new(); + reader + .read_line(&mut answer) + .await + .expect("the daemon answers"); + serde_json::from_str(&answer).unwrap_or_else(|err| panic!("{answer:?}: {err}")) + } + + /// The hook's report that a context began, which is what makes the + /// daemon create an account for it. Returns the DID the context is + /// told it is, or the trouble the daemon named instead. + async fn began(&self, session: &str, context: &str) -> Result { + let answer = self + .ask(json!({ + "asks": "report", + "version": 2, + "observed": "began", + "session": session, + "context": context, + "asker": "scenarios", + })) + .await; + match (answer["identity"].as_str(), answer["trouble"].as_str()) { + (Some(did), _) => Ok(did.to_owned()), + (None, Some(trouble)) => Err(trouble.to_owned()), + (None, None) => Err(format!("neither an identity nor trouble: {answer}")), + } + } + + /// An agent for a fresh context, which the daemon creates beneath its + /// host. + async fn agent(&self, context: &str) -> String { + let session = format!("session-{context}"); + wait_for( + &format!("the daemon to create an agent for {context}"), + || { + let session = session.clone(); + async move { self.began(&session, context).await.ok() } + }, + ) + .await + } +} + +impl Drop for Daemon { + fn drop(&mut self) { + let _ = self.child.start_kill(); + if std::thread::panicking() { + if let Ok(log) = std::fs::read_to_string(&self.log) { + eprintln!("--- {} ---\n{log}", self.log.display()); + } + } + } +} + +// --------------------------------------------------------------------------- +// The stack: the human, an issuer, and servers the human has claimed +// --------------------------------------------------------------------------- + +struct Stack { + work: Work, + human: Human, + issuer: Issuer, + servers: Vec, + /// The human's own machine, holding their session and the tokens + /// `didbot operate` keeps. + laptop: Machine, +} + +impl Stack { + /// `labels.len()` servers, each on its own zone, each claimed by the + /// human with `didbot operate ` and found by the server's + /// poll. + async fn start(labels: &[&str]) -> Self { + Self::start_with(labels, Server::CONFIG, false).await + } + + /// The same, with every server on `config` and, when `durable`, on a + /// `--data` directory it can be restarted onto. + async fn start_with(labels: &[&str], config: &str, durable: bool) -> Self { + let started = Instant::now(); + let work = Work::new(labels[0]); + let human = Human::start().await; + let issuer = Issuer::start().await; + let mut servers = Vec::new(); + for label in labels { + let data = durable.then(|| work.dir(&format!("{label}-data"))); + servers.push(Server::start_with(&work, label, &human, config, free_port(), data).await); + } + let laptop = Machine::new(&work, "human"); + + // The grant the human's sign-in would have left: the operator + // collection, and one service-auth call at each server. + let mut scope = admit_scope(&servers[0].did); + for server in &servers[1..] { + let rpc = admit_scope(&server.did); + let rpc = rpc.rsplit(' ').next().expect("an rpc atom"); + scope.push(' '); + scope.push_str(rpc); + } + human.seed_session(&laptop.config, &scope).await; + + let stack = Self { + work, + human, + issuer, + servers, + laptop, + }; + for server in &stack.servers { + let claimed = stack + .laptop + .didbot(&["operate", &server.host, stack.human.did(), "--json"], &[]) + .await + .ok(); + assert_eq!(claimed["subject"], server.did, "{claimed}"); + assert_eq!( + claimed["record"], + format!( + "at://{}/bot.did.operator/{}", + stack.human.did(), + rkey_of(&server.host) + ) + ); + } + let claimed = started.elapsed(); + for server in &stack.servers { + server.nudge().await; + } + for server in &stack.servers { + server.wait_claimed().await; + } + eprintln!( + "{}: up and claimed in {:?} (the poll found the claim after {:?})", + labels.join("+"), + claimed, + started.elapsed() - claimed + ); + stack + } + + fn server(&self) -> &Server { + &self.servers[0] + } + + /// `didbot operate --check ` from a machine holding nothing: the + /// walk a stranger runs. Returns the hops on success and the refusal + /// line otherwise. + async fn check(&self, name: &str) -> Result, String> { + let stranger = Machine::new(&self.work, "stranger"); + let ran = stranger + .didbot(&["operate", "--check", name, "--json"], &[]) + .await; + match ran.status { + 0 => { + let answer = ran.ok(); + assert_eq!(answer["verified"], true, "{answer}"); + Ok(answer["hops"].as_array().cloned().unwrap_or_default()) + } + _ => Err(ran.refused()), + } + } + + /// The parent each hop of a verified walk from `name` names, bottom + /// up, without the last hop: the server's own, which every walk ends + /// with, naming the human. + async fn walk(&self, name: &str) -> Vec { + let hops = self + .check(name) + .await + .unwrap_or_else(|refusal| panic!("--check {name} refused: {refusal}")); + let (server, edges) = hops.split_last().expect("a walk has hops"); + assert_eq!(server["kind"], "server", "{server}"); + assert_eq!(server["parent"], self.human.did(), "{server}"); + assert!( + self.servers.iter().any(|s| s.did == server["account"]), + "{server}" + ); + edges + .iter() + .map(|hop| hop["parent"].as_str().unwrap_or_default().to_owned()) + .collect() + } + + /// `didbot register host ` on `machine` and `didbot operate + /// --fingerprint --creates agent` on the human's, in the order + /// a person runs them: the fingerprint printed on the host is what the + /// human types. Returns once the server has read the allowance. + async fn register_host(&self, machine: &Machine, server: &Server, name: &str) -> String { + let mut register = machine + .command( + &[ + "register", + "host", + name, + "--server", + &server.host, + "--timeout", + "60", + "--json", + ], + &[], + ) + .spawn() + .expect("didbot register starts"); + let stdout = register.stdout.take().expect("stdout is piped"); + let mut stderr = register.stderr.take().expect("stderr is piped"); + let mut lines = BufReader::new(stdout).lines(); + let mut printed = Vec::new(); + let mut fingerprint = None; + while let Some(line) = lines.next_line().await.expect("register prints") { + if let Some(rest) = line.split("--fingerprint ").nth(1) { + fingerprint = Some(rest.trim().to_owned()); + } + printed.push(line); + if fingerprint.is_some() { + break; + } + } + let fingerprint = fingerprint + .unwrap_or_else(|| panic!("register printed no fingerprint:\n{}", printed.join("\n"))); + assert!(fingerprint.starts_with("SHA256:"), "{fingerprint}"); + + let admitted = self + .laptop + .didbot( + &[ + "operate", + name, + self.human.did(), + "--fingerprint", + &fingerprint, + "--creates", + "agent", + "--json", + ], + &[], + ) + .await + .ok(); + assert_eq!(admitted["kind"], "host", "{admitted}"); + assert_eq!(admitted["did"], did_of(name), "{admitted}"); + assert_eq!(admitted["operator"], self.human.did(), "{admitted}"); + assert_eq!( + admitted["creates"], + json!([{ "scope": "subject", "kinds": ["agent"] }]), + "{admitted}" + ); + let record = self + .human + .record("bot.did.operator", &rkey_of(name)) + .expect("the human's record for the host"); + assert_eq!(record["subject"], did_of(name), "{record}"); + assert_eq!(record["creates"], admitted["creates"], "{record}"); + // The parked key is now the host's own credential record. + let held = credentials(server, &did_of(name)).await; + assert_eq!(held.len(), 1, "{held:?}"); + assert!( + is_key_credential(&held[0]), + "the host logs in with the key it parked: {held:?}" + ); + server.read_allowances().await; + + while let Some(line) = lines.next_line().await.expect("register prints") { + printed.push(line); + } + let status = register.wait().await.expect("register exits"); + let mut complaint = String::new(); + let _ = stderr.read_to_string(&mut complaint).await; + assert!( + status.success(), + "didbot register exited {status}:\n{}\nstderr:\n{complaint}", + printed.join("\n") + ); + let last: Value = printed + .iter() + .rev() + .find_map(|line| serde_json::from_str(line).ok()) + .expect("register prints its answer as JSON"); + assert_eq!(last["did"], did_of(name), "{last}"); + assert!( + machine.token_file(name).is_file(), + "the host's session is kept where didbot-oauth reads it" + ); + did_of(name) + } +} + +// --------------------------------------------------------------------------- +// One agent on my laptop, by hand +// --------------------------------------------------------------------------- + +/// `didbot operate kestrel. --kind agent`: one record in the human's +/// repository, one create proved by their PDS's service-auth token, and a +/// walk of one edge. +#[tokio::test(flavor = "multi_thread")] +async fn an_agent_by_hand() { + let stack = Stack::start(&["hand"]).await; + let server = stack.server(); + let name = format!("kestrel.{}", server.host); + let did = did_of(&name); + + let admitted = stack + .laptop + .didbot( + &[ + "operate", + &name, + stack.human.did(), + "--kind", + "agent", + "--json", + ], + &[], + ) + .await + .ok(); + assert_eq!(admitted["did"], did, "{admitted}"); + assert_eq!(admitted["kind"], "agent"); + assert_eq!(admitted["operator"], stack.human.did()); + assert_eq!(admitted["creates"], json!([]), "{admitted}"); + + // The agent's registration and document name the human; the human's + // repository holds the record naming the agent, written with the + // session, and it names nothing to create. Nothing logs in as the + // agent but the session the server handed back. + let registration = registration(server, &did).await; + assert_eq!(registration["did"], did); + assert_eq!(registration["operator"], stack.human.did()); + assert_eq!(registration["kind"], "agent"); + assert!( + registration.get("lineage").is_none(), + "admitted by the human directly: {registration}" + ); + assert_eq!(document(&name).await["operator"], stack.human.did()); + let record = stack + .human + .record("bot.did.operator", &rkey_of(&name)) + .expect("the human's record for the agent"); + assert!(record.get("creates").is_none(), "{record}"); + assert_eq!(credentials(server, &did).await, Vec::::new()); + assert_eq!( + stack.human.record_keys("bot.did.operator").await, + vec![rkey_of(&server.host), rkey_of(&name)] + ); + assert!( + stack + .human + .write_auth() + .iter() + .all(|scheme| scheme == "DPoP"), + "every write at the human's PDS carried the session: {:?}", + stack.human.write_auth() + ); + + // A stranger walks it: one edge, ending at the human. + assert_eq!(stack.walk(&name).await, vec![stack.human.did().to_owned()]); + + // The session the server returned writes as the agent. + let token = std::fs::read_to_string(stack.laptop.token_file(&name)).expect("the token file"); + let (status, written) = post_json( + &format!("{}/xrpc/com.atproto.repo.createRecord", server.origin), + Some(token.trim()), + json!({ + "repo": did, + "collection": "com.example.thing", + "record": { "text": "hello", "emoji": "\u{1f9ee}", "createdAt": now_rfc3339() }, + }), + ) + .await; + assert_eq!(status, 200, "{written}"); +} + +// --------------------------------------------------------------------------- +// One agent host on my laptop, automatic +// --------------------------------------------------------------------------- + +/// A host the daemon stands up on this stack: registered from its own +/// machine, admitted from the human's, its daemon running. +struct Laptop { + machine: Machine, + name: String, + did: String, + daemon: Daemon, +} + +impl Laptop { + async fn stand_up(stack: &Stack, server: &Server, label: &str) -> Self { + let machine = Machine::new(&stack.work, label); + let name = format!("{label}.{}", server.host); + let did = stack.register_host(&machine, server, &name).await; + let daemon = Daemon::start(&machine, server).await; + Self { + machine, + name, + did, + daemon, + } + } +} + +/// `didbot register host` here, `didbot operate --fingerprint` there, and +/// the daemon creating an agent beneath the host with a JWT its key signs: +/// the host's repository gains the record, the human's gains nothing, and +/// the walk climbs two edges. +#[tokio::test(flavor = "multi_thread")] +async fn a_laptop_host_and_its_daemon() { + let stack = Stack::start(&["laptop"]).await; + let server = stack.server(); + let laptop = Laptop::stand_up(&stack, server, "laptop").await; + + let held = credentials(server, &laptop.did).await; + assert!( + held.iter().any(is_key_credential), + "the host's own key is its credential record: {held:?}" + ); + assert_eq!( + registration(server, &laptop.did).await["operator"], + stack.human.did() + ); + + let agent = laptop.daemon.agent("ctx-1").await; + let agent_name = agent.trim_start_matches("did:web:").replace("%3A", ":"); + assert!( + agent_name.ends_with(&server.host), + "the agent is under the server: {agent}" + ); + let registration = registration(server, &agent).await; + assert_eq!( + registration["operator"], + stack.human.did(), + "the human, not the host" + ); + assert_eq!(registration["lineage"], json!([laptop.did])); + assert_eq!(registration["kind"], "agent"); + assert_eq!(document(&agent_name).await["operator"], stack.human.did()); + + let (status, record) = get_record( + &server.origin, + &laptop.did, + "bot.did.operator", + &rkey_of(&agent_name), + ) + .await; + assert_eq!( + status, 200, + "the host's repository names the agent: {record}" + ); + assert_eq!(record["value"]["subject"], agent); + let mut expected = vec![rkey_of(&server.host), rkey_of(&laptop.name)]; + expected.sort(); + assert_eq!( + stack.human.record_keys("bot.did.operator").await, + expected, + "nothing was written to the human's repository for the agent" + ); + + assert_eq!( + stack.walk(&agent_name).await, + vec![laptop.did.clone(), stack.human.did().to_owned()] + ); +} + +/// Deleting the human's record for the host stops the host and everything +/// beneath it at the next poll: the walk fails at once, and the server +/// refuses their creates and writes once it has read the deletion. +#[tokio::test(flavor = "multi_thread")] +async fn deleting_the_hosts_record_stops_its_subtree() { + let stack = Stack::start(&["revoke"]).await; + let server = stack.server(); + let laptop = Laptop::stand_up(&stack, server, "laptop").await; + let agent = laptop.daemon.agent("ctx-1").await; + let agent_name = agent.trim_start_matches("did:web:").replace("%3A", ":"); + + // A second agent, admitted with the host's own session so this test + // holds a token for something beneath the host. + let host_token = laptop.machine.token_file(&laptop.name); + let by_hand = format!("byhand.{}", server.host); + let created = laptop + .machine + .didbot( + &[ + "register", + "agent", + &by_hand, + "--under", + &laptop.name, + "--server", + &server.host, + "--token-file", + host_token.to_str().expect("utf-8"), + "--json", + ], + &[], + ) + .await + .ok(); + assert_eq!(created["did"], did_of(&by_hand), "{created}"); + assert_eq!(stack.walk(&by_hand).await.len(), 2); + + let thing = + || json!({ "text": "still here", "emoji": "\u{1f9ee}", "createdAt": now_rfc3339() }); + let write_as = |name: &str, machine: &Machine| { + let token = std::fs::read_to_string(machine.token_file(name)).expect("a token"); + let url = format!("{}/xrpc/com.atproto.repo.createRecord", server.origin); + let body = + json!({ "repo": did_of(name), "collection": "com.example.thing", "record": thing() }); + async move { post_json(&url, Some(token.trim()), body).await } + }; + let (status, _) = write_as(&by_hand, &laptop.machine).await; + assert_eq!(status, 200, "the agent writes while its host stands"); + + // The human deletes their record for the host. + stack + .human + .delete("bot.did.operator", &rkey_of(&laptop.name)); + assert_eq!( + stack.human.record_keys("bot.did.operator").await, + vec![rkey_of(&server.host)] + ); + + // The walk reads the human's repository directly and fails at once, + // from every account beneath the host. + for name in [&laptop.name, &agent_name, &by_hand] { + let refusal = stack + .check(name) + .await + .expect_err("the walk fails once the record is gone"); + assert!(refusal.contains("not verified"), "{name}: {refusal}"); + } + + // The server acts at its next poll: the host and its agents are + // refused, by name. + server.nudge().await; + let refused = wait_for("the server to stop the host's subtree", || async { + let (status, body) = write_as(&by_hand, &laptop.machine).await; + (status == 403).then_some(body) + }) + .await; + assert_eq!(refused["error"], "AccountNotWritable", "{refused}"); + let (status, body) = write_as(&laptop.name, &laptop.machine).await; + assert_eq!(status, 403, "the host itself is stopped: {body}"); + assert_eq!(body["error"], "AccountNotWritable"); + + // The daemon can create nothing more beneath the host. + let trouble = laptop + .daemon + .began("session-after", "ctx-after") + .await + .expect_err("the daemon's create is refused"); + assert!( + trouble.contains("refused") || trouble.contains("403"), + "the daemon names the refusal: {trouble}" + ); + let refusal = laptop + .machine + .didbot( + &[ + "register", + "agent", + &format!("late.{}", server.host), + "--under", + &laptop.name, + "--server", + &server.host, + "--token-file", + host_token.to_str().expect("utf-8"), + ], + &[], + ) + .await + .refused(); + assert!( + refusal.contains("didbot register:"), + "the refusal is one line on stderr: {refusal}" + ); +} + +// --------------------------------------------------------------------------- +// Two servers, hosts I registered +// --------------------------------------------------------------------------- + +/// Two deployments on two zones, one host on each: the human's repository +/// holds four records, and each host verifies on its own server. +#[tokio::test(flavor = "multi_thread")] +async fn two_servers_two_hosts() { + let stack = Stack::start(&["pds-a", "pds-b"]).await; + let (a, b) = (&stack.servers[0], &stack.servers[1]); + assert_ne!(a.did, b.did); + + let h1 = Laptop::stand_up(&stack, a, "h1").await; + let h2 = Laptop::stand_up(&stack, b, "h2").await; + + let mut expected = vec![ + rkey_of(&a.host), + rkey_of(&b.host), + rkey_of(&h1.name), + rkey_of(&h2.name), + ]; + expected.sort(); + assert_eq!( + stack.human.record_keys("bot.did.operator").await, + expected, + "two servers and one host on each" + ); + + for (server, host) in [(a, &h1), (b, &h2)] { + assert_eq!( + registration(server, &host.did).await["operator"], + stack.human.did() + ); + assert_eq!( + stack.walk(&host.name).await, + vec![stack.human.did().to_owned()] + ); + let agent = host.daemon.agent("ctx-1").await; + let agent_name = agent.trim_start_matches("did:web:").replace("%3A", ":"); + assert!( + agent_name.ends_with(&server.host), + "{agent} is on {}", + server.host + ); + assert_eq!( + stack.walk(&agent_name).await, + vec![host.did.clone(), stack.human.did().to_owned()] + ); + } +} + +// --------------------------------------------------------------------------- +// Any agents from any host in an autoscaling pool +// --------------------------------------------------------------------------- + +/// A service that is an OpenID Connect identity, admitted with `--creates +/// host --creates-beneath agent`, admits an instance by the platform's +/// token; the instance's daemon then creates agents. A token with the +/// wrong claim admits nothing, and a token is spent once. +#[tokio::test(flavor = "multi_thread")] +async fn an_autoscaling_pool() { + let stack = Stack::start(&["pool"]).await; + let server = stack.server(); + let web = format!("web.{}", server.host); + let web_did = did_of(&web); + + let admitted = stack + .laptop + .didbot( + &[ + "operate", + &web, + stack.human.did(), + "--kind", + "service", + "--oidc", + &stack.issuer.base, + "email=web@pool", + "--creates", + "host", + "--creates-beneath", + "agent", + "--json", + ], + &[], + ) + .await + .ok(); + assert_eq!(admitted["did"], web_did, "{admitted}"); + let record = stack + .human + .record("bot.did.operator", &rkey_of(&web)) + .expect("the human's record for the service"); + assert_eq!( + record["creates"], + json!([ + { "scope": "subject", "kinds": ["host"] }, + { "scope": "descendants", "kinds": ["agent"] }, + ]), + "{record}" + ); + server.read_allowances().await; + assert_eq!(document(&web).await["operator"], stack.human.did()); + assert_eq!( + credentials(server, &web_did).await, + vec![json!({ + "$type": "bot.did.credential#oidcIdentity", + "issuer": stack.issuer.base, + "claims": [{ "name": "email", "value": "web@pool" }], + })], + "the identity is the service's own credential record" + ); + + let mut instances = Vec::new(); + for i in 1..=2 { + let machine = Machine::new(&stack.work, &format!("i-{i}")); + let name = format!("i-{i}.{}", server.host); + let token = stack + .issuer + .id_token(&server.did, json!({ "email": "web@pool" })); + let token_file = machine.secret_file("id-token", &token); + let registered = machine + .didbot( + &[ + "register", + "host", + &name, + "--under", + &web, + "--server", + &server.host, + "--token-file", + token_file.to_str().expect("utf-8"), + "--json", + ], + &[], + ) + .await + .ok(); + assert_eq!(registered["did"], did_of(&name), "{registered}"); + let registered = registration(server, &did_of(&name)).await; + assert_eq!(registered["operator"], stack.human.did()); + assert_eq!(registered["lineage"], json!([web_did])); + let (status, record) = get_record( + &server.origin, + &web_did, + "bot.did.operator", + &rkey_of(&name), + ) + .await; + assert_eq!( + status, 200, + "the service's repository names the host: {record}" + ); + assert_eq!(record["value"]["subject"], did_of(&name), "{record}"); + let held = credentials(server, &did_of(&name)).await; + assert!( + held.len() == 1 && is_key_credential(&held[0]), + "the key the instance registered with: {held:?}" + ); + + let daemon = Daemon::start(&machine, server).await; + // A harness's context id names the agent, so each instance's + // contexts carry the instance in theirs. + let agent = daemon.agent(&format!("i{i}-ctx-1")).await; + let agent_name = agent.trim_start_matches("did:web:").replace("%3A", ":"); + let registered = registration(server, &agent).await; + assert_eq!(registered["operator"], stack.human.did()); + assert_eq!( + registered["lineage"], + json!([web_did, did_of(&name)]), + "the pool, then the host" + ); + assert_eq!( + stack.walk(&agent_name).await, + vec![did_of(&name), web_did.clone(), stack.human.did().to_owned()], + "agent -> host -> service -> human" + ); + instances.push((machine, name, token_file, daemon)); + } + assert_eq!( + stack.human.record_keys("bot.did.operator").await, + vec![rkey_of(&server.host), rkey_of(&web)], + "the human wrote nothing for the instances" + ); + + // A token whose named claim is somebody else's admits nothing. + let stranger = Machine::new(&stack.work, "i-3"); + let wrong = stack + .issuer + .id_token(&server.did, json!({ "email": "other@pool" })); + let wrong_file = stranger.secret_file("id-token", &wrong); + let refusal = stranger + .didbot( + &[ + "register", + "host", + &format!("i-3.{}", server.host), + "--under", + &web, + "--server", + &server.host, + "--token-file", + wrong_file.to_str().expect("utf-8"), + ], + &[], + ) + .await + .refused(); + assert!( + refusal.contains("didbot register:") && refusal.contains("refused"), + "{refusal}" + ); + + // The first instance's token, presented again, is a replay. + let (_, _, spent, _) = &instances[0]; + let replay = Machine::new(&stack.work, "i-4"); + let refusal = replay + .didbot( + &[ + "register", + "host", + &format!("i-4.{}", server.host), + "--under", + &web, + "--server", + &server.host, + "--token-file", + spent.to_str().expect("utf-8"), + ], + &[], + ) + .await + .refused(); + assert!(refusal.contains("refused"), "{refusal}"); + assert_eq!( + list_record_keys(&server.origin, &web_did, "bot.did.operator").await, + vec![rkey_of(&instances[0].1), rkey_of(&instances[1].1)], + "the service's repository names its two instances and nothing else" + ); +} + +// --------------------------------------------------------------------------- +// Any run of one OIDC pipeline +// --------------------------------------------------------------------------- + +/// Every run of the pipeline signs in with its own ID token and is the +/// pipeline; two runs at once hold two sessions; a run's agent is created +/// beneath the pipeline; a token is spent once. +#[tokio::test(flavor = "multi_thread")] +async fn a_pipelines_runs() { + let stack = Stack::start(&["pipe"]).await; + let server = stack.server(); + let deploy = format!("deploy.{}", server.host); + let deploy_did = did_of(&deploy); + + let admitted = stack + .laptop + .didbot( + &[ + "operate", + &deploy, + stack.human.did(), + "--kind", + "pipeline", + "--oidc", + &stack.issuer.base, + "repository_id=456789", + "--creates", + "agent", + "--json", + ], + &[], + ) + .await + .ok(); + assert_eq!(admitted["did"], deploy_did, "{admitted}"); + let record = stack + .human + .record("bot.did.operator", &rkey_of(&deploy)) + .expect("the human's record for the pipeline"); + assert_eq!( + record["creates"], + json!([{ "scope": "subject", "kinds": ["agent"] }]), + "{record}" + ); + assert_eq!( + credentials(server, &deploy_did).await, + vec![json!({ + "$type": "bot.did.credential#oidcIdentity", + "issuer": stack.issuer.base, + "claims": [{ "name": "repository_id", "value": "456789" }], + })], + "the identity is the pipeline's own credential record" + ); + server.read_allowances().await; + + // Two runs, each with the token its platform minted, each a session as + // the pipeline, at the same time. + let run_a = Machine::new(&stack.work, "run-a"); + let run_b = Machine::new(&stack.work, "run-b"); + let token_for = |run: &Machine| { + run.secret_file( + "id-token", + &stack + .issuer + .id_token(&server.did, json!({ "repository_id": "456789" })), + ) + }; + let (token_a, token_b) = (token_for(&run_a), token_for(&run_b)); + let (token_a, token_b) = ( + token_a.to_str().expect("utf-8"), + token_b.to_str().expect("utf-8"), + ); + let pds = [("DIDBOT_PDS", server.origin.as_str())]; + let args_a = ["oauth", "pending", "--token-file", token_a]; + let args_b = ["oauth", "pending", "--token-file", token_b]; + let (a, b) = tokio::join!(run_a.didbot(&args_a, &pds), run_b.didbot(&args_b, &pds)); + a.ok(); + b.ok(); + + // A session each, minted through the same route the command uses, so + // this test holds one: both stand at once, and one used after the + // other was minted still writes as the pipeline. + let session = |token: String| { + let url = format!("{}/xrpc/bot.did.createSession", server.origin); + let deploy_did = deploy_did.clone(); + async move { + let (status, body) = post_json(&url, Some(&token), json!({})).await; + assert_eq!(status, 200, "{body}"); + assert_eq!(body["did"], deploy_did, "{body}"); + body["accountToken"].as_str().expect("a session").to_owned() + } + }; + let fresh_a = stack + .issuer + .id_token(&server.did, json!({ "repository_id": "456789" })); + let session_a = session(fresh_a.clone()).await; + let session_b = session( + stack + .issuer + .id_token(&server.did, json!({ "repository_id": "456789" })), + ) + .await; + assert_ne!(session_a, session_b); + let (status, body) = post_json( + &format!("{}/xrpc/com.atproto.repo.createRecord", server.origin), + Some(&session_a), + json!({ + "repo": deploy_did, + "collection": "com.example.thing", + "record": { "text": "run a", "emoji": "\u{1f9ee}", "createdAt": now_rfc3339() }, + }), + ) + .await; + assert_eq!(status, 200, "the first session still writes: {body}"); + + // The token behind session A, presented again, is a replay. + let (status, body) = post_json( + &format!("{}/xrpc/bot.did.createSession", server.origin), + Some(&fresh_a), + json!({}), + ) + .await; + assert_eq!(status, 401, "a spent token: {body}"); + + // Run A creates its agent with its session. + let session_file = run_a.secret_file("session", &session_a); + let agent = format!("run-a.{}", server.host); + let created = run_a + .didbot( + &[ + "register", + "agent", + &agent, + "--under", + &deploy, + "--server", + &server.host, + "--token-file", + session_file.to_str().expect("utf-8"), + "--json", + ], + &[], + ) + .await + .ok(); + assert_eq!(created["did"], did_of(&agent), "{created}"); + let registration = registration(server, &did_of(&agent)).await; + assert_eq!(registration["operator"], stack.human.did()); + assert_eq!(registration["lineage"], json!([deploy_did])); + assert_eq!( + stack.walk(&agent).await, + vec![deploy_did.clone(), stack.human.did().to_owned()], + "agent -> pipeline -> human" + ); + assert_eq!( + stack.human.record_keys("bot.did.operator").await, + vec![rkey_of(&deploy), rkey_of(&server.host)] + ); +} + +// --------------------------------------------------------------------------- +// The allowance: the human says what a host creates +// --------------------------------------------------------------------------- + +/// The laptop was admitted with `--creates agent`: its attempt to create a +/// service is refused by name, the refusal reaches the command line's +/// stderr, and the agent its allowance names still lands. A host admitted +/// with no allowance creates nothing. +#[tokio::test(flavor = "multi_thread")] +async fn a_host_creates_only_the_kinds_its_allowance_names() { + let stack = Stack::start(&["allow"]).await; + let server = stack.server(); + let laptop = Laptop::stand_up(&stack, server, "laptop").await; + + let host_token = laptop.machine.token_file(&laptop.name); + let refusal = laptop + .machine + .didbot( + &[ + "register", + "service", + &format!("svc.{}", server.host), + "--under", + &laptop.name, + "--server", + &server.host, + "--token-file", + host_token.to_str().expect("utf-8"), + ], + &[], + ) + .await + .refused(); + assert!( + refusal.contains("NotAllowedToCreate"), + "the refusal names the allowance the human did not write: {refusal}" + ); + let agent = laptop.daemon.agent("ctx-1").await; + let agent_name = agent.trim_start_matches("did:web:").replace("%3A", ":"); + assert_eq!(stack.walk(&agent_name).await.len(), 2); + + let bare = format!("bare.{}", server.host); + stack + .laptop + .didbot( + &[ + "operate", + &bare, + stack.human.did(), + "--kind", + "host", + "--json", + ], + &[], + ) + .await + .ok(); + server.read_allowances().await; + let refusal = register_agent( + &stack.laptop, + server, + &format!("orphan.{}", server.host), + &bare, + &stack.laptop.token_file(&bare), + ) + .await + .refused(); + assert!(refusal.contains("NotAllowedToCreate"), "{refusal}"); +} + +// --------------------------------------------------------------------------- +// A server somebody else runs, answering whatever it likes +// --------------------------------------------------------------------------- + +/// A server the stranger's walk may be led to: it serves a `did:web` +/// document for any host it is asked for, `describeServer`, and whatever +/// records it was given, and remembers every URL it was asked. +struct Evil { + origin: String, + host: String, + did: String, + docs: Arc>>, + records: Arc>, + asked: Arc>>, +} + +/// Records by repository, collection and key. +type Held = HashMap<(String, String, String), Value>; + +impl Evil { + async fn start(label: &str) -> Self { + let listener = tokio::net::TcpListener::bind("[::]:0").await.expect("bind"); + let port = listener.local_addr().expect("local addr").port(); + let host = format!("{label}.localhost:{port}"); + let did = did_of(&host); + let evil = Self { + origin: format!("http://{host}"), + host, + did, + docs: Arc::default(), + records: Arc::default(), + asked: Arc::default(), + }; + let (docs, records, asked, did) = ( + evil.docs.clone(), + evil.records.clone(), + evil.asked.clone(), + evil.did.clone(), + ); + let remember = { + let asked = asked.clone(); + move |uri: &axum::http::Uri| asked.lock().expect("asked").push(uri.to_string()) + }; + let doc_route = { + let remember = remember.clone(); + move |headers: HeaderMap, uri: axum::http::Uri| { + let docs = docs.clone(); + let remember = remember.clone(); + async move { + remember(&uri); + let host = headers + .get("host") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_owned(); + match docs.lock().expect("docs").get(&host) { + Some(doc) => Json(doc.clone()).into_response(), + None => StatusCode::NOT_FOUND.into_response(), + } + } + } + }; + let describe_route = { + let remember = remember.clone(); + move |uri: axum::http::Uri| { + let did = did.clone(); + let remember = remember.clone(); + async move { + remember(&uri); + Json(json!({ "did": did, "availableUserDomains": [] })) + } + } + }; + let record_route = { + let remember = remember.clone(); + move |Query(query): Query>, uri: axum::http::Uri| { + let records = records.clone(); + let remember = remember.clone(); + async move { + remember(&uri); + let key = ( + query.get("repo").cloned().unwrap_or_default(), + query.get("collection").cloned().unwrap_or_default(), + query.get("rkey").cloned().unwrap_or_default(), + ); + match records.lock().expect("records").get(&key) { + Some(value) => Json(json!({ + "uri": format!("at://{}/{}/{}", key.0, key.1, key.2), + "cid": "bafyevil", + "value": value, + })) + .into_response(), + None => ( + StatusCode::BAD_REQUEST, + Json(json!({ "error": "RecordNotFound", "message": key.2 })), + ) + .into_response(), + } + } + } + }; + let list_route = move |uri: axum::http::Uri| { + let remember = remember.clone(); + async move { + remember(&uri); + Json(json!({ "accounts": [] })) + } + }; + let router = Router::new() + .route("/.well-known/did.json", get(doc_route)) + .route( + "/xrpc/com.atproto.server.describeServer", + get(describe_route), + ) + .route("/xrpc/com.atproto.repo.getRecord", get(record_route)) + .route("/xrpc/bot.did.listAccounts", get(list_route)); + tokio::spawn(async move { + axum::serve(listener, router).await.ok(); + }); + evil + } + + /// A hosted account's document, naming this server as its PDS unless + /// `pds` says otherwise. + fn document(&self, name: &str, pds: Option<&str>) { + let did = did_of(name); + let service = |endpoint: &str| { + json!({ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": endpoint, + }) + }; + let services = match pds { + Some(other) => vec![service(other), service(&self.origin)], + None => vec![service(&self.origin)], + }; + self.docs.lock().expect("docs").insert( + name.to_owned(), + json!({ + "@context": ["https://www.w3.org/ns/did/v1"], + "id": did, + "verificationMethod": [], + "service": services, + }), + ); + } + + fn record(&self, repo: &str, collection: &str, rkey: &str, value: Value) { + self.records.lock().expect("records").insert( + (repo.to_owned(), collection.to_owned(), rkey.to_owned()), + value, + ); + } + + /// An account here, registered as operated by `operator`. + fn account(&self, name: &str, operator: &str) { + self.document(name, None); + self.record( + &did_of(name), + "bot.did.registration", + "self", + json!({ "$type": "bot.did.registration", "did": did_of(name), "operator": operator, "kind": "agent" }), + ); + } + + /// `operator`'s record naming `name`, in a repository this server + /// claims to hold. + fn vouch(&self, operator: &str, name: &str) { + self.record( + operator, + "bot.did.operator", + &rkey_of(name), + json!({ "$type": "bot.did.operator", "subject": did_of(name), "createdAt": now_rfc3339() }), + ); + } + + fn asked(&self) -> Vec { + self.asked.lock().expect("asked").clone() + } +} + +/// A port that accepts every connection and answers none of them. +async fn silent_port() -> u16 { + let listener = tokio::net::TcpListener::bind("[::]:0").await.expect("bind"); + let port = listener.local_addr().expect("local addr").port(); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((stream, _)) = listener.accept().await { + held.push(stream); + } + }); + port +} + +/// `didbot operate --check ` from a machine holding nothing, against +/// no stack in particular. +async fn stranger_check(work: &Work, args: &[&str]) -> Ran { + let mut full = vec!["operate", "--check"]; + full.extend_from_slice(args); + full.push("--json"); + Machine::new(work, "stranger").didbot(&full, &[]).await +} + +// --------------------------------------------------------------------------- +// Fooling the stranger +// --------------------------------------------------------------------------- + +/// A server that is not the human's serves accounts whose registrations +/// name the human, a forged copy of the human's repository, loops, and +/// operators nobody can reach. The stranger's walk passes none of them. +#[tokio::test(flavor = "multi_thread")] +async fn a_forged_tree_does_not_verify() { + let work = Work::new("forged"); + let human = Human::start().await; + let evil = Evil::start("evil").await; + let stranger = |args: Vec| { + let work = &work; + async move { + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + stranger_check(work, &args).await + } + }; + + // 1. An account whose registration names the human, with the evil + // server holding a forged copy of the human's repository that + // vouches for it. The human's real repository holds nothing. + let forged = format!("forged.{}", evil.host); + evil.account(&forged, human.did()); + evil.vouch(human.did(), &forged); + let refusal = stranger(vec![forged.clone()]).await.refused(); + assert!( + refusal.contains("no such record") && refusal.contains(human.did()), + "the walk reads the human's own repository, not the forged one: {refusal}" + ); + assert!( + !evil + .asked() + .iter() + .any(|url| url.contains("getRecord") && url.contains(&human.did().replace(':', "%3A"))), + "the forged copy of the human's repository was never read: {:?}", + evil.asked() + ); + + // 2. Two accounts that operate each other. + let a = format!("a.{}", evil.host); + let b = format!("b.{}", evil.host); + evil.account(&a, &did_of(&b)); + evil.account(&b, &did_of(&a)); + evil.vouch(&did_of(&b), &a); + evil.vouch(&did_of(&a), &b); + let refusal = stranger(vec![a.clone()]).await.refused(); + assert!( + refusal.contains("too deep") && refusal.contains("after 3 edges"), + "a loop is refused at the depth bound: {refusal}" + ); + + // 3. An account that operates itself. + let own = format!("own.{}", evil.host); + evil.account(&own, &did_of(&own)); + evil.vouch(&did_of(&own), &own); + let refusal = stranger(vec![own.clone()]).await.refused(); + assert!(refusal.contains("too deep"), "{refusal}"); + + // 4. An operator on a server nobody answers at, and one that answers + // nothing. + let orphan = format!("orphan.{}", evil.host); + evil.account(&orphan, "did:web:nowhere.localhost%3A1"); + let refusal = stranger(vec![orphan.clone()]).await.refused(); + assert!( + refusal.contains("could not resolve did:web:nowhere.localhost%3A1"), + "{refusal}" + ); + let silent = silent_port().await; + let stalled = format!("stalled.{}", evil.host); + evil.account(&stalled, &format!("did:web:silent.localhost%3A{silent}")); + let started = Instant::now(); + let refusal = stranger(vec![stalled.clone()]).await.refused(); + assert!( + refusal.contains("could not resolve did:web:silent.localhost"), + "{refusal}" + ); + assert!( + started.elapsed() < Duration::from_secs(30), + "a silent operator is given up on: {:?}", + started.elapsed() + ); + + // 5. A document with two `#atproto_pds` entries: the first is where + // the repository is read, so a second one changes nothing. + let twin = format!("twin.{}", evil.host); + evil.document(&twin, Some("http://elsewhere.localhost:1")); + evil.record( + &did_of(&twin), + "bot.did.registration", + "self", + json!({ "operator": human.did(), "kind": "agent" }), + ); + let refusal = stranger(vec![twin.clone()]).await.refused(); + assert!( + refusal.contains("elsewhere.localhost:1") || refusal.contains("bot.did.registration/self"), + "the first endpoint is the one read: {refusal}" + ); + + // Nothing above asked the server who it hosts. + assert!( + !evil.asked().iter().any(|url| url.contains("listAccounts")), + "the walk never asks listAccounts: {:?}", + evil.asked() + ); +} + +/// The honest tree, read by a stranger who types the name in every way +/// a person might, and after the human withdraws from the server. +#[tokio::test(flavor = "multi_thread")] +async fn a_stranger_reads_the_honest_tree() { + let stack = Stack::start(&["read"]).await; + let server = stack.server(); + let kestrel = format!("kestrel.{}", server.host); + let admitted = stack + .laptop + .didbot( + &[ + "operate", + &kestrel, + stack.human.did(), + "--kind", + "agent", + "--creates", + "--json", + ], + &[], + ) + .await + .ok(); + assert_eq!(admitted["did"], did_of(&kestrel), "{admitted}"); + assert_eq!( + admitted["creates"], + json!([{ "scope": "subject" }]), + "a bare --creates is any kind: {admitted}" + ); + server.read_allowances().await; + + // A second edge: kestrel's own session creates an account beneath it. + let token = std::fs::read_to_string(stack.laptop.token_file(&kestrel)).expect("a token"); + let pup = format!("pup.{}", server.host); + let (status, created) = post_json( + &format!("{}/xrpc/bot.did.createAccount", server.origin), + Some(token.trim()), + json!({ "name": pup, "kind": "agent" }), + ) + .await; + assert_eq!(status, 200, "{created}"); + assert_eq!(created["operator"], stack.human.did(), "{created}"); + assert_eq!( + stack.walk(&pup).await, + vec![did_of(&kestrel), stack.human.did().to_owned()] + ); + + // Spelled differently, the name is a different did:web, and nothing + // answers for it. + let upper = pup.to_uppercase(); + let refusal = stranger_check(&stack.work, &[&upper]).await.refused(); + eprintln!("uppercase: {refusal}"); + let encoded = pup.replace(':', "%3A"); + let ran = stranger_check(&stack.work, &[&encoded]).await; + eprintln!( + "percent-encoded: status {} stderr {}", + ran.status, ran.stderr + ); + assert_ne!(ran.status, 0, "a percent-encoded name is not the account"); + + // `--server` spelled unlike the endpoint: does the walk still climb? + let stranger = Machine::new(&stack.work, "stranger"); + let upper_server = server.host.to_uppercase(); + let misspelled = |name: String| { + let stranger = &stranger; + let upper_server = upper_server.clone(); + async move { + stranger + .didbot( + &[ + "operate", + "--check", + &name, + "--server", + &upper_server, + "--json", + ], + &[], + ) + .await + } + }; + let before = misspelled(pup.clone()).await; + eprintln!( + "--server uppercase: status {} stdout {} stderr {}", + before.status, before.stdout, before.stderr + ); + + // The human withdraws from kestrel: the honest walk from pup fails at + // the second edge. + stack.human.delete("bot.did.operator", &rkey_of(&kestrel)); + let refusal = stack + .check(&pup) + .await + .expect_err("the second edge is gone"); + assert!(refusal.contains("after 1 edges"), "{refusal}"); + let after = misspelled(pup.clone()).await; + eprintln!( + "--server uppercase after withdrawal: status {} stdout {} stderr {}", + after.status, after.stdout, after.stderr + ); + + // The human withdraws from the server itself. The server pauses; what + // does the walk from kestrel say, and what does the server show? + stack.human.write("bot.did.operator", &rkey_of(&kestrel), + json!({ "$type": "bot.did.operator", "subject": did_of(&kestrel), "createdAt": now_rfc3339() })); + assert!(stack.check(&pup).await.is_ok()); + stack + .human + .delete("bot.did.operator", &rkey_of(&server.host)); + server.nudge().await; + wait_for("the server to pause", || async { + (server.create_probe().await != "NameRequired").then_some(()) + }) + .await; + let walked = stack.check(&kestrel).await; + eprintln!("walk from kestrel with the top edge gone: {walked:?}"); + let dashboard = get_json(&format!("{}/dashboard/api/about", server.origin)).await; + let stats = get_json(&format!("{}/xrpc/bot.did.stats", server.origin)).await; + let capabilities = get_json(&format!("{}/dashboard/api/capabilities", server.origin)).await; + let listing = get_json(&format!("{}/xrpc/bot.did.listAccounts", server.origin)).await; + eprintln!( + "about: {dashboard}\nstats: {stats}\ncapabilities: {capabilities}\nlistAccounts: {listing}" + ); + + assert!( + !(before.status == 0 && after.status == 0), + "a misspelled --server passes a walk whose second edge is gone" + ); + let refusal = walked.expect_err("nobody answers for the server"); + assert!( + refusal.contains(&format!( + "bot.did.operator/{}: no such record", + rkey_of(&server.host) + )), + "the walk ends at the human's record for the server: {refusal}" + ); +} + +// --------------------------------------------------------------------------- +// The human's PDS is down +// --------------------------------------------------------------------------- + +/// What the server does when the human's repository stops answering: the +/// poll's verdict, a create the human authenticates, a create beneath a +/// root, an empty listing, a record naming somebody else, and a start and +/// a restart during the outage. +#[tokio::test(flavor = "multi_thread")] +async fn the_humans_pds_is_down() { + const PATIENT: &str = "[operator]\ngrace_window_hours = 1\n"; + let work = Work::new("outage"); + let human = Human::start().await; + let issuer = Issuer::start().await; + let data = work.dir("outage-data"); + let port = free_port(); + let server = + Server::start_with(&work, "outage", &human, PATIENT, port, Some(data.clone())).await; + let laptop = Machine::new(&work, "human"); + human + .seed_session(&laptop.config, &admit_scope(&server.did)) + .await; + let stack = Stack { + work, + human, + issuer, + servers: vec![server], + laptop, + }; + let claimed = stack + .laptop + .didbot( + &["operate", &stack.server().host, stack.human.did(), "--json"], + &[], + ) + .await + .ok(); + assert_eq!(claimed["subject"], stack.server().did, "{claimed}"); + stack.server().nudge().await; + stack.server().wait_claimed().await; + let server = stack.server(); + + let kestrel = format!("kestrel.{}", server.host); + stack + .laptop + .didbot( + &[ + "operate", + &kestrel, + stack.human.did(), + "--kind", + "agent", + "--creates", + "agent", + "--json", + ], + &[], + ) + .await + .ok(); + server.read_allowances().await; + let token = std::fs::read_to_string(stack.laptop.token_file(&kestrel)).expect("a token"); + let create_beneath = |name: String, token: String| { + let url = format!("{}/xrpc/bot.did.createAccount", server.origin); + async move { + post_json( + &url, + Some(token.trim()), + json!({ "name": name, "kind": "agent" }), + ) + .await + } + }; + let write_as_kestrel = || { + let url = format!("{}/xrpc/com.atproto.repo.createRecord", server.origin); + let body = json!({ "repo": did_of(&kestrel), "collection": "com.example.thing", + "record": { "text": "hi", "emoji": "\u{1f9ee}", "createdAt": now_rfc3339() } }); + let token = token.clone(); + async move { post_json(&url, Some(token.trim()), body).await } + }; + let (status, _) = write_as_kestrel().await; + assert_eq!(status, 200); + let (status, _) = create_beneath(format!("c1.{}", server.host), token.clone()).await; + assert_eq!(status, 200); + + // The human's PDS answers 500 to everything. The poll keeps its + // verdict inside the grace window, and says so. + let log_before = server.log().len(); + stack.human.set_outage(Outage::Failing); + server.nudge().await; + wait_for("the poll to report the outage", || async { + server.log()[log_before..] + .contains("operator poll could not reach the operator's repository") + .then_some(()) + }) + .await; + assert_eq!( + server.create_probe().await, + "NameRequired", + "creates stay open in grace" + ); + let (status, _) = write_as_kestrel().await; + assert_eq!(status, 200, "writes stay open in grace"); + + // A create the human authenticates, with a token their PDS minted + // before it went down: named, and not "invalid token". + let service_auth = + Human::mint_service_auth(&stack.human.state, &server.did, "bot.did.createAccount"); + let c2 = format!("c2.{}", server.host); + stack.human.write( + "bot.did.operator", + &rkey_of(&c2), + json!({ "$type": "bot.did.operator", "subject": did_of(&c2), "createdAt": now_rfc3339() }), + ); + let (status, body) = create_beneath(c2.clone(), service_auth.clone()).await; + eprintln!("human create during a 500 outage: {status} {body}"); + assert_eq!(body["error"], "OperatorUnreachable", "{status} {body}"); + + // A create beneath a root, once the 60 s cache lapses, re-reads the + // record and cannot: the poll's verdict stands. + let started = Instant::now(); + let (status, body) = create_beneath(format!("c3.{}", server.host), token.clone()).await; + eprintln!( + "root create during a 500 outage: {status} {body} after {:?}", + started.elapsed() + ); + assert_eq!(status, 200, "{body}"); + + // The human's PDS hangs instead. The first create beneath a root + // waits out the server's client timeout before it is allowed; the + // next one inside the cache's minute does not wait again. + stack.human.set_outage(Outage::Hanging); + tokio::time::sleep(Duration::from_secs(61)).await; + let patient = didbot_http::builder() + .timeout(Duration::from_secs(60)) + .build() + .expect("a client"); + let create_patiently = |name: String| { + let url = format!("{}/xrpc/bot.did.createAccount", server.origin); + let token = token.clone(); + let patient = patient.clone(); + async move { + let started = Instant::now(); + let response = patient + .post(&url) + .bearer_auth(token.trim()) + .json(&json!({ "name": name, "kind": "agent" })) + .send() + .await + .expect("the server answers"); + let status = response.status().as_u16(); + let body: Value = response.json().await.unwrap_or(Value::Null); + (status, body, started.elapsed()) + } + }; + let (status, body, took) = create_patiently(format!("c4.{}", server.host)).await; + eprintln!("root create during a hang: {status} {body} after {took:?}"); + assert_eq!(status, 200, "{body}"); + assert!( + took >= Duration::from_secs(9), + "the first create waits out the read: {took:?}" + ); + let (status, body, took) = create_patiently(format!("c4b.{}", server.host)).await; + eprintln!("second root create during a hang: {status} after {took:?}"); + assert_eq!(status, 200, "{body}"); + assert!( + took < Duration::from_secs(3), + "a failed read is remembered for the cache's minute: {took:?}" + ); + let log_before = server.log().len(); + server.nudge().await; + wait_for("the poll to report the hang", || async { + server.log()[log_before..] + .contains("could not reach the operator's repository") + .then_some(()) + }) + .await; + assert_eq!(server.create_probe().await, "NameRequired"); + + // A kind the allowance does not name is refused by name. + stack.human.set_outage(Outage::None); + let (status, body) = post_json( + &format!("{}/xrpc/bot.did.createAccount", server.origin), + Some(token.trim()), + json!({ "name": format!("svc.{}", server.host), "kind": "service" }), + ) + .await; + assert_eq!(body["error"], "NotAllowedToCreate", "{status} {body}"); + + // Every listing comes back empty while records still read one by + // one: the poll installs what it listed, which is nothing, and every + // hosted creator is refused until a listing carries the records again. + stack.human.set_outage(Outage::EmptyListings); + server.read_allowances().await; + let (status, body) = create_beneath(format!("c-empty.{}", server.host), token.clone()).await; + eprintln!("agent create beneath kestrel on an empty listing: {status} {body}"); + assert_eq!(body["error"], "NotAllowedToCreate", "{status} {body}"); + stack.human.set_outage(Outage::None); + server.read_allowances().await; + let (status, body) = create_beneath(format!("c-back.{}", server.host), token.clone()).await; + assert_eq!( + status, 200, + "the allowance is back with the listing: {body}" + ); + + // The human's record for kestrel names somebody else. + stack.human.write( + "bot.did.operator", + &rkey_of(&kestrel), + json!({ "$type": "bot.did.operator", "subject": did_of(&c2), "createdAt": now_rfc3339() }), + ); + tokio::time::sleep(Duration::from_secs(61)).await; + let (status, body) = create_beneath(format!("c5.{}", server.host), token.clone()).await; + assert_eq!(body["error"], "OperatorRecordMissing", "{status} {body}"); + let refusal = stack + .check(&kestrel) + .await + .expect_err("the record names somebody else"); + assert!(refusal.contains("names did:web:c2"), "{refusal}"); + let log_before = server.log().len(); + server.nudge().await; + wait_for("the poll to see the wrong subject", || async { + server.log()[log_before..] + .contains("names a different did") + .then_some(()) + }) + .await; + let (status, _) = write_as_kestrel().await; + assert_eq!(status, 200, "inside the grace window kestrel still writes"); + stack.human.write("bot.did.operator", &rkey_of(&kestrel), + json!({ "$type": "bot.did.operator", "subject": did_of(&kestrel), "createdAt": now_rfc3339() })); +} + +/// A server started, and one restarted, while the human's PDS is down. +#[tokio::test(flavor = "multi_thread")] +async fn a_restart_while_the_humans_pds_is_down() { + const PATIENT: &str = "[operator]\ngrace_window_hours = 1\n"; + let work = Work::new("restart-outage"); + let human = Human::start().await; + let issuer = Issuer::start().await; + let data = work.dir("restart-data"); + let port = free_port(); + let server = + Server::start_with(&work, "restart", &human, PATIENT, port, Some(data.clone())).await; + let laptop = Machine::new(&work, "human"); + human + .seed_session(&laptop.config, &admit_scope(&server.did)) + .await; + let mut stack = Stack { + work, + human, + issuer, + servers: vec![server], + laptop, + }; + let claimed = stack + .laptop + .didbot( + &["operate", &stack.server().host, stack.human.did(), "--json"], + &[], + ) + .await + .ok(); + assert_eq!(claimed["subject"], stack.server().did, "{claimed}"); + stack.server().nudge().await; + stack.server().wait_claimed().await; + let kestrel = format!("kestrel.{}", stack.server().host); + stack + .laptop + .didbot( + &[ + "operate", + &kestrel, + stack.human.did(), + "--kind", + "agent", + "--json", + ], + &[], + ) + .await + .ok(); + let token = std::fs::read_to_string(stack.laptop.token_file(&kestrel)).expect("a token"); + + // A fresh server, started during the outage, waits for the claim + // rather than crashing. + stack.human.set_outage(Outage::Failing); + let mut fresh = Server::start_with( + &stack.work, + "fresh", + &stack.human, + PATIENT, + free_port(), + None, + ) + .await; + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(fresh.create_probe().await, "ServerNotReady"); + assert!(fresh + .log() + .contains("operator poll could not reach the operator's repository")); + eprintln!( + "fresh start during the outage, log tail:\n{}", + fresh + .log() + .lines() + .rev() + .take(6) + .collect::>() + .join("\n") + ); + fresh.stop(); + + // The claimed server restarts on its data during the outage. + let mut server = stack.servers.pop().expect("the server"); + server.stop(); + let server = Server::start_with( + &stack.work, + "restart", + &stack.human, + PATIENT, + port, + Some(data.clone()), + ) + .await; + let write = || { + let url = format!("{}/xrpc/com.atproto.repo.createRecord", server.origin); + let body = json!({ "repo": did_of(&kestrel), "collection": "com.example.thing", + "record": { "text": "after restart", "emoji": "\u{1f9ee}", "createdAt": now_rfc3339() } }); + let token = token.clone(); + async move { post_json(&url, Some(token.trim()), body).await } + }; + let (status, body) = write().await; + eprintln!("write as kestrel after a restart during the outage: {status} {body}"); + eprintln!( + "restart log tail:\n{}", + server + .log() + .lines() + .rev() + .take(10) + .collect::>() + .join("\n") + ); + let during = (status, body); + stack.human.set_outage(Outage::None); + server.nudge().await; + server.wait_claimed().await; + let (status, body) = write().await; + assert_eq!( + status, 200, + "once the human's PDS answers, kestrel writes again: {body}" + ); + // Finding: a restart inside the grace window does not keep writes open. + eprintln!("during the outage the write was {} {}", during.0, during.1); +} + +/// A deleted account's name is never reissued, and the human's record for +/// it outlives it: `operate` for the name again is refused and the record +/// survives the refusal. +#[tokio::test(flavor = "multi_thread")] +async fn records_a_deleted_name_is_not_reissued() { + let stack = Stack::start(&["typo"]).await; + let server = stack.server(); + let name = format!("oops.{}", server.host); + let did = did_of(&name); + let admitted = stack + .laptop + .didbot( + &[ + "operate", + &name, + stack.human.did(), + "--kind", + "host", + "--json", + ], + &[], + ) + .await + .ok(); + assert_eq!(admitted["kind"], "host"); + let token = std::fs::read_to_string(stack.laptop.token_file(&name)).expect("the token file"); + // The account deletes itself, as a harness would at the end of a session. + let (status, body) = post_json( + &format!("{}/xrpc/bot.did.deleteAccount", server.origin), + Some(token.trim()), + json!({ "did": did }), + ) + .await; + assert_eq!(status, 200, "{body}"); + // The document is still served; the registration is gone, so the walk + // fails; the human's record still names it. + let (status, _) = get_json_status(&format!("http://{name}/.well-known/did.json")).await; + assert_eq!(status, 200); + assert!(stack.check(&name).await.is_err()); + assert!(stack + .human + .record_keys("bot.did.operator") + .await + .contains(&rkey_of(&name))); + // The name, as an agent this time, is refused for good, and the refusal + // costs the human nothing. + let refusal = stack + .laptop + .didbot( + &[ + "operate", + &name, + stack.human.did(), + "--kind", + "agent", + "--json", + ], + &[], + ) + .await + .refused(); + assert!(refusal.contains("AccountAlreadyExists"), "{refusal}"); + assert!(stack + .human + .record_keys("bot.did.operator") + .await + .contains(&rkey_of(&name))); +} +async fn get_json_status(url: &str) -> (u16, Value) { + let response = http() + .get(url) + .send() + .await + .unwrap_or_else(|err| panic!("GET {url}: {err}")); + let status = response.status().as_u16(); + (status, response.json().await.unwrap_or(Value::Null)) +} + +// --------------------------------------------------------------------------- +// Revocation two edges down +// --------------------------------------------------------------------------- +/// The human deletes their record for a service. A create the service +/// itself makes is refused once the server's re-read of the top edge +/// expires; a create by a host beneath the service has to be refused at +/// the same moment, because the host's chain ends at the same record. +/// +/// Prints the measured windows so the report can quote them. +#[tokio::test(flavor = "multi_thread")] +async fn a_revoked_roots_grandchildren_stop_creating_with_it() { + let stack = Stack::start(&["deep"]).await; + let server = stack.server(); + let web = format!("web.{}", server.host); + let web_did = did_of(&web); + stack + .laptop + .didbot( + &[ + "operate", + &web, + stack.human.did(), + "--kind", + "service", + "--oidc", + &stack.issuer.base, + "email=web@pool", + "--creates", + "host", + "--creates-beneath", + "agent", + "--json", + ], + &[], + ) + .await + .ok(); + server.read_allowances().await; + let machine = Machine::new(&stack.work, "i-1"); + let host = format!("i-1.{}", server.host); + let token_file = machine.secret_file( + "id-token", + &stack + .issuer + .id_token(&server.did, json!({ "email": "web@pool" })), + ); + machine + .didbot( + &[ + "register", + "host", + &host, + "--under", + &web, + "--server", + &server.host, + "--token-file", + token_file.to_str().expect("utf-8"), + "--json", + ], + &[], + ) + .await + .ok(); + let daemon = Daemon::start(&machine, server).await; + daemon.agent("ctx-1").await; + // A create by the service itself, with a fresh ID token each time. + let create_beneath_web = |n: usize| { + let token = stack + .issuer + .id_token(&server.did, json!({ "email": "web@pool" })); + let url = format!("{}/xrpc/bot.did.createAccount", server.origin); + let body = json!({ "name": format!("probe-{n}.{}", server.host), "kind": "host" }); + async move { post_json(&url, Some(&token), body).await } + }; + let (status, body) = create_beneath_web(0).await; + assert_eq!(status, 200, "{body}"); + let deleted_at = Instant::now(); + stack.human.delete("bot.did.operator", &rkey_of(&web)); + // The service's own creates stop once the cached top edge expires. + let mut n = 1; + let refused = loop { + let (status, body) = create_beneath_web(n).await; + n += 1; + if status == 403 { + break body; + } + assert!( + deleted_at.elapsed() < Duration::from_secs(90), + "the service still creates {:?} after its record was deleted", + deleted_at.elapsed() + ); + tokio::time::sleep(Duration::from_secs(1)).await; + }; + let root_window = deleted_at.elapsed(); + assert_eq!(refused["error"], "OperatorRecordMissing", "{refused}"); + eprintln!("deep: the service's own creates were refused {root_window:?} after the delete"); + // The host beneath it is on the same chain: its daemon's create has + // to be refused now too, not at the next poll. + let trouble = daemon.began("session-2", "ctx-2").await; + eprintln!("deep: the host's create {root_window:?} after the delete: {trouble:?}"); + let trouble = trouble.expect_err("a host beneath a revoked service creates nothing"); + assert!( + trouble.contains("refused") || trouble.contains("403"), + "the daemon names the refusal: {trouble}" + ); + // And the poll, once nudged, stops the host's writes; measure that too. + let host_token = std::fs::read_to_string(machine.token_file(&host)).expect("a token"); + let write = || { + let url = format!("{}/xrpc/com.atproto.repo.createRecord", server.origin); + let body = json!({ + "repo": did_of(&host), + "collection": "com.example.thing", + "record": { "text": "x", "emoji": "\u{1f9ee}", "createdAt": now_rfc3339() }, + }); + let token = host_token.trim().to_owned(); + async move { post_json(&url, Some(&token), body).await } + }; + let (status, _) = write().await; + assert_eq!(status, 200, "before the poll the host still writes"); + let nudged_at = Instant::now(); + server.nudge().await; + wait_for("the poll to quarantine the host", || async { + let (status, _) = write().await; + (status == 403).then_some(()) + }) + .await; + eprintln!( + "deep: the host's writes were refused {:?} after the nudge ({:?} after the delete)", + nudged_at.elapsed(), + deleted_at.elapsed() + ); + // What landed beneath the service is what the cached top edge let + // through, and nothing beneath the host after that. + let probes = list_record_keys(&server.origin, &web_did, "bot.did.operator").await; + eprintln!( + "deep: {} accounts were created beneath the service inside the window", + probes.len() - 1 + ); + assert_eq!( + list_record_keys(&server.origin, &did_of(&host), "bot.did.operator").await, + vec![rkey_of(&format!("ctx-1.{}", server.host))], + "nothing landed beneath the host after the service's record went" + ); +} + +// --------------------------------------------------------------------------- +// Sessions: the live-token cap, expiry, and a stolen token after revocation +// --------------------------------------------------------------------------- +/// The account-creation token counts toward the eight-token cap and is never +/// the one evicted (it expires last), so a pipeline the operator created has +/// only seven usable proof-session slots. The eighth proof session evicts the +/// first — one run sooner than the plain cap of eight suggests — and what run +/// one then sees is `InvalidToken`: a token this server no longer issued, +/// distinct from `Halted` or `AccountNotWritable`. +#[tokio::test(flavor = "multi_thread")] +async fn the_creation_token_pins_a_slot_and_the_eighth_session_evicts_the_first() { + let stack = Stack::start(&["evict"]).await; + let server = stack.server(); + let deploy = format!("deploy.{}", server.host); + let deploy_did = did_of(&deploy); + stack + .laptop + .didbot( + &[ + "operate", + &deploy, + stack.human.did(), + "--kind", + "pipeline", + "--oidc", + &stack.issuer.base, + "repository_id=456789", + "--json", + ], + &[], + ) + .await + .ok(); + // The long-lived token operate kept when it created the pipeline. + let long = std::fs::read_to_string(stack.laptop.token_file(&deploy)) + .expect("the creation token") + .trim() + .to_owned(); + let mint = || { + let url = format!("{}/xrpc/bot.did.createSession", server.origin); + let token = stack + .issuer + .id_token(&server.did, json!({ "repository_id": "456789" })); + async move { + let (status, body) = post_json(&url, Some(&token), json!({})).await; + assert_eq!(status, 200, "a run mints its session: {body}"); + body["accountToken"].as_str().expect("a session").to_owned() + } + }; + let write = |token: String| { + let url = format!("{}/xrpc/com.atproto.repo.createRecord", server.origin); + let body = json!({ + "repo": deploy_did, + "collection": "com.example.thing", + "record": { "text": "x", "emoji": "\u{1f9ee}", "createdAt": now_rfc3339() }, + }); + async move { post_json(&url, Some(&token), body).await } + }; + // Mint proof sessions one at a time, watching for the first to fall out. + let mut first: Option = None; + let mut evicted_at = None; + for n in 1..=8 { + let session = mint().await; + if first.is_none() { + first = Some(session.clone()); + continue; + } + let (status, _) = write(first.clone().unwrap()).await; + if status == 401 { + evicted_at = Some(n); + break; + } + } + let evicted_at = evicted_at.expect("the first session is evicted within eight mints"); + eprintln!( + "runs: the first proof session is evicted at proof-session #{evicted_at} (the creation token pins one of the eight slots)" + ); + assert_eq!( + evicted_at, 8, + "seven proof sessions coexist beside the pinned creation token" + ); + let (status, body) = write(first.unwrap()).await; + assert_eq!(status, 401, "run one's session is gone: {body}"); + assert_eq!(body["error"], "InvalidToken", "{body}"); + // The creation token, expiring last, is never evicted and still writes. + let (status, body) = write(long).await; + assert_eq!( + status, 200, + "the long-lived creation token survives: {body}" + ); +} +/// A proof-minted session lasts `[sessions].proof_ttl_secs`. Past it a write +/// is `ExpiredToken`, and nothing re-mints it for the run: an account token +/// is not an OAuth session, so it carries no refresh token. The run that +/// wants to keep going presents its proof again. +#[tokio::test(flavor = "multi_thread")] +async fn a_session_expires_mid_run_and_carries_no_refresh() { + let config = "[operator]\ngrace_window_hours = 0\n[sessions]\nproof_ttl_secs = 3\n"; + let stack = Stack::start_with(&["ttl"], config, false).await; + let server = stack.server(); + let deploy = format!("deploy.{}", server.host); + let deploy_did = did_of(&deploy); + stack + .laptop + .didbot( + &[ + "operate", + &deploy, + stack.human.did(), + "--kind", + "pipeline", + "--oidc", + &stack.issuer.base, + "repository_id=456789", + "--json", + ], + &[], + ) + .await + .ok(); + let token = stack + .issuer + .id_token(&server.did, json!({ "repository_id": "456789" })); + let (status, body) = post_json( + &format!("{}/xrpc/bot.did.createSession", server.origin), + Some(&token), + json!({}), + ) + .await; + assert_eq!(status, 200, "{body}"); + let session = body["accountToken"].as_str().expect("a session").to_owned(); + let write = || { + let url = format!("{}/xrpc/com.atproto.repo.createRecord", server.origin); + let session = session.clone(); + let body = json!({ + "repo": deploy_did, + "collection": "com.example.thing", + "record": { "text": "x", "emoji": "\u{1f9ee}", "createdAt": now_rfc3339() }, + }); + async move { post_json(&url, Some(&session), body).await } + }; + let (status, _) = write().await; + assert_eq!(status, 200, "the fresh session writes"); + tokio::time::sleep(Duration::from_secs(4)).await; + let (status, body) = write().await; + assert_eq!(status, 401, "the session has expired: {body}"); + eprintln!( + "runs: a session past its ttl is refused as {}", + body["error"] + ); + assert_eq!(body["error"], "ExpiredToken", "{body}"); +} +/// A leaked 1 h session, used after the operator has revoked the account's +/// host: the token is *not* invalidated by the self-pause. The write is +/// refused because the account is quarantined (`AccountNotWritable`), and a +/// fresh session cannot be minted (`CreatorNotActive`). But the stolen token +/// is still a token this server issued; there is no per-account way to end +/// it short of deleting the account. +#[tokio::test(flavor = "multi_thread")] +async fn a_stolen_session_is_refused_by_quarantine_not_invalidated() { + let stack = Stack::start(&["stolen"]).await; + let server = stack.server(); + let deploy = format!("deploy.{}", server.host); + let deploy_did = did_of(&deploy); + stack + .laptop + .didbot( + &[ + "operate", + &deploy, + stack.human.did(), + "--kind", + "pipeline", + "--oidc", + &stack.issuer.base, + "repository_id=456789", + "--json", + ], + &[], + ) + .await + .ok(); + let proof = stack + .issuer + .id_token(&server.did, json!({ "repository_id": "456789" })); + let (status, body) = post_json( + &format!("{}/xrpc/bot.did.createSession", server.origin), + Some(&proof), + json!({}), + ) + .await; + assert_eq!(status, 200, "{body}"); + let stolen = body["accountToken"].as_str().expect("a session").to_owned(); + // The human revokes the pipeline. The poll, once nudged, quarantines it. + stack.human.delete("bot.did.operator", &rkey_of(&deploy)); + server.nudge().await; + let write = || { + let url = format!("{}/xrpc/com.atproto.repo.createRecord", server.origin); + let stolen = stolen.clone(); + let body = json!({ + "repo": deploy_did, + "collection": "com.example.thing", + "record": { "text": "x", "emoji": "\u{1f9ee}", "createdAt": now_rfc3339() }, + }); + async move { post_json(&url, Some(&stolen), body).await } + }; + let refused = wait_for("the poll to quarantine the pipeline", || async { + let (status, body) = write().await; + (status == 403).then_some(body) + }) + .await; + eprintln!( + "runs: the stolen session after revocation is refused as {} (the token itself still verifies)", + refused["error"] + ); + assert_eq!( + refused["error"], "AccountNotWritable", + "the write is refused by the quarantine, not by the token being gone: {refused}" + ); + // And a fresh proof mints nothing while quarantined. + let fresh = stack + .issuer + .id_token(&server.did, json!({ "repository_id": "456789" })); + let (status, body) = post_json( + &format!("{}/xrpc/bot.did.createSession", server.origin), + Some(&fresh), + json!({}), + ) + .await; + assert_eq!(status, 403, "no new session while quarantined: {body}"); + assert_eq!(body["error"], "AccountLocked", "{body}"); +} +// --------------------------------------------------------------------------- +// Operator rotation: one operator DID is launch configuration +// --------------------------------------------------------------------------- +/// Rule 7 calls operator rotation "a delete and a new claim inside the grace +/// window." It is not: the operator DID is `--operator` at launch and the +/// server polls only that repository. Human A deletes the server's top edge; +/// Human B writes a fresh `bot.did.operator` naming the same server. The +/// server never reads B, so it lapses on A and pauses the whole deployment. +/// There is no operation that repoints it at B short of a restart. +#[tokio::test(flavor = "multi_thread")] +async fn a_second_human_cannot_rotate_in_as_operator() { + let stack = Stack::start(&["rotate"]).await; + let server = stack.server(); + // A create the server serves while A's claim stands. + let (status, body) = post_json( + &format!("{}/xrpc/bot.did.createAccount", server.origin), + None, + json!({ "kind": "agent" }), + ) + .await; + assert_eq!( + body["error"], "NameRequired", + "the server serves creates: {body}" + ); + assert_eq!(status, 400); + // Human B stands up their own PDS and writes a claim naming this server. + let human_b = Human::start().await; + human_b.write( + "bot.did.operator", + &rkey_of(&server.host), + json!({ + "$type": "bot.did.operator", + "subject": server.did, + "createdAt": now_rfc3339(), + }), + ); + // A rotates out: deletes their record for the server. + stack + .human + .delete("bot.did.operator", &rkey_of(&server.host)); + server.nudge().await; + // The server pauses: it polls A, not B, and B's fresh claim is invisible. + let paused = wait_for("the server to pause on A's lapse", || async { + let (status, body) = post_json( + &format!("{}/xrpc/bot.did.createAccount", server.origin), + None, + json!({ "kind": "agent" }), + ) + .await; + (status != 400).then_some((status, body)) + }) + .await; + eprintln!( + "rotate: after A deletes and B writes, the server answers a create with {} {}", + paused.0, paused.1["error"] + ); + assert!( + paused.1["error"] == "Halted" || paused.1["error"] == "ServerNotReady", + "the deployment paused rather than following B: {:?}", + paused.1 + ); + // B's record stands and names the server; nobody read it. + assert_eq!( + human_b.record_keys("bot.did.operator").await, + vec![rkey_of(&server.host)], + "B's claim is there to be read; the server simply never reads it" + ); +} + +// --------------------------------------------------------------------------- +// The two stand-ins, left running for a person driving the stack by hand +// --------------------------------------------------------------------------- +/// Starts the human's PDS and the issuer on fixed ports, seeds the session +/// `didbot operate` resumes under `NEWCOMER_CONFIG`, and stays up until +/// killed. `NEWCOMER_SERVERS` is the comma-separated DIDs of the servers the +/// session may admit accounts at; `NEWCOMER_HUMAN_PORT`, `NEWCOMER_ISSUER_PORT` +/// and `NEWCOMER_MINT_PORT` move the three listeners (3415, 3416, 3417). +/// `GET /?aud=&=...` on the mint port answers an +/// ID token the issuer signed. +/// +/// `cargo test -p didbot --test scenarios by_hand -- --ignored --nocapture` +#[ignore] +#[tokio::test(flavor = "multi_thread")] +async fn the_stand_ins_by_hand() { + let port = |name: &str, default: u16| { + std::env::var(name) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) + }; + let config = PathBuf::from(std::env::var("NEWCOMER_CONFIG").expect("NEWCOMER_CONFIG is set")); + let servers = std::env::var("NEWCOMER_SERVERS").unwrap_or_default(); + let human = Human::start_on(port("NEWCOMER_HUMAN_PORT", 3415)).await; + let issuer = Issuer::start_on(port("NEWCOMER_ISSUER_PORT", 3416)).await; + let mut scope = String::new(); + for (index, server) in servers.split(',').filter(|s| !s.is_empty()).enumerate() { + let atoms = admit_scope(server); + if index == 0 { + scope = atoms; + } else { + scope.push(' '); + scope.push_str(atoms.rsplit(' ').next().expect("an rpc atom")); + } + } + human.seed_session(&config, &scope).await; + let key = issuer.key.clone(); + let base = issuer.base.clone(); + let mint = Router::new().route( + "/", + get(move |Query(mut query): Query>| { + let key = key.clone(); + let base = base.clone(); + async move { + let aud = query.remove("aud").unwrap_or_default(); + let claims: Value = query + .into_iter() + .map(|(k, v)| (k, Value::String(v))) + .collect(); + Issuer { base, key }.id_token(&aud, claims) + } + }), + ); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", port("NEWCOMER_MINT_PORT", 3417))) + .await + .expect("bind"); + tokio::spawn(async move { + axum::serve(listener, mint).await.ok(); + }); + println!("human {} at {}", human.did(), human.origin()); + println!("issuer {}", issuer.base); + println!( + "mint http://127.0.0.1:{}/?aud=&claim=value", + port("NEWCOMER_MINT_PORT", 3417) + ); + println!( + "session for scope {scope:?} written under {}", + config.display() + ); + loop { + tokio::time::sleep(Duration::from_secs(3600)).await; + } +} +/// `com.atproto.repo.createRecord` as `name`, with `token`. +async fn write_as(server: &Server, name: &str, token: &str) -> (u16, Value) { + post_json( + &format!("{}/xrpc/com.atproto.repo.createRecord", server.origin), + Some(token.trim()), + json!({ + "repo": did_of(name), + "collection": "com.example.thing", + "record": { "text": "still here", "emoji": "\u{1f9ee}", "createdAt": now_rfc3339() }, + }), + ) + .await +} + +/// The session `didbot operate` or `didbot register` kept for `name`. +fn token_of(machine: &Machine, name: &str) -> String { + std::fs::read_to_string(machine.token_file(name)) + .expect("a token file") + .trim() + .to_owned() +} + +/// `bot.did.createAccount` with a bearer, as a caller with a token does. +async fn create_with(server: &Server, token: &str, name: &str, kind: &str) -> (u16, Value) { + post_json( + &format!("{}/xrpc/bot.did.createAccount", server.origin), + Some(token), + json!({ "name": name, "kind": kind }), + ) + .await +} + +/// `bot.did.createSession` with a proof. +async fn session_with(server: &Server, proof: &str) -> (u16, Value) { + post_json( + &format!("{}/xrpc/bot.did.createSession", server.origin), + Some(proof), + json!({}), + ) + .await +} + +/// `didbot register agent --under ` with `token`. +async fn register_agent( + machine: &Machine, + server: &Server, + name: &str, + under: &str, + token_file: &Path, +) -> Ran { + machine + .didbot( + &[ + "register", + "agent", + name, + "--under", + under, + "--server", + &server.host, + "--token-file", + token_file.to_str().expect("utf-8"), + "--json", + ], + &[], + ) + .await +} + +/// The whole tree on one durable server — a host with its daemon and an +/// agent, an agent by hand beneath the host, a pipeline with a session — +/// then `didbot-pds` killed and started again on the same `--data`. Every +/// account, session and edge is where it was; what the log does not hold +/// is measured and printed: a key-signed proof and an OpenID Connect +/// token, each spent before the restart, presented again after it. +#[tokio::test(flavor = "multi_thread")] +async fn a_restart_keeps_the_tree() { + let mut stack = Stack::start_with(&["restart"], Server::CONFIG, true).await; + let server = stack.server(); + let laptop = Laptop::stand_up(&stack, server, "laptop").await; + let agent = laptop.daemon.agent("ctx-1").await; + let agent_name = agent.trim_start_matches("did:web:").replace("%3A", ":"); + let host_token = laptop.machine.token_file(&laptop.name); + let by_hand = format!("byhand.{}", server.host); + register_agent(&laptop.machine, server, &by_hand, &laptop.name, &host_token) + .await + .ok(); + + // A pipeline, and a session as it from a token its platform minted. + let deploy = format!("deploy.{}", server.host); + stack + .laptop + .didbot( + &[ + "operate", + &deploy, + stack.human.did(), + "--kind", + "pipeline", + "--oidc", + &stack.issuer.base, + "repository_id=456789", + "--json", + ], + &[], + ) + .await + .ok(); + let oidc_token = stack + .issuer + .id_token(&server.did, json!({ "repository_id": "456789" })); + let (status, body) = session_with(server, &oidc_token).await; + assert_eq!(status, 200, "{body}"); + let session_a = body["accountToken"].as_str().expect("a session").to_owned(); + let (status, _) = session_with(server, &oidc_token).await; + assert_eq!(status, 401, "the token is spent while the server runs"); + + // A key-signed proof from the human, spent on one create; presenting + // it for a second name is a replay while the server runs. + let j1 = format!("j1.{}", server.host); + let j2 = format!("j2.{}", server.host); + vouch(&stack.human, &j1); + vouch(&stack.human, &j2); + let key_proof = stack + .human + .service_auth(&server.did, "bot.did.createAccount"); + let (status, body) = create_with(server, &key_proof, &j1, "agent").await; + assert_eq!(status, 200, "{body}"); + let (status, body) = create_with(server, &key_proof, &j2, "agent").await; + assert_eq!( + status, 401, + "the proof is spent while the server runs: {body}" + ); + + let human_records = stack.human.record_keys("bot.did.operator").await; + let host_records = list_record_keys(&server.origin, &laptop.did, "bot.did.operator").await; + assert_eq!(host_records.len(), 2, "{host_records:?}"); + + // Killed and started again on the same directory. + let down = stack.servers[0].restart().await; + let server = stack.server(); + server.nudge().await; + server.wait_claimed().await; + eprintln!("restart: the server was unreachable for {down:?}"); + + // Held: every session the log recorded still writes. + for (name, token) in [ + ( + laptop.name.as_str(), + token_of(&laptop.machine, &laptop.name), + ), + (by_hand.as_str(), token_of(&laptop.machine, &by_hand)), + (deploy.as_str(), session_a.clone()), + ] { + let (status, body) = write_as(server, name, &token).await; + assert_eq!(status, 200, "{name} still writes after the restart: {body}"); + } + // Held: the daemon and the host still create beneath the host. + let agent_2 = laptop.daemon.agent("ctx-2").await; + assert_ne!(agent_2, agent); + let late = format!("late.{}", server.host); + register_agent(&laptop.machine, server, &late, &laptop.name, &host_token) + .await + .ok(); + // Held: every edge still verifies, and the repositories hold what they + // held. + for name in [ + laptop.name.as_str(), + agent_name.as_str(), + by_hand.as_str(), + deploy.as_str(), + j1.as_str(), + late.as_str(), + ] { + stack.walk(name).await; + } + assert_eq!( + stack.human.record_keys("bot.did.operator").await, + human_records + ); + assert_eq!( + list_record_keys(&server.origin, &laptop.did, "bot.did.operator") + .await + .len(), + 4, + "the host names its two agents from before and two from after" + ); + + // Measured: the proofs spent before the restart, presented again. + let (status, body) = create_with(server, &key_proof, &j2, "agent").await; + eprintln!("restart: a key-signed proof spent before the restart answers {status} {body}"); + let (status, body) = session_with(server, &oidc_token).await; + eprintln!("restart: an OpenID Connect token spent before the restart answers {status} {body}"); +} + +/// `didbot register host ` started on `machine` and left waiting on +/// its park: the process, its stdout so far, and the fingerprint it +/// printed. +async fn park_host( + machine: &Machine, + server: &Server, + name: &str, + timeout: &str, +) -> ( + tokio::process::Child, + tokio::io::Lines>, + String, +) { + let mut register = machine + .command( + &[ + "register", + "host", + name, + "--server", + &server.host, + "--timeout", + timeout, + "--json", + ], + &[], + ) + .spawn() + .expect("didbot register starts"); + let stdout = register.stdout.take().expect("stdout is piped"); + let mut lines = BufReader::new(stdout).lines(); + let mut fingerprint = None; + while let Some(line) = lines.next_line().await.expect("register prints") { + if let Some(rest) = line.split("--fingerprint ").nth(1) { + fingerprint = Some(rest.trim().to_owned()); + break; + } + } + ( + register, + lines, + fingerprint.expect("register printed a fingerprint"), + ) +} + +/// A key is parked, the server restarts, and the human admits the +/// fingerprint they were shown: what each side says, and how long the +/// machine waits. +#[tokio::test(flavor = "multi_thread")] +async fn a_restart_forgets_a_parked_key() { + let mut stack = Stack::start_with(&["park"], Server::CONFIG, true).await; + let server = stack.server(); + let machine = Machine::new(&stack.work, "parked"); + let name = format!("parked.{}", server.host); + let (mut register, mut lines, fingerprint) = park_host(&machine, server, &name, "20").await; + + let down = stack.servers[0].restart().await; + let server = stack.server(); + server.nudge().await; + server.wait_claimed().await; + eprintln!("park: the server was unreachable for {down:?}"); + + let admitted = stack + .laptop + .didbot( + &[ + "operate", + &name, + stack.human.did(), + "--fingerprint", + &fingerprint, + "--json", + ], + &[], + ) + .await; + eprintln!( + "park: `didbot operate --fingerprint` after the restart exited {}\nstderr: {}", + admitted.status, + admitted.stderr.trim() + ); + + // What the refusal tells the operator to do instead. + let with_kind = stack + .laptop + .didbot( + &[ + "operate", + &name, + stack.human.did(), + "--kind", + "host", + "--json", + ], + &[], + ) + .await; + eprintln!( + "park: `didbot operate --kind host`, which the refusal suggests, exited {}", + with_kind.status + ); + if with_kind.status == 0 { + let held = credentials(server, &did_of(&name)).await; + eprintln!("park: the account it made logs in with {held:?}"); + } + + let waited = Instant::now(); + let mut printed = Vec::new(); + while let Some(line) = lines.next_line().await.expect("register prints") { + printed.push(line); + } + let status = register.wait().await.expect("register exits"); + let mut complaint = String::new(); + if let Some(mut stderr) = register.stderr.take() { + let _ = stderr.read_to_string(&mut complaint).await; + } + eprintln!( + "park: `didbot register host` exited {status} after {:?}\nstdout:\n{}\nstderr: {}", + waited.elapsed(), + printed.join("\n"), + complaint.trim() + ); +} + +/// Writes as `name` every `BEAT` for up to `PATIENCE`, until the answer is +/// `until`; returns the statuses seen and how long the first `until` took. +async fn writes_until( + server: &Server, + name: &str, + token: &str, + until: u16, +) -> (Vec, Option) { + let started = Instant::now(); + let mut seen = Vec::new(); + loop { + let (status, _) = write_as(server, name, token).await; + seen.push(status); + if status == until { + return (seen, Some(started.elapsed())); + } + if started.elapsed() > PATIENCE { + return (seen, None); + } + tokio::time::sleep(BEAT).await; + } +} + +/// Writes as `name` every `BEAT` for `span`, and answers every status seen. +async fn sample_writes(server: &Server, name: &str, token: &str, span: Duration) -> Vec { + let started = Instant::now(); + let mut seen = Vec::new(); + while started.elapsed() < span { + let (status, _) = write_as(server, name, token).await; + seen.push(status); + tokio::time::sleep(BEAT).await; + } + seen +} + +/// The human's record for a host is gone and the server has paused the +/// host's subtree. The server restarts under the default six-hour grace +/// window, the way a deployment that lapsed hours ago would: the +/// quarantine is this server's own record that the window already ran +/// out, so the subtree stays stopped rather than getting another window. +#[tokio::test(flavor = "multi_thread")] +async fn a_restart_does_not_give_a_lapsed_root_another_grace_window() { + let mut stack = Stack::start_with(&["lapsed"], Server::CONFIG, true).await; + let server = stack.server(); + let laptop = Laptop::stand_up(&stack, server, "laptop").await; + let host_token = laptop.machine.token_file(&laptop.name); + let by_hand = format!("byhand.{}", server.host); + register_agent(&laptop.machine, server, &by_hand, &laptop.name, &host_token) + .await + .ok(); + let agent_token = token_of(&laptop.machine, &by_hand); + let (status, _) = write_as(server, &by_hand, &agent_token).await; + assert_eq!(status, 200, "the agent writes while its host stands"); + + stack + .human + .delete("bot.did.operator", &rkey_of(&laptop.name)); + server.nudge().await; + let (_, paused) = writes_until(server, &by_hand, &agent_token, 403).await; + assert!( + paused.is_some(), + "the subtree is paused once the poll reads the deletion" + ); + + // Restarted under the grace window a deployment actually runs. The + // record is still gone; the lapse it already served does not restart + // with the process. + std::fs::write(&server.config, "[operator]\ngrace_window_hours = 6\n").expect("config"); + let down = stack.servers[0].restart().await; + let server = stack.server(); + server.nudge().await; + server.wait_claimed().await; + let seen = sample_writes(server, &by_hand, &agent_token, Duration::from_secs(3)).await; + eprintln!("lapsed: unreachable {down:?}; writes over the 3 s after the restart: {seen:?}"); + assert!( + seen.iter().all(|status| *status == 403), + "the subtree stays stopped across the restart: {seen:?}" + ); + let (status, body) = create_with( + server, + &agent_token, + &format!("deep.{}", server.host), + "agent", + ) + .await; + assert_eq!( + status, 403, + "nothing beneath the lapsed root creates either: {body}" + ); + let refusal = register_agent( + &laptop.machine, + server, + &format!("late.{}", server.host), + &laptop.name, + &host_token, + ) + .await + .refused(); + assert!(refusal.contains("didbot register:"), "{refusal}"); + assert!(stack.check(&by_hand).await.is_err(), "the walk still fails"); +} + +/// `didbot operate --kind agent` from the human's machine. +async fn operate_agent(stack: &Stack, name: &str) -> Ran { + stack + .laptop + .didbot( + &[ + "operate", + name, + stack.human.did(), + "--kind", + "agent", + "--json", + ], + &[], + ) + .await +} + +/// `[tree]` bounds across a restart: the per-parent hourly budget and the +/// children count. Two creates an hour, four children. +#[tokio::test(flavor = "multi_thread")] +async fn a_restart_and_the_trees_bounds() { + const CONFIG: &str = + "[operator]\ngrace_window_hours = 0\n[tree]\ncreates_per_hour = 2\nmax_children = 4\n\ + [park]\nrate_limit = 100\n"; + let mut stack = Stack::start_with(&["bounds"], CONFIG, true).await; + let server = stack.server(); + let laptop = Laptop::stand_up(&stack, server, "laptop").await; + let host_token = laptop.machine.token_file(&laptop.name); + let host = server.host.clone(); + let a = |n: u32| format!("a{n}.{host}"); + operate_agent(&stack, &a(2)).await.ok(); + let refusal = operate_agent(&stack, &a(3)).await.refused(); + assert!(refusal.contains("CreatesTooFast"), "{refusal}"); + for n in 1..=2 { + register_agent( + &laptop.machine, + server, + &format!("c{n}.{}", server.host), + &laptop.name, + &host_token, + ) + .await + .ok(); + } + let refusal = register_agent( + &laptop.machine, + server, + &format!("c3.{}", server.host), + &laptop.name, + &host_token, + ) + .await + .refused(); + assert!(refusal.contains("CreatesTooFast"), "{refusal}"); + + let down = stack.servers[0].restart().await; + let server = stack.server(); + server.nudge().await; + server.wait_claimed().await; + + let third = operate_agent(&stack, &a(3)).await; + eprintln!( + "bounds: unreachable {down:?}; the operator's third create of the hour after the restart: exit {} {}{}", + third.status, + third.stdout.trim(), + third.stderr.trim() + ); + let fourth = operate_agent(&stack, &a(4)).await; + eprintln!( + "bounds: the operator's fourth root after the restart: exit {} {}{}", + fourth.status, + fourth.stdout.trim(), + fourth.stderr.trim() + ); + let c3 = register_agent( + &laptop.machine, + server, + &format!("c3.{}", server.host), + &laptop.name, + &host_token, + ) + .await; + eprintln!( + "bounds: the host's third create of the hour after the restart: exit {} {}{}", + c3.status, + c3.stdout.trim(), + c3.stderr.trim() + ); +} + +/// A data directory stamped by the build before this stack — layout 16, +/// a different entry shape — is refused by name, before anything is read. +#[tokio::test(flavor = "multi_thread")] +async fn a_data_directory_from_layout_16_is_refused_by_name() { + let mut stack = Stack::start_with(&["layout"], Server::CONFIG, true).await; + let server = stack.server(); + let name = format!("kestrel.{}", server.host); + stack + .laptop + .didbot( + &[ + "operate", + &name, + stack.human.did(), + "--kind", + "agent", + "--json", + ], + &[], + ) + .await + .ok(); + stack.servers[0].kill(); + let server = &mut stack.servers[0]; + let stamp_path = server.data_dir().join("pds.layout"); + let mut stamp: Value = + serde_json::from_str(&std::fs::read_to_string(&stamp_path).expect("a stamp")) + .expect("json"); + assert_eq!(stamp["layout"], 17, "{stamp}"); + // What the build before this stack wrote: its own number, and a shape + // that is not this one's (the `agentToken*` entries were renamed). + stamp["layout"] = json!(16); + stamp["shape"] = json!(stamp["shape"].as_u64().expect("a shape") ^ 1); + std::fs::write(&stamp_path, stamp.to_string()).expect("the stamp rewrites"); + let wal_before = std::fs::read(server.data_dir().join("pds.wal.000000")).expect("the log"); + + let (code, logged) = server.start_again_and_wait(); + eprintln!("layout: exit {code}\n{logged}"); + assert_ne!(code, 0); + assert!(logged.contains("written as version 16"), "{logged}"); + assert!( + logged.contains("check out the build that wrote it, or delete the directory"), + "{logged}" + ); + assert_eq!( + std::fs::read(server.data_dir().join("pds.wal.000000")).expect("the log"), + wal_before, + "the refusal touched nothing" + ); +} + +/// Every account this server says it holds, by DID. +async fn listed_accounts(server: &Server) -> Vec { + get_json(&format!("{}/xrpc/bot.did.listAccounts", server.origin)).await["accounts"] + .as_array() + .map(|accounts| { + accounts + .iter() + .filter_map(|account| account["did"].as_str()) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default() +} + +/// `didbot-pds` killed partway through a `bot.did.createAccount`, at +/// several offsets across the window the call takes: what the directory +/// holds when the process comes back. +/// +/// The account is either there whole — a row, a document, a registration +/// record and the operator's record in the creator's repository — or it is +/// not there at all and its name can be taken again. Anything between +/// those two is a torn account. +#[ignore = "reproduces a finding: every such kill leaves a row that holds its name"] +#[tokio::test(flavor = "multi_thread")] +async fn a_kill_partway_through_a_create_leaves_no_torn_account() { + let mut stack = Stack::start_with(&["torn"], Server::CONFIG, true).await; + // How long a create takes here, so the kills land inside it. + let warm = format!("warm.{}", stack.server().host); + vouch(&stack.human, &warm); + let timed = Instant::now(); + let proof = stack + .human + .service_auth(&stack.server().did, "bot.did.createAccount"); + let (status, _) = create_with(stack.server(), &proof, &warm, "agent").await; + assert_eq!(status, 200); + let takes = timed.elapsed(); + eprintln!("torn: one create takes {takes:?}"); + + let mut torn = Vec::new(); + for (step, kind) in [(2, "agent"), (4, "host"), (6, "host"), (8, "agent")] { + let name = format!("t{step}{kind}.{}", stack.server().host); + let did = did_of(&name); + vouch(&stack.human, &name); + let at = takes.mul_f64(f64::from(step) / 9.0); + let proof = stack + .human + .service_auth(&stack.server().did, "bot.did.createAccount"); + let origin = stack.server().origin.clone(); + let creating = { + let name = name.clone(); + tokio::spawn(async move { + post_json( + &format!("{origin}/xrpc/bot.did.createAccount"), + Some(&proof), + json!({ "name": name, "kind": kind }), + ) + .await + }) + }; + tokio::time::sleep(at).await; + stack.servers[0].kill(); + let answered = creating.await.ok(); + stack.servers[0].start_again().await; + let server = stack.server(); + server.nudge().await; + server.wait_claimed().await; + + let listed = listed_accounts(server).await.contains(&did); + let (document, _) = get_record(&server.origin, &did, "bot.did.registration", "self").await; + let registered = document == 200; + let (operator_record, _) = get_record( + &server.origin, + &server.did, + "bot.did.operator", + &rkey_of(&name), + ) + .await; + // Whether the name can be taken again, which is the only honest + // answer for an account that is not there. + let retry = create_with( + server, + &stack + .human + .service_auth(&server.did, "bot.did.createAccount"), + &name, + kind, + ) + .await; + // What a sweep would ever do about a row nobody finished: only a + // retention that can run out is reaped. + let retention = get_json(&format!("{}/xrpc/bot.did.listAccounts", server.origin)).await + ["accounts"] + .as_array() + .and_then(|accounts| { + accounts + .iter() + .find(|account| account["did"] == did) + .map(|account| account.to_string()) + }) + .unwrap_or_default(); + eprintln!( + "torn: a {kind}, killed {at:?} in; the call answered {:?}; listed={listed} \ + registration={registered} operator_record={operator_record}; \ + creating it again: {} {}; the row: {retention}", + answered.map(|(status, _)| status), + retry.0, + retry.1["error"] + ); + let whole = listed && registered; + let absent = !listed && !registered && retry.0 == 200; + if !(whole || absent) { + torn.push(format!("{name} at {at:?}")); + } + } + assert!(torn.is_empty(), "torn accounts: {torn:?}"); +} + +/// A `bot.did.operator` record in the human's repository naming `name`, the +/// way `didbot operate` writes one, without running the command. +fn vouch(human: &Human, name: &str) { + human.write( + "bot.did.operator", + &rkey_of(name), + json!({ + "$type": "bot.did.operator", + "subject": did_of(name), + "createdAt": now_rfc3339(), + }), + ); +} + +// --------------------------------------------------------------------------- +// A GitHub Actions pipeline, ported from AWS OIDC +// --------------------------------------------------------------------------- + +/// The owner ID `org` carries, whichever of its repositories a run is in. +const GITHUB_OWNER_ID: &str = "123456"; + +/// The claim set GitHub mints for a run, as `docs/pipelines.md` shows it: +/// `sub` in the immutable form, and each ID beside the name it belongs to. +fn github_claims(repository: &str, repository_id: &str, environment: Option<&str>) -> Value { + let owner = repository.split('/').next().unwrap_or_default(); + let name = repository.rsplit('/').next().unwrap_or_default(); + let subject = format!("repo:{owner}@{GITHUB_OWNER_ID}/{name}@{repository_id}"); + let mut claims = json!({ + "sub": match environment { + Some(environment) => format!("{subject}:environment:{environment}"), + None => format!("{subject}:ref:refs/heads/main"), + }, + "repository": repository, + "repository_id": repository_id, + "repository_owner": owner, + "repository_owner_id": GITHUB_OWNER_ID, + "ref": "refs/heads/main", + "ref_type": "branch", + "workflow": "deploy", + "job_workflow_ref": format!("{repository}/.github/workflows/deploy.yml@refs/heads/main"), + "actor": "octocat", + "run_id": "12345", + "nbf": OffsetDateTime::now_utc().unix_timestamp() - 30, + "exp": (OffsetDateTime::now_utc() + time::Duration::minutes(10)).unix_timestamp(), + }); + if let Some(environment) = environment { + claims["environment"] = Value::String(environment.to_owned()); + } + claims +} + +/// `deploy` admits every run of one repository, `deploy-prod` only a run +/// in its `production` environment; each run's token is one of the two +/// and nobody else's; a token is one command, and a stale one is refused. +#[tokio::test(flavor = "multi_thread")] +async fn a_github_pipeline_ported_from_aws() { + let stack = Stack::start(&["gh"]).await; + let server = stack.server(); + let human = stack.human.did(); + let deploy = format!("deploy.{}", server.host); + let prod = format!("deploy-prod.{}", server.host); + let (laptop, issuer) = (&stack.laptop, stack.issuer.base.as_str()); + let operate = |name: &str, claims: &[&str]| { + let mut args = vec![ + "operate", name, human, "--kind", "pipeline", "--oidc", issuer, + ]; + args.extend_from_slice(claims); + args.extend(["--creates", "agent", "--json"]); + let args: Vec = args.into_iter().map(str::to_owned).collect(); + async move { + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + laptop.didbot(&args, &[]).await.ok() + } + }; + assert_eq!( + operate(&deploy, &["repository_id=456789"]).await["did"], + did_of(&deploy) + ); + assert_eq!( + operate(&prod, &["repository_id=456789", "environment=production"]).await["did"], + did_of(&prod) + ); + server.read_allowances().await; + + let pds = [("DIDBOT_PDS", server.origin.as_str())]; + let run = Machine::new(&stack.work, "run-41"); + let mint = |claims: Value| stack.issuer.id_token(&server.did, claims); + + // Step one of the workflow: the run's token is a session as `deploy`. + let token = run.secret_file( + "id-token-1", + &mint(github_claims("org/repo", "456789", None)), + ); + run.didbot( + &[ + "oauth", + "pending", + "--token-file", + token.to_str().expect("utf-8"), + ], + &pds, + ) + .await + .ok(); + // The same token, again, is a replay: a token is one command. + let refusal = run + .didbot( + &[ + "oauth", + "pending", + "--token-file", + token.to_str().expect("utf-8"), + ], + &pds, + ) + .await + .refused(); + eprintln!("replayed: {}", refusal.trim()); + + // Step two: a second token creates the run's agent beneath `deploy`. + let token = run.secret_file( + "id-token-2", + &mint(github_claims("org/repo", "456789", None)), + ); + let agent = format!("run-41.{}", server.host); + let created = run + .didbot( + &[ + "register", + "agent", + &agent, + "--under", + &deploy, + "--server", + &server.host, + "--token-file", + token.to_str().expect("utf-8"), + "--json", + ], + &[], + ) + .await + .ok(); + assert_eq!(created["did"], did_of(&agent), "{created}"); + assert_eq!( + stack.walk(&agent).await, + vec![did_of(&deploy), human.to_owned()] + ); + + // A production run carries `environment`, and is the narrower account. + let session = |token: String| async move { + post_json( + &format!("{}/xrpc/bot.did.createSession", server.origin), + Some(&token), + json!({}), + ) + .await + }; + let (status, body) = session(mint(github_claims( + "org/repo", + "456789", + Some("production"), + ))) + .await; + assert_eq!(status, 200, "{body}"); + assert_eq!(body["did"], did_of(&prod), "{body}"); + let (status, body) = session(mint(github_claims("org/repo", "456789", Some("staging")))).await; + assert_eq!(status, 200, "{body}"); + assert_eq!( + body["did"], + did_of(&deploy), + "a staging run is only `deploy`: {body}" + ); + + // Another repository, even under the same owner, is nobody here. + let (status, body) = session(mint(github_claims("org/other", "456790", None))).await; + assert_eq!(status, 401, "{body}"); + eprintln!("other repository: {}", body["message"]); + + // A token minted six minutes ago is stale, whatever its `exp`. + let mut old = github_claims("org/repo", "456789", None); + old["iat"] = json!(OffsetDateTime::now_utc().unix_timestamp() - 6 * 60); + let (status, body) = session(mint(old)).await; + assert_eq!(status, 401, "{body}"); + eprintln!("six minutes old: {}", body["message"]); +} + +// --------------------------------------------------------------------------- +// A Google service account and a Kubernetes service account +// --------------------------------------------------------------------------- + +/// The service account's numeric unique ID, which Google never hands out +/// a second time however often the address is. +const GOOGLE_SUB: &str = "104521389012345678901"; + +/// The UID Kubernetes gave the `runner` service account, new again if that +/// account is deleted and recreated under the same name. +const K8S_ACCOUNT_UID: &str = "6f2a0d3e-9c41-4a86-8f0b-1d5e7c2b9a44"; + +/// Google's shape: a numeric `sub`, `email`, `email_verified`, no `jti`, +/// an hour long. `instance` is what `format=full` adds and what tells two +/// instances of one service account apart; without it their tokens are the +/// same bytes and the second is a replay. +fn google_claims(email: &str, instance: Option<&str>) -> Value { + let mut claims = json!({ + "sub": GOOGLE_SUB, + "email": email, + "email_verified": true, + "azp": GOOGLE_SUB, + "jti": null, + "exp": (OffsetDateTime::now_utc() + time::Duration::hours(1)).unix_timestamp(), + }); + if let Some(instance) = instance { + claims["google"] = json!({ "compute_engine": { "instance_id": instance } }); + } + claims +} + +/// Kubernetes' shape: `aud` as an array, `sub` naming the service account, +/// the pod and account, each with its UID, under a nested `kubernetes.io`, +/// an hour long. +fn kubernetes_claims(aud: &str, namespace: &str, account: &str, pod: &str) -> Value { + json!({ + "aud": [aud], + "sub": format!("system:serviceaccount:{namespace}:{account}"), + // A projected token carries one, so two pods' tokens differ. + "jti": OffsetDateTime::now_utc().unix_timestamp_nanos().to_string(), + "kubernetes.io": { + "namespace": namespace, + "pod": { "name": pod, "uid": "b6e1c7f2-3a58-4d90-9e12-77c4ab0e5d31" }, + "serviceaccount": { "name": account, "uid": K8S_ACCOUNT_UID }, + }, + "exp": (OffsetDateTime::now_utc() + time::Duration::hours(1)).unix_timestamp(), + }) +} + +/// Twenty instances of a Google-identified service register at once; a +/// Kubernetes service account registers by its UID, and one of its pods by +/// that and a nested claim; the token a metadata server hands out twice is +/// a replay the second time and stale after five minutes. +#[tokio::test(flavor = "multi_thread")] +async fn google_and_kubernetes_pools() { + let stack = Stack::start(&["pools"]).await; + let server = stack.server(); + let human = stack.human.did(); + let email = "web@project.iam.gserviceaccount.com"; + let web = format!("web.{}", server.host); + let k8s = format!("k8s.{}", server.host); + let pod = format!("pod.{}", server.host); + for (name, claims) in [ + (&web, vec![format!("sub={GOOGLE_SUB}")]), + ( + &k8s, + vec![format!( + "/kubernetes.io/serviceaccount/uid={K8S_ACCOUNT_UID}" + )], + ), + ( + &pod, + vec![ + format!("/kubernetes.io/serviceaccount/uid={K8S_ACCOUNT_UID}"), + "/kubernetes.io/pod/name=runner-0".to_owned(), + ], + ), + ] { + let mut args = vec![ + "operate", + name, + human, + "--kind", + "service", + "--oidc", + &stack.issuer.base, + ]; + args.extend(claims.iter().map(String::as_str)); + args.extend(["--creates", "host", "--json"]); + let admitted = stack.laptop.didbot(&args, &[]).await.ok(); + assert_eq!(admitted["did"], did_of(name), "{admitted}"); + } + server.read_allowances().await; + + // Twenty instances boot at once, each with the token its metadata + // server minted for it. + let mut boots = Vec::new(); + for i in 1..=20 { + let machine = Machine::new(&stack.work, &format!("g-{i}")); + let name = format!("g-{i}.{}", server.host); + let token = machine.secret_file( + "id-token", + &stack + .issuer + .id_token(&server.did, google_claims(email, Some(&format!("i-{i}")))), + ); + let (web, host) = (web.clone(), server.host.clone()); + boots.push(tokio::spawn(async move { + let ran = machine + .didbot( + &[ + "register", + "host", + &name, + "--under", + &web, + "--server", + &host, + "--token-file", + token.to_str().expect("utf-8"), + "--json", + ], + &[], + ) + .await; + (name, ran) + })); + } + let mut names = Vec::new(); + for boot in boots { + let (name, ran) = boot.await.expect("the boot task ran"); + assert_eq!(ran.ok()["did"], did_of(&name)); + names.push(rkey_of(&name)); + } + names.sort(); + assert_eq!( + list_record_keys(&server.origin, &did_of(&web), "bot.did.operator").await, + names + ); + assert_eq!( + stack.walk(&format!("g-7.{}", server.host)).await, + vec![did_of(&web), human.to_owned()] + ); + + // The metadata server hands the same token out again within the hour: + // the second boot with it is a replay. Six minutes on, it is stale. + let again = Machine::new(&stack.work, "g-again"); + let token = stack + .issuer + .id_token(&server.did, google_claims(email, Some("i-again"))); + let file = again.secret_file("id-token", &token); + let register = |machine: &Machine, name: String, file: PathBuf| { + let (web, host) = (web.clone(), server.host.clone()); + let machine = Machine { + home: machine.home.clone(), + config: machine.config.clone(), + state: machine.state.clone(), + }; + async move { + machine + .didbot( + &[ + "register", + "host", + &name, + "--under", + &web, + "--server", + &host, + "--token-file", + file.to_str().expect("utf-8"), + "--json", + ], + &[], + ) + .await + } + }; + register(&again, format!("g-again.{}", server.host), file.clone()) + .await + .ok(); + let refusal = register(&again, format!("g-again-2.{}", server.host), file) + .await + .refused(); + eprintln!("same token twice: {}", refusal.trim()); + let mut stale = google_claims(email, Some("i-stale")); + stale["iat"] = json!(OffsetDateTime::now_utc().unix_timestamp() - 6 * 60); + let file = again.secret_file("id-token-stale", &stack.issuer.id_token(&server.did, stale)); + let refusal = register(&again, format!("g-stale.{}", server.host), file) + .await + .refused(); + eprintln!("six minutes old: {}", refusal.trim()); + + // Two instances of one service account whose tokens carry nothing to + // tell them apart mint the same bytes, and the same bytes are one + // credential: the second is a replay. `format=full` is what avoids it. + let plain = stack + .issuer + .id_token(&server.did, google_claims(email, None)); + assert_eq!( + plain, + stack + .issuer + .id_token(&server.did, google_claims(email, None)), + "two instances of one service account mint the same token" + ); + + // A Kubernetes pod's projected token. `runner-0`'s carries both the + // service account's `sub` and the pod's nested name, so it is the + // account naming both; naming the other as `--under` is refused and + // says which account the token actually is. + // One `didbot register host` per machine: a node that already + // registered is already a host. + let register_k8s = |node: &'static str, claims: Value, name: String, under: String| { + let (host, machine) = (server.host.clone(), Machine::new(&stack.work, node)); + let file = machine.secret_file("token", &stack.issuer.id_token(&server.did, claims)); + async move { + machine + .didbot( + &[ + "register", + "host", + &name, + "--under", + &under, + "--server", + &host, + "--token-file", + file.to_str().expect("utf-8"), + "--json", + ], + &[], + ) + .await + } + }; + let name = format!("runner-0.{}", server.host); + let created = register_k8s( + "node-1", + kubernetes_claims(&server.did, "agents", "runner", "runner-0"), + name.clone(), + pod.clone(), + ) + .await + .ok(); + assert_eq!(created["did"], did_of(&name), "{created}"); + let registered = registration(server, &did_of(&name)).await; + assert_eq!(registered["operator"], human); + assert_eq!(registered["lineage"], json!([did_of(&pod)])); + + let refusal = register_k8s( + "node-2", + kubernetes_claims(&server.did, "agents", "runner", "runner-0"), + format!("runner-0b.{}", server.host), + k8s.clone(), + ) + .await + .refused(); + eprintln!( + "the pod's token, named under the service account: {}", + refusal.trim() + ); + + // Another pod of the same service account matches the account's UID + // alone, so it is the service account's own instance. + let name = format!("runner-9.{}", server.host); + let created = register_k8s( + "node-3", + kubernetes_claims(&server.did, "agents", "runner", "runner-9"), + name.clone(), + k8s.clone(), + ) + .await + .ok(); + assert_eq!(created["did"], did_of(&name), "{created}"); + let registered = registration(server, &did_of(&name)).await; + assert_eq!(registered["operator"], human); + assert_eq!(registered["lineage"], json!([did_of(&k8s)])); +} + +// --------------------------------------------------------------------------- +// A consumer that knows nothing of didbot +// --------------------------------------------------------------------------- + +/// How long `com.atproto.sync.subscribeRepos` has to stay quiet before a +/// drain is over. +const QUIET: Duration = Duration::from_secs(2); + +/// One op of a `#commit`, with its record read out of the frame's own CAR. +#[derive(Debug, Clone)] +struct Op { + action: String, + path: String, + record: Option, +} + +/// One frame, decoded the way a consumer reading only the lexicon would. +#[derive(Debug, Clone)] +struct Ev { + seq: i64, + tag: String, + did: String, + body: Value, + ops: Vec, +} + +fn dag_str(fields: &BTreeMap, key: &str) -> String { + match fields.get(key) { + Some(Dag::String(value)) => value.clone(), + _ => String::new(), + } +} + +impl Ev { + fn parse(bytes: &[u8]) -> Self { + let (header, body) = (1..bytes.len()) + .find_map(|split| { + let header = dag_cbor::decode(&bytes[..split]).ok()?; + let body = dag_cbor::decode(&bytes[split..]).ok()?; + Some((header, body)) + }) + .expect("a frame is two concatenated DAG-CBOR values"); + let header = header.to_json(); + let tag = header["t"].as_str().unwrap_or("").to_owned(); + let mut ops = Vec::new(); + if let (true, Dag::Map(fields)) = (tag == "#commit", &body) { + let blocks = match fields.get("blocks") { + Some(Dag::Bytes(car)) if !car.is_empty() => { + didbot_repo::car::read(car).expect("a CAR").blocks + } + _ => BTreeMap::new(), + }; + if let Some(Dag::List(list)) = fields.get("ops") { + for op in list { + let Dag::Map(op) = op else { continue }; + let record = match op.get("cid") { + Some(Dag::Link(cid)) => blocks + .get(cid) + .map(|block| dag_cbor::decode(block).expect("a record").to_json()), + _ => None, + }; + ops.push(Op { + action: dag_str(op, "action"), + path: dag_str(op, "path"), + record, + }); + } + } + } + let body = body.to_json(); + let did = body["did"] + .as_str() + .or(body["repo"].as_str()) + .unwrap_or("") + .to_owned(); + let seq = body["seq"].as_i64().unwrap_or(-1); + Self { + seq, + tag, + did, + body, + ops, + } + } + + /// The ops in one collection, as `(rkey, op)`. + fn ops_in<'a>(&'a self, collection: &'a str) -> impl Iterator { + self.ops.iter().filter_map(move |op| { + op.path + .strip_prefix(collection) + .and_then(|rest| rest.strip_prefix('/')) + .map(|rkey| (rkey, op)) + }) + } + + /// One line per frame, the way the report prints a sequence. + fn line(&self) -> String { + let did = self.did.replace("%3A", ":"); + match self.tag.as_str() { + "#commit" => format!( + "{:>4} #commit {did} rev={} {}", + self.seq, + self.body["rev"].as_str().unwrap_or(""), + self.ops + .iter() + .map(|op| format!("{}:{}", op.action, op.path)) + .collect::>() + .join(" ") + ), + "#identity" => format!( + "{:>4} #identity {did} handle={}", + self.seq, + self.body["handle"].as_str().unwrap_or("-") + ), + "#account" => format!( + "{:>4} #account {did} active={} status={}", + self.seq, + self.body["active"], + self.body["status"].as_str().unwrap_or("-") + ), + _ => format!("{:>4} {} {}", self.seq, self.tag, self.body), + } + } +} + +/// One connection to the stream, and everything it has read. +struct Consumer { + client: support::ws::Client, + seen: Vec, +} + +impl Consumer { + async fn open(server: &Server, cursor: Option) -> Self { + let path = match cursor { + Some(cursor) => format!("/xrpc/com.atproto.sync.subscribeRepos?cursor={cursor}"), + None => "/xrpc/com.atproto.sync.subscribeRepos".to_owned(), + }; + Self { + client: support::ws::Client::open(&server.addr(), &path).await, + seen: Vec::new(), + } + } + + /// Everything the stream carries until it has been quiet for `QUIET`. + async fn drain(&mut self) -> Vec { + let mut out = Vec::new(); + while let Ok(Some(bytes)) = tokio::time::timeout(QUIET, self.client.next()).await { + out.push(Ev::parse(&bytes)); + } + self.seen.extend(out.iter().cloned()); + out + } + + fn last_seq(&self) -> i64 { + self.seen.last().map_or(0, |ev| ev.seq) + } +} + +fn print_frames(title: &str, events: &[Ev]) { + eprintln!("--- {title} ({} frames) ---", events.len()); + for ev in events { + eprintln!("{}", ev.line()); + } +} + +/// The tree as a consumer rebuilds it from the stream alone. +#[derive(Default, Debug)] +struct Tree { + /// DID → (seq, record) of its `bot.did.registration/self`. + registrations: BTreeMap, + /// Child → (parent, seq): a `bot.did.operator` record in a hosted + /// repository, removed again when the record is deleted. + edges: BTreeMap, + identities: BTreeMap>, + /// DID → every `#account` as (seq, active, status). + accounts: BTreeMap>, + first_commit: BTreeMap, +} + +fn rebuild(events: &[Ev]) -> Tree { + let mut tree = Tree::default(); + for ev in events { + match ev.tag.as_str() { + "#identity" => tree + .identities + .entry(ev.did.clone()) + .or_default() + .push(ev.seq), + "#account" => tree.accounts.entry(ev.did.clone()).or_default().push(( + ev.seq, + ev.body["active"].as_bool().unwrap_or(false), + ev.body["status"].as_str().unwrap_or("").to_owned(), + )), + "#commit" => { + tree.first_commit.entry(ev.did.clone()).or_insert(ev.seq); + for (rkey, op) in ev.ops_in("bot.did.registration") { + if let ("self", Some(record)) = (rkey, &op.record) { + tree.registrations + .insert(ev.did.clone(), (ev.seq, record.clone())); + } + } + for (rkey, op) in ev.ops_in("bot.did.operator") { + match (op.action.as_str(), &op.record) { + ("delete", _) => { + tree.edges.remove(&did_of(rkey)); + } + (_, Some(record)) => { + let subject = record["subject"].as_str().unwrap_or("").to_owned(); + tree.edges.insert(subject, (ev.did.clone(), ev.seq)); + } + _ => {} + } + } + } + _ => {} + } + } + tree +} + +/// `com.atproto.sync.listRepos`, as anyone reads it: `(did, active)`. +async fn list_repos(server: &Server) -> Vec<(String, bool)> { + let body = get_json(&format!( + "{}/xrpc/com.atproto.sync.listRepos?limit=1000", + server.origin + )) + .await; + body["repos"] + .as_array() + .map(|repos| { + repos + .iter() + .filter_map(|repo| { + Some(( + repo["did"].as_str()?.to_owned(), + repo["active"].as_bool().unwrap_or(true), + )) + }) + .collect() + }) + .unwrap_or_default() +} + +/// A `com.atproto.sync.*` read of one repository, as anyone makes it: the +/// DID goes through the query encoder, so a development port's `%3A` +/// survives. +async fn sync_read(server: &Server, method: &str, did: &str) -> (u16, Value) { + let response = http() + .get(format!("{}/xrpc/{method}", server.origin)) + .query(&[("did", did)]) + .send() + .await + .unwrap_or_else(|err| panic!("{method} {did}: {err}")); + let status = response.status().as_u16(); + (status, response.json().await.unwrap_or(Value::Null)) +} + +fn name_of(did: &str) -> String { + did.trim_start_matches("did:web:").replace("%3A", ":") +} + +/// What a resolver finds at a hosted DID: the endpoint and the one key the +/// document names, the human it names as `operator`, and nothing that is +/// not a public half. What logs in as the account is not in it. +async fn inspect_document(server: &Server, human: &str, did: &str) { + let doc = document(&name_of(did)).await; + assert_eq!(doc["id"], did, "{doc}"); + let services = doc["service"].as_array().cloned().unwrap_or_default(); + let pds = services + .iter() + .find(|service| { + service["id"] + .as_str() + .is_some_and(|id| id.ends_with("#atproto_pds")) + }) + .unwrap_or_else(|| panic!("{did} names no #atproto_pds: {doc}")); + assert_eq!(pds["serviceEndpoint"], server.origin, "{did}: {doc}"); + let methods = doc["verificationMethod"] + .as_array() + .cloned() + .unwrap_or_default(); + for method in &methods { + let keys: BTreeSet<&str> = method + .as_object() + .expect("a map") + .keys() + .map(String::as_str) + .collect(); + assert_eq!( + keys, + ["controller", "id", "publicKeyMultibase", "type"] + .into_iter() + .collect(), + "{did}: a verification method carries more than a public key: {method}" + ); + assert_eq!(method["type"], "Multikey"); + let multibase = method["publicKeyMultibase"].as_str().expect("a string"); + assert!(multibase.starts_with('z'), "{did}: {multibase}"); + assert!( + didbot_key::VerifyingKey::from_multibase(multibase).is_ok() + || didbot_key::secp256r1::VerifyingKey::from_multibase(multibase).is_ok(), + "{did}: {multibase} is not a public key this deployment's own parsers read" + ); + } + assert_eq!(methods.len(), 1, "{did}: one verification method: {doc}"); + assert!( + methods[0]["id"] + .as_str() + .is_some_and(|id| id.ends_with("#atproto")), + "{did}: the one key is #atproto: {doc}" + ); + assert!( + doc.get("authentication").is_none(), + "{did}: what logs in is a credential record, never the document: {doc}" + ); + let text = doc.to_string(); + for needle in ["\"d\"", "privateKey", "secret", "BEGIN"] { + assert!( + !text.contains(needle), + "{did}: the document carries {needle}: {doc}" + ); + } + assert_eq!( + doc["operator"], human, + "{did}: the document names the human: {doc}" + ); +} + +/// The full tree, read off `com.atproto.sync.subscribeRepos` from the +/// start and from a late cursor by something that has never heard of +/// `bot.did.listAccounts`: every account's registration and every hosted +/// edge are on the stream, in an order a consumer can apply, once each; +/// the edges into the human's repository are not, and the registration +/// names the human and every account between. Then what a deletion and a +/// revocation put on the stream. +#[tokio::test(flavor = "multi_thread")] +async fn a_consumer_rebuilds_the_tree_from_the_firehose() { + // Every command here runs from one address, and `didbot operate` asks + // what is parked on every create, so the park budget is raised past + // what one afternoon of admitting accounts spends. + const CONFIG: &str = "[operator]\ngrace_window_hours = 0\n\n[park]\nrate_limit = 1000\n"; + let stack = Stack::start_with(&["consumer"], CONFIG, false).await; + let server = stack.server(); + let human = stack.human.did().to_owned(); + let mut from_start = Consumer::open(server, Some(0)).await; + let boot = from_start.drain().await; + print_frames("boot: the server's own account", &boot); + // The server's own account opens the stream the way every other + // account does, so a consumer learns the DID before it is handed a + // commit in its repository. + assert_eq!( + boot.iter() + .map(|ev| (ev.tag.as_str(), ev.did.as_str())) + .collect::>(), + vec![ + ("#identity", server.did.as_str()), + ("#account", server.did.as_str()), + ("#commit", server.did.as_str()), + ], + "the server's own account is announced before its first commit" + ); + + // One agent by hand. + let kestrel = format!("kestrel.{}", server.host); + stack + .laptop + .didbot( + &["operate", &kestrel, &human, "--kind", "agent", "--json"], + &[], + ) + .await + .ok(); + let after_kestrel = from_start.drain().await; + print_frames("an agent created by the human", &after_kestrel); + + // A host, then its daemon's first agent. + let laptop = Laptop::stand_up(&stack, server, "laptop").await; + let after_host = from_start.drain().await; + print_frames("a host admitted by parked key", &after_host); + let mark = from_start.last_seq(); + let agent = laptop.daemon.agent("ctx-1").await; + let after_agent = from_start.drain().await; + print_frames("the host's daemon creates an agent", &after_agent); + + // A service that is an OIDC identity, an instance beneath it, and the + // instance's agent: the deepest walk the default tree allows. + let web = format!("web.{}", server.host); + stack + .laptop + .didbot( + &[ + "operate", + &web, + &human, + "--kind", + "service", + "--oidc", + &stack.issuer.base, + "email=web@pool", + "--creates", + "host", + "--creates-beneath", + "agent", + "--json", + ], + &[], + ) + .await + .ok(); + server.read_allowances().await; + let instance = Machine::new(&stack.work, "i-1"); + let i1 = format!("i-1.{}", server.host); + let token = stack + .issuer + .id_token(&server.did, json!({ "email": "web@pool" })); + let token_file = instance.secret_file("id-token", &token); + instance + .didbot( + &[ + "register", + "host", + &i1, + "--under", + &web, + "--server", + &server.host, + "--token-file", + token_file.to_str().expect("utf-8"), + "--json", + ], + &[], + ) + .await + .ok(); + let i1_daemon = Daemon::start(&instance, server).await; + let deep = i1_daemon.agent("i1-ctx-1").await; + let after_pool = from_start.drain().await; + print_frames("a service, an instance and its agent", &after_pool); + + // A late consumer sees exactly the frames after its cursor, once each. + let mut late = Consumer::open(server, Some(mark)).await; + let tail: Vec = late.drain().await.iter().map(|ev| ev.seq).collect(); + let expected: Vec = from_start + .seen + .iter() + .filter(|ev| ev.seq > mark) + .map(|ev| ev.seq) + .collect(); + assert_eq!(tail, expected, "a resume from {mark} is the tail, once"); + for window in from_start.seen.windows(2) { + assert!( + window[0].seq < window[1].seq, + "sequence numbers rise: {} then {}", + window[0].seq, + window[1].seq + ); + } + assert!( + from_start + .seen + .iter() + .all(|ev| ev.tag.starts_with('#') && ev.tag != "#info"), + "no #info or error frames" + ); + + // The consumer's tree, against what a public listing says exists. + let tree = rebuild(&from_start.seen); + let listed = list_repos(server).await; + let hosted: Vec = listed + .iter() + .map(|(did, _)| did.clone()) + .filter(|did| did != &server.did) + .collect(); + assert!( + hosted.len() >= 6, + "kestrel, laptop, its agent, web, i-1 and its agent: {hosted:?}" + ); + for did in &hosted { + let (seq, registration) = tree + .registrations + .get(did) + .unwrap_or_else(|| panic!("{did}'s registration never reached the stream")); + let operator = registration["operator"].as_str().expect("an operator"); + assert_eq!(operator, human, "{did}: the registration names the human"); + let parent = registration["lineage"] + .as_array() + .and_then(|lineage| lineage.last()) + .and_then(Value::as_str); + let identity = tree + .identities + .get(did) + .unwrap_or_else(|| panic!("{did} had no #identity")); + assert_eq!(identity.len(), 1, "{did}: one #identity"); + let accounts = &tree.accounts[did]; + assert_eq!(accounts.len(), 1, "{did}: one #account: {accounts:?}"); + assert!(accounts[0].1, "{did}: announced active"); + assert!( + identity[0] < accounts[0].0 && accounts[0].0 < tree.first_commit[did], + "{did}: #identity, then #account, then its commits" + ); + assert!(*seq >= tree.first_commit[did]); + match parent { + None => { + assert!( + !tree.edges.contains_key(did), + "{did}: the human's edge is not on this stream" + ); + // The join: the registration names the human; the human's + // document names their PDS; the record is keyed by the + // hostname. + let operator_doc = + get_json(&format!("{}/.well-known/did.json", stack.human.origin())).await; + let pds = operator_doc["service"] + .as_array() + .and_then(|s| s.iter().find(|s| s["id"] == "#atproto_pds")) + .and_then(|s| s["serviceEndpoint"].as_str()) + .expect("the human's PDS"); + let (status, record) = + get_record(pds, operator, "bot.did.operator", &name_of(did)).await; + assert_eq!(status, 200, "{did}: the human's record reads: {record}"); + assert_eq!(record["value"]["subject"], *did); + } + Some(parent) => { + let (edge_parent, edge_seq) = tree + .edges + .get(did) + .unwrap_or_else(|| panic!("{did}: no hosted edge on the stream")); + assert_eq!( + edge_parent, parent, + "{did}: the edge and the lineage name the same parent" + ); + assert!(*edge_seq < identity[0], "{did}: the parent's operator record ({edge_seq}) precedes the child's #identity ({})", identity[0]); + assert!( + tree.registrations.contains_key(parent), + "{did}: the parent {parent} is itself registered" + ); + } + } + } + + // What a resolver finds at each DID, and what a stranger's walk says. + for did in &hosted { + let kind = tree.registrations[did].1["kind"] + .as_str() + .unwrap_or("") + .to_owned(); + inspect_document(server, &human, did).await; + let held = credentials(server, did).await; + eprintln!("document {did}: kind={kind} credentials={held:?}"); + if kind == "host" { + assert!( + held.iter().any(is_key_credential), + "{did}: a host holds its own key" + ); + } + stack.walk(&name_of(did)).await; + } + inspect_document(server, &human, &server.did).await; + + // A hosted parent cannot end its own edge: the record is the server's. + let host_token = + std::fs::read_to_string(laptop.machine.token_file(&laptop.name)).expect("the host's token"); + let (status, body) = post_json( + &format!("{}/xrpc/com.atproto.repo.deleteRecord", server.origin), + Some(host_token.trim()), + json!({ "repo": laptop.did, "collection": "bot.did.operator", "rkey": name_of(&agent) }), + ) + .await; + assert_eq!( + status, 403, + "the host may not delete its operator record: {body}" + ); + assert!( + from_start.drain().await.is_empty(), + "a refused write puts nothing on the stream" + ); + + // An account deleting itself: the parent's edge goes, then the account. + let by_hand = format!("byhand.{}", server.host); + let host_token_file = laptop.machine.token_file(&laptop.name); + laptop + .machine + .didbot( + &[ + "register", + "agent", + &by_hand, + "--under", + &laptop.name, + "--server", + &server.host, + "--token-file", + host_token_file.to_str().expect("utf-8"), + "--json", + ], + &[], + ) + .await + .ok(); + from_start.drain().await; + let by_hand_token = + std::fs::read_to_string(laptop.machine.token_file(&by_hand)).expect("the agent's token"); + let (status, body) = post_json( + &format!("{}/xrpc/bot.did.deleteAccount", server.origin), + Some(by_hand_token.trim()), + json!({ "did": did_of(&by_hand) }), + ) + .await; + assert_eq!(status, 200, "{body}"); + let on_delete = from_start.drain().await; + print_frames("an agent deletes itself", &on_delete); + let lines: Vec = on_delete + .iter() + .map(|ev| format!("{} {}", ev.tag, ev.did)) + .collect(); + assert_eq!( + lines, + vec![ + format!("#commit {}", laptop.did), + format!("#account {}", did_of(&by_hand)) + ], + "the parent's edge is removed, then the account is announced gone" + ); + assert_eq!(on_delete[0].ops[0].action, "delete"); + assert_eq!( + on_delete[0].ops[0].path, + format!("bot.did.operator/{by_hand}") + ); + assert_eq!(on_delete[1].body["active"], false); + assert_eq!(on_delete[1].body["status"], "deleted"); + let (_, status) = sync_read(server, "com.atproto.sync.getRepoStatus", &did_of(&by_hand)).await; + assert_eq!(status["active"], false, "{status}"); + + // The human revokes the host: the server pauses the subtree at its + // next poll, and the stream says what it says. + stack + .human + .delete("bot.did.operator", &rkey_of(&laptop.name)); + server.nudge().await; + let write_as_agent = || async { + post_json( + &format!("{}/xrpc/com.atproto.repo.createRecord", server.origin), + Some(host_token.trim()), + json!({ "repo": laptop.did, "collection": "com.example.thing", "record": { "text": "still here", "emoji": "\u{1f9ee}", "createdAt": now_rfc3339() } }), + ) + .await + }; + let refused = wait_for("the server to stop the host's subtree", || async { + let (status, body) = write_as_agent().await; + (status == 403).then_some(body) + }) + .await; + assert_eq!(refused["error"], "AccountNotWritable", "{refused}"); + let on_revoke = from_start.drain().await; + print_frames( + "the human deletes the host's record; the server pauses the subtree", + &on_revoke, + ); + // What the pause puts on the wire, and what it does not: the subtree + // refuses writes, `getRepoStatus` still says active, `#account` says + // nothing, and each paused account's registration is the one written + // when it was created: a commit naming it carries no new block for it. + for did in [&laptop.did, &agent] { + let (_, status) = sync_read(server, "com.atproto.sync.getRepoStatus", did).await; + assert_eq!( + status["active"], true, + "{did}: a quarantined repository still reads: {status}" + ); + let (_, at_creation) = &tree.registrations[did]; + for ev in on_revoke + .iter() + .filter(|ev| ev.tag == "#commit" && &ev.did == did) + { + for (_, op) in ev.ops_in("bot.did.registration") { + assert!( + !op.record + .as_ref() + .is_some_and(|record| record != at_creation), + "{did}: the registration is written once: {op:?}" + ); + } + } + assert_eq!( + registration(server, did).await, + *at_creation, + "{did}: the registration is written once" + ); + } + assert!( + !on_revoke.iter().any(|ev| ev.tag == "#account"), + "the pause is not on #account: {:?}", + on_revoke.iter().map(Ev::line).collect::>() + ); + assert!( + stack.check(&name_of(&agent)).await.is_err(), + "the walk fails once the human's record is gone" + ); + drop(deep); +} + +/// One creator making a hundred accounts in well under a minute and then +/// erasing half: every frame numbered once and rising, every deletion +/// announced, the heads the stream implies being the heads the server +/// serves, and a restart on the same log never reissuing a number. +#[tokio::test(flavor = "multi_thread")] +async fn a_hundred_creates_and_fifty_deletes_keep_the_stream_whole() { + const CONFIG: &str = "[operator]\ngrace_window_hours = 0\n\n[tree]\nmax_children = 200\ncreates_per_hour = 1000\n"; + let mut stack = Stack::start_with(&["load"], CONFIG, true).await; + let server = stack.server(); + let machine = Machine::new(&stack.work, "host"); + let host = format!("host.{}", server.host); + let host_did = stack.register_host(&machine, server, &host).await; + let token = std::fs::read_to_string(machine.token_file(&host)) + .expect("the host's token") + .trim() + .to_owned(); + + let mut consumer = Consumer::open(server, Some(0)).await; + consumer.drain().await; + + let started = Instant::now(); + let mut children: Vec<(String, String, String)> = Vec::new(); + for batch in (0..100).collect::>().chunks(10) { + let mut tasks = Vec::new(); + for i in batch { + let url = format!("{}/xrpc/bot.did.createAccount", server.origin); + let token = token.clone(); + let name = format!("a{i}.{}", server.host); + tasks.push(tokio::spawn(async move { + let (status, body) = + post_json(&url, Some(&token), json!({ "name": name, "kind": "agent" })).await; + (name, status, body) + })); + } + for task in tasks { + let (name, status, body) = task.await.expect("the create task ran"); + assert_eq!(status, 200, "{name}: {body}"); + children.push(( + name, + body["did"].as_str().expect("a did").to_owned(), + body["accountToken"].as_str().expect("a token").to_owned(), + )); + } + } + let creating = started.elapsed(); + eprintln!("100 creates in {creating:?}"); + assert!( + creating < Duration::from_secs(60), + "a hundred creates in {creating:?}" + ); + + let deleted: BTreeSet = children + .iter() + .step_by(2) + .map(|(_, did, _)| did.clone()) + .collect(); + for (name, did, child_token) in children.iter().step_by(2) { + let (status, body) = post_json( + &format!("{}/xrpc/bot.did.deleteAccount", server.origin), + Some(child_token), + json!({ "did": did }), + ) + .await; + assert_eq!(status, 200, "{name}: {body}"); + } + consumer.drain().await; + + let seqs: Vec = consumer.seen.iter().map(|ev| ev.seq).collect(); + for window in seqs.windows(2) { + assert!( + window[0] < window[1], + "sequence numbers rise: {} then {}", + window[0], + window[1] + ); + } + let gaps = seqs.windows(2).filter(|w| w[1] != w[0] + 1).count(); + eprintln!( + "{} frames, seq {}..={}, {gaps} gaps", + seqs.len(), + seqs[0], + seqs[seqs.len() - 1] + ); + assert_eq!(gaps, 0, "no number is skipped within one run"); + assert!( + consumer + .seen + .iter() + .all(|ev| ev.tag != "#info" && !ev.tag.is_empty()), + "no #info or error frames" + ); + + let tree = rebuild(&consumer.seen); + for (name, did, _) in &children { + assert_eq!( + tree.identities.get(did).map(Vec::len), + Some(1), + "{name}: one #identity" + ); + assert_eq!( + tree.registrations[did].1["operator"], + stack.human.did(), + "{name}" + ); + assert_eq!( + tree.registrations[did].1["lineage"], + json!([host_did]), + "{name}" + ); + let accounts: Vec<(bool, &str)> = tree.accounts[did] + .iter() + .map(|(_, active, status)| (*active, status.as_str())) + .collect(); + if deleted.contains(did) { + assert_eq!(accounts, vec![(true, ""), (false, "deleted")], "{name}"); + assert!( + !tree.edges.contains_key(did), + "{name}: the edge is gone from the stream" + ); + let (_, status) = sync_read(server, "com.atproto.sync.getRepoStatus", did).await; + assert_eq!(status["active"], false, "{name}: {status}"); + assert_eq!(status["status"], "deleted", "{name}: {status}"); + let (gone, body) = sync_read(server, "com.atproto.sync.getRepo", did).await; + assert_ne!( + gone, 200, + "{name}: getRepo serves a deleted repository: {body}" + ); + } else { + assert_eq!(accounts, vec![(true, "")], "{name}"); + assert_eq!(tree.edges[did].0, host_did, "{name}"); + let last = consumer + .seen + .iter() + .rev() + .find(|ev| ev.tag == "#commit" && &ev.did == did) + .expect("a commit"); + let (_, head) = sync_read(server, "com.atproto.sync.getLatestCommit", did).await; + assert_eq!( + head["cid"], last.body["commit"]["$link"], + "{name}: the head is the last commit on the stream" + ); + assert_eq!(head["rev"], last.body["rev"], "{name}"); + } + } + let listed = list_repos(server).await; + let live: BTreeSet<&str> = listed + .iter() + .filter(|(_, active)| *active) + .map(|(did, _)| did.as_str()) + .collect(); + for (name, did, _) in &children { + assert_eq!( + live.contains(did.as_str()), + !deleted.contains(did), + "{name}: listRepos agrees with the stream" + ); + } + + // A restart on the same log: nothing is renumbered, nothing is replayed + // as new, and the heads survive. + let before = consumer.last_seq(); + let heads: Vec<(String, Value)> = { + let mut heads = Vec::new(); + for (_, did, _) in children + .iter() + .filter(|(_, did, _)| !deleted.contains(did)) + .take(5) + { + heads.push(( + did.clone(), + sync_read(server, "com.atproto.sync.getLatestCommit", did) + .await + .1, + )); + } + heads + }; + stack.servers[0].restart().await; + let server = stack.server(); + server.nudge().await; + server.wait_claimed().await; + server.read_allowances().await; + let mut resumed = Consumer::open(server, Some(before)).await; + let on_resume = resumed.drain().await; + print_frames( + &format!("a consumer resuming from {before} after the restart"), + &on_resume, + ); + let mut fresh = Consumer::open(server, Some(0)).await; + let replay = fresh.drain().await; + print_frames("a consumer from 0 after the restart", &replay); + for (did, head) in &heads { + let (_, again) = sync_read(server, "com.atproto.sync.getLatestCommit", did).await; + assert_eq!(&again, head, "{did}: the head survives the restart"); + } + let (status, body) = post_json( + &format!("{}/xrpc/bot.did.createAccount", server.origin), + Some(&token), + json!({ "name": format!("after.{}", server.host), "kind": "agent" }), + ) + .await; + assert_eq!( + status, 200, + "the host's session survives the restart: {body}" + ); + let after = fresh.drain().await; + print_frames("the first create after the restart", &after); + let lowest = after + .iter() + .map(|ev| ev.seq) + .min() + .expect("the create is announced"); + assert!( + lowest > before, + "the restart reissued a number: {lowest} after {before} was already used" + ); + for ev in on_resume.iter().chain(replay.iter()) { + assert!( + ev.seq > before || ev.tag == "#info", + "a number from before the restart came back as new: {}", + ev.line() + ); + } +} diff --git a/scripts/scenarios.sh b/scripts/scenarios.sh new file mode 100755 index 00000000..72ac8489 --- /dev/null +++ b/scripts/scenarios.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# The ownership scenarios, end to end, with the real binaries. +# +# One server per scenario on a `.localhost` zone, `didbot` dispatching to +# `didbot-operator`, `didbot-register` and `didbot-oauth`, and `didbot-agentd` +# on every host; the human's own PDS and an OpenID Connect issuer are stood +# in for inside the test. crates/didbot/tests/scenarios.rs is the test, and +# this is the way to run it from a shell and watch it go. +# +# Usage: scripts/scenarios.sh [] +set -euo pipefail + +cd "$(dirname "$0")/.." + +export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$PWD/target}" +exec cargo test -p didbot --test scenarios -- --nocapture "$@"