From ff72f82f400a8299bbe8503c23e7cd2a491998ce Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Mon, 31 Aug 2026 12:36:38 -0400 Subject: [PATCH] build!: remove didbot-index, didbot-query and the canvas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They now live at vibescrobble.com, split out of this repository on 2026-08-28; these were the leftovers never deleted. Everything that writes stays here — the PDS, hook, swarm, setup, keys, attestation, agent accounts — and everything that only reads goes. The built-in development stack stops describing processes this repository cannot start: `ServiceKind` still names an index, a query service and a canvas, so a profile can point at one running out of the other checkout, but the defaults no longer wire one up. Co-Authored-By: Claude Opus 5 (1M context) Change-Id: I842c17de3eb539e9c21a06c2c94c886ce0bca223 --- Cargo.lock | 36 - Cargo.toml | 2 - crates/didbot-index/Cargo.toml | 38 - crates/didbot-index/src/bin/didbot-index.rs | 641 ------ crates/didbot-index/src/bin/didbot-replay.rs | 189 -- crates/didbot-index/src/bin/didbot-survey.rs | 299 --- crates/didbot-index/src/directory.rs | 485 ---- crates/didbot-index/src/feed.rs | 205 -- crates/didbot-index/src/firehose.rs | 510 ----- crates/didbot-index/src/follow.rs | 243 -- crates/didbot-index/src/handle.rs | 192 -- crates/didbot-index/src/http.rs | 154 -- crates/didbot-index/src/lib.rs | 154 -- crates/didbot-index/src/nsid.rs | 42 - crates/didbot-index/src/registration.rs | 215 -- crates/didbot-index/src/replay.rs | 488 ---- crates/didbot-index/src/report.rs | 258 --- crates/didbot-index/src/resolve.rs | 263 --- crates/didbot-index/src/server.rs | 260 --- crates/didbot-index/src/service.rs | 355 --- crates/didbot-index/src/survey.rs | 508 ----- crates/didbot-index/src/tests.rs | 931 -------- crates/didbot-index/src/view.rs | 1486 ------------- crates/didbot-index/src/vouch.rs | 328 --- crates/didbot-index/src/work.rs | 1027 --------- crates/didbot-index/tests/clustering.rs | 207 -- crates/didbot-index/tests/corpus/mod.rs | 257 --- crates/didbot-index/tests/firehose_wire.rs | 90 - crates/didbot-index/tests/reconnect.rs | 432 ---- crates/didbot-query/Cargo.toml | 37 - crates/didbot-query/src/bin/didbot-query.rs | 105 - crates/didbot-query/src/lib.rs | 100 - crates/didbot-query/src/read.rs | 370 ---- crates/didbot-query/tests/read.rs | 379 ---- crates/didbot-stack/src/defaults.rs | 55 +- scripts/dev-index.sh | 38 - scripts/dev-query.sh | 34 - scripts/dev-web.sh | 29 - web/app.js | 2076 ------------------ web/avatar.js | 233 -- web/detail.css | 184 -- web/detail.js | 449 ---- web/index.html | 93 - web/live.js | 226 -- web/mock.js | 513 ----- web/mockups.html | 59 - web/mockups.js | 392 ---- web/style.css | 221 -- web/work.js | 439 ---- 49 files changed, 8 insertions(+), 16319 deletions(-) delete mode 100644 crates/didbot-index/Cargo.toml delete mode 100644 crates/didbot-index/src/bin/didbot-index.rs delete mode 100644 crates/didbot-index/src/bin/didbot-replay.rs delete mode 100644 crates/didbot-index/src/bin/didbot-survey.rs delete mode 100644 crates/didbot-index/src/directory.rs delete mode 100644 crates/didbot-index/src/feed.rs delete mode 100644 crates/didbot-index/src/firehose.rs delete mode 100644 crates/didbot-index/src/follow.rs delete mode 100644 crates/didbot-index/src/handle.rs delete mode 100644 crates/didbot-index/src/http.rs delete mode 100644 crates/didbot-index/src/lib.rs delete mode 100644 crates/didbot-index/src/nsid.rs delete mode 100644 crates/didbot-index/src/registration.rs delete mode 100644 crates/didbot-index/src/replay.rs delete mode 100644 crates/didbot-index/src/report.rs delete mode 100644 crates/didbot-index/src/resolve.rs delete mode 100644 crates/didbot-index/src/server.rs delete mode 100644 crates/didbot-index/src/service.rs delete mode 100644 crates/didbot-index/src/survey.rs delete mode 100644 crates/didbot-index/src/tests.rs delete mode 100644 crates/didbot-index/src/view.rs delete mode 100644 crates/didbot-index/src/vouch.rs delete mode 100644 crates/didbot-index/src/work.rs delete mode 100644 crates/didbot-index/tests/clustering.rs delete mode 100644 crates/didbot-index/tests/corpus/mod.rs delete mode 100644 crates/didbot-index/tests/firehose_wire.rs delete mode 100644 crates/didbot-index/tests/reconnect.rs delete mode 100644 crates/didbot-query/Cargo.toml delete mode 100644 crates/didbot-query/src/bin/didbot-query.rs delete mode 100644 crates/didbot-query/src/lib.rs delete mode 100644 crates/didbot-query/src/read.rs delete mode 100644 crates/didbot-query/tests/read.rs delete mode 100755 scripts/dev-index.sh delete mode 100755 scripts/dev-query.sh delete mode 100755 scripts/dev-web.sh delete mode 100644 web/app.js delete mode 100644 web/avatar.js delete mode 100644 web/detail.css delete mode 100644 web/detail.js delete mode 100644 web/index.html delete mode 100644 web/live.js delete mode 100644 web/mock.js delete mode 100644 web/mockups.html delete mode 100644 web/mockups.js delete mode 100644 web/style.css delete mode 100644 web/work.js diff --git a/Cargo.lock b/Cargo.lock index f9b46451..593bbab3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -879,25 +879,6 @@ dependencies = [ "thiserror 2.0.20", ] -[[package]] -name = "didbot-index" -version = "0.1.0" -dependencies = [ - "axum", - "didbot-identity", - "didbot-lexicon", - "futures-core", - "reqwest", - "serde", - "serde_json", - "thiserror 2.0.20", - "time", - "tokio", - "tower-http", - "tracing", - "tracing-subscriber", -] - [[package]] name = "didbot-key" version = "0.1.0" @@ -981,23 +962,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "didbot-query" -version = "0.1.0" -dependencies = [ - "axum", - "didbot-identity", - "didbot-index", - "reqwest", - "serde", - "serde_json", - "tokio", - "tower", - "tower-http", - "tracing", - "tracing-subscriber", -] - [[package]] name = "didbot-repo" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 2303e6b4..949991dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ publish = false [workspace.dependencies] didbot-config = { version = "0.1.0", path = "crates/didbot-config" } -didbot-index = { version = "0.1.0", path = "crates/didbot-index" } didbot-attest = { version = "0.1.0", path = "crates/didbot-attest" } didbot-avatar = { version = "0.1.0", path = "crates/didbot-avatar" } didbot-dns = { version = "0.1.0", path = "crates/didbot-dns" } @@ -24,7 +23,6 @@ didbot-pds = { version = "0.1.0", path = "crates/didbot-pds" } didbot-hookd = { version = "0.1.0", path = "crates/didbot-hookd" } didbot-mcp = { version = "0.1.0", path = "crates/didbot-mcp" } didbot-serve = { version = "0.1.0", path = "crates/didbot-serve" } -didbot-query = { version = "0.1.0", path = "crates/didbot-query" } didbot-swarm = { version = "0.1.0", path = "crates/didbot-swarm" } didbot-hook = { version = "0.1.0", path = "crates/didbot-hook" } didbot-identity = { version = "0.1.0", path = "crates/didbot-identity" } diff --git a/crates/didbot-index/Cargo.toml b/crates/didbot-index/Cargo.toml deleted file mode 100644 index 9c3ab3e0..00000000 --- a/crates/didbot-index/Cargo.toml +++ /dev/null @@ -1,38 +0,0 @@ -[package] -name = "didbot-index" -description = "Finds didbot personal data servers and reads the agent feeds worth indexing." -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -repository.workspace = true -publish.workspace = true - -[dependencies] -axum.workspace = true -# Only for the `Stream` trait axum's server-sent events want; axum does not -# re-export it. -futures-core = "0.3" -reqwest.workspace = true -serde.workspace = true -serde_json.workspace = true -thiserror.workspace = true -# Vouch records carry `createdAt` and `expiresAt`, and an expired vouch has to -# be dropped rather than followed. -time.workspace = true -tokio.workspace = true -tracing.workspace = true -tracing-subscriber.workspace = true -tower-http.workspace = true -# Containment is checked here with the same comparison the server mints under, -# rather than a second implementation of the label walk. -didbot-identity.workspace = true - -[dev-dependencies] -# Only for `nsid::copies_agree_with_the_definitions`. The indexer reads records -# over the wire and names the collections itself; this holds the two in step -# while both live in one workspace. -didbot-lexicon.workspace = true - -[lints] -workspace = true diff --git a/crates/didbot-index/src/bin/didbot-index.rs b/crates/didbot-index/src/bin/didbot-index.rs deleted file mode 100644 index ef9b86d0..00000000 --- a/crates/didbot-index/src/bin/didbot-index.rs +++ /dev/null @@ -1,641 +0,0 @@ -//! Follows personal data servers and keeps an index of what is on them. -//! -//! Usage: `didbot-index [--pds ] [--voucher ] [--listen ]`. -//! -//! Four loops over one view, and each one answers a question the others -//! cannot. The lifecycle stream says when an account appears or goes away, at -//! the moment it happens. The firehose says when a record is written, at the -//! moment it happens. A survey on a timer says what actually exists, which is -//! how anything either stream missed is noticed at all. And a catch-up sweep -//! reads repositories directly, which is the only way to recover records -//! written while nothing was connected. -//! -//! The sweep is the one that looks redundant now that there is a firehose, and -//! it is not. A stream can only carry what happened while somebody was -//! listening: an index starting against a server that already holds a thousand -//! records has no cursor that reaches them, and a server's replay buffer is -//! finite, so a long enough disconnection is a hole no cursor can close. What -//! the firehose changed is the sweep's job — it went from being how records -//! are found to being how the gaps are filled, and its interval went up by an -//! order of magnitude to match. -//! -//! There is a fifth loop that is not about following anything: the clustering -//! pass, which works out which agents are doing the same thing from the text of -//! their status lines. It runs here rather than in each browser tab so that two -//! people looking at the same feed see the same groups under the same ids. -//! -//! What it serves is deliberately thin: the whole view, what changes after -//! that, and the groups. The query service is what people's browsers talk to. - -#![forbid(unsafe_code)] - -use std::collections::HashSet; -use std::net::SocketAddr; -use std::process::ExitCode; -use std::sync::Arc; -use std::time::Duration; - -use didbot_identity::Zone; -use didbot_index::directory::{ServerDirectory, StaticDirectory, Target, VouchDirectory}; -use didbot_index::firehose::{subscribe, FirehoseFilter, Update}; -use didbot_index::resolve::DidResolver; -use didbot_index::view::{Agent, Event, Operator}; -use didbot_index::{app, follow, Indexer, ReqwestGet, Shared}; -use didbot_index::{read_profile, read_record}; -use tokio::net::TcpListener; -use tokio::sync::{mpsc, Notify}; -use tracing::{debug, info, warn}; - -/// The development personal data server. -const DEFAULT_PDS: &str = "http://localhost:3000"; -/// Where the index listens, one above the MCP host. -const DEFAULT_LISTEN: &str = "127.0.0.1:3002"; -/// How often the whole population is surveyed again. -const DEFAULT_SURVEY: u64 = 10; -/// How often another handful of repositories is read as a backstop. -/// -/// Slow, because the firehose is what carries a record now and this is only -/// here for what the firehose could not: history from before this index -/// connected, and anything a replay buffer had already dropped. -const DEFAULT_FEED: u64 = 15; -/// How many repositories are read on one of those beats. -const DEFAULT_BATCH: usize = 64; -/// How many records are asked for from each. -const DEFAULT_LIMIT: usize = 12; -/// How often the population is reclustered. -/// -/// The browser did this every 0.8 seconds because it was drawing the result at -/// sixty frames a second and a group that lagged behind the dots was visible. -/// Here the result goes out over a stream to be drawn by somebody else, and a -/// pass costs the whole population against a handful of leaders, so two seconds -/// buys most of the responsiveness for a quarter of the work. -const DEFAULT_CLUSTER: u64 = 2; - -const USAGE: &str = "\ -didbot-index - follow personal data servers and index what is on them - -Usage: didbot-index [options] - -Options: - --pds Servers to follow, comma or space separated - (default http://localhost:3000) - --voucher DIDs whose vouches name servers to follow - --listen Address to serve on (default 127.0.0.1:3002) - --survey-every Seconds between full surveys (default 10) - --feed-every Seconds between catch-up repository reads (default 15) - --feed-batch Repositories read per beat (default 64) - --feed-limit Records asked for from each (default 12) - --cluster-every Seconds between clustering passes (default 2, 0 is off) - -h, --help Print this message -"; - -struct Args { - servers: String, - vouchers: String, - listen: String, - survey: Duration, - feed: Duration, - batch: usize, - limit: usize, - cluster: Duration, -} - -impl Default for Args { - fn default() -> Self { - Self { - servers: String::new(), - vouchers: String::new(), - listen: DEFAULT_LISTEN.to_owned(), - survey: Duration::from_secs(DEFAULT_SURVEY), - feed: Duration::from_secs(DEFAULT_FEED), - batch: DEFAULT_BATCH, - limit: DEFAULT_LIMIT, - cluster: Duration::from_secs(DEFAULT_CLUSTER), - } - } -} - -fn parse_args>(mut args: I) -> Result { - let mut parsed = Args::default(); - while let Some(flag) = args.next() { - let mut value = || { - args.next() - .ok_or_else(|| format!("{flag} needs a value\n\n{USAGE}")) - }; - let seconds = |raw: String, name: &str| -> Result { - raw.parse::() - .map(Duration::from_secs) - .map_err(|_| format!("{name}: `{raw}` is not a number of seconds")) - }; - match flag.as_str() { - "--pds" => parsed.servers = value()?, - "--voucher" | "--vouchers" => parsed.vouchers = value()?, - "--listen" => parsed.listen = value()?, - "--survey-every" => parsed.survey = seconds(value()?, "--survey-every")?, - "--feed-every" => parsed.feed = seconds(value()?, "--feed-every")?, - "--cluster-every" => parsed.cluster = seconds(value()?, "--cluster-every")?, - "--feed-batch" => { - let raw = value()?; - parsed.batch = raw - .parse() - .map_err(|_| format!("--feed-batch: `{raw}` is not a count"))?; - } - "--feed-limit" => { - let raw = value()?; - parsed.limit = raw - .parse() - .map_err(|_| format!("--feed-limit: `{raw}` is not a count"))?; - } - "-h" | "--help" => return Err(USAGE.to_owned()), - other => return Err(format!("unknown option `{other}`\n\n{USAGE}")), - } - } - // A bare invocation follows the development server, so the dev stack - // connects up without being told how. - if parsed.servers.is_empty() && parsed.vouchers.is_empty() { - parsed.servers = DEFAULT_PDS.to_owned(); - } - if parsed.batch == 0 || parsed.limit == 0 { - return Err("--feed-batch and --feed-limit must be greater than zero".to_owned()); - } - Ok(parsed) -} - -#[tokio::main] -async fn main() -> ExitCode { - let args = match parse_args(std::env::args().skip(1)) { - Ok(args) => args, - Err(message) => { - println!("{message}"); - return ExitCode::from(2); - } - }; - didbot_index::init_tracing(); - - match run(args).await { - Ok(()) => ExitCode::SUCCESS, - Err(message) => { - tracing::error!("{message}"); - ExitCode::FAILURE - } - } -} - -async fn run(args: Args) -> Result<(), String> { - let http = ReqwestGet::new(); - // The clustering view: this process is the one that works out the groups, - // and everything downstream mirrors what it decides. - let shared = Shared::clustering(); - - let configured = StaticDirectory::parse(&args.servers) - .map_err(|err| format!("--pds: {err}"))? - .targets() - .await; - let (mut targets, refused) = configured.into_parts(); - for rejection in &refused { - warn!(%rejection, "not followed"); - } - - if !args.vouchers.is_empty() { - let vouchers = VouchDirectory::::parse_vouchers(&args.vouchers); - let directory = VouchDirectory::new(DidResolver::new(http.clone()), vouchers); - let (vouched, rejections) = directory.targets().await.into_parts(); - for rejection in &rejections { - warn!(%rejection, "not followed"); - } - targets.extend(vouched); - } - if targets.is_empty() { - return Err("nothing to follow".to_owned()); - } - - let addr: SocketAddr = args - .listen - .parse() - .map_err(|_| format!("--listen: `{}` is not an address", args.listen))?; - - println!("didbot-index"); - for target in &targets { - println!(" following {}", target.server); - } - println!(" listening http://{addr}"); - println!(); - println!(" curl http://{addr}/view"); - println!(" curl http://{addr}/work"); - println!(" curl -N http://{addr}/events"); - println!(); - - // Two things wake a loop early. A survey that has not run yet is what - // turns an account the firehose just mentioned into one this index will - // accept records for; a catch-up sweep is what fills a hole the firehose - // said it could not replay. Both are `Notify` rather than a channel - // because the message is "there is work", not "here is what it is", and - // two of them arriving before the loop wakes is still one piece of work. - let survey_now = Arc::new(Notify::new()); - let catch_up_now = Arc::new(Notify::new()); - - // The lifecycle stream, one follower per server, all feeding one channel. - let (lifecycle, mut incoming) = mpsc::channel(256); - for target in &targets { - tokio::spawn(follow(target.server.clone(), lifecycle.clone())); - } - drop(lifecycle); - - // The record stream, likewise. - let (commits, incoming_commits) = mpsc::channel(1024); - for target in &targets { - // `All`, and deliberately: `firehose_loop` records a commit's head - // before it decides whether the collection is one this index reads - // (see the comment there), which is what `/health` compares against - // `com.atproto.sync.getLatestCommit`. Filtering here instead would - // make that comparison wrong for every collection dropped, silently. - tokio::spawn(subscribe( - target.server.clone(), - FirehoseFilter::All, - commits.clone(), - )); - } - drop(commits); - let following = targets - .iter() - .map(|target| target.server.base_url().to_owned()) - .collect::>() - .join(", "); - shared.pulse.following(&following, true).await; - let streamer = tokio::spawn(firehose_loop( - shared.clone(), - incoming_commits, - survey_now.clone(), - catch_up_now.clone(), - following, - )); - - let surveyor = tokio::spawn(survey_loop( - shared.clone(), - http.clone(), - targets.clone(), - args.survey, - survey_now.clone(), - )); - let reader = tokio::spawn(feed_loop( - shared.clone(), - http.clone(), - targets.clone(), - args.feed, - args.batch, - args.limit, - catch_up_now.clone(), - )); - - let clusterer = tokio::spawn(cluster_loop(shared.clone(), args.cluster)); - - let listener = TcpListener::bind(addr) - .await - .map_err(|err| format!("could not bind {addr}: {err}"))?; - - let serving = shared.clone(); - let server = tokio::spawn(async move { - axum::serve(listener, app(serving)).await.ok(); - }); - - // Lifecycle arrives here rather than in its own task so that everything - // touching the view is in one place and its ordering is obvious. - while let Some(event) = incoming.recv().await { - match event.kind.as_str() { - // A provisioned account is announced before anything is known - // about it beyond its identity; the survey fills in the rest. - "provisioned" => info!(did = event.did, "provisioned"), - "deleted" => { - let announced = shared.view.lock().await.forget(&event.did); - shared.announce(announced).await; - } - "pinned" | "unpinned" => { - let pinned = event.kind == "pinned"; - let announced = shared.view.lock().await.set_pinned(&event.did, pinned); - shared.announce(announced).await; - } - other => info!(kind = other, did = event.did, "transition"), - } - } - - surveyor.abort(); - reader.abort(); - streamer.abort(); - clusterer.abort(); - server.abort(); - Ok(()) -} - -/// Surveys every target on a timer, and reconciles what it finds. -/// -/// `wake` runs one early: the firehose mentioning a DID this index does not -/// know is exactly the moment a survey is worth running, and waiting out the -/// interval would mean dropping that account's records until it expired. -async fn survey_loop( - shared: Shared, - http: ReqwestGet, - targets: Vec, - every: Duration, - wake: Arc, -) { - let indexer = Indexer::new(http); - let mut ticker = tokio::time::interval(every); - loop { - tokio::select! { - _ = ticker.tick() => {} - () = wake.notified() => debug!("surveying early"), - } - let survey = indexer.survey(&targets).await; - for rejection in survey.rejections() { - warn!(%rejection, "refused"); - } - - let mut present = HashSet::new(); - for source in survey.sources() { - present.insert(source.did().as_str().to_owned()); - - let operator = operator_of(source); - let agent = Agent { - did: source.did().as_str().to_owned(), - agent_id: source.agent_id().to_owned(), - handle: source.handle().map(str::to_owned), - operator: operator.did.clone(), - pinned: source.pinned(), - created_at: source.created_at().to_owned(), - // A server's account listing carries none of the four fields - // below: lineage, type, harness and model all come from the - // account's own profile record. `see_agent` keeps whatever is - // already known, so a survey beat does not flatten the tree - // every ten seconds or blank a label it did not fetch. - parent: None, - agent_type: None, - harness: None, - model: None, - }; - let announced = { - let mut view = shared.view.lock().await; - let first = view.see_operator(operator); - let second = view.see_agent(agent); - (first, second) - }; - shared.announce(announced.0).await; - shared.announce(announced.1).await; - } - - // Anything the index holds that the survey did not see is gone. This - // is the backstop for a `deleted` frame that arrived while nothing - // was listening. - let stale: Vec = { - let view = shared.view.lock().await; - view.snapshot() - .agents - .into_iter() - .map(|agent| agent.did) - .filter(|did| !present.contains(did)) - .collect() - }; - for did in stale { - let announced = shared.view.lock().await.forget(&did); - shared.announce(announced).await; - } - } -} - -/// Reads a handful of repositories on every beat, round-robin. -/// -/// Round-robin rather than all at once: a hundred agents polled together is a -/// hundred requests landing on one server in the same instant, which is a -/// thundering herd this index would be the sole author of. -/// -/// The exception is a beat the firehose asked for. A gap it could not replay -/// is unbounded — anything at all may be in it — so that pass reads every -/// repository rather than a slice, because a slice would leave the rest of the -/// hole open for however long a full cycle takes. -async fn feed_loop( - shared: Shared, - http: ReqwestGet, - targets: Vec, - every: Duration, - batch: usize, - limit: usize, - wake: Arc, -) { - let indexer = Indexer::new(http); - let mut ticker = tokio::time::interval(every); - let mut cursor = 0usize; - loop { - let whole_population = tokio::select! { - _ = ticker.tick() => false, - () = wake.notified() => true, - }; - let survey = indexer.survey(&targets).await; - let sources = survey.sources(); - if sources.is_empty() { - continue; - } - let reach = if whole_population { - debug!( - repositories = sources.len(), - "catching up on every repository" - ); - sources.len() - } else { - batch.min(sources.len()) - }; - for step in 0..reach { - let source = &sources[(cursor + step) % sources.len()]; - if !shared.view.lock().await.knows(source.did().as_str()) { - continue; - } - // The profile before the scrobbles. It is the authoritative - // statement of the account's lineage, so reading it first means a - // record's copy is compared against the profile rather than - // announced and then corrected. - if let Some(profile) = indexer.profile(source).await { - let announced = shared.view.lock().await.see_profile(&profile); - shared.announce(announced).await; - } - let page = indexer.feed(source, limit).await; - for rejection in page.rejections() { - warn!(%rejection, "record skipped"); - } - for scrobble in page.scrobbles() { - let announced = shared.view.lock().await.see_scrobble(scrobble.clone()); - shared.announce(announced).await; - } - } - cursor = cursor.wrapping_add(reach); - } -} - -/// Reclusters the population on a timer and announces what changed. -/// -/// A pass that comes out the same as the last one announces nothing, which on -/// a quiet population is most of them; see -/// [`View::recluster`](didbot_index::View::recluster). -async fn cluster_loop(shared: Shared, every: Duration) { - if every.is_zero() { - info!("clustering is off"); - return; - } - let mut ticker = tokio::time::interval(every); - loop { - ticker.tick().await; - let announced = { - let mut view = shared.view.lock().await; - view.recluster() - }; - if let Some(Event::Work { groups }) = &announced { - debug!( - groups = groups.len(), - grouped = groups - .iter() - .map(|group| group.members.len()) - .sum::(), - "reclustered" - ); - } - shared.announce(announced).await; - } -} - -/// Applies what the firehose says, and asks for help when it cannot. -/// -/// Only scrobbles, and only for accounts a survey has already admitted. Both -/// filters are about trust rather than tidiness: the survey is where a DID was -/// checked against the server's zone and the vouch that bounded it, so a -/// commit naming anything else is a server describing a repository it has no -/// authority over. Dropping it and waking the survey means a genuinely new -/// account arrives the same way every other one does, a moment later. -/// -/// It is also what `/health` reports the firehose's state from: a gap means -/// the stream admitted it could not replay something, and a commit means it -/// is delivering again. A server that goes away entirely produces neither, -/// which is what `quietFor` is for — nothing here can distinguish a dead -/// upstream from a quiet one except by how long it has been silent. -async fn firehose_loop( - shared: Shared, - mut incoming: mpsc::Receiver, - survey_now: Arc, - catch_up_now: Arc, - following: String, -) { - let mut applied = 0u64; - while let Some(update) = incoming.recv().await { - match update { - Update::Gap { - server, - oldest, - message, - } => { - warn!( - server = server.base_url(), - oldest, message, "the firehose left a gap; reading every repository" - ); - shared.pulse.following(&following, false).await; - survey_now.notify_one(); - catch_up_now.notify_one(); - } - Update::Commit { server, commit } => { - shared.pulse.following(&following, true).await; - // Recorded before the two filters below, and for every - // collection: where a repository is, is a fact about the - // repository rather than about the records this index cares - // to keep. It is what `/health` compares against - // `com.atproto.sync.getLatestCommit`. - shared - .pulse - .head(&commit.did, &commit.commit, &commit.rev, commit.seq) - .await; - let collection = commit.collection.clone(); - if collection != didbot_index::nsid::SCROBBLE - && collection != didbot_index::nsid::REGISTRATION - { - debug!(collection, "nothing this index reads; ignored"); - continue; - } - if !shared.view.lock().await.knows(&commit.did) { - debug!( - server = server.base_url(), - did = commit.did, - "a commit for an account no survey has admitted; surveying" - ); - // The survey, and deliberately not the sweep. Once the - // survey admits the account its next commit is applied - // normally, and the handful in between are picked up by - // the sweep's ordinary beat. Waking the sweep here instead - // means every agent that ever provisions triggers a read - // of every repository, which is measurably worse than the - // poll this whole change exists to retire. - survey_now.notify_one(); - continue; - } - if collection == didbot_index::nsid::REGISTRATION { - match read_profile(&commit.did, &commit.uri, commit.record) { - Ok(profile) => { - let announced = shared.view.lock().await.see_profile(&profile); - shared.announce(announced).await; - } - Err(reason) => warn!( - server = server.base_url(), - uri = commit.uri, - reason, - "profile skipped" - ), - } - continue; - } - match read_record(&commit.did, &commit.uri, commit.record) { - Ok(scrobble) => { - let announced = shared.view.lock().await.see_scrobble(scrobble); - if !announced.is_empty() { - applied += 1; - if applied.is_multiple_of(100) { - info!(applied, "records taken in from the firehose"); - } - } - shared.announce(announced).await; - } - Err(reason) => warn!( - server = server.base_url(), - uri = commit.uri, - reason, - "record skipped" - ), - } - } - } - } -} - -/// The operator record for whatever server a feed came from. -/// -/// The service DID is rebuilt from the zone rather than asked for, because -/// that is the rule the server itself mints under: the zone hostname, with a -/// development port if it has one. -fn operator_of(source: &didbot_index::FeedSource) -> Operator { - let server = source.server(); - let zone = source.zone(); - let did = Zone::new(zone) - .map(|plain| match port_of(server.base_url()) { - Some(port) => plain.clone().with_port(port).unwrap_or(plain), - None => plain, - }) - .map(|zone| zone.service_did()) - .unwrap_or_else(|_| format!("did:web:{zone}")); - - Operator { - did, - base_url: server.base_url().to_owned(), - handle: zone.to_owned(), - zone: zone.to_owned(), - } -} - -/// The port in a base URL, if it names one. -fn port_of(base_url: &str) -> Option { - let authority = base_url.split("://").nth(1)?; - let authority = authority.split(['/', '?', '#']).next()?; - let (_, port) = authority.rsplit_once(':')?; - port.parse().ok() -} diff --git a/crates/didbot-index/src/bin/didbot-replay.rs b/crates/didbot-index/src/bin/didbot-replay.rs deleted file mode 100644 index 8cb61d11..00000000 --- a/crates/didbot-index/src/bin/didbot-replay.rs +++ /dev/null @@ -1,189 +0,0 @@ -//! Record a firehose, and play it back with nothing else running. -//! -//! ```text -//! didbot-replay record --pds http://localhost:3000 --out session.jsonl -//! didbot-replay serve --file session.jsonl --listen 127.0.0.1:3010 -//! didbot-index --pds http://127.0.0.1:3010 -//! ``` -//! -//! Why it exists, and what it is not, is in -//! [`didbot_index::replay`](../didbot_index/replay/index.html). - -#![forbid(unsafe_code)] - -use std::io::Write; -use std::path::PathBuf; - -use didbot_index::follow::{read_frames, Frame}; -use didbot_index::replay::{router, Pace, Recorded, Recording}; -use didbot_index::ServerRef; -use tokio::sync::mpsc; - -/// What `--help` prints. -const USAGE: &str = "\ -didbot-replay — record a firehose, and play it back - - didbot-replay record --out [--pds ] [--frames ] - didbot-replay serve --file [--listen ] [--rate ] - -Options - --pds Server to record from (default http://localhost:3000) - --out Where to write the recording - --frames Stop after this many frames (default: until interrupted) - --file Recording to play back - --listen Address to serve it on (default 127.0.0.1:3010) - --rate Frames per second, or 0 for as fast as it will go - (default 5) - -A recording is one JSON object per line, so trimming it to the frames that -reproduce something is a text editor and nothing else. -"; - -/// Where to record from when nothing says otherwise. -const DEFAULT_PDS: &str = "http://localhost:3000"; - -/// Where to serve a playback when nothing says otherwise. -/// -/// Not a port any other component uses, so a replay can run beside a real -/// stack rather than instead of it. -const DEFAULT_LISTEN: &str = "127.0.0.1:3010"; - -/// Frames per second a playback hands over when nothing says otherwise. -/// -/// Slow enough that a canvas watching it looks like something is happening -/// rather than flashing once and stopping. -const DEFAULT_RATE: f64 = 5.0; - -#[tokio::main] -async fn main() -> Result<(), Box> { - let args: Vec = std::env::args().skip(1).collect(); - if args.is_empty() || args.iter().any(|arg| arg == "--help" || arg == "-h") { - print!("{USAGE}"); - return Ok(()); - } - - didbot_index::init_tracing(); - - match args[0].as_str() { - "record" => record(&args[1..]).await, - "serve" => serve(&args[1..]).await, - other => { - eprintln!("didbot-replay: no such command: {other}\n\n{USAGE}"); - std::process::exit(2); - } - } -} - -/// Subscribes to a firehose and writes what it sees. -/// -/// Appends as it goes rather than buffering to the end, so that a recording -/// interrupted with Ctrl-C is a recording rather than nothing. That is the -/// ordinary way to end one: a developer records until the thing they are -/// chasing happens. -async fn record(args: &[String]) -> Result<(), Box> { - let mut pds = DEFAULT_PDS.to_owned(); - let mut out: Option = None; - let mut limit: Option = None; - - let mut rest = args.iter(); - while let Some(arg) = rest.next() { - match arg.as_str() { - "--pds" => pds = rest.next().ok_or("--pds wants a url")?.clone(), - "--out" => out = Some(PathBuf::from(rest.next().ok_or("--out wants a path")?)), - "--frames" => { - limit = Some(rest.next().ok_or("--frames wants a count")?.parse()?); - } - other => return Err(format!("unexpected argument: {other}").into()), - } - } - let out = out.ok_or("record needs --out")?; - - let server = ServerRef::parse(&pds)?; - let file = std::fs::File::create(&out)?; - let mut file = std::io::BufWriter::new(file); - - let (sender, mut frames) = mpsc::channel::(didbot_index::follow::FRAME_BUFFER); - let url = format!("{}/firehose", server.base_url()); - tracing::info!(%url, path = %out.display(), "recording"); - - let http = reqwest::Client::new(); - let reading = tokio::spawn(async move { - if let Err(err) = read_frames(&http, &url, &sender).await { - tracing::warn!(error = %err, "the stream ended"); - } - }); - - let mut written = 0usize; - while let Some(frame) = frames.recv().await { - let recorded = Recorded::from(frame); - writeln!(file, "{}", serde_json::to_string(&recorded)?)?; - // Flushed per frame: the ordinary end of a recording is Ctrl-C, and a - // buffer that had not reached the disk would take the interesting part - // with it. - file.flush()?; - written += 1; - if limit.is_some_and(|limit| written >= limit) { - break; - } - } - reading.abort(); - tracing::info!(frames = written, path = %out.display(), "recorded"); - Ok(()) -} - -/// Serves a recording as a firehose. -async fn serve(args: &[String]) -> Result<(), Box> { - let mut file: Option = None; - let mut listen: Option = None; - let mut rate = DEFAULT_RATE; - - let mut rest = args.iter(); - while let Some(arg) = rest.next() { - match arg.as_str() { - "--file" => file = Some(PathBuf::from(rest.next().ok_or("--file wants a path")?)), - "--listen" => listen = Some(rest.next().ok_or("--listen wants an address")?.clone()), - "--rate" => rate = rest.next().ok_or("--rate wants a number")?.parse()?, - other => return Err(format!("unexpected argument: {other}").into()), - } - } - let file = file.ok_or("serve needs --file")?; - - let recording = Recording::parse(&std::fs::read_to_string(&file)?)?; - let frames = recording.frames.len(); - let accounts = recording.accounts().len(); - // Where the recording came from, unless told otherwise: an index checks - // that a server's zone sits under the host it was pointed at, so a - // playback on any other port is refused however correct its frames are. - let listen = listen - .or_else(|| recording.listen_hint()) - .unwrap_or_else(|| DEFAULT_LISTEN.to_owned()); - let pace = Pace { - per_second: (rate > 0.0).then_some(rate), - }; - - let listener = tokio::net::TcpListener::bind(&listen).await?; - let addr = listener.local_addr()?; - println!( - "replaying {} — {frames} frame(s), {accounts} account(s)", - file.display() - ); - println!(" firehose http://{addr}/firehose"); - println!( - " follow it didbot-index --pds http://localhost:{}", - addr.port() - ); - if let Some(zone) = recording.zone() { - println!(" minted under {zone}"); - } - println!( - " pace {}", - match pace.per_second { - Some(rate) => format!("{rate} frame(s) per second"), - None => "as fast as the socket takes them".to_owned(), - } - ); - println!(); - - axum::serve(listener, router(recording, pace)).await?; - Ok(()) -} diff --git a/crates/didbot-index/src/bin/didbot-survey.rs b/crates/didbot-index/src/bin/didbot-survey.rs deleted file mode 100644 index 11862bb2..00000000 --- a/crates/didbot-index/src/bin/didbot-survey.rs +++ /dev/null @@ -1,299 +0,0 @@ -//! Surveys the configured didbot servers and prints what would be -//! indexed. -//! -//! Usage: `didbot-survey [--voucher ] [--servers ] [--feed -//! ] [--verify-handles]`. -//! -//! This is the discovery half of the indexer with a terminal attached. Given -//! a voucher it walks the whole chain — the human's DID, their repository, -//! the vouch records in it, the servers those name — and prints the feeds it -//! would index, the claims it refused and why, and optionally the most recent -//! scrobbles from each feed. Given `--servers` instead it skips the vouch and -//! trusts a server for its own hostname, which is the development path. -//! -//! It writes nothing and stores nothing. - -#![forbid(unsafe_code)] - -use std::process::ExitCode; - -use didbot_index::{ - verify_handle, DidResolver, Discovered, FeedSource, Indexer, Rejection, ReqwestGet, - ServerDirectory, StaticDirectory, Survey, Target, VouchDirectory, DEFAULT_PLC_DIRECTORY, -}; - -/// Servers surveyed when neither `--voucher` nor `--servers` is given. -const DEFAULT_SERVERS: &str = "http://localhost:3000"; - -/// What `--help` prints. -const USAGE: &str = "\ -didbot-survey - find the agent feeds an appview would index - -Usage: didbot-survey [--voucher ] [--servers ] [--feed ] - [--verify-handles] - -Options: - --voucher DIDs of the humans whose vouches to follow, comma or - space separated - --servers Base URLs to read without a vouch, comma or space - separated (the development path; default - http://localhost:3000 when no --voucher is given) - --plc-directory Where did:plc is resolved (default https://plc.directory) - --feed Also print each feed's n most recent scrobbles - --verify-handles Resolve every handle and check the account's DID - document claims it back (two requests per account) - -h, --help Print this message -"; - -/// Parsed command line. -struct Args { - /// DIDs of the humans whose vouches to follow. - vouchers: String, - /// Base URLs to read with no vouch behind them. - servers: String, - /// Where `did:plc` is resolved. - plc_directory: String, - /// How many scrobbles to print per feed; zero prints none. - feed: usize, - /// Whether to complete atproto's bidirectional handle check. - verify_handles: bool, -} - -/// Parses the command line, or returns the text to print and exit on. -/// -/// Hand-rolled, like every other binary here: three flags do not justify a -/// dependency. -fn parse_args>(mut args: I) -> Result { - let mut parsed = Args { - vouchers: String::new(), - servers: String::new(), - plc_directory: DEFAULT_PLC_DIRECTORY.to_owned(), - feed: 0, - verify_handles: false, - }; - while let Some(flag) = args.next() { - let mut value = || { - args.next() - .ok_or_else(|| format!("{flag} needs a value\n\n{USAGE}")) - }; - match flag.as_str() { - "--voucher" | "--vouchers" => parsed.vouchers = value()?, - "--servers" => parsed.servers = value()?, - "--plc-directory" => parsed.plc_directory = value()?, - "--feed" => { - let raw = value()?; - parsed.feed = raw - .parse() - .map_err(|_| format!("--feed: `{raw}` is not a count"))?; - } - "--verify-handles" => parsed.verify_handles = true, - "-h" | "--help" => return Err(USAGE.to_owned()), - other => return Err(format!("unknown option `{other}`\n\n{USAGE}")), - } - } - // A bare invocation is the development one. Naming a voucher and no - // server is the real one, and must not quietly also read localhost. - if parsed.vouchers.is_empty() && parsed.servers.is_empty() { - parsed.servers = DEFAULT_SERVERS.to_owned(); - } - Ok(parsed) -} - -#[tokio::main] -async fn main() -> ExitCode { - let args = match parse_args(std::env::args().skip(1)) { - Ok(args) => args, - Err(message) => { - println!("{message}"); - return ExitCode::from(2); - } - }; - - // Rejections are printed, so the tracing warnings that also report them - // are off unless the operator asks for them. - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("error")), - ) - .with_writer(std::io::stderr) - .init(); - - match run(args).await { - Ok(code) => code, - Err(message) => { - eprintln!("didbot-survey: {message}"); - ExitCode::FAILURE - } - } -} - -/// Surveys, prints, and reports whether anything was found. -async fn run(args: Args) -> Result { - let http = ReqwestGet::new(); - let mut found = Discovered::default(); - - if !args.vouchers.is_empty() { - let vouchers = VouchDirectory::::parse_vouchers(&args.vouchers); - println!("following {} voucher(s)", vouchers.len()); - for voucher in &vouchers { - println!(" {voucher}"); - } - let resolver = DidResolver::with_plc_directory(http.clone(), &args.plc_directory); - found = VouchDirectory::new(resolver, vouchers).targets().await; - } - - if !args.servers.is_empty() { - let configured = StaticDirectory::parse(&args.servers) - .map_err(|err| format!("--servers: {err}"))? - .targets() - .await; - found = merge(found, configured); - } - - let (targets, rejections) = found.into_parts(); - print_targets(&targets, &rejections); - - let indexer = Indexer::new(http.clone()); - let survey = indexer.survey(&targets).await; - print_survey(&survey); - - if args.verify_handles { - let resolver = DidResolver::with_plc_directory(http, &args.plc_directory); - verify_handles(&resolver, &survey).await; - } - - if args.feed > 0 { - for source in survey.sources() { - print_feed(&indexer, source, args.feed).await; - } - } - - // An empty survey is not an error: a dev server with no agents on it is a - // perfectly good answer. It is reported through the exit code anyway, so a - // script can wait for one without parsing this output. - Ok(if survey.sources().is_empty() { - ExitCode::from(1) - } else { - ExitCode::SUCCESS - }) -} - -/// Concatenates two discoveries, keeping both sets of refusals. -fn merge(first: Discovered, second: Discovered) -> Discovered { - let (mut targets, mut rejections) = first.into_parts(); - let (more_targets, more_rejections) = second.into_parts(); - targets.extend(more_targets); - rejections.extend(more_rejections); - Discovered::from_parts(targets, rejections) -} - -/// Prints what the directories found, and what they refused on the way. -fn print_targets(targets: &[Target], rejections: &[Rejection]) { - println!(); - println!("servers to read: {}", targets.len()); - for target in targets { - println!(" {}", target.server); - match target.vouch.as_ref() { - Some(vouch) => println!( - " covers {} · vouched by {}", - target.authority, vouch.voucher - ), - None => println!(" covers {} · no vouch", target.authority), - } - } - print_rejections("not followed", rejections); - println!(); -} - -/// Prints the feeds and the refusals. -fn print_survey(survey: &Survey) { - println!("indexable feeds: {}", survey.sources().len()); - for source in survey.sources() { - let pin = if source.pinned() { - "pinned" - } else { - "unpinned" - }; - println!(" {}", source.did()); - println!( - " agent {} · handle {} · zone {} · {pin} · since {}", - source.agent_id(), - source.handle().unwrap_or("-"), - source.zone(), - source.created_at() - ); - } - print_rejections("refused", survey.rejections()); - println!(); -} - -/// Completes atproto's bidirectional handle check for every surveyed feed. -/// -/// The survey has already dropped any handle the server had no authority to -/// claim; this asks the remaining question, which needs the network: does the -/// handle actually resolve to this account, and does this account's own DID -/// document claim it back. Two requests per account, which is why it is a flag -/// and not the default — see `didbot_index::handle`. -async fn verify_handles(resolver: &DidResolver, survey: &Survey) { - let checked: Vec<&FeedSource> = survey - .sources() - .iter() - .filter(|source| source.handle().is_some()) - .collect(); - println!("handles verified bidirectionally: {}", checked.len()); - let mut failures = Vec::new(); - for source in checked { - let Some(handle) = source.handle() else { - continue; - }; - // The port is development's, and comes off the server that was - // dialled: a whole loopback zone is one listener, and a handle string - // carries no port of its own. - let port = source.server().port(); - match verify_handle(resolver, handle, source.did().as_str(), port).await { - Ok(()) => println!(" {handle} ✓ {}", source.did()), - Err(failure) => { - println!(" {handle} ✗ {failure}"); - failures.push(handle.to_owned()); - } - } - } - if failures.is_empty() { - println!(" every handle resolves to the account that claims it"); - } else { - println!(" {} handle(s) did not verify", failures.len()); - } - println!(); -} - -/// Prints a list of refusals under a heading, or nothing when there are none. -fn print_rejections(heading: &str, rejections: &[Rejection]) { - if rejections.is_empty() { - return; - } - println!(); - println!("{heading}: {}", rejections.len()); - for rejection in rejections { - println!(" {rejection}"); - } -} - -/// Prints one feed's most recent scrobbles. -async fn print_feed(indexer: &Indexer, source: &FeedSource, limit: usize) { - let page = indexer.feed(source, limit).await; - println!("{} ({})", source.agent_id(), source.did()); - if page.scrobbles().is_empty() && page.rejections().is_empty() { - println!(" (no scrobbles yet)"); - } - for scrobble in page.scrobbles() { - println!( - " {} {} {}", - scrobble.emoji, scrobble.created_at, scrobble.text - ); - } - for rejection in page.rejections() { - println!(" ! {rejection}"); - } - println!(); -} diff --git a/crates/didbot-index/src/directory.rs b/crates/didbot-index/src/directory.rs deleted file mode 100644 index b52de0b0..00000000 --- a/crates/didbot-index/src/directory.rs +++ /dev/null @@ -1,485 +0,0 @@ -//! Where the servers to index come from, and what each one may speak for. -//! -//! [`VouchDirectory`] is the real answer: a human writes a vouch record in -//! their own repository naming the personal data server they run, the appview -//! is configured with the humans it is willing to listen to, and everything -//! else — which server, which accounts, for how long — follows from records -//! those humans signed. [`StaticDirectory`] is the development shortcut, a -//! server URL somebody typed with no vouch behind it. -//! -//! Both produce [`Target`]s, and a target is deliberately two separate facts: -//! **where** to fetch, and **what may be claimed** from what comes back. The -//! first is delegable — a DID document may name any host as its personal data -//! server, exactly as an atproto account hosted by a provider does. The second -//! is not: it is read off the vouched DID itself, so a server cannot answer -//! for names the vouch did not cover. - -use std::fmt; -use std::future::Future; - -use crate::nsid; -use didbot_identity::{hostname_is_at_or_below, AgentDid}; -use time::format_description::well_known::Rfc3339; -use time::OffsetDateTime; -use tracing::{debug, warn}; - -use crate::http::HttpGet; -use crate::report::{fetch_json, RecordsView, Rejection}; -use crate::resolve::{DidResolver, Resolved}; -use crate::server::{ServerRef, UrlError}; -use crate::vouch::{in_force, Dropped, Vouch}; - -/// How many vouch records are read from one repository. -/// -/// A hundred is the ceiling atproto's own `listRecords` applies, and a person -/// with more than a hundred live vouches has a different problem. -pub const VOUCH_PAGE_LIMIT: usize = 100; - -/// What may be indexed under one target. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Authority { - /// Every account at or below this hostname. - /// - /// A vouched `did:web` yields this, whether it names a server or one - /// account: an account hostname has nothing below it, so the same rule - /// covers a whole zone and a single account without a mode switch. - Host(String), - /// Exactly this DID, matched as a string. - /// - /// For a subject that names no hostname — a `did:plc` — there is no - /// containment to check, so nothing but the DID itself is covered. - Did(String), -} - -impl Authority { - /// Whether `did` is a name this authority covers. - pub fn covers(&self, did: &AgentDid) -> bool { - match self { - Self::Host(host) => hostname_is_at_or_below(did.host(), host), - Self::Did(exact) => did.as_str() == exact, - } - } -} - -impl fmt::Display for Authority { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Host(host) => f.write_str(host), - Self::Did(did) => f.write_str(did), - } - } -} - -/// The vouch a target came from. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct VouchRef { - /// The DID of the human who wrote it. - pub voucher: String, - /// The AT-URI of the vouch record. - pub uri: String, - /// The DID it vouched for. - pub subject: String, -} - -/// A server to read, and the bound on what it may offer. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Target { - /// Where to fetch. Whatever the subject's DID document named. - pub server: ServerRef, - /// What may be claimed from it, read off the subject's DID. - pub authority: Authority, - /// Who vouched, or `None` for a server somebody configured directly. - pub vouch: Option, -} - -impl Target { - /// A server with no vouch behind it, trusted for its own hostname. - pub fn configured(server: ServerRef) -> Self { - let authority = Authority::Host(server.host().to_owned()); - Self { - server, - authority, - vouch: None, - } - } - - /// The DID of the human who vouched for this target, if one did. - pub fn vouched_by(&self) -> Option<&str> { - self.vouch.as_ref().map(|vouch| vouch.voucher.as_str()) - } -} - -/// What a directory found, and what it declined on the way. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct Discovered { - targets: Vec, - rejections: Vec, -} - -impl Discovered { - /// The targets to survey. - pub fn targets(&self) -> &[Target] { - &self.targets - } - - /// Everything declined while assembling them. - pub fn rejections(&self) -> &[Rejection] { - &self.rejections - } - - /// Splits into what was found and what was refused. - pub fn into_parts(self) -> (Vec, Vec) { - (self.targets, self.rejections) - } - - /// Reassembles what [`Discovered::into_parts`] took apart. - /// - /// For a caller combining more than one directory: the vouched servers - /// and, alongside them, whatever a developer named on the command line. - pub fn from_parts(targets: Vec, rejections: Vec) -> Self { - Self { - targets, - rejections, - } - } - - /// Adds a target, dropping one that is already covered. - /// - /// Two people vouching for the same server is normal and is not worth - /// reporting: the second vouch adds no name the first did not already - /// cover, and the accounts themselves are deduplicated by the survey. - fn push(&mut self, target: Target) { - if self - .targets - .iter() - .any(|held| held.server == target.server && held.authority == target.authority) - { - debug!(server = target.server.base_url(), "target already covered"); - return; - } - self.targets.push(target); - } -} - -/// A source of targets. -pub trait ServerDirectory { - /// The targets known right now. - fn targets(&self) -> impl Future + Send; -} - -/// Servers somebody configured, with no vouch behind them. -/// -/// The development path. Each server is trusted for its own hostname and -/// nothing else, which is the same bound a vouch would give it if a human had -/// written one naming that host. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct StaticDirectory { - servers: Vec, -} - -impl StaticDirectory { - /// Builds a directory from already-parsed references, keeping the first - /// mention of each server and dropping repeats. - pub fn new(servers: impl IntoIterator) -> Self { - let mut deduped: Vec = Vec::new(); - for server in servers { - if !deduped.contains(&server) { - deduped.push(server); - } - } - Self { servers: deduped } - } - - /// Parses a list of base URLs separated by commas or whitespace. - /// - /// The shape a `--servers` flag or an environment variable arrives in. An - /// unparseable entry fails the whole list rather than being skipped: a - /// survey that quietly indexed three of the four servers it was asked - /// about would look exactly like a survey of four. - pub fn parse(list: &str) -> Result { - let servers = split_list(list) - .map(ServerRef::parse) - .collect::, _>>()?; - Ok(Self::new(servers)) - } - - /// The servers, in the order they were configured. - pub fn entries(&self) -> &[ServerRef] { - &self.servers - } - - /// Whether the directory names no servers at all. - pub fn is_empty(&self) -> bool { - self.servers.is_empty() - } -} - -impl ServerDirectory for StaticDirectory { - async fn targets(&self) -> Discovered { - let mut found = Discovered::default(); - for server in &self.servers { - found.push(Target::configured(server.clone())); - } - found - } -} - -/// Servers reached through the vouch records of humans this appview listens -/// to. -/// -/// The chain is: a voucher's DID resolves to their personal data server, that -/// server holds their `bot.did.vouch` records, each live vouch -/// names a subject, and the subject's DID resolves to the didbot server -/// to read and bounds what may be read from it. -/// -/// The list of vouchers is configuration, and it is the only configuration -/// that matters. Indexing anyone who vouches for anything needs a relay to -/// hear vouches on; until then the appview is told whose word it takes. -#[derive(Debug, Clone)] -pub struct VouchDirectory { - resolver: DidResolver, - vouchers: Vec, - clock: Option, -} - -impl VouchDirectory { - /// Listens to the vouches written by `vouchers`. - pub fn new(resolver: DidResolver, vouchers: impl IntoIterator) -> Self { - Self { - resolver, - vouchers: vouchers.into_iter().collect(), - clock: None, - } - } - - /// Splits a comma or whitespace separated list of voucher DIDs. - pub fn parse_vouchers(list: &str) -> Vec { - split_list(list).map(str::to_owned).collect() - } - - /// Pins the clock expiry is judged against, for tests. - pub fn with_clock(mut self, now: OffsetDateTime) -> Self { - self.clock = Some(now); - self - } - - /// The vouchers this directory listens to. - pub fn vouchers(&self) -> &[String] { - &self.vouchers - } - - fn now(&self) -> OffsetDateTime { - self.clock.unwrap_or_else(OffsetDateTime::now_utc) - } - - /// Reads one voucher's live vouches and turns them into targets. - async fn targets_from(&self, voucher: &str, found: &mut Discovered) { - let resolved = match self.resolve(voucher).await { - Ok(resolved) => resolved, - Err(rejection) => { - warn!(voucher, %rejection, "voucher not read"); - found.rejections.push(rejection); - return; - } - }; - - let vouches = match self.read_vouches(voucher, &resolved.1).await { - Ok(vouches) => vouches, - Err(rejection) => { - warn!(voucher, %rejection, "vouches not read"); - found.rejections.push(rejection); - return; - } - }; - - let (parsed, unreadable) = split_vouches(voucher, vouches); - found.rejections.extend(unreadable); - - let (live, dropped) = in_force(parsed, self.now()); - for (vouch, reason) in dropped { - let rejection = lapsed(&vouch, &reason); - warn!(uri = vouch.uri, %rejection, "vouch not followed"); - found.rejections.push(rejection); - } - - for vouch in live { - match self.target_for(&vouch).await { - Ok(target) => { - debug!( - subject = vouch.subject, - server = target.server.base_url(), - "vouch followed" - ); - found.push(target); - } - Err(rejection) => { - warn!(subject = vouch.subject, %rejection, "subject not followed"); - found.rejections.push(rejection); - } - } - } - } - - /// Resolves a DID to its document and the server holding its repository. - async fn resolve(&self, did: &str) -> Result<(Resolved, ServerRef), Rejection> { - let resolved = - self.resolver - .resolve(did) - .await - .map_err(|reason| Rejection::Unresolvable { - did: did.to_owned(), - reason, - })?; - let server = resolved.pds().map_err(|reason| Rejection::Unresolvable { - did: did.to_owned(), - reason, - })?; - Ok((resolved, server)) - } - - /// Reads the vouch collection out of a voucher's repository. - async fn read_vouches( - &self, - voucher: &str, - server: &ServerRef, - ) -> Result { - let url = server.list_records_url(voucher, nsid::VOUCH, VOUCH_PAGE_LIMIT); - fetch_json(self.resolver.http(), voucher, &url).await - } - - /// Turns one live vouch into the target it names. - async fn target_for(&self, vouch: &Vouch) -> Result { - let (resolved, server) = self.resolve(&vouch.subject).await?; - // The hostname comes off the identifier, never out of the document: - // the document is served by whoever holds the name, so it cannot be - // the thing that says how much the name covers. - let authority = match resolved.host() { - Some(host) => Authority::Host(host.to_owned()), - None => Authority::Did(vouch.subject.clone()), - }; - Ok(Target { - server, - authority, - vouch: Some(VouchRef { - voucher: vouch.voucher.clone(), - uri: vouch.uri.clone(), - subject: vouch.subject.clone(), - }), - }) - } -} - -impl ServerDirectory for VouchDirectory { - async fn targets(&self) -> Discovered { - let mut found = Discovered::default(); - for voucher in &self.vouchers { - self.targets_from(voucher, &mut found).await; - } - found - } -} - -/// Reads every record in a vouch listing, keeping the failures. -fn split_vouches(voucher: &str, listing: RecordsView) -> (Vec, Vec) { - let mut parsed = Vec::new(); - let mut rejections = Vec::new(); - for record in listing.records { - match Vouch::read(voucher, &record.uri, &record.value) { - Ok(vouch) => parsed.push(vouch), - Err(reason) => rejections.push(Rejection::UnreadableVouch { - uri: record.uri, - reason: reason.to_string(), - }), - } - } - (parsed, rejections) -} - -/// Reports a vouch that is no longer in force. -fn lapsed(vouch: &Vouch, reason: &Dropped) -> Rejection { - match reason { - Dropped::Expired { at } => Rejection::ExpiredVouch { - uri: vouch.uri.clone(), - subject: vouch.subject.clone(), - expired_at: at.format(&Rfc3339).unwrap_or_else(|_| at.to_string()), - }, - Dropped::Revoked { by } => Rejection::RevokedVouch { - uri: vouch.uri.clone(), - subject: vouch.subject.clone(), - by: by.clone(), - }, - } -} - -/// Splits a comma or whitespace separated list, dropping empty entries. -fn split_list(list: &str) -> impl Iterator { - list.split([',', ' ', '\t', '\n', '\r']) - .filter(|entry| !entry.is_empty()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_a_mixed_separator_list() { - let directory = StaticDirectory::parse( - "http://localhost:3000, https://agents.example\nhttp://b.example", - ) - .expect("valid"); - let urls: Vec<&str> = directory - .entries() - .iter() - .map(ServerRef::base_url) - .collect(); - assert_eq!( - urls, - [ - "http://localhost:3000", - "https://agents.example", - "http://b.example" - ] - ); - } - - #[test] - fn collapses_a_server_named_twice() { - let directory = - StaticDirectory::parse("http://localhost:3000 http://localhost:3000/").expect("valid"); - assert_eq!(directory.entries().len(), 1); - } - - #[test] - fn one_bad_entry_fails_the_list() { - assert!(StaticDirectory::parse("http://ok.example, nonsense").is_err()); - } - - #[test] - fn an_empty_string_is_an_empty_directory() { - assert!(StaticDirectory::parse(" ").expect("valid").is_empty()); - } - - #[test] - fn a_host_authority_covers_its_subdomains_and_nothing_beside_them() { - let authority = Authority::Host("agents.example".to_owned()); - let inside = AgentDid::parse("did:web:kestrel.agents.example").expect("valid"); - // Shares a suffix, shares no label boundary. The entire attack. - let beside = AgentDid::parse("did:web:kestrel.evilagents.example").expect("valid"); - - assert!(authority.covers(&inside)); - assert!(!authority.covers(&beside)); - } - - #[test] - fn an_account_authority_covers_exactly_one_did() { - let did = "did:web:kestrel.agents.example"; - let authority = Authority::Host("kestrel.agents.example".to_owned()); - let itself = AgentDid::parse(did).expect("valid"); - let sibling = AgentDid::parse("did:web:marmot.agents.example").expect("valid"); - - // A vouch naming one account bounds to that account's hostname, and - // nothing is ever minted below an account. - assert!(authority.covers(&itself)); - assert!(!authority.covers(&sibling)); - } -} diff --git a/crates/didbot-index/src/feed.rs b/crates/didbot-index/src/feed.rs deleted file mode 100644 index 55e1315c..00000000 --- a/crates/didbot-index/src/feed.rs +++ /dev/null @@ -1,205 +0,0 @@ -//! Reading one agent's scrobbles off the server that hosts it. - -use crate::nsid; -use serde_json::Value; -use tracing::warn; - -use crate::http::HttpGet; -use crate::report::{fetch_json, RecordView, RecordsView, Rejection}; -use crate::survey::{FeedSource, Indexer}; - -/// How many records a feed read asks for when the caller does not choose. -pub const DEFAULT_FEED_LIMIT: usize = 50; - -/// Placeholder for a record that arrived without a URI to name it by. -const UNNAMED: &str = "(no uri)"; - -/// One scrobble, as an indexer needs it. -/// -/// A flattened view rather than the raw record: the fields this project -/// defines are named, and `value` keeps the rest so an indexer built against -/// today's lexicon does not silently drop a field added tomorrow. -/// -/// It serializes without `value`, because everything a reader needs is -/// already named beside it and shipping every raw record to every connected -/// client multiplies the size of a snapshot for something none of them read. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Scrobble { - /// The record's AT-URI. - pub uri: String, - /// The DID of the agent that wrote it. - pub did: String, - /// What the agent said it was doing. - pub text: String, - /// The single emoji it chose. - pub emoji: String, - /// The client-declared creation timestamp, RFC 3339. - pub created_at: String, - /// The harness's task identifier, if the agent had one. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub task: Option, - /// The reasoning effort the harness reported, if any. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub effort: Option, - /// The model that produced it, if the harness said. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub model: Option, - /// The kind of agent that wrote it, if the harness said. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub agent_type: Option, - /// The DID of the agent that spawned this one, if the harness could tell. - /// - /// A denormalized copy of the author's profile, and the reason it is - /// on the record at all: the tree is the first thing a canvas draws, and a - /// reader that had to fetch a profile per record to find out where a - /// scrobble hangs would be fetching a profile per record. The profile - /// stays authoritative; nothing here writes one yet. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub parent: Option, - /// AT-URIs of artifacts the scrobble references. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub artifacts: Vec, - /// The record exactly as it was served. - #[serde(skip)] - pub value: Value, -} - -/// One read of one feed. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct FeedPage { - scrobbles: Vec, - rejections: Vec, -} - -impl FeedPage { - /// The scrobbles that were readable, newest first as the server ordered - /// them. - pub fn scrobbles(&self) -> &[Scrobble] { - &self.scrobbles - } - - /// Records that were served but could not be read as scrobbles. - pub fn rejections(&self) -> &[Rejection] { - &self.rejections - } - - /// Splits the page into what it read and what it refused. - pub fn into_parts(self) -> (Vec, Vec) { - (self.scrobbles, self.rejections) - } -} - -impl Indexer { - /// Reads up to `limit` of a source's most recent scrobbles. - /// - /// A failure to reach the feed at all comes back as a page holding one - /// rejection and no scrobbles, so that indexing a list of sources does not - /// stop at the first server that went away mid-run. - pub async fn feed(&self, source: &FeedSource, limit: usize) -> FeedPage { - let server = source.server(); - let url = server.list_records_url(source.did().as_str(), nsid::SCROBBLE, limit); - - let listing: RecordsView = match fetch_json(self.http(), server.base_url(), &url).await { - Ok(listing) => listing, - Err(rejection) => { - warn!(did = source.did().as_str(), %rejection, "feed not read"); - return FeedPage { - scrobbles: Vec::new(), - rejections: vec![rejection], - }; - } - }; - - let mut page = FeedPage::default(); - for record in listing.records { - match read_scrobble(source, record) { - Ok(scrobble) => page.scrobbles.push(scrobble), - Err(rejection) => { - warn!(did = source.did().as_str(), %rejection, "record skipped"); - page.rejections.push(rejection); - } - } - } - page - } -} - -/// Reads one served record as a scrobble, in the shape a listing needs. -/// -/// A thin wrapper over [`read_record`] that names the server in the rejection, -/// which is the one thing a listing knows and a bare record does not. -fn read_scrobble(source: &FeedSource, record: RecordView) -> Result { - let uri = if record.uri.is_empty() { - UNNAMED.to_owned() - } else { - record.uri - }; - read_record(source.did().as_str(), &uri, record.value).map_err(|reason| { - Rejection::UnreadableRecord { - server: source.server().base_url().to_owned(), - uri, - reason, - } - }) -} - -/// Reads one record as a scrobble, wherever it came from. -/// -/// Strict about the three fields the lexicon requires and about `$type`, and -/// lenient about everything else. A record missing its text or its emoji is -/// not a scrobble with a gap in it; it is something else in the scrobble -/// collection, and guessing a default for it would put a blank line in a feed -/// with no way to tell where it came from. -/// -/// Shared with [`crate::firehose`], which reads the same records off a stream -/// rather than out of a listing. Two readers would be two sets of rules for -/// the same lexicon, and the one that drifted would be the one nobody was -/// looking at. -pub fn read_record(did: &str, uri: &str, value: Value) -> Result { - let Some(object) = value.as_object() else { - return Err("the record is not a JSON object".to_owned()); - }; - - // The collection was named in the request, so a record declaring a - // different type is the server answering a question nobody asked. An - // absent `$type` is accepted: the collection already said what this is. - if let Some(declared) = object.get("$type").and_then(Value::as_str) { - if declared != nsid::SCROBBLE { - return Err(format!( - "declares $type {declared:?}, not {:?}", - nsid::SCROBBLE - )); - } - } - - let string = |key: &str| object.get(key).and_then(Value::as_str); - let text = string("text").ok_or("no text".to_owned())?; - let emoji = string("emoji").ok_or("no emoji".to_owned())?; - let created_at = string("createdAt").ok_or("no createdAt".to_owned())?; - - Ok(Scrobble { - uri: uri.to_owned(), - did: did.to_owned(), - text: text.to_owned(), - emoji: emoji.to_owned(), - created_at: created_at.to_owned(), - task: string("task").map(str::to_owned), - effort: string("effort").map(str::to_owned), - model: string("model").map(str::to_owned), - agent_type: string("agentType").map(str::to_owned), - parent: string("parent").map(str::to_owned), - artifacts: object - .get("artifacts") - .and_then(Value::as_array) - .map(|items| { - items - .iter() - .filter_map(Value::as_str) - .map(str::to_owned) - .collect() - }) - .unwrap_or_default(), - value, - }) -} diff --git a/crates/didbot-index/src/firehose.rs b/crates/didbot-index/src/firehose.rs deleted file mode 100644 index 632735b2..00000000 --- a/crates/didbot-index/src/firehose.rs +++ /dev/null @@ -1,510 +0,0 @@ -//! Subscribing to a server's record stream, and resuming after a gap. -//! -//! The survey answers "which accounts exist"; this answers "what have they -//! just written". Before it there was no answer to the second question at all, -//! so the index re-read every repository on a timer and worked out what was -//! new by comparing AT-URIs. That still happens — see the argument in -//! [`crate::survey`] and in the index binary — but it is now the backstop -//! rather than the mechanism. -//! -//! # The cursor is the whole point -//! -//! A subscriber remembers the last frame it applied and hands the server that -//! cursor when it reconnects. The server replays what came after it. What that -//! buys is the two failure modes a stream without one has to choose between: -//! resume from nothing and miss whatever happened during the gap, or re-read -//! everything and pay for it every time. Neither is necessary when the server -//! can say "here is what you missed". -//! -//! It cannot always say that. The buffer is finite and lives in the server's -//! memory, so a subscriber that was away too long, or that is talking to a -//! server which has restarted, gets [`Update::Gap`] instead — the server -//! saying, in as many words, that there are records between the cursor and the -//! oldest frame it still holds. That is not an error and it is not recoverable -//! from the stream; the only thing that can fill it is reading the -//! repositories, which is why the poll does not go away. -//! -//! # What a frame says about the repository -//! -//! A frame names the commit it landed in — the record's CID, the commit's CID -//! and its revision. That is what makes the stream reconcilable rather -//! than merely believable: an index that has applied every frame up to some -//! sequence number holds exactly the records the commit named by that frame -//! covers, and can check that by asking the server for -//! `com.atproto.sync.getLatestCommit` or by reading an export. `/health` -//! reports the head this index last heard for each repository, which is the -//! left-hand side of that comparison. -//! -//! The blocks that would *prove* any of it are not in the frame and are a -//! request away at `com.atproto.sync.getRecord`. Nothing here verifies one: -//! this crate has no DAG-CBOR decoder, and the frame does not -//! carry the proof for it. -//! -//! # What is deliberately not trusted -//! -//! A commit names a repository, and a server can name any repository it likes. -//! Nothing here checks containment, because nothing here has to: the index -//! applies a commit only for an account the survey has already admitted, and -//! the survey is where the DID was checked against both the server's zone and -//! the vouch that bounded it (see [`crate::survey`]). A commit for anything -//! else is dropped and the survey is woken, so an account that is legitimately -//! new is picked up in the ordinary way rather than smuggled in by a stream. - -use std::collections::BTreeSet; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use tokio::sync::mpsc; -use tracing::{debug, info, warn}; - -use crate::follow::{read_frames, Frame, FRAME_BUFFER, RETRY}; -use crate::server::ServerRef; - -/// One record write, as the server puts it on the wire. -/// -/// A mirror of the server's own frame type, declared here rather than shared, -/// for the same reason [`Lifecycle`](crate::follow::Lifecycle) is: an index -/// reading somebody else's server must not compile against that server's -/// types, or it can only ever read servers built from this repository. -#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct Commit { - /// Where the frame sits in the stream. - pub seq: u64, - /// Which run of the server minted [`Self::seq`]. - pub instance: String, - /// The repository written to. - pub did: String, - /// The collection the record went into. - pub collection: String, - /// The record key. - pub rkey: String, - /// The AT-URI naming the record. - pub uri: String, - /// The CID of the record itself. - /// - /// Defaulted, like every field a server might not send: an index reads - /// servers it did not build, and one that predates the commit being named - /// must not stop it dead. An empty string is the honest reading of "this - /// server did not say". - #[serde(default)] - pub cid: String, - /// The CID of the signed commit the write landed in. - /// - /// This is what a repository is reconciled on. A revision is derived on - /// the server that sent it and does not move for every write, - /// so two frames can share one and still cover different records. - #[serde(default)] - pub commit: String, - /// The revision that commit is at. - #[serde(default)] - pub rev: String, - /// When the server stored it, RFC 3339. - #[serde(default)] - pub time: String, - /// The record itself. - pub record: Value, -} - -impl Commit { - /// The cursor to hand back to resume after this frame. - pub fn cursor(&self) -> String { - format!("{}:{}", self.instance, self.seq) - } -} - -/// The stream's opening frame, as the server describes itself. -/// -/// Unknown `name` values are carried rather than refused: a server that grows -/// a fourth thing to say about a connection should not stop an index that has -/// not heard of it, and every name except `OutdatedCursor` means "carry on". -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct StreamInfo { - /// `Live`, `Replay`, `OutdatedCursor`, or something newer. - pub name: String, - /// The run of the server that minted the sequence numbers. - #[serde(default)] - pub instance: String, - /// The oldest sequence number still replayable. - #[serde(default)] - pub oldest: u64, - /// The newest sequence number assigned. - #[serde(default)] - pub newest: u64, - /// What the server has to say about it, for a log line. - #[serde(default)] - pub message: String, -} - -/// The name a server uses when a cursor is out of reach. -const OUTDATED: &str = "OutdatedCursor"; - -/// Something the stream told this index. -#[derive(Debug, Clone, PartialEq)] -pub enum Update { - /// A record was written. - Commit { - /// Which server said so. - server: ServerRef, - /// The write. - commit: Box, - }, - /// Records were missed and cannot be recovered from the stream. - /// - /// The consumer's cursor is older than anything the server still holds, so - /// the only way to find out what is in the gap is to read the - /// repositories. - Gap { - /// Which server said so. - server: ServerRef, - /// The oldest sequence number it can still replay. - oldest: u64, - /// What it said, for a log line. - message: String, - }, -} - -/// What a subscriber wants off the stream, decided once per connection. -/// -/// Named when the subscription first landed and left unbuilt: every commit -/// was read and every commit was forwarded, and the caller filtered -/// afterwards. That is workable for one index reading a handful of -/// collections from a handful of servers and is exactly the cost this exists -/// to cut for anything larger — bandwidth and parse work for records nobody -/// asked for. -/// -/// atproto's own `com.atproto.sync.subscribeRepos` carries no filter at all: -/// a relay aggregates every repository on the network, so there is no -/// narrower thing to ask for and every consumer reads everything and decides -/// for itself. Jetstream — the ecosystem's answer for a consumer that is not -/// a relay — filters on exactly two things a commit carries without decoding -/// the record itself: `wantedCollections` and `wantedDids`. These three modes -/// mirror that pair, plus the case of wanting neither restriction: -/// -/// * [`FirehoseFilter::All`] is `subscribeRepos`' own shape: nothing is -/// dropped, so a consumer that wants to mirror an unfamiliar deployment can. -/// * [`FirehoseFilter::Collections`] is Jetstream's `wantedCollections`: keep -/// only the record types a consumer understands, which is what this index -/// has always wanted and used to get by throwing frames away after they had -/// already been parsed and forwarded. -/// * [`FirehoseFilter::Repos`] is Jetstream's `wantedDids`: keep only the -/// repositories a consumer already holds, which is the shape a survey-first -/// index needs — see [`crate::survey`] for why an unadmitted account's -/// commits are dropped rather than trusted. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum FirehoseFilter { - /// Every commit, on every collection, from every repository. - All, - /// Only commits whose collection is in this set. - Collections(BTreeSet), - /// Only commits whose repository is in this set. - Repos(BTreeSet), -} - -impl FirehoseFilter { - /// A filter down to exactly these collections. - pub fn collections, S: Into>(wanted: I) -> Self { - Self::Collections(wanted.into_iter().map(Into::into).collect()) - } - - /// A filter down to exactly these repositories. - pub fn repos, S: Into>(wanted: I) -> Self { - Self::Repos(wanted.into_iter().map(Into::into).collect()) - } - - /// Whether `commit` is one this filter keeps. - fn admits(&self, commit: &Commit) -> bool { - match self { - FirehoseFilter::All => true, - FirehoseFilter::Collections(wanted) => wanted.contains(&commit.collection), - FirehoseFilter::Repos(wanted) => wanted.contains(&commit.did), - } - } -} - -/// Where a server's record stream is read from, resuming after `cursor`. -fn firehose_url(server: &ServerRef, cursor: Option<&str>) -> String { - match cursor { - // A cursor is `:` and the colon is reserved in a query, - // so it is encoded rather than interpolated. The server percent-decodes - // once on arrival and gets back exactly what it minted. - Some(cursor) => format!( - "{}/firehose?cursor={}", - server.base_url(), - encode_query_value(cursor) - ), - None => format!("{}/firehose", server.base_url()), - } -} - -/// Percent-encodes everything outside the unreserved set. -fn encode_query_value(value: &str) -> String { - let mut out = String::with_capacity(value.len()); - for byte in value.bytes() { - match byte { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => { - out.push(byte as char); - } - other => out.push_str(&format!("%{other:02X}")), - } - } - out -} - -/// Follows one server's firehose from its head, forever. -/// -/// Reconnects on its own, and does so *with* the cursor it has been keeping, -/// which is what makes a restart of either side cost a replay rather than a -/// re-survey. Returns only when the receiver is gone. -pub async fn subscribe(server: ServerRef, filter: FirehoseFilter, out: mpsc::Sender) { - subscribe_from(server, None, filter, out).await; -} - -/// Follows one server's firehose, starting after `cursor`. -/// -/// The starting cursor is a parameter rather than always `None` because an -/// index that outlives one run of itself would hand back what it had stored, -/// and because a resume is not a thing that can be tested end to end if the -/// only way to reach one is to wait for a connection to break. -pub async fn subscribe_from( - server: ServerRef, - cursor: Option, - filter: FirehoseFilter, - out: mpsc::Sender, -) { - let http = reqwest::Client::new(); - let mut cursor = cursor; - loop { - let url = firehose_url(&server, cursor.as_deref()); - debug!(url, "connecting to the firehose"); - - let (frames, mut incoming) = mpsc::channel::(FRAME_BUFFER); - let reader = tokio::spawn({ - let http = http.clone(); - let url = url.clone(); - async move { - match read_frames(&http, &url, &frames).await { - Ok(()) => debug!(url, "firehose ended"), - Err(err) => warn!(url, %err, "firehose failed"), - } - } - }); - - let mut applied = 0usize; - while let Some(frame) = incoming.recv().await { - match frame.event.as_deref() { - Some("info") => { - let Some(info) = frame.parse::() else { - continue; - }; - if info.name == OUTDATED { - warn!( - server = server.base_url(), - oldest = info.oldest, - message = info.message, - "the firehose cannot replay from our cursor" - ); - // The cursor is worthless now: keeping it would ask - // for the same impossible replay on every reconnect. - cursor = None; - let gap = Update::Gap { - server: server.clone(), - oldest: info.oldest, - message: info.message, - }; - if out.send(gap).await.is_err() { - reader.abort(); - return; - } - } else { - info!( - server = server.base_url(), - name = info.name, - oldest = info.oldest, - newest = info.newest, - "firehose connected" - ); - } - } - Some("commit") => { - let Some(commit) = frame.parse::() else { - continue; - }; - // Advanced before the filter and regardless of what - // becomes of the frame: a commit this index chose not to - // keep has still been seen, and asking for it again after - // a reconnect would replay it forever. - cursor = Some(commit.cursor()); - if !filter.admits(&commit) { - debug!( - collection = commit.collection, - did = commit.did, - "firehose frame filtered out" - ); - continue; - } - applied += 1; - let update = Update::Commit { - server: server.clone(), - commit: Box::new(commit), - }; - if out.send(update).await.is_err() { - reader.abort(); - return; - } - } - // A frame this index has no name for. Carried past rather than - // refused, and the cursor is untouched, because a server that - // grows a frame type must not strand an older consumer. - other => debug!(event = other, "firehose frame ignored"), - } - } - - reader.abort(); - if out.is_closed() { - return; - } - debug!( - server = server.base_url(), - applied, - resuming = cursor.is_some(), - "firehose disconnected; retrying" - ); - tokio::time::sleep(RETRY).await; - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn server() -> ServerRef { - ServerRef::parse("http://localhost:3000").expect("valid") - } - - #[test] - fn a_cursorless_subscription_asks_for_the_head() { - assert_eq!( - firehose_url(&server(), None), - "http://localhost:3000/firehose" - ); - } - - #[test] - fn a_cursor_is_encoded_rather_than_interpolated() { - // The colon is reserved in a query string; interpolating it works by - // luck and stops working the moment the instance identifier does not. - assert_eq!( - firehose_url(&server(), Some("18e4a2f:97")), - "http://localhost:3000/firehose?cursor=18e4a2f%3A97" - ); - } - - #[test] - fn a_commit_frame_reads_back_into_a_cursor() { - let frame = Frame { - event: Some("commit".to_owned()), - data: serde_json::json!({ - "seq": 12, - "instance": "18e4a2f", - "did": "did:web:a.agents.localhost%3A3000", - "collection": "com.vibescrobble.scrobble", - "rkey": "3lq", - "uri": "at://did:web:a/com.vibescrobble.scrobble/3lq", - "cid": "bafyreiarecord", - "commit": "bafyreiacommit", - "rev": "3lqmnopqrstuv", - "time": "2026-08-25T10:00:00Z", - "record": {"text": "reading the cart", "emoji": "🔍"}, - }) - .to_string(), - }; - let commit = frame.parse::().expect("a commit frame"); - assert_eq!(commit.cursor(), "18e4a2f:12"); - assert_eq!(commit.record["emoji"], "🔍"); - assert_eq!(commit.commit, "bafyreiacommit"); - } - - /// A server that names no commit is still readable. An index reads - /// servers it did not build, and refusing a frame for a missing field - /// would make one older server stop this one dead. - #[test] - fn a_frame_from_a_server_that_names_no_commit_still_parses() { - let frame = Frame { - event: Some("commit".to_owned()), - data: serde_json::json!({ - "seq": 1, - "instance": "18e4a2f", - "did": "did:web:a", - "collection": "com.vibescrobble.scrobble", - "rkey": "3lq", - "uri": "at://did:web:a/com.vibescrobble.scrobble/3lq", - "record": {}, - }) - .to_string(), - }; - let commit = frame.parse::().expect("a commit frame"); - assert!(commit.commit.is_empty()); - } - - #[test] - fn an_info_frame_this_index_has_not_heard_of_is_not_a_gap() { - let info: StreamInfo = - serde_json::from_str(r#"{"name":"Rebalancing","instance":"x"}"#).expect("readable"); - assert_ne!(info.name, OUTDATED); - assert_eq!(info.oldest, 0); - } - - fn commit_from(did: &str, collection: &str) -> Commit { - Commit { - seq: 1, - instance: "18e4a2f".to_owned(), - did: did.to_owned(), - collection: collection.to_owned(), - rkey: "3lq".to_owned(), - uri: format!("at://{did}/{collection}/3lq"), - cid: String::new(), - commit: String::new(), - rev: String::new(), - time: String::new(), - record: Value::Null, - } - } - - #[test] - fn the_all_filter_admits_any_collection_and_any_repository() { - let filter = FirehoseFilter::All; - assert!(filter.admits(&commit_from("did:web:a", "com.vibescrobble.scrobble"))); - assert!(filter.admits(&commit_from("did:web:b", "com.vibescrobble.registration"))); - } - - #[test] - fn the_collections_filter_admits_a_named_collection_from_any_repository() { - let filter = FirehoseFilter::collections(["com.vibescrobble.scrobble"]); - assert!(filter.admits(&commit_from("did:web:a", "com.vibescrobble.scrobble"))); - assert!(filter.admits(&commit_from("did:web:b", "com.vibescrobble.scrobble"))); - } - - #[test] - fn the_collections_filter_excludes_an_unnamed_collection() { - let filter = FirehoseFilter::collections(["com.vibescrobble.scrobble"]); - assert!(!filter.admits(&commit_from("did:web:a", "com.vibescrobble.registration"))); - } - - #[test] - fn the_repos_filter_admits_a_named_repository_on_any_collection() { - let filter = FirehoseFilter::repos(["did:web:a"]); - assert!(filter.admits(&commit_from("did:web:a", "com.vibescrobble.scrobble"))); - assert!(filter.admits(&commit_from("did:web:a", "com.vibescrobble.registration"))); - } - - #[test] - fn the_repos_filter_excludes_an_unnamed_repository() { - let filter = FirehoseFilter::repos(["did:web:a"]); - assert!(!filter.admits(&commit_from( - "did:web:elsewhere", - "com.vibescrobble.scrobble" - ))); - } -} diff --git a/crates/didbot-index/src/follow.rs b/crates/didbot-index/src/follow.rs deleted file mode 100644 index 57663ea8..00000000 --- a/crates/didbot-index/src/follow.rs +++ /dev/null @@ -1,243 +0,0 @@ -//! Following a personal data server's event stream. -//! -//! The server announces every account transition on `/events` as it happens. -//! Polling would find the same transitions eventually, and eventually is the -//! problem: an agent that provisions, says one thing and ends inside a poll -//! interval is invisible to a poller and obvious to a follower. -//! -//! The server-sent-events reader lives here too, because it is not only the -//! lifecycle stream that needs one: [`crate::firehose`] reads the same format -//! and wants the frames raw, event name and all, rather than parsed into one -//! fixed shape. [`read_frames`] is that reader. - -use std::time::Duration; - -use serde::Deserialize; -use tokio::sync::mpsc; -use tracing::{debug, warn}; - -use crate::http::TransportError; -use crate::server::ServerRef; - -/// How long to wait before reconnecting a stream that ended. -pub const RETRY: Duration = Duration::from_secs(3); - -/// How many frames are buffered between the reader and whatever consumes them. -pub const FRAME_BUFFER: usize = 256; - -/// One frame off a server-sent-event stream, before anything is made of it. -/// -/// The event name is kept because a stream that carries more than one kind of -/// thing — the firehose carries `info` and `commit` — cannot be told apart by -/// its payload alone without every payload growing a discriminant it only -/// needs because the name was thrown away. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Frame { - /// The `event:` name, if the frame carried one. - pub event: Option, - /// Everything from the frame's `data:` lines, concatenated. - pub data: String, -} - -impl Frame { - /// Reads the frame's data as JSON, or nothing if it is not what was asked - /// for. - pub fn parse(&self) -> Option { - serde_json::from_str(&self.data).ok() - } -} - -/// One frame off a server's `/events`. -/// -/// The server's own type, as it puts it on the wire: a `kind` discriminant -/// and the fields that go with it. Unknown kinds are carried rather than -/// refused, because a server that grows a transition should not stop an index -/// that has not heard of it. -#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Lifecycle { - /// Which transition happened. - pub kind: String, - /// The account it applies to. - pub did: String, - /// The agent the account was minted for. Only on `provisioned`. - #[serde(default)] - pub agent_id: Option, - /// Which attestation backend admitted it. Only on `provisioned`. - #[serde(default)] - pub backend: Option, - /// How much that backend's check proves. Only on `provisioned`. - #[serde(default)] - pub assurance: Option, -} - -/// Follows one personal data server's stream. -pub async fn follow(server: ServerRef, out: mpsc::Sender) { - stream(format!("{}/events", server.base_url()), out).await; -} - -/// Follows any server-sent-event endpoint, forwarding frames it can read. -/// -/// Reconnects on its own. A service restarting is an ordinary event in -/// development and should cost its readers a gap, not a process. The gap is -/// real, though: whatever happened while the connection was down is missed, -/// which is why everything that reads one of these also has a way to catch -/// up from a snapshot. -pub async fn stream( - url: String, - out: mpsc::Sender, -) { - let http = reqwest::Client::new(); - loop { - match stream_once(&http, &url, &out).await { - Ok(()) => debug!(url, "event stream ended"), - Err(err) => warn!(url, %err, "event stream failed"), - } - if out.is_closed() { - return; - } - tokio::time::sleep(RETRY).await; - } -} - -/// One connection's worth of frames, translated into `T`. -/// -/// The reader runs in its own task because a frame has to be handed on while -/// the next one is still arriving; doing both in one task means a `select!` -/// over a pinned stream for no gain over a channel and a spawn. -async fn stream_once( - http: &reqwest::Client, - url: &str, - out: &mpsc::Sender, -) -> Result<(), TransportError> { - let (frames, mut incoming) = mpsc::channel::(FRAME_BUFFER); - let out = out.clone(); - let forwarding = tokio::spawn(async move { - while let Some(frame) = incoming.recv().await { - if let Some(value) = frame.parse::() { - if out.send(value).await.is_err() { - return; - } - } - } - }); - let result = read_frames(http, url, &frames).await; - drop(frames); - forwarding.await.ok(); - result -} - -/// Reads one connection's raw frames into `out`, until the body ends. -/// -/// A deliberately small server-sent-events reader: the format is a few lines -/// and a blank one, `reqwest` hands over a body as it arrives, and the -/// alternative is a dependency that also knows about retry policies and -/// last-event-ids that nothing here uses. -pub async fn read_frames( - http: &reqwest::Client, - url: &str, - out: &mpsc::Sender, -) -> Result<(), TransportError> { - use futures_core::Stream; - use std::pin::Pin; - use std::task::Context; - - let response = http - .get(url) - .send() - .await - .map_err(|err| TransportError::new(url, err.to_string()))?; - if !response.status().is_success() { - return Err(TransportError::new( - url, - format!("answered HTTP {}", response.status().as_u16()), - )); - } - - let mut body = response.bytes_stream(); - let mut pending = String::new(); - // `bytes_stream` is a `Stream`, and polling one by hand is the price of - // not depending on `futures` for a single `next()`. - loop { - let chunk = - std::future::poll_fn(|cx: &mut Context<'_>| Pin::new(&mut body).poll_next(cx)).await; - let Some(chunk) = chunk else { return Ok(()) }; - let chunk = chunk.map_err(|err| TransportError::new(url, err.to_string()))?; - pending.push_str(&String::from_utf8_lossy(&chunk)); - - // Frames are separated by a blank line; anything after the last one is - // a frame still arriving and stays in the buffer. - while let Some(split) = pending.find("\n\n") { - let frame = pending[..split].to_owned(); - pending.drain(..split + 2); - if let Some(frame) = read_frame(&frame) { - if out.send(frame).await.is_err() { - return Ok(()); - } - } - } - } -} - -/// Splits one frame into its event name and its data. -/// -/// Comments — the keep-alive lines a server sends to hold an idle connection -/// open — start with a colon and carry no data, so they read as nothing and -/// are dropped without ceremony. So does a frame that names an event and -/// carries no payload. -fn read_frame(frame: &str) -> Option { - let mut event = None; - let mut data = String::new(); - for line in frame.lines() { - if let Some(rest) = line.strip_prefix("event:") { - event = Some(rest.trim().to_owned()); - } else if let Some(rest) = line.strip_prefix("data:") { - data.push_str(rest.trim_start()); - } - } - if data.is_empty() { - return None; - } - Some(Frame { event, data }) -} - -/// Reads one frame's payload as `T`. -#[cfg(test)] -fn parse(frame: &str) -> Option { - read_frame(frame)?.parse() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn reads_a_frame_the_server_would_send() { - let frame = "event: provisioned\ndata: {\"kind\":\"provisioned\",\"did\":\"did:web:a.example\",\"agentId\":\"a\"}"; - let event = parse::(frame).expect("a frame with data is an event"); - assert_eq!(event.kind, "provisioned"); - assert_eq!(event.did, "did:web:a.example"); - assert_eq!(event.agent_id.as_deref(), Some("a")); - } - - #[test] - fn ignores_the_keep_alive() { - assert!(parse::(":").is_none()); - assert!(parse::(": ping").is_none()); - assert!(parse::("event: provisioned").is_none()); - } - - #[test] - fn carries_a_transition_it_has_never_heard_of() { - let event = - parse::("data: {\"kind\":\"hibernated\",\"did\":\"did:web:a.example\"}") - .expect("an unknown kind is still an event"); - assert_eq!(event.kind, "hibernated"); - } - - #[test] - fn refuses_a_frame_that_is_not_a_transition() { - assert!(parse::("data: not json").is_none()); - assert!(parse::("data: {\"kind\":\"provisioned\"}").is_none()); - } -} diff --git a/crates/didbot-index/src/handle.rs b/crates/didbot-index/src/handle.rs deleted file mode 100644 index 01cab1ee..00000000 --- a/crates/didbot-index/src/handle.rs +++ /dev/null @@ -1,192 +0,0 @@ -//! Verifying a handle the way atproto says a handle must be verified. -//! -//! The rule is bidirectional and neither half stands alone. A handle resolves -//! to a DID — here through `/.well-known/atproto-did` on the handle's own -//! hostname — and that DID's document must claim the handle back in -//! `alsoKnownAs`. A resolver that stops after the forward half has learned -//! only that *somebody* answered for a name, which is exactly what a party -//! asserting a name they do not hold is able to arrange. -//! -//! # Why this is not on the survey path -//! -//! [`Indexer::survey`](crate::Indexer::survey) checks a listed handle against -//! the bounds it can decide from the strings in hand — it is a handle, it sits -//! at or below the zone the server mints under, no other account in the survey -//! claims it — and stops there. This module is the rest of the rule, and it -//! costs two requests per account: one for the well-known, one for the DID -//! document. A survey runs on a short timer over the whole population, so -//! paying that on every pass would multiply the traffic by the number of -//! accounts for a check that, on a `did:web`, both directions of are served by -//! the same deployment. -//! -//! That last point is the substance rather than a caveat. For an atproto -//! `did:plc` the two directions come from two parties — a website or a DNS -//! zone on one side, the PLC directory on the other — and the round trip is -//! what stops either from renaming an account alone. For a `did:web` under a -//! didbot zone, the same server answers both, so a hostile one can make -//! itself agree with itself and the round trip proves only self-consistency. -//! What it cannot do is hold a name outside its zone, and that is the check -//! the survey makes for free. -//! -//! So this is an on-demand check: `didbot-survey --verify-handles` runs -//! it, and it is what an operator reaches for when they want to know whether a -//! deployment's handles actually resolve rather than whether they are -//! plausible. - -use didbot_identity::{atproto_did_url, parse_atproto_did, HandleError}; - -use crate::http::{preview, HttpGet}; -use crate::resolve::{DidResolver, ResolveFailure}; - -/// How much of an unreadable body is quoted back. -const BODY_PREVIEW: usize = 160; - -/// Why a handle is not this account's handle. -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -pub enum HandleFailure { - /// The string is not a handle, so there is nothing to resolve. - #[error("{handle:?} is not a handle: {reason}")] - NotAHandle { - /// The string that was offered. - handle: String, - /// Why it is not one. - reason: HandleError, - }, - /// Nothing answered at the well-known path. - #[error("could not reach {url}: {reason}")] - Unreachable { - /// The URL that was requested. - url: String, - /// What the transport reported. - reason: String, - }, - /// Something answered, with a status that is not a success. - /// - /// A 404 lands here like any other. It is the honest answer for a name - /// nobody holds, and it is not distinguished from a 500: either way this - /// handle did not resolve, and the status is in the message for whoever - /// has to fix it. - #[error("{url} answered HTTP {status}: {body}")] - BadStatus { - /// The URL that was requested. - url: String, - /// The status it returned. - status: u16, - /// The start of the body it returned. - body: String, - }, - /// The body was not a DID. - #[error("{url} answered with something that is not a did: {body:?} ({reason})")] - NotADid { - /// The URL that was requested. - url: String, - /// The start of the body it returned. - body: String, - /// What the validator said. - reason: String, - }, - /// The handle resolves, to somebody else. - #[error("{handle} resolves to {found}, not to {expected}")] - ResolvesElsewhere { - /// The handle that was checked. - handle: String, - /// The DID that was expected to hold it. - expected: String, - /// The DID the well-known named. - found: String, - }, - /// The DID the handle resolves to would not resolve itself. - #[error("{did} would not resolve: {reason}")] - Unresolvable { - /// The DID that was being resolved. - did: String, - /// Why it would not. - reason: ResolveFailure, - }, - /// The reverse direction: the document does not claim the handle back. - /// - /// The failure the whole check exists for. Everything up to here can be - /// arranged by whoever answers for the handle's hostname; only the DID's - /// own document can say the account agrees. - #[error("{did} does not claim {handle} back; its document claims {claimed:?}")] - NotClaimedBack { - /// The DID that was resolved. - did: String, - /// The handle it was expected to claim. - handle: String, - /// What its document claims instead, if anything. - claimed: Option, - }, -} - -/// Verifies `handle` belongs to `did`, in both directions. -/// -/// `port` is development's escape hatch and follows atproto's own rule for a -/// `did:web`: a port is allowed on loopback and refused anywhere else. A whole -/// development zone is one listener on one ephemeral port, and a handle string -/// — unlike a DID — carries no port of its own, so the caller supplies it. -/// [`ServerRef::port`](crate::ServerRef::port) is where it comes from. -/// -/// The forward direction is checked first and the document is only fetched -/// when it agrees, because a handle that resolves somewhere else is already -/// answered and the second request would be asked on behalf of a claim that -/// has already failed. -pub async fn verify( - resolver: &DidResolver, - handle: &str, - did: &str, - port: Option, -) -> Result<(), HandleFailure> { - let url = atproto_did_url(handle, port).map_err(|reason| HandleFailure::NotAHandle { - handle: handle.to_owned(), - reason, - })?; - - let response = resolver - .http() - .get(&url) - .await - .map_err(|err| HandleFailure::Unreachable { - url: url.clone(), - reason: err.reason, - })?; - if !response.is_success() { - return Err(HandleFailure::BadStatus { - url, - status: response.status, - body: preview(&response.body, BODY_PREVIEW), - }); - } - - let found = parse_atproto_did(&response.body).map_err(|reason| HandleFailure::NotADid { - url: url.clone(), - body: preview(&response.body, BODY_PREVIEW), - reason: reason.to_string(), - })?; - // Byte for byte. A DID is compared exactly across the network — two - // spellings would be two identities — and the handle is the only half of - // this pair that normalizes. - if found != did { - return Err(HandleFailure::ResolvesElsewhere { - handle: handle.to_owned(), - expected: did.to_owned(), - found: found.to_owned(), - }); - } - - let resolved = resolver - .resolve(did) - .await - .map_err(|reason| HandleFailure::Unresolvable { - did: did.to_owned(), - reason, - })?; - if !resolved.document().claims_handle(handle) { - return Err(HandleFailure::NotClaimedBack { - did: did.to_owned(), - handle: handle.to_owned(), - claimed: resolved.document().handle().map(str::to_owned), - }); - } - Ok(()) -} diff --git a/crates/didbot-index/src/http.rs b/crates/didbot-index/src/http.rs deleted file mode 100644 index d55ca477..00000000 --- a/crates/didbot-index/src/http.rs +++ /dev/null @@ -1,154 +0,0 @@ -//! The HTTP boundary, kept narrow enough to stub. -//! -//! Everything this crate fetches is a GET that returns JSON, so the trait it -//! is generic over has exactly one method. That is not abstraction for its own -//! sake: it is what lets the survey and feed logic — which is where the -//! interesting rules live — be tested against fixed responses, including the -//! malicious ones a real server would have to be persuaded to send. - -use std::future::Future; - -/// A response that reached a status code. -/// -/// The body is read whatever the status. On a failure it is the only -/// actionable detail an operator gets, and on a success it is the payload. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HttpResponse { - /// The status code. - pub status: u16, - /// The body, as text. - pub body: String, -} - -impl HttpResponse { - /// Builds a response, for tests and for transport implementations. - pub fn new(status: u16, body: impl Into) -> Self { - Self { - status, - body: body.into(), - } - } - - /// Whether the status is in the 2xx range. - pub fn is_success(&self) -> bool { - (200..300).contains(&self.status) - } -} - -/// A request that never reached a status code: no route, no listener, timeout. -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -#[error("could not reach {url}: {reason}")] -pub struct TransportError { - /// The URL that was attempted. - pub url: String, - /// What went wrong, as the transport described it. - pub reason: String, -} - -impl TransportError { - /// Records a failed attempt against `url`. - pub fn new(url: impl Into, reason: impl Into) -> Self { - Self { - url: url.into(), - reason: reason.into(), - } - } -} - -/// Something that can perform an HTTP GET. -pub trait HttpGet { - /// Fetches `url`. - /// - /// A non-2xx status is a successful call returning an unhappy - /// [`HttpResponse`], not an `Err`: this crate reports "the server said no" - /// and "the server said nothing" differently, and collapsing them here - /// would make that impossible upstream. - fn get(&self, url: &str) -> impl Future> + Send; -} - -/// Lets one transport back a directory and an indexer at the same time. -/// -/// Both hold their client by value, and a deployment wants them sharing one: -/// the same connection pool, and one place to set a timeout. -impl HttpGet for std::sync::Arc -where - T: HttpGet + Send + Sync + ?Sized, -{ - fn get(&self, url: &str) -> impl Future> + Send { - (**self).get(url) - } -} - -/// The [`HttpGet`] implementation that actually talks to the network. -#[derive(Debug, Clone, Default)] -pub struct ReqwestGet { - http: reqwest::Client, -} - -impl ReqwestGet { - /// Builds a client with `reqwest`'s defaults. - pub fn new() -> Self { - Self::default() - } - - /// Builds a client over a caller-supplied one, for timeouts or a proxy. - pub fn with_client(http: reqwest::Client) -> Self { - Self { http } - } -} - -impl HttpGet for ReqwestGet { - async fn get(&self, url: &str) -> Result { - let response = self - .http - .get(url) - .send() - .await - .map_err(|err| TransportError::new(url, err.to_string()))?; - let status = response.status().as_u16(); - let body = response - .text() - .await - .map_err(|err| TransportError::new(url, err.to_string()))?; - Ok(HttpResponse { status, body }) - } -} - -/// Shortens a body for an error message. -/// -/// Errors quoting a server's response are read in a terminal, and a server -/// that answers an XRPC call with a megabyte of HTML should not be able to -/// fill it. -pub(crate) fn preview(body: &str, limit: usize) -> String { - let trimmed = body.trim(); - if trimmed.chars().count() <= limit { - return trimmed.to_owned(); - } - let head: String = trimmed.chars().take(limit).collect(); - format!("{head}…") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn preview_leaves_a_short_body_alone() { - assert_eq!(preview(" {\"ok\":true} ", 100), "{\"ok\":true}"); - } - - #[test] - fn preview_truncates_on_characters_not_bytes() { - // Slicing by byte offset would panic here, which is exactly the bug - // this function exists to not have. - assert_eq!(preview("ααααα", 3), "ααα…"); - } - - #[test] - fn success_is_the_two_hundreds_only() { - assert!(HttpResponse::new(200, "").is_success()); - assert!(HttpResponse::new(204, "").is_success()); - assert!(!HttpResponse::new(302, "").is_success()); - assert!(!HttpResponse::new(404, "").is_success()); - } -} diff --git a/crates/didbot-index/src/lib.rs b/crates/didbot-index/src/lib.rs deleted file mode 100644 index ae4fc9ac..00000000 --- a/crates/didbot-index/src/lib.rs +++ /dev/null @@ -1,154 +0,0 @@ -//! Finding the agent feeds to index, and reading them. -//! -//! An appview, in atproto's sense, is the whole read side of an application: -//! the lexicons it defines, the service that indexes other people's -//! repositories, and whatever renders the result. This crate is the indexing -//! part of that — no HTTP surface of its own, no opinion about what is drawn -//! with what it collects. -//! -//! Before anything can be indexed, two questions have to be answered, and they -//! are not the same question: -//! -//! 1. **Whose agents are these, and who says so?** A human writes a -//! [`Vouch`] record in their own repository naming the personal data server -//! they run: these are my agents, and I take responsibility for them. The -//! appview is configured with the humans it is willing to listen to, and -//! everything else follows from records those humans signed — -//! [`VouchDirectory`] resolves the voucher, reads their vouches, and -//! resolves each subject to the server it names. -//! 2. **Which accounts on that server does the vouch actually cover?** The -//! server answers `bot.did.listAgents` with the accounts it claims -//! to host. A claim is not a fact. [`Indexer::survey`] admits an account -//! only when it sits at or below both the zone the server mints under and -//! the name the vouch bounded it to. -//! -//! # Where to fetch and what may be claimed are different facts -//! -//! A [`Target`] carries both, deliberately apart. The *endpoint* comes out of -//! the subject's DID document and may be any host: an atproto account is free -//! to name a hosting provider, and a didbot server is no different. The -//! *[`Authority`]* comes off the vouched DID itself — for a `did:web`, the -//! hostname in the identifier — so no matter which host answers, it can only -//! answer for names at or below the one the human vouched for. -//! -//! That is containment read from the other end. The server enforces -//! containment when it mints; this crate enforces it again when it reads, -//! because the reader is the party that gets hurt if it does not. A server -//! listing `did:web:someagent.trusted.example` is describing an account it -//! has no authority over, and an indexer taking that at face value would file -//! its scrobbles under somebody else's agent. The check needs no round trip: -//! the DID string and the vouched name are enough. -//! -//! A listed handle is the same kind of claim and gets the same treatment. It -//! has to be a handle, it has to sit at or below the server's zone, and no -//! other account in the survey may claim it; one that fails is dropped and the -//! account keeps its DID. Completing atproto's bidirectional rule — resolving -//! the handle and reading the account's own DID document back — is -//! [`handle::verify`], which is not on the survey path because it is two -//! requests per account and, for a `did:web`, both directions are served by -//! the same deployment. -//! -//! Three things are deliberately *not* trusted: -//! -//! * A DID document that claims to be a different DID. `did:web` documents -//! are served by whoever holds the name, so the `id` is checked every time. -//! * The zone a server reports on `/health`. It is read, but a server whose -//! zone neither contains nor sits under the vouched name is refused outright -//! — it has nothing to do with what was vouched for. -//! * A vouch that has expired, or that a later record in the same repository -//! revokes. Revocation is repository-local, so nobody can cancel somebody -//! else's vouch. -//! -//! # Three ways of hearing about the same thing -//! -//! A server's `/events` stream carries account lifecycle transitions, its -//! `/firehose` carries every record as it is written, and a survey is a -//! snapshot of what exists right now. All three are needed and none is -//! redundant: the streams say what changed but only while somebody is -//! listening, and the survey says what is true but only at the moment it runs. -//! [`firehose::subscribe`] is the record stream and [`Indexer::feed`] is the -//! catch-up read that fills whatever the stream could not replay. -//! -//! It stores nothing beyond [`View`]: parsed scrobbles are handed back and -//! where they go is the caller's problem. -//! -//! # Example -//! -//! ```no_run -//! # async fn run() -> Result<(), Box> { -//! use std::sync::Arc; -//! use didbot_index::{DidResolver, Indexer, ReqwestGet, ServerDirectory, VouchDirectory}; -//! -//! let http = Arc::new(ReqwestGet::new()); -//! let vouchers = ["did:plc:nlzmjyfv6loqtxyzvdcznwgf".to_owned()]; -//! let directory = VouchDirectory::new(DidResolver::new(http.clone()), vouchers); -//! -//! let (targets, refused) = directory.targets().await.into_parts(); -//! for rejection in &refused { -//! eprintln!("not followed: {rejection}"); -//! } -//! -//! let indexer = Indexer::new(http); -//! let survey = indexer.survey(&targets).await; -//! for source in survey.sources() { -//! for scrobble in indexer.feed(source, 20).await.scrobbles() { -//! println!("{} {} {}", scrobble.emoji, source.agent_id(), scrobble.text); -//! } -//! } -//! # Ok(()) -//! # } -//! ``` - -#![forbid(unsafe_code)] - -pub mod directory; -pub mod feed; -pub mod firehose; -pub mod follow; -pub mod handle; -pub mod http; -pub mod nsid; -pub mod registration; -pub mod replay; -pub mod report; -pub mod resolve; -pub mod server; -pub mod service; -pub mod survey; -pub mod view; -pub mod vouch; -pub mod work; - -pub use directory::{ - Authority, Discovered, ServerDirectory, StaticDirectory, Target, VouchDirectory, VouchRef, -}; -pub use feed::{read_record, FeedPage, Scrobble}; -pub use firehose::{subscribe, subscribe_from, Commit, FirehoseFilter, StreamInfo, Update}; -pub use follow::{follow, Lifecycle}; -pub use handle::{verify as verify_handle, HandleFailure}; -pub use http::{HttpGet, HttpResponse, ReqwestGet, TransportError}; -pub use registration::{read_profile, Profile}; -pub use report::Rejection; -pub use resolve::{DidResolver, ResolveFailure, Resolved, DEFAULT_PLC_DIRECTORY}; -pub use server::{ServerRef, UrlError}; -pub use service::{app, Head, Pulse, Shared}; -pub use survey::{FeedSource, Indexer, Survey}; -pub use view::{Agent, Event, Lineage, Operator, Snapshot, View, GROUPS}; -pub use vouch::{Vouch, VouchError}; -pub use work::{WorkGroup, WorkIndex, MIN_CLUSTER}; - -#[cfg(test)] -mod tests; - -/// Sets up logging the way every binary in this crate wants it. -/// -/// `info` by default and `RUST_LOG` when somebody wants more, which is the -/// same contract the rest of the workspace's services offer. -pub fn init_tracing() { - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), - ) - .init(); -} diff --git a/crates/didbot-index/src/nsid.rs b/crates/didbot-index/src/nsid.rs deleted file mode 100644 index 0c6b55c0..00000000 --- a/crates/didbot-index/src/nsid.rs +++ /dev/null @@ -1,42 +0,0 @@ -//! The record collections this indexer reads, named here rather than imported. -//! -//! An indexer is a consumer. It learns what an account wrote by asking the -//! server over XRPC and by reading the firehose, so the only thing it needs -//! from a lexicon is the collection's name — not its schema, and not the -//! authority that defines it. Naming them here keeps the indexer's dependency -//! on the hosting server at the wire, which is where a consumer's dependency -//! on a server belongs. -//! -//! These are copies. While the definitions and the indexer share a workspace, -//! `copies_agree_with_the_definitions` holds them to it. - -/// Intent-level status update. One per interesting turn, and the record the -/// feed is built from. -pub const SCROBBLE: &str = "com.vibescrobble.scrobble"; - -/// Signed ownership or delegation attestation. Read to decide which accounts -/// an authority vouches for. -pub const VOUCH: &str = "bot.did.vouch"; - -/// The hosting server's record of registering an account. Read for the account -/// metadata the directory and the canvas display. -pub const REGISTRATION: &str = "bot.did.registration"; - -#[cfg(test)] -mod tests { - use super::*; - - /// The indexer's copies must name the same collections the definitions do. - /// - /// This is the only place the indexer refers to the lexicon crate, and it - /// is a development dependency: a drift between the two would otherwise - /// show up as an indexer that silently reads nothing. - #[test] - fn copies_agree_with_the_definitions() { - use didbot_lexicon::nsid as defined; - - assert_eq!(SCROBBLE, defined::SCROBBLE); - assert_eq!(VOUCH, defined::VOUCH); - assert_eq!(REGISTRATION, defined::REGISTRATION); - } -} diff --git a/crates/didbot-index/src/registration.rs b/crates/didbot-index/src/registration.rs deleted file mode 100644 index cd9f6a0e..00000000 --- a/crates/didbot-index/src/registration.rs +++ /dev/null @@ -1,215 +0,0 @@ -//! Reading an account's registration record, which is what it says before it -//! speaks. -//! -//! The index used to read lineage off the scrobbles, and then off a profile -//! the harness supplied. It reads the registration record now, which is -//! strictly better for the same fields: the hosting server composes it from -//! its own bookkeeping, writes it before the repository is readable, and only -//! names a parent it hosts itself. An agent that never scrobbles still has -//! one. -//! -//! # Two ways in, one reader -//! -//! It arrives on the firehose the moment an account is provisioned, and it is -//! re-read from the repository on the poll's ordinary beat. Both go through -//! [`read_profile`], for the reason [`crate::feed::read_record`] is shared -//! between the same two paths: two readers would be two sets of rules for one -//! lexicon, and the one that drifted would be the one nobody was looking at. -//! -//! The poll is not redundant with the stream. A commit for an account no -//! survey has admitted yet is dropped — the account is provisioned and this -//! record is written in the same breath, so its frame is exactly the one most -//! likely to arrive before the index has heard of its author. -//! -//! The model is not here. It changes every turn, so it is stamped on the -//! records a turn writes and the index learns it from those. - -use crate::nsid; -use serde_json::Value; -use tracing::warn; - -use crate::http::HttpGet; -use crate::report::{fetch_json, RecordsView}; -use crate::survey::{FeedSource, Indexer}; - -/// The record key a profile always sits at. -/// -/// `bot.did.registration` declares `key: "literal:self"`, so this is -/// not a convention: a repository has exactly one profile and it is here. A -/// record served from this collection under any other key is a server that -/// does not implement the lexicon, and is refused rather than read. -pub const SELF: &str = "self"; - -/// One agent's profile, as an indexer needs it. -/// -/// Only the fields this index acts on. The record is not kept: unlike a -/// scrobble, a profile is a singleton that is overwritten in place, so an -/// index holding the raw value would be holding a copy of something that has -/// no history to preserve. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct Profile { - /// The DID of the account whose profile this is. - pub did: String, - /// The kind of agent, as its harness names it. - pub agent_type: Option, - /// The harness that names it. - pub harness: Option, - /// The DID the hosting server says spawned it. - /// - /// Stronger than the claim this used to read: the server writes it only - /// for a parent it hosts itself. [`View::see_profile`](crate::View::see_profile) - /// still applies the same refusals, because the server saying it is not - /// this index trusting it. - pub parent: Option, -} - -impl Indexer { - /// Reads one account's profile, if it has written one. - /// - /// `None` covers every way of not having an answer, and they are all - /// ordinary: the server did not respond, the repository holds no profile, - /// or what it holds is not one. An account without a profile is an account - /// this index knows nothing extra about, which is the state every account - /// was in until now. - /// - /// One record is asked for, because there is only ever one. - pub async fn profile(&self, source: &FeedSource) -> Option { - let server = source.server(); - let did = source.did().as_str(); - let url = server.list_records_url(did, nsid::REGISTRATION, 1); - - let listing: RecordsView = match fetch_json(self.http(), server.base_url(), &url).await { - Ok(listing) => listing, - Err(rejection) => { - warn!(did, %rejection, "profile not read"); - return None; - } - }; - let record = listing.records.into_iter().next()?; - match read_profile(did, &record.uri, record.value) { - Ok(profile) => Some(profile), - Err(reason) => { - warn!(did, uri = record.uri, reason, "profile skipped"); - None - } - } - } -} - -/// Reads one served record as a profile, wherever it came from. -/// -/// Strict about two things and lenient about everything else. `$type`, for -/// the reason [`crate::feed::read_record`] checks it: a record declaring -/// something else is the server answering a question nobody asked. And the -/// record key, which must be `self` — a profile filed under a TID is a -/// repository holding several profiles with no way to say which is the one, -/// which is precisely the defect fixed on the writing side, and an index -/// that read them anyway would hide it. -/// -/// Every field is optional, including all four this index reads. A profile -/// carrying nothing but its `createdAt` is a valid profile and says the -/// harness had nothing to report, which is not the same as a malformed one. -pub fn read_profile(did: &str, uri: &str, value: Value) -> Result { - let Some(object) = value.as_object() else { - return Err("the record is not a JSON object".to_owned()); - }; - - if let Some(declared) = object.get("$type").and_then(Value::as_str) { - if declared != nsid::REGISTRATION { - return Err(format!( - "declares $type {declared:?}, not {:?}", - nsid::REGISTRATION - )); - } - } - - // Checked against the AT-URI rather than a separate key field, because the - // URI is what a listing and a firehose frame both carry. A URI that does - // not end in `/self` names a record this collection cannot hold. - if !uri.is_empty() && !uri.ends_with(&format!("/{SELF}")) { - return Err(format!( - "is filed at {uri:?}, and a profile lives at {SELF:?}" - )); - } - - // The kind-specific facts sit inside the union member, so an account that - // is not an agent simply answers nothing here rather than being refused: - // this index is looking for lineage and a harness, and a pipeline has - // neither. - let actor = object.get("actor").and_then(Value::as_object); - let member = |key: &str| { - actor - .and_then(|actor| actor.get(key)) - .and_then(Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) - }; - - Ok(Profile { - did: did.to_owned(), - agent_type: member("agentType"), - harness: member("harness"), - parent: member("parent"), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - fn uri(rkey: &str) -> String { - format!("at://did:web:a.example/{}/{rkey}", nsid::REGISTRATION) - } - - #[test] - fn reads_the_fields_the_index_acts_on() { - let value = json!({ - "$type": nsid::REGISTRATION, - "did": "did:web:a.example", - "provisionedAt": "2024-01-01T00:00:00Z", - "operator": "did:web:agents.example", - "owner": "did:web:owner.example", - "admittedBy": "dev-shared-secret", - "assurance": "shared-secret", - "revision": 1, - "actor": { - "$type": "bot.did.registration#agent", - "harness": "claude-code", - "agentType": "Explore", - "parent": "did:web:parent.example", - }, - }); - let profile = read_profile("did:web:a.example", &uri(SELF), value).expect("read"); - assert_eq!(profile.agent_type.as_deref(), Some("Explore")); - assert_eq!(profile.harness.as_deref(), Some("claude-code")); - assert_eq!(profile.parent.as_deref(), Some("did:web:parent.example")); - } - - #[test] - fn a_profile_with_only_its_timestamp_is_still_a_profile() { - let value = json!({ "createdAt": "2024-01-01T00:00:00Z" }); - let profile = read_profile("did:web:a.example", &uri(SELF), value).expect("read"); - assert_eq!( - profile, - Profile { - did: "did:web:a.example".to_owned(), - ..Profile::default() - } - ); - } - - #[test] - fn a_profile_filed_anywhere_but_self_is_refused() { - let value = json!({ "createdAt": "2024-01-01T00:00:00Z" }); - let reason = read_profile("did:web:a.example", &uri("3lbxyz"), value).expect_err("refused"); - assert!(reason.contains("self"), "{reason}"); - } - - #[test] - fn a_record_claiming_another_type_is_refused() { - let value = json!({ "$type": nsid::SCROBBLE, "createdAt": "2024-01-01T00:00:00Z" }); - read_profile("did:web:a.example", &uri(SELF), value).expect_err("refused"); - } -} diff --git a/crates/didbot-index/src/replay.rs b/crates/didbot-index/src/replay.rs deleted file mode 100644 index 0c976c26..00000000 --- a/crates/didbot-index/src/replay.rs +++ /dev/null @@ -1,488 +0,0 @@ -//! A recorded firehose, and a server that plays it back. -//! -//! # What this is for -//! -//! Working on the index, the query service or the canvas needs a personal data -//! server and a swarm in front of it: three terminals and a population, before -//! anything downstream has a byte to look at. The canvas already ships a mock -//! for exactly that reason. This is the same idea one layer down — a recording -//! of a real stream, played back with nothing else running. -//! -//! What it buys over keeping a `--data` directory is that a recording is a -//! *sequence*, not a state. The same frames arrive in the same order every -//! time, as fast or as slowly as you ask, so a read-side bug is reproduced -//! rather than re-provoked, and a recording can be trimmed by hand to the -//! three frames that cause it. -//! -//! # The format is one frame per line -//! -//! JSON Lines, each carrying the server-sent event's name and its data. Not a -//! clever container: the point is that a person can open a recording, delete -//! the frames that are not the bug, and hand the file to somebody else. -//! -//! # What it does not pretend to be -//! -//! A personal data server. It answers the firehose and the account listing an -//! index asks for on every pass, and nothing else — no provisioning, no -//! repositories, no blobs. An index following it fills in what it can and -//! retries what it cannot, which is what it does against a real server that is -//! part way up. - -use std::sync::Arc; - -use axum::extract::{Query, State}; -use axum::response::sse::Sse; -use axum::response::IntoResponse; -use axum::routing::get; -use axum::Router; -use serde::{Deserialize, Serialize}; - -use crate::follow::Frame; - -/// One frame as a recording holds it. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct Recorded { - /// The `event:` name the frame carried, if any. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub event: Option, - /// Everything from the frame's `data:` lines. - pub data: String, -} - -impl From for Recorded { - fn from(frame: Frame) -> Self { - Self { - event: frame.event, - data: frame.data, - } - } -} - -impl Recorded { - /// The sequence number this frame carries, if it is a commit. - /// - /// Read out of the data rather than stored beside it, so that a recording - /// somebody has edited by hand cannot disagree with itself. - pub fn seq(&self) -> Option { - serde_json::from_str::(&self.data) - .ok()? - .get("seq")? - .as_u64() - } - - /// The DID this frame is about, if it names one. - pub fn did(&self) -> Option { - serde_json::from_str::(&self.data) - .ok()? - .get("did")? - .as_str() - .map(str::to_owned) - } -} - -/// A recording, in the order it was captured. -#[derive(Debug, Clone, Default)] -pub struct Recording { - /// Every frame, in order. - pub frames: Vec, -} - -impl Recording { - /// Parses a recording, skipping blank lines and refusing bad ones by line. - /// - /// A line that will not parse is an error naming the line, because a - /// recording is a file people edit and the useful answer to a typo is - /// which line it is on. - pub fn parse(text: &str) -> Result { - let mut frames = Vec::new(); - for (number, line) in text.lines().enumerate() { - if line.trim().is_empty() { - continue; - } - let frame = serde_json::from_str(line).map_err(|source| RecordingError::Line { - line: number + 1, - source, - })?; - frames.push(frame); - } - Ok(Self { frames }) - } - - /// Renders a recording back to the format [`Recording::parse`] reads. - pub fn render(&self) -> String { - let mut out = String::new(); - for frame in &self.frames { - out.push_str(&serde_json::to_string(frame).expect("a frame serializes")); - out.push('\n'); - } - out - } - - /// Every account the recording mentions, in the order first seen. - /// - /// An index asks a server what accounts it has on every survey pass. A - /// recording knows only the accounts that wrote something, which is fewer - /// than a real server would list and is the honest answer: nothing here - /// saw the others. - pub fn accounts(&self) -> Vec { - let mut seen = Vec::new(); - for frame in &self.frames { - if let Some(did) = frame.did() { - if !seen.contains(&did) { - seen.push(did); - } - } - } - seen - } - - /// The zone the recorded accounts were minted under, if it can be told. - /// - /// An index refuses to survey a server whose `/health` does not name a - /// zone — that is how it tells a didbot server from anything else - /// answering on a port — so a replay has to name one or it is a firehose - /// nobody will read past. - /// - /// It is derived rather than recorded: a `did:web` is the zone with a - /// label in front of it, so dropping the leftmost label of the first - /// account in the recording gives the zone that minted it. Derived, so a - /// recording somebody trimmed by hand cannot end up claiming a zone none - /// of its frames are from. - pub fn zone(&self) -> Option { - let (zone, _port) = self.zone_and_port()?; - Some(zone) - } - - /// The zone and, where the recording came from a development stack, its - /// port. - /// - /// They are separated because a server declares its zone *without* a port — - /// `agents.localhost`, not `agents.localhost:3400` — while the account - /// hostnames it mints carry one. A replay that declared the port would be - /// refused by the containment check for a reason that reads like a trust - /// failure and is a formatting difference. - fn zone_and_port(&self) -> Option<(String, Option)> { - let did = self.accounts().into_iter().next()?; - let host = did.strip_prefix("did:web:")?.replace("%3A", ":"); - let (_label, rest) = host.split_once('.')?; - Some(match rest.rsplit_once(':') { - Some((zone, port)) => (zone.to_owned(), port.parse().ok()), - None => (rest.to_owned(), None), - }) - .filter(|(zone, _)| !zone.is_empty()) - } - - /// The address a playback should listen on to be believed. - /// - /// An index does not take a server's word for the zone it mints under: it - /// checks that the zone sits under the host it was pointed at, which is - /// the containment rule the whole trust model rests on. A replay is - /// serving somebody else's recording, so it satisfies that rule only by - /// listening where the recording came from. - /// - /// Loopback and the recorded port, then — `agents.localhost:3400` becomes - /// `127.0.0.1:3400`, and an index pointed at `http://localhost:3400` - /// surveys it as it would the server that was recorded. `None` when the - /// recording names no port, which is a recording from a deployment rather - /// than from a development stack. - pub fn listen_hint(&self) -> Option { - let (_zone, port) = self.zone_and_port()?; - Some(format!("127.0.0.1:{}", port?)) - } - - /// The frames after `cursor`, or all of them when there is none. - /// - /// A cursor is `:` as the firehose defines it, and only the - /// sequence number is used: a recording is one instance by construction. - pub fn after(&self, cursor: Option<&str>) -> &[Recorded] { - let Some(seq) = cursor - .and_then(|cursor| cursor.rsplit(':').next()) - .and_then(|seq| seq.parse::().ok()) - else { - return &self.frames; - }; - let start = self - .frames - .iter() - .position(|frame| frame.seq().is_some_and(|found| found > seq)) - .unwrap_or(self.frames.len()); - &self.frames[start..] - } -} - -/// Why a recording could not be read. -#[derive(Debug, thiserror::Error)] -pub enum RecordingError { - /// One line is not a frame. - #[error("line {line} is not a recorded frame: {source}")] - Line { - /// Which line, counting from one. - line: usize, - /// What serde said. - source: serde_json::Error, - }, -} - -/// How fast a replay hands frames over. -#[derive(Debug, Clone, Copy)] -pub struct Pace { - /// Frames per second, or `None` for as fast as the socket takes them. - pub per_second: Option, -} - -impl Pace { - /// The gap between two frames, if there is one. - pub fn gap(self) -> Option { - let rate = self.per_second?; - (rate > 0.0).then(|| std::time::Duration::from_secs_f64(1.0 / rate)) - } -} - -/// What the replay server is serving. -#[derive(Clone)] -struct Playback { - recording: Arc, - pace: Pace, -} - -/// A router that serves `recording` as a firehose. -pub fn router(recording: Recording, pace: Pace) -> Router { - let state = Playback { - recording: Arc::new(recording), - pace, - }; - Router::new() - .route("/firehose", get(firehose)) - .route("/health", get(health)) - .route("/xrpc/bot.did.listAgents", get(list_agents)) - .with_state(state) -} - -/// The recording, as a server-sent event stream. -async fn firehose( - State(playback): State, - Query(params): Query>, -) -> impl IntoResponse { - let cursor = params.get("cursor").cloned(); - let frames: Vec = playback.recording.after(cursor.as_deref()).to_vec(); - let gap = playback.pace.gap(); - - let stream = async_stream::stream(frames, gap); - Sse::new(stream) -} - -/// What this is and how much of it there is. -async fn health(State(playback): State) -> impl IntoResponse { - axum::Json(serde_json::json!({ - "status": "ok", - // The field an index looks for before it will read anything else here. - "zone": playback.recording.zone(), - // And the ones that say this is not the server it is pretending to be. - "replay": true, - "frames": playback.recording.frames.len(), - "accounts": playback.recording.accounts().len(), - })) -} - -/// The accounts the recording mentions, shaped as the real listing is. -async fn list_agents(State(playback): State) -> impl IntoResponse { - let agents: Vec = playback - .recording - .accounts() - .into_iter() - .map(|did| serde_json::json!({ "did": did })) - .collect(); - axum::Json(serde_json::json!({ "agents": agents })) -} - -/// The stream half, kept apart because it is the only part that is a `Stream`. -mod async_stream { - use std::convert::Infallible; - use std::future::Future; - use std::pin::Pin; - use std::task::{Context, Poll}; - use std::time::Duration; - - use axum::response::sse::Event; - use futures_core::Stream; - - use super::Recorded; - - /// A stream that hands over `frames`, waiting `gap` between them. - /// - /// Hand-rolled for the reason [`crate::follow::read_frames`] is: the crate - /// that would provide this also provides a great deal that nothing here - /// uses, and the state machine is one index and one sleep. - pub(super) fn stream( - frames: Vec, - gap: Option, - ) -> impl Stream> + Send + 'static { - Playing { - frames, - at: 0, - gap, - waiting: None, - } - } - - struct Playing { - frames: Vec, - at: usize, - gap: Option, - waiting: Option>>, - } - - impl Stream for Playing { - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); - if let Some(sleep) = this.waiting.as_mut() { - match sleep.as_mut().poll(cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(()) => this.waiting = None, - } - } - let Some(frame) = this.frames.get(this.at) else { - // The end of a recording is the end of the stream, and a - // subscriber that reconnects gets it again from its cursor — - // which is what a real server does when it has nothing more. - return Poll::Ready(None); - }; - this.at += 1; - let mut event = Event::default().data(frame.data.clone()); - if let Some(name) = &frame.event { - event = event.event(name.clone()); - } - if let Some(gap) = this.gap { - this.waiting = Some(Box::pin(tokio::time::sleep(gap))); - } - Poll::Ready(Some(Ok(event))) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn recording() -> Recording { - Recording::parse( - r#"{"event":"info","data":"{\"instance\":\"abc\",\"oldest\":1}"} -{"event":"commit","data":"{\"seq\":1,\"did\":\"did:web:a\"}"} -{"event":"commit","data":"{\"seq\":2,\"did\":\"did:web:b\"}"} -{"event":"commit","data":"{\"seq\":3,\"did\":\"did:web:a\"}"} -"#, - ) - .expect("the fixture parses") - } - - #[test] - fn a_recording_round_trips_through_its_own_format() { - let first = recording(); - let again = Recording::parse(&first.render()).expect("re-read"); - assert_eq!(first.frames, again.frames); - } - - #[test] - fn a_blank_line_is_not_a_frame() { - let parsed = Recording::parse("\n\n{\"data\":\"{}\"}\n\n").expect("parses"); - assert_eq!(parsed.frames.len(), 1); - } - - #[test] - fn a_line_that_will_not_parse_says_which_line() { - let err = Recording::parse("{\"data\":\"{}\"}\nnot json\n").expect_err("refused"); - assert!(matches!(err, RecordingError::Line { line: 2, .. }), "{err}"); - } - - #[test] - fn a_cursor_hands_back_only_what_came_after_it() { - let recording = recording(); - let after = recording.after(Some("abc:1")); - assert_eq!(after.len(), 2); - assert_eq!(after[0].seq(), Some(2)); - } - - #[test] - fn no_cursor_is_the_whole_recording() { - assert_eq!(recording().after(None).len(), 4); - } - - #[test] - fn a_cursor_past_the_end_hands_back_nothing() { - assert!(recording().after(Some("abc:99")).is_empty()); - } - - #[test] - fn a_cursor_that_is_not_one_is_ignored_rather_than_refused() { - // A recording is a development tool and a mistyped cursor should show - // you the stream, not an error page. - assert_eq!(recording().after(Some("nonsense")).len(), 4); - } - - #[test] - fn the_zone_comes_off_the_accounts_rather_than_being_declared() { - let recording = Recording::parse( - r#"{"event":"commit","data":"{\"seq\":1,\"did\":\"did:web:marmot.agents.localhost%3A3400\"}"} -"#, - ) - .expect("parses"); - // Without the port: that is how a server declares its own zone, and - // an index compares the two. - assert_eq!(recording.zone().as_deref(), Some("agents.localhost")); - } - - #[test] - fn a_playback_listens_where_the_recording_came_from() { - // Not a convenience: an index checks that a server's zone sits under - // the host it was pointed at, so a playback anywhere else is refused - // however correct its frames are. - let recording = Recording::parse( - r#"{"event":"commit","data":"{\"seq\":1,\"did\":\"did:web:marmot.agents.localhost%3A3400\"}"} -"#, - ) - .expect("parses"); - assert_eq!(recording.listen_hint().as_deref(), Some("127.0.0.1:3400")); - } - - #[test] - fn a_recording_with_no_port_suggests_no_address() { - let recording = Recording::parse( - r#"{"event":"commit","data":"{\"seq\":1,\"did\":\"did:web:kestrel.agents.example\"}"} -"#, - ) - .expect("parses"); - assert_eq!(recording.zone().as_deref(), Some("agents.example")); - assert_eq!(recording.listen_hint(), None); - } - - #[test] - fn a_recording_of_nothing_claims_no_zone() { - // Better than guessing one: an index that is told a zone reads on, and - // reading on into an empty recording is not an improvement. - assert_eq!(Recording::default().zone(), None); - } - - #[test] - fn the_accounts_are_the_ones_that_said_something() { - assert_eq!(recording().accounts(), vec!["did:web:a", "did:web:b"]); - } - - #[test] - fn a_pace_of_none_waits_for_nothing() { - assert!(Pace { per_second: None }.gap().is_none()); - assert!(Pace { - per_second: Some(0.0) - } - .gap() - .is_none()); - assert_eq!( - Pace { - per_second: Some(4.0) - } - .gap(), - Some(std::time::Duration::from_millis(250)) - ); - } -} diff --git a/crates/didbot-index/src/report.rs b/crates/didbot-index/src/report.rs deleted file mode 100644 index 893525db..00000000 --- a/crates/didbot-index/src/report.rs +++ /dev/null @@ -1,258 +0,0 @@ -//! What the appview declined, and why. -//! -//! Rejections are values rather than log lines, and they are one type rather -//! than one per stage. An operator who pointed the appview at a voucher and -//! got nothing back needs to be told which link of the chain gave way — the -//! voucher would not resolve, the vouch had expired, the server it named -//! answered with something else — and a count of indexed feeds cannot say -//! any of that. - -use serde::de::DeserializeOwned; - -use crate::http::{preview, HttpGet}; -use crate::resolve::ResolveFailure; - -/// How much of a server's answer is quoted back in a rejection. -const BODY_PREVIEW: usize = 160; - -/// A claim this appview declined, and why. -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -pub enum Rejection { - /// Nothing answered. - #[error("{context}: unreachable: {reason}")] - Unreachable { - /// What was being read: a server, or the DID being resolved. - context: String, - /// What the transport reported. - reason: String, - }, - /// Something answered, with a status that is not a success. - #[error("{context}: {url} answered HTTP {status}: {body}")] - BadStatus { - /// What was being read. - context: String, - /// The endpoint that was called. - url: String, - /// The status it returned. - status: u16, - /// The start of the body it returned. - body: String, - }, - /// The answer was not the shape this appview reads. - #[error("{context}: {url} answered with something unreadable: {reason}")] - Malformed { - /// What was being read. - context: String, - /// The endpoint that was called. - url: String, - /// What could not be read. - reason: String, - }, - /// A DID in the chain would not resolve. - #[error("{did}: {reason}")] - Unresolvable { - /// The DID that was being resolved. - did: String, - /// Why it would not resolve, or would not yield a server. - reason: ResolveFailure, - }, - /// A record in the voucher's vouch collection is not a vouch. - #[error("{uri}: not a readable vouch: {reason}")] - UnreadableVouch { - /// The record's AT-URI. - uri: String, - /// What was wrong with it. - reason: String, - }, - /// A vouch that has lapsed. - #[error("{uri}: the vouch for {subject} expired at {expired_at}")] - ExpiredVouch { - /// The record's AT-URI. - uri: String, - /// Who it vouched for. - subject: String, - /// When it stopped being valid, RFC 3339. - expired_at: String, - }, - /// A vouch a later record in the same repository cancelled. - #[error("{uri}: the vouch for {subject} was revoked by {by}")] - RevokedVouch { - /// The record's AT-URI. - uri: String, - /// Who it vouched for. - subject: String, - /// The AT-URI of the record that revoked it. - by: String, - }, - /// The health endpoint answered, but not as a didbot server. - /// - /// Its own zone is the one thing a server must be able to state: what it - /// may speak for is decided against that zone. - #[error("{server}: answered /health without a zone; this is not a didbot server")] - NotDidbot { - /// The server that was read. - server: String, - }, - /// The server's zone and the vouched name have nothing to do with each - /// other. - /// - /// One has to contain the other. A server minting under `agents.example` - /// can answer for a vouch naming `agents.example` or one naming a single - /// account inside it, and for nothing else. - #[error("{server}: mints under {zone:?}, which neither contains nor sits under the vouched {authority:?}")] - ZoneOutsideAuthority { - /// The server that was read. - server: String, - /// The zone it reported. - zone: String, - /// The name the vouch bounded it to. - authority: String, - }, - /// An account was listed with a DID this appview cannot parse. - #[error("{server}: listed {did:?}, which is not a usable did:web: {reason}")] - UnusableDid { - /// The server that listed it. - server: String, - /// The DID as listed. - did: String, - /// Why it would not parse. - reason: String, - }, - /// An account was listed that the server has no authority over. - #[error("{server}: listed {did:?}, which is not at or below its zone {zone:?}")] - DidEscapesZone { - /// The server that listed it. - server: String, - /// The DID as listed. - did: String, - /// The zone it had to sit under. - zone: String, - }, - /// A vouch named one account, and the server it pointed at does not host - /// it. - #[error("{server}: does not host the vouched account {did}")] - AccountNotHosted { - /// The server that was read. - server: String, - /// The account the vouch named. - did: String, - }, - /// A record in an indexed feed was not a readable scrobble. - #[error("{server}: record {uri} is not a readable scrobble: {reason}")] - UnreadableRecord { - /// The server the record came from. - server: String, - /// The record's AT-URI, or a placeholder when it had none. - uri: String, - /// What was wrong with it. - reason: String, - }, - /// An account was listed with a handle the server has no authority over. - /// - /// Containment read as a rule about handles. A handle resolves through the name - /// itself — `/.well-known/atproto-did` on the handle's own hostname — so a - /// handle outside the server's zone is one the server cannot make resolve - /// and has no business claiming. The account is still indexed: its DID - /// passed containment and is what everything downstream is decided on. - /// The handle is what is dropped, because nothing else can be. - #[error("{server}: listed {did} with the handle {handle:?}, which is not at or below its zone {zone:?}; the handle is dropped")] - HandleEscapesZone { - /// The server that listed it. - server: String, - /// The account it was listed against. - did: String, - /// The handle as listed. - handle: String, - /// The zone it had to sit under. - zone: String, - }, - /// An account was listed with a handle that is not a handle. - #[error("{server}: listed {did} with {handle:?}, which is not a handle: {reason}; the handle is dropped")] - UnusableHandle { - /// The server that listed it. - server: String, - /// The account it was listed against. - did: String, - /// The handle as listed. - handle: String, - /// Why it is not one. - reason: String, - }, - /// Two accounts were listed under one handle. - /// - /// A handle resolves to exactly one DID, so a second claim on a name is - /// one the reverse direction would refuse. Which of the two is the real - /// holder is not knowable from the listing, so neither keeps it: the - /// first is left alone only because dropping both on the arrival of a - /// forgery would let a server erase a name by claiming it twice. - #[error("{server}: listed {did} under the handle {handle:?}, already claimed by {first}; the handle is dropped")] - DuplicateHandle { - /// The server whose copy was dropped. - server: String, - /// The account whose handle was dropped. - did: String, - /// The handle claimed twice. - handle: String, - /// The account that claimed it first. - first: String, - }, - /// Two vouches, or two servers, offered the same account. - #[error("{server}: also offers {did:?}, already indexed from {first}; the copy is ignored")] - DuplicateDid { - /// The server whose copy was dropped. - server: String, - /// The DID offered twice. - did: String, - /// The server that offered it first. - first: String, - }, -} - -/// GETs `url` and parses it, mapping every failure to a [`Rejection`]. -/// -/// `context` is what the operator should be told was being read: a server's -/// base URL, or the DID whose repository this is. -pub(crate) async fn fetch_json(http: &H, context: &str, url: &str) -> Result -where - T: DeserializeOwned, - H: HttpGet, -{ - let response = http.get(url).await.map_err(|err| Rejection::Unreachable { - context: context.to_owned(), - reason: err.reason, - })?; - - if !response.is_success() { - return Err(Rejection::BadStatus { - context: context.to_owned(), - url: url.to_owned(), - status: response.status, - body: preview(&response.body, BODY_PREVIEW), - }); - } - - serde_json::from_str(&response.body).map_err(|err| Rejection::Malformed { - context: context.to_owned(), - url: url.to_owned(), - reason: err.to_string(), - }) -} - -/// The `com.atproto.repo.listRecords` response. -/// -/// Shared by the vouch read and the feed read: the same XRPC method, on two -/// different servers, for two different collections. -#[derive(Debug, serde::Deserialize)] -pub(crate) struct RecordsView { - #[serde(default)] - pub(crate) records: Vec, -} - -/// One entry in that response. -#[derive(Debug, serde::Deserialize)] -pub(crate) struct RecordView { - #[serde(default)] - pub(crate) uri: String, - #[serde(default)] - pub(crate) value: serde_json::Value, -} diff --git a/crates/didbot-index/src/resolve.rs b/crates/didbot-index/src/resolve.rs deleted file mode 100644 index c1122691..00000000 --- a/crates/didbot-index/src/resolve.rs +++ /dev/null @@ -1,263 +0,0 @@ -//! Resolving a DID to the personal data server that holds its repository. -//! -//! Two methods, because those are the two this system actually meets. A human -//! vouching for their agents is an ordinary atproto account, so their DID is -//! almost always a `did:plc` resolved through the PLC directory. The thing -//! they vouch *for* is a didbot server or an account on one, which is a -//! `did:web` resolved from the hostname it names. -//! -//! Neither answer is trusted further than it goes. The document's `id` is -//! checked against the DID that was asked for — atproto is blunt about this, -//! "the DID declared in the document (in the `id` field) should always be -//! verified against what was expected" — and the endpoint it names is only a -//! statement about *where* the repository is. What may be claimed from that -//! endpoint is bounded by the DID, not by the server; see -//! [`Authority`](crate::directory::Authority). - -use didbot_identity::{AgentDid, DidDocument}; - -use crate::http::{preview, HttpGet}; -use crate::server::{ServerRef, UrlError}; - -/// The PLC directory used when a caller does not name one. -pub const DEFAULT_PLC_DIRECTORY: &str = "https://plc.directory"; - -/// How much of an unreadable body is quoted back. -const BODY_PREVIEW: usize = 160; - -/// Length of the identifier in a `did:plc`. -const PLC_IDENTIFIER_LENGTH: usize = 24; - -/// Why a DID would not resolve. -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -pub enum ResolveFailure { - /// A method this appview cannot resolve. - #[error("{0:?} is not a did:web or a did:plc")] - UnsupportedMethod(String), - /// A `did:web` that is not a legal one. - #[error("not a usable did:web: {0}")] - BadDidWeb(String), - /// A `did:plc` whose identifier is not base32-sortable. - /// - /// Checked before the identifier is put in a URL. A directory lookup is a - /// path join, and a path join with an unvalidated identifier in it is how - /// a resolver ends up requesting something other than a DID document. - #[error("{0:?} is not a well-formed did:plc identifier")] - BadDidPlc(String), - /// Nothing answered. - #[error("could not reach {url}: {reason}")] - Unreachable { - /// The URL that was requested. - url: String, - /// What the transport reported. - reason: String, - }, - /// Something answered, with a status that is not a success. - #[error("{url} answered HTTP {status}: {body}")] - BadStatus { - /// The URL that was requested. - url: String, - /// The status it returned. - status: u16, - /// The start of the body it returned. - body: String, - }, - /// The body was not a DID document. - #[error("the document at {url} could not be read: {reason}")] - Malformed { - /// The URL that was requested. - url: String, - /// What the deserializer reported. - reason: String, - }, - /// The document describes a different DID. - #[error("the document at {url} claims {found:?}, not {expected:?}")] - IdMismatch { - /// The URL that was requested. - url: String, - /// The DID that was asked for. - expected: String, - /// The DID the document claimed. - found: String, - }, - /// The document names no personal data server. - #[error("{did} resolves to a document with no #atproto_pds service")] - NoPdsEndpoint { - /// The DID that was resolved. - did: String, - }, - /// The endpoint it names is not a base URL this appview will fetch from. - #[error("{did} names the endpoint {endpoint:?}, which is unusable: {reason}")] - BadEndpoint { - /// The DID that was resolved. - did: String, - /// The endpoint the document named. - endpoint: String, - /// Why it could not be used. - reason: UrlError, - }, -} - -/// A DID that resolved, and what its document said. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Resolved { - did: String, - host: Option, - document: DidDocument, -} - -impl Resolved { - /// The DID that was resolved. - pub fn did(&self) -> &str { - &self.did - } - - /// The hostname the DID names, for a `did:web`. - /// - /// `None` for a `did:plc`, which names no host. This is the hostname that - /// bounds what may be claimed under the DID, and it is read off the - /// identifier rather than out of the document: the document is served by - /// whoever holds the name, so it cannot be the thing that says what the - /// name covers. - pub fn host(&self) -> Option<&str> { - self.host.as_deref() - } - - /// The resolved document. - pub fn document(&self) -> &DidDocument { - &self.document - } - - /// The personal data server holding this DID's repository. - pub fn pds(&self) -> Result { - let endpoint = - self.document - .pds_endpoint() - .ok_or_else(|| ResolveFailure::NoPdsEndpoint { - did: self.did.clone(), - })?; - ServerRef::parse(endpoint).map_err(|reason| ResolveFailure::BadEndpoint { - did: self.did.clone(), - endpoint: endpoint.to_owned(), - reason, - }) - } -} - -/// Resolves DIDs over an HTTP transport. -#[derive(Debug, Clone)] -pub struct DidResolver { - http: H, - plc_directory: String, -} - -impl DidResolver { - /// A resolver using [`DEFAULT_PLC_DIRECTORY`]. - pub fn new(http: H) -> Self { - Self { - http, - plc_directory: DEFAULT_PLC_DIRECTORY.to_owned(), - } - } - - /// A resolver against a different PLC directory, or a mirror of one. - pub fn with_plc_directory(http: H, plc_directory: &str) -> Self { - Self { - http, - plc_directory: plc_directory.trim_end_matches('/').to_owned(), - } - } - - /// The transport this resolver fetches with. - pub fn http(&self) -> &H { - &self.http - } - - /// Resolves `did` and checks that the document admits to being it. - pub async fn resolve(&self, did: &str) -> Result { - let (url, host) = self.document_url(did)?; - let document = self.fetch_document(&url).await?; - if document.id != did { - return Err(ResolveFailure::IdMismatch { - url, - expected: did.to_owned(), - found: document.id, - }); - } - Ok(Resolved { - did: did.to_owned(), - host, - document, - }) - } - - /// Where `did`'s document lives, and the hostname the DID names. - fn document_url(&self, did: &str) -> Result<(String, Option), ResolveFailure> { - if did.starts_with("did:web:") { - // `AgentDid` is this project's validated `did:web`. A service DID - // is not an agent, but the parse it needs — percent-decoding, the - // path-based refusal, ports only on loopback — is the same one, - // and a second copy of it would be a second thing to get wrong. - let parsed = - AgentDid::parse(did).map_err(|err| ResolveFailure::BadDidWeb(err.to_string()))?; - return Ok((parsed.did_json_url(), Some(parsed.host().to_owned()))); - } - if let Some(identifier) = did.strip_prefix("did:plc:") { - if !is_plc_identifier(identifier) { - return Err(ResolveFailure::BadDidPlc(did.to_owned())); - } - return Ok((format!("{}/{did}", self.plc_directory), None)); - } - Err(ResolveFailure::UnsupportedMethod(did.to_owned())) - } - - /// GETs a DID document. - async fn fetch_document(&self, url: &str) -> Result { - let response = self - .http - .get(url) - .await - .map_err(|err| ResolveFailure::Unreachable { - url: url.to_owned(), - reason: err.reason, - })?; - if !response.is_success() { - return Err(ResolveFailure::BadStatus { - url: url.to_owned(), - status: response.status, - body: preview(&response.body, BODY_PREVIEW), - }); - } - serde_json::from_str(&response.body).map_err(|err| ResolveFailure::Malformed { - url: url.to_owned(), - reason: err.to_string(), - }) - } -} - -/// Whether `identifier` is the base32-sortable body of a `did:plc`. -/// -/// The specification fixes both the length and the alphabet, and this runs -/// before the identifier is joined into a directory URL. -fn is_plc_identifier(identifier: &str) -> bool { - identifier.len() == PLC_IDENTIFIER_LENGTH - && identifier - .bytes() - .all(|b| b.is_ascii_lowercase() || (b'2'..=b'7').contains(&b)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_plc_identifier_is_fixed_length_base32() { - assert!(is_plc_identifier("nlzmjyfv6loqtxyzvdcznwgf")); - // Too short, uppercase, a digit outside the alphabet, and the one - // that matters: anything that would climb out of the directory path. - assert!(!is_plc_identifier("nlzmjyfv6loqtxyzvdcznwg")); - assert!(!is_plc_identifier("NLZMJYFV6LOQTXYZVDCZNWGF")); - assert!(!is_plc_identifier("nlzmjyfv8loqtxyzvdcznwgf")); - assert!(!is_plc_identifier("../../../etc/passwd0000000")); - } -} diff --git a/crates/didbot-index/src/server.rs b/crates/didbot-index/src/server.rs deleted file mode 100644 index ba7491f2..00000000 --- a/crates/didbot-index/src/server.rs +++ /dev/null @@ -1,260 +0,0 @@ -//! A server this appview is willing to talk to, and the URLs it is reached at. - -use std::fmt; - -/// Path the health probe reads the zone from. -pub const HEALTH_PATH: &str = "/health"; - -/// Why a base URL is not usable as a server reference. -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -pub enum UrlError { - /// No `scheme://` at all. A bare hostname is ambiguous about transport. - #[error("{0:?} is not an absolute url; write it as http://host or https://host")] - NotAbsolute(String), - /// A scheme this appview does not fetch over. - #[error("{url:?} uses the {scheme:?} scheme; only http and https are indexed")] - UnsupportedScheme { - /// The URL as given. - url: String, - /// The scheme found on it. - scheme: String, - }, - /// Credentials in the authority. - /// - /// Rejected rather than stripped. `http://trusted.example@evil.example` - /// reads as `trusted.example` to a human and resolves to `evil.example`, - /// and every containment check in this crate is anchored on the hostname, - /// so the one input that must never be misread is this one. - #[error("{0:?} carries userinfo before the hostname")] - Userinfo(String), - /// Nothing between the scheme and the path. - #[error("{0:?} has no hostname")] - NoHost(String), -} - -/// A didbot personal data server, as a base URL and the host it names. -/// -/// The hostname is kept beside the URL because it is the anchor for every -/// containment decision this crate makes, and re-parsing it at each use is how -/// two call sites end up disagreeing about what was dialled. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ServerRef { - base_url: String, - host: String, -} - -impl ServerRef { - /// Parses a base URL, e.g. `http://localhost:3000`. - /// - /// A trailing slash is trimmed so joining a path cannot double the - /// separator. The parse is deliberately strict and deliberately small: it - /// accepts an absolute http or https URL with a bare authority and - /// nothing else, and anything with more shape than that is a URL somebody - /// should look at rather than one this crate should guess about. - pub fn parse(base_url: &str) -> Result { - let raw = base_url.trim(); - let (scheme, rest) = raw - .split_once("://") - .ok_or_else(|| UrlError::NotAbsolute(raw.to_owned()))?; - if !scheme.eq_ignore_ascii_case("http") && !scheme.eq_ignore_ascii_case("https") { - return Err(UrlError::UnsupportedScheme { - url: raw.to_owned(), - scheme: scheme.to_owned(), - }); - } - - let authority = rest - .split(['/', '?', '#']) - .next() - .unwrap_or(rest) - .to_owned(); - if authority.contains('@') { - return Err(UrlError::Userinfo(raw.to_owned())); - } - let host = host_without_port(&authority); - if host.is_empty() { - return Err(UrlError::NoHost(raw.to_owned())); - } - - Ok(Self { - base_url: format!( - "{}://{}", - scheme.to_ascii_lowercase(), - authority.to_ascii_lowercase() - ), - host: host.to_ascii_lowercase(), - }) - } - - /// The base URL, normalized: lowercased, no path, no trailing slash. - pub fn base_url(&self) -> &str { - &self.base_url - } - - /// The hostname that was dialled, lowercased and without a port. - /// - /// This is the authority every DID offered by this server is checked - /// against. The port is not part of it: a port says where to reach a host, - /// not which host it is. - pub fn host(&self) -> &str { - &self.host - } - - /// The port that was dialled, when the URL named one. - /// - /// Only development has any use for it: a whole zone served on one - /// ephemeral loopback listener means a handle under that zone is reached - /// on the same port, and a handle string carries no port of its own the - /// way a `did:web` does. In production the answer is `None` and the - /// handle is reached on 443, which is the case that needs no help. - pub fn port(&self) -> Option { - let authority = self.base_url.rsplit("://").next()?; - let (_, port) = authority.rsplit_once(':')?; - port.parse().ok() - } - - /// Where the health probe reads the server's zone from. - pub fn health_url(&self) -> String { - format!("{}{HEALTH_PATH}", self.base_url) - } - - /// Where an XRPC method is called. - pub fn xrpc_url(&self, nsid: &str) -> String { - format!("{}/xrpc/{nsid}", self.base_url) - } - - /// Where this server lists the accounts it claims to host. - pub fn list_agents_url(&self) -> String { - self.xrpc_url("bot.did.listAgents") - } - - /// Where one repository's records in one collection are read. - /// - /// `repo` is percent-encoded rather than interpolated. An agent DID for a - /// development port already contains `%3A`, and a query parameter is - /// decoded once on arrival, so passing it through raw would ask the server - /// for `did:web:host:3000` — a DID it has never minted and cannot match. - pub fn list_records_url(&self, repo: &str, collection: &str, limit: usize) -> String { - format!( - "{}?repo={}&collection={}&limit={limit}", - self.xrpc_url("com.atproto.repo.listRecords"), - encode_query_value(repo), - encode_query_value(collection), - ) - } -} - -impl fmt::Display for ServerRef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.base_url) - } -} - -impl std::str::FromStr for ServerRef { - type Err = UrlError; - - fn from_str(s: &str) -> Result { - Self::parse(s) - } -} - -/// Strips a port from an authority, unbracketing an IPv6 literal with it. -fn host_without_port(authority: &str) -> &str { - if let Some(rest) = authority.strip_prefix('[') { - return rest.split(']').next().unwrap_or(rest); - } - match authority.rsplit_once(':') { - Some((host, port)) if !port.is_empty() && port.bytes().all(|b| b.is_ascii_digit()) => host, - _ => authority, - } -} - -/// Percent-encodes everything outside the unreserved set. -/// -/// Written out rather than pulled in: the only values this crate puts in a -/// query are DIDs and NSIDs, the unreserved set is four punctuation marks, and -/// a dependency whose job is a sixteen-line loop is a dependency to explain. -fn encode_query_value(value: &str) -> String { - let mut out = String::with_capacity(value.len()); - for byte in value.bytes() { - match byte { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => { - out.push(byte as char); - } - other => out.push_str(&format!("%{other:02X}")), - } - } - out -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_a_development_url() { - let server = ServerRef::parse("http://localhost:3000/").expect("valid"); - assert_eq!(server.base_url(), "http://localhost:3000"); - assert_eq!(server.host(), "localhost"); - assert_eq!(server.health_url(), "http://localhost:3000/health"); - } - - #[test] - fn normalizes_scheme_and_host_case() { - let server = ServerRef::parse("HTTPS://Agents.Example").expect("valid"); - assert_eq!(server.base_url(), "https://agents.example"); - assert_eq!(server.host(), "agents.example"); - } - - #[test] - fn keeps_the_port_in_the_url_and_out_of_the_host() { - let server = ServerRef::parse("http://[::1]:3000").expect("valid"); - assert_eq!(server.base_url(), "http://[::1]:3000"); - assert_eq!(server.host(), "::1"); - } - - #[test] - fn drops_a_path() { - let server = ServerRef::parse("http://example:80/xrpc/whatever?a=b").expect("valid"); - assert_eq!(server.base_url(), "http://example:80"); - } - - #[test] - fn refuses_userinfo() { - // The whole attack: reads as trusted.example, resolves to evil.example. - let err = ServerRef::parse("http://trusted.example@evil.example").expect_err("refused"); - assert!(matches!(err, UrlError::Userinfo(_))); - } - - #[test] - fn refuses_a_url_it_cannot_anchor_a_check_on() { - assert!(matches!( - ServerRef::parse("localhost:3000"), - Err(UrlError::NotAbsolute(_)) - )); - assert!(matches!( - ServerRef::parse("file:///etc/passwd"), - Err(UrlError::UnsupportedScheme { .. }) - )); - assert!(matches!( - ServerRef::parse("http:///lonely/path"), - Err(UrlError::NoHost(_)) - )); - } - - #[test] - fn encodes_the_percent_in_a_development_did() { - let server = ServerRef::parse("http://localhost:3000").expect("valid"); - let url = server.list_records_url( - "did:web:kestrel.agents.localhost%3A3000", - "com.vibescrobble.scrobble", - 20, - ); - assert_eq!( - url, - "http://localhost:3000/xrpc/com.atproto.repo.listRecords\ - ?repo=did%3Aweb%3Akestrel.agents.localhost%253A3000\ - &collection=com.vibescrobble.scrobble&limit=20" - ); - } -} diff --git a/crates/didbot-index/src/service.rs b/crates/didbot-index/src/service.rs deleted file mode 100644 index 0245e49b..00000000 --- a/crates/didbot-index/src/service.rs +++ /dev/null @@ -1,355 +0,0 @@ -//! The index's own HTTP surface. -//! -//! Two things, and deliberately only two: the whole view for something that -//! has just connected, and a stream of what changes after that. Everything -//! else a reader might want — a feed, a search, a page of history — belongs to -//! whatever serves people, because the moment this starts answering those it -//! stops being an index and becomes the application. -//! -//! It is not the browser's endpoint either. That is the query service's job; -//! this is what the query service reads. - -use std::collections::BTreeMap; -use std::convert::Infallible; -use std::pin::Pin; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use std::task::{Context, Poll}; -use std::time::Instant; - -use axum::extract::State; -use axum::response::sse::{Event as SseEvent, KeepAlive, Sse}; -use axum::response::IntoResponse; -use axum::routing::get; -use axum::{Json, Router}; -use futures_core::Stream; -use tokio::sync::{broadcast, mpsc, Mutex}; -use tower_http::cors::{Any, CorsLayer}; -use tracing::{debug, warn}; - -use crate::view::{Event, View}; -use crate::work::WorkGroup; - -/// How many events the broadcast channel holds for a lagging subscriber. -pub const EVENT_CAPACITY: usize = 1024; - -/// How many events are buffered per connected client. -const CLIENT_BUFFER: usize = 64; - -/// What a handler needs. -#[derive(Clone)] -pub struct Shared { - /// The view, shared with whatever is updating it. - pub view: Arc>, - /// Where changes are announced. - pub events: broadcast::Sender, - /// How this process is doing, for whoever has to operate it. - pub pulse: Arc, -} - -/// The handful of numbers that say whether this process is actually working. -/// -/// Counts of things held are already on `/health` and they are not the same -/// question: an index holding four hundred agents looks identical whether it -/// heard about them a second ago or lost its upstream an hour ago and has -/// been serving the same snapshot since. What distinguishes the two is how -/// long it has been since anything changed, and whether it currently believes -/// it is connected to whatever it follows. -/// -/// Deliberately not a metrics endpoint. There is no registry, no histograms -/// and no scrape format, because nothing here is deployed anywhere that has a -/// collector, and a Prometheus surface nobody scrapes is a dependency and a -/// wire format to keep working in exchange for nothing. These are the fields -/// an operator reads in a terminal with `curl`, which is the operating story -/// this project actually has. -#[derive(Debug)] -pub struct Pulse { - started: Instant, - /// Events this process has announced since it started. - announced: AtomicU64, - /// When something last changed the view. - changed: Mutex>, - /// What this process follows, and whether it currently thinks it is - /// connected to it. - upstream: Mutex>, - /// The head this process last heard for each repository. - /// - /// Not a count of anything held, which is what the paragraph above says - /// this is not for. It is the left-hand side of one comparison an - /// operator can actually make: the commit this index believes a - /// repository is at, against what `com.atproto.sync.getLatestCommit` - /// answers for the same DID. Equal means this index is exactly caught up - /// with that repository; different means it is behind, and says by which - /// commit. - heads: Mutex>, -} - -/// Where one repository was, the last time a frame said. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] -pub struct Head { - /// The CID of the signed commit. - pub commit: String, - /// The revision that commit is at. - pub rev: String, - /// The sequence number of the frame that said so. - pub seq: u64, -} - -impl Pulse { - /// A process that has just started and heard nothing yet. - pub fn new() -> Self { - Self { - started: Instant::now(), - announced: AtomicU64::new(0), - changed: Mutex::new(None), - upstream: Mutex::new(None), - heads: Mutex::new(BTreeMap::new()), - } - } - - /// Records where a frame said a repository now is. - /// - /// A frame naming no commit is ignored rather than stored as an empty - /// one: some servers do not name theirs, and "this index has never been - /// told" is what an absent entry already says. - pub async fn head(&self, did: &str, commit: &str, rev: &str, seq: u64) { - if commit.is_empty() { - return; - } - self.heads.lock().await.insert( - did.to_owned(), - Head { - commit: commit.to_owned(), - rev: rev.to_owned(), - seq, - }, - ); - } - - /// Records what this process follows, and whether it is reaching it. - /// - /// Called on every reconnection rather than once at startup, because - /// "which index am I mirroring" is static and "am I reaching it" is the - /// half worth reporting. - pub async fn following(&self, upstream: impl Into, connected: bool) { - *self.upstream.lock().await = Some((upstream.into(), connected)); - } - - /// Notes that something changed the view. - async fn beat(&self) { - self.announced.fetch_add(1, Ordering::Relaxed); - *self.changed.lock().await = Some(Instant::now()); - } - - /// The numbers, as `/health` reports them. - async fn report(&self) -> serde_json::Value { - let (upstream, connected) = match self.upstream.lock().await.clone() { - Some((url, connected)) => (Some(url), Some(connected)), - None => (None, None), - }; - serde_json::json!({ - "uptime": self.started.elapsed().as_secs(), - "announced": self.announced.load(Ordering::Relaxed), - // Absent rather than zero when nothing has happened yet: a fresh - // process and a stalled one both report no change, and zero - // seconds ago would say the opposite of what is true. - "quietFor": self - .changed - .lock() - .await - .map(|at| at.elapsed().as_secs()), - "upstream": upstream, - "connected": connected, - // Keyed by DID so an operator can compare one line of this with - // one call to `getLatestCommit`. - "heads": self.heads.lock().await.clone(), - }) - } -} - -impl Default for Pulse { - fn default() -> Self { - Self::new() - } -} - -impl Shared { - /// Builds the shared state and the sender the updater keeps. - /// - /// The view does not cluster: this is what a mirror uses, and a mirror - /// takes its groups from upstream. - pub fn new() -> Self { - Self::over(View::new()) - } - - /// The same, with a view that works out its own work groups. - pub fn clustering() -> Self { - Self::over(View::clustering()) - } - - /// Shares an already-built view. - fn over(view: View) -> Self { - let (events, _) = broadcast::channel(EVENT_CAPACITY); - Self { - view: Arc::new(Mutex::new(view)), - events, - pulse: Arc::new(Pulse::new()), - } - } - - /// Applies an event to the view and announces it if it changed anything. - /// - /// The two halves are together on purpose: an event that did not change - /// the view must not be announced, and something that changed the view - /// without announcing it leaves every connected client wrong until it - /// reconnects. - /// - /// Takes anything iterable so that both shapes the view produces fit: an - /// `Option` for the observations that change one thing, and a `Vec` for - /// the ones that cascade — a record that teaches this index an agent's - /// parent, or a deletion that hands a subtree up. - pub async fn announce(&self, events: impl IntoIterator) { - for event in events { - self.pulse.beat().await; - // `send` fails only when nobody is subscribed, which is the normal - // state of an index with no query service attached yet. - match self.events.send(event) { - Ok(subscribers) => debug!(subscribers, "announced"), - Err(_) => debug!("nobody listening"), - } - } - } -} - -impl Default for Shared { - fn default() -> Self { - Self::new() - } -} - -/// Builds the router. -pub fn app(shared: Shared) -> Router { - Router::new() - .route("/health", get(health)) - .route("/view", get(view)) - .route("/work", get(work)) - .route("/events", get(events)) - .with_state(shared) - // Wide open, because in development the browser may well talk to this - // directly, and there is nothing here that is not already public: it - // is an index of records other people published. - .layer(CorsLayer::new().allow_origin(Any).allow_methods(Any)) -} - -/// `GET /health` -/// -/// Two kinds of number, and the second kind is the one worth having. What is -/// held says how big the view is; [`Pulse`] says whether anything is still -/// arriving, which is the difference between a working index and one that -/// lost its upstream an hour ago and has been serving the same answer since. -async fn health(State(shared): State) -> impl IntoResponse { - let mut body = shared.pulse.report().await; - let view = shared.view.lock().await; - let held = serde_json::json!({ - "status": "ok", - "operators": view.operators(), - "agents": view.agents(), - "scrobbles": view.scrobbles(), - "groups": view.groups().len(), - "clustering": view.clusters(), - }); - // Merged rather than nested, so the shape `/health` already had is - // unchanged and anything reading it keeps working. - if let (Some(body), Some(held)) = (body.as_object_mut(), held.as_object()) { - for (key, value) in held { - body.insert(key.clone(), value.clone()); - } - } - Json(body) -} - -/// `GET /view` -async fn view(State(shared): State) -> impl IntoResponse { - Json(shared.view.lock().await.snapshot()) -} - -/// `GET /work` -/// -/// The work groups on their own, for a caller that wants the grouping without -/// the population it was computed over. The same list `/view` carries, so a -/// client following the stream has no reason to call it; it is here because -/// "which agents are on the same thing" is the question this index answers -/// that nothing else can, and it should be one request to ask it. -async fn work(State(shared): State) -> impl IntoResponse { - let view = shared.view.lock().await; - let groups: Vec = view.groups().to_vec(); - Json(serde_json::json!({ "groups": groups })) -} - -/// `GET /events` -async fn events(State(shared): State) -> impl IntoResponse { - Sse::new(Feed::subscribe(&shared.events)).keep_alive(KeepAlive::default()) -} - -/// The per-client stream handed to axum's SSE responder. -/// -/// A broadcast receiver cannot be polled directly, so a forwarding task moves -/// events onto an mpsc channel, which can. The task ends when the client -/// disconnects and the receiver is dropped. -pub struct Feed { - rx: mpsc::Receiver, -} - -impl Feed { - /// Subscribes and starts forwarding into a fresh client stream. - pub fn subscribe(tx: &broadcast::Sender) -> Self { - let mut source = tx.subscribe(); - let (out, rx) = mpsc::channel(CLIENT_BUFFER); - tokio::spawn(async move { - loop { - match source.recv().await { - Ok(event) => { - if out.send(event).await.is_err() { - debug!("subscriber disconnected"); - break; - } - } - Err(broadcast::error::RecvError::Lagged(skipped)) => { - // The client's view is now wrong and it has no way to - // know, so say so loudly: the fix is a reconnect, - // which fetches the whole view again. - warn!(skipped, "subscriber fell behind; its view has a hole in it"); - } - Err(broadcast::error::RecvError::Closed) => break, - } - } - }); - Self { rx } - } -} - -impl Stream for Feed { - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.get_mut().rx.poll_recv(cx).map(|event| { - event.map(|event| { - // Named after its kind, so a browser can use - // `addEventListener("scrobble", ..)` instead of parsing every - // frame to find out what it is. - let name = match &event { - Event::Operator(_) => "operator", - Event::Provisioned(_) => "provisioned", - Event::Scrobble(_) => "scrobble", - Event::Pinned { .. } => "pinned", - Event::Deleted { .. } => "deleted", - Event::Work { .. } => "work", - }; - Ok(SseEvent::default() - .event(name) - .json_data(&event) - .unwrap_or_else(|_| SseEvent::default().event(name).data("{}"))) - }) - }) - } -} diff --git a/crates/didbot-index/src/survey.rs b/crates/didbot-index/src/survey.rs deleted file mode 100644 index 480fdac1..00000000 --- a/crates/didbot-index/src/survey.rs +++ /dev/null @@ -1,508 +0,0 @@ -//! Turning targets into the agent feeds this appview is willing to index. - -use didbot_identity::{hostname_is_at_or_below, validate_handle, AgentDid, HandleError}; -use serde::Deserialize; -use tracing::{debug, info, warn}; - -use crate::directory::{Authority, Target, VouchRef}; -use crate::http::HttpGet; -use crate::report::{fetch_json, Rejection}; -use crate::server::ServerRef; - -/// An agent feed worth reading, and everything that says so. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FeedSource { - did: AgentDid, - agent_id: String, - handle: Option, - pinned: bool, - created_at: String, - zone: String, - server: ServerRef, - vouch: Option, -} - -impl FeedSource { - /// The account's DID, already parsed and already checked for containment. - pub fn did(&self) -> &AgentDid { - &self.did - } - - /// The agent label the account was minted for, as the server reports it. - /// - /// The server's own word, not derived from the DID. They normally agree; - /// where they do not, [`AgentDid::agent_id`] is the one containment was - /// checked against and this is a display string. - pub fn agent_id(&self) -> &str { - &self.agent_id - } - - /// The name this account answers to, when the server issued it one that - /// this appview is willing to repeat. - /// - /// `None` covers three different things on purpose, because a reader does - /// the same thing with all of them: the server said nothing, the server - /// said the account's own hostname — which is the label the DID was minted - /// from and therefore not a name — or the server claimed a handle outside - /// its own zone, which is the one case worth being suspicious about and is - /// dropped rather than displayed. See [`readable_handle`]. - /// - /// Two more join that list: a handle that is not a handle, and one a - /// second account in the same survey also claims. Every one of them is - /// decided from strings already in hand and none of them fetches - /// anything, so completing atproto's bidirectional rule — resolving the - /// handle and reading the account's own DID document back — is - /// [`crate::handle::verify`] and is not on this path. - pub fn handle(&self) -> Option<&str> { - self.handle.as_deref() - } - - /// Whether the server says this account survives a sweep of unpinned ones. - pub fn pinned(&self) -> bool { - self.pinned - } - - /// When the server says the account was provisioned, RFC 3339. - pub fn created_at(&self) -> &str { - &self.created_at - } - - /// The zone the hosting server mints under. - pub fn zone(&self) -> &str { - &self.zone - } - - /// The server this feed is read from. - pub fn server(&self) -> &ServerRef { - &self.server - } - - /// The vouch this feed is indexed under, if a human wrote one. - /// - /// `None` for a server that was configured directly, which is the - /// development path and carries nobody's name. - pub fn vouch(&self) -> Option<&VouchRef> { - self.vouch.as_ref() - } - - /// The DID of the human who takes responsibility for this agent. - pub fn vouched_by(&self) -> Option<&str> { - self.vouch.as_ref().map(|vouch| vouch.voucher.as_str()) - } -} - -/// The result of surveying every target. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct Survey { - sources: Vec, - rejections: Vec, -} - -impl Survey { - /// The feeds that may be indexed. - pub fn sources(&self) -> &[FeedSource] { - &self.sources - } - - /// Everything that was declined, in the order it was declined. - pub fn rejections(&self) -> &[Rejection] { - &self.rejections - } - - /// Splits the survey into what it found and what it refused. - pub fn into_parts(self) -> (Vec, Vec) { - (self.sources, self.rejections) - } -} - -/// Reads didbot servers over `H`. -#[derive(Debug, Clone)] -pub struct Indexer { - http: H, -} - -impl Indexer { - /// Builds an indexer over a transport. - pub fn new(http: H) -> Self { - Self { http } - } - - /// The transport this indexer fetches with. - pub fn http(&self) -> &H { - &self.http - } - - /// Surveys every target, in order, and merges the results. - /// - /// Sequential rather than concurrent, and that is a choice worth stating: - /// duplicate accounts are resolved first-target-wins, so a concurrent - /// survey would need an ordering rule anyway, and a handful of servers - /// polled on an interval is not where the time goes. - pub async fn survey(&self, targets: &[Target]) -> Survey { - let mut survey = Survey::default(); - for target in targets { - let (sources, rejections) = self.survey_target(target).await.into_parts(); - survey.rejections.extend(rejections); - for mut source in sources { - if let Some(existing) = survey - .sources - .iter() - .find(|existing| existing.did() == source.did()) - { - survey.rejections.push(Rejection::DuplicateDid { - server: target.server.base_url().to_owned(), - did: source.did().as_str().to_owned(), - first: existing.server().base_url().to_owned(), - }); - continue; - } - // A handle resolves to one DID, so a second claim on a name is - // one the reverse direction of atproto's rule would refuse. - // Checked here rather than in `survey_target` because it is - // the only question about a handle that is about two accounts - // at once, and two servers can collide as easily as two - // accounts on one. - if let Some(handle) = source.handle.take() { - match survey.sources.iter().find(|existing| { - existing - .handle() - .is_some_and(|held| held.eq_ignore_ascii_case(&handle)) - }) { - Some(first) => { - let rejection = Rejection::DuplicateHandle { - server: target.server.base_url().to_owned(), - did: source.did().as_str().to_owned(), - handle, - first: first.did().as_str().to_owned(), - }; - warn!(%rejection, "handle claimed twice"); - survey.rejections.push(rejection); - } - None => source.handle = Some(handle), - } - } - survey.sources.push(source); - } - } - info!( - targets = targets.len(), - sources = survey.sources.len(), - rejected = survey.rejections.len(), - "survey finished" - ); - survey - } - - /// Reads one target and lists the feeds it may speak for. - pub async fn survey_target(&self, target: &Target) -> Survey { - let server = &target.server; - let zone = match self.probe_zone(server, &target.authority).await { - Ok(zone) => zone, - Err(rejection) => { - warn!(server = server.base_url(), %rejection, "target not surveyed"); - return refused(rejection); - } - }; - - let url = server.list_agents_url(); - let agents: AgentsView = match fetch_json(&self.http, server.base_url(), &url).await { - Ok(agents) => agents, - Err(rejection) => { - warn!(server = server.base_url(), %rejection, "agents not listed"); - return refused(rejection); - } - }; - - let mut survey = Survey::default(); - for agent in agents.agents { - match self.admit(target, &zone, agent, &mut survey.rejections) { - Ok(Some(source)) => { - debug!( - server = server.base_url(), - did = source.did().as_str(), - handle = source.handle().unwrap_or("-"), - "feed found" - ); - survey.sources.push(source); - } - // Hosted, inside the server's own zone, and simply not one of - // the names this vouch covers. Ordinary for a vouch naming a - // single account on a busy server, so it is not a refusal. - Ok(None) => {} - Err(rejection) => { - warn!(server = server.base_url(), %rejection, "account not indexed"); - survey.rejections.push(rejection); - } - } - } - - // A vouch that named one account, pointed at a server that does not - // host it, is a broken chain and not an empty one. - if survey.sources.is_empty() && names_one_account(&target.authority, &zone) { - survey.rejections.push(Rejection::AccountNotHosted { - server: server.base_url().to_owned(), - did: target.authority.to_string(), - }); - } - survey - } - - /// Reads the zone a server mints under, and checks it against the bound. - /// - /// The two have to be comparable: either the server mints under a name the - /// vouch covers, or the vouch names one account inside the server's zone. - /// Anything else means the vouch and the server have nothing to do with - /// each other, and nothing the server lists can be checked at all. - async fn probe_zone( - &self, - server: &ServerRef, - authority: &Authority, - ) -> Result { - let url = server.health_url(); - let health: HealthView = fetch_json(&self.http, server.base_url(), &url).await?; - let zone = - health - .zone - .filter(|zone| !zone.is_empty()) - .ok_or_else(|| Rejection::NotDidbot { - server: server.base_url().to_owned(), - })?; - - if let Authority::Host(host) = authority { - if !hostname_is_at_or_below(&zone, host) && !hostname_is_at_or_below(host, &zone) { - return Err(Rejection::ZoneOutsideAuthority { - server: server.base_url().to_owned(), - zone, - authority: host.clone(), - }); - } - } - Ok(zone) - } - - /// Decides whether one listed account may be indexed under `target`. - /// - /// `Ok(None)` means the account is legitimately the server's but is not a - /// name this target covers. - fn admit( - &self, - target: &Target, - zone: &str, - agent: AgentView, - notes: &mut Vec, - ) -> Result, Rejection> { - let server = &target.server; - let did = AgentDid::parse(&agent.did).map_err(|err| Rejection::UnusableDid { - server: server.base_url().to_owned(), - did: agent.did.clone(), - reason: err.to_string(), - })?; - - // Two independent bounds, and both have to hold. The zone is what the - // server may mint at all; the authority is what the vouch covers. A - // DID outside the zone is the server overreaching, and is reported. - if !hostname_is_at_or_below(did.host(), zone) { - return Err(Rejection::DidEscapesZone { - server: server.base_url().to_owned(), - did: agent.did, - zone: zone.to_owned(), - }); - } - if !target.authority.covers(&did) { - debug!(did = agent.did, authority = %target.authority, "not covered by this vouch"); - return Ok(None); - } - - let agent_id = if agent.agent_id.is_empty() { - did.agent_id().to_owned() - } else { - agent.agent_id - }; - - // A refused handle is reported and the account is kept: the DID - // passed containment and is what everything downstream is decided on, - // so the name is the only thing there is to drop. - let handle = match believable_handle(agent.handle.as_deref(), did.host(), zone) { - Ok(handle) => handle, - Err(refusal) => { - let rejection = refusal.into_rejection(server, did.as_str()); - warn!(server = server.base_url(), %rejection, "handle not believed"); - notes.push(rejection); - None - } - }; - - Ok(Some(FeedSource { - did, - agent_id, - handle, - pinned: agent.pinned, - created_at: agent.created_at, - zone: zone.to_owned(), - server: server.clone(), - vouch: target.vouch.clone(), - })) - } -} - -/// The handle worth showing for an account, out of what its server claimed. -/// -/// A handle is the server's own word about one of its accounts, and this -/// appview reads servers it was pointed at rather than servers it trusts. Two -/// filters, for two different reasons: -/// -/// * A handle that is not at or below the zone the server admits to minting is -/// dropped. Every containment decision in this crate is anchored on that -/// rule, and a name is the one field a person reads instead of the -/// DID — a server that could get `stripe.com` drawn under someone else's -/// agent would be handed the whole point of the display. -/// * A handle that is the account's own hostname is dropped, because it is not -/// a name. It is the label the DID was minted from, which a reader already -/// has, and repeating it as a handle would make an unnamed deployment look -/// like a named one. -/// * A handle that is not a syntactically valid handle is dropped, because -/// nothing could resolve it: the forward direction of atproto's rule is a -/// request to the handle's own hostname, and a string that is not a hostname -/// has none. -/// -/// Uniqueness is the one rule that is not here, because it is the one that is -/// about two accounts rather than one; [`Indexer::survey`] applies it. -pub fn readable_handle(handle: Option<&str>, host: &str, zone: &str) -> Option { - match believable_handle(handle, host, zone) { - Ok(handle) => handle, - Err(refusal) => { - warn!(%refusal, zone, "handle not believed; not shown"); - None - } - } -} - -/// Why a listed handle is not one this appview will repeat. -/// -/// Separate from [`Rejection`] so the rule can be stated without a server URL -/// and a DID to hang it on; `into_rejection` adds those where they are known. -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -enum HandleRefusal { - /// The string is not a handle. - #[error("{handle:?} is not a handle: {reason}")] - NotAHandle { - /// The string that was listed. - handle: String, - /// Why it is not one. - reason: HandleError, - }, - /// The handle is outside the zone the server admits to minting under. - #[error("{handle:?} is not at or below the zone {zone:?}")] - EscapesZone { - /// The handle that was listed. - handle: String, - /// The zone it had to sit under. - zone: String, - }, -} - -impl HandleRefusal { - /// Names the server and the account this refusal was about. - fn into_rejection(self, server: &ServerRef, did: &str) -> Rejection { - let server = server.base_url().to_owned(); - let did = did.to_owned(); - match self { - Self::NotAHandle { handle, reason } => Rejection::UnusableHandle { - server, - did, - handle, - reason: reason.to_string(), - }, - Self::EscapesZone { handle, zone } => Rejection::HandleEscapesZone { - server, - did, - handle, - zone, - }, - } - } -} - -/// [`readable_handle`], with the refusal kept rather than logged. -/// -/// `Ok(None)` is "the server named no handle, or named the account's own -/// hostname", which is an absence and not a claim. `Err` is a claim that did -/// not stand up, and the caller that has a DID to attach it to reports it. -fn believable_handle( - handle: Option<&str>, - host: &str, - zone: &str, -) -> Result, HandleRefusal> { - let handle = handle.map(str::trim).unwrap_or_default(); - if handle.is_empty() || handle.eq_ignore_ascii_case(host) { - return Ok(None); - } - if let Err(reason) = validate_handle(handle) { - return Err(HandleRefusal::NotAHandle { - handle: handle.to_owned(), - reason, - }); - } - if !hostname_is_at_or_below(handle, zone) { - return Err(HandleRefusal::EscapesZone { - handle: handle.to_owned(), - zone: zone.to_owned(), - }); - } - Ok(Some(handle.to_owned())) -} - -/// A survey holding one refusal and nothing else. -fn refused(rejection: Rejection) -> Survey { - Survey { - sources: Vec::new(), - rejections: vec![rejection], - } -} - -/// Whether this authority names a single account rather than a whole zone. -fn names_one_account(authority: &Authority, zone: &str) -> bool { - match authority { - // Strictly below the zone: a name the server could have minted, not - // the zone itself. - Authority::Host(host) => { - hostname_is_at_or_below(host, zone) && !host.eq_ignore_ascii_case(zone) - } - Authority::Did(_) => true, - } -} - -/// The part of `/health` this appview reads. -/// -/// `status` is deliberately absent: a server that answers 200 with a zone is -/// answering, and a second field saying so adds a way for a healthy server to -/// be skipped over a spelling. -#[derive(Debug, Deserialize)] -struct HealthView { - zone: Option, -} - -/// The `bot.did.listAgents` response. -#[derive(Debug, Deserialize)] -struct AgentsView { - #[serde(default)] - agents: Vec, -} - -/// One entry in that response. -/// -/// Every field but the DID defaults: a server that grows a field, or omits one -/// this appview only displays, should not cost an entire listing. -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct AgentView { - did: String, - #[serde(default)] - agent_id: String, - #[serde(default)] - handle: Option, - #[serde(default)] - pinned: bool, - #[serde(default)] - created_at: String, -} diff --git a/crates/didbot-index/src/tests.rs b/crates/didbot-index/src/tests.rs deleted file mode 100644 index 31051044..00000000 --- a/crates/didbot-index/src/tests.rs +++ /dev/null @@ -1,931 +0,0 @@ -//! Tests for the discovery rules, driven from fixed server answers. -//! -//! Two things are worth testing here and neither needs a network. The first is -//! the vouch chain: a human's DID, their repository, the records in it, and -//! the servers those name. The second is the set of claims a cooperating -//! server never makes — a zone unrelated to the vouched name, an account -//! belonging to somebody else, the same account offered twice — which are the -//! cases the containment rules exist for. - -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; - -use crate::nsid; -use didbot_identity::{AgentDid, DidDocument, Zone}; -use serde_json::{json, Value}; -use time::macros::datetime; -use time::OffsetDateTime; - -use crate::directory::{ - Authority, ServerDirectory, StaticDirectory, Target, VouchDirectory, VOUCH_PAGE_LIMIT, -}; -use crate::feed::DEFAULT_FEED_LIMIT; -use crate::http::{HttpGet, HttpResponse, TransportError}; -use crate::report::Rejection; -use crate::resolve::DidResolver; -use crate::server::ServerRef; -use crate::survey::{Indexer, Survey}; - -/// A transport that answers from a table and remembers what it was asked. -#[derive(Debug, Default)] -struct StubHttp { - answers: HashMap>, - asked: Mutex>, -} - -impl StubHttp { - fn new() -> Self { - Self::default() - } - - fn json(mut self, url: &str, body: Value) -> Self { - self.answers - .insert(url.to_owned(), Ok(HttpResponse::new(200, body.to_string()))); - self - } - - fn raw(mut self, url: &str, status: u16, body: &str) -> Self { - self.answers - .insert(url.to_owned(), Ok(HttpResponse::new(status, body))); - self - } - - fn down(mut self, url: &str, reason: &str) -> Self { - self.answers.insert(url.to_owned(), Err(reason.to_owned())); - self - } - - fn asked(&self) -> Vec { - self.asked.lock().expect("stub lock").clone() - } -} - -impl HttpGet for StubHttp { - async fn get(&self, url: &str) -> Result { - self.asked.lock().expect("stub lock").push(url.to_owned()); - match self.answers.get(url) { - Some(Ok(response)) => Ok(response.clone()), - Some(Err(reason)) => Err(TransportError::new(url, reason.clone())), - // An unstubbed URL is a 404 rather than a panic: "the server does - // not have this route" is a real answer a real server gives. - None => Ok(HttpResponse::new(404, "{\"error\":\"NotFound\"}")), - } - } -} - -// --------------------------------------------------------------------------- -// The deployment every test here talks about -// --------------------------------------------------------------------------- - -const DEV: &str = "http://localhost:3000"; -const ZONE: &str = "agents.localhost"; -const SERVICE_DID: &str = "did:web:agents.localhost%3A3000"; -const KESTREL: &str = "did:web:kestrel.agents.localhost%3A3000"; -const MARMOT: &str = "did:web:marmot.agents.localhost%3A3000"; - -/// The human doing the vouching, and where their own repository lives. -const VOUCHER: &str = "did:plc:nlzmjyfv6loqtxyzvdcznwgf"; -const VOUCHER_PDS: &str = "https://pds.example"; -const PLC: &str = "https://plc.directory"; - -/// A clock late enough that a vouch expiring in 2026-08-10 has lapsed. -fn now() -> OffsetDateTime { - datetime!(2026-08-24 00:00 UTC) -} - -fn server() -> ServerRef { - ServerRef::parse(DEV).expect("valid base url") -} - -fn voucher_pds() -> ServerRef { - ServerRef::parse(VOUCHER_PDS).expect("valid base url") -} - -/// A target with no vouch behind it: the development path. -fn configured() -> Vec { - vec![Target::configured(server())] -} - -fn agent(did: &str, agent_id: &str) -> Value { - json!({ - "did": did, - "agentId": agent_id, - "pinned": false, - "createdAt": "2026-08-24T10:00:00Z", - }) -} - -/// A server that answers `/health` with `zone` and lists `agents`. -fn stub_with(zone: &str, agents: Value) -> StubHttp { - let server = server(); - StubHttp::new() - .json( - &server.health_url(), - json!({ "status": "ok", "zone": zone }), - ) - .json(&server.list_agents_url(), json!({ "agents": agents })) -} - -// --------------------------------------------------------------------------- -// Containment, against a server nobody vouched for -// --------------------------------------------------------------------------- - -#[tokio::test] -async fn indexes_an_account_the_server_may_speak_for() { - let http = stub_with(ZONE, json!([agent(KESTREL, "kestrel")])); - let survey = Indexer::new(http).survey(&configured()).await; - - assert!(survey.rejections().is_empty(), "{:?}", survey.rejections()); - let source = &survey.sources()[0]; - assert_eq!(source.did().as_str(), KESTREL); - assert_eq!(source.agent_id(), "kestrel"); - assert_eq!(source.zone(), ZONE); - assert_eq!(source.server().base_url(), DEV); - assert_eq!(source.vouched_by(), None); -} - -/// A listing entry that also claims a handle. -fn named(did: &str, agent_id: &str, handle: &str) -> Value { - let mut entry = agent(did, agent_id); - entry["handle"] = json!(handle); - entry -} - -#[tokio::test] -async fn keeps_the_name_a_server_issued_under_its_own_zone() { - let http = stub_with( - ZONE, - json!([named(KESTREL, "kestrel", "basalt-otter.agents.localhost")]), - ); - let survey = Indexer::new(http).survey(&configured()).await; - - assert_eq!( - survey.sources()[0].handle(), - Some("basalt-otter.agents.localhost") - ); -} - -#[tokio::test] -async fn drops_a_name_that_escapes_the_servers_zone() { - // The one field a person reads instead of the DID, so a server that could - // get somebody else's domain drawn under its own agent would be handed the - // whole point of the display. - let http = stub_with( - ZONE, - json!([named(KESTREL, "kestrel", "basalt-otter.example.com")]), - ); - let survey = Indexer::new(http).survey(&configured()).await; - - assert_eq!(survey.sources()[0].handle(), None); - // Reported rather than only logged: a name the server had no authority to - // claim is the one handle failure worth telling an operator about. - assert!( - matches!( - &survey.rejections()[0], - Rejection::HandleEscapesZone { handle, .. } if handle == "basalt-otter.example.com" - ), - "{:?}", - survey.rejections() - ); -} - -#[tokio::test] -async fn an_accounts_own_hostname_is_not_a_name() { - // What a deployment that does not name its agents reports. Repeating it - // would make an unnamed population look like a named one. - let http = stub_with( - ZONE, - json!([named(KESTREL, "kestrel", "kestrel.agents.localhost:3000")]), - ); - let survey = Indexer::new(http).survey(&configured()).await; - - assert_eq!(survey.sources()[0].handle(), None); -} - -#[tokio::test] -async fn refuses_an_account_below_somebody_elses_zone() { - let http = stub_with( - ZONE, - json!([ - agent(KESTREL, "kestrel"), - agent("did:web:someagent.trusted.example", "someagent"), - ]), - ); - let survey = Indexer::new(http).survey(&configured()).await; - - assert_eq!(survey.sources().len(), 1); - assert!(matches!( - &survey.rejections()[0], - Rejection::DidEscapesZone { did, zone, .. } - if did == "did:web:someagent.trusted.example" && zone == ZONE - )); -} - -#[tokio::test] -async fn a_suffix_is_not_containment() { - // `evilagents.localhost` shares a suffix with `agents.localhost` and - // shares no label boundary with it. A string suffix test would index it. - let http = stub_with( - ZONE, - json!([agent("did:web:kestrel.evilagents.localhost", "kestrel")]), - ); - let survey = Indexer::new(http).survey(&configured()).await; - - assert!(survey.sources().is_empty()); - assert!(matches!( - &survey.rejections()[0], - Rejection::DidEscapesZone { .. } - )); -} - -#[tokio::test] -async fn refuses_a_server_whose_zone_is_unrelated_to_the_bound() { - // Reached at `localhost`, claiming to mint under `trusted.example`. The - // whole target is dropped: nothing it lists can be checked against a name - // it does not hold. - let http = stub_with("trusted.example", json!([agent(KESTREL, "kestrel")])); - let survey = Indexer::new(http).survey(&configured()).await; - - assert!(survey.sources().is_empty()); - assert_eq!(survey.rejections().len(), 1); - assert!(matches!( - &survey.rejections()[0], - Rejection::ZoneOutsideAuthority { zone, authority, .. } - if zone == "trusted.example" && authority == "localhost" - )); -} - -#[tokio::test] -async fn does_not_list_agents_from_a_target_it_rejected() { - let http = stub_with("trusted.example", json!([agent(KESTREL, "kestrel")])); - let indexer = Indexer::new(http); - let _ = indexer.survey(&configured()).await; - - assert_eq!(indexer.http().asked(), [server().health_url()]); -} - -#[tokio::test] -async fn reports_a_server_that_is_not_one_of_ours() { - let http = StubHttp::new().json(&server().health_url(), json!({ "status": "ok" })); - let survey = Indexer::new(http).survey(&configured()).await; - - assert!(matches!( - &survey.rejections()[0], - Rejection::NotDidbot { .. } - )); -} - -/// Silence, refusal and nonsense are three different answers, and an -/// operator needs to be told which one they got. -#[tokio::test] -async fn reports_a_server_that_never_answered() { - let http = StubHttp::new().down(&server().health_url(), "connection refused"); - let survey = Indexer::new(http).survey(&configured()).await; - - assert!(survey.sources().is_empty()); - assert!(matches!( - &survey.rejections()[0], - Rejection::Unreachable { .. } - )); -} - -#[tokio::test] -async fn reports_a_server_that_answered_with_a_failure() { - let http = StubHttp::new().raw(&server().health_url(), 503, "unavailable"); - let survey = Indexer::new(http).survey(&configured()).await; - - assert!(matches!( - &survey.rejections()[0], - Rejection::BadStatus { status: 503, .. } - )); -} - -#[tokio::test] -async fn reports_a_server_that_answered_with_something_else_entirely() { - let http = StubHttp::new().raw(&server().health_url(), 200, "hello"); - let survey = Indexer::new(http).survey(&configured()).await; - - assert!(matches!( - &survey.rejections()[0], - Rejection::Malformed { .. } - )); -} - -#[tokio::test] -async fn reports_an_unparseable_did_without_dropping_the_listing() { - let http = stub_with( - ZONE, - json!([ - agent("did:plc:notourbusiness", "elsewhere"), - agent(KESTREL, "kestrel"), - ]), - ); - let survey = Indexer::new(http).survey(&configured()).await; - - assert_eq!(survey.sources().len(), 1); - assert!(matches!( - &survey.rejections()[0], - Rejection::UnusableDid { .. } - )); -} - -#[tokio::test] -async fn the_first_target_to_offer_an_account_keeps_it() { - let second = ServerRef::parse("http://localhost:3100").expect("valid base url"); - let http = stub_with(ZONE, json!([agent(KESTREL, "kestrel")])) - .json( - &second.health_url(), - json!({ "status": "ok", "zone": ZONE }), - ) - .json( - &second.list_agents_url(), - json!({ "agents": [agent(KESTREL, "kestrel")] }), - ); - - let targets = vec![Target::configured(server()), Target::configured(second)]; - let survey = Indexer::new(http).survey(&targets).await; - - assert_eq!(survey.sources().len(), 1); - assert_eq!(survey.sources()[0].server().base_url(), DEV); - assert!(matches!( - &survey.rejections()[0], - Rejection::DuplicateDid { server, first, .. } - if server == "http://localhost:3100" && first == DEV - )); -} - -#[tokio::test] -async fn falls_back_to_the_did_when_a_listing_omits_the_agent_id() { - let http = stub_with(ZONE, json!([{ "did": KESTREL }])); - let survey = Indexer::new(http).survey(&configured()).await; - - assert_eq!(survey.sources()[0].agent_id(), "kestrel"); -} - -#[tokio::test] -async fn a_configured_directory_feeds_the_survey() { - let directory = StaticDirectory::parse(DEV).expect("valid list"); - let targets = directory.targets().await; - let http = stub_with(ZONE, json!([agent(KESTREL, "kestrel")])); - - assert_eq!( - targets.targets()[0].authority, - Authority::Host("localhost".to_owned()) - ); - assert_eq!( - Indexer::new(http) - .survey(targets.targets()) - .await - .sources() - .len(), - 1 - ); -} - -// --------------------------------------------------------------------------- -// The vouch chain -// --------------------------------------------------------------------------- - -fn vouch_uri(rkey: &str) -> String { - format!("at://{VOUCHER}/{}/{rkey}", nsid::VOUCH) -} - -/// One record as the voucher's personal data server would serve it. -fn vouch_record(rkey: &str, value: Value) -> Value { - json!({ "uri": vouch_uri(rkey), "value": value }) -} - -/// A vouch for `subject` with no expiry. -fn plain_vouch(subject: &str) -> Value { - json!({ - "$type": nsid::VOUCH, - "subject": subject, - "createdAt": "2026-08-01T00:00:00Z", - }) -} - -/// The service document the didbot server serves on its zone host. -fn service_document() -> Value { - let zone = Zone::new(ZONE) - .expect("zone host is valid") - .with_port(3000) - .expect("localhost may carry a port"); - let document = DidDocument::for_service(&zone.service_did(), DEV); - serde_json::to_value(document).expect("a document serializes") -} - -/// A DID document for one agent account, pointing at the same server. -fn account_document(did: &str) -> Value { - let parsed = AgentDid::parse(did).expect("a valid agent did"); - let document = DidDocument::for_account(&parsed, "zPlaceholderMultibase", parsed.host(), DEV); - serde_json::to_value(document).expect("a document serializes") -} - -/// Where a `did:web` document is fetched from. -fn did_web_url(did: &str) -> String { - AgentDid::parse(did) - .expect("a valid did:web") - .did_json_url() -} - -/// The whole chain, stubbed: the voucher's DID, their repository, the vouch -/// records in it, the subject's DID, and the didbot server itself. -fn chain(records: Vec, agents: Value) -> Arc { - Arc::new(chain_stub(records, agents)) -} - -/// The same chain, before it is shared, so a test can overwrite one answer. -fn chain_stub(records: Vec, agents: Value) -> StubHttp { - let voucher_document = json!({ - "@context": ["https://www.w3.org/ns/did/v1"], - "id": VOUCHER, - "alsoKnownAs": ["at://quernstone.example"], - "service": [{ - "id": "#atproto_pds", - "type": "AtprotoPersonalDataServer", - "serviceEndpoint": VOUCHER_PDS, - }], - }); - - stub_with(ZONE, agents) - .json(&format!("{PLC}/{VOUCHER}"), voucher_document) - .json( - &voucher_pds().list_records_url(VOUCHER, nsid::VOUCH, VOUCH_PAGE_LIMIT), - json!({ "records": records }), - ) - .json(&did_web_url(SERVICE_DID), service_document()) - .json(&did_web_url(KESTREL), account_document(KESTREL)) -} - -fn directory(http: Arc) -> VouchDirectory> { - VouchDirectory::new(DidResolver::new(http), [VOUCHER.to_owned()]).with_clock(now()) -} - -/// Walks the chain end to end, as a deployment does: discover, then survey. -async fn walk(http: Arc) -> (Survey, Vec) { - let (targets, rejections) = directory(http.clone()).targets().await.into_parts(); - (Indexer::new(http).survey(&targets).await, rejections) -} - -#[tokio::test] -async fn a_vouch_for_a_server_covers_every_agent_on_it() { - let http = chain( - vec![vouch_record("a", plain_vouch(SERVICE_DID))], - json!([agent(KESTREL, "kestrel"), agent(MARMOT, "marmot")]), - ); - let (targets, rejections) = directory(http).targets().await.into_parts(); - - assert!(rejections.is_empty(), "{rejections:?}"); - assert_eq!(targets.len(), 1); - assert_eq!(targets[0].server.base_url(), DEV); - assert_eq!(targets[0].authority, Authority::Host(ZONE.to_owned())); - assert_eq!(targets[0].vouched_by(), Some(VOUCHER)); -} - -#[tokio::test] -async fn a_vouch_for_one_account_covers_only_that_account() { - let http = chain( - vec![vouch_record("a", plain_vouch(KESTREL))], - json!([agent(KESTREL, "kestrel"), agent(MARMOT, "marmot")]), - ); - let (targets, _) = directory(http).targets().await.into_parts(); - - assert_eq!( - targets[0].authority, - Authority::Host("kestrel.agents.localhost".to_owned()) - ); -} - -#[tokio::test] -async fn an_expired_vouch_is_not_followed() { - let http = chain( - vec![vouch_record( - "a", - json!({ - "$type": nsid::VOUCH, - "subject": SERVICE_DID, - "createdAt": "2026-08-01T00:00:00Z", - "expiresAt": "2026-08-10T00:00:00Z", - }), - )], - json!([agent(KESTREL, "kestrel")]), - ); - let (targets, rejections) = directory(http).targets().await.into_parts(); - - assert!(targets.is_empty()); - assert!(matches!( - &rejections[0], - Rejection::ExpiredVouch { subject, .. } if subject == SERVICE_DID - )); -} - -#[tokio::test] -async fn a_revoked_vouch_is_not_followed() { - let http = chain( - vec![ - vouch_record("older", plain_vouch(SERVICE_DID)), - vouch_record( - "newer", - json!({ - "$type": nsid::VOUCH, - "subject": MARMOT, - "createdAt": "2026-08-02T00:00:00Z", - "revokes": vouch_uri("older"), - }), - ), - ], - json!([agent(KESTREL, "kestrel")]), - ); - let (targets, rejections) = directory(http).targets().await.into_parts(); - - assert!(matches!( - &rejections[0], - Rejection::RevokedVouch { subject, by, .. } - if subject == SERVICE_DID && by == &vouch_uri("newer") - )); - // The revoking vouch is itself a vouch, and its own subject would not - // resolve here, so the chain reports that rather than silently stopping. - assert!(targets.is_empty()); - assert!(matches!(&rejections[1], Rejection::Unresolvable { did, .. } if did == MARMOT)); -} - -#[tokio::test] -async fn a_record_that_is_not_a_vouch_is_reported_and_skipped() { - let http = chain( - vec![ - vouch_record("bad", json!({ "$type": nsid::SCROBBLE, "text": "hi" })), - vouch_record("good", plain_vouch(SERVICE_DID)), - ], - json!([agent(KESTREL, "kestrel")]), - ); - let (targets, rejections) = directory(http).targets().await.into_parts(); - - assert_eq!(targets.len(), 1); - assert!(matches!( - &rejections[0], - Rejection::UnreadableVouch { uri, .. } if uri == &vouch_uri("bad") - )); -} - -#[tokio::test] -async fn a_voucher_that_will_not_resolve_is_reported() { - let http = Arc::new(StubHttp::new().raw(&format!("{PLC}/{VOUCHER}"), 404, "{}")); - let (targets, rejections) = directory(http).targets().await.into_parts(); - - assert!(targets.is_empty()); - assert!(matches!( - &rejections[0], - Rejection::Unresolvable { did, .. } if did == VOUCHER - )); -} - -#[tokio::test] -async fn a_document_claiming_a_different_did_is_refused() { - // did:web is served by whoever holds the name, so the one check that - // cannot be skipped is that the document admits to being what was asked - // for. - let mut imposter = service_document(); - imposter["id"] = json!("did:web:trusted.example"); - let http = Arc::new( - chain_stub(vec![vouch_record("a", plain_vouch(SERVICE_DID))], json!([])) - .json(&did_web_url(SERVICE_DID), imposter), - ); - - let (targets, rejections) = directory(http).targets().await.into_parts(); - - assert!(targets.is_empty()); - assert!(matches!( - &rejections[0], - Rejection::Unresolvable { did, .. } if did == SERVICE_DID - )); -} - -#[tokio::test] -async fn a_vouched_account_the_server_does_not_host_is_reported() { - let http = chain( - vec![vouch_record("a", plain_vouch(KESTREL))], - json!([agent(MARMOT, "marmot")]), - ); - let (survey, _) = walk(http).await; - - assert!(survey.sources().is_empty()); - assert!(matches!( - &survey.rejections()[0], - Rejection::AccountNotHosted { did, .. } if did == "kestrel.agents.localhost" - )); -} - -#[tokio::test] -async fn the_survey_carries_the_vouch_that_admitted_each_feed() { - let http = chain( - vec![vouch_record("a", plain_vouch(SERVICE_DID))], - json!([agent(KESTREL, "kestrel"), agent(MARMOT, "marmot")]), - ); - let (survey, _) = walk(http).await; - - assert_eq!(survey.sources().len(), 2); - for source in survey.sources() { - assert_eq!(source.vouched_by(), Some(VOUCHER)); - let vouch = source.vouch().expect("a vouched source carries its vouch"); - assert_eq!(vouch.subject, SERVICE_DID); - assert_eq!(vouch.uri, vouch_uri("a")); - } -} - -#[tokio::test] -async fn a_vouch_for_one_account_leaves_its_siblings_alone() { - let http = chain( - vec![vouch_record("a", plain_vouch(KESTREL))], - json!([agent(KESTREL, "kestrel"), agent(MARMOT, "marmot")]), - ); - let (survey, _) = walk(http).await; - - // Marmot is a perfectly good account that nobody vouched for. Not a - // refusal, just not covered. - assert_eq!(survey.sources().len(), 1); - assert_eq!(survey.sources()[0].did().as_str(), KESTREL); - assert!(survey.rejections().is_empty(), "{:?}", survey.rejections()); -} - -// --------------------------------------------------------------------------- -// Reading a feed -// --------------------------------------------------------------------------- - -/// A survey and a feed read against one stub, which is how the crate is used. -#[tokio::test] -async fn reads_a_feed_from_a_discovered_source() { - let server = server(); - let records = json!({ - "records": [ - { - "uri": format!("at://{KESTREL}/com.vibescrobble.scrobble/3mtufhkqapfi5"), - "value": { - "$type": "com.vibescrobble.scrobble", - "text": "quernstone placeholder work", - "emoji": "🧭", - "createdAt": "2026-08-24T18:00:00Z", - "effort": "high", - "artifacts": ["at://example/one", 7], - }, - }, - // No emoji: something else in the scrobble collection. - { - "uri": "at://x/com.vibescrobble.scrobble/2", - "value": { "text": "half a record", "createdAt": "2026-08-24T18:01:00Z" }, - }, - // A record answering a question nobody asked. - { - "uri": "at://x/com.vibescrobble.scrobble/3", - "value": { "$type": "bot.did.memory", "text": "m", "emoji": "x", - "createdAt": "2026-08-24T18:02:00Z" }, - }, - ] - }); - let feed_url = server.list_records_url(KESTREL, nsid::SCROBBLE, DEFAULT_FEED_LIMIT); - let http = stub_with(ZONE, json!([agent(KESTREL, "kestrel")])).json(&feed_url, records); - - let indexer = Indexer::new(http); - let survey = indexer.survey(&configured()).await; - let page = indexer.feed(&survey.sources()[0], DEFAULT_FEED_LIMIT).await; - - assert_eq!(page.scrobbles().len(), 1); - let scrobble = &page.scrobbles()[0]; - assert_eq!(scrobble.text, "quernstone placeholder work"); - assert_eq!(scrobble.emoji, "🧭"); - assert_eq!(scrobble.effort.as_deref(), Some("high")); - assert_eq!(scrobble.task, None); - assert_eq!(scrobble.artifacts, ["at://example/one"]); - assert_eq!(scrobble.did, KESTREL); - - assert_eq!(page.rejections().len(), 2); - assert!(page - .rejections() - .iter() - .all(|rejection| matches!(rejection, Rejection::UnreadableRecord { .. }))); -} - -#[tokio::test] -async fn a_feed_that_cannot_be_read_is_a_rejection_not_a_panic() { - let http = stub_with(ZONE, json!([agent(KESTREL, "kestrel")])); - let indexer = Indexer::new(http); - let survey = indexer.survey(&configured()).await; - let page = indexer.feed(&survey.sources()[0], 10).await; - - assert!(page.scrobbles().is_empty()); - assert!(matches!( - &page.rejections()[0], - Rejection::BadStatus { status: 404, .. } - )); -} - -// --------------------------------------------------------------------------- -// Handles: what a listed name has to survive before it is repeated -// --------------------------------------------------------------------------- - -#[tokio::test] -async fn a_handle_suffix_is_not_containment_either() { - let http = stub_with( - ZONE, - json!([named(KESTREL, "kestrel", "otter.evilagents.localhost")]), - ); - let survey = Indexer::new(http).survey(&configured()).await; - - assert_eq!(survey.sources()[0].handle(), None); - assert!(matches!( - &survey.rejections()[0], - Rejection::HandleEscapesZone { .. } - )); -} - -#[tokio::test] -async fn drops_a_handle_that_is_not_a_handle() { - let http = stub_with(ZONE, json!([named(KESTREL, "kestrel", "not a handle")])); - let survey = Indexer::new(http).survey(&configured()).await; - - assert_eq!(survey.sources()[0].handle(), None); - assert!(matches!( - &survey.rejections()[0], - Rejection::UnusableHandle { handle, .. } if handle == "not a handle" - )); -} - -#[tokio::test] -async fn two_accounts_may_not_be_listed_under_one_handle() { - // A handle resolves to one DID, so the second claim is one the reverse - // direction would refuse. The first keeps the name: dropping both would - // let a server erase a name by claiming it twice. - let handle = "basalt-otter.agents.localhost"; - let http = stub_with( - ZONE, - json!([ - named(KESTREL, "kestrel", handle), - named(MARMOT, "marmot", handle), - ]), - ); - let survey = Indexer::new(http).survey(&configured()).await; - - assert_eq!(survey.sources().len(), 2); - assert_eq!(survey.sources()[0].handle(), Some(handle)); - assert_eq!(survey.sources()[1].handle(), None); - assert!( - matches!( - &survey.rejections()[0], - Rejection::DuplicateHandle { did, first, .. } if did == MARMOT && first == KESTREL - ), - "{:?}", - survey.rejections() - ); -} - -#[tokio::test] -async fn a_handle_claimed_twice_is_matched_without_regard_to_case() { - let http = stub_with( - ZONE, - json!([ - named(KESTREL, "kestrel", "basalt-otter.agents.localhost"), - named(MARMOT, "marmot", "Basalt-Otter.Agents.Localhost"), - ]), - ); - let survey = Indexer::new(http).survey(&configured()).await; - - assert_eq!(survey.sources()[1].handle(), None); - assert!(matches!( - &survey.rejections()[0], - Rejection::DuplicateHandle { .. } - )); -} - -// --------------------------------------------------------------------------- -// The bidirectional check itself -// --------------------------------------------------------------------------- - -/// The handle used by the verification tests, and where it is resolved. -const OTTER: &str = "basalt-otter.agents.localhost"; - -fn otter_well_known() -> String { - format!("http://{OTTER}:3000/.well-known/atproto-did") -} - -/// A document for `KESTREL` claiming `handle`. -fn kestrel_doc(handle: &str) -> Value { - json!({ - "@context": ["https://www.w3.org/ns/did/v1"], - "id": KESTREL, - "alsoKnownAs": [format!("at://{handle}")], - "service": [{ - "id": format!("{KESTREL}#atproto_pds"), - "type": "AtprotoPersonalDataServer", - "serviceEndpoint": DEV, - }], - }) -} - -/// Where `KESTREL`'s document is fetched from, port and all. -fn kestrel_doc_url() -> String { - AgentDid::parse(KESTREL) - .expect("a valid did:web") - .did_json_url() -} - -#[tokio::test] -async fn a_handle_that_resolves_and_is_claimed_back_verifies() { - let http = StubHttp::new() - .raw(&otter_well_known(), 200, KESTREL) - .json(&kestrel_doc_url(), kestrel_doc(OTTER)); - let resolver = DidResolver::new(http); - - crate::handle::verify(&resolver, OTTER, KESTREL, Some(3000)) - .await - .expect("both directions agree"); -} - -#[tokio::test] -async fn a_handle_the_document_does_not_claim_back_is_refused() { - // The failure the whole check exists for. The well-known says the handle - // is this account's, and the account's own document says otherwise; a - // resolver that stopped after the first answer would file records under - // an account that never claimed the name. - let http = StubHttp::new().raw(&otter_well_known(), 200, KESTREL).json( - &kestrel_doc_url(), - kestrel_doc("mossy-vole.agents.localhost"), - ); - let resolver = DidResolver::new(http); - - let failure = crate::handle::verify(&resolver, OTTER, KESTREL, Some(3000)) - .await - .expect_err("the document claims a different name"); - assert!( - matches!( - failure, - crate::handle::HandleFailure::NotClaimedBack { ref claimed, .. } - if claimed.as_deref() == Some("mossy-vole.agents.localhost") - ), - "got {failure}" - ); -} - -#[tokio::test] -async fn a_handle_that_resolves_to_another_account_is_refused() { - let http = StubHttp::new() - .raw(&otter_well_known(), 200, MARMOT) - .json(&kestrel_doc_url(), kestrel_doc(OTTER)); - let resolver = DidResolver::new(http); - - let failure = crate::handle::verify(&resolver, OTTER, KESTREL, Some(3000)) - .await - .expect_err("the handle is somebody else's"); - assert!( - matches!(failure, crate::handle::HandleFailure::ResolvesElsewhere { ref found, .. } if found == MARMOT), - "got {failure}" - ); - // The document is not fetched for a claim that has already failed. - assert!(!resolver.http().asked().contains(&kestrel_doc_url())); -} - -#[tokio::test] -async fn a_handle_nobody_answers_for_is_refused() { - // A 404 from the well-known is the honest answer for a name nobody holds, - // and it has to reach the caller as a refusal rather than as an absence. - let resolver = DidResolver::new(StubHttp::new()); - - let failure = crate::handle::verify(&resolver, OTTER, KESTREL, Some(3000)) - .await - .expect_err("nothing answers for this handle"); - assert!( - matches!( - failure, - crate::handle::HandleFailure::BadStatus { status: 404, .. } - ), - "got {failure}" - ); -} - -#[tokio::test] -async fn a_well_known_body_that_is_not_a_did_is_refused() { - // A proxy that answers 200 with an error page, or a server that answers - // with JSON. Neither is a DID and neither may be looked up. - let http = StubHttp::new().raw(&otter_well_known(), 200, "nope"); - let resolver = DidResolver::new(http); - - let failure = crate::handle::verify(&resolver, OTTER, KESTREL, Some(3000)) - .await - .expect_err("an html page is not a did"); - assert!( - matches!(failure, crate::handle::HandleFailure::NotADid { .. }), - "got {failure}" - ); -} - -#[tokio::test] -async fn a_trailing_newline_on_the_well_known_is_tolerated() { - // The body is "the DID and nothing else", and a text file served off a - // web server acquires a trailing newline the moment somebody edits it. - let http = StubHttp::new() - .raw(&otter_well_known(), 200, &format!("{KESTREL}\n")) - .json(&kestrel_doc_url(), kestrel_doc(OTTER)); - let resolver = DidResolver::new(http); - - crate::handle::verify(&resolver, OTTER, KESTREL, Some(3000)) - .await - .expect("surrounding whitespace is not a disagreement"); -} diff --git a/crates/didbot-index/src/view.rs b/crates/didbot-index/src/view.rs deleted file mode 100644 index 45398979..00000000 --- a/crates/didbot-index/src/view.rs +++ /dev/null @@ -1,1486 +0,0 @@ -//! What the index holds, and what it hands on. -//! -//! One structure, updated from two sources that answer different questions. -//! The server's event stream says when an account appears or goes away, which -//! is a thing that happens at a moment. Reading a repository says what an -//! agent has been saying, which is a thing that accumulates. Neither can -//! answer the other's question, so the view is what happens when both are -//! applied to the same place. -//! -//! Everything here is in memory and dies with the process. That is honest for -//! what this is: an index over other people's repositories, all of which -//! remain the durable copy. Losing it costs a re-survey, not a record. - -use std::collections::{BTreeMap, BTreeSet, HashMap}; - -use serde::{Deserialize, Serialize}; -use tracing::warn; - -use crate::feed::Scrobble; -use crate::registration::Profile; -use crate::work::{WorkGroup, WorkIndex}; - -/// Where a claim about an agent's parent came from. -/// -/// Two sources say the same thing and one of them is authoritative. The -/// profile is the account's durable statement of its own lineage; the copy on -/// a scrobble is a denormalization put there so a reader of the firehose does -/// not have to fetch a profile per record. When they disagree the -/// profile wins, and this is how [`View`] can tell which it is holding. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Lineage { - /// From the copy on a record the account wrote. - Record, - /// From the account's `bot.did.registration`. - Profile, -} - -/// How many scrobbles are kept for one agent. -/// -/// A canvas draws the last thing an agent said and a panel wants the run-up; -/// nothing needs the whole history, and the repository has it anyway. -pub const HISTORY: usize = 32; - -/// A personal data server the index is following. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Operator { - /// The service DID, which is what a vouch names. - pub did: String, - /// How the server is reached. - pub base_url: String, - /// The zone it mints under, and the handle a person would recognise. - pub handle: String, - /// The zone hostname. - pub zone: String, -} - -/// One agent account being indexed. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Agent { - /// The account's DID. - pub did: String, - /// The label it was minted under. - pub agent_id: String, - /// The name it answers to, when its server issued one. - /// - /// The full handle, zone and all, because the whole thing is the name: a - /// bare label resolves nowhere and would name a different agent under a - /// different zone. Absent for an account - /// whose deployment does not name its agents, and absent when the name its - /// server claimed was not one this index will repeat — see - /// [`readable_handle`](crate::survey::readable_handle). - /// - /// Defaulted, so a snapshot taken by an index older than this field still - /// reads as an unnamed population rather than failing to parse. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub handle: Option, - /// The server it lives on, by service DID. - pub operator: String, - /// Whether it survives a sweep of unpinned accounts. - pub pinned: bool, - /// When it was provisioned, RFC 3339, as the server reports it. - pub created_at: String, - /// The DID of the agent that spawned this one, when one is known. - /// - /// Learned from the records the account writes rather than from the - /// server's account listing, which carries no lineage: a scrobble names - /// its author's parent and this is where that copy is folded into - /// something durable, so a client is told once rather than having to - /// reconstruct the tree from a stream of records. - /// - /// `None` means "nothing has said", which is not the same as "no parent". - /// A session started by a human genuinely has none; an agent that has not - /// spoken yet has one nobody has been told about. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub parent: Option, - /// The kind of agent, as its harness names it. - /// - /// From the account's profile record and from nowhere else. A scrobble - /// carries a copy of it too, but the copy is there so a reader of - /// the firehose can judge one record without a fetch; an account-level - /// fact belongs to the account, and reading it here is what lets a client - /// label an agent that has not said anything. - /// - /// Free-form and meaningless without [`Self::harness`], which is why the - /// two arrive together. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub agent_type: Option, - /// The harness that operates the account, such as `claude-code`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub harness: Option, - /// The model the account's most recent scrobble was written by. - /// - /// A statement about the account rather than about any one turn: a - /// scrobble says what wrote that sentence, and this says what the agent - /// was running when its profile was last written. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub model: Option, -} - -/// How many work groups an index will report at once. -/// -/// A canvas draws a handful and a panel lists eight; a tail of forty groups of -/// three is noise on the wire and noise on the screen. Groups are sorted -/// biggest first, so this cuts the tail rather than an arbitrary slice. -pub const GROUPS: usize = 24; - -/// Something that happened, in the shape a client consumes. -/// -/// Deliberately not the server's own event type. This is the index's account -/// of what changed, including changes no server announces — a record -/// appearing is not a lifecycle transition anywhere upstream — and a client -/// should not have to know which of those it is reading. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "camelCase")] -pub enum Event { - /// A server came into view. - Operator(Operator), - /// An account appeared. - Provisioned(Agent), - /// An account said something. - Scrobble(Scrobble), - /// An account was pinned or unpinned. - Pinned { - /// The account. - did: String, - /// Its new state. - pinned: bool, - }, - /// An account went away. - Deleted { - /// The account. - did: String, - }, - /// The work groups were recomputed and came out different. - /// - /// The whole set rather than a diff. A group is a list of members and a - /// reclustering pass can move any of them, so a diff would be most of the - /// set most of the time; and a client that missed one diff would be wrong - /// with no way to notice, where a client that misses a whole set is - /// briefly stale and right again on the next pass. - Work { - /// Every group, biggest first. - groups: Vec, - }, -} - -/// Everything the index knows, at one moment. -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Snapshot { - /// The servers being followed. - pub operators: Vec, - /// Every account currently indexed. - pub agents: Vec, - /// Recent records, oldest first. - pub scrobbles: Vec, - /// Which agents are doing the same work, biggest group first. - /// - /// Defaulted rather than required, so a snapshot from an index that is not - /// clustering — or from one older than this field — reads as "no groups" - /// instead of failing to parse. - #[serde(default)] - pub groups: Vec, -} - -/// The index's state. -#[derive(Debug, Default)] -pub struct View { - operators: BTreeMap, - agents: BTreeMap, - /// A handle, lowercased, to every DID currently holding it. - /// - /// A lookup rather than a scan: [`Self::agent_by_handle`] used to walk - /// every account in the population to answer one question, which is fine - /// at a few hundred accounts and shows up in a profile well before ten - /// thousand. Kept in step here, at the two places an account's handle can - /// change — [`Self::see_agent`] and [`Self::forget`] — rather than - /// rebuilt from the population on read, which is the shape that would - /// drift. - /// - /// A set rather than a single DID: a handle is only unique among live - /// accounts by construction elsewhere in the pipeline, not by anything - /// enforced here, and this index reads servers it does not control. - /// [`Self::agent_by_handle`] resolves a collision the same way the linear - /// scan it replaced did — the lowest DID, which is the iteration order a - /// [`BTreeSet`] already gives for free. - by_handle: HashMap>, - history: BTreeMap>, - seen: BTreeMap, - /// Accounts whose parent was learned from their own profile record. - /// - /// A profile wins over the copy on a record, and this is the whole of the - /// machinery for - /// it: a record's claim is refused for an account that is in here and - /// disagrees. It is not a second copy of the lineage — the parent itself - /// lives on the [`Agent`] — only a note of where the one held came from. - /// - /// Not in the snapshot. A client is told the tree, not the archaeology of - /// how each edge was learned, and a mirror that had to reproduce this - /// would be reproducing a decision the index has already made. - profiled: std::collections::BTreeSet, - /// The groups this view is currently reporting. - groups: Vec, - /// The clustering state, in the one process that is allowed to have it. - /// - /// `None` everywhere downstream. The whole point of moving clustering off - /// the client is that there is one answer, and a mirror that recomputed - /// its own would produce a second one from a different event history — the - /// exact problem, moved one hop. A mirror takes [`Event::Work`] and stores - /// it; it cannot do anything else, because it has nothing to do it with. - work: Option, -} - -impl View { - /// An empty view that reports whatever groups it is told about. - pub fn new() -> Self { - Self::default() - } - - /// An empty view that works out its own groups. - /// - /// For the index, and for nothing else: a mirror takes its groups from - /// upstream, and a mirror that recomputed them would produce a second - /// answer from a different event history. - pub fn clustering() -> Self { - Self { - work: Some(WorkIndex::new()), - ..Self::default() - } - } - - /// The groups as they stand. - pub fn groups(&self) -> &[WorkGroup] { - &self.groups - } - - /// Whether this view computes its own groups. - pub fn clusters(&self) -> bool { - self.work.is_some() - } - - /// Recomputes the groups, announcing them only if they changed. - /// - /// Nothing when this view does not cluster, and nothing when the pass came - /// out the same as the last one — which is the common case on a quiet - /// population, and the difference between a stream that says something - /// when something happened and one that talks continuously. - pub fn recluster(&mut self) -> Option { - let work = self.work.as_mut()?; - let mut groups = work.cluster(); - groups.truncate(GROUPS); - if groups == self.groups { - return None; - } - self.groups = groups.clone(); - Some(Event::Work { groups }) - } - - /// Takes groups computed somewhere else. - pub fn see_groups(&mut self, groups: Vec) -> Option { - if groups == self.groups { - return None; - } - self.groups = groups.clone(); - Some(Event::Work { groups }) - } - - /// How many accounts are indexed. - pub fn agents(&self) -> usize { - self.agents.len() - } - - /// How many servers are being followed. - pub fn operators(&self) -> usize { - self.operators.len() - } - - /// Records the index has taken in, across every agent. - pub fn scrobbles(&self) -> usize { - self.seen.len() - } - - /// Adds or replaces a server. - /// - /// Returns the event to announce, or nothing when this changed nothing — - /// a survey runs on a timer and mostly finds what it found last time. - pub fn see_operator(&mut self, operator: Operator) -> Option { - if self.operators.get(&operator.did) == Some(&operator) { - return None; - } - self.operators - .insert(operator.did.clone(), operator.clone()); - Some(Event::Operator(operator)) - } - - /// Adds or replaces an account. - /// - /// Lineage is merged rather than replaced. A survey rebuilds an `Agent` - /// from a server's account listing, which carries no parent, so a plain - /// replace would clear a parent learned from a record on every survey - /// beat and announce the loss — a tree that fell apart every ten seconds - /// and grew back. An incoming parent wins when there is one; otherwise - /// what is already known is kept. - pub fn see_agent(&mut self, mut agent: Agent) -> Option { - let previous_handle = if let Some(known) = self.agents.get(&agent.did) { - if agent.parent.is_none() { - agent.parent.clone_from(&known.parent); - } - // The same argument, for the same reason, for the three fields a - // profile is the only source of. A survey rebuilds an `Agent` - // from an account listing that carries none of them. - if agent.agent_type.is_none() { - agent.agent_type.clone_from(&known.agent_type); - } - if agent.harness.is_none() { - agent.harness.clone_from(&known.harness); - } - if agent.model.is_none() { - agent.model.clone_from(&known.model); - } - if known == &agent { - return None; - } - known.handle.clone() - } else { - None - }; - if previous_handle.as_deref() != agent.handle.as_deref() { - self.unindex_handle(&agent.did, previous_handle.as_deref()); - self.index_handle(&agent.did, agent.handle.as_deref()); - } - self.agents.insert(agent.did.clone(), agent.clone()); - Some(Event::Provisioned(agent)) - } - - /// Adds `did` under `handle`, lowercased, in [`Self::by_handle`]. - fn index_handle(&mut self, did: &str, handle: Option<&str>) { - if let Some(handle) = handle { - let holders = self.by_handle.entry(handle.to_lowercase()).or_default(); - holders.insert(did.to_owned()); - if holders.len() > 1 { - warn!( - handle, - did, - holders = holders.len(), - "more than one account claims this handle; the lowest did answers for it" - ); - } - } - } - - /// Removes `did` from under `handle` in [`Self::by_handle`]. - /// - /// Drops the entry entirely once its set is empty, so a handle nobody - /// holds is absent rather than mapping to nothing. - fn unindex_handle(&mut self, did: &str, handle: Option<&str>) { - if let Some(handle) = handle { - let key = handle.to_lowercase(); - if let Some(holders) = self.by_handle.get_mut(&key) { - holders.remove(did); - if holders.is_empty() { - self.by_handle.remove(&key); - } - } - } - } - - /// Records that `child` was spawned by `parent`, if that is believable. - /// - /// The claim arrives on a record, which means it arrives from the server - /// hosting the author — so it is the same kind of claim discovery refuses to - /// take at face value everywhere else, and it gets the same treatment. A - /// parent is accepted only when it is an account this index already holds - /// **on the same operator**, because a server naming an agent on another - /// server as the parent is describing a repository it has no authority - /// over, and an index that believed it would hang somebody else's subtree - /// under a name of the server's choosing. - /// - /// Three further refusals, none of them about trust: - /// - /// * an agent is not its own parent; - /// * a parent already below the child is refused, because a cycle is not - /// a tree and every consumer of this walks it upward; - /// * a parent that is already recorded changes nothing, so nothing is - /// announced. Lineage is constant for the life of an account and every - /// record the account writes repeats it, so this is the common case. - pub fn learn_parent(&mut self, child: &str, parent: &str) -> Option { - self.learn_parent_from(child, parent, Lineage::Record) - } - - /// The same, saying where the claim came from. - /// - /// This was left open once: lineage was read off the records because - /// nothing wrote profiles, and a profile was to win when one existed. This - /// is where it wins. A parent learned from a profile is - /// remembered as such, and a later record claiming a different one is - /// refused rather than applied — a record repeats the account's lineage on - /// every write, so without this rule one stale scrobble in a repository - /// would drag the tree back to a shape the account has since corrected. - /// - /// Disagreement is worth a log line rather than a silence, because both - /// claims come from the same harness reading the same sidecar. The two - /// differing means something upstream is inconsistent, and the only place - /// that is visible is here. - pub fn learn_parent_from( - &mut self, - child: &str, - parent: &str, - source: Lineage, - ) -> Option { - if child == parent { - return None; - } - let known = self.agents.get(child)?; - if known.parent.as_deref() == Some(parent) { - if source == Lineage::Profile { - self.profiled.insert(child.to_owned()); - } - return None; - } - if source == Lineage::Record && self.profiled.contains(child) { - tracing::debug!( - child, - claimed = parent, - held = known.parent.as_deref().unwrap_or("-"), - "a record disagrees with the account's own profile; keeping the profile" - ); - return None; - } - let operator = known.operator.clone(); - let claimed = self.agents.get(parent)?; - if claimed.operator != operator { - return None; - } - if self.descends_from(parent, child) { - return None; - } - if source == Lineage::Profile { - self.profiled.insert(child.to_owned()); - } - let child = self.agents.get_mut(child)?; - child.parent = Some(parent.to_owned()); - Some(Event::Provisioned(child.clone())) - } - - /// Takes in one account's profile record. - /// - /// The reason an agent that has never scrobbled is no longer a root - /// forever. A profile is written when the account is provisioned, so this - /// answers the three questions that could once only be answered after an agent spoke — - /// what spawned it and what kind of agent it is — - /// for an agent that never does. - /// - /// Returns at most one event, because everything it changes is on one - /// account. Nothing when the profile said nothing this view did not - /// already hold, which is every re-read after the first. - pub fn see_profile(&mut self, profile: &Profile) -> Vec { - let Some(agent) = self.agents.get_mut(&profile.did) else { - return Vec::new(); - }; - let mut changed = false; - for (held, incoming) in [ - (&mut agent.agent_type, &profile.agent_type), - (&mut agent.harness, &profile.harness), - ] { - // An absent field is the server having had nothing to record, not - // a retraction. Taking a field away is a thing the writer does by - // sending a different value, not by silence. - if incoming.is_some() && held != incoming { - held.clone_from(incoming); - changed = true; - } - } - let announced = changed.then(|| Event::Provisioned(agent.clone())); - // After the fields, so an account that learned both in one pass - // announces the lineage last and a client hangs the node off a parent - // it has already been told about. - let lineage = profile - .parent - .as_deref() - .and_then(|parent| self.learn_parent_from(&profile.did, parent, Lineage::Profile)); - match (announced, lineage) { - // The lineage event carries the whole agent, fields and all, so - // announcing both would say the same thing twice. - (_, Some(event)) => vec![event], - (Some(event), None) => vec![event], - (None, None) => Vec::new(), - } - } - - /// Whether `did` has `ancestor` somewhere above it. - /// - /// Bounded by the number of accounts held rather than trusted to - /// terminate: this is the check that keeps a cycle out, so it must not - /// assume there is not one already. - fn descends_from(&self, did: &str, ancestor: &str) -> bool { - let mut node = did; - for _ in 0..self.agents.len() { - let Some(agent) = self.agents.get(node) else { - return false; - }; - let Some(parent) = agent.parent.as_deref() else { - return false; - }; - if parent == ancestor { - return true; - } - node = parent; - } - false - } - - /// Takes in one record, if it has not been seen before. - /// - /// Identity is the AT-URI, because a poller re-reads the same page of a - /// repository every time it looks and every record on it is already - /// known except the ones that are not. - /// - /// Returns more than one event when the record taught this view something - /// beyond its own text: a scrobble carries its author's parent, and the - /// first one that does turns an account drawn as a session root into a - /// child of the agent that spawned it. The lineage goes out first, so a - /// client has the tree in the shape the record describes before it is - /// asked to hang anything off it. - pub fn see_scrobble(&mut self, scrobble: Scrobble) -> Vec { - if self.seen.contains_key(&scrobble.uri) { - return Vec::new(); - } - self.seen.insert(scrobble.uri.clone(), ()); - // The one place a status line enters this view, and therefore the one - // place it can be folded into a profile without a second call site to - // forget about. - if let Some(work) = self.work.as_mut() { - work.observe(&scrobble.did, &scrobble.text); - } - let history = self.history.entry(scrobble.did.clone()).or_default(); - history.push(scrobble.clone()); - if history.len() > HISTORY { - history.remove(0); - } - let mut events = Vec::new(); - if let Some(parent) = scrobble.parent.clone() { - events.extend(self.learn_parent(&scrobble.did, &parent)); - } - // The account-level model comes from here now, and only from here. It - // used to be read off a profile the harness rewrote, which made the - // account's copy and the record's copy two answers to compare; a model - // changes per turn, so the last turn to say anything is the honest - // source and the record it wrote is where it said it. - if let Some(model) = scrobble.model.clone() { - if let Some(agent) = self.agents.get_mut(&scrobble.did) { - if agent.model.as_deref() != Some(model.as_str()) { - agent.model = Some(model); - events.push(Event::Provisioned(agent.clone())); - } - } - } - events.push(Event::Scrobble(scrobble)); - events - } - - /// Marks an account pinned or unpinned. - pub fn set_pinned(&mut self, did: &str, pinned: bool) -> Option { - let agent = self.agents.get_mut(did)?; - if agent.pinned == pinned { - return None; - } - agent.pinned = pinned; - Some(Event::Pinned { - did: did.to_owned(), - pinned, - }) - } - - /// Forgets an account and everything it said. - /// - /// The records go with it. They are gone from the repository too — an - /// account's deletion takes its data — so an index that kept them would - /// be the only place they still existed, which is not what an index is. - /// - /// Anything it spawned is handed up to its own parent first. A subagent - /// routinely outlives the agent that asked for it, and leaving the child - /// pointing at a name this index no longer holds would put a hole in the - /// middle of every tree drawn from a later snapshot. Handing it up is the - /// same repair the canvas already makes locally, made once and announced, - /// so a client that connects afterwards is told the same thing as one - /// that was watching. - pub fn forget(&mut self, did: &str) -> Vec { - let Some(gone) = self.agents.remove(did) else { - return Vec::new(); - }; - self.unindex_handle(did, gone.handle.as_deref()); - // A child handed up below inherits an edge this index no longer has a - // profile for, so it must not keep the mark that would refuse a later - // record's claim. - self.profiled.remove(did); - if let Some(work) = self.work.as_mut() { - work.forget(did); - } - if let Some(history) = self.history.remove(did) { - for scrobble in history { - self.seen.remove(&scrobble.uri); - } - } - - let orphans: Vec = self - .agents - .values() - .filter(|agent| agent.parent.as_deref() == Some(did)) - .map(|agent| agent.did.clone()) - .collect(); - let mut events = Vec::with_capacity(orphans.len() + 1); - for orphan in orphans { - self.profiled.remove(&orphan); - if let Some(agent) = self.agents.get_mut(&orphan) { - agent.parent.clone_from(&gone.parent); - events.push(Event::Provisioned(agent.clone())); - } - } - events.push(Event::Deleted { - did: did.to_owned(), - }); - events - } - - /// Applies an event from upstream, as if it had been observed here. - /// - /// This is what makes a second service possible: the index observes and - /// announces, and anything downstream reaches the same state by replaying - /// the announcements rather than by doing the observing again. - pub fn apply(&mut self, event: Event) -> Vec { - match event { - Event::Operator(operator) => self.see_operator(operator).into_iter().collect(), - Event::Provisioned(agent) => self.see_agent(agent).into_iter().collect(), - Event::Scrobble(scrobble) => self.see_scrobble(scrobble), - Event::Pinned { did, pinned } => self.set_pinned(&did, pinned).into_iter().collect(), - Event::Deleted { did } => self.forget(&did), - Event::Work { groups } => self.see_groups(groups).into_iter().collect(), - } - } - - /// Replaces everything with a snapshot taken somewhere else. - /// - /// Used when a downstream service connects, or reconnects after a gap: - /// catching up event by event is impossible when the events are gone, and - /// the whole view is small enough to send. - pub fn adopt(&mut self, snapshot: Snapshot) { - let work = self.work.take().map(|_| WorkIndex::new()); - *self = Self { - work, - ..Self::new() - }; - for operator in snapshot.operators { - self.see_operator(operator); - } - for agent in snapshot.agents { - self.see_agent(agent); - } - for scrobble in snapshot.scrobbles { - self.see_scrobble(scrobble); - } - self.groups = snapshot.groups; - } - - /// One account, by DID. - pub fn agent(&self, did: &str) -> Option<&Agent> { - self.agents.get(did) - } - - /// One account, by the handle its deployment issued it. - /// - /// Matched case-insensitively, because a handle is a hostname and a - /// hostname is not case-sensitive; a person typing one out of a log - /// should not have to reproduce its case. A lookup against - /// `by_handle` rather than a scan of the population — kept in - /// step at the two places a handle can change, so there is one answer - /// rather than a second index that can drift from the first. - /// - /// A handle is not unique across time — one is released after a hold — - /// but it is unique among live accounts, which is what this searches. If - /// two accounts nonetheless claim the same one, the lowest DID wins, the - /// same tie-break the scan this replaced made by visiting accounts in DID - /// order and returning the first match. - pub fn agent_by_handle(&self, handle: &str) -> Option<&Agent> { - let did = self.by_handle.get(&handle.to_lowercase())?.iter().next()?; - self.agents.get(did) - } - - /// One server, by service DID. - pub fn operator(&self, did: &str) -> Option<&Operator> { - self.operators.get(did) - } - - /// Every server being followed. - pub fn operator_list(&self) -> Vec { - self.operators.values().cloned().collect() - } - - /// Every account this one spawned, in DID order. - pub fn children_of(&self, did: &str) -> Vec<&Agent> { - self.agents - .values() - .filter(|agent| agent.parent.as_deref() == Some(did)) - .collect() - } - - /// The chain from an account up to its session root, nearest first. - /// - /// Bounded by the size of the population rather than trusted to - /// terminate. [`Self::learn_parent`] refuses to build a cycle, but a view - /// that adopted a snapshot took its tree from somewhere else, and the one - /// place a cycle would do real damage is a loop that walks it. - pub fn ancestors_of(&self, did: &str) -> Vec<&Agent> { - let mut chain = Vec::new(); - let mut node = self - .agents - .get(did) - .and_then(|agent| agent.parent.as_deref()); - for _ in 0..self.agents.len() { - let Some(current) = node else { break }; - let Some(agent) = self.agents.get(current) else { - break; - }; - chain.push(agent); - node = agent.parent.as_deref(); - } - chain - } - - /// What one account has been saying, oldest first. - pub fn history_of(&self, did: &str) -> &[Scrobble] { - self.history.get(did).map_or(&[], Vec::as_slice) - } - - /// Accounts in DID order, starting strictly after `after`. - /// - /// DID order rather than arrival order, because a cursor has to name a - /// position that still means the same thing on the next request. Accounts - /// arrive and go away constantly here, so an offset would skip or repeat - /// whenever the population changed under a paging client; a DID is stable, - /// total, and still a valid place to resume from after the account it - /// names has itself been deleted. - pub fn agents_after<'a>( - &'a self, - after: Option<&str>, - keep: impl Fn(&Agent) -> bool + 'a, - ) -> impl Iterator + 'a { - let start = after.map(|did| did.to_owned()); - self.agents - .iter() - .filter(move |(did, _)| match &start { - Some(after) => did.as_str() > after.as_str(), - None => true, - }) - .map(|(_, agent)| agent) - .filter(move |agent| keep(agent)) - } - - /// Every record held, newest first. - /// - /// Sorted on the way out rather than kept sorted: records are held per - /// account, because that is what bounds them, and one ordered list beside - /// that would be a second copy to keep in step. The whole set is at most - /// [`HISTORY`] per account and a mirror serving this is doing it once per - /// request, not once per record. - pub fn recent(&self, keep: impl Fn(&Scrobble) -> bool) -> Vec<&Scrobble> { - let mut all: Vec<&Scrobble> = self - .history - .values() - .flatten() - .filter(|scrobble| keep(scrobble)) - .collect(); - // Newest first, and the URI breaks a tie so that two records written - // in the same second have one order rather than an arbitrary one — a - // cursor over an unstable sort silently skips rows. - all.sort_by(|a, b| { - b.created_at - .cmp(&a.created_at) - .then_with(|| b.uri.cmp(&a.uri)) - }); - all - } - - /// One work group, by id. - pub fn group(&self, id: &str) -> Option<&WorkGroup> { - self.groups.iter().find(|group| group.id == id) - } - - /// Whether an account is known. - pub fn knows(&self, did: &str) -> bool { - self.agents.contains_key(did) - } - - /// Everything, for a client that has just connected. - pub fn snapshot(&self) -> Snapshot { - let mut scrobbles: Vec = self - .history - .values() - .flat_map(|history| history.iter().cloned()) - .collect(); - // Oldest first, so a client applying them in order ends up where the - // index already is. - scrobbles.sort_by(|a, b| a.created_at.cmp(&b.created_at)); - Snapshot { - operators: self.operators.values().cloned().collect(), - agents: self.lineage_order(), - scrobbles, - groups: self.groups.clone(), - } - } - - /// Every account, each one after whatever spawned it. - /// - /// A snapshot is replayed as though it were a stream of arrivals, and a - /// client building a tree can only attach a child to a parent it already - /// has. The map is keyed by DID, so its natural order is alphabetical and - /// says nothing about who came first; a child sorted before its parent - /// would be drawn as a root and stay one, which is exactly the flat - /// picture this lineage exists to replace. - /// - /// Whatever is left when no more parents can be satisfied is emitted - /// anyway, in DID order. That is a cycle, which [`Self::learn_parent`] - /// refuses to create — but a snapshot adopted from upstream was built by - /// somebody else, and dropping accounts on the floor is a worse answer - /// than drawing a few of them as roots. - fn lineage_order(&self) -> Vec { - let mut ordered: Vec = Vec::with_capacity(self.agents.len()); - let mut placed: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new(); - let mut waiting: Vec<&Agent> = self.agents.values().collect(); - while !waiting.is_empty() { - let mut ready = Vec::new(); - let mut later = Vec::new(); - for agent in waiting { - match agent.parent.as_deref() { - Some(parent) - if self.agents.contains_key(parent) && !placed.contains(parent) => - { - later.push(agent); - } - _ => ready.push(agent), - } - } - if ready.is_empty() { - ordered.extend(later.into_iter().cloned()); - break; - } - for agent in ready { - placed.insert(agent.did.as_str()); - ordered.push(agent.clone()); - } - waiting = later; - } - ordered - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn agent(did: &str) -> Agent { - Agent { - did: did.to_owned(), - agent_id: "one".to_owned(), - handle: Some("basalt-otter.agents.example".to_owned()), - operator: "did:web:agents.example".to_owned(), - pinned: false, - created_at: "2026-08-25T10:00:00Z".to_owned(), - parent: None, - agent_type: None, - harness: None, - model: None, - } - } - - fn scrobble(uri: &str, did: &str) -> Scrobble { - Scrobble { - uri: uri.to_owned(), - did: did.to_owned(), - text: "reading through the cart".to_owned(), - emoji: "🔍".to_owned(), - created_at: "2026-08-25T10:01:00Z".to_owned(), - task: None, - effort: Some("high".to_owned()), - model: None, - agent_type: None, - parent: None, - artifacts: Vec::new(), - value: serde_json::Value::Null, - } - } - - #[test] - fn seeing_the_same_thing_twice_announces_it_once() { - let mut view = View::new(); - assert!(view.see_agent(agent("did:web:a.example")).is_some()); - assert!(view.see_agent(agent("did:web:a.example")).is_none()); - assert_eq!(view.agents(), 1); - } - - #[test] - fn a_changed_account_is_announced_again() { - let mut view = View::new(); - view.see_agent(agent("did:web:a.example")); - let mut pinned = agent("did:web:a.example"); - pinned.pinned = true; - assert!(view.see_agent(pinned).is_some()); - } - - #[test] - fn a_record_is_taken_in_once_however_often_it_is_read() { - let mut view = View::new(); - view.see_agent(agent("did:web:a.example")); - assert!( - view.see_scrobble(scrobble("at://a/1", "did:web:a.example")) - .len() - == 1 - ); - assert!(view - .see_scrobble(scrobble("at://a/1", "did:web:a.example")) - .is_empty()); - assert_eq!(view.scrobbles(), 1); - } - - #[test] - fn history_is_bounded_per_agent() { - let mut view = View::new(); - view.see_agent(agent("did:web:a.example")); - for i in 0..(HISTORY + 20) { - view.see_scrobble(scrobble(&format!("at://a/{i}"), "did:web:a.example")); - } - assert_eq!(view.snapshot().scrobbles.len(), HISTORY); - } - - #[test] - fn forgetting_an_account_forgets_what_it_said() { - let mut view = View::new(); - view.see_agent(agent("did:web:a.example")); - view.see_scrobble(scrobble("at://a/1", "did:web:a.example")); - assert!(!view.forget("did:web:a.example").is_empty()); - assert_eq!(view.agents(), 0); - assert_eq!(view.scrobbles(), 0); - assert!(view.snapshot().scrobbles.is_empty()); - // And forgetting one that was never there is not an event. - assert!(view.forget("did:web:a.example").is_empty()); - } - - #[test] - fn a_handle_is_found_case_insensitively() { - let mut view = View::new(); - view.see_agent(agent("did:web:a.example")); - assert_eq!( - view.agent_by_handle("BASALT-Otter.Agents.EXAMPLE") - .map(|a| a.did.as_str()), - Some("did:web:a.example") - ); - } - - #[test] - fn an_unknown_handle_finds_nothing() { - let view = View::new(); - assert!(view.agent_by_handle("nobody.agents.example").is_none()); - } - - #[test] - fn a_forgotten_account_s_handle_stops_resolving() { - let mut view = View::new(); - view.see_agent(agent("did:web:a.example")); - view.forget("did:web:a.example"); - assert!(view - .agent_by_handle("basalt-otter.agents.example") - .is_none()); - // And the freed handle is unaffected by having been held before. - let mut later = agent("did:web:b.example"); - later.handle = Some("basalt-otter.agents.example".to_owned()); - view.see_agent(later); - assert_eq!( - view.agent_by_handle("basalt-otter.agents.example") - .map(|a| a.did.as_str()), - Some("did:web:b.example") - ); - } - - #[test] - fn a_changed_handle_answers_only_to_the_new_one() { - let mut view = View::new(); - view.see_agent(agent("did:web:a.example")); - let mut renamed = agent("did:web:a.example"); - renamed.handle = Some("cinder-wren.agents.example".to_owned()); - view.see_agent(renamed); - - assert!(view - .agent_by_handle("basalt-otter.agents.example") - .is_none()); - assert_eq!( - view.agent_by_handle("cinder-wren.agents.example") - .map(|a| a.did.as_str()), - Some("did:web:a.example") - ); - } - - /// Two accounts nonetheless claiming the same handle is resolved the same - /// way the linear scan this replaced resolved it: the lowest DID, which - /// was simply the first one the scan visited in the population's DID - /// order. - #[test] - fn a_colliding_handle_resolves_to_the_lowest_did() { - let mut view = View::new(); - view.see_agent(agent("did:web:z.example")); - view.see_agent(agent("did:web:a.example")); - assert_eq!( - view.agent_by_handle("basalt-otter.agents.example") - .map(|a| a.did.as_str()), - Some("did:web:a.example") - ); - - // The winner leaving does not strand the handle: the other holder is - // still found under it. - view.forget("did:web:a.example"); - assert_eq!( - view.agent_by_handle("basalt-otter.agents.example") - .map(|a| a.did.as_str()), - Some("did:web:z.example") - ); - } - - /// A lookup against a population large enough that a linear scan would - /// show up in a profile. `agent_by_handle` used to walk every account to - /// answer one question; this asserts the cost of a request no longer - /// grows with the size of the population, by timing a batch of lookups - /// against a population that would make a scan visibly slow and giving - /// the lookup-based version a budget generous enough to never flake but - /// far short of what a scan over this many accounts costs. - /// - /// The searched-for handle belongs to the account with the - /// alphabetically last DID, which is also the last entry a scan in DID - /// order would reach — the worst case for the implementation this - /// replaces, and an ordinary case for a lookup. - #[test] - fn a_handle_lookup_does_not_scan_the_population() { - const ACCOUNTS: usize = 200_000; - const LOOKUPS: usize = 200; - - let mut view = View::new(); - for i in 0..ACCOUNTS { - let did = format!("did:web:acct-{i:07}.example"); - let mut account = agent(&did); - account.handle = Some(format!("acct-{i:07}.agents.example")); - view.see_agent(account); - } - let last = format!("acct-{:07}.agents.example", ACCOUNTS - 1); - - let start = std::time::Instant::now(); - for _ in 0..LOOKUPS { - assert!(view.agent_by_handle(&last).is_some()); - } - let elapsed = start.elapsed(); - - // A hash lookup answers each of these in microseconds; a scan over - // 200,000 accounts per call, times 200 calls, costs tens of - // milliseconds at the very least. 20ms total leaves ample room for a - // slow CI host while still failing hard against the old shape. - assert!( - elapsed < std::time::Duration::from_millis(20), - "{LOOKUPS} lookups against {ACCOUNTS} accounts took {elapsed:?}; \ - agent_by_handle is scanning the population again" - ); - } - - #[test] - fn a_snapshot_is_ordered_oldest_first() { - let mut view = View::new(); - view.see_agent(agent("did:web:a.example")); - let mut later = scrobble("at://a/2", "did:web:a.example"); - later.created_at = "2026-08-25T11:00:00Z".to_owned(); - view.see_scrobble(later); - view.see_scrobble(scrobble("at://a/1", "did:web:a.example")); - - let snapshot = view.snapshot(); - assert_eq!(snapshot.scrobbles[0].uri, "at://a/1"); - assert_eq!(snapshot.scrobbles[1].uri, "at://a/2"); - } - - /// Says one thing several times, which is what makes a profile sharp - /// enough to cluster on. - fn says(view: &mut View, did: &str, text: &str, times: usize) { - view.see_agent(agent(did)); - for i in 0..times { - let mut line = scrobble(&format!("at://{did}/{i}"), did); - line.text = text.to_owned(); - view.see_scrobble(line); - } - } - - #[test] - fn a_plain_view_never_invents_a_group() { - let mut view = View::new(); - assert!(!view.clusters()); - says(&mut view, "did:web:a", "walking the coupon code", 8); - says(&mut view, "did:web:b", "walking the coupon code", 8); - says(&mut view, "did:web:c", "walking the coupon code", 8); - assert!(view.recluster().is_none(), "a mirror must not cluster"); - assert!(view.groups().is_empty()); - } - - #[test] - fn a_clustering_view_groups_what_it_hears_and_says_so_once() { - let mut view = View::clustering(); - assert!(view.clusters()); - for did in ["did:web:a", "did:web:b", "did:web:c"] { - says( - &mut view, - did, - "walking the coupon code through the cart", - 8, - ); - } - let Some(Event::Work { groups }) = view.recluster() else { - panic!("the first pass found a group and must announce it"); - }; - assert_eq!(groups.len(), 1); - assert_eq!(groups[0].members.len(), 3); - // Nothing was said in between, so nothing changed, so nothing is said. - assert!(view.recluster().is_none()); - } - - #[test] - fn a_mirror_reaches_the_same_groups_by_replaying_the_announcement() { - let mut index = View::clustering(); - for did in ["did:web:a", "did:web:b", "did:web:c"] { - says(&mut index, did, "draining the dead letter queue", 8); - } - let announced = index.recluster().expect("a group was found"); - - let mut mirror = View::new(); - for did in ["did:web:a", "did:web:b", "did:web:c"] { - mirror.see_agent(agent(did)); - } - assert!(!mirror.apply(announced).is_empty()); - assert_eq!(mirror.groups(), index.groups()); - // And a snapshot carries them too, for a mirror that connects late. - let mut late = View::new(); - late.adopt(index.snapshot()); - assert_eq!(late.groups(), index.groups()); - } - - #[test] - fn a_deleted_account_leaves_the_grouping() { - let mut view = View::clustering(); - for did in ["did:web:a", "did:web:b", "did:web:c", "did:web:d"] { - says( - &mut view, - did, - "reconciling the invoice against the meter", - 8, - ); - } - view.recluster(); - assert_eq!(view.groups()[0].members.len(), 4); - view.forget("did:web:a"); - let Some(Event::Work { groups }) = view.recluster() else { - panic!("losing a member changes the group"); - }; - assert_eq!(groups[0].members.len(), 3); - assert!(!groups[0].members.iter().any(|did| did == "did:web:a")); - } - - /// An agent, its scrobble, and the parent the record names. - fn child_of(view: &mut View, child: &str, parent: Option<&str>) -> Vec { - view.see_agent(agent(child)); - let mut line = scrobble(&format!("at://{child}/1"), child); - line.parent = parent.map(str::to_owned); - view.see_scrobble(line) - } - - #[test] - fn a_record_teaches_the_index_who_spawned_its_author() { - let mut view = View::new(); - view.see_agent(agent("did:web:parent")); - let events = child_of(&mut view, "did:web:child", Some("did:web:parent")); - - // The lineage first, then the record that carried it. - assert_eq!(events.len(), 2); - let Event::Provisioned(child) = &events[0] else { - panic!( - "the first event is the reparented agent, got {:?}", - events[0] - ); - }; - assert_eq!(child.parent.as_deref(), Some("did:web:parent")); - assert!(matches!(events[1], Event::Scrobble(_))); - - // Every record repeats it, and saying it again is not news. - let mut next = scrobble("at://did:web:child/2", "did:web:child"); - next.parent = Some("did:web:parent".to_owned()); - let again = view.see_scrobble(next); - assert_eq!(again.len(), 1, "only the record itself: {again:?}"); - } - - #[test] - fn a_survey_does_not_flatten_a_tree_it_knows_nothing_about() { - // The whole reason `see_agent` merges. A server's account listing - // carries no lineage, so a survey rebuilds every agent with no - // parent, and a plain replace would drop the tree on every beat. - let mut view = View::new(); - view.see_agent(agent("did:web:parent")); - child_of(&mut view, "did:web:child", Some("did:web:parent")); - - assert!( - view.see_agent(agent("did:web:child")).is_none(), - "a survey that learned nothing new must announce nothing" - ); - let held = view.snapshot(); - let child = held - .agents - .iter() - .find(|held| held.did == "did:web:child") - .expect("still there"); - assert_eq!(child.parent.as_deref(), Some("did:web:parent")); - } - - #[test] - fn a_parent_on_another_server_is_not_believed() { - // The discovery rule, applied to lineage: a server may only speak for the - // accounts it hosts, and hanging somebody else's subtree under a name - // of its choosing is exactly what it must not be able to do. - let mut view = View::new(); - let mut elsewhere = agent("did:web:parent"); - elsewhere.operator = "did:web:other.example".to_owned(); - view.see_agent(elsewhere); - - let events = child_of(&mut view, "did:web:child", Some("did:web:parent")); - assert_eq!(events.len(), 1, "the record, and no lineage: {events:?}"); - } - - #[test] - fn a_parent_nothing_has_heard_of_is_not_invented() { - let mut view = View::new(); - let events = child_of(&mut view, "did:web:child", Some("did:web:nowhere")); - assert_eq!(events.len(), 1, "the record, and no lineage: {events:?}"); - } - - #[test] - fn nothing_can_be_talked_into_a_cycle() { - let mut view = View::new(); - view.see_agent(agent("did:web:a")); - child_of(&mut view, "did:web:b", Some("did:web:a")); - child_of(&mut view, "did:web:c", Some("did:web:b")); - - // c is below a, so a cannot be below c, and an agent is not its own. - assert!(view.learn_parent("did:web:a", "did:web:c").is_none()); - assert!(view.learn_parent("did:web:a", "did:web:a").is_none()); - } - - #[test] - fn a_deleted_agent_hands_its_children_up() { - let mut view = View::new(); - view.see_agent(agent("did:web:root")); - child_of(&mut view, "did:web:middle", Some("did:web:root")); - child_of(&mut view, "did:web:leaf", Some("did:web:middle")); - - let events = view.forget("did:web:middle"); - let Some(Event::Provisioned(leaf)) = events.first() else { - panic!("the orphan is handed up before the deletion: {events:?}"); - }; - assert_eq!(leaf.did, "did:web:leaf"); - assert_eq!(leaf.parent.as_deref(), Some("did:web:root")); - assert!(matches!(events.last(), Some(Event::Deleted { .. }))); - } - - #[test] - fn a_root_taking_its_children_with_it_leaves_them_roots() { - let mut view = View::new(); - view.see_agent(agent("did:web:root")); - child_of(&mut view, "did:web:leaf", Some("did:web:root")); - - view.forget("did:web:root"); - let leaf = view - .snapshot() - .agents - .into_iter() - .find(|agent| agent.did == "did:web:leaf") - .expect("still there"); - assert_eq!(leaf.parent, None, "it has no grandparent to be handed to"); - } - - #[test] - fn a_snapshot_lists_every_parent_before_its_children() { - // The map is keyed by DID, so its own order is alphabetical: `zzz` - // spawning `aaa` is the case that catches a snapshot handed out in - // key order, because the child sorts first and would be drawn as a - // root by anything replaying it. - let mut view = View::new(); - view.see_agent(agent("did:web:zzz")); - child_of(&mut view, "did:web:aaa", Some("did:web:zzz")); - child_of(&mut view, "did:web:mmm", Some("did:web:aaa")); - - let order: Vec = view - .snapshot() - .agents - .into_iter() - .map(|agent| agent.did) - .collect(); - let at = |did: &str| order.iter().position(|held| held == did).expect("listed"); - assert!(at("did:web:zzz") < at("did:web:aaa")); - assert!(at("did:web:aaa") < at("did:web:mmm")); - } - - #[test] - fn a_mirror_that_replays_a_snapshot_ends_up_with_the_same_tree() { - let mut index = View::clustering(); - index.see_agent(agent("did:web:root")); - child_of(&mut index, "did:web:child", Some("did:web:root")); - - let mut mirror = View::new(); - mirror.adopt(index.snapshot()); - assert_eq!(mirror.snapshot().agents, index.snapshot().agents); - } - - /// A profile for `did`, in the shape the reader produces. - fn profile(did: &str, parent: Option<&str>) -> Profile { - Profile { - did: did.to_owned(), - agent_type: Some("Explore".to_owned()), - harness: Some("claude-code".to_owned()), - parent: parent.map(str::to_owned), - } - } - - /// The gap closed: this agent has said nothing at all. - #[test] - fn a_profile_gives_lineage_to_an_agent_that_has_never_scrobbled() { - let mut view = View::new(); - view.see_agent(agent("did:web:root")); - view.see_agent(agent("did:web:child")); - assert!(view.snapshot().agents.iter().all(|a| a.parent.is_none())); - - let events = view.see_profile(&profile("did:web:child", Some("did:web:root"))); - assert_eq!(events.len(), 1, "the tree changed and must be announced"); - let child = view.agent("did:web:child").expect("still held"); - assert_eq!(child.parent.as_deref(), Some("did:web:root")); - assert_eq!(child.agent_type.as_deref(), Some("Explore")); - assert_eq!(view.scrobbles(), 0, "and it still has not said anything"); - } - - #[test] - fn a_profile_that_repeats_itself_announces_nothing() { - let mut view = View::new(); - view.see_agent(agent("did:web:root")); - view.see_agent(agent("did:web:child")); - assert!(!view - .see_profile(&profile("did:web:child", Some("did:web:root"))) - .is_empty()); - assert!( - view.see_profile(&profile("did:web:child", Some("did:web:root"))) - .is_empty(), - "a poll re-reads the same profile on every beat" - ); - } - - /// A profile wins over a record's claim. - #[test] - fn a_record_does_not_overrule_the_account_s_own_profile() { - let mut view = View::new(); - for did in ["did:web:root", "did:web:other", "did:web:child"] { - view.see_agent(agent(did)); - } - view.see_profile(&profile("did:web:child", Some("did:web:root"))); - - let mut line = scrobble("at://child/1", "did:web:child"); - line.parent = Some("did:web:other".to_owned()); - view.see_scrobble(line); - - assert_eq!( - view.agent("did:web:child").expect("held").parent.as_deref(), - Some("did:web:root"), - "the profile is authoritative and the copy on a record is not" - ); - } - - /// The other direction: a profile corrects what a record taught. - #[test] - fn a_profile_overrules_what_a_record_taught() { - let mut view = View::new(); - for did in ["did:web:root", "did:web:other", "did:web:child"] { - view.see_agent(agent(did)); - } - child_of(&mut view, "did:web:child", Some("did:web:other")); - - assert!(!view - .see_profile(&profile("did:web:child", Some("did:web:root"))) - .is_empty()); - assert_eq!( - view.agent("did:web:child").expect("held").parent.as_deref(), - Some("did:web:root") - ); - } - - /// The same containment rule the record path applies, applied to a - /// claim that arrived on a profile. - #[test] - fn a_profile_naming_a_parent_on_another_server_is_refused() { - let mut view = View::new(); - view.see_agent(agent("did:web:child")); - let mut elsewhere = agent("did:web:stranger"); - elsewhere.operator = "did:web:somewhere.else".to_owned(); - view.see_agent(elsewhere); - - assert!(view - .see_profile(&profile("did:web:child", Some("did:web:stranger"))) - .iter() - .all(|event| !matches!(event, Event::Provisioned(a) if a.parent.is_some()))); - assert!(view.agent("did:web:child").expect("held").parent.is_none()); - } - - /// A survey rebuilds an account from a listing that carries none of this. - #[test] - fn a_survey_beat_does_not_blank_what_a_profile_taught() { - let mut view = View::new(); - view.see_agent(agent("did:web:child")); - view.see_profile(&profile("did:web:child", None)); - - assert!( - view.see_agent(agent("did:web:child")).is_none(), - "the listing says nothing new, so nothing is announced" - ); - let child = view.agent("did:web:child").expect("held"); - assert_eq!(child.agent_type.as_deref(), Some("Explore")); - } - - /// Silence is not a retraction: a profile rewritten to add a model must - /// not drop the type it was written with. - #[test] - fn a_profile_field_the_harness_stopped_reporting_is_kept() { - let mut view = View::new(); - view.see_agent(agent("did:web:child")); - view.see_profile(&profile("did:web:child", None)); - view.see_profile(&Profile { - did: "did:web:child".to_owned(), - ..Profile::default() - }); - - let child = view.agent("did:web:child").expect("held"); - assert_eq!(child.agent_type.as_deref(), Some("Explore")); - } - - #[test] - fn a_profile_for_an_account_this_index_does_not_hold_changes_nothing() { - let mut view = View::new(); - assert!(view - .see_profile(&profile("did:web:stranger", None)) - .is_empty()); - assert_eq!(view.agents(), 0); - } - - #[test] - fn pinning_only_announces_a_change() { - let mut view = View::new(); - view.see_agent(agent("did:web:a.example")); - assert!(view.set_pinned("did:web:a.example", true).is_some()); - assert!(view.set_pinned("did:web:a.example", true).is_none()); - assert!(view.set_pinned("did:web:missing.example", true).is_none()); - } -} diff --git a/crates/didbot-index/src/vouch.rs b/crates/didbot-index/src/vouch.rs deleted file mode 100644 index ff5548d8..00000000 --- a/crates/didbot-index/src/vouch.rs +++ /dev/null @@ -1,328 +0,0 @@ -//! The record a human writes to say "these are my agents". -//! -//! A vouch is an ordinary atproto record in the voucher's own repository, -//! naming a subject DID. That is the whole of the trust chain this appview -//! starts from: the human is accountable for what they signed, the record -//! says which server or account they are accountable for, and nothing the -//! vouched-for server says can widen it. -//! -//! There is one vouch per personal data server rather than one per -//! agent, and every agent on that server inherits it. The lexicon also allows -//! an account DID as the subject, and this module does not care which it is: -//! a subject is a DID, and what it covers is worked out from the DID itself. -//! -//! `scopes` is read and ignored. "A vouch with an empty or absent scope set -//! asserts ownership or operation only, and grants no delegated authority" — -//! and indexing a public feed needs no delegated authority, only somebody -//! willing to say whose agents these are. - -use crate::nsid; -use serde_json::Value; -use time::format_description::well_known::Rfc3339; -use time::OffsetDateTime; - -/// Why a record in the vouch collection is not a vouch. -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -pub enum VouchError { - /// Not a JSON object at all. - #[error("the record is not a JSON object")] - NotAnObject, - /// A record of some other type, in the vouch collection. - #[error("declares $type {0:?}, not the vouch type")] - WrongType(String), - /// No subject, so it vouches for nothing. - #[error("no subject")] - NoSubject, - /// A timestamp that is not RFC 3339. - #[error("{field} is not an RFC 3339 timestamp")] - BadTimestamp { - /// Which field would not parse. - field: &'static str, - }, -} - -/// Why an otherwise well-formed vouch is not being followed. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Dropped { - /// Its `expiresAt` has passed. - Expired { - /// When it stopped being valid. - at: OffsetDateTime, - }, - /// A later record in the same repository revokes it. - Revoked { - /// The AT-URI of the record that revoked it. - by: String, - }, -} - -/// A parsed vouch record. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Vouch { - /// The AT-URI of the record itself. - pub uri: String, - /// The DID of the repository that holds it: the human doing the vouching. - pub voucher: String, - /// The DID being vouched for: a server, or one account on one. - pub subject: String, - /// Scopes the vouch delegates. Read, recorded, and not acted on. - pub scopes: Vec, - /// When the voucher says they issued it. - pub created_at: OffsetDateTime, - /// When it stops being valid, if it does. - pub expires_at: Option, - /// The AT-URI of a prior vouch this one revokes. - pub revokes: Option, -} - -impl Vouch { - /// Reads one record from the voucher's repository. - pub fn read(voucher: &str, uri: &str, value: &Value) -> Result { - let object = value.as_object().ok_or(VouchError::NotAnObject)?; - // The collection was named in the request, so an absent `$type` is - // accepted and a disagreeing one is not. - if let Some(declared) = object.get("$type").and_then(Value::as_str) { - if declared != nsid::VOUCH { - return Err(VouchError::WrongType(declared.to_owned())); - } - } - - let string = |key: &str| object.get(key).and_then(Value::as_str); - let subject = string("subject").ok_or(VouchError::NoSubject)?; - let time_at = |field: &'static str| -> Result, VouchError> { - match string(field) { - Some(raw) => OffsetDateTime::parse(raw, &Rfc3339) - .map(Some) - .map_err(|_| VouchError::BadTimestamp { field }), - None => Ok(None), - } - }; - - Ok(Self { - uri: uri.to_owned(), - voucher: voucher.to_owned(), - subject: subject.to_owned(), - scopes: object - .get("scopes") - .and_then(Value::as_array) - .map(|items| { - items - .iter() - .filter_map(Value::as_str) - .map(str::to_owned) - .collect() - }) - .unwrap_or_default(), - // A vouch with no `createdAt` is invalid under the lexicon, but - // nothing here needs the value, so a missing one is not worth - // refusing a live vouch over. The epoch stands in for "unstated". - created_at: time_at("createdAt")?.unwrap_or(OffsetDateTime::UNIX_EPOCH), - expires_at: time_at("expiresAt")?, - revokes: string("revokes").map(str::to_owned), - }) - } - - /// Whether `now` is past this vouch's expiry. - pub fn is_expired(&self, now: OffsetDateTime) -> bool { - self.expires_at.is_some_and(|expires| now >= expires) - } - - /// The prefix every record in this vouch's own repository starts with. - fn repo_prefix(&self) -> String { - format!("at://{}/", self.voucher) - } -} - -/// Reduces a repository's vouch records to the ones still in force. -/// -/// Two rules, applied in that order: -/// -/// * **Revocation first, expiry second.** A revoking record is honoured even -/// if it has itself expired. Revocation is a statement that something -/// stopped being true, and letting the revoker lapse would resurrect what -/// it revoked — which is the one direction this must never fail in. -/// * **Revocation is repository-local.** "The revoked record is expected to -/// reside in the same repository as this one", so a `revokes` pointing -/// anywhere else is ignored rather than honoured. Otherwise one voucher -/// could cancel another's vouch by writing a record in their own repo. -pub fn in_force(vouches: Vec, now: OffsetDateTime) -> (Vec, Vec<(Vouch, Dropped)>) { - let mut revoked_by: Vec<(String, String)> = Vec::new(); - for vouch in &vouches { - if let Some(target) = &vouch.revokes { - if target.starts_with(&vouch.repo_prefix()) { - revoked_by.push((target.clone(), vouch.uri.clone())); - } - } - } - - let mut live = Vec::new(); - let mut dropped = Vec::new(); - for vouch in vouches { - if let Some((_, by)) = revoked_by.iter().find(|(target, _)| *target == vouch.uri) { - let by = by.clone(); - dropped.push((vouch, Dropped::Revoked { by })); - continue; - } - if vouch.is_expired(now) { - // `is_expired` is true only when `expires_at` is set. - let at = vouch.expires_at.unwrap_or(now); - dropped.push((vouch, Dropped::Expired { at })); - continue; - } - live.push(vouch); - } - (live, dropped) -} - -#[cfg(test)] -mod tests { - use serde_json::json; - use time::macros::datetime; - - use super::*; - - const VOUCHER: &str = "did:plc:nlzmjyfv6loqtxyzvdcznwgf"; - const SUBJECT: &str = "did:web:agents.localhost%3A3000"; - - fn uri(rkey: &str) -> String { - format!("at://{VOUCHER}/{}/{rkey}", nsid::VOUCH) - } - - fn vouch(rkey: &str, value: Value) -> Vouch { - Vouch::read(VOUCHER, &uri(rkey), &value).expect("a readable vouch") - } - - fn plain(rkey: &str) -> Vouch { - vouch( - rkey, - json!({ - "$type": nsid::VOUCH, - "subject": SUBJECT, - "createdAt": "2026-08-01T00:00:00Z", - }), - ) - } - - #[test] - fn reads_every_field_the_lexicon_defines() { - let read = vouch( - "a", - json!({ - "$type": nsid::VOUCH, - "subject": SUBJECT, - "scopes": ["repo:com.vibescrobble.scrobble", 7], - "createdAt": "2026-08-01T00:00:00Z", - "expiresAt": "2027-08-01T00:00:00Z", - "revokes": uri("older"), - }), - ); - - assert_eq!(read.subject, SUBJECT); - assert_eq!(read.voucher, VOUCHER); - assert_eq!(read.scopes, ["repo:com.vibescrobble.scrobble"]); - assert_eq!(read.created_at, datetime!(2026-08-01 00:00 UTC)); - assert_eq!(read.expires_at, Some(datetime!(2027-08-01 00:00 UTC))); - assert_eq!(read.revokes.as_deref(), Some(uri("older").as_str())); - } - - #[test] - fn refuses_a_record_that_is_not_a_vouch() { - let cases = [ - (json!("a string"), VouchError::NotAnObject), - ( - json!({ "$type": nsid::SCROBBLE, "subject": SUBJECT }), - VouchError::WrongType(nsid::SCROBBLE.to_owned()), - ), - ( - json!({ "createdAt": "2026-08-01T00:00:00Z" }), - VouchError::NoSubject, - ), - ( - json!({ "subject": SUBJECT, "createdAt": "last tuesday" }), - VouchError::BadTimestamp { field: "createdAt" }, - ), - ]; - for (value, expected) in cases { - assert_eq!(Vouch::read(VOUCHER, &uri("x"), &value), Err(expected)); - } - } - - #[test] - fn an_expired_vouch_is_not_in_force() { - let expiring = vouch( - "a", - json!({ - "subject": SUBJECT, - "createdAt": "2026-08-01T00:00:00Z", - "expiresAt": "2026-08-10T00:00:00Z", - }), - ); - let (live, dropped) = in_force(vec![expiring], datetime!(2026-08-24 00:00 UTC)); - - assert!(live.is_empty()); - assert_eq!( - dropped[0].1, - Dropped::Expired { - at: datetime!(2026-08-10 00:00 UTC) - } - ); - } - - #[test] - fn a_revoked_vouch_is_not_in_force() { - let older = plain("older"); - let revoker = vouch( - "newer", - json!({ - "subject": SUBJECT, - "createdAt": "2026-08-02T00:00:00Z", - "revokes": uri("older"), - }), - ); - let (live, dropped) = in_force(vec![older, revoker], datetime!(2026-08-24 00:00 UTC)); - - assert_eq!(live.len(), 1); - assert_eq!(live[0].uri, uri("newer")); - assert_eq!(dropped[0].1, Dropped::Revoked { by: uri("newer") }); - } - - #[test] - fn an_expired_revoker_still_revokes() { - // The revoked vouch must not come back when the record that cancelled - // it lapses. This is the direction that must never fail. - let older = plain("older"); - let revoker = vouch( - "newer", - json!({ - "subject": SUBJECT, - "createdAt": "2026-08-02T00:00:00Z", - "expiresAt": "2026-08-03T00:00:00Z", - "revokes": uri("older"), - }), - ); - let (live, dropped) = in_force(vec![older, revoker], datetime!(2026-08-24 00:00 UTC)); - - assert!(live.is_empty()); - assert_eq!(dropped.len(), 2); - } - - #[test] - fn a_revocation_cannot_reach_into_another_repository() { - let mine = plain("mine"); - let theirs = Vouch::read( - "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa", - "at://did:plc:aaaaaaaaaaaaaaaaaaaaaaaa/bot.did.vouch/x", - &json!({ - "subject": SUBJECT, - "createdAt": "2026-08-02T00:00:00Z", - "revokes": uri("mine"), - }), - ) - .expect("a readable vouch"); - - let (live, dropped) = in_force(vec![mine, theirs], datetime!(2026-08-24 00:00 UTC)); - - assert_eq!(live.len(), 2); - assert!(dropped.is_empty()); - } -} diff --git a/crates/didbot-index/src/work.rs b/crates/didbot-index/src/work.rs deleted file mode 100644 index 5f1fc278..00000000 --- a/crates/didbot-index/src/work.rs +++ /dev/null @@ -1,1027 +0,0 @@ -//! Finding agents that are doing the same work. -//! -//! The question this answers: out of several hundred agents on several dozen -//! servers, which ones are working on the same thing right now? Nobody tells -//! the index that. All it has is a stream of short status lines. -//! -//! This is a port of `web/work.js`, which ran the same algorithm in every -//! browser tab. It moved here because the ids it hands out have to mean the -//! same thing to two different readers, and because a fingerprint built from -//! an agent's last dozen lines should not die when somebody closes a tab. -//! -//! # Why this algorithm -//! -//! The constraint was a CPU and a browser tab: no matrix library, no embedding -//! model, no quadratic pass over the population on every event. What fits is -//! the classic near-duplicate detection stack, which is streaming and -//! near-linear. The constraint has loosened — this is a server now — and the -//! algorithm has deliberately not changed with it, because the reasoning below -//! is about what the *data* supports and not about what the machine can afford. -//! -//! 1. **Shingle** each line into tokens, stopwords dropped, and fold it into -//! the agent's running profile: every existing weight decays a little, and -//! the words just used gain. What an agent is working on is the exponential -//! moving average of what it has been saying, not its last few lines. Say -//! the same thing ten times and the profile is sharp and stable; start -//! saying something else and it drifts across to the new subject over a -//! handful of updates, which is the behaviour a person watching would call -//! correct. -//! 2. **Weight** each token by inverse document frequency, kept as a running -//! count. Without this the phrases every job shares — the scaffolding words -//! a template puts around the interesting nouns — dominate the fingerprint -//! and every agent looks like every other. IDF is what makes the rare word -//! the one that decides. This is not a tuning preference; it is the -//! difference between clusters and one undifferentiated blob. -//! 3. **SimHash** the weighted profile into one 32-bit fingerprint, and -//! **band** it into four 8-bit keys. This is locality-sensitive hashing, and -//! its job here is recall, not judgement: two agents that collide in a band -//! are worth comparing properly, and everything else can be skipped without -//! being looked at. -//! 4. **Score the candidates by cosine similarity** over the profiles -//! themselves, held as arrays sorted by interned word id so a comparison is -//! a two-pointer merge rather than a walk of a hash map. A fingerprint is a -//! lossy summary and thirty-two bits is a blunt instrument — real topics sit -//! closer together than random noise but not by much, so a Hamming threshold -//! either misses most of them or merges unrelated ones. -//! 5. **Assign to leaders**, not to each other. Each cluster is represented by -//! one profile, and an agent joins the most similar leader above the -//! threshold or becomes a leader itself. -//! -//! The last step is the one worth defending, and it is the second thing here -//! that is the result of a failure. Joining agents to *each other* and taking -//! connected components is the obvious move and it collapses: single linkage -//! chains A to B to C until one cluster holds most of the population, however -//! tight the threshold. Leaders cost the same to compute, hold their identity -//! from pass to pass, and cannot chain. -//! -//! One thing did change in the port, and only because moving here changed what -//! the result is for: a leader's profile is now the mean of its members rather -//! than a live reference to the agent that opened it. See `Leader::profile` -//! for what that reference did to a shared id. -//! -//! # What it is not -//! -//! SimHash measures word overlap, not meaning. Two agents describing the same -//! work in different vocabularies will not be grouped, and that is the honest -//! limit of a method with no model behind it. The upgrade path — the same LSH -//! structure over embedding vectors, keyed by random hyperplane signs — is the -//! same code with a different fingerprint function. - -use std::collections::{BTreeMap, HashMap}; - -use serde::{Deserialize, Serialize}; - -/// Words that say nothing about which job an agent is on. -const STOPWORDS: &[&str] = &[ - "the", "a", "an", "and", "or", "of", "to", "in", "on", "at", "for", "with", "after", "before", - "left", "lands", "refused", "re", "is", "it", "this", "that", "from", "into", "over", "under", - "then", "now", "still", "again", -]; - -/// How many bands the fingerprint is split into for the index. -const BANDS: u32 = 4; -/// Bits per band. Four bands of eight is one 32-bit fingerprint. -const BAND_BITS: u32 = 8; -/// Profiles at least this alike are the same work. -const MIN_SIMILARITY: f64 = 0.45; -/// Leaders compared against when the band index turns nothing up. -const FALLBACK_LEADERS: usize = 160; -/// How much of an agent's profile survives each new line. -/// -/// At 0.82 a word said once is down to a tenth of its weight after twelve -/// updates, so a fingerprint has a memory of roughly the last dozen lines -/// without being pinned to any of them. Lower and clusters flicker; higher and -/// an agent that has moved on keeps its old company for too long. -const DECAY: f64 = 0.82; -/// Weights below this are dropped, so a profile stays a handful of words. -const PRUNE: f64 = 0.05; -/// A cluster smaller than this is not a group, it is a coincidence. -pub const MIN_CLUSTER: usize = 3; - -/// One group of agents doing the same work, as a reader sees it. -/// -/// Members are DIDs, so a client that has an agent can ask which group it is -/// in and a client that has a group can ask who is in it, without either of -/// them holding anything this index invented. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct WorkGroup { - /// A stable handle for this group; see [`WorkIndex::cluster`]. - pub id: String, - /// The two words this group says and other groups do not. - pub label: String, - /// The DID of the agent whose profile the group was opened on. - /// - /// Carried so the id is explicable rather than magic: it is derived from - /// exactly this string, disambiguated only if that agent has opened more - /// than one group that is still going. The agent may since have moved to - /// another group, or gone away, and the group keeps its identity either - /// way. - pub founder: String, - /// Every member's DID, sorted. - pub members: Vec, -} - -impl WorkGroup { - /// How many agents are in the group. - pub fn size(&self) -> usize { - self.members.len() - } -} - -/// Splits a status line into the words worth hashing. -pub fn tokenize(text: &str) -> Vec { - let mut out = Vec::new(); - for raw in text - .to_lowercase() - .split(|c: char| !c.is_ascii_alphanumeric()) - { - if raw.len() < 3 || STOPWORDS.contains(&raw) { - continue; - } - // Numbers are counters — "17 left" — and say nothing about the job. - if raw.bytes().all(|b| b.is_ascii_digit()) { - continue; - } - out.push(raw.to_owned()); - } - out -} - -/// FNV-1a. Cheap, well spread, and no dependency. -/// -/// Over bytes rather than UTF-16 code units, which is what the browser version -/// hashed. [`tokenize`] keeps only ASCII alphanumerics, so for every input this -/// function will ever see the two agree. -pub fn hash32(word: &str) -> u32 { - let mut hash: u32 = 0x811c_9dc5; - for byte in word.bytes() { - hash ^= u32::from(byte); - hash = hash.wrapping_mul(0x0100_0193); - } - hash -} - -/// SimHash of a weighted token list. -/// -/// Each token votes on all 32 bits with its weight; the sign of each column -/// becomes the fingerprint bit. Two texts built from the same rare words agree -/// on most of their columns, however the words are ordered. -pub fn simhash<'a, I: IntoIterator>(tokens: I) -> u32 { - let mut columns = [0.0f64; 32]; - for (token, weight) in tokens { - let hash = hash32(token); - for (bit, column) in columns.iter_mut().enumerate() { - *column += if (hash >> bit) & 1 == 1 { - weight - } else { - -weight - }; - } - } - let mut fingerprint = 0u32; - for (bit, column) in columns.iter().enumerate() { - if *column > 0.0 { - fingerprint |= 1 << bit; - } - } - fingerprint -} - -/// Population count of the difference: how far apart two fingerprints are. -pub fn hamming(a: u32, b: u32) -> u32 { - (a ^ b).count_ones() -} - -/// One agent's profile, laid out for comparison. -/// -/// `ids` is sorted, and `values` runs beside it, so a similarity is the merge -/// step of a merge sort: one pass, no hashing, no allocation. -#[derive(Debug, Clone, Default, PartialEq)] -struct Profile { - /// Raw decayed counts, before inverse document frequency. - counts: BTreeMap, - /// Those counts multiplied by inverse document frequency. - /// - /// Kept beside the raw counts because it is what a group's centroid is - /// averaged over: two profiles can only be added together once they are in - /// the same weighting. - weighted: BTreeMap, - /// Interned word ids, ascending. - ids: Vec, - /// The weight of each, in the same order. - values: Vec, - /// The Euclidean norm of `values`, never zero. - norm: f64, - /// The fingerprint the bands are cut from. - fingerprint: u32, - /// Which band buckets this profile currently sits in. - keys: Vec, -} - -impl Profile { - /// Fills in everything derived from [`Self::weighted`]. - /// - /// One function, called for an agent's profile and for a group's centroid - /// alike, because the two are compared against each other and a difference - /// in how they were laid out would be a difference in what the comparison - /// meant. - fn seal(&mut self, tokens: &mut Interner) { - self.fingerprint = simhash( - self.weighted - .iter() - .map(|(token, value)| (token.as_str(), *value)), - ); - let mut pairs: Vec<(u32, f64)> = self - .weighted - .iter() - .map(|(token, value)| (tokens.intern(token), *value)) - .collect(); - pairs.sort_by_key(|(id, _)| *id); - let square: f64 = pairs.iter().map(|(_, value)| value * value).sum(); - self.ids = pairs.iter().map(|(id, _)| *id).collect(); - self.values = pairs.into_iter().map(|(_, value)| value).collect(); - // Never zero: a profile with no words would otherwise divide every - // similarity by nothing. - self.norm = square.sqrt().max(f64::MIN_POSITIVE); - self.keys = band_keys(self.fingerprint); - } - - /// Cosine similarity of two profiles. - fn similarity(&self, other: &Self) -> f64 { - let (mut i, mut j) = (0usize, 0usize); - let mut dot = 0.0; - while i < self.ids.len() && j < other.ids.len() { - match self.ids[i].cmp(&other.ids[j]) { - std::cmp::Ordering::Equal => { - dot += self.values[i] * other.values[j]; - i += 1; - j += 1; - } - std::cmp::Ordering::Less => i += 1, - std::cmp::Ordering::Greater => j += 1, - } - } - dot / (self.norm * other.norm) - } -} - -/// Words, interned. -/// -/// Every profile is compared against several leaders on every pass, and a -/// string-keyed comparison turns that into hundreds of thousands of hash -/// lookups. Interning once, at the point a word is first seen, turns the same -/// comparison into integer arithmetic over sorted arrays. -#[derive(Debug, Default)] -struct Interner { - ids: HashMap, -} - -impl Interner { - /// The id for a word, assigning one if this is the first sighting. - fn intern(&mut self, token: &str) -> u32 { - if let Some(id) = self.ids.get(token) { - return *id; - } - // `u32` because the ids are compared millions of times and the - // vocabulary of a status feed is thousands of words, not billions. - let id = u32::try_from(self.ids.len()).unwrap_or(u32::MAX); - self.ids.insert(token.to_owned(), id); - id - } -} - -/// A cluster, as the index carries it between passes. -#[derive(Debug, Clone)] -struct Leader { - /// The stable handle handed out to readers. - id: String, - /// The agent the group was opened on. - founder: String, - /// The profile the group is compared against: the mean of its members. - /// - /// **This is the one deliberate divergence from `web/work.js`**, and the - /// stable-id requirement is what forces it. There, a leader held a - /// reference to the founding agent's own profile object, so the group's - /// profile silently followed that one agent. The consequence is invisible - /// in a tab and fatal to a shared id: when the founder changes subject, the - /// group changes subject with it under the same id, the members who did not - /// move fail to match it and open a new group, and a link to "the group - /// working on the cart" now opens a group of one working on something else. - /// - /// Averaging over the members instead makes the profile mean what its name - /// says. A group with several members moves slowly and only when its - /// members do, an agent that leaves takes only its own share with it, and - /// nothing else about the algorithm changes: assignment is still one round - /// against leaders, so it still cannot chain. - profile: Profile, - /// Members assigned on the most recent pass, sorted. - members: Vec, -} - -/// The streaming index: what each agent is working on, and who agrees. -#[derive(Debug, Default)] -pub struct WorkIndex { - agents: BTreeMap, - /// Band key to the agents sitting in that bucket. - buckets: HashMap>, - /// Token to how many lines it has appeared in. - frequency: HashMap, - /// The shared vocabulary every profile's ids are drawn from. - tokens: Interner, - /// How many lines have been seen at all. - lines: usize, - /// Cluster leaders, carried between passes so groups hold their identity. - leaders: Vec, -} - -impl WorkIndex { - /// An index with nothing in it. - pub fn new() -> Self { - Self::default() - } - - /// How many agents have profiles. - pub fn agents(&self) -> usize { - self.agents.len() - } - - /// How many status lines have been folded in. - pub fn lines(&self) -> usize { - self.lines - } - - /// How much one token's vote counts. - /// - /// Inverse document frequency: a word in every third line is worth almost - /// nothing, a word in one line in fifty decides the fingerprint. - fn weight(&self, token: &str) -> f64 { - let seen = self.frequency.get(token).copied().unwrap_or(0); - #[allow(clippy::cast_precision_loss)] - let ratio = (self.lines + 1) as f64 / (seen + 1) as f64; - ratio.ln() + 0.2 - } - - /// Records one status line against an agent. - pub fn observe(&mut self, did: &str, text: &str) { - let line = tokenize(text); - - self.lines += 1; - // Per line, not per occurrence: document frequency counts documents. - let mut once: Vec<&String> = line.iter().collect(); - once.sort(); - once.dedup(); - for token in once { - *self.frequency.entry(token.clone()).or_insert(0) += 1; - } - - let mut profile = self.agents.remove(did).unwrap_or_default(); - self.unindex(did, &profile); - - // Everything the agent has said before fades by a fixed fraction, and - // what it just said is added on top. - profile.counts.retain(|_, weight| { - *weight *= DECAY; - *weight >= PRUNE - }); - for token in &line { - *profile.counts.entry(token.clone()).or_insert(0.0) += 1.0; - } - - // Re-weighted by inverse document frequency once, here, so neither the - // fingerprint nor a similarity has to do it again. - profile.weighted = profile - .counts - .iter() - .map(|(token, count)| (token.clone(), count * self.weight(token))) - .collect(); - profile.seal(&mut self.tokens); - - for key in &profile.keys { - self.buckets.entry(*key).or_default().push(did.to_owned()); - } - self.agents.insert(did.to_owned(), profile); - } - - /// Takes an agent out of every band bucket it sits in. - fn unindex(&mut self, did: &str, profile: &Profile) { - for key in &profile.keys { - if let Some(bucket) = self.buckets.get_mut(key) { - bucket.retain(|member| member != did); - if bucket.is_empty() { - self.buckets.remove(key); - } - } - } - } - - /// Drops an agent that has ended. - pub fn forget(&mut self, did: &str) { - if let Some(profile) = self.agents.remove(did) { - self.unindex(did, &profile); - } - } - - /// Groups the population into work clusters. - /// - /// Each agent joins the closest leader within the threshold, or becomes a - /// leader. Leaders survive between passes, so an agent that keeps saying - /// the same thing keeps its group. - /// - /// # The ids - /// - /// A group's id is [`hash32`] of the DID of the agent it was opened on, - /// in hex. That gives the three properties a shared id has to have: two - /// clients asking this index get the same answer, because there is one - /// index; a group keeps its id across passes, because its leader does; and - /// the id is derived from a published identifier rather than from a - /// counter, so it is explicable and reproducible rather than an artefact of - /// the order leaders happened to be created in — which is exactly what made - /// the browser version's ids meaningless to a second reader. - /// - /// What it is not is durable across a restart of this index. The - /// population is re-observed from scratch, different agents open the - /// groups, and the ids differ. That is honest: after a restart these are - /// genuinely different groupings, and an id that survived would be claiming - /// otherwise. - pub fn cluster(&mut self) -> Vec { - for leader in &mut self.leaders { - leader.members.clear(); - } - - // Candidate leaders are looked up through the band index first, so this - // stays sub-linear in the number of leaders as the population grows. - let mut by_key: HashMap> = HashMap::new(); - for (index, leader) in self.leaders.iter().enumerate() { - for key in &leader.profile.keys { - by_key.entry(*key).or_default().push(index); - } - } - - // `BTreeMap` keys, so the pass does not depend on hash order and the - // same observations produce the same clusters. - let dids: Vec = self.agents.keys().cloned().collect(); - for did in dids { - let Some(profile) = self.agents.get(&did) else { - continue; - }; - let mut best: Option = None; - let mut best_score = MIN_SIMILARITY; - let mut seen: Vec = Vec::new(); - for key in &profile.keys { - for index in by_key.get(key).into_iter().flatten() { - if seen.contains(index) { - continue; - } - seen.push(*index); - let score = profile.similarity(&self.leaders[*index].profile); - if score > best_score { - best_score = score; - best = Some(*index); - } - } - } - // The bands are a recall device and they do miss: a profile can be - // plainly about the same subject as a leader and share no eight-bit - // slice with it. When they turn nothing up, the leaders are few - // enough to look at directly rather than open a new group on no - // evidence. - if best.is_none() && self.leaders.len() <= FALLBACK_LEADERS { - for index in 0..self.leaders.len() { - if seen.contains(&index) { - continue; - } - let score = profile.similarity(&self.leaders[index].profile); - if score > best_score { - best_score = score; - best = Some(index); - } - } - } - - let index = match best { - Some(index) => index, - None => { - let index = self.leaders.len(); - for key in &profile.keys { - by_key.entry(*key).or_default().push(index); - } - let id = self.mint_id(&did); - self.leaders.push(Leader { - id, - founder: did.clone(), - profile: profile.clone(), - members: Vec::new(), - }); - index - } - }; - self.leaders[index].members.push(did); - } - - self.leaders.retain(|leader| !leader.members.is_empty()); - self.recentre(); - - let mut groups: Vec = self - .leaders - .iter() - .filter(|leader| leader.members.len() >= MIN_CLUSTER) - .map(|leader| WorkGroup { - id: leader.id.clone(), - label: self.label(&leader.members), - founder: leader.founder.clone(), - members: leader.members.clone(), - }) - .collect(); - // Biggest first, and by id where two are the same size, so a client - // taking the top eight takes the same eight as every other client. - groups.sort_by(|a, b| b.size().cmp(&a.size()).then_with(|| a.id.cmp(&b.id))); - groups - } - - /// A group id that no live group already answers to. - /// - /// [`hash32`] of the founder's DID, and then of the DID with a bump on the - /// end until it is free. Two things need that. An agent can open a second - /// group while the first is still going — it drifts away from a group it - /// founded, matches nothing, and opens another, while the members it left - /// behind keep the original alive — and two live groups answering to one id - /// would make a link ambiguous, which is the entire property these ids - /// exist to have. Thirty-two bits also collide between different DIDs - /// eventually, and the same loop covers it. - fn mint_id(&self, founder: &str) -> String { - let mut id = format!("{:08x}", hash32(founder)); - let mut bump = 0u32; - while self.leaders.iter().any(|leader| leader.id == id) { - bump += 1; - id = format!("{:08x}", hash32(&format!("{founder}#{bump}"))); - } - id - } - - /// Recomputes every leader's profile as the mean of its members'. - /// - /// After assignment, not before: a centroid is a summary of who is in the - /// group, and computing it from last pass's membership would leave every - /// group one pass behind the population it is describing. Recomputed rather - /// than updated incrementally because inverse document frequency moves - /// under it — a word that was rare when the group opened may be - /// commonplace by now, and a stored centroid would still be scoring it as - /// rare. - fn recentre(&mut self) { - for index in 0..self.leaders.len() { - let mut centroid: BTreeMap = BTreeMap::new(); - let members = self.leaders[index].members.len(); - for did in &self.leaders[index].members { - let Some(profile) = self.agents.get(did) else { - continue; - }; - for (token, value) in &profile.weighted { - *centroid.entry(token.clone()).or_insert(0.0) += value; - } - } - #[allow(clippy::cast_precision_loss)] - let divisor = members.max(1) as f64; - for value in centroid.values_mut() { - *value /= divisor; - } - // A word one member of a large group used once says nothing about - // the group, and carrying every such word makes a centroid that - // matches everything a little. The same threshold the profiles - // themselves are pruned at keeps it a handful of words. - centroid.retain(|_, value| *value >= PRUNE); - let mut profile = Profile { - weighted: centroid, - ..Profile::default() - }; - profile.seal(&mut self.tokens); - self.leaders[index].profile = profile; - } - } - - /// Names a group after the words its members are weighted toward. - /// - /// Weighted by both the profile and the inverse document frequency, so the - /// name is what this group says and other groups do not. - fn label(&self, members: &[String]) -> String { - let mut counts: BTreeMap<&str, f64> = BTreeMap::new(); - for did in members { - let Some(profile) = self.agents.get(did) else { - continue; - }; - for (token, weight) in &profile.counts { - *counts.entry(token.as_str()).or_insert(0.0) += weight * self.weight(token); - } - } - let mut ranked: Vec<(&str, f64)> = counts.into_iter().collect(); - // By score, then alphabetically, so two words that tie do not order - // themselves differently on two passes over the same data. - ranked.sort_by(|a, b| { - b.1.partial_cmp(&a.1) - .unwrap_or(std::cmp::Ordering::Equal) - .then_with(|| a.0.cmp(b.0)) - }); - ranked - .into_iter() - .take(2) - .map(|(token, _)| token) - .collect::>() - .join(" · ") - } -} - -/// The band buckets one fingerprint falls in. -/// -/// The band number is packed into the key alongside the slice, so a slice of -/// `0x3f` in band one and the same slice in band two are different buckets; -/// without that the index would be one 8-bit table pretending to be four. -fn band_keys(fingerprint: u32) -> Vec { - (0..BANDS) - .map(|band| { - let slice = (fingerprint >> (band * BAND_BITS)) & ((1 << BAND_BITS) - 1); - (band << BAND_BITS) | slice - }) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn tokenizing_drops_what_says_nothing_about_the_job() { - assert_eq!( - tokenize("Reading the cart, 17 left after the coupon"), - vec!["reading", "cart", "coupon"] - ); - } - - #[test] - fn a_band_key_carries_which_band_it_came_from() { - // Every band of an all-ones fingerprint holds the same eight bits, and - // four buckets is what has to come out of it, not one. - let keys = band_keys(u32::MAX); - let mut unique = keys.clone(); - unique.sort_unstable(); - unique.dedup(); - assert_eq!(keys.len(), 4); - assert_eq!(unique.len(), 4); - } - - #[test] - fn the_same_words_hash_to_the_same_fingerprint_whatever_the_order() { - let a = simhash([("cart", 1.0), ("coupon", 2.0), ("receipt", 0.5)]); - let b = simhash([("receipt", 0.5), ("cart", 1.0), ("coupon", 2.0)]); - assert_eq!(a, b); - } - - #[test] - fn near_texts_are_nearer_than_unrelated_ones() { - let near = |words: &[&str]| simhash(words.iter().map(|word| (*word, 1.0))); - let cart = near(&["cart", "coupon", "checkout", "receipt"]); - let cart2 = near(&["cart", "coupon", "checkout", "tax"]); - let other = near(&["ranking", "autocomplete", "synonym", "typo"]); - assert!( - hamming(cart, cart2) < hamming(cart, other), - "one word apart must be closer than a different subject" - ); - } - - /// Feeds one agent a line several times, which is what makes a profile - /// sharp enough to cluster on. - fn say(index: &mut WorkIndex, did: &str, text: &str, times: usize) { - for _ in 0..times { - index.observe(did, text); - } - } - - #[test] - fn agents_saying_the_same_thing_land_in_one_group() { - let mut index = WorkIndex::new(); - for i in 0..5 { - say( - &mut index, - &format!("did:web:cart-{i}"), - "walking the coupon code through the cart", - 6, - ); - } - for i in 0..5 { - say( - &mut index, - &format!("did:web:rank-{i}"), - "retraining the ranking model on click signal", - 6, - ); - } - - let groups = index.cluster(); - assert_eq!(groups.len(), 2, "two subjects, two groups: {groups:?}"); - for group in &groups { - assert_eq!(group.size(), 5); - let prefix = &group.members[0][.."did:web:cart".len()]; - assert!( - group.members.iter().all(|did| did.starts_with(prefix)), - "a group must not mix subjects: {group:?}" - ); - } - } - - #[test] - fn a_group_keeps_its_id_across_passes() { - let mut index = WorkIndex::new(); - for i in 0..4 { - say( - &mut index, - &format!("did:web:ledger-{i}"), - "reconciling the invoice against the usage meter", - 6, - ); - } - let first = index.cluster(); - assert_eq!(first.len(), 1); - - // A fifth agent joins, and everyone says the same thing again. - say( - &mut index, - "did:web:ledger-4", - "reconciling the invoice against the usage meter", - 6, - ); - for i in 0..4 { - say( - &mut index, - &format!("did:web:ledger-{i}"), - "reconciling the invoice against the usage meter", - 1, - ); - } - let second = index.cluster(); - assert_eq!(second.len(), 1); - assert_eq!( - first[0].id, second[0].id, - "the group grew; it did not become a different group" - ); - assert_eq!(second[0].size(), 5); - } - - #[test] - fn an_id_is_derived_from_the_agent_the_group_was_opened_on() { - let mut index = WorkIndex::new(); - for i in 0..3 { - say( - &mut index, - &format!("did:web:queue-{i}"), - "draining the dead letter queue by partition key", - 6, - ); - } - let groups = index.cluster(); - assert_eq!(groups.len(), 1); - assert_eq!(groups[0].id, format!("{:08x}", hash32(&groups[0].founder))); - } - - #[test] - fn two_groups_never_answer_to_one_id() { - // The case: an agent opens a group, drifts off it while the members it - // left keep it alive, and opens another. Both are live, and a link has - // to name one of them. - let mut index = WorkIndex::new(); - for i in 0..4 { - say( - &mut index, - &format!("did:web:aaa-{i}"), - "walking the coupon code through the cart", - 8, - ); - } - index.cluster(); - // `aaa-0` opened the group; it now says something nothing else says. - say( - &mut index, - "did:web:aaa-0", - "recalibrating the sourdough hygrometer", - 14, - ); - for i in 1..4 { - say( - &mut index, - &format!("did:web:aaa-{i}"), - "walking the coupon code through the cart", - 2, - ); - } - // And two more join it, so its new group is reportable. - for i in 0..2 { - say( - &mut index, - &format!("did:web:zzz-{i}"), - "recalibrating the sourdough hygrometer", - 8, - ); - } - let groups = index.cluster(); - let mut ids: Vec<&str> = groups.iter().map(|group| group.id.as_str()).collect(); - let total = ids.len(); - ids.sort_unstable(); - ids.dedup(); - assert_eq!( - ids.len(), - total, - "two live groups share an id: {:?}", - groups - .iter() - .map(|group| (&group.id, &group.label)) - .collect::>() - ); - assert!( - total >= 2, - "the test needs both groups to be live: {groups:?}" - ); - } - - #[test] - fn a_group_smaller_than_three_is_a_coincidence_and_is_not_reported() { - let mut index = WorkIndex::new(); - say(&mut index, "did:web:alone", "polishing the plan picker", 6); - say(&mut index, "did:web:pair", "sweeping the synonym list", 6); - assert!(index.cluster().is_empty()); - } - - #[test] - fn an_agent_that_changes_subject_changes_group() { - let mut index = WorkIndex::new(); - for i in 0..4 { - say( - &mut index, - &format!("did:web:cart-{i}"), - "walking the coupon code through the cart", - 8, - ); - } - for i in 0..4 { - say( - &mut index, - &format!("did:web:rank-{i}"), - "retraining the ranking model on click signal", - 8, - ); - } - let before = index.cluster(); - let cart_group = before - .iter() - .find(|group| group.members.iter().any(|did| did == "did:web:cart-0")) - .expect("cart-0 is in a group") - .id - .clone(); - - // Twelve lines on the new subject, which is the decay's stated memory. - say( - &mut index, - "did:web:cart-0", - "retraining the ranking model on click signal", - 12, - ); - let after = index.cluster(); - let moved = after - .iter() - .find(|group| group.members.iter().any(|did| did == "did:web:cart-0")) - .expect("cart-0 is still in a group"); - assert_ne!( - moved.id, cart_group, - "an agent that has moved on must not keep its old company" - ); - assert!(moved.members.iter().any(|did| did == "did:web:rank-0")); - } - - #[test] - fn a_group_survives_the_agent_it_was_opened_on_walking_away() { - // The case the centroid exists for. In the browser version the leader - // held the founder's own profile, so this turned the ledger group into - // a ranking group under the ledger group's id. - let mut index = WorkIndex::new(); - for i in 0..4 { - say( - &mut index, - &format!("did:web:ledger-{i}"), - "reconciling the invoice against the usage meter", - 8, - ); - } - for i in 0..4 { - say( - &mut index, - &format!("did:web:rank-{i}"), - "retraining the ranking model on click signal", - 8, - ); - } - let before = index.cluster(); - let ledger = before - .iter() - .find(|group| group.members.iter().any(|did| did == "did:web:ledger-1")) - .expect("ledger-1 is in a group") - .clone(); - assert_eq!( - ledger.founder, "did:web:ledger-0", - "the first did alphabetically opens the group" - ); - - say( - &mut index, - "did:web:ledger-0", - "retraining the ranking model on click signal", - 12, - ); - let after = index.cluster(); - let still = after - .iter() - .find(|group| group.members.iter().any(|did| did == "did:web:ledger-1")) - .expect("the rest of the ledger group is still a group"); - assert_eq!( - still.id, ledger.id, - "the work continued, so the group must keep its identity" - ); - assert!( - !still.members.iter().any(|did| did == "did:web:ledger-0"), - "the founder left: {still:?}" - ); - assert!( - still.label.contains("invoice") || still.label.contains("meter"), - "the group is still about the ledger: {:?}", - still.label - ); - } - - #[test] - fn a_forgotten_agent_leaves_its_group() { - let mut index = WorkIndex::new(); - for i in 0..4 { - say( - &mut index, - &format!("did:web:gone-{i}"), - "backfilling the schema registry", - 6, - ); - } - assert_eq!(index.cluster()[0].size(), 4); - index.forget("did:web:gone-0"); - assert_eq!(index.agents(), 3); - let groups = index.cluster(); - assert_eq!(groups[0].size(), 3); - assert!(!groups[0].members.iter().any(|did| did == "did:web:gone-0")); - } - - #[test] - fn a_group_is_named_after_what_it_says_and_others_do_not() { - let mut index = WorkIndex::new(); - // A phrase every agent shares, so the label has to look past it. - for i in 0..4 { - say( - &mut index, - &format!("did:web:tax-{i}"), - "working on the tax exemption for the invoice", - 6, - ); - } - for i in 0..4 { - say( - &mut index, - &format!("did:web:cart-{i}"), - "working on the abandoned cart for the coupon", - 6, - ); - } - let groups = index.cluster(); - let labels: Vec<&str> = groups.iter().map(|group| group.label.as_str()).collect(); - assert!( - labels.iter().all(|label| !label.contains("working")), - "the word both groups share must not name either of them: {labels:?}" - ); - } - - #[test] - fn the_same_observations_produce_the_same_groups_twice() { - let feed = |index: &mut WorkIndex| { - for i in 0..6 { - say( - index, - &format!("did:web:a-{i}"), - "tracing how the refresh token is minted", - 5, - ); - say( - index, - &format!("did:web:b-{i}"), - "widening the autocomplete typo tolerance", - 5, - ); - } - }; - let mut first = WorkIndex::new(); - feed(&mut first); - let mut second = WorkIndex::new(); - feed(&mut second); - assert_eq!(first.cluster(), second.cluster()); - } -} diff --git a/crates/didbot-index/tests/clustering.rs b/crates/didbot-index/tests/clustering.rs deleted file mode 100644 index bab92b51..00000000 --- a/crates/didbot-index/tests/clustering.rs +++ /dev/null @@ -1,207 +0,0 @@ -//! The work clustering, against a corpus of the prose a swarm actually writes. -//! -//! The unit tests beside the algorithm run it on hand-written lines, which says -//! it does what it says and nothing about whether the thing it does is useful. -//! This says the useful part: given the status lines a population of synthetic -//! agents writes — templates filled from a project's own vocabulary, moving -//! through phases, with the scaffolding words every project shares — the -//! grouping recovers who is working on what. -//! -//! The lines are composed here rather than driven through a server, because -//! what is being tested is the text and not the transport. The composition is -//! the same one the swarm performs: a phase template with `{part}` and -//! `{other}` filled from one project. See `corpus` for why the vocabulary is a -//! copy rather than an import. - -mod corpus; - -use std::collections::{HashMap, HashSet}; - -use didbot_index::work::WorkIndex; - -use corpus::{Phase, Project, PHASES, PROJECTS}; - -/// How many agents sit on each part. -const TEAM: usize = 6; -/// How many lines each of them writes. -const LINES: usize = 14; -/// How many project-and-part teams the corpus holds. -const TEAMS: usize = 8; - -/// A linear congruential generator, so a failure is reproducible. -/// -/// Written out rather than pulled in: this test needs a deterministic sequence -/// of small integers and nothing else, and a seeded generator from a -/// dependency would still have to be pinned to keep the corpus stable. -struct Rng(u64); - -impl Rng { - fn next(&mut self, bound: usize) -> usize { - self.0 = self - .0 - .wrapping_mul(6_364_136_223_846_793_005) - .wrapping_add(1_442_695_040_888_963_407); - usize::try_from(self.0 >> 33).unwrap_or(0) % bound.max(1) - } -} - -/// One status line, composed the way the swarm composes one. -fn compose(rng: &mut Rng, project: &Project, part: &str, phase: &Phase) -> String { - let mut other = part; - for _ in 0..6 { - if other != part { - break; - } - other = if rng.next(2) == 0 { - project.terms[rng.next(project.terms.len())] - } else { - project.parts[rng.next(project.parts.len())] - }; - } - phase.lines[rng.next(phase.lines.len())] - .replace("{part}", part) - .replace("{other}", other) - .replace("{n}", &(rng.next(12) + 1).to_string()) -} - -/// The teams the corpus is built from: a project and one of its parts. -/// -/// One part from each of the first `TEAMS` projects, so the corpus is spread -/// across subjects rather than concentrated in one. -fn teams() -> Vec<(usize, usize)> { - (0..TEAMS.min(PROJECTS.len())) - .map(|project| (project, project % PROJECTS[project].parts.len())) - .collect() -} - -/// Builds the corpus and folds it into an index. -/// -/// Interleaved rather than agent by agent, because that is the order a real -/// feed arrives in and inverse document frequency is a running count: an index -/// fed one team at a time would score the first team's words as rare and the -/// last team's as commonplace, purely from the order. -fn indexed() -> (WorkIndex, HashMap) { - let mut rng = Rng(0x5eed_1234); - let teams = teams(); - let mut index = WorkIndex::new(); - let mut team_of = HashMap::new(); - - for team in 0..teams.len() { - for member in 0..TEAM { - team_of.insert(format!("did:web:t{team}-a{member}"), team); - } - } - - for line in 0..LINES { - for (team, (project, part)) in teams.iter().enumerate() { - let project = &PROJECTS[*project]; - let part = project.parts[*part]; - // The whole team moves through the phases together, which is what - // a real team of agents on one part looks like. - let phase = &PHASES[(line / 4) % PHASES.len()]; - for member in 0..TEAM { - let text = compose(&mut rng, project, part, phase); - index.observe(&format!("did:web:t{team}-a{member}"), &text); - } - } - } - (index, team_of) -} - -#[test] -fn the_grouping_recovers_who_is_working_on_what() { - let (mut index, team_of) = indexed(); - let population = team_of.len(); - - let groups = index.cluster(); - assert!( - !groups.is_empty(), - "a corpus of {population} agents on {TEAMS} parts must produce groups" - ); - - // Purity: every group is dominated by one team. This is the property that - // matters — a group mixing two subjects is worse than no group, because a - // reader acts on it. - let mut grouped = 0usize; - let mut recovered = HashSet::new(); - for group in &groups { - let mut counts: HashMap = HashMap::new(); - for did in &group.members { - let team = team_of.get(did).copied().unwrap_or(usize::MAX); - *counts.entry(team).or_insert(0) += 1; - } - let (team, share) = counts - .into_iter() - .max_by_key(|(_, count)| *count) - .unwrap_or((usize::MAX, 0)); - #[allow(clippy::cast_precision_loss)] - let purity = share as f64 / group.members.len() as f64; - assert!( - purity >= 0.8, - "a group must be about one thing: {purity:.2} of {:?} is team {team}, label {:?}", - group.members, - group.label - ); - grouped += group.members.len(); - recovered.insert(team); - } - - // Recall: most of the population is placed somewhere, and most teams are - // found. Not all, and that is the honest limit — an agent whose last dozen - // lines happened to be mostly about `{other}` reads as a different - // subject, because on the evidence available it is one. - #[allow(clippy::cast_precision_loss)] - let placed = grouped as f64 / population as f64; - assert!( - placed >= 0.6, - "only {placed:.2} of the population was grouped, in {} groups", - groups.len() - ); - assert!( - recovered.len() >= TEAMS / 2, - "only {} of {TEAMS} teams were recovered", - recovered.len() - ); -} - -#[test] -fn a_group_is_named_after_the_work_and_not_the_template() { - let (mut index, _) = indexed(); - let groups = index.cluster(); - assert!(!groups.is_empty()); - - // The scaffolding words the phase templates wrap around every part must - // not name a group: that is what inverse document frequency is in the - // algorithm for, and without it every group here is called the same thing. - let scaffolding = ["looking", "wiring", "adding", "checking", "reading"]; - for group in &groups { - assert!( - !group.label.is_empty(), - "a group with no name is not usable in a panel" - ); - for word in scaffolding { - assert!( - !group.label.contains(word), - "{:?} names the template, not the work", - group.label - ); - } - } - - // And two groups must not answer to the same name, or the panel is a list - // of identical rows. - let mut labels: Vec<&str> = groups.iter().map(|group| group.label.as_str()).collect(); - labels.sort_unstable(); - let total = labels.len(); - labels.dedup(); - assert_eq!(labels.len(), total, "two groups share a name"); -} - -#[test] -fn the_same_corpus_produces_the_same_groups_and_the_same_ids() { - // The reason the algorithm moved behind the index: two readers asking - // about the same feed get the same answer. - let (mut one, _) = indexed(); - let (mut two, _) = indexed(); - assert_eq!(one.cluster(), two.cluster()); -} diff --git a/crates/didbot-index/tests/corpus/mod.rs b/crates/didbot-index/tests/corpus/mod.rs deleted file mode 100644 index 687669ce..00000000 --- a/crates/didbot-index/tests/corpus/mod.rs +++ /dev/null @@ -1,257 +0,0 @@ -//! The vocabulary a synthetic swarm writes with, as a fixed corpus. -//! -//! This is a third copy. `didbot-swarm` holds the one the real write -//! path exercises, `web/mock.js` holds the one the canvas runs on with no -//! backend, and this one feeds the clustering test beside it. Each exists -//! because taking it from either of the others would mean a dependency the -//! taker should not have: the indexer reads records off a server and has no -//! business linking the generator that produced them. -//! -//! Being a copy, it is a snapshot. It does not track the swarm's vocabulary as -//! that grows, and it does not need to: what the test beside it asks is -//! whether clustering recovers structure from realistic prose, and prose that -//! was realistic stays realistic. Only the structure matters, not the words. - -// Kept structurally identical to the vocabulary it was copied from, so the two -// can be compared by eye. The clustering test reads the templates and nothing -// else, which leaves the identifier and emoji fields unread here. -#![allow(dead_code)] - -/// A project, and the words that only come up while working on it. -pub struct Project { - /// Short identifier, used in log lines. - pub id: &'static str, - /// The parts it is built from. An agent settles on one of these. - pub parts: &'static [&'static str], - /// Terms that belong to the project rather than to any one part. - pub terms: &'static [&'static str], -} - -/// A stage of work, and how an agent talks during it. -pub struct Phase { - /// Short identifier, used in log lines. - pub id: &'static str, - /// Templates, with `{part}`, `{other}` and `{n}` to fill in. - pub lines: &'static [&'static str], - /// What an agent reaches for while it is here. - pub emoji: &'static [&'static str], -} - -/// Ten projects, which is enough that a few hundred agents overlap without -/// every one of them being on the same thing. -pub const PROJECTS: &[Project] = &[ - Project { - id: "checkout", - parts: &[ - "cart", - "coupon code", - "tax calculation", - "payment intent", - "receipt email", - "address form", - ], - terms: &[ - "idempotency key", - "currency rounding", - "abandoned cart", - "gift card", - ], - }, - Project { - id: "auth", - parts: &[ - "login form", - "session store", - "refresh token", - "password reset", - "device list", - "signup flow", - ], - terms: &["token rotation", "rate limit", "audit log", "magic link"], - }, - Project { - id: "search", - parts: &[ - "query parser", - "ranking model", - "autocomplete", - "synonym list", - "search index", - "results page", - ], - terms: &[ - "stop word", - "typo tolerance", - "faceted filter", - "click signal", - ], - }, - Project { - id: "billing", - parts: &[ - "invoice", - "proration", - "subscription", - "dunning email", - "usage meter", - "plan picker", - ], - terms: &[ - "grace period", - "webhook retry", - "tax exemption", - "trial period", - ], - }, - Project { - id: "ingest", - parts: &[ - "consumer group", - "backfill job", - "dedupe step", - "schema registry", - "dead letter queue", - "batch writer", - ], - terms: &[ - "consumer lag", - "replay window", - "watermark", - "partition key", - ], - }, - Project { - id: "mobile", - parts: &[ - "onboarding screen", - "push notification", - "deep link", - "offline cache", - "photo picker", - "settings page", - ], - terms: &[ - "cold start", - "background refresh", - "permission prompt", - "crash report", - ], - }, - Project { - id: "infra", - parts: &[ - "terraform module", - "autoscaler", - "canary deploy", - "alert rule", - "log pipeline", - "secret rotation", - ], - terms: &[ - "blast radius", - "rollback plan", - "noisy alert", - "capacity headroom", - ], - }, - Project { - id: "models", - parts: &[ - "embedding job", - "eval set", - "prompt template", - "tokenizer", - "feature store", - "drift monitor", - ], - terms: &[ - "holdout split", - "context window", - "label noise", - "inference cost", - ], - }, - Project { - id: "docs", - parts: &[ - "changelog", - "migration guide", - "api reference", - "quickstart", - "example project", - "error catalogue", - ], - terms: &["broken link", "code sample", "screenshot", "release note"], - }, - Project { - id: "performance", - parts: &[ - "n+1 query", - "cache layer", - "bundle size", - "image pipeline", - "connection pool", - "startup path", - ], - terms: &["hit rate", "tail latency", "memory ceiling", "flame graph"], - }, -]; - -/// The stages, in the order work goes through them. -pub const PHASES: &[Phase] = &[ - Phase { - id: "reading", - lines: &[ - "reading through the {part}", - "tracing how the {part} is wired up", - "mapping the {part} end to end", - "checking whether the {part} already handles {other}", - "looking for where {other} is set on the {part}", - ], - emoji: &["🔍", "🧭", "📘", "🗺️", "🪞", "🕯️", "📐", "🔬"], - }, - Phase { - id: "building", - lines: &[ - "adding {other} handling to the {part}", - "splitting the {part} into smaller pieces", - "moving the {part} behind a feature flag", - "rewriting the {part} so {other} is not special-cased", - "wiring the {part} up to {other}", - ], - emoji: &["🛠️", "🔧", "🧱", "🪵", "🧵", "🔩", "🪛", "📦"], - }, - Phase { - id: "debugging", - lines: &[ - "the {part} times out once {other} is involved", - "narrowed the {part} failure down to {other}", - "the {part} returns stale data after {other} changes", - "reproduced the {part} bug locally", - "{other} and the {part} disagree about ordering", - ], - emoji: &["🔥", "🚨", "🐛", "💥", "🪤", "🕸️", "⚡", "🧯"], - }, - Phase { - id: "testing", - lines: &[ - "writing tests for the {part}", - "{n} tests still failing in the {part} suite", - "adding coverage around {other}", - "the {part} test is flaky under load", - "checking the {part} against {other} one more time", - ], - emoji: &["🧪", "🔬", "🧊", "⏳", "🧾", "🔋", "🐝", "🧲"], - }, - Phase { - id: "shipping", - lines: &[ - "opened a pull request for the {part}", - "rebasing the {part} branch onto main", - "documenting the new {part} behaviour", - "cleaning up after the {part} change", - "waiting on review for the {part}", - ], - emoji: &["✅", "🎉", "🏁", "🧹", "♻️", "📬", "🚀", "🌅"], - }, -]; diff --git a/crates/didbot-index/tests/firehose_wire.rs b/crates/didbot-index/tests/firehose_wire.rs deleted file mode 100644 index 394bb14e..00000000 --- a/crates/didbot-index/tests/firehose_wire.rs +++ /dev/null @@ -1,90 +0,0 @@ -//! What this index reads off a firehose, against the recorded vectors. -//! -//! The consumer's half of `vectors/firehose/`. It asserts that the recorded -//! bodies deserialize into the values this crate acts on, so a server built -//! somewhere else stays readable and a change to the types here fails against -//! the record rather than in production. -//! -//! Two of the vectors exist only on this side. No server built from this -//! repository emits them; they record what an index must tolerate from one -//! that is older or newer than itself. - -use std::path::PathBuf; - -use didbot_index::firehose::{Commit, StreamInfo}; -use serde_json::Value; - -/// Reads one vector, returning its event name and body. -fn vector(name: &str) -> (String, Value) { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../vectors/firehose") - .join(format!("{name}.json")); - let text = std::fs::read_to_string(&path) - .unwrap_or_else(|err| panic!("reading {}: {err}", path.display())); - let doc: Value = serde_json::from_str(&text) - .unwrap_or_else(|err| panic!("parsing {}: {err}", path.display())); - let event = doc["event"].as_str().expect("an event name").to_owned(); - (event, doc["data"].clone()) -} - -#[test] -fn the_opening_frame_of_a_live_connection_reads() { - let (event, data) = vector("info-live"); - let info: StreamInfo = serde_json::from_value(data).expect("a readable info frame"); - - assert_eq!(event, "info"); - assert_eq!(info.name, "Live"); - assert_eq!(info.instance, "01JEXAMPLEINSTANCE"); - assert_eq!((info.oldest, info.newest), (0, 0)); -} - -#[test] -fn an_unreachable_cursor_reads_as_the_name_that_means_a_gap() { - let (_, data) = vector("info-outdated-cursor"); - let info: StreamInfo = serde_json::from_value(data).expect("a readable info frame"); - - // The one name that is not "carry on". The index compares this string, so - // the vector is what stops it drifting from what a server sends. - assert_eq!(info.name, "OutdatedCursor"); - assert_eq!(info.oldest, 41); -} - -#[test] -fn an_info_name_this_index_has_not_heard_of_is_carried_rather_than_refused() { - let (_, data) = vector("info-unknown-name"); - let info: StreamInfo = serde_json::from_value(data).expect("a readable info frame"); - - assert_eq!(info.name, "Rebalancing"); - assert_ne!(info.name, "OutdatedCursor", "this must not read as a gap"); -} - -#[test] -fn a_write_reads_with_every_field_a_server_sent() { - let (event, data) = vector("commit"); - let commit: Commit = serde_json::from_value(data).expect("a readable commit frame"); - - assert_eq!(event, "commit"); - assert_eq!(commit.seq, 42); - assert_eq!(commit.did, "did:web:kestrel.agents.localhost"); - assert_eq!(commit.collection, "com.vibescrobble.scrobble"); - assert_eq!(commit.rkey, "3lqm0000abcd2"); - assert_eq!(commit.cursor(), "01JEXAMPLEINSTANCE:42"); - assert_eq!(commit.record["text"], "reading the firehose reconnect path"); -} - -#[test] -fn a_write_from_a_server_that_predates_four_fields_still_reads() { - let (_, data) = vector("commit-sparse"); - let commit: Commit = serde_json::from_value(data).expect("a readable commit frame"); - - // The point of the vector: an index reads servers it did not build, and - // one that does not name a commit must not stop it dead. Empty is the - // honest reading of "this server did not say". - assert_eq!(commit.seq, 43); - assert_eq!(commit.cid, ""); - assert_eq!(commit.commit, ""); - assert_eq!(commit.rev, ""); - assert_eq!(commit.time, ""); - assert_eq!(commit.cursor(), "01JEXAMPLEINSTANCE:43"); - assert_eq!(commit.record["text"], "a frame from an older server"); -} diff --git a/crates/didbot-index/tests/reconnect.rs b/crates/didbot-index/tests/reconnect.rs deleted file mode 100644 index f5fb03d3..00000000 --- a/crates/didbot-index/tests/reconnect.rs +++ /dev/null @@ -1,432 +0,0 @@ -//! A server that goes away mid-stream, and this index surviving it. -//! -//! The consumer's side. What is asserted here is the client's own behaviour: -//! that it reconnects on a timer, resumes from a cursor it kept to itself, -//! and arrives at the right place rather than silently missing frames nobody -//! will ever notice were missing. -//! -//! The failure it is about is ordinary: a developer restarts `dev-pds.sh` -//! while an index is following it. -//! -//! The server here is a stub that speaks the firehose and nothing else. A real -//! one would drag the whole personal data server in for a property that is not -//! about it — what the real server puts on the wire is asserted on its own -//! side, and that the two agree is `vectors/firehose/`'s job. -//! -//! This is the slow test in this crate and unavoidably so: the subscriber -//! waits out its reconnect interval, which is a number chosen for a developer -//! watching a terminal rather than for a test. - -use std::net::SocketAddr; -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use didbot_index::firehose::{subscribe_from, FirehoseFilter, Update}; -use didbot_index::ServerRef; -use serde_json::{json, Value}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::sync::mpsc; - -/// How long a step may take before it counts as a hang. Generous: the only -/// thing it protects against is a test that would otherwise never finish. -const PATIENCE: Duration = Duration::from_secs(20); - -/// The run of the "server" that minted the sequence numbers. -const INSTANCE: &str = "stub-instance"; - -const DID: &str = "did:web:stormcock.agents.localhost"; -const COLLECTION: &str = "com.vibescrobble.scrobble"; - -/// A firehose and nothing else, on a port nobody chose. -/// -/// Written against the socket rather than on a web framework because the test -/// needs to take the connection away from a client that is *already -/// connected*, and owning the socket is what makes that possible. Aborting a -/// framework's listener does not do it: each accepted connection is served -/// from its own task, so a stream nobody closes goes on being served through a -/// listener that no longer exists, and the subscriber never disconnects at -/// all. A test built on that passes whether the resume works or not. -struct Stub { - addr: SocketAddr, - /// Every commit frame written so far, in sequence order. - frames: Arc>>, - /// The accept loop and every connection it has opened. - live: Arc>>>, -} - -impl Stub { - async fn start() -> Self { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("loopback is bindable"); - let stub = Self { - addr: listener.local_addr().expect("the port reads back"), - frames: Arc::new(Mutex::new(Vec::new())), - live: Arc::new(Mutex::new(Vec::new())), - }; - stub.accept_on(listener); - stub - } - - fn accept_on(&self, listener: tokio::net::TcpListener) { - let frames = Arc::clone(&self.frames); - let owned = Arc::clone(&self.live); - let live = Arc::clone(&self.live); - let accepting = tokio::spawn(async move { - while let Ok((socket, _)) = listener.accept().await { - let frames = Arc::clone(&frames); - let serving = tokio::spawn(async move { serve(socket, frames).await }); - owned.lock().expect("stub lock").push(serving); - } - }); - live.lock().expect("stub lock").push(accepting); - } - - /// Drops every connection and stops accepting new ones. - fn cut(&self) { - for task in self.live.lock().expect("stub lock").drain(..) { - task.abort(); - } - } - - /// Accepts again, on the same port. - async fn resume(&self) { - // The port was in use a moment ago and is not always released - // instantly; retrying is the difference between a test that exercises - // a reconnect and one that fails on a loaded machine. - let listener = loop { - match tokio::net::TcpListener::bind(self.addr).await { - Ok(listener) => break listener, - Err(_) => tokio::time::sleep(Duration::from_millis(20)).await, - } - }; - self.accept_on(listener); - } - - /// Publishes one commit frame, and answers with its sequence number. - fn write(&self, text: &str) -> u64 { - self.write_from(DID, COLLECTION, text) - } - - /// Publishes one commit frame naming `did` and `collection`, for a test - /// that cares which repository or collection a commit belongs to rather - /// than only what it says. - fn write_from(&self, did: &str, collection: &str, text: &str) -> u64 { - let mut frames = self.frames.lock().expect("stub lock"); - let seq = frames.len() as u64 + 1; - let rkey = format!("3lqm{seq:08}"); - frames.push(json!({ - "seq": seq, - "instance": INSTANCE, - "did": did, - "collection": collection, - "rkey": rkey, - "uri": format!("at://{did}/{collection}/{rkey}"), - "record": { "$type": collection, "text": text }, - })); - seq - } - - fn server(&self) -> ServerRef { - ServerRef::parse(&format!("http://{}", self.addr)).expect("a usable base url") - } -} - -/// Serves one connection: the opening frame, whatever the cursor missed, and -/// everything published from then on. -async fn serve(mut socket: tokio::net::TcpStream, frames: Arc>>) { - let Some(request) = read_request(&mut socket).await else { - return; - }; - let after = cursor_of(&request); - - // No content length and no chunking: the body ends when the connection - // does, which is exactly what a stream that is cut should look like. - let opening = json!({ - "name": if after.is_some() { "Replay" } else { "Live" }, - "instance": INSTANCE, - "oldest": 1, - "newest": frames.lock().expect("stub lock").len(), - "message": "a stub firehose", - }); - let head = format!( - "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nCache-Control: no-cache\r\n\r\n\ - event: info\r\ndata: {opening}\n\n" - ); - if socket.write_all(head.as_bytes()).await.is_err() { - return; - } - - let mut sent = after.unwrap_or(0); - loop { - let pending: Vec = frames - .lock() - .expect("stub lock") - .iter() - .skip(sent as usize) - .cloned() - .collect(); - for frame in pending { - let block = format!("event: commit\ndata: {frame}\n\n"); - if socket.write_all(block.as_bytes()).await.is_err() { - return; - } - sent += 1; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } -} - -/// Reads request bytes up to the blank line that ends the headers. -async fn read_request(socket: &mut tokio::net::TcpStream) -> Option { - let mut buffered = Vec::new(); - let mut chunk = [0u8; 1024]; - loop { - let read = socket.read(&mut chunk).await.ok()?; - if read == 0 { - return None; - } - buffered.extend_from_slice(&chunk[..read]); - if buffered.windows(4).any(|window| window == b"\r\n\r\n") { - return Some(String::from_utf8_lossy(&buffered).into_owned()); - } - } -} - -/// The sequence number a request's cursor names, if it carries one. -/// -/// A cursor is `:` and the colon is reserved in a query, so a -/// client encodes it. Decoding just that one escape is enough here. -fn cursor_of(request: &str) -> Option { - let line = request.lines().next()?; - let query = line.split_whitespace().nth(1)?.split_once('?')?.1; - let value = query - .split('&') - .find_map(|pair| pair.strip_prefix("cursor="))?; - value.replace("%3A", ":").split_once(':')?.1.parse().ok() -} - -/// The next update, or a failure naming what was being waited for. -async fn next(incoming: &mut mpsc::Receiver, what: &str) -> Update { - tokio::time::timeout(PATIENCE, incoming.recv()) - .await - .unwrap_or_else(|_| panic!("timed out waiting for {what}")) - .unwrap_or_else(|| panic!("the stream closed while waiting for {what}")) -} - -/// Unwraps a commit, failing on anything else. -fn commit(update: Update) -> Box { - match update { - Update::Commit { commit, .. } => commit, - Update::Gap { message, .. } => panic!("expected a commit, got a gap: {message}"), - } -} - -#[tokio::test] -async fn a_subscriber_survives_the_server_going_away_and_misses_nothing() { - let stub = Stub::start().await; - let (out, mut incoming) = mpsc::channel(64); - let following = tokio::spawn(subscribe_from( - stub.server(), - None, - FirehoseFilter::All, - out, - )); - - // Connected, and reading. - stub.write("before the outage"); - let first = commit(next(&mut incoming, "the first frame").await); - assert_eq!(first.record["text"], "before the outage"); - - // The server goes away. The writes do not: the stream fills with frames - // nobody is connected to read. - stub.cut(); - for i in 0..3 { - stub.write(&format!("written while nothing was listening {i}")); - } - stub.resume().await; - - // Everything written during the outage, in order, with nothing repeated - // and nothing lost. - let mut missed = Vec::new(); - for _ in 0..3 { - missed.push(commit( - next(&mut incoming, "a frame from after the outage").await, - )); - } - let texts: Vec<&str> = missed - .iter() - .map(|frame| frame.record["text"].as_str().unwrap_or_default()) - .collect(); - assert_eq!( - texts, - [ - "written while nothing was listening 0", - "written while nothing was listening 1", - "written while nothing was listening 2", - ], - "the subscriber did not resume where it left off" - ); - - // Sequence numbers say the same thing the texts do, and say it about the - // stream rather than about the records: consecutive, and strictly after - // the frame the subscriber had before the outage. Nothing replayed. - assert_eq!( - missed[0].seq, - first.seq + 1, - "the resume skipped or repeated" - ); - assert_eq!(missed[1].seq, missed[0].seq + 1); - assert_eq!(missed[2].seq, missed[1].seq + 1); - - // And it is still following, rather than having reconnected once and - // stopped: a retry loop that gives up after one success is a retry loop - // that works in a test and not on a machine. - stub.write("and still following"); - assert_eq!( - commit(next(&mut incoming, "a frame after the recovery").await).record["text"], - "and still following" - ); - - following.abort(); -} - -/// A second repository, for the filter tests: real vouched deployments follow -/// more than one account, and a filter that only ever saw one DID in a test -/// could not tell "matches this DID" from "matches anything at all". -const OTHER_DID: &str = "did:web:otherbird.agents.localhost"; -/// A second collection, likewise. -const OTHER_COLLECTION: &str = "com.vibescrobble.registration"; - -#[tokio::test] -async fn the_all_filter_admits_every_collection_and_every_repository() { - let stub = Stub::start().await; - let (out, mut incoming) = mpsc::channel(64); - let following = tokio::spawn(subscribe_from( - stub.server(), - None, - FirehoseFilter::All, - out, - )); - - stub.write_from(DID, COLLECTION, "mine"); - stub.write_from(OTHER_DID, OTHER_COLLECTION, "somebody else's"); - - let first = commit(next(&mut incoming, "the first commit").await); - assert_eq!(first.record["text"], "mine"); - let second = commit(next(&mut incoming, "the second commit").await); - assert_eq!(second.record["text"], "somebody else's"); - - following.abort(); -} - -#[tokio::test] -async fn the_collections_filter_admits_only_the_named_collections() { - let stub = Stub::start().await; - let (out, mut incoming) = mpsc::channel(64); - let filter = FirehoseFilter::collections([COLLECTION]); - let following = tokio::spawn(subscribe_from(stub.server(), None, filter, out)); - - // Excluded: a different collection, from either repository. - stub.write_from(DID, OTHER_COLLECTION, "excluded: wrong collection"); - stub.write_from( - OTHER_DID, - OTHER_COLLECTION, - "excluded: wrong collection too", - ); - // Admitted: the named collection, from either repository — the filter - // does not care which account wrote it. - stub.write_from(DID, COLLECTION, "admitted: mine"); - stub.write_from( - OTHER_DID, - COLLECTION, - "admitted: somebody else's, same collection", - ); - - let first = commit(next(&mut incoming, "the first admitted commit").await); - assert_eq!(first.record["text"], "admitted: mine"); - let second = commit(next(&mut incoming, "the second admitted commit").await); - assert_eq!( - second.record["text"], - "admitted: somebody else's, same collection" - ); - - // Nothing else arrives: the excluded frames never reached the channel. - assert!( - tokio::time::timeout(Duration::from_millis(200), incoming.recv()) - .await - .is_err(), - "an excluded collection was forwarded" - ); - - following.abort(); -} - -#[tokio::test] -async fn the_repos_filter_admits_only_the_named_repositories() { - let stub = Stub::start().await; - let (out, mut incoming) = mpsc::channel(64); - let filter = FirehoseFilter::repos([DID]); - let following = tokio::spawn(subscribe_from(stub.server(), None, filter, out)); - - // Excluded: the other repository, on either collection. - stub.write_from(OTHER_DID, COLLECTION, "excluded: wrong repo"); - stub.write_from(OTHER_DID, OTHER_COLLECTION, "excluded: wrong repo too"); - // Admitted: the named repository, on either collection — the filter does - // not care which collection the record went into. - stub.write_from(DID, COLLECTION, "admitted: named repo"); - stub.write_from( - DID, - OTHER_COLLECTION, - "admitted: named repo, other collection", - ); - - let first = commit(next(&mut incoming, "the first admitted commit").await); - assert_eq!(first.record["text"], "admitted: named repo"); - let second = commit(next(&mut incoming, "the second admitted commit").await); - assert_eq!( - second.record["text"], - "admitted: named repo, other collection" - ); - - assert!( - tokio::time::timeout(Duration::from_millis(200), incoming.recv()) - .await - .is_err(), - "an excluded repository was forwarded" - ); - - following.abort(); -} - -#[tokio::test] -async fn a_filtered_out_frame_still_advances_the_cursor() { - // The whole point of filtering after the cursor moves rather than before: - // a reconnect must not re-request a frame this subscriber has already - // decided not to keep, or a filtered collection would be replayed on - // every single reconnect forever. - let stub = Stub::start().await; - let (out, mut incoming) = mpsc::channel(64); - let filter = FirehoseFilter::collections([COLLECTION]); - let following = tokio::spawn(subscribe_from(stub.server(), None, filter, out)); - - stub.write_from(OTHER_DID, OTHER_COLLECTION, "filtered out"); // seq 1 - stub.write_from(DID, COLLECTION, "kept"); // seq 2 - let kept = commit(next(&mut incoming, "the kept commit").await); - assert_eq!(kept.seq, 2, "the filtered frame's sequence number was seen"); - - stub.cut(); - stub.write_from( - OTHER_DID, - OTHER_COLLECTION, - "filtered out during the outage", - ); // seq 3 - stub.write_from(DID, COLLECTION, "kept after the outage"); // seq 4 - stub.resume().await; - - let after = commit(next(&mut incoming, "the frame after the outage").await); - assert_eq!(after.record["text"], "kept after the outage"); - assert_eq!(after.seq, 4, "the reconnect replayed a frame already seen"); - - following.abort(); -} diff --git a/crates/didbot-query/Cargo.toml b/crates/didbot-query/Cargo.toml deleted file mode 100644 index 6fa9ff83..00000000 --- a/crates/didbot-query/Cargo.toml +++ /dev/null @@ -1,37 +0,0 @@ -[package] -name = "didbot-query" -description = "Answers questions about the index, for the people looking at it." -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -repository.workspace = true -publish.workspace = true - -[dependencies] -axum.workspace = true -reqwest.workspace = true -serde.workspace = true -serde_json.workspace = true -tokio.workspace = true -tracing.workspace = true -tracing-subscriber.workspace = true -# The view, the event shapes and the stream reader are the index's, and a -# second copy of any of them would be a second thing to keep in step. -didbot-index.workspace = true -# For `canonical_did`: a development DID carries a percent-encoded port, and -# the repair for a URL that arrived with a bare colon belongs beside the DID -# type rather than in a copy here. -didbot-identity.workspace = true -# The reader-facing endpoints are open to a browser on another origin, for -# the same reason the index's are: everything here is already public. -tower-http.workspace = true - -[dev-dependencies] -# `oneshot` against the router, because everything worth testing here is the -# routing and the paging rather than the transport under it. -serde_json.workspace = true -tower.workspace = true - -[lints] -workspace = true diff --git a/crates/didbot-query/src/bin/didbot-query.rs b/crates/didbot-query/src/bin/didbot-query.rs deleted file mode 100644 index 675378b2..00000000 --- a/crates/didbot-query/src/bin/didbot-query.rs +++ /dev/null @@ -1,105 +0,0 @@ -//! Serves the index's view to whoever is watching it. -//! -//! Usage: `didbot-query [--index ] [--listen ]`. -//! -//! Holds no truth of its own: it mirrors the index and fans the result out to -//! browsers. Restarting it costs its clients a reconnect and nothing else. - -#![forbid(unsafe_code)] - -use std::net::SocketAddr; -use std::process::ExitCode; - -use didbot_index::Shared; -use didbot_query::{app, mirror}; -use tokio::net::TcpListener; - -/// The index, where it listens by default. -const DEFAULT_INDEX: &str = "http://127.0.0.1:3002"; -/// Where this listens, one above the index. -const DEFAULT_LISTEN: &str = "127.0.0.1:3003"; - -const USAGE: &str = "\ -didbot-query - serve the index's view to whoever is watching - -Usage: didbot-query [options] - -Options: - --index The index to mirror (default http://127.0.0.1:3002) - --listen Address to serve on (default 127.0.0.1:3003) - -h, --help Print this message -"; - -struct Args { - index: String, - listen: String, -} - -fn parse_args>(mut args: I) -> Result { - let mut parsed = Args { - index: DEFAULT_INDEX.to_owned(), - listen: DEFAULT_LISTEN.to_owned(), - }; - while let Some(flag) = args.next() { - let mut value = || { - args.next() - .ok_or_else(|| format!("{flag} needs a value\n\n{USAGE}")) - }; - match flag.as_str() { - "--index" => parsed.index = value()?, - "--listen" => parsed.listen = value()?, - "-h" | "--help" => return Err(USAGE.to_owned()), - other => return Err(format!("unknown option `{other}`\n\n{USAGE}")), - } - } - parsed.index = parsed.index.trim_end_matches('/').to_owned(); - Ok(parsed) -} - -#[tokio::main] -async fn main() -> ExitCode { - let args = match parse_args(std::env::args().skip(1)) { - Ok(args) => args, - Err(message) => { - println!("{message}"); - return ExitCode::from(2); - } - }; - didbot_index::init_tracing(); - - let addr: SocketAddr = match args.listen.parse() { - Ok(addr) => addr, - Err(_) => { - tracing::error!("--listen: `{}` is not an address", args.listen); - return ExitCode::FAILURE; - } - }; - - let shared = Shared::new(); - tokio::spawn(mirror(args.index.clone(), shared.clone())); - - println!("didbot-query"); - println!(" mirroring {}", args.index); - println!(" listening http://{addr}"); - println!(); - println!(" curl http://{addr}/view"); - println!(" curl 'http://{addr}/agents?limit=20'"); - println!(" curl http://{addr}/agents/"); - println!(" curl 'http://{addr}/scrobbles?limit=20'"); - println!(" curl http://{addr}/work"); - println!(" curl -N http://{addr}/events"); - println!(); - - let listener = match TcpListener::bind(addr).await { - Ok(listener) => listener, - Err(err) => { - tracing::error!("could not bind {addr}: {err}"); - return ExitCode::FAILURE; - } - }; - if let Err(err) = axum::serve(listener, app(shared)).await { - tracing::error!("server stopped: {err}"); - return ExitCode::FAILURE; - } - ExitCode::SUCCESS -} diff --git a/crates/didbot-query/src/lib.rs b/crates/didbot-query/src/lib.rs deleted file mode 100644 index 72f84298..00000000 --- a/crates/didbot-query/src/lib.rs +++ /dev/null @@ -1,100 +0,0 @@ -//! What people's browsers talk to. -//! -//! The index observes: it follows servers, reads repositories, and announces -//! what changed. This answers: it holds a copy of what the index built and -//! serves it to however many clients are watching. The split is not -//! ceremony — the two have different shapes of load and different reasons to -//! restart. Ingest must not drop an event and is one consumer of one upstream; -//! serving is many short-lived readers who can be dropped and reconnect. Put -//! them in one process and a slow client is an ingest problem. -//! -//! What it serves is the whole reader-facing surface: the snapshot and the -//! stream the canvas follows, and beside them the paged, filtered lookups -//! that the index deliberately does not offer — a page of agents, one agent -//! with its lineage and its work, a filtered feed. See [`read`]. -//! -//! It holds no truth of its own. Everything here can be rebuilt by asking the -//! index again, which is exactly what it does when it starts or when its -//! stream breaks. That includes the work groups: they are computed once, in -//! the index, and arrive here as an ordinary event. A mirror that recomputed -//! them would produce a second answer from a different event history, which is -//! the problem clustering in the browser had, moved one hop closer. - -#![forbid(unsafe_code)] - -pub mod read; - -pub use read::{app, AgentDetail, Page, Paging, DEFAULT_LIMIT, MAX_LIMIT}; - -use std::time::Duration; - -use didbot_index::follow::stream; -use didbot_index::view::{Event, Snapshot}; -use didbot_index::Shared; -use tokio::sync::mpsc; -use tracing::{info, warn}; - -/// How long to wait before asking the index for the whole view again. -const RESEED: Duration = Duration::from_secs(5); - -/// Keeps a local copy of the index's view in step with it. -/// -/// Seeds from a snapshot, then follows the stream. When the stream breaks it -/// seeds again rather than resuming, because the gap is unbounded and a view -/// that is quietly missing a deletion is worse than one that is briefly late. -pub async fn mirror(index: String, shared: Shared) { - loop { - match seed(&index, &shared).await { - Ok(counts) => info!( - operators = counts.0, - agents = counts.1, - scrobbles = counts.2, - groups = counts.3, - "seeded from the index" - ), - Err(err) => { - warn!(%err, "could not read the index"); - shared.pulse.following(&index, false).await; - tokio::time::sleep(RESEED).await; - continue; - } - } - shared.pulse.following(&index, true).await; - - let (tx, mut rx) = mpsc::channel::(512); - let following = tokio::spawn(stream(format!("{index}/events"), tx)); - - while let Some(event) = rx.recv().await { - let announced = shared.view.lock().await.apply(event); - shared.announce(announced).await; - } - - following.abort(); - shared.pulse.following(&index, false).await; - warn!("the index's stream ended; seeding again"); - tokio::time::sleep(RESEED).await; - } -} - -/// Fetches the whole view and adopts it, returning what arrived. -async fn seed(index: &str, shared: &Shared) -> Result<(usize, usize, usize, usize), String> { - let url = format!("{index}/view"); - let response = reqwest::get(&url) - .await - .map_err(|err| format!("{url}: {err}"))?; - if !response.status().is_success() { - return Err(format!("{url}: HTTP {}", response.status().as_u16())); - } - let snapshot: Snapshot = response - .json() - .await - .map_err(|err| format!("{url}: unreadable: {err}"))?; - let counts = ( - snapshot.operators.len(), - snapshot.agents.len(), - snapshot.scrobbles.len(), - snapshot.groups.len(), - ); - shared.view.lock().await.adopt(snapshot); - Ok(counts) -} diff --git a/crates/didbot-query/src/read.rs b/crates/didbot-query/src/read.rs deleted file mode 100644 index da975a8f..00000000 --- a/crates/didbot-query/src/read.rs +++ /dev/null @@ -1,370 +0,0 @@ -//! The endpoints people's browsers and scripts actually call. -//! -//! The index serves two things and says why it serves only two: the whole -//! view, and what changes after it. Everything a reader might want beyond -//! that — a page of agents, one agent's lineage, a filtered feed — is -//! deliberately not there, because an index that answered those would stop -//! being an index and become the application. This is where they live -//! instead. -//! -//! Everything here reads the mirrored [`View`](didbot_index::View) and -//! nothing here writes to it. -//! That is what makes these safe to add: a slow client paging through forty -//! thousand records cannot delay ingest, because ingest is in another process -//! and this one is a copy that can be thrown away and rebuilt. -//! -//! # Paging -//! -//! Every list endpoint takes `limit` and `cursor` and returns a `cursor` when -//! there is more. The cursor is a position in a total order and never an -//! offset: this population turns over continuously — agents provision and are -//! deleted while a client is halfway through reading them — and an offset -//! into a list that is changing underneath silently skips rows and repeats -//! others. Agents are ordered by DID and records by timestamp then URI, both -//! of which still name a place to resume from after the row they name has -//! gone away. -//! -//! A cursor is opaque and clients must treat it that way, but it is not -//! encoded or signed, because there is nothing in it to hide: it is a -//! position in public data that the same request would have reached anyway. - -use std::collections::BTreeMap; - -use axum::extract::{Path, Query, State}; -use axum::http::StatusCode; -use axum::response::IntoResponse; -use axum::routing::get; -use axum::{Json, Router}; -use didbot_identity::canonical_did; -use didbot_index::view::{Agent, Operator}; -use didbot_index::{Scrobble, Shared, WorkGroup}; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use tower_http::cors::{Any, CorsLayer}; - -/// How many rows a list returns when the caller does not ask. -pub const DEFAULT_LIMIT: usize = 50; - -/// The most rows a list will return however large a `limit` is asked for. -/// -/// A ceiling rather than an error: a client asking for ten thousand agents -/// wants as many as it can have, and refusing the request teaches it to ask -/// for exactly the ceiling instead, which is the same request with an extra -/// round trip in front of it. The returned cursor says there is more. -pub const MAX_LIMIT: usize = 500; - -/// Builds the reader-facing router over the mirrored view. -/// -/// Wraps the index's own router rather than replacing it: `/view`, `/events` -/// and `/work` are what the canvas follows and what a second mirror would -/// seed from, and they behave identically here and on the index by -/// construction, because they are the same handlers. -pub fn app(shared: Shared) -> Router { - Router::new() - .route("/operators", get(operators)) - .route("/agents", get(agents)) - .route("/agents/{did}", get(agent)) - .route("/scrobbles", get(scrobbles)) - .route("/work/{id}", get(group)) - .with_state(shared.clone()) - .layer(CorsLayer::new().allow_origin(Any).allow_methods(Any)) - .merge(didbot_index::app(shared)) -} - -/// What every list endpoint accepts, plus whatever it filters on. -#[derive(Debug, Clone, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Paging { - /// Where to resume from, as returned by the previous page. - pub cursor: Option, - /// How many rows to return, capped at [`MAX_LIMIT`]. - pub limit: Option, - /// Only accounts on this server, by service DID. - pub operator: Option, - /// Only accounts spawned by this one, by DID. - /// - /// `parent=` with an empty value means the opposite: only session roots, - /// the agents nothing here spawned. Absent means no filter at all, which - /// is a third answer and the reason this is not a bare `Option` - /// read as "roots when missing". - pub parent: Option, - /// Only the account with this handle, or this DID. - pub handle: Option, - /// Only records written by this account. - pub did: Option, - /// Only accounts that are pinned, or only those that are not. - pub pinned: Option, -} - -impl Paging { - /// The page size to use, clamped into something a server will serve. - fn limit(&self) -> usize { - self.limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT) - } -} - -/// One page of anything, in the one shape every list here returns. -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct Page { - /// The rows. - pub items: Vec, - /// Where to resume, absent when this was the last page. - /// - /// Absent rather than null or empty: a client checking `if (page.cursor)` - /// is the common case and all three spellings should read the same to it, - /// but only one of them says "there is no next page" without a client - /// having to know that an empty string is not a position. - #[serde(skip_serializing_if = "Option::is_none")] - pub cursor: Option, -} - -impl Page { - /// Takes `limit` rows and reports where the next page starts. - /// - /// The cursor comes off the last row returned rather than from a lookahead - /// row, so a page that exactly empties the list still returns a cursor. - /// Following it costs one more request that comes back empty, which is - /// the honest answer: nothing here can promise that no row will have been - /// written by the time that request arrives. - fn take(rows: impl Iterator, limit: usize, position: impl Fn(&T) -> String) -> Self { - let items: Vec = rows.take(limit).collect(); - let cursor = if items.len() < limit { - None - } else { - items.last().map(&position) - }; - Self { items, cursor } - } -} - -/// `GET /operators` -/// -/// Unpaged, and deliberately: an operator is a personal data server somebody -/// wrote a vouch for, and an appview following more than a screenful of them -/// is not a thing that happens. Paging it would be surface nobody calls. -async fn operators(State(shared): State) -> impl IntoResponse { - let view = shared.view.lock().await; - Json(json!({ "operators": view.operator_list() })) -} - -/// `GET /agents` -/// -/// The population, paged, with the filters a reader actually asks for: whose -/// server, what spawned it, and whether it is pinned. -async fn agents(State(shared): State, Query(paging): Query) -> impl IntoResponse { - let view = shared.view.lock().await; - - // A handle names one account, so it answers as a one-row page rather than - // as a filter. Asking for a handle and a cursor together is a client - // paging through a single row, which is harmless and comes out right. - if let Some(handle) = &paging.handle { - let found = view - .agent_by_handle(handle) - .or_else(|| view.agent(handle)) - .or_else(|| view.agent(&canonical_did(handle))) - .cloned(); - return Json(Page { - items: found.into_iter().collect::>(), - cursor: None, - }) - .into_response(); - } - - let operator = paging.operator.as_deref().map(canonical_did); - // Left as given when it is empty: that spelling asks for session roots - // rather than naming an agent, and `canonical_did` would pass it through - // anyway. Kept explicit so the two meanings stay visible together. - let parent = paging.parent.as_deref().map(canonical_did); - let pinned = paging.pinned; - let keep = move |agent: &Agent| { - if let Some(wanted) = &operator { - if &agent.operator != wanted { - return false; - } - } - if let Some(wanted) = &parent { - // An empty value asks for session roots: the agents this index has - // not been told a parent for. - let matches = if wanted.is_empty() { - agent.parent.is_none() - } else { - agent.parent.as_deref() == Some(wanted.as_str()) - }; - if !matches { - return false; - } - } - if let Some(wanted) = pinned { - if agent.pinned != wanted { - return false; - } - } - true - }; - - let page = Page::take( - view.agents_after(paging.cursor.as_deref(), keep).cloned(), - paging.limit(), - |agent: &Agent| agent.did.clone(), - ); - Json(page).into_response() -} - -/// One account with everything about it that needs a second lookup. -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AgentDetail { - /// The account itself. - pub agent: Agent, - /// The server it lives on, when that server is still being followed. - #[serde(skip_serializing_if = "Option::is_none")] - pub operator: Option, - /// The chain up to its session root, nearest ancestor first. - pub ancestors: Vec, - /// Everything it spawned that this index still holds. - pub children: Vec, - /// What it has been saying, newest first. - pub scrobbles: Vec, - /// The work groups it is a member of. - pub groups: Vec, -} - -/// `GET /agents/{did}` -/// -/// One agent and its place in everything: who spawned it, what it spawned, -/// what it has said, and who it is working alongside. One request rather than -/// five, because these are the five things a panel opens with and a client -/// making five round trips to draw one card is the reason this endpoint -/// exists at all. -/// -/// Accepts a handle as well as a DID. A person copying a name out of a log -/// has the handle, and refusing it here would mean a lookup call in front of -/// every fetch. -async fn agent(State(shared): State, Path(did): Path) -> impl IntoResponse { - let view = shared.view.lock().await; - let canonical = canonical_did(&did); - let Some(agent) = view - .agent(&did) - .or_else(|| view.agent(&canonical)) - .or_else(|| view.agent_by_handle(&did)) - else { - return ( - StatusCode::NOT_FOUND, - Json(json!({ "error": "NotFound", "message": format!("no agent {did}") })), - ) - .into_response(); - }; - let agent = agent.clone(); - - let mut scrobbles = view.history_of(&agent.did).to_vec(); - scrobbles.reverse(); - Json(AgentDetail { - operator: view.operator(&agent.operator).cloned(), - ancestors: view.ancestors_of(&agent.did).into_iter().cloned().collect(), - children: view.children_of(&agent.did).into_iter().cloned().collect(), - scrobbles, - groups: view - .groups() - .iter() - .filter(|group| group.members.contains(&agent.did)) - .cloned() - .collect(), - agent, - }) - .into_response() -} - -/// `GET /scrobbles` -/// -/// The feed, newest first, paged, and filterable by account or by server. -/// -/// What it can return is bounded by what the index keeps, which is the last -/// [`HISTORY`](didbot_index::view::HISTORY) records per account and not -/// the whole of anybody's repository. A client that wants further back reads -/// the repository, which is the durable copy and the only complete one. -async fn scrobbles( - State(shared): State, - Query(paging): Query, -) -> impl IntoResponse { - let view = shared.view.lock().await; - - // Resolved once rather than per record: the operator filter is a property - // of the author's account, and looking it up inside the predicate would - // be a map lookup for every record held. - let authors: Option> = - paging - .operator - .as_deref() - .map(canonical_did) - .map(|operator| { - view.agents_after(None, |agent: &Agent| agent.operator == operator) - .map(|agent| (agent.did.clone(), ())) - .collect() - }); - - let did = paging.did.as_deref().map(canonical_did); - let all = view.recent(|scrobble: &Scrobble| { - if let Some(wanted) = &did { - if &scrobble.did != wanted { - return false; - } - } - if let Some(authors) = &authors { - if !authors.contains_key(&scrobble.did) { - return false; - } - } - true - }); - - let cursor = paging.cursor.clone(); - let rows = all - .into_iter() - .skip_while(move |scrobble| match &cursor { - // Newest first, so "after the cursor" is "strictly older than it". - Some(at) => position_of(scrobble).as_str() >= at.as_str(), - None => false, - }) - .cloned(); - Json(Page::take(rows, paging.limit(), |scrobble: &Scrobble| { - position_of(scrobble) - })) -} - -/// Where one record sits in the feed's order. -/// -/// Timestamp then URI, matching [`View::recent`](didbot_index::View), -/// and joined by a character that cannot appear in an RFC 3339 timestamp so -/// the two halves cannot be confused for one another. -fn position_of(scrobble: &Scrobble) -> String { - format!("{} {}", scrobble.created_at, scrobble.uri) -} - -/// `GET /work/{id}` -/// -/// One work group with its members resolved, rather than as the list of DIDs -/// the stream carries. A client drawing a group wants to name its agents, and -/// the alternative is a fetch per member. -async fn group(State(shared): State, Path(id): Path) -> impl IntoResponse { - let view = shared.view.lock().await; - let Some(group) = view.group(&id) else { - return ( - StatusCode::NOT_FOUND, - Json(json!({ "error": "NotFound", "message": format!("no work group {id}") })), - ) - .into_response(); - }; - let members: Vec = group - .members - .iter() - .filter_map(|did| view.agent(did).cloned()) - .collect(); - Json(json!({ - "id": group.id, - "label": group.label, - "founder": group.founder, - "members": members, - })) - .into_response() -} diff --git a/crates/didbot-query/tests/read.rs b/crates/didbot-query/tests/read.rs deleted file mode 100644 index 872ab0c3..00000000 --- a/crates/didbot-query/tests/read.rs +++ /dev/null @@ -1,379 +0,0 @@ -//! The reader-facing endpoints, against a view built by hand. -//! -//! `oneshot` against the router rather than a listener: what is worth pinning -//! here is the paging and the filtering, and a socket in front of them tests -//! axum rather than this crate. - -use axum::body::Body; -use axum::http::{Request, StatusCode}; -use didbot_index::view::{Agent, Operator}; -use didbot_index::{Scrobble, Shared}; -use didbot_query::app; -use serde_json::Value; -use tower::ServiceExt; - -const OPERATOR: &str = "did:web:agents.example"; - -fn agent(did: &str, parent: Option<&str>) -> Agent { - Agent { - did: did.to_owned(), - agent_id: did.rsplit(':').next().unwrap_or(did).to_owned(), - operator: OPERATOR.to_owned(), - pinned: false, - created_at: "2026-08-25T10:00:00Z".to_owned(), - handle: Some(format!( - "{}.agents.example", - did.rsplit(':').next().unwrap() - )), - parent: parent.map(str::to_owned), - agent_type: None, - harness: None, - model: None, - } -} - -fn scrobble(did: &str, n: usize) -> Scrobble { - Scrobble { - uri: format!("at://{did}/{n}"), - did: did.to_owned(), - text: "walking the coupon code through the cart".to_owned(), - emoji: "🧭".to_owned(), - created_at: format!("2026-08-25T10:{n:02}:00Z"), - task: None, - effort: None, - model: None, - agent_type: None, - parent: None, - artifacts: Vec::new(), - value: Value::Null, - } -} - -/// A mirror holding one server, `agents` accounts and a few records each. -async fn populated(agents: usize) -> Shared { - let shared = Shared::new(); - { - let mut view = shared.view.lock().await; - view.see_operator(Operator { - did: OPERATOR.to_owned(), - base_url: "http://agents.example".to_owned(), - handle: "agents.example".to_owned(), - zone: "agents.example".to_owned(), - }); - for i in 0..agents { - let did = format!("did:web:a{i:03}"); - // Every third account is spawned by the one before it. - let parent = (i % 3 == 2).then(|| format!("did:web:a{:03}", i - 1)); - view.see_agent(agent(&did, parent.as_deref())); - for n in 0..3 { - view.see_scrobble(scrobble(&did, n)); - } - } - } - shared -} - -async fn get(shared: &Shared, url: &str) -> (StatusCode, Value) { - let response = app(shared.clone()) - .oneshot( - Request::builder() - .uri(url) - .body(Body::empty()) - .expect("a request"), - ) - .await - .expect("the router answers"); - let status = response.status(); - let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) - .await - .expect("the body reads"); - ( - status, - serde_json::from_slice(&bytes).unwrap_or(Value::Null), - ) -} - -#[tokio::test] -async fn a_client_can_page_the_whole_population_without_taking_a_snapshot() { - let shared = populated(120).await; - - let mut seen: Vec = Vec::new(); - let mut cursor: Option = None; - for _ in 0..20 { - let url = match &cursor { - Some(at) => format!("/agents?limit=25&cursor={at}"), - None => "/agents?limit=25".to_owned(), - }; - let (status, page) = get(&shared, &url).await; - assert_eq!(status, StatusCode::OK); - for item in page["items"].as_array().expect("items") { - seen.push(item["did"].as_str().expect("a did").to_owned()); - } - match page["cursor"].as_str() { - Some(next) => cursor = Some(next.to_owned()), - None => break, - } - } - - assert_eq!(seen.len(), 120, "every agent is reached exactly once"); - let mut unique = seen.clone(); - unique.sort(); - unique.dedup(); - assert_eq!(unique.len(), 120, "no agent is served on two pages"); - assert!(seen.windows(2).all(|pair| pair[0] < pair[1]), "in order"); -} - -#[tokio::test] -async fn a_limit_is_capped_rather_than_refused() { - let shared = populated(30).await; - let (status, page) = get(&shared, "/agents?limit=100000").await; - assert_eq!(status, StatusCode::OK); - assert_eq!(page["items"].as_array().expect("items").len(), 30); -} - -#[tokio::test] -async fn agents_can_be_asked_for_by_who_spawned_them() { - let shared = populated(30).await; - - let (_, children) = get(&shared, "/agents?parent=did:web:a001").await; - let items = children["items"].as_array().expect("items"); - assert_eq!(items.len(), 1); - assert_eq!(items[0]["did"], "did:web:a002"); - - // An empty parent asks the opposite question: only the session roots. - let (_, roots) = get(&shared, "/agents?parent=&limit=500").await; - let roots = roots["items"].as_array().expect("items"); - assert_eq!(roots.len(), 20, "two in every three are roots"); - assert!(roots.iter().all(|agent| agent["parent"].is_null())); -} - -#[tokio::test] -async fn an_agent_is_found_by_the_handle_a_person_would_type() { - let shared = populated(5).await; - - let (status, found) = get(&shared, "/agents?handle=A002.Agents.Example").await; - assert_eq!(status, StatusCode::OK, "a handle is not case-sensitive"); - let items = found["items"].as_array().expect("items"); - assert_eq!(items.len(), 1); - assert_eq!(items[0]["did"], "did:web:a002"); - - // And the detail endpoint takes one in place of a DID. - let (status, detail) = get(&shared, "/agents/a002.agents.example").await; - assert_eq!(status, StatusCode::OK); - assert_eq!(detail["agent"]["did"], "did:web:a002"); -} - -#[tokio::test] -async fn a_handle_nobody_holds_finds_nothing() { - let shared = populated(5).await; - let (status, page) = get(&shared, "/agents?handle=nobody.agents.example").await; - assert_eq!(status, StatusCode::OK); - assert!(page["items"].as_array().expect("items").is_empty()); -} - -#[tokio::test] -async fn a_deleted_account_s_handle_stops_resolving() { - let shared = populated(3).await; - { - let mut view = shared.view.lock().await; - view.forget("did:web:a001"); - } - let (status, page) = get(&shared, "/agents?handle=a001.agents.example").await; - assert_eq!(status, StatusCode::OK); - assert!( - page["items"].as_array().expect("items").is_empty(), - "a deleted account's handle must not still resolve" - ); - // The other accounts are unaffected. - let (_, still) = get(&shared, "/agents?handle=a002.agents.example").await; - assert_eq!(still["items"][0]["did"], "did:web:a002"); -} - -#[tokio::test] -async fn a_colliding_handle_resolves_to_the_lowest_did() { - let shared = populated(2).await; - { - let mut view = shared.view.lock().await; - // A third account claims the same handle as the one already there. - let mut colliding = agent("did:web:z999", None); - colliding.handle = Some("a000.agents.example".to_owned()); - view.see_agent(colliding); - } - let (status, page) = get(&shared, "/agents?handle=a000.agents.example").await; - assert_eq!(status, StatusCode::OK); - let items = page["items"].as_array().expect("items"); - assert_eq!(items.len(), 1); - assert_eq!(items[0]["did"], "did:web:a000", "the lowest did wins"); -} - -#[tokio::test] -async fn one_agent_comes_back_with_its_place_in_the_tree() { - let shared = populated(9).await; - let (status, detail) = get(&shared, "/agents/did:web:a002").await; - assert_eq!(status, StatusCode::OK); - - assert_eq!(detail["agent"]["did"], "did:web:a002"); - assert_eq!(detail["operator"]["did"], OPERATOR); - assert_eq!(detail["ancestors"][0]["did"], "did:web:a001"); - assert_eq!( - detail["scrobbles"].as_array().expect("records").len(), - 3, - "what it has been saying comes with it" - ); - // Newest first, which is the order a panel renders. - assert_eq!(detail["scrobbles"][0]["uri"], "at://did:web:a002/2"); - - let (status, _) = get(&shared, "/agents/did:web:nobody").await; - assert_eq!(status, StatusCode::NOT_FOUND); -} - -#[tokio::test] -async fn the_feed_pages_newest_first_and_repeats_nothing() { - let shared = populated(40).await; - - let mut seen: Vec = Vec::new(); - let mut cursor: Option = None; - for _ in 0..40 { - let url = match &cursor { - Some(at) => format!("/scrobbles?limit=17&cursor={}", urlencode(at)), - None => "/scrobbles?limit=17".to_owned(), - }; - let (status, page) = get(&shared, &url).await; - assert_eq!(status, StatusCode::OK); - for item in page["items"].as_array().expect("items") { - seen.push(item["uri"].as_str().expect("a uri").to_owned()); - } - match page["cursor"].as_str() { - Some(next) => cursor = Some(next.to_owned()), - None => break, - } - } - - assert_eq!(seen.len(), 120, "every record, once"); - let mut unique = seen.clone(); - unique.sort(); - unique.dedup(); - assert_eq!(unique.len(), 120); -} - -#[tokio::test] -async fn the_feed_can_be_narrowed_to_one_account_or_one_server() { - let shared = populated(6).await; - - let (_, mine) = get(&shared, "/scrobbles?did=did:web:a003").await; - let items = mine["items"].as_array().expect("items"); - assert_eq!(items.len(), 3); - assert!(items.iter().all(|row| row["did"] == "did:web:a003")); - - let (_, ours) = get( - &shared, - "/scrobbles?operator=did:web:agents.example&limit=500", - ) - .await; - assert_eq!(ours["items"].as_array().expect("items").len(), 18); - - let (_, nobodys) = get(&shared, "/scrobbles?operator=did:web:elsewhere").await; - assert!(nobodys["items"].as_array().expect("items").is_empty()); -} - -#[tokio::test] -async fn health_says_whether_anything_is_still_arriving() { - let shared = populated(3).await; - let (status, health) = get(&shared, "/health").await; - assert_eq!(status, StatusCode::OK); - - assert_eq!(health["agents"], 3); - assert_eq!(health["status"], "ok"); - assert!(health["uptime"].is_number()); - // Nothing has been announced through `Shared`, so there is no last - // change to report, and the field says so rather than claiming zero - // seconds ago. - assert!(health["quietFor"].is_null()); - - shared.pulse.following("http://index.example", false).await; - let (_, health) = get(&shared, "/health").await; - assert_eq!(health["upstream"], "http://index.example"); - assert_eq!(health["connected"], false); -} - -#[tokio::test] -async fn the_index_endpoints_are_still_there() { - // The canvas follows these, and a mirror that stopped serving them in - // exchange for the new ones would have completed nothing. - let shared = populated(4).await; - for url in ["/view", "/work", "/operators"] { - let (status, _) = get(&shared, url).await; - assert_eq!(status, StatusCode::OK, "{url} is served"); - } -} - -/// Percent-encodes a cursor for a query string. -fn urlencode(value: &str) -> String { - value - .bytes() - .map(|byte| match byte { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => { - (byte as char).to_string() - } - other => format!("%{other:02X}"), - }) - .collect() -} - -#[tokio::test] -async fn a_development_did_is_found_with_its_port_written_either_way() { - // This project mints `did:web:host%3A3400` locally, so naming one in a URL path - // means double-encoding the `%`. A person with curl and a DID out of a - // log writes the bare colon, and that must not be a 404 for an agent - // that is plainly there. - let shared = Shared::new(); - { - let mut view = shared.view.lock().await; - view.see_agent(agent("did:web:kestrel.agents.localhost%3A3400", None)); - } - - for path in [ - "/agents/did%3Aweb%3Akestrel.agents.localhost%253A3400", - "/agents/did%3Aweb%3Akestrel.agents.localhost%3A3400", - ] { - let (status, detail) = get(&shared, path).await; - assert_eq!(status, StatusCode::OK, "{path}"); - assert_eq!( - detail["agent"]["did"], - "did:web:kestrel.agents.localhost%3A3400" - ); - } - - // And it is still a 404 when there is genuinely no such agent. - let (status, _) = get(&shared, "/agents/did%3Aweb%3Aother.example%3A3400").await; - assert_eq!(status, StatusCode::NOT_FOUND); -} - -#[tokio::test] -async fn a_filter_naming_a_development_did_is_repaired_too() { - // A filter that silently matched nothing would be worse than the 404 - // above: a caller reads an empty page as "no such children" rather than - // as "you spelled the DID the other way". - let shared = Shared::new(); - { - let mut view = shared.view.lock().await; - let root = "did:web:root.agents.localhost%3A3400"; - view.see_agent(agent(root, None)); - view.see_agent(agent("did:web:leaf.agents.localhost%3A3400", Some(root))); - } - - for spelling in [ - "did%3Aweb%3Aroot.agents.localhost%253A3400", - "did%3Aweb%3Aroot.agents.localhost%3A3400", - ] { - let (_, page) = get(&shared, &format!("/agents?parent={spelling}")).await; - let items = page["items"].as_array().expect("items"); - assert_eq!(items.len(), 1, "{spelling} finds the child"); - assert_eq!(items[0]["did"], "did:web:leaf.agents.localhost%3A3400"); - } - - // The empty parent still means "only the session roots" and is not a DID - // that anything tries to repair. - let (_, roots) = get(&shared, "/agents?parent=").await; - assert_eq!(roots["items"].as_array().expect("items").len(), 1); -} diff --git a/crates/didbot-stack/src/defaults.rs b/crates/didbot-stack/src/defaults.rs index 8b369c7a..454cbedf 100644 --- a/crates/didbot-stack/src/defaults.rs +++ b/crates/didbot-stack/src/defaults.rs @@ -80,48 +80,6 @@ pub(crate) fn builtin() -> Config { names: None, }, ); - services.insert( - "index".to_owned(), - Service { - kind: ServiceKind::Index, - port: 3002, - host: host(), - pds: Some("pds".to_owned()), - index: None, - query: None, - zone: None, - data: None, - names: None, - }, - ); - services.insert( - "query".to_owned(), - Service { - kind: ServiceKind::Query, - port: 3003, - host: host(), - index: Some("index".to_owned()), - pds: None, - query: None, - zone: None, - data: None, - names: None, - }, - ); - services.insert( - "web".to_owned(), - Service { - kind: ServiceKind::Web, - port: 8137, - host: host(), - query: Some("query".to_owned()), - pds: None, - index: None, - zone: None, - data: None, - names: None, - }, - ); let mut profiles = BTreeMap::new(); profiles.insert( @@ -129,9 +87,14 @@ pub(crate) fn builtin() -> Config { Profile { pds: "pds".to_owned(), mcp: Some("mcp".to_owned()), - index: Some("index".to_owned()), - query: Some("query".to_owned()), - web: Some("web".to_owned()), + // The index, the query service and the canvas are + // vibescrobble.com's processes now. `ServiceKind` still names + // them, so a developer running both can declare them in their + // own profile; what is built in is what this repository can + // start. + index: None, + query: None, + web: None, }, ); @@ -160,8 +123,6 @@ mod tests { for (script, variable, service) in [ ("dev-pds.sh", "DIDBOT_PDS_PORT", "pds"), ("dev-mcp.sh", "DIDBOT_MCP_PORT", "mcp"), - ("dev-index.sh", "DIDBOT_INDEX_PORT", "index"), - ("dev-query.sh", "DIDBOT_QUERY_PORT", "query"), ] { let source = std::fs::read_to_string(root.join(script)) .unwrap_or_else(|err| panic!("read {script}: {err}")); diff --git a/scripts/dev-index.sh b/scripts/dev-index.sh deleted file mode 100755 index 9dad8f3b..00000000 --- a/scripts/dev-index.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# Rebuild and run the index in the foreground. -# -# It follows the development personal data server, holds what it finds, and -# serves that to the query service. This terminal is where you watch things -# being taken in: an account appearing, a repository being read, a survey -# reconciling what a stream missed. -set -euo pipefail - -cd "$(dirname "$0")/.." - -# This directory's profile, if it has one. See dev-profile.sh. -. "$(dirname "$0")/dev-profile.sh" -. "$(dirname "$0")/dev-watch.sh" - -PORT="${DIDBOT_INDEX_PORT:-3002}" -PDS_PORT="${DIDBOT_PDS_PORT:-3000}" -PDS_URL="${DIDBOT_PDS_URL:-http://localhost:${PDS_PORT}}" - -say() { printf '\033[1m%s\033[0m\n' "$*"; } - -# `--watch` reruns this script whenever the source moves. It never returns. -watch_or_continue "$@" - -say "building" -cargo test --quiet -p didbot-index - -# Warned about rather than waited for. The index runs perfectly well with the -# server down — it retries — and blocking here would make the order the -# terminals are started in matter, which it should not. -if ! curl -sf "${PDS_URL}/health" >/dev/null 2>&1; then - say "note: no personal data server answering at ${PDS_URL}" - echo " the index will keep trying; start ./scripts/dev-pds.sh" - echo -fi - -exec cargo run --quiet -p didbot-index --bin didbot-index -- \ - --pds "${PDS_URL}" --listen "127.0.0.1:${PORT}" "$@" diff --git a/scripts/dev-query.sh b/scripts/dev-query.sh deleted file mode 100755 index 1322aeaf..00000000 --- a/scripts/dev-query.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Rebuild and run the query service in the foreground. -# -# It mirrors the index and serves the browser. This terminal is quiet by -# design: if it is saying anything other than that it seeded, something -# upstream is wrong. -set -euo pipefail - -cd "$(dirname "$0")/.." - -# This directory's profile, if it has one. See dev-profile.sh. -. "$(dirname "$0")/dev-profile.sh" -. "$(dirname "$0")/dev-watch.sh" - -PORT="${DIDBOT_QUERY_PORT:-3003}" -INDEX_PORT="${DIDBOT_INDEX_PORT:-3002}" -INDEX_URL="${DIDBOT_INDEX_URL:-http://127.0.0.1:${INDEX_PORT}}" - -say() { printf '\033[1m%s\033[0m\n' "$*"; } - -# `--watch` reruns this script whenever the source moves. It never returns. -watch_or_continue "$@" - -say "building" -cargo test --quiet -p didbot-query - -if ! curl -sf "${INDEX_URL}/health" >/dev/null 2>&1; then - say "note: no index answering at ${INDEX_URL}" - echo " the query service will keep asking; start ./scripts/dev-index.sh" - echo -fi - -exec cargo run --quiet -p didbot-query --bin didbot-query -- \ - --index "${INDEX_URL}" --listen "127.0.0.1:${PORT}" "$@" diff --git a/scripts/dev-web.sh b/scripts/dev-web.sh deleted file mode 100755 index 0814ed10..00000000 --- a/scripts/dev-web.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# Serve the appview canvas on a local port. -# -# Static files and nothing else: the page runs on mocked events out of -# web/mock.js, so it needs no personal data server and no appview behind it. -# Python's server is used rather than anything installed, because the whole -# point of the page having no build step is that it has no build step. -set -euo pipefail - -cd "$(dirname "$0")/.." - -# This directory's profile, if it has one. See dev-profile.sh. -. "$(dirname "$0")/dev-profile.sh" -. "$(dirname "$0")/dev-watch.sh" - -cd web - -PORT="${DIDBOT_WEB_PORT:-8137}" - -# `--watch` reruns this script whenever the source moves. It never returns. -# The canvas has no build step, so watching it only matters for the banner — -# the page itself is reloaded by the browser. -watch_or_continue "$@" - -printf '\033[1m%s\033[0m\n' "appview canvas on http://localhost:${PORT}/" -echo " mocked data; nothing is fetched" -echo - -exec python3 -m http.server "${PORT}" --bind 127.0.0.1 diff --git a/web/app.js b/web/app.js deleted file mode 100644 index c8416e5f..00000000 --- a/web/app.js +++ /dev/null @@ -1,2076 +0,0 @@ -/* The appview canvas. - * - * Geometry is drawn in world space under a camera transform; every piece of - * text is drawn afterwards in screen space, at a fixed size, from a projected - * point. That split is the whole readability trick: the shape of a few hundred - * agents can be zoomed out to nothing, and the words on top stay legible. - * - * Nothing here knows where its events come from. `apply` takes the shapes the - * personal data server's lifecycle stream and scrobble records already have, - * so a live subscription replaces mock.js and leaves this file alone. - */ - -(() => { - "use strict"; - - // --- layout constants ----------------------------------------------------- - - /** Distance between operator hubs, in world units. */ - const HUB_SPACING = 360; - /** Golden angle, for placing the hubs. */ - const GOLDEN = 2.399963229728653; - - // --- the simulation ------------------------------------------------------- - // - // Two attractions pull on every agent at once, and the shape on screen is - // the argument between them. Its household pulls it home: a spring along the - // link that spawned it, and a weak gravity toward the operator's hub. Its - // work pulls it away: a spring toward the middle of whatever other agents - // are saying the same thing, wherever they are hosted. Local repulsion keeps - // the result from collapsing into a point. - // - // Turn the work pull down and the picture is thirty separate households. - // Turn it up and threads reach between them, because agents on the same job - // are rarely on the same server. The interesting settings are in between, - // where a cluster holds its shape and still puts out processes toward its - // neighbours. - - /** Rest length of the spring from a session root to its operator's hub. */ - const REST_ROOT = 96; - /** Rest length of the spring from a subagent to whatever spawned it. */ - const REST_CHILD = 48; - /** Stiffness of those springs. */ - const K_SPRING = 0.09; - /** Gravity toward the operator's hub. Weak: the springs do the shaping. */ - const K_HUB = 0.013; - /** Stiffness of the springs along a work fibre, at full strength. */ - const K_WORK = 0.055; - /** Rest length of those springs. */ - const REST_WORK = 86; - /** Ceiling on one fibre spring, so a long thread cannot drag a household - * apart faster than its own springs can hold it together. */ - const MAX_WORK_FORCE = 1.6; - /** Cohesion toward a cluster's own middle, which only matters at high pull. */ - const K_WORK_CENTRE = 0.05; - /** Most neighbours one agent is pushed away from in a single tick. */ - const MAX_NEIGHBOURS = 24; - /** Agents closer than this push each other apart. */ - const REPULSE_RADIUS = 26; - /** How hard they push. */ - const K_REPULSE = 1.15; - /** Velocity kept between frames. */ - const DAMPING = 0.86; - /** Ceiling on speed, so nothing can be flung off the canvas. */ - const MAX_SPEED = 26; - /** - * How strongly a branch holds the shape it grew in. - * - * This is what makes the trees dendritic rather than merely connected. A - * plain spring only says how far a child should be from its parent, so a - * subtree settles into whatever knot the repulsion leaves room for. This - * says where: out along the direction the parent itself grew, turned by a - * fixed angle that siblings share out between them. Branches then keep - * going the way they were heading, split at a consistent angle, and get - * shorter each generation — which is most of what a dendrite is. - */ - const K_DENDRITE = 0.075; - /** How far apart siblings are set, in radians, at the first branching. */ - const BRANCH_ANGLE = 0.62; - /** How hard a foreign operator's hub pushes an agent away. */ - const K_FOREIGN = 900; - /** Beyond this an operator's hub ignores other people's agents. */ - const FOREIGN_RADIUS = 150; - /** Physics ticks per drawn frame. */ - const SUBSTEPS = 4; - /** How quickly the drawn position catches up with the simulated one. */ - const SMOOTHING = 0.28; - /** How long a scrobble's words stay up, in seconds. */ - const TEXT_TTL = 3.6; - /** - * How long its emoji stays up: ten times longer. - * - * The words are the detail and they have to clear or the screen is - * unreadable. The emoji is the trace — one glyph, no reading required — so - * it can hang around, and a few hundred of them drifting is the record of - * what has just happened everywhere at once. - */ - const EMOJI_TTL = 36; - /** Most floats kept alive at once. Past this the oldest are dropped. */ - const FLOAT_CAP = 420; - /** How many floats carry their text as well as their emoji. */ - const FEATURED = 6; - /** Seconds an agent stays lit after it scrobbles. */ - const HEAT_DECAY = 6; - /** Zoom at which agent labels start being drawn. */ - const LABEL_ZOOM = 1.15; - /** The resolution an operator's picture is generated at. */ - const PICTURE_SIZE = 72; - - /** - * How much bigger an agent is drawn for the effort it reported. - * - * The harness reports what a turn actually cost, so this is the one number - * on a scrobble that is about the work rather than about the words. Size is - * the channel it gets: it survives being zoomed out, it needs no legend, - * and a field where the expensive agents are visibly the large ones answers - * "where is the money going" at a glance. - * - * Size was free to give it because the thing it used to encode — how many - * agents something spawned — is already in the thickness of the branch - * below it. - */ - const EFFORT_SCALE = { - low: 0.7, - medium: 1, - high: 1.32, - xhigh: 1.68, - max: 2.1, - }; - - // --- state ---------------------------------------------------------------- - - const canvas = document.getElementById("stage"); - const ctx = canvas.getContext("2d"); - const tooltip = document.getElementById("tooltip"); - const logList = document.getElementById("log"); - - const camera = { x: 0, y: 0, scale: 0.5 }; - const view = { width: 0, height: 0, dpr: 1 }; - - const state = { - operators: new Map(), - agents: new Map(), - floats: [], - scrobbleTimes: [], - totals: { agents: 0, scrobbles: 0, depth: 0 }, - options: { links: true, labels: true, text: true, work: true, emoji: true }, - /** How hard shared work pulls agents together, 0 to 1. */ - workPull: 0.92, - work: { clusters: [], of: new Map() }, - focus: null, - hover: null, - query: "", - matches: 0, - paused: false, - // Set once the operator moves the camera themselves, after which the - // view stops re-fitting itself on a window resize. - touched: false, - rate: 12, - now: 0, - /** Smoothed cost of one frame's physics, in milliseconds. */ - stepMs: 0, - }; - - /** - * Where the events come from. - * - * Live if a query service answers, invented if one does not. Both hand over - * the same shapes, so nothing below this line knows the difference — which - * is what the mock's event shapes were copied from the server for. - */ - let feed = new Mock.MockFeed({ operators: 30, agents: 420 }); - let living = false; - const workIndex = new Work.WorkIndex(); - - /** How often the whole population is reclustered, in seconds. */ - const RECLUSTER = 0.8; - /** How many work clusters get drawn on the canvas at once. */ - const DRAWN_CLUSTERS = 8; - - // --- events --------------------------------------------------------------- - - /** Applies one event. `live` is false while seeding the backlog. */ - function apply(event, live) { - switch (event.kind) { - case "operator": - addOperator(event); - break; - case "provisioned": - // The index re-announces an agent when it learns something new about - // it, and the thing it learns late is lineage: a scrobble names its - // author's parent, and the account was provisioned before it spoke. - // So this is an arrival the first time and a reparent afterwards. - if (state.agents.has(event.did)) reparent(event); - else addAgent(event, live); - break; - case "scrobble": - addScrobble(event, live); - break; - case "deleted": - endAgent(event.did, live); - break; - case "work": - adoptGroups(event.groups); - break; - case "pinned": - case "unpinned": { - const agent = state.agents.get(event.did); - if (agent) agent.pinned = event.kind === "pinned"; - break; - } - default: - break; - } - if (live) note(event); - } - - function addOperator(event) { - const index = state.operators.size; - const angle = index * GOLDEN; - // The picture is made first because the colour is read off it. An - // operator who changes their avatar changes the colour of every agent - // they run, which is the right way round: the colour is a property of the - // picture, not a number assigned to the account. - const picture = Avatar.make(event.did, event.handle, PICTURE_SIZE); - const hue = Avatar.dominantHue(picture); - const radius = HUB_SPACING * Math.sqrt(index + 0.55); - // Stretched sideways, because a screen is. A round phyllotaxis fills a - // 16:9 viewport by leaving both corners empty and shrinking everything - // until the tallest part fits. - state.operators.set(event.did, { - did: event.did, - handle: event.handle, - zone: event.zone, - picture, - hue: hue === null ? Avatar.hueOf(event.did) : hue, - x: Math.cos(angle) * radius * 1.45, - y: Math.sin(angle) * radius * 0.82, - // Hubs do not move, so their drawn position is their position. Named - // the same as an agent's so every drawing routine can take either. - rx: Math.cos(angle) * radius * 1.45, - ry: Math.sin(angle) * radius * 0.82, - roots: [], - agents: 0, - phase: index * 1.7, - }); - } - - function addAgent(event, live) { - const operator = state.operators.get(event.operator); - if (!operator) return; - const parent = event.parent ? state.agents.get(event.parent) : null; - const depth = parent ? parent.depth + 1 : 0; - const agent = { - did: event.did, - agentId: event.agentId, - // What the agent is called, and what it is keyed on. An account issued - // a handle is drawn by its name everywhere a person reads the canvas; - // one that was never named falls back to the label its DID was minted - // from, which is what the whole canvas showed before names existed. - name: event.name || event.agentId, - handle: event.handle || null, - operator, - parent, - depth, - children: [], - // Size of everything below this agent, kept up to date as the tree - // changes. Branch thickness comes from it. - weight: 1, - // Born next to whatever spawned it, with a nudge, and then argued into - // position by the simulation. - x: 0, - y: 0, - vx: 0, - vy: 0, - // What is drawn, chasing the simulated position. The simulation is - // allowed to be twitchy; the picture is not. - rx: 0, - ry: 0, - born: state.now, - dying: 0, - heat: live ? 1 : 0, - scrobbles: 0, - pinned: false, - emoji: "", - phase: Math.random() * 6.28, - }; - // Where this branch turns relative to the one it grew from. Siblings - // share the angle out between them, alternating sides, so a parent with - // three children makes a fan and not a bundle. - const rank = parent ? parent.children.length : operator.roots.length; - agent.branch = - parent === null - ? rank * GOLDEN - : (rank % 2 === 0 ? 1 : -1) * BRANCH_ANGLE * (1 + Math.floor(rank / 2) * 0.55); - - const from = parent || operator; - const scatter = parent ? REST_CHILD : REST_ROOT; - const bearing = parent ? agent.branch : rank * GOLDEN; - agent.x = from.x + Math.cos(bearing) * scatter; - agent.y = from.y + Math.sin(bearing) * scatter; - agent.rx = agent.x; - agent.ry = agent.y; - - state.agents.set(agent.did, agent); - if (parent) parent.children.push(agent); - else operator.roots.push(agent); - reweigh(agent.parent, 1); - operator.agents += 1; - state.totals.agents += 1; - state.totals.depth = Math.max(state.totals.depth, depth); - } - - function addScrobble(event, live) { - const agent = state.agents.get(event.did); - if (!agent) return; - agent.scrobbles += 1; - // Seeded scrobbles are backlog: they leave the agent warm rather than - // fully lit, so the first frame shows a population with a history and not - // a field of dots that all fired at once. - agent.heat = live ? 1 : 0.15 + Math.random() * 0.5; - agent.emoji = event.emoji; - agent.text = event.text; - // Unknown levels are carried rather than rejected: the lexicon calls this - // an open set, and an effort this page has not heard of should still draw - // as something rather than vanish. - if (event.effort) agent.effort = event.effort; - // Handed over whether or not a panel is open: an agent somebody clicks on - // has usually been talking for a while before anybody looked at it, and - // the run-up is most of what makes the panel worth opening. - Detail.record(agent.did, { - emoji: event.emoji, - text: event.text, - effort: event.effort, - at: state.now, - }); - // Only the mock's page clusters locally; a live page is told its groups. - if (!living) workIndex.observe(agent.did, event.text); - state.totals.scrobbles += 1; - if (!live) return; - state.scrobbleTimes.push(state.now); - for (const float of state.floats) float.featured = false; - state.floats.push({ - agent, - emoji: event.emoji, - text: event.text, - born: state.now, - drift: (Math.random() - 0.5) * 26, - featured: true, - }); - let featured = 0; - for (let i = state.floats.length - 1; i >= 0 && featured < FEATURED; i -= 1) { - state.floats[i].featured = true; - featured += 1; - } - if (state.floats.length > FLOAT_CAP) { - state.floats.splice(0, state.floats.length - FLOAT_CAP); - } - } - - function endAgent(did, live) { - const agent = state.agents.get(did); - if (!agent || agent.dying) return; - if (!live) { - forget(agent); - return; - } - agent.dying = state.now; - } - - /** Adds `delta` to an agent's ancestors, which is where taper comes from. */ - function reweigh(agent, delta) { - for (let node = agent; node; node = node.parent) { - node.weight += delta; - } - } - - /** - * Moves an agent under a different parent, or out to a session root. - * - * The same surgery `forget` does when an agent's parent goes away, driven - * from the other end: detach from wherever it currently hangs, take its - * weight off its old ancestors, attach it to the new one and give the - * weight back. The subtree comes with it, which is the whole point — an - * agent usually learns its parent after its own children have arrived, and - * a reparent that dropped them would flatten the tree it was fixing. - * - * Position is deliberately not reset. The simulation pulls the agent toward - * its new parent over the next second or so, and a branch that snapped into - * place would read as a glitch rather than as something being learned. - */ - function reparent(event) { - const agent = state.agents.get(event.did); - if (!agent) return; - const parent = event.parent ? state.agents.get(event.parent) || null : null; - if (agent.parent === parent) return; - // An agent cannot hang below itself. The index refuses to publish a cycle, - // so this is a guard against a stale event rather than an expected case. - for (let node = parent; node; node = node.parent) { - if (node === agent) return; - } - - const before = agent.parent ? agent.parent.children : agent.operator.roots; - const at = before.indexOf(agent); - if (at >= 0) before.splice(at, 1); - reweigh(agent.parent, -agent.weight); - - agent.parent = parent; - const after = parent ? parent.children : agent.operator.roots; - const rank = after.length; - after.push(agent); - reweigh(parent, agent.weight); - - // Its branch angle is its place among its new siblings, and everything - // below it moves down a generation with it. - agent.branch = - parent === null - ? rank * GOLDEN - : (rank % 2 === 0 ? 1 : -1) * BRANCH_ANGLE * (1 + Math.floor(rank / 2) * 0.55); - redepth(agent, parent ? parent.depth + 1 : 0); - } - - /** Sets an agent's depth and walks the change down through its children. */ - function redepth(agent, depth) { - agent.depth = depth; - state.totals.depth = Math.max(state.totals.depth, depth); - for (const child of agent.children) redepth(child, depth + 1); - } - - function forget(agent) { - const siblings = agent.parent ? agent.parent.children : agent.operator.roots; - const index = siblings.indexOf(agent); - if (index >= 0) siblings.splice(index, 1); - - // A parent can outlive its own removal only as a gap in the tree, so its - // children are handed up rather than left springing toward a dead node. - reweigh(agent.parent, -agent.weight); - for (const child of agent.children) { - child.parent = agent.parent; - if (agent.parent) agent.parent.children.push(child); - else agent.operator.roots.push(child); - reweigh(agent.parent, child.weight); - } - agent.children = []; - - agent.operator.agents -= 1; - state.agents.delete(agent.did); - workIndex.forget(agent.did); - Detail.forget(agent.did); - if (state.focus && state.focus.did === agent.did) setFocus(null); - } - - // --- the rolling log ------------------------------------------------------ - - const LOG_LIMIT = 34; - - function note(event) { - const agent = state.agents.get(event.did); - let who = ""; - let what = ""; - if (event.kind === "provisioned") { - who = event.name || event.agentId; - what = event.parent ? `spawned by ${labelOf(event.parent)}` : `session on ${event.operatorHandle}`; - } else if (event.kind === "scrobble") { - who = `${event.emoji} ${labelOf(event.did)}`; - what = event.text; - } else if (event.kind === "deleted") { - who = labelOf(event.did); - what = "ended"; - } else if (event.kind === "pinned") { - who = labelOf(event.did); - what = "pinned"; - } else { - return; - } - if (agent && agent.dying) return; - - const item = document.createElement("li"); - item.className = `kind-${event.kind}`; - const whoEl = document.createElement("span"); - whoEl.className = "who"; - whoEl.textContent = who; - const whatEl = document.createElement("span"); - whatEl.className = "what"; - whatEl.textContent = what; - item.append(whoEl, whatEl); - logList.prepend(item); - while (logList.children.length > LOG_LIMIT) { - logList.lastElementChild.remove(); - } - } - - /** The readable part of a DID: the agent label, or the DID if it is odd. */ - function labelOf(did) { - const agent = state.agents.get(did); - if (agent) return agent.name; - const host = did.replace("did:web:", ""); - return host.split(".")[0] || did; - } - - // --- layout and drawing --------------------------------------------------- - - /** - * The centre of gravity of each work cluster, recomputed every frame. - * - * Kept as a map from cluster id so both the simulation and the labels read - * the same point, and so a cluster whose members are scattered across six - * operators still has one place to pull toward. - */ - const workCentres = new Map(); - - function measureWork() { - workCentres.clear(); - if (state.workPull <= 0 && !state.options.work) return; - for (const agent of state.agents.values()) { - const id = state.work.of.get(agent.did); - if (id === undefined) continue; - let centre = workCentres.get(id); - if (!centre) { - centre = { x: 0, y: 0, count: 0 }; - workCentres.set(id, centre); - } - centre.x += agent.rx; - centre.y += agent.ry; - centre.count += 1; - } - for (const centre of workCentres.values()) { - centre.x /= centre.count; - centre.y /= centre.count; - } - } - - /** - * A uniform grid over the agents, rebuilt each frame. - * - * Repulsion only has to reach a couple of node widths, so every pair worth - * computing is in the same cell or one beside it. Without this the tick is - * quadratic and five hundred agents is a hundred and twenty thousand pairs; - * with it the work is proportional to the population. - */ - const grid = new Map(); - - /** - * A cell's key as one integer rather than a string. - * - * Building `"12:-7"` for every agent in every substep is thousands of short - * strings a frame, and the garbage collector eventually asks for the frame - * back. Packing the two coordinates into a number keeps the whole tick - * allocation-free. - */ - const CELL_SPAN = 100000; - - function cellKey(x, y) { - return ( - (Math.floor(x / REPULSE_RADIUS) + CELL_SPAN / 2) * CELL_SPAN + - (Math.floor(y / REPULSE_RADIUS) + CELL_SPAN / 2) - ); - } - - function fillGrid() { - grid.clear(); - for (const agent of state.agents.values()) { - const key = cellKey(agent.x, agent.y); - let cell = grid.get(key); - if (!cell) { - cell = []; - grid.set(key, cell); - } - cell.push(agent); - } - } - - /** - * Advances the simulation and then eases the drawn positions toward it. - * - * Several small ticks rather than one big one: the springs are stiff enough - * that a single step at a low frame rate overshoots and the whole picture - * rings. Three cheap ticks are steadier than one expensive one, and the - * repulsion grid is rebuilt for each because that is what makes them cheap. - */ - function step(dt) { - const started = performance.now(); - const slice = Math.min(2, dt * 60) / SUBSTEPS; - // Once per frame, not once per tick: a cluster's centre does not move far - // enough inside one frame to be worth four passes over the population. - measureWork(); - for (let i = 0; i < SUBSTEPS; i += 1) tick(slice); - // Kept so the physics budget is something you can read rather than guess. - state.stepMs = state.stepMs * 0.9 + (performance.now() - started) * 0.1; - - for (const agent of state.agents.values()) { - agent.rx += (agent.x - agent.rx) * SMOOTHING; - agent.ry += (agent.y - agent.ry) * SMOOTHING; - } - } - - function tick(scale) { - fillGrid(); - - // How much of the household forces survive at this setting of the work - // pull. Never all the way to zero: a session with nothing holding it - // together stops being legible as a session. - const homeward = 1 - 0.72 * state.workPull; - - for (const agent of state.agents.values()) { - let fx = 0; - let fy = 0; - - // The link that spawned it: a session root hangs off its operator, a - // subagent off whatever asked it to exist. - const anchor = agent.parent || agent.operator; - const rest = agent.parent ? REST_CHILD : REST_ROOT; - let dx = anchor.x - agent.x; - let dy = anchor.y - agent.y; - let distance = Math.hypot(dx, dy) || 0.001; - const pull = K_SPRING * homeward * (distance - rest); - fx += (dx / distance) * pull; - fy += (dy / distance) * pull; - - // Its household. Weak, and over any distance, so a cluster that gets - // stretched toward its work still knows where home is. It gives way as - // the work pull rises: at the top of the slider the question being asked - // is "who is working on this", and holding households together would be - // answering a different one. - dx = agent.operator.x - agent.x; - dy = agent.operator.y - agent.y; - fx += dx * K_HUB * homeward; - fy += dy * K_HUB * homeward; - - // Grow the way the branch was already growing, turned by this agent's - // own share of the angle. The target is a point, not a nudge: out from - // the parent, along the parent's direction of growth, at the rest - // length for this generation. - { - const grand = agent.parent ? agent.parent.parent || agent.parent.operator : null; - let ax; - let ay; - if (grand) { - ax = anchor.x - grand.x; - ay = anchor.y - grand.y; - } else { - // A session root has no grain to follow, so it takes the bearing it - // was born with and the hub becomes a soma with dendrites off it. - ax = Math.cos(agent.branch); - ay = Math.sin(agent.branch); - } - const reach = Math.hypot(ax, ay) || 0.001; - ax /= reach; - ay /= reach; - const turn = grand ? agent.branch : 0; - const cos = Math.cos(turn); - const sin = Math.sin(turn); - const tx = anchor.x + (ax * cos - ay * sin) * rest; - const ty = anchor.y + (ax * sin + ay * cos) * rest; - // Only partly softened by the work pull. The other household forces - // are about where a branch sits and should give way; this one is - // about what shape it is, and a branch dragged across the canvas by - // its work should arrive still looking like a branch. - const shaping = 0.4 + 0.6 * homeward; - fx += (tx - agent.x) * K_DENDRITE * shaping; - fy += (ty - agent.y) * K_DENDRITE * shaping; - } - - // Other people's hubs push back, so two households that drift together - // stay two households rather than one smear. - for (const operator of state.operators.values()) { - if (operator === agent.operator) continue; - dx = agent.x - operator.x; - dy = agent.y - operator.y; - const squared = dx * dx + dy * dy; - if (squared > FOREIGN_RADIUS * FOREIGN_RADIUS) continue; - distance = Math.sqrt(squared) || 0.001; - const push = K_FOREIGN / (squared * distance); - fx += dx * push; - fy += dy * push; - } - - // Its cluster's own middle. Weighted by the square of the pull, so it - // is nothing at the settings where the fibre alone reads well and takes - // over at the top, where the point is to see a work group as one object - // rather than as a thread. A chain gathers members; only a centre packs - // them. - if (state.workPull > 0.35) { - const id = state.work.of.get(agent.did); - const centre = id === undefined ? null : workCentres.get(id); - if (centre && centre.count > 2) { - dx = centre.x - agent.x; - dy = centre.y - agent.y; - distance = Math.hypot(dx, dy) || 0.001; - const strength = K_WORK_CENTRE * state.workPull * state.workPull * Math.sqrt(distance); - fx += (dx / distance) * strength; - fy += (dy / distance) * strength; - } - } - - // Everything nearby, pushing back. Bounded: a work group packed tight - // at full pull can put fifty agents in one cell, and the tick must not - // get quadratic inside a knot of its own making. - let neighbours = 0; - const cx = Math.floor(agent.x / REPULSE_RADIUS); - const cy = Math.floor(agent.y / REPULSE_RADIUS); - for (let ox = -1; ox <= 1 && neighbours < MAX_NEIGHBOURS; ox += 1) { - for (let oy = -1; oy <= 1 && neighbours < MAX_NEIGHBOURS; oy += 1) { - const cell = grid.get( - (cx + ox + CELL_SPAN / 2) * CELL_SPAN + (cy + oy + CELL_SPAN / 2) - ); - if (!cell) continue; - for (const other of cell) { - if (other === agent) continue; - if (neighbours >= MAX_NEIGHBOURS) break; - dx = agent.x - other.x; - dy = agent.y - other.y; - const squared = dx * dx + dy * dy; - if (squared > REPULSE_RADIUS * REPULSE_RADIUS) continue; - distance = Math.sqrt(squared) || 0.001; - const push = (K_REPULSE * (REPULSE_RADIUS - distance)) / distance; - fx += dx * push; - fy += dy * push; - neighbours += 1; - } - } - } - - agent.fx = fx; - agent.fy = fy; - } - - // Work, as springs along each cluster's fibre rather than a pull toward - // its middle. A shared centre is the wrong model: it is a point in empty - // space that every member is dragged to, so clusters collapse inward and - // the picture becomes a wheel. A chain through the members has no centre, - // costs one spring per agent, and lets a group of six on the same job - // stretch into a thread between the households running them. - if (state.workPull > 0) { - const pull = state.workPull; - // Everything about the work spring scales together, and faster than the - // slider does: stiffer, allowed to pull harder, and wanting the members - // closer. At the top the fibre wins outright and a work group collapses - // into a knot of mixed colours, which is the honest picture of six - // people's agents on one problem. - const stiffness = K_WORK * pull * (1 + 3 * pull); - const cap = MAX_WORK_FORCE * (1 + 9 * pull); - const rest = REST_WORK * (1 - 0.6 * pull); - for (const cluster of state.work.clusters) { - const path = cluster.path; - if (!path || path.length < 2) continue; - for (let i = 1; i < path.length; i += 1) { - const a = state.agents.get(path[i - 1]); - const b = state.agents.get(path[i]); - if (!a || !b) continue; - const dx = b.x - a.x; - const dy = b.y - a.y; - const distance = Math.hypot(dx, dy) || 0.001; - const force = clamp(stiffness * (distance - rest), -cap, cap); - const pull = force / distance; - a.fx += dx * pull; - a.fy += dy * pull; - b.fx -= dx * pull; - b.fy -= dy * pull; - } - } - } - - for (const agent of state.agents.values()) { - const fx = agent.fx; - const fy = agent.fy; - agent.vx = (agent.vx + fx * scale) * DAMPING; - agent.vy = (agent.vy + fy * scale) * DAMPING; - const speed = Math.hypot(agent.vx, agent.vy); - if (speed > MAX_SPEED) { - agent.vx = (agent.vx / speed) * MAX_SPEED; - agent.vy = (agent.vy / speed) * MAX_SPEED; - } - agent.x += agent.vx * scale; - agent.y += agent.vy * scale; - } - } - - function toScreen(x, y) { - return { - x: (x - camera.x) * camera.scale + view.width / 2, - y: (y - camera.y) * camera.scale + view.height / 2, - }; - } - - function toWorld(x, y) { - return { - x: (x - view.width / 2) / camera.scale + camera.x, - y: (y - view.height / 2) / camera.scale + camera.y, - }; - } - - /** - * Orders a cluster's members into a chain, nearest neighbour first. - * - * Greedy and good enough: start at the member furthest from the middle and - * keep taking the closest one not yet used. The result is a path that - * mostly does not cross itself, which is what makes the fibre readable and - * stops the springs from tying the cluster in a knot. Quadratic in the size - * of one cluster, which is a couple of dozen agents, and it runs once a - * second rather than once a frame. - */ - function fibre(members) { - const nodes = []; - for (const did of members) { - const agent = state.agents.get(did); - if (agent) nodes.push(agent); - } - if (nodes.length < 2) return nodes.map((agent) => agent.did); - - let cx = 0; - let cy = 0; - for (const agent of nodes) { - cx += agent.rx; - cy += agent.ry; - } - cx /= nodes.length; - cy /= nodes.length; - - let start = 0; - let furthest = -1; - nodes.forEach((agent, index) => { - const distance = (agent.rx - cx) ** 2 + (agent.ry - cy) ** 2; - if (distance > furthest) { - furthest = distance; - start = index; - } - }); - - const used = new Array(nodes.length).fill(false); - const path = []; - let current = start; - for (let step = 0; step < nodes.length; step += 1) { - used[current] = true; - path.push(nodes[current].did); - let next = -1; - let best = Infinity; - for (let i = 0; i < nodes.length; i += 1) { - if (used[i]) continue; - const distance = - (nodes[i].rx - nodes[current].rx) ** 2 + (nodes[i].ry - nodes[current].ry) ** 2; - if (distance < best) { - best = distance; - next = i; - } - } - if (next < 0) break; - current = next; - } - return path; - } - - /** - * The hue a group is drawn in. Derived from its id, so it holds still. - * - * Ids from the index are hex strings rather than counters, and the same - * group has to come out the same colour in two people's tabs, so the string - * is folded down rather than parsed: any id that is the same produces the - * same hue, whatever shape it is. - */ - function clusterHue(id) { - const text = String(id); - let hash = 0x811c9dc5; - for (let i = 0; i < text.length; i += 1) { - hash ^= text.charCodeAt(i); - hash = Math.imul(hash, 0x01000193); - } - return (hash >>> 0) % 360; - } - - /** - * The hue one agent is drawn in: always its operator. - * - * The two questions this canvas answers are "whose agents are these" and - * "who is working on the same thing", and they need different channels or - * one of them is unanswerable. Colour is the owner. Position is the work — - * the simulation pulls agents on the same job together, across whichever - * operators happen to be running them. A thread of six colours reaching - * across the canvas is six people on one problem, and it reads as that - * without a legend. - */ - function agentHue(agent) { - return agent.operator.hue; - } - - /** How lit a node is: 1 for the focused chain, low for everything else. */ - function emphasis(agent) { - if (!state.focus) return 1; - if (state.focus.kind === "operator") { - return agent.operator.did === state.focus.did ? 1 : 0.12; - } - if (state.focus.kind === "work") { - return state.work.of.get(agent.did) === state.focus.id ? 1 : 0.09; - } - return inChain(agent) ? 1 : 0.1; - } - - function inChain(agent) { - const target = state.agents.get(state.focus.did); - if (!target) return false; - for (let node = agent; node; node = node.parent) { - if (node === target) return true; - } - for (let node = target; node; node = node.parent) { - if (node === agent) return true; - } - return false; - } - - function matches(agent) { - if (!state.query) return false; - return ( - agent.name.includes(state.query) || - agent.agentId.includes(state.query) || - agent.operator.handle.includes(state.query) - ); - } - - function draw(time) { - ctx.setTransform(view.dpr, 0, 0, view.dpr, 0, 0); - ctx.clearRect(0, 0, view.width, view.height); - - ctx.save(); - ctx.translate(view.width / 2, view.height / 2); - ctx.scale(camera.scale, camera.scale); - ctx.translate(-camera.x, -camera.y); - - if (state.options.links) drawLinks(); - if (state.options.work) drawWork(); - drawAgents(time); - ctx.restore(); - - drawHubs(time); - if (state.options.emoji) drawEmojiNodes(); - drawLabels(); - drawFloats(time); - } - - function drawLinks() { - for (const agent of state.agents.values()) { - // Thickness follows the cube root of everything hanging off this - // branch, which is roughly how a real one is sized: a trunk carrying - // eight tips is twice the width of one carrying one, not eight times. - ctx.lineWidth = (0.7 + 1.5 * Math.cbrt(agent.weight)) / camera.scale; - const alpha = emphasis(agent) * (agent.depth === 0 ? 0.3 : 0.22); - if (alpha < 0.03) continue; - const from = agent.parent || agent.operator; - ctx.strokeStyle = `hsla(${agent.operator.hue}, 45%, 68%, ${alpha * fade(agent)})`; - // Bowed a little, away from the straight line. A field of straight - // spokes reads as a starburst; a field of slight curves reads as growth. - const midX = (from.rx + agent.rx) / 2 - (agent.ry - from.ry) * 0.12; - const midY = (from.ry + agent.ry) / 2 + (agent.rx - from.rx) * 0.12; - ctx.beginPath(); - ctx.moveTo(from.rx, from.ry); - ctx.quadraticCurveTo(midX, midY, agent.rx, agent.ry); - ctx.stroke(); - } - } - - /** - * Draws each cluster as one thread through its members. - * - * The same chain the springs pull along, so the line on screen is the force - * that put the agents where they are rather than a decoration over the top - * of it. - */ - function drawWork() { - const drawn = new Map(); - // A focused operator brings its own threads forward: the answer to "what - // is this person's estate working on, and who else is on it" is the set of - // fibres any of their agents sits on. - const operator = state.focus && state.focus.kind === "operator" ? state.focus.did : null; - const single = state.focus && state.focus.kind === "work" ? state.focus.id : null; - - if (single !== null) { - for (const cluster of state.work.clusters) { - if (cluster.id === single) drawn.set(cluster, 1); - } - } else if (operator !== null) { - // A busy operator is on dozens of jobs at once, and drawing all of them - // is a ball of wool. The ones worth showing are where most of their - // agents are, so the fibres are ranked by how much of this estate is on - // each and the tail is left out. - const ranked = []; - for (const cluster of state.work.clusters) { - let mine = 0; - for (const did of cluster.members) { - const agent = state.agents.get(did); - if (agent && agent.operator.did === operator) mine += 1; - } - if (mine > 0) ranked.push({ cluster, mine }); - } - ranked.sort((a, b) => b.mine - a.mine); - for (const entry of ranked.slice(0, DRAWN_CLUSTERS)) { - drawn.set(entry.cluster, 0.85); - } - } else { - for (const cluster of state.work.clusters) { - if (drawn.size >= DRAWN_CLUSTERS) break; - drawn.set(cluster, 0.3); - } - } - - ctx.lineJoin = "round"; - ctx.lineCap = "round"; - for (const [cluster, weight] of drawn) { - const path = cluster.path || []; - if (path.length < 2) continue; - const points = []; - for (const did of path) { - const agent = state.agents.get(did); - if (agent) points.push(agent); - } - if (points.length < 2) continue; - - const hue = clusterHue(cluster.id); - // Two passes: a wide, nearly invisible one for the bloom, and a thin - // bright one for the line itself. Cheaper than a shadow blur and it - // survives being zoomed. - for (const [width, alpha] of [ - [6 / camera.scale, weight * 0.1], - [(weight > 0.5 ? 1.9 : 1.2) / camera.scale, weight], - ]) { - ctx.strokeStyle = `hsla(${hue}, 72%, 74%, ${alpha})`; - ctx.lineWidth = width; - ctx.beginPath(); - ctx.moveTo(points[0].rx, points[0].ry); - for (let i = 1; i < points.length; i += 1) { - ctx.lineTo(points[i].rx, points[i].ry); - } - ctx.stroke(); - } - - let sumX = 0; - let sumY = 0; - for (const point of points) { - sumX += point.rx; - sumY += point.ry; - } - cluster.cx = sumX / points.length; - cluster.cy = sumY / points.length; - cluster.drawn = true; - } - for (const cluster of state.work.clusters) { - if (!drawn.has(cluster)) cluster.drawn = false; - } - } - - /** - * The size an operator's picture is drawn at, in screen pixels. - * - * Bounded at both ends. A human is a fixed point in a view that zooms over - * two orders of magnitude, and letting the picture scale with the camera - * either loses it at the wide end or turns it into wallpaper at the near - * one. - */ - function hubSize() { - return clamp(40 * camera.scale, 26, 64); - } - - /** - * Draws the humans. - * - * A person is not a bigger dot. They get a picture, a ring in the colour - * every agent of theirs is drawn in, and an arc around it for how much of - * the population they are running — three channels on one object, none of - * which an agent has, so a hub is never mistaken for one at any zoom. - */ - function drawHubs(time) { - const size = hubSize(); - let busiest = 1; - for (const operator of state.operators.values()) { - busiest = Math.max(busiest, operator.agents); - } - - for (const operator of state.operators.values()) { - const lit = !state.focus || state.focus.did === operator.did ? 1 : 0.3; - const point = toScreen(operator.rx, operator.ry); - if (!onScreen(point, size * 2)) continue; - - const radius = size / 2; - const pulse = 1 + 0.03 * Math.sin(time * 0.9 + operator.phase); - - // Lit from behind, so the picture sits above its own branches. - const halo = ctx.createRadialGradient(point.x, point.y, radius, point.x, point.y, radius * 2.4); - halo.addColorStop(0, `hsla(${operator.hue}, 70%, 60%, ${0.34 * lit})`); - halo.addColorStop(1, `hsla(${operator.hue}, 70%, 60%, 0)`); - ctx.fillStyle = halo; - ctx.beginPath(); - ctx.arc(point.x, point.y, radius * 2.4, 0, Math.PI * 2); - ctx.fill(); - - ctx.globalAlpha = lit; - ctx.drawImage(pictureFor(operator), point.x - radius, point.y - radius, size, size); - ctx.globalAlpha = 1; - - ctx.strokeStyle = `hsla(${operator.hue}, 62%, 74%, ${0.95 * lit})`; - ctx.lineWidth = 2.5; - ctx.beginPath(); - ctx.arc(point.x, point.y, radius * pulse + 2, 0, Math.PI * 2); - ctx.stroke(); - - // How much of everything running is theirs. - const share = Math.min(1, operator.agents / busiest); - ctx.strokeStyle = `rgba(120, 130, 145, ${0.3 * lit})`; - ctx.lineWidth = 3; - ctx.beginPath(); - ctx.arc(point.x, point.y, radius + 6, 0, Math.PI * 2); - ctx.stroke(); - ctx.strokeStyle = `hsla(${operator.hue}, 70%, 68%, ${0.9 * lit})`; - ctx.lineCap = "round"; - ctx.beginPath(); - ctx.arc(point.x, point.y, radius + 6, -Math.PI / 2, -Math.PI / 2 + share * Math.PI * 2); - ctx.stroke(); - ctx.lineCap = "butt"; - } - } - - /** - * The operator's picture, made once and kept. - * - * Generated from the DID today. When an appview resolves the account it can - * fetch the avatar its personal data server holds and put the image here - * instead; nothing else on this page has to change, including the colour, - * which is read back off whatever ends up in this slot. - */ - function pictureFor(operator) { - if (!operator.picture) { - operator.picture = Avatar.make(operator.did, operator.handle, PICTURE_SIZE); - const hue = Avatar.dominantHue(operator.picture); - if (hue !== null) operator.hue = hue; - } - return operator.picture; - } - - /** How visible a node is: 0 while it is being born or dying. */ /** How visible a node is: 0 while it is being born or dying. */ - function fade(agent) { - if (agent.dying) { - return Math.max(0, 1 - (state.now - agent.dying) / 1.1); - } - return Math.min(1, (state.now - agent.born) / 0.5); - } - - function drawAgents(time) { - for (const agent of state.agents.values()) { - const lit = emphasis(agent) * fade(agent); - if (lit < 0.02) continue; - const heat = agent.heat; - const hue = agentHue(agent); - // A world radius with a screen-space floor: zoomed all the way out, a - // few hundred agents must still be dots and not dust. - const effort = EFFORT_SCALE[agent.effort] || 1; - const radius = clamp( - (agent.depth === 0 ? 6.2 : 4.4) * effort, - 2.4 / camera.scale, - 16 / camera.scale - ); - const light = 52 + heat * 24 - agent.depth * 4; - - if (heat > 0.02) { - const reach = radius * 6; - ctx.globalAlpha = 0.34 * heat * lit; - ctx.drawImage(glowFor(hue), agent.rx - reach, agent.ry - reach, reach * 2, reach * 2); - ctx.globalAlpha = 1; - } - - // In emoji mode an agent that has said something is drawn as what it - // said; one that has not is still a dot, so a fresh session is visible - // from the moment it exists. - if (!state.options.emoji || !agent.emoji) { - ctx.fillStyle = `hsla(${hue}, ${45 + heat * 25}%, ${light}%, ${lit})`; - ctx.beginPath(); - ctx.arc(agent.rx, agent.ry, radius, 0, Math.PI * 2); - ctx.fill(); - } - - if (agent.pinned) { - ctx.strokeStyle = `hsla(48, 70%, 76%, ${0.8 * lit})`; - ctx.lineWidth = 1.4 / camera.scale; - ctx.beginPath(); - ctx.arc(agent.rx, agent.ry, radius + 2.6, 0, Math.PI * 2); - ctx.stroke(); - } - - // A ring that expands once, as the account is minted. - const age = state.now - agent.born; - if (age < 0.9 && !agent.dying) { - const grow = age / 0.9; - ctx.strokeStyle = `hsla(${hue}, 80%, 78%, ${(1 - grow) * 0.75})`; - ctx.lineWidth = 1.6 / camera.scale; - ctx.beginPath(); - ctx.arc(agent.rx, agent.ry, radius + grow * 26, 0, Math.PI * 2); - ctx.stroke(); - } - - if (matches(agent)) { - ctx.strokeStyle = "rgba(143, 208, 160, 0.95)"; - ctx.lineWidth = 1.6 / camera.scale; - ctx.beginPath(); - ctx.arc(agent.rx, agent.ry, radius + 5, 0, Math.PI * 2); - ctx.stroke(); - } - } - void time; - } - - /** Operator handles always; agent labels only when there is room. */ - function drawLabels() { - occupied = []; - if (!state.options.labels) return; - ctx.textAlign = "center"; - ctx.textBaseline = "middle"; - - // Operators first: they are the structure, and they win a collision. - ctx.font = "12px ui-monospace, monospace"; - for (const operator of state.operators.values()) { - const focused = !state.focus || state.focus.did === operator.did; - const point = toScreen(operator.rx, operator.ry); - if (!onScreen(point, 80)) continue; - // Below the picture rather than above it: the picture is the thing - // being named, and a name over the top of it hides what it names. - const drop = hubSize() / 2 + 18; - const width = ctx.measureText(operator.handle).width; - if (!claim(point.x - width / 2 - 5, point.y + drop - 8, width + 10, 16)) continue; - ctx.fillStyle = `rgba(11, 13, 17, ${focused ? 0.65 : 0.4})`; - ctx.fillRect(point.x - width / 2 - 5, point.y + drop - 8, width + 10, 16); - ctx.fillStyle = focused - ? `hsla(${operator.hue}, 65%, 82%, 0.95)` - : "rgba(120, 130, 145, 0.45)"; - ctx.fillText(operator.handle, point.x, point.y + drop); - } - - if (state.options.work) { - ctx.font = "12px ui-monospace, monospace"; - for (const cluster of state.work.clusters) { - if (!cluster.drawn) continue; - if (cluster.cx === undefined) continue; - const point = toScreen(cluster.cx, cluster.cy); - if (!onScreen(point, 60)) continue; - const caption = `${cluster.label} · ${cluster.size}`; - const width = ctx.measureText(caption).width; - if (!claim(point.x - width / 2 - 6, point.y - 9, width + 12, 18)) continue; - ctx.fillStyle = "rgba(11, 13, 17, 0.78)"; - ctx.fillRect(point.x - width / 2 - 6, point.y - 9, width + 12, 18); - ctx.fillStyle = `hsla(${clusterHue(cluster.id)}, 70%, 80%, 0.95)`; - ctx.fillText(caption, point.x, point.y); - } - } - - if (camera.scale >= LABEL_ZOOM) { - ctx.font = "11px ui-monospace, monospace"; - for (const agent of state.agents.values()) { - const lit = emphasis(agent) * fade(agent); - if (lit < 0.5) continue; - const point = toScreen(agent.rx, agent.ry); - if (!onScreen(point, 60)) continue; - const width = ctx.measureText(agent.name).width; - if (!claim(point.x - width / 2 - 3, point.y - 20, width + 6, 14)) continue; - ctx.fillStyle = `rgba(223, 228, 236, ${0.6 * lit})`; - ctx.fillText(agent.name, point.x, point.y - 13); - } - } - } - - /** - * Pre-rendered glow sprites, one per hue bucket. - * - * A radial gradient built per agent per frame is four hundred gradient - * objects a second thrown away sixty times over, and it shows up as the - * frame rate rather than as anything on screen. Twenty-four sprites drawn - * once and blitted with `drawImage` cost nothing and look identical. - */ - const GLOW_BUCKETS = 24; - const GLOW_SIZE = 96; - const glows = []; - - function buildGlows() { - for (let bucket = 0; bucket < GLOW_BUCKETS; bucket += 1) { - const hue = (bucket * 360) / GLOW_BUCKETS; - const sprite = document.createElement("canvas"); - sprite.width = GLOW_SIZE; - sprite.height = GLOW_SIZE; - const paint = sprite.getContext("2d"); - const half = GLOW_SIZE / 2; - const gradient = paint.createRadialGradient(half, half, 0, half, half, half); - gradient.addColorStop(0, `hsla(${hue}, 80%, 70%, 1)`); - gradient.addColorStop(0.45, `hsla(${hue}, 80%, 70%, 0.35)`); - gradient.addColorStop(1, `hsla(${hue}, 80%, 70%, 0)`); - paint.fillStyle = gradient; - paint.fillRect(0, 0, GLOW_SIZE, GLOW_SIZE); - glows.push(sprite); - } - } - - function glowFor(hue) { - const bucket = Math.round((hue / 360) * GLOW_BUCKETS) % GLOW_BUCKETS; - return glows[bucket]; - } - - /** Keeps a value inside a range. */ - function clamp(value, low, high) { - return Math.min(Math.max(value, low), high); - } - - /** - * Screen rectangles already claimed by text this frame. - * - * Labels and floating text are the only things on the canvas that cannot - * overlap and stay useful, so each one claims a box and anything that would - * land on top of a claim is dropped. Structural text claims first: operator - * handles, then agent labels, then the floating scrobbles. - */ - let occupied = []; - - function claim(x, y, width, height) { - for (const box of occupied) { - if (x < box.x + box.w && x + width > box.x && y < box.y + box.h && y + height > box.y) { - return false; - } - } - occupied.push({ x, y, w: width, h: height }); - return true; - } - - function onScreen(point, margin) { - return ( - point.x > -margin && - point.x < view.width + margin && - point.y > -margin && - point.y < view.height + margin - ); - } - - /** Emoji rise from the agent that scrobbled; only a few carry their text. */ - function drawFloats() { - ctx.textAlign = "left"; - ctx.textBaseline = "middle"; - for (const float of state.floats) { - const life = (state.now - float.born) / EMOJI_TTL; - if (life >= 1) continue; - // Most of the climb happens in the first second or two and then it - // slows, so a fresh scrobble reads as movement and an old one has - // settled into the background rather than sailing off the top. - const lift = 14 + 62 * Math.sqrt(life); - const point = toScreen(float.agent.rx, float.agent.ry); - point.x += float.drift * Math.sqrt(life) * 2.2; - point.y -= lift; - if (!onScreen(point, 140)) continue; - - const emphasised = emphasis(float.agent); - const alpha = (1 - life) ** 1.25 * emphasised; - if (alpha < 0.03) continue; - ctx.font = "12px ui-monospace, monospace"; - ctx.fillStyle = `rgba(255, 255, 255, ${alpha})`; - ctx.fillText(float.emoji, point.x - 7, point.y); - - const said = (state.now - float.born) / TEXT_TTL; - if (!state.options.text || !float.featured || said >= 1) continue; - const readable = (1 - said) ** 2 * emphasised; - ctx.font = "11px ui-monospace, monospace"; - const width = ctx.measureText(float.text).width; - if (!claim(point.x + 10, point.y - 8, width + 10, 16)) continue; - ctx.fillStyle = `rgba(11, 13, 17, ${readable * 0.8})`; - ctx.fillRect(point.x + 10, point.y - 8, width + 10, 16); - ctx.fillStyle = `hsla(${float.agent.operator.hue}, 55%, 84%, ${readable})`; - ctx.fillText(float.text, point.x + 15, point.y); - } - } - - /** - * Draws each agent as the last thing it said, instead of as a dot. - * - * In screen space and at a fixed size, so the glyphs stay readable however - * far out the camera is. The owner's colour is still behind them in the - * glow, which is the only reason this can replace the dot without losing - * the answer to whose agent it is. - */ - function drawEmojiNodes() { - ctx.textAlign = "center"; - ctx.textBaseline = "middle"; - for (const agent of state.agents.values()) { - if (!agent.emoji) continue; - const lit = emphasis(agent) * fade(agent); - if (lit < 0.06) continue; - const point = toScreen(agent.rx, agent.ry); - if (!onScreen(point, 30)) continue; - - // A ring in the owner's colour, because the glyph itself is somebody - // else's palette and the one thing that must survive this mode is being - // able to see whose agent it is. It fades out as the camera pulls back: - // four hundred rings at arm's length is a texture, not information. - const effort = EFFORT_SCALE[agent.effort] || 1; - const ring = (agent.depth === 0 ? 10 : 8.5) * (0.72 + 0.28 * effort); - ctx.globalAlpha = (0.4 + 0.5 * agent.heat) * lit * clamp(camera.scale * 2.2, 0.25, 1); - ctx.strokeStyle = `hsl(${agent.operator.hue}, 60%, 66%)`; - ctx.lineWidth = agent.depth === 0 ? 2 : 1.4; - ctx.beginPath(); - ctx.arc(point.x, point.y, ring, 0, Math.PI * 2); - ctx.stroke(); - - // Set explicitly: a colour emoji brings its own palette, but a glyph - // with no colour version of it — an arrow, a gear — takes this, and - // whatever the last fill happened to be is not a colour to draw with. - ctx.fillStyle = "#f2f5f9"; - ctx.globalAlpha = (0.62 + 0.38 * agent.heat) * lit; - ctx.font = `${(11 + effort * 3.5).toFixed(1)}px ui-monospace, monospace`; - ctx.fillText(agent.emoji, point.x, point.y); - } - ctx.globalAlpha = 1; - } - - // --- the frame loop ------------------------------------------------------- - - let last = performance.now(); - let reclusterDue = 0; - let reclustering = false; - - /** - * Takes the groups the index worked out and draws with them. - * - * Members that this page has not heard of are dropped rather than waited - * for: the index announces a group the moment it recomputes one, and a tab - * that opened a second ago may not have the agent yet. It will on the next - * pass, and a group missing one dot for two seconds is a better answer than - * no group at all. - */ - /** - * A work group named in the address bar that has not arrived yet. - * - * Cleared once it is applied, or once a set of groups arrives without it — - * the group has gone, which a link to an ephemeral thing has to survive. - */ - let wantedGroup = null; - - /** Focuses the group the address bar asked for, if it is here. */ - function applyWantedGroup() { - if (wantedGroup === null) return; - const cluster = state.work.clusters.find((entry) => entry.id === wantedGroup); - if (!cluster) { - // Only give up once groups have actually arrived; before that there is - // nothing to conclude from not finding it. - if (state.work.clusters.length > 0) wantedGroup = null; - return; - } - wantedGroup = null; - setFocus({ kind: "work", id: cluster.id }); - setWork(true); - } - - function adoptGroups(groups) { - const clusters = []; - const of = new Map(); - for (const group of groups || []) { - const members = (group.members || []).filter((did) => state.agents.has(did)); - if (members.length < 2) continue; - const cluster = { id: group.id, label: group.label, members, size: members.length }; - cluster.path = fibre(members); - clusters.push(cluster); - for (const did of members) of.set(did, cluster.id); - } - state.work = { clusters, of }; - renderClusters(); - applyWantedGroup(); - } - - /** - * Reclusters the population, out of the way of the frame. - * - * Only when this page is running on the mock. A live page is told what the - * groups are: the index computes them once so that two people looking at the - * same feed see the same groups under the same ids, and a tab that - * recomputed its own would be back to inventing a private answer — which is - * the whole thing that moving the algorithm behind the index was for. The - * mock has no backend to ask, so it still clusters here. - * - * The pass is the single most expensive thing this page does — a few - * milliseconds over several hundred agents — and running it inside a frame - * costs that frame. `requestIdleCallback` hands it whatever slack the - * browser has instead, with a deadline so it still happens on a busy page. - */ - function scheduleRecluster() { - if (living) return; - if (reclustering || state.now < reclusterDue) return; - reclustering = true; - reclusterDue = state.now + RECLUSTER; - - const run = () => { - const grouped = workIndex.cluster(); - // The index hands out string ids; the local pass hands out counters. - // Made strings here so that nothing downstream — the hue, the address - // bar, the focus comparison — has to know which one it is looking at. - for (const cluster of grouped.clusters) cluster.id = String(cluster.id); - grouped.of = new Map([...grouped.of].map(([did, id]) => [did, String(id)])); - state.work = grouped; - for (const cluster of state.work.clusters) cluster.path = fibre(cluster.members); - renderClusters(); - reclustering = false; - }; - - if (typeof requestIdleCallback === "function") { - requestIdleCallback(run, { timeout: RECLUSTER * 1000 }); - } else { - setTimeout(run, 0); - } - } - - function frame(stamp) { - const dt = Math.min(0.05, (stamp - last) / 1000); - last = stamp; - state.now += dt; - - if (!state.paused) { - for (const event of feed.tick(dt, state.rate)) apply(event, !event.backlog); - } - - for (const agent of state.agents.values()) { - if (agent.heat > 0) agent.heat = Math.max(0, agent.heat - dt / HEAT_DECAY); - if (agent.dying && state.now - agent.dying > 1.2) forget(agent); - } - state.floats = state.floats.filter((float) => state.now - float.born < EMOJI_TTL); - - scheduleRecluster(); - - if (state.now >= addressDue) { - addressDue = state.now + 0.5; - writeAddress(); - } - - step(dt); - draw(state.now); - updateStats(); - requestAnimationFrame(frame); - } - - // --- panel ---------------------------------------------------------------- - - const stats = { - operators: document.getElementById("stat-operators"), - agents: document.getElementById("stat-agents"), - live: document.getElementById("stat-live"), - scrobbles: document.getElementById("stat-scrobbles"), - rate: document.getElementById("stat-rate"), - depth: document.getElementById("stat-depth"), - busiest: document.getElementById("stat-busiest"), - }; - - let statsDue = 0; - - function updateStats() { - if (state.now < statsDue) return; - statsDue = state.now + 0.25; - const cutoff = state.now - 60; - while (state.scrobbleTimes.length && state.scrobbleTimes[0] < cutoff) { - state.scrobbleTimes.shift(); - } - stats.operators.textContent = state.operators.size; - stats.agents.textContent = state.totals.agents; - stats.live.textContent = state.agents.size; - stats.scrobbles.textContent = state.totals.scrobbles; - stats.rate.textContent = state.scrobbleTimes.length; - stats.depth.textContent = state.totals.depth + 1; - let busiest = 0; - for (const operator of state.operators.values()) { - busiest = Math.max(busiest, operator.agents); - } - stats.busiest.textContent = busiest; - Detail.refresh(state); - } - - const clusterList = document.getElementById("clusters"); - const clusterCount = document.getElementById("work-count"); - - /** The top clusters, as a clickable list. */ - function renderClusters() { - const top = state.work.clusters.slice(0, 8); - clusterCount.textContent = `${state.work.clusters.length} groups`; - clusterList.textContent = ""; - for (const cluster of top) { - const operators = new Set(); - for (const did of cluster.members) { - const agent = state.agents.get(did); - if (agent) operators.add(agent.operator.did); - } - const item = document.createElement("li"); - item.classList.toggle( - "on", - Boolean(state.focus && state.focus.kind === "work" && state.focus.id === cluster.id) - ); - const swatch = document.createElement("i"); - swatch.className = "cluster-swatch"; - swatch.style.background = `hsl(${clusterHue(cluster.id)}, 65%, 62%)`; - const label = document.createElement("span"); - label.className = "cluster-label"; - label.textContent = `${cluster.label} · ${operators.size} op`; - const size = document.createElement("span"); - size.className = "cluster-size"; - size.textContent = cluster.size; - item.append(swatch, label, size); - item.addEventListener("click", () => { - const already = state.focus && state.focus.kind === "work" && state.focus.id === cluster.id; - setFocus(already ? null : { kind: "work", id: cluster.id }); - if (!already && !state.options.work) setWork(true); - }); - clusterList.append(item); - } - } - - const focusBox = document.getElementById("focus"); - const focusName = document.getElementById("focus-name"); - const focusDetail = document.getElementById("focus-detail"); - - function setFocus(target) { - state.focus = target; - if (target && target.kind === "agent") { - const agent = state.agents.get(target.did); - if (agent) Detail.show(agent, state); - } else { - Detail.hide(); - } - if (!target) { - focusBox.hidden = true; - return; - } - focusBox.hidden = false; - if (target.kind === "work") { - const cluster = state.work.clusters.find((entry) => entry.id === target.id); - if (!cluster) { - focusBox.hidden = true; - state.focus = null; - return; - } - const operators = new Set(); - for (const did of cluster.members) { - const agent = state.agents.get(did); - if (agent) operators.add(agent.operator.handle); - } - focusName.textContent = cluster.label; - focusDetail.textContent = - `${cluster.size} agents doing the same work, across ${operators.size} operators: ` + - `${[...operators].slice(0, 6).join(", ")}`; - return; - } - if (target.kind === "operator") { - const operator = state.operators.get(target.did); - focusName.textContent = operator.handle; - focusDetail.textContent = `${operator.agents} live agents · zone ${operator.zone} · ${operator.did}`; - } else { - const agent = state.agents.get(target.did); - focusName.textContent = agent.name; - focusDetail.textContent = - `depth ${agent.depth} · ${agent.children.length} spawned · ` + - `${agent.scrobbles} scrobbles · ${agent.did}`; - } - } - - // --- input ---------------------------------------------------------------- - - function resize() { - const previous = view.dpr; - view.dpr = Math.min(2, window.devicePixelRatio || 1); - if (previous !== view.dpr) { - for (const operator of state.operators.values()) operator.picture = null; - } - view.width = window.innerWidth; - view.height = window.innerHeight; - canvas.width = Math.floor(view.width * view.dpr); - canvas.height = Math.floor(view.height * view.dpr); - canvas.style.width = `${view.width}px`; - canvas.style.height = `${view.height}px`; - } - - function fit() { - let minX = Infinity; - let minY = Infinity; - let maxX = -Infinity; - let maxY = -Infinity; - for (const operator of state.operators.values()) { - minX = Math.min(minX, operator.rx); - minY = Math.min(minY, operator.ry); - maxX = Math.max(maxX, operator.rx); - maxY = Math.max(maxY, operator.ry); - } - // The agents, not an estimate of where they ought to be: the simulation - // decides that now, and a cluster stretched toward its work reaches - // further than any formula here would guess. - for (const agent of state.agents.values()) { - minX = Math.min(minX, agent.rx); - minY = Math.min(minY, agent.ry); - maxX = Math.max(maxX, agent.rx); - maxY = Math.max(maxY, agent.ry); - } - minX -= 70; - minY -= 70; - maxX += 70; - maxY += 70; - if (minX === Infinity) return; - // The panel covers the right-hand edge on a wide screen, so the picture - // is centred in what is left of the viewport rather than in the viewport. - const panel = view.width > 1100 ? 344 : 0; - const usable = view.width - panel; - camera.scale = Math.min( - (usable - 60) / (maxX - minX), - (view.height - 120) / (maxY - minY) - ); - camera.x = (minX + maxX) / 2 + panel / 2 / camera.scale; - camera.y = (minY + maxY) / 2; - } - - /** The agent or operator under a screen point, if anything is. */ - function pick(sx, sy) { - const world = toWorld(sx, sy); - const reach = 14 / camera.scale; - let best = null; - let bestDistance = reach * reach; - for (const agent of state.agents.values()) { - const dx = agent.rx - world.x; - const dy = agent.ry - world.y; - const distance = dx * dx + dy * dy; - if (distance < bestDistance) { - bestDistance = distance; - best = { kind: "agent", did: agent.did, agent }; - } - } - for (const operator of state.operators.values()) { - const dx = operator.rx - world.x; - const dy = operator.ry - world.y; - const distance = dx * dx + dy * dy; - if (distance < Math.max(bestDistance, (hubSize() / 2 / camera.scale) ** 2)) { - bestDistance = distance; - best = { kind: "operator", did: operator.did, operator }; - } - } - return best; - } - - let dragging = false; - let moved = false; - let pointer = { x: 0, y: 0 }; - - canvas.addEventListener("pointerdown", (event) => { - dragging = true; - moved = false; - pointer = { x: event.clientX, y: event.clientY }; - canvas.classList.add("dragging"); - canvas.setPointerCapture(event.pointerId); - }); - - canvas.addEventListener("pointermove", (event) => { - if (dragging) { - const dx = event.clientX - pointer.x; - const dy = event.clientY - pointer.y; - if (Math.abs(dx) + Math.abs(dy) > 3) { - moved = true; - state.touched = true; - } - camera.x -= dx / camera.scale; - camera.y -= dy / camera.scale; - pointer = { x: event.clientX, y: event.clientY }; - tooltip.hidden = true; - return; - } - const hit = pick(event.clientX, event.clientY); - state.hover = hit; - if (!hit) { - tooltip.hidden = true; - return; - } - tooltip.hidden = false; - tooltip.style.left = `${Math.min(event.clientX + 14, view.width - 320)}px`; - tooltip.style.top = `${event.clientY + 16}px`; - if (hit.kind === "operator") { - tooltip.innerHTML = - `${hit.operator.handle}
${hit.operator.agents} live agents` + - `
${hit.operator.did}`; - } else { - const agent = hit.agent; - tooltip.innerHTML = - `${agent.emoji ? `${agent.emoji} ` : ""}${agent.name}` + - `
${agent.operator.handle} · depth ${agent.depth} · ` + - `${agent.children.length} spawned · ${agent.scrobbles} scrobbles` + - `
${agent.did}`; - } - }); - - function release(event) { - if (dragging && !moved) { - const hit = pick(event.clientX, event.clientY); - setFocus(hit ? { kind: hit.kind, did: hit.did } : null); - } - dragging = false; - canvas.classList.remove("dragging"); - } - - canvas.addEventListener("pointerup", release); - canvas.addEventListener("pointercancel", () => { - dragging = false; - canvas.classList.remove("dragging"); - }); - canvas.addEventListener("pointerleave", () => { - tooltip.hidden = true; - }); - - canvas.addEventListener( - "wheel", - (event) => { - event.preventDefault(); - const before = toWorld(event.clientX, event.clientY); - state.touched = true; - const factor = Math.exp(-event.deltaY * 0.0016); - camera.scale = Math.min(6, Math.max(0.08, camera.scale * factor)); - const after = toWorld(event.clientX, event.clientY); - camera.x += before.x - after.x; - camera.y += before.y - after.y; - }, - { passive: false } - ); - - // --- controls ------------------------------------------------------------- - - const rateInput = document.getElementById("rate"); - const rateOut = document.getElementById("rate-out"); - rateInput.addEventListener("input", () => { - state.rate = Number(rateInput.value); - rateOut.textContent = `${state.rate}/s`; - }); - - const pullInput = document.getElementById("pull"); - const pullOut = document.getElementById("pull-out"); - pullInput.addEventListener("input", () => { - state.workPull = Number(pullInput.value) / 100; - pullOut.textContent = `${pullInput.value}%`; - }); - - const pauseButton = document.getElementById("pause"); - pauseButton.addEventListener("click", () => { - state.paused = !state.paused; - pauseButton.textContent = state.paused ? "resume" : "pause"; - pauseButton.classList.toggle("on", state.paused); - pauseButton.setAttribute("aria-pressed", String(state.paused)); - }); - - /** The button for one option, so state and control cannot disagree. */ - const optionButtons = {}; - - function setOption(key, on) { - state.options[key] = on; - const button = optionButtons[key]; - if (!button) return; - button.classList.toggle("on", on); - button.setAttribute("aria-pressed", String(on)); - if (key === "work" && !on && state.focus && state.focus.kind === "work") { - setFocus(null); - } - } - - function toggle(id, key) { - const button = document.getElementById(id); - optionButtons[key] = button; - button.addEventListener("click", () => setOption(key, !state.options[key])); - } - /** Turns work mode on or off, keeping the button in step. */ - function setWork(on) { - setOption("work", on); - } - - toggle("toggle-work", "work"); - toggle("toggle-emoji", "emoji"); - toggle("toggle-links", "links"); - toggle("toggle-labels", "labels"); - toggle("toggle-text", "text"); - - document.getElementById("fit").addEventListener("click", () => { - state.touched = false; - fit(); - }); - document.getElementById("focus-clear").addEventListener("click", () => setFocus(null)); - - const search = document.getElementById("search"); - search.addEventListener("input", () => { - state.query = search.value.trim().toLowerCase(); - }); - - window.addEventListener("resize", () => { - resize(); - // A window that changes size has not changed what is in it, so the view - // re-fits — unless the operator has already chosen where to look. - if (!state.touched) fit(); - }); - - document.addEventListener("keydown", (event) => { - if (event.key === "Escape") { - setFocus(null); - search.blur(); - } - }); - - // --- the address bar ------------------------------------------------------ - - /** - * What of the view survives being copied out of the address bar. - * - * Camera, controls and focus, including a focused work group — which it did - * not used to. Group ids were assigned in the order leaders appeared in one - * tab, so a link to one opened on somebody else's group. The index hands out - * the ids now, derived from the DID of the agent a group was opened on, so - * two people asking get the same answer and a link means what it says. - * - * Two caveats, and the link is written anyway because a group that has gone - * is handled the same way a deleted agent already is — the focus clears and - * the page is otherwise as the link asked for. A group's id does not survive - * the index restarting, because the population is re-observed from scratch - * and the groups are genuinely different; and a page running on the mock - * invents its own ids, so a link from one only means something in another - * mock with the same seed. - * - * An operator or an agent is a DID: the same identifier for everyone who has - * one, which is the whole argument for building on this protocol. - */ - let addressDue = 0; - - function writeAddress() { - const fields = new URLSearchParams(); - fields.set("x", Math.round(camera.x)); - fields.set("y", Math.round(camera.y)); - fields.set("z", camera.scale.toFixed(3)); - fields.set("pull", String(Math.round(state.workPull * 100))); - fields.set("rate", String(state.rate)); - const on = Object.entries(state.options) - .filter(([, value]) => value) - .map(([name]) => name); - fields.set("show", on.join(",")); - if (state.paused) fields.set("paused", "1"); - if (state.focus) { - const what = state.focus.kind === "work" ? state.focus.id : state.focus.did; - fields.set("focus", `${state.focus.kind}:${what}`); - } - // `replaceState`, not a hash assignment: panning should not fill the back - // button with every frame the camera passed through. - history.replaceState(null, "", `#${fields}`); - } - - function readAddress() { - const fields = new URLSearchParams(location.hash.slice(1)); - if ([...fields].length === 0) return; - - const number = (name, fallback) => { - const value = Number(fields.get(name)); - return Number.isFinite(value) ? value : fallback; - }; - - if (fields.has("z")) { - camera.scale = clamp(number("z", camera.scale), 0.08, 6); - camera.x = number("x", camera.x); - camera.y = number("y", camera.y); - state.touched = true; - } - if (fields.has("pull")) { - state.workPull = clamp(number("pull", 92) / 100, 0, 1); - pullInput.value = String(Math.round(state.workPull * 100)); - pullOut.textContent = `${pullInput.value}%`; - } - if (fields.has("rate")) { - state.rate = clamp(number("rate", state.rate), 0, 40); - rateInput.value = String(state.rate); - rateOut.textContent = `${state.rate}/s`; - } - if (fields.has("show")) { - const on = new Set(fields.get("show").split(",").filter(Boolean)); - for (const name of Object.keys(state.options)) { - setOption(name, on.has(name)); - } - } - if (fields.get("paused") === "1") pauseButton.click(); - - // `warm=` runs the feed forward before the first frame. It exists - // because a screenshot, a demo and a bug report all want the same thing: - // a population that has been running for a while, arrived at the same way - // every time. The last few seconds are applied as live events so there are - // floats in the air and lines in the stream, rather than a still frame of - // a history nobody watched. - const warm = clamp(number("warm", 0), 0, 600); - if (warm > 0) { - const live = Math.min(4, warm); - for (let second = 0; second < warm; second += 1) { - state.now += 1; - for (const event of feed.tick(1, state.rate)) { - apply(event, second >= warm - live); - } - } - if (!fields.has("z")) fit(); - } - - const focus = fields.get("focus"); - if (focus) { - const [kind, ...rest] = focus.split(":"); - const did = rest.join(":"); - // A work group named in a link cannot be resolved yet on a live page: - // the groups arrive over the network, seconds after this runs. It is - // held instead and applied by `adoptGroups` when they land. - if (kind === "work") { - wantedGroup = did; - applyWantedGroup(); - return; - } - // The agent may have ended between the link being made and being - // followed, which is not an error — it is what an ephemeral population - // means, and the view opens without a focus rather than not at all. - const subject = - kind === "operator" ? state.operators.get(did) : state.agents.get(did); - if (subject) { - setFocus({ kind, did }); - // A link that names something and not a camera means "show me this", - // so the view arrives looking at it rather than wherever the last - // person happened to be. - if (!fields.has("z")) { - camera.scale = kind === "operator" ? 0.9 : 1.4; - const panel = view.width > 1100 ? 344 : 0; - camera.x = subject.rx + panel / 2 / camera.scale; - camera.y = subject.ry; - state.touched = true; - } - } - } - } - - // --- start ---------------------------------------------------------------- - - Detail.mount(document.getElementById("detail-root")); - buildGlows(); - resize(); - for (const event of feed.seed()) apply(event, false); - fit(); - readAddress(); - - // And if something real is running, hand over to it. The mock's population - // is cleared first: showing invented agents beside live ones would be the - // one thing this view must never do. - (async () => { - const fields = new URLSearchParams(location.hash.slice(1)); - if (fields.get("live") === "off") return; - const live = await Live.connect(fields.get("query")); - if (!live) return; - for (const did of [...state.agents.keys()]) forget(state.agents.get(did)); - state.operators.clear(); - state.floats = []; - state.totals = { agents: 0, scrobbles: 0, depth: 0 }; - feed = live; - living = true; - const badge = document.getElementById("source"); - badge.textContent = "live"; - badge.classList.add("on", "live"); - badge.title = `following ${live.base}`; - - // The camera was framed around the mock population that has just been - // cleared. Give the snapshot a moment to land and frame what is actually - // there — unless somebody has already moved it themselves. - state.touched = false; - setTimeout(() => { - if (!state.touched) fit(); - }, 2000); - })(); - requestAnimationFrame(frame); - - // A deliberate handle on the running view. Everything above is closed over, - // and without this there is no way to drive the thing from a devtools - // console or from a screenshot script. - window.appview = { - camera, - state, - view, - fit, - setFocus, - setWork, - workIndex, - live: () => living, - }; -})(); diff --git a/web/avatar.js b/web/avatar.js deleted file mode 100644 index 22d9de7c..00000000 --- a/web/avatar.js +++ /dev/null @@ -1,233 +0,0 @@ -/* Profile pictures for the humans on the canvas. - * - * A real deployment has one: an operator is an atproto account, and an - * atproto account has an avatar blob its personal data server will serve. - * This file is what stands in for it until that fetch exists, and what fills - * the gap afterwards — plenty of accounts have no avatar set, and a canvas - * that leaves a hole for them is worse than one that draws something. - * - * Everything here is generated from the DID, so a given operator always looks - * the same, and drawn once into an offscreen canvas that the main loop blits. - * Nothing is fetched and nothing is stored. - * - * The picture is also where the operator's colour on the canvas comes from. - * [`dominantHue`] reads it back off the pixels rather than the palette being - * handed in, which sounds like the long way round and is the point: it works - * the same on a photograph somebody uploaded. Change your avatar and your - * agents change colour with it, because the colour was never a property of - * the account — it was a property of the picture all along. - */ - -const Avatar = (() => { - "use strict"; - - /** FNV-1a over a string, as the seed for everything below. */ - function seedOf(text) { - let hash = 0x811c9dc5; - for (let i = 0; i < text.length; i += 1) { - hash ^= text.charCodeAt(i); - hash = Math.imul(hash, 0x01000193); - } - return hash >>> 0; - } - - /** The hue a DID's generated picture is built around. */ - function hueOf(did) { - return (seedOf(did) % 360000) / 1000; - } - - /** A small deterministic generator, so one seed gives one picture. */ - function stream(seed) { - let state = seed >>> 0; - return () => { - state = (state + 0x6d2b79f5) >>> 0; - let t = Math.imul(state ^ (state >>> 15), 1 | state); - t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; - return ((t ^ (t >>> 14)) >>> 0) / 4294967296; - }; - } - - /** An offscreen canvas of `size` square, at device resolution. */ - function surface(size) { - const ratio = Math.min(2, window.devicePixelRatio || 1); - const canvas = document.createElement("canvas"); - canvas.width = Math.round(size * ratio); - canvas.height = Math.round(size * ratio); - const paint = canvas.getContext("2d"); - paint.scale(ratio, ratio); - return { canvas, paint }; - } - - /** - * Blocks: a symmetric grid, two colours. - * - * The identicon everyone already knows how to read. Holds up smallest of - * the four because it is nothing but hard edges. - */ - function blocks(paint, size, seed, hue) { - const next = stream(seed); - const cells = 5; - const cell = size / cells; - paint.fillStyle = `hsl(${hue}, 22%, 14%)`; - paint.fillRect(0, 0, size, size); - for (let y = 0; y < cells; y += 1) { - for (let x = 0; x < Math.ceil(cells / 2); x += 1) { - if (next() < 0.45) continue; - paint.fillStyle = next() < 0.3 ? `hsl(${(hue + 40) % 360}, 65%, 68%)` : `hsl(${hue}, 62%, 60%)`; - paint.fillRect(x * cell, y * cell, cell + 0.5, cell + 0.5); - paint.fillRect((cells - 1 - x) * cell, y * cell, cell + 0.5, cell + 0.5); - } - } - } - - /** - * Orb: soft overlapping fields of colour. - * - * The one that reads as a photograph at twelve pixels, which matters - * because a photograph is what will eventually be there. - */ - function orb(paint, size, seed, hue) { - const next = stream(seed); - paint.fillStyle = `hsl(${hue}, 45%, 30%)`; - paint.fillRect(0, 0, size, size); - for (let i = 0; i < 4; i += 1) { - const cx = next() * size; - const cy = next() * size; - const radius = size * (0.35 + next() * 0.45); - const tint = (hue + (i - 1.5) * 46 + 360) % 360; - const wash = paint.createRadialGradient(cx, cy, 0, cx, cy, radius); - wash.addColorStop(0, `hsla(${tint}, ${55 + next() * 25}%, ${58 + next() * 22}%, 0.92)`); - wash.addColorStop(1, `hsla(${tint}, 60%, 45%, 0)`); - paint.fillStyle = wash; - paint.fillRect(0, 0, size, size); - } - } - - /** - * Initials: the first two letters of the handle. - * - * The only one of the four that says something true about the account - * rather than about its hash. - */ - function initials(paint, size, seed, hue, label) { - paint.fillStyle = `hsl(${hue}, 40%, 26%)`; - paint.fillRect(0, 0, size, size); - paint.fillStyle = `hsl(${hue}, 70%, 82%)`; - paint.font = `600 ${size * 0.46}px ui-monospace, monospace`; - paint.textAlign = "center"; - paint.textBaseline = "middle"; - paint.fillText((label || "??").slice(0, 2).toUpperCase(), size / 2, size / 2 + size * 0.02); - } - - /** - * Bands: horizontal strata, like a core sample. - * - * Quietest of the four. Distinguishable side by side, less so across a - * crowded canvas. - */ - function bands(paint, size, seed, hue) { - const next = stream(seed); - let y = 0; - while (y < size) { - const height = size * (0.08 + next() * 0.22); - const tint = (hue + (next() - 0.5) * 60 + 360) % 360; - paint.fillStyle = `hsl(${tint}, ${35 + next() * 35}%, ${28 + next() * 42}%)`; - paint.fillRect(0, y, size, height + 0.5); - y += height; - } - } - - const STYLES = { blocks, orb, initials, bands }; - - /** - * Renders one avatar and returns the canvas to blit. - * - * @param {string} did identity the picture is generated from - * @param {string} handle used by the styles that show letters - * @param {number} size in CSS pixels - * @param {string} style one of `blocks`, `orb`, `initials`, `bands` - */ - function make(did, handle, size, style = "orb") { - const { canvas, paint } = surface(size); - const seed = seedOf(did); - const hue = hueOf(did); - const draw = STYLES[style] || orb; - // Clipped round, so a real photograph dropped in later needs no other - // change to sit correctly in the ring. - paint.save(); - paint.beginPath(); - paint.arc(size / 2, size / 2, size / 2, 0, Math.PI * 2); - paint.clip(); - draw(paint, size, seed, hue, handle); - paint.restore(); - return canvas; - } - - /** - * The dominant hue of a picture, as a circular mean weighted by saturation. - * - * Sampled from a sixteen-pixel thumbnail, because the question is what - * colour a person would call the picture and that survives being thrown - * away at this scale. Weighting by saturation is what stops a photograph - * that is nine tenths grey wall from being called grey: the wall has no - * hue to contribute and the coat in front of it does. - * - * Averaged as vectors rather than as numbers, since hue wraps — the mean of - * 350 and 10 is 0, and the arithmetic mean says 180, which is the opposite - * colour. - * - * @param {CanvasImageSource} picture anything `drawImage` accepts - * @returns {number|null} degrees, or null if there is no colour in it - */ - function dominantHue(picture) { - const size = 16; - const { canvas, paint } = surface(size); - try { - paint.drawImage(picture, 0, 0, size, size); - } catch (error) { - return null; - } - let pixels; - try { - pixels = paint.getImageData(0, 0, canvas.width, canvas.height).data; - } catch (error) { - // A picture from another origin taints the canvas and reading it back - // throws. Nothing here is cross-origin today, but a fetched avatar will - // be, and losing the colour is better than losing the frame. - return null; - } - - let x = 0; - let y = 0; - let total = 0; - for (let i = 0; i < pixels.length; i += 4) { - const alpha = pixels[i + 3] / 255; - if (alpha < 0.5) continue; - const r = pixels[i] / 255; - const g = pixels[i + 1] / 255; - const b = pixels[i + 2] / 255; - const max = Math.max(r, g, b); - const min = Math.min(r, g, b); - const span = max - min; - if (span < 0.04) continue; - const light = (max + min) / 2; - const saturation = span / (1 - Math.abs(2 * light - 1) || 1); - let hue; - if (max === r) hue = ((g - b) / span + (g < b ? 6 : 0)) * 60; - else if (max === g) hue = ((b - r) / span + 2) * 60; - else hue = ((r - g) / span + 4) * 60; - // Mid-lightness pixels describe the picture; the near-black and - // near-white ones are its shadows and highlights and say little. - const weight = alpha * saturation * (1 - Math.abs(2 * light - 1) * 0.8); - const radians = (hue * Math.PI) / 180; - x += Math.cos(radians) * weight; - y += Math.sin(radians) * weight; - total += weight; - } - if (total <= 0) return null; - const mean = (Math.atan2(y, x) * 180) / Math.PI; - return (mean + 360) % 360; - } - - return { make, dominantHue, hueOf, styles: Object.keys(STYLES), seedOf }; -})(); diff --git a/web/detail.css b/web/detail.css deleted file mode 100644 index 20999de9..00000000 --- a/web/detail.css +++ /dev/null @@ -1,184 +0,0 @@ -/* The detail panel. Same glass and the same monospace as the stats panel on - the other side of the viewport, because they are two readings of one thing - and a second visual language would say otherwise. - - Every selector is under .detail: this file is loaded beside style.css and - must not have an opinion about anything it did not build. */ - -.detail { - position: fixed; - top: 70px; - left: 12px; - bottom: 46px; - width: 330px; - display: flex; - flex-direction: column; - gap: 10px; - padding: 12px; - background: var(--panel); - border: 1px solid var(--line); - border-radius: 8px; - backdrop-filter: blur(6px); - font: 13px/1.5 var(--mono); - color: var(--ink); - /* Above the canvas, below the tooltip, which is the pointer's own feedback - and has to survive being over the panel. */ - z-index: 4; -} - -/* The panel is a flex column, and `display: flex` would otherwise win over - the hidden attribute and leave a closed panel on screen. */ -.detail[hidden] { display: none; } - -.detail-head { - display: flex; - align-items: flex-start; - gap: 8px; - padding-bottom: 9px; - border-bottom: 1px solid var(--line); -} - -/* The operator's colour, the same hue their agents are drawn in on the - canvas, so the panel and the picture agree about whose agent this is. */ -.detail-swatch { - width: 10px; - height: 10px; - margin-top: 5px; - border-radius: 2px; - flex: none; -} - -.detail-heading { min-width: 0; flex: 1; } -.detail-name { font-weight: normal; color: var(--ink); word-break: break-all; } -.detail-operator { color: var(--ink-dim); word-break: break-all; } - -.detail-close { - flex: none; - padding: 2px 8px; - color: var(--ink-dim); - background: rgba(255, 255, 255, 0.03); - border: 1px solid var(--line); - border-radius: 5px; - font: inherit; - cursor: pointer; -} -.detail-close:hover { color: var(--ink); } - -/* The whole body scrolls as one. A panel with four independently scrolling - regions is a puzzle, and only the scrobble list is ever long. */ -.detail-body { - flex: 1; - min-height: 0; - overflow-y: auto; - overflow-x: hidden; - display: flex; - flex-direction: column; - gap: 12px; - scrollbar-width: thin; - scrollbar-color: var(--line) transparent; -} - -.detail-block { display: flex; flex-direction: column; gap: 4px; } - -.detail-caption { - margin: 0; - font-size: 13px; - font-weight: normal; - color: var(--ink-dim); -} - -.detail-sub { color: var(--ink-dim); opacity: 0.8; } -.detail-line { color: var(--ink-dim); } - -/* A caption with its copy button pushed to the far end, so a value that - wraps to three lines wraps under nothing. */ -.detail-label-row { - display: flex; - align-items: baseline; - justify-content: space-between; - gap: 8px; -} - -.detail-value { - display: block; - /* Selectable and breakable: a DID is one long unbreakable-looking token and - the point of putting it here is that it can leave. */ - user-select: text; - word-break: break-all; - color: var(--ink); - font: inherit; -} - -.detail-copy-button { - flex: none; - padding: 1px 7px; - color: var(--ink-dim); - background: rgba(255, 255, 255, 0.03); - border: 1px solid var(--line); - border-radius: 5px; - font: inherit; - cursor: pointer; -} -.detail-copy-button:hover { color: var(--accent); } - -.detail-chain { - display: flex; - flex-wrap: wrap; - align-items: baseline; - gap: 4px; - color: var(--ink-dim); -} -.detail-sep { color: var(--line); } -.detail-hop { color: var(--ink-dim); } -.detail-hop-here { color: var(--ink); } - -.detail-work { color: var(--accent); word-break: break-word; } -.detail-work-alone { color: var(--ink-dim); } - -.detail-scrobbles { - margin: 0; - padding: 0; - list-style: none; - display: flex; - flex-direction: column; - gap: 4px; -} - -.detail-scrobble { - display: flex; - align-items: baseline; - gap: 6px; - /* Never shrink, for the same reason the stream list does not: a run of - short lines squeezed under their own line height overprints. */ - flex: none; -} -.detail-emoji { flex: none; width: 1.3em; } -.detail-said { - flex: 1; - min-width: 0; - color: var(--ink); - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} -.detail-age { - flex: none; - color: var(--ink-dim); - font-variant-numeric: tabular-nums; -} -.detail-effort { flex: none; color: var(--ink-dim); opacity: 0.7; } - -.detail-quiet { color: var(--ink-dim); } -.detail-quiet[hidden] { display: none; } - -/* An agent whose account has been deleted while the panel was open. The - identity and the URIs stay readable — they are the reason somebody would - still be looking — and the frame says the thing is over. */ -.detail-gone { border-color: rgba(224, 160, 143, 0.4); } -.detail-gone .detail-name { color: var(--warn); } - -/* Hidden with the other panels rather than stacked on top of the canvas: at - this width there is no canvas left to put one beside. */ -@media (max-width: 1100px) { - .detail { display: none; } -} diff --git a/web/detail.js b/web/detail.js deleted file mode 100644 index b9e246a7..00000000 --- a/web/detail.js +++ /dev/null @@ -1,449 +0,0 @@ -/* The detail panel: one agent, read closely. - * - * The canvas answers "what is happening" at a glance and refuses to answer - * anything else — a dot has no room for a DID. This panel is the other half - * of that trade. It takes one agent and lays out the things a person needs - * when they have stopped scanning and started asking: who this is, who - * spawned it, what it is working on alongside whom, what it has been saying, - * and where a client would go to read the records for itself. - * - * Nothing here reaches into app.js. It is handed an agent and the state map - * it came from, so the panel works the same over a live subscription as it - * does over mock.js. - */ - -const Detail = (() => { - "use strict"; - - /** Scrobbles kept per agent. Two dozen is about a screen and a half. */ - const HISTORY = 24; - /** - * Agents whose history is kept at all. - * - * `forget` is the intended way a history ends, but a stream that drops a - * `deleted` event leaks one array per agent forever. The map is ordered by - * how recently each agent spoke, so the oldest can simply be dropped off - * the front once there are too many. - */ - const TRACKED = 2000; - /** How long a copy button stays showing what happened, in milliseconds. */ - const FLASH = 1400; - - /** did -> array of scrobbles, oldest first. */ - const history = new Map(); - /** Rises with every scrobble recorded, so the list knows what it has drawn. */ - let stamped = 0; - - /** The panel, and every node in it `refresh` is allowed to touch. */ - let panel = null; - const nodes = {}; - /** The agent the panel is currently about, held as a DID because the agent - * object itself can be dropped from the population while the panel is up. */ - let subject = null; - /** The highest stamp already in the scrobble list. */ - let drawn = 0; - - function el(tag, className, text) { - const node = document.createElement(tag); - if (className) node.className = className; - if (text !== undefined) node.textContent = text; - return node; - } - - /** - * A labelled block, returned with the parts that change. - * - * Every section of the panel is the same shape — a dim caption over a value - * — and building them from one function is what keeps `refresh` able to - * write into a known node instead of rebuilding a subtree. - */ - function block(title) { - const wrap = el("div", "detail-block"); - const head = el("div", "detail-label-row"); - head.append(el("h3", "detail-caption", title)); - wrap.append(head); - // The caption row comes back with the block because a copy button belongs - // up there beside the name of the thing being copied: put it next to the - // value and a DID that wraps to two lines wraps around it. - return { wrap, head }; - } - - /** - * A value with a copy button beside it. - * - * The DID and the AT-URIs are the only things on this page anybody is - * likely to want in another window, so they are selectable text first and - * button-driven second: `navigator.clipboard` is absent over plain HTTP and - * refused when the document is not focused, and a value you can drag over - * still works in both cases. - */ - function copyRow(head) { - const value = el("code", "detail-value"); - const button = el("button", "detail-copy-button", "copy"); - button.type = "button"; - let timer = 0; - button.addEventListener("click", () => { - const text = value.textContent; - window.clearTimeout(timer); - const flash = (word) => { - button.textContent = word; - timer = window.setTimeout(() => { - button.textContent = "copy"; - }, FLASH); - }; - try { - navigator.clipboard.writeText(text).then( - () => flash("copied"), - () => flash(select(value) ? "selected" : "failed") - ); - } catch (error) { - flash(select(value) ? "selected" : "failed"); - } - }); - head.append(button); - return value; - } - - /** Selects a node's text, so a refused clipboard still leaves ctrl-C usable. */ - function select(node) { - try { - const range = document.createRange(); - range.selectNodeContents(node); - const selection = window.getSelection(); - selection.removeAllRanges(); - selection.addRange(range); - return true; - } catch (error) { - return false; - } - } - - /** Writes text only when it has changed, because `refresh` runs four times - * a second and a written text node invalidates layout whether or not the - * string differs. */ - function set(node, text) { - if (node && node.textContent !== text) node.textContent = text; - } - - function mount(root) { - if (panel) return; - panel = el("section", "detail"); - panel.hidden = true; - - const head = el("header", "detail-head"); - nodes.swatch = el("i", "detail-swatch"); - const heading = el("div", "detail-heading"); - nodes.name = el("strong", "detail-name", "—"); - nodes.operator = el("div", "detail-operator"); - heading.append(nodes.name, nodes.operator); - const close = el("button", "detail-close", "close"); - close.type = "button"; - close.addEventListener("click", () => { - hide(); - // The panel can close itself, but only the wiring knows whether the - // canvas focus should go with it, so the decision is passed along - // rather than guessed at. - panel.dispatchEvent(new CustomEvent("detail-close", { bubbles: true })); - }); - head.append(nodes.swatch, heading, close); - - const body = el("div", "detail-body"); - - const identity = block("identity"); - const handleRow = el("div", "detail-label-row"); - handleRow.append(el("span", "detail-sub", "handle")); - nodes.handle = copyRow(handleRow); - const didRow = el("div", "detail-label-row"); - didRow.append(el("span", "detail-sub", "did")); - nodes.did = copyRow(didRow); - // Both, and in this order. The handle is the one a person reads and the - // one that can be reissued; the DID is the one every record is written - // under and the one that never changes. A panel that showed only the - // pretty half would be inviting somebody to paste a name into a place - // that needs the permanent identifier. - identity.wrap.append(handleRow, nodes.handle, didRow, nodes.did); - - const lineage = block("spawned from"); - nodes.chain = el("div", "detail-chain"); - nodes.lineage = el("div", "detail-line"); - lineage.wrap.append(nodes.chain, nodes.lineage); - - const work = block("work"); - nodes.workLabel = el("div", "detail-work"); - nodes.workLine = el("div", "detail-line"); - work.wrap.append(nodes.workLabel, nodes.workLine); - - const said = block("recent"); - nodes.scrobbles = el("ul", "detail-scrobbles"); - nodes.empty = el("li", "detail-quiet", "nothing recorded yet"); - nodes.scrobbles.append(nodes.empty); - said.wrap.append(nodes.scrobbles); - - const records = block("records"); - const collection = el("div", "detail-label-row"); - collection.append(el("span", "detail-sub", "collection")); - nodes.collection = copyRow(collection); - const zone = el("div", "detail-label-row"); - zone.append(el("span", "detail-sub", "zone")); - nodes.zone = copyRow(zone); - records.wrap.append(collection, nodes.collection, zone, nodes.zone); - - body.append(identity.wrap, lineage.wrap, work.wrap, said.wrap, records.wrap); - panel.append(head, body); - root.append(panel); - } - - /** - * The chain from the operator down to this agent. - * - * Walked upward and reversed, because an agent knows its parent and a - * parent's children are everything it spawned rather than the one branch - * that leads here. The operator goes on the front: a session root's real - * parent is a person, and a lineage that starts at an agent leaves out the - * only part of it that is accountable. - */ - function chainOf(agent) { - const names = []; - // Bounded rather than trusted: a reparented tree is edited live in - // `forget`, and a walk with no ceiling is one bad splice from hanging the - // page. - let guard = 0; - for (let node = agent; node && guard < 64; node = node.parent) { - names.push(node.name || node.agentId); - guard += 1; - } - names.reverse(); - return names; - } - - function renderChain(agent) { - nodes.chain.textContent = ""; - const operator = agent.operator; - const handle = el("span", "detail-hop detail-hop-operator", operator ? operator.handle : "?"); - if (operator) handle.style.color = `hsl(${operator.hue}, 62%, 74%)`; - nodes.chain.append(handle); - const names = chainOf(agent); - names.forEach((name, index) => { - nodes.chain.append(el("span", "detail-sep", "/")); - const hop = el("span", "detail-hop", name); - // The last hop is the agent the panel is about, and a chain where you - // cannot see which end you are standing on is a list, not a path. - if (index === names.length - 1) hop.classList.add("detail-hop-here"); - nodes.chain.append(hop); - }); - } - - /** The cluster an agent is in, or null if it has none or the pass has moved - * on since the panel was drawn. */ - function clusterOf(did, state) { - if (!state || !state.work || !state.work.of) return null; - const id = state.work.of.get(did); - if (id === undefined) return null; - const clusters = state.work.clusters || []; - for (const cluster of clusters) { - if (cluster.id === id) return cluster; - } - return null; - } - - function renderWork(agent, state) { - const cluster = clusterOf(agent.did, state); - if (!cluster) { - set(nodes.workLabel, agent.text || "—"); - // Said plainly, because the alternative is a blank space that reads as - // a bug. A population where nothing else matches is a real answer and - // the commonest one for a fresh agent. - set(nodes.workLine, "no other agent is saying this"); - nodes.workLabel.classList.add("detail-work-alone"); - return; - } - nodes.workLabel.classList.remove("detail-work-alone"); - set(nodes.workLabel, cluster.label); - const operators = new Set(); - let live = 0; - for (const did of cluster.members || []) { - const other = state.agents ? state.agents.get(did) : null; - if (!other) continue; - live += 1; - if (other.operator) operators.add(other.operator.did); - } - const plural = operators.size === 1 ? "operator" : "operators"; - set(nodes.workLine, `${live} agents across ${operators.size} ${plural}`); - } - - /** How long ago something was said, in one short token. */ - function ago(wall) { - const seconds = Math.max(0, Math.round((Date.now() - wall) / 1000)); - if (seconds < 60) return `${seconds}s`; - if (seconds < 3600) return `${Math.round(seconds / 60)}m`; - return `${Math.round(seconds / 3600)}h`; - } - - /** - * Brings the scrobble list up to date without rebuilding it. - * - * Newest first, so the list reads the way the stream panel does. Only the - * entries recorded since the last pass are built; everything already on - * screen keeps its node and has its age rewritten, which is the difference - * between a panel that ticks and one that flickers. - */ - function renderScrobbles() { - const lines = history.get(subject) || []; - nodes.empty.hidden = lines.length > 0; - - // Walked oldest-first over the new lines, each one put on the front, so - // the newest ends up at the top however many arrived between two passes. - let first = lines.length; - while (first > 0 && lines[first - 1].stamp > drawn) first -= 1; - for (let i = first; i < lines.length; i += 1) { - const line = lines[i]; - const item = el("li", "detail-scrobble"); - item.append(el("span", "detail-emoji", line.emoji || "")); - const text = el("span", "detail-said", line.text || ""); - const age = el("span", "detail-age", ago(line.wall)); - item.append(text, age); - if (line.effort) { - item.append(el("span", "detail-effort", line.effort)); - } - item.dataset.wall = String(line.wall); - // After the empty note, so the note stays at the bottom of an empty - // list and never above the newest line. - nodes.scrobbles.insertBefore(item, nodes.scrobbles.firstChild); - } - drawn = lines.length ? lines[lines.length - 1].stamp : drawn; - - // The empty note lives at the bottom of the list and is not a line, so - // the oldest line is whatever sits above it. - while (nodes.scrobbles.children.length > HISTORY + 1) { - let oldest = nodes.scrobbles.lastElementChild; - if (oldest === nodes.empty) oldest = oldest.previousElementSibling; - if (!oldest) break; - oldest.remove(); - } - for (const item of nodes.scrobbles.children) { - if (item === nodes.empty) continue; - const age = item.querySelector(".detail-age"); - if (age) set(age, ago(Number(item.dataset.wall))); - } - } - - function show(agent, state) { - if (!panel || !agent) return; - subject = agent.did; - drawn = 0; - // Everything static about this agent is written once here; `refresh` only - // touches what can change while the panel is open. - nodes.scrobbles.textContent = ""; - nodes.scrobbles.append(nodes.empty); - - const operator = agent.operator || null; - const hue = operator ? operator.hue : 0; - nodes.swatch.style.background = `hsl(${hue}, 62%, 62%)`; - set(nodes.name, agent.name || agent.agentId || agent.did); - set(nodes.operator, operator ? operator.handle : "unknown operator"); - if (operator) nodes.operator.style.color = `hsl(${hue}, 55%, 76%)`; - set(nodes.did, agent.did); - // An agent nobody named has no handle to show, and a dash says so without - // pretending the minted label is one. - set(nodes.handle, agent.handle || "—"); - - renderChain(agent); - // The two AT-URIs a client would actually dereference: the agent's own - // scrobble collection, and the zone its account was minted under. Shown - // rather than linked, because there is no appview at the other end of - // them yet and a dead link is worse than a string you can paste. - set(nodes.collection, `at://${agent.did}/com.vibescrobble.scrobble`); - set(nodes.zone, operator ? operator.zone : "—"); - - panel.hidden = false; - refresh(state); - } - - function hide() { - if (!panel) return; - panel.hidden = true; - subject = null; - } - - function isOpen() { - return Boolean(panel && !panel.hidden && subject); - } - - /** - * The numbers that move, four times a second. - * - * Every lookup here is guarded and the agent is fetched afresh by DID: a - * session can end between two of these calls, and the honest response to - * that is to say so and leave the identity and the URIs on screen, not to - * throw inside somebody's frame loop. - */ - function refresh(state) { - if (!isOpen()) return; - const agent = state && state.agents ? state.agents.get(subject) : null; - if (!agent) { - panel.classList.add("detail-gone"); - set(nodes.lineage, "this agent's account is gone"); - set(nodes.workLine, ""); - renderScrobbles(); - return; - } - panel.classList.remove("detail-gone"); - const spawned = agent.children ? agent.children.length : 0; - const noun = spawned === 1 ? "agent" : "agents"; - set( - nodes.lineage, - `depth ${agent.depth} · spawned ${spawned} ${noun} · ` + - `${agent.scrobbles} scrobbles` - ); - renderWork(agent, state); - renderScrobbles(); - } - - /** - * Keeps one line of what an agent said. - * - * app.js keeps only the last scrobble on the agent, because the canvas only - * ever draws the last one. A panel wants the run-up, so the history lives - * here and the caller hands over every scrobble as it applies it, open - * panel or not — an agent clicked on has usually been talking for a while - * before anybody looked at it. - */ - function record(did, line) { - if (!did || !line) return; - let lines = history.get(did); - if (lines) { - // Re-inserted so the map stays ordered by who spoke last, which is what - // makes dropping from the front the right thing to drop. - history.delete(did); - } else { - lines = []; - } - stamped += 1; - lines.push({ - emoji: line.emoji || "", - text: line.text || "", - effort: line.effort || "", - at: line.at, - // Wall time is kept alongside whatever the caller's clock says, so an - // age can be shown without this file having to know whether `at` counts - // seconds since the page loaded or milliseconds since 1970. - wall: Date.now(), - stamp: stamped, - }); - if (lines.length > HISTORY) lines.splice(0, lines.length - HISTORY); - history.set(did, lines); - while (history.size > TRACKED) { - const oldest = history.keys().next(); - if (oldest.done || oldest.value === subject) break; - history.delete(oldest.value); - } - if (did === subject && isOpen()) renderScrobbles(); - } - - /** Drops an agent's history, for when its account is deleted. */ - function forget(did) { - history.delete(did); - } - - return { mount, show, hide, isOpen, refresh, record, forget }; -})(); diff --git a/web/index.html b/web/index.html deleted file mode 100644 index b6ed0121..00000000 --- a/web/index.html +++ /dev/null @@ -1,93 +0,0 @@ - - - - - - didbot appview - - - - - - - -
- didbot - mock - - - - - - - - - - - - - - -
- - - -
- - - -
- operator & their load - session root - subagent - pinned - colour is the owner · position is the work - drag to pan · wheel to zoom · click to focus -
- - - - - - - - - diff --git a/web/live.js b/web/live.js deleted file mode 100644 index a2cec94d..00000000 --- a/web/live.js +++ /dev/null @@ -1,226 +0,0 @@ -/* The live feed: the same events, from a real query service. - * - * `mock.js` invents a population so the canvas can be worked on with nothing - * behind it. This reads one that actually exists. Both hand app.js the same - * shapes, so the canvas does not know or care which it is drawing — that is - * the whole reason the mock was written in the event shapes the server uses - * rather than in whatever was convenient. - * - * Two differences from the mock the canvas has to live with: - * - * * **The tree is learned rather than given.** An account's profile record - * names the agent that spawned it, and a scrobble carries a copy of the - * same fact; the index folds either onto the account. The mock knows - * the whole tree up front because it invented it; here the shape is being - * reported by the population itself. A profile is written when the account - * is provisioned, so an agent that never scrobbles has a place in the tree - * too, and an agent still arrives as a root when nothing has said. - * * **The snapshot arrives as backlog.** What the index already knew when the - * page opened is applied without ceremony, so a fresh tab does not fire - * three hundred spawn rings and three hundred notes at once. The index - * lists every parent before its children, so a tree replayed from a - * snapshot comes out the same shape as one watched live. - * - * One thing this feed carries that the mock cannot: the work groups. Which - * agents are on the same job is computed once, in the index, so two people - * with this page open see the same groups under the same ids. The mock has no - * backend to ask, so `app.js` clusters locally when it is running on one. - */ - -const Live = (() => { - "use strict"; - - /** Where the query service listens in the development stack. */ - const DEFAULT_QUERY = "http://127.0.0.1:3003"; - - /** How long to wait for the query service before falling back to the mock. */ - const REACH_TIMEOUT = 1200; - - /** - * A feed with the same shape as `Mock.MockFeed`, fed by a real service. - * - * `seed` and `tick` are the whole interface, and both are synchronous - * because that is what the frame loop wants. Everything asynchronous - * happens off to the side and lands in a queue. - */ - class LiveFeed { - constructor(base) { - this.base = base.replace(/\/$/, ""); - this.queue = []; - this.operators = new Map(); - this.source = null; - this.connected = false; - } - - /** Nothing yet: the snapshot arrives over the network, not from here. */ - seed() { - return []; - } - - /** Whatever has arrived since the last frame. */ - tick() { - const events = this.queue; - this.queue = []; - return events; - } - - /** Fetches the snapshot, then follows the stream. */ - async connect() { - const snapshot = await fetch(`${this.base}/view`).then((response) => { - if (!response.ok) throw new Error(`HTTP ${response.status}`); - return response.json(); - }); - this.absorb(snapshot); - this.listen(); - this.connected = true; - } - - /** Turns a snapshot into the events that would have produced it. */ - absorb(snapshot) { - for (const operator of snapshot.operators || []) { - this.push(this.operator(operator), true); - } - for (const agent of snapshot.agents || []) { - this.push(this.provisioned(agent), true); - } - for (const scrobble of snapshot.scrobbles || []) { - this.push(this.scrobble(scrobble), true); - } - // Last, because a group is a list of agents and the agents have to - // exist before anything can be drawn joining them together. - if (snapshot.groups) this.push({ kind: "work", groups: snapshot.groups }, true); - } - - /** - * Follows the stream, reconnecting on its own. - * - * `EventSource` retries by itself, which is most of why it is here rather - * than a hand-rolled reader over `fetch`. What it cannot do is tell us - * that the gap it just bridged had events in it, so a reconnection - * re-fetches the snapshot. - */ - listen() { - this.source = new EventSource(`${this.base}/events`); - const take = (kind) => (message) => { - try { - this.push(this.translate(kind, JSON.parse(message.data)), false); - } catch (error) { - // One unreadable frame is not worth tearing the stream down for. - } - }; - for (const kind of ["operator", "provisioned", "scrobble", "pinned", "deleted", "work"]) { - this.source.addEventListener(kind, take(kind)); - } - this.source.addEventListener("error", () => { - // The browser is already reconnecting. What it cannot do is tell us - // what we missed, so ask for the whole view again once it is back. - if (this.source.readyState === EventSource.CONNECTING) { - this.resync(); - } - }); - } - - /** Re-fetches the snapshot after a gap, at most once at a time. */ - resync() { - if (this.resyncing) return; - this.resyncing = true; - setTimeout(async () => { - try { - const snapshot = await fetch(`${this.base}/view`).then((r) => r.json()); - this.absorb(snapshot); - } catch (error) { - // Still down. The next error event will try again. - } - this.resyncing = false; - }, 1500); - } - - push(event, backlog) { - if (!event) return; - if (backlog) event.backlog = true; - this.queue.push(event); - } - - translate(kind, payload) { - if (kind === "operator") return this.operator(payload); - if (kind === "provisioned") return this.provisioned(payload); - if (kind === "scrobble") return this.scrobble(payload); - if (kind === "work") return { kind: "work", groups: payload.groups || [] }; - if (kind === "deleted") return { kind: "deleted", did: payload.did }; - if (kind === "pinned" && payload.pinned) return { kind: "pinned", did: payload.did }; - return null; - } - - operator(operator) { - this.operators.set(operator.did, operator); - return { - kind: "operator", - did: operator.did, - handle: operator.handle, - zone: operator.zone, - }; - } - - provisioned(agent) { - const operator = this.operators.get(agent.operator); - return { - kind: "provisioned", - did: agent.did, - agentId: agent.agentId, - handle: agent.handle || null, - name: shortName(agent.handle), - operator: agent.operator, - operatorHandle: operator ? operator.handle : agent.operator, - // Absent until the account's profile or one of its records says - // otherwise, and `null` rather than `undefined` because the canvas - // reads it as "no parent" and drawing a root is the right answer - // while nothing has said. - parent: agent.parent || null, - }; - } - - scrobble(scrobble) { - return { - kind: "scrobble", - did: scrobble.did, - emoji: scrobble.emoji, - text: scrobble.text, - effort: scrobble.effort, - }; - } - } - - /** - * The part of a handle a person says out loud. - * - * `basalt-otter.agents.localhost` is what resolves and `basalt-otter` is - * what anybody calls it; the zone after it is the same on every agent on the - * canvas, so it is width spent to say nothing. Null when the index reported - * no handle, which is what a deployment that does not name its agents looks - * like, and the canvas falls back to the minted label. - */ - function shortName(handle) { - if (typeof handle !== "string") return null; - const label = handle.split(".")[0].trim(); - return label || null; - } - - /** - * Connects to a query service, or reports that there is not one. - * - * The canvas defaults to live and falls back to the mock, rather than the - * other way round: somebody running the development stack should not have - * to ask for the data they just started three processes to produce, and - * somebody working on the canvas alone should not have to run any of them. - */ - async function connect(base) { - const feed = new LiveFeed(base || DEFAULT_QUERY); - const reached = await Promise.race([ - feed.connect().then(() => true).catch(() => false), - new Promise((resolve) => setTimeout(() => resolve(false), REACH_TIMEOUT)), - ]); - return reached && feed.connected ? feed : null; - } - - return { connect, LiveFeed, DEFAULT_QUERY }; -})(); diff --git a/web/mock.js b/web/mock.js deleted file mode 100644 index f249a4f3..00000000 --- a/web/mock.js +++ /dev/null @@ -1,513 +0,0 @@ -/* Mocked appview data. - * - * Every event here has the shape the real thing produces: `provisioned`, - * `deleted` and `pinned` are the server's lifecycle stream, and `scrobble` is - * a record written to an agent's repository. Nothing in app.js knows this file - * is a fake, so swapping it for a live subscription is a change here and a - * change to the wiring, and not to the view. - * - * Three things are modelled deliberately, because the view is meant to be - * judged at scale and a flat simulation would flatter it: - * - * * **Operators are not equal.** How many agents a person runs follows a Zipf - * weight, so a handful run dozens and the tail runs one or two. A uniform - * draw would make every cluster the same size, which is the one thing real - * populations never are. - * * **Sessions end.** Agents leave as well as arrive, and a session usually - * ends all at once — the root and everything it spawned — rather than one - * leaf at a time. - * * **Agents work on something.** Each session takes a job, subagents inherit - * their parent's, and the words come from the job. So agents doing the same - * work really are describable as one group, and a clustering pass over the - * text has something to find rather than noise to hallucinate. - * - * Every string is a placeholder. The words are deliberately obscure and - * greppable so nobody mistakes them for copy. - */ - -const Mock = (() => { - "use strict"; - - /** Agent labels. One DNS label each, because that is what a DID allows. */ - const AGENT_WORDS = [ - "kestrel", "marmot", "junco", "obsidian", "tanager", "quillback", "sarsen", - "ganister", "pintle", "wold", "gorse", "lammas", "quernstone", "fettle", - "thrimble", "brail", "snath", "withy", "garron", "tholepin", "ravelin", - "kern", "spandrel", "coomb", "linhay", "mullein", "teasel", "sedge", - "bracken", "whin", "birk", "rowan", "alder", "sallow", "chert", "flint", - "gabbro", "gneiss", "schist", "marl", "loess", "scree", "tarn", "ghyll", - "clough", "holt", "spinney", "shaw", "carr", "haugh", "fell", "sike", - "beck", "gill", "rill", "mere", "flash", "dale", "moss", "brant", - ]; - - /** Operator handles. The humans who wrote the vouch records. */ - const OPERATOR_WORDS = [ - "hoggin", "wappen", "brack", "dorse", "felloe", "girt", "keld", "lade", - "nabble", "orrel", "pight", "quop", "rample", "sneck", "tine", "umber", - "verge", "wath", "yett", "addle", "byre", "croft", "dyke", "ewer", - "forge", "garth", "hearth", "ingle", "kiln", "lench", "morrow", "nogging", - ]; - - /** - * What a session is working on. - * - * A project is a small world of real words: the parts it is built from, and - * the terms that only come up while working on it. An agent picks one part - * of one project and mostly keeps saying things about that part, because - * that is what working on something looks like — the same subject over and - * over, with the verbs changing as it goes from reading to building to - * debugging to shipping. - * - * The vocabulary is ordinary English on purpose. Obscure placeholder words - * would still cluster, but nobody reading the screen could tell whether the - * grouping meant anything, and the whole question this view answers is - * whether two agents are working on the same thing. - */ - const PROJECTS = [ - { - id: "checkout", - parts: ["cart", "coupon code", "tax calculation", "payment intent", "receipt email", "address form"], - terms: ["idempotency key", "currency rounding", "abandoned cart", "gift card"], - }, - { - id: "auth", - parts: ["login form", "session store", "refresh token", "password reset", "device list", "signup flow"], - terms: ["token rotation", "rate limit", "audit log", "magic link"], - }, - { - id: "search", - parts: ["query parser", "ranking model", "autocomplete", "synonym list", "search index", "results page"], - terms: ["stop word", "typo tolerance", "faceted filter", "click signal"], - }, - { - id: "billing", - parts: ["invoice", "proration", "subscription", "dunning email", "usage meter", "plan picker"], - terms: ["grace period", "webhook retry", "tax exemption", "trial period"], - }, - { - id: "ingest", - parts: ["consumer group", "backfill job", "dedupe step", "schema registry", "dead letter queue", "batch writer"], - terms: ["consumer lag", "replay window", "watermark", "partition key"], - }, - { - id: "mobile", - parts: ["onboarding screen", "push notification", "deep link", "offline cache", "photo picker", "settings page"], - terms: ["cold start", "background refresh", "permission prompt", "crash report"], - }, - { - id: "infra", - parts: ["terraform module", "autoscaler", "canary deploy", "alert rule", "log pipeline", "secret rotation"], - terms: ["blast radius", "rollback plan", "noisy alert", "capacity headroom"], - }, - { - id: "models", - parts: ["embedding job", "eval set", "prompt template", "tokenizer", "feature store", "drift monitor"], - terms: ["holdout split", "context window", "label noise", "inference cost"], - }, - { - id: "docs", - parts: ["changelog", "migration guide", "api reference", "quickstart", "example project", "error catalogue"], - terms: ["broken link", "code sample", "screenshot", "release note"], - }, - { - id: "performance", - parts: ["n+1 query", "cache layer", "bundle size", "image pipeline", "connection pool", "startup path"], - terms: ["hit rate", "tail latency", "memory ceiling", "flame graph"], - }, - ]; - - /** - * How an agent talks at each stage of a piece of work. - * - * Ordered: an agent moves along this list and does not go back, which is - * roughly how a session reads from the outside even when the work itself is - * messier than that. - */ - const PHASES = [ - { - id: "reading", - lines: [ - "reading through the {part}", - "tracing how the {part} is wired up", - "mapping the {part} end to end", - "checking whether the {part} already handles {other}", - "looking for where {other} is set on the {part}", - ], - }, - { - id: "building", - lines: [ - "adding {other} handling to the {part}", - "splitting the {part} into smaller pieces", - "moving the {part} behind a feature flag", - "rewriting the {part} so {other} is not special-cased", - "wiring the {part} up to {other}", - ], - }, - { - id: "debugging", - lines: [ - "the {part} times out once {other} is involved", - "narrowed the {part} failure down to {other}", - "the {part} returns stale data after {other} changes", - "reproduced the {part} bug locally", - "{other} and the {part} disagree about ordering", - ], - }, - { - id: "testing", - lines: [ - "writing tests for the {part}", - "{n} tests still failing in the {part} suite", - "adding coverage around {other}", - "the {part} test is flaky under load", - "checking the {part} against {other} one more time", - ], - }, - { - id: "shipping", - lines: [ - "opened a pull request for the {part}", - "rebasing the {part} branch onto main", - "documenting the new {part} behaviour", - "cleaning up after the {part} change", - "waiting on review for the {part}", - ], - }, - ]; - - const EMOJI = [ - "🧭", "🌱", "🔗", "📬", "🪵", "🧵", "🪨", "🛠️", "🔍", "🧪", "📐", "🫧", - "🌾", "🪞", "🕯️", "🧊", "🌗", "🍂", "⚙️", "🪤", "🧱", "🐚", "🪶", "🜂", - ]; - - const EFFORT = ["low", "medium", "medium", "high", "high", "xhigh", "max"]; - - /** Share of new agents that are spawned by another agent, not by a human. */ - const SUBAGENT_SHARE = 0.58; - - /** - * How much less likely each generation is to spawn than the one above it. - * - * A session root delegates freely, what it spawned delegates occasionally, - * and below that almost nothing does. Weighting a candidate parent by this - * to the power of its depth gives roughly half the population at depth - * zero, a third at depth one, under a tenth at depth two, and a couple of - * agents at depth three anywhere on the screen at once. Four and five exist - * and are a thing you have to go looking for. - */ - const DEPTH_FALLOFF = 0.3; - - /** Nothing spawns below this, however the dice fall. */ - const MAX_DEPTH = 5; - - /** Deterministic RNG, so a screenshot of this is reproducible. */ - function mulberry32(seed) { - let a = seed >>> 0; - return function random() { - a = (a + 0x6d2b79f5) >>> 0; - let t = Math.imul(a ^ (a >>> 15), 1 | a); - t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; - return ((t ^ (t >>> 14)) >>> 0) / 4294967296; - }; - } - - /** The zone one operator's agents are minted under. */ - function zoneOf(handle) { - return `agents.${handle}.example`; - } - - class MockFeed { - /** - * @param {object} options - * @param {number} options.operators how many humans have vouched - * @param {number} options.agents how many agents to seed - * @param {number} options.seed RNG seed - * @param {number} options.skew Zipf exponent for agents per operator - */ - constructor({ operators = 30, agents = 420, seed = 20260825, skew = 0.85 } = {}) { - this.random = mulberry32(seed); - this.operatorCount = operators; - this.seedAgents = agents; - this.skew = skew; - this.operators = []; - this.weights = []; - this.live = new Map(); - this.roots = []; - this.counter = 0; - this.pending = 0; - } - - pick(list) { - return list[Math.floor(this.random() * list.length)]; - } - - /** An operator, drawn against the Zipf weights. */ - pickOperator() { - const roll = this.random() * this.weights[this.weights.length - 1]; - let low = 0; - let high = this.weights.length - 1; - while (low < high) { - const mid = (low + high) >> 1; - if (this.weights[mid] < roll) low = mid + 1; - else high = mid; - } - return this.operators[low]; - } - - /** A fresh agent label that no live agent is using. */ - label() { - this.counter += 1; - const word = this.pick(AGENT_WORDS); - // Two agents may hold the same word at different times, but never at - // once: the counter is what keeps the hostname unique. - return `${word}-${this.counter.toString(36)}`; - } - - /** - * Builds the operator set and the weights that decide how busy each is. - * - * Zipf rather than uniform: with 30 operators and an exponent near one, - * the busiest runs on the order of twenty times what the quietest does, - * which is the shape a population of people running agents actually has. - */ - buildOperators() { - const words = OPERATOR_WORDS.slice(0, this.operatorCount); - this.operators = words.map((word, index) => ({ - did: `did:plc:${word.padEnd(24, "quernstone").slice(0, 24)}`, - handle: `${word}.example`, - zone: zoneOf(word), - index, - })); - // Ranks are shuffled so the busiest operator is not always the first - // one drawn, which would put it in the middle of the canvas every time. - const ranks = this.operators.map((_, index) => index); - for (let i = ranks.length - 1; i > 0; i -= 1) { - const j = Math.floor(this.random() * (i + 1)); - [ranks[i], ranks[j]] = [ranks[j], ranks[i]]; - } - let total = 0; - this.weights = this.operators.map((operator, index) => { - operator.weight = 1 / Math.pow(ranks[index] + 1, this.skew); - total += operator.weight; - return total; - }); - return this.operators; - } - - /** One `provisioned` event, spawned under `parent` when there is one. */ - provision(operator, parent) { - const agentId = this.label(); - const did = `did:web:${agentId}.${operator.zone}`; - // A subagent inherits the project and usually the part: it was spawned - // to help with this, not to go and do something else. - const project = parent ? parent.project : this.pick(PROJECTS); - const part = parent && this.random() < 0.7 ? parent.part : this.pick(project.parts); - const event = { - kind: "provisioned", - did, - agentId, - operator: operator.did, - operatorHandle: operator.handle, - parent: parent ? parent.did : null, - assurance: parent ? "self-asserted" : "shared-secret", - }; - const agent = { - did, - operator, - project, - part, - phase: 0, - depth: parent ? parent.depth + 1 : 0, - parent: parent || null, - children: [], - }; - this.live.set(did, agent); - if (parent) parent.children.push(agent); - else this.roots.push(agent); - return event; - } - - /** - * Picks the agent that spawns the next one, or nothing for a new session. - * - * Weighted by depth, in one pass rather than by building a list: with a - * few hundred agents and a spawn a second, that list was the largest - * allocation this simulation made. Each candidate replaces the one held - * with probability proportional to its weight, which samples correctly - * without knowing the total in advance. - */ - pickParent(operator) { - let chosen = null; - let total = 0; - for (const agent of this.live.values()) { - if (agent.operator !== operator || agent.depth >= MAX_DEPTH) continue; - const weight = Math.pow(DEPTH_FALLOFF, agent.depth); - total += weight; - if (this.random() < weight / total) chosen = agent; - } - return chosen; - } - - /** Chooses who spawns next: a session root, or a subagent under one. */ - nextSpawn() { - const operator = this.pickOperator(); - const parent = this.random() < SUBAGENT_SHARE ? this.pickParent(operator) : null; - return this.provision(operator, parent); - } - - /** One `scrobble` event from a live agent, in its session's vocabulary. */ - scrobble() { - const agents = [...this.live.values()]; - if (agents.length === 0) return null; - const agent = this.pick(agents); - return { - kind: "scrobble", - did: agent.did, - emoji: this.pick(EMOJI), - text: this.text(agent), - effort: this.pick(EFFORT), - }; - } - - /** - * One status line from an agent, about whatever it is working on. - * - * Mostly the part it has been on all along, sometimes a neighbouring part - * of the same project, occasionally one of the project's own terms. That - * ratio is the whole point: repetition is what makes two agents on the - * same thing recognisable, and the variation is what stops the recognition - * from being trivial. - */ - text(agent) { - // Work moves forward, slowly, and never backward. - if (this.random() < 0.06 && agent.phase < PHASES.length - 1) { - agent.phase += 1; - } - const roll = this.random(); - const part = - roll < 0.68 - ? agent.part - : roll < 0.9 - ? this.pick(agent.project.parts) - : this.pick(agent.project.terms); - // Never the same thing twice in one line: "the cart already handles the - // cart" is the kind of sentence only a generator writes. - let other = part; - for (let tries = 0; tries < 6 && other === part; tries += 1) { - other = - this.random() < 0.5 ? this.pick(agent.project.terms) : this.pick(agent.project.parts); - } - return this.pick(PHASES[agent.phase].lines) - .replace("{part}", part) - .replace("{other}", other) - .replace("{n}", String(1 + Math.floor(this.random() * 12))); - } - - /** Detaches one agent from its parent and the live set. */ - drop(agent) { - const siblings = agent.parent ? agent.parent.children : this.roots; - const index = siblings.indexOf(agent); - if (index >= 0) siblings.splice(index, 1); - this.live.delete(agent.did); - } - - /** - * Ends a whole session: the root and everything it spawned. - * - * Deepest first, so a reader of the stream never sees a parent leave - * before its children. - */ - endSession() { - if (this.roots.length === 0) return []; - const root = this.pick(this.roots); - const doomed = []; - const stack = [root]; - while (stack.length > 0) { - const agent = stack.pop(); - doomed.push(agent); - for (const child of agent.children) stack.push(child); - } - doomed.sort((a, b) => b.depth - a.depth); - for (const agent of doomed) this.drop(agent); - return doomed.map((agent) => ({ kind: "deleted", did: agent.did })); - } - - /** - * Ends one agent that has nothing under it. - * - * Roots included: a session that never delegated finishes the same way a - * subagent does, and excluding them leaves the population slowly filling - * up with root agents that only a whole-session teardown can clear. - */ - endLeaf() { - const leaves = [...this.live.values()].filter((agent) => agent.children.length === 0); - if (leaves.length === 0) return []; - const agent = this.pick(leaves); - this.drop(agent); - return [{ kind: "deleted", did: agent.did }]; - } - - /** Everything that "already happened" when the view opens. */ - seed() { - const events = []; - for (const operator of this.buildOperators()) { - events.push({ - kind: "operator", - did: operator.did, - handle: operator.handle, - zone: operator.zone, - }); - } - for (let i = 0; i < this.seedAgents; i += 1) { - events.push(this.nextSpawn()); - } - // Enough backlog that every agent has said something, so an opening - // screenshot is not a field of silent dots. - for (let i = 0; i < this.seedAgents * 2; i += 1) { - const event = this.scrobble(); - if (event) events.push(event); - } - return events; - } - - /** - * Advances the stream. - * - * @param {number} dt seconds since the last call - * @param {number} rate events per second - * @returns {object[]} events to apply - */ - tick(dt, rate) { - this.pending += dt * rate; - const events = []; - while (this.pending >= 1) { - this.pending -= 1; - const roll = this.random(); - // Births and deaths are balanced against each other: a session - // teardown takes a whole tree, which averages a shade over two agents, - // so it is worth about half as much as its share of the rolls - // suggests. Get this wrong and the population either drains away or - // fills the screen over the course of ten minutes. - if (roll < 0.68) { - const event = this.scrobble(); - if (event) events.push(event); - } else if (roll < 0.86) { - events.push(this.nextSpawn()); - } else if (roll < 0.935) { - events.push(...this.endLeaf()); - } else if (roll < 0.985) { - events.push(...this.endSession()); - } else { - const agents = [...this.live.values()]; - if (agents.length > 0) { - events.push({ kind: "pinned", did: this.pick(agents).did }); - } - } - } - return events; - } - } - - return { MockFeed, PROJECTS, PHASES, EMOJI, AGENT_WORDS, OPERATOR_WORDS }; -})(); diff --git a/web/mockups.html b/web/mockups.html deleted file mode 100644 index 1c6bc9e8..00000000 --- a/web/mockups.html +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - operator hub mockups - - - - - -

operator hub — mockups

-

- Everything below is drawn at the size it appears on the canvas, on the - canvas ground. Pictures are generated from the DID; a real deployment - puts the account’s own avatar in the same place. -

- -

picture

-
- -

effort

-

- The harness reports the effort a turn actually spent — five levels, - on every scrobble. Which channel carries that without a legend? -

-
- -

framing

-
- - - - - diff --git a/web/mockups.js b/web/mockups.js deleted file mode 100644 index f08f2a39..00000000 --- a/web/mockups.js +++ /dev/null @@ -1,392 +0,0 @@ -/* Draws the mockup sheet. - * - * Deliberately separate from app.js: this is a sheet for choosing between - * treatments, and the moment it shares code with the canvas it stops being - * able to show a treatment the canvas does not implement. - */ - -(() => { - "use strict"; - - const OPERATORS = [ - { did: "did:plc:hogginquernstonequernsto", handle: "hoggin.example", hue: 8, agents: 87 }, - { did: "did:plc:wappenquernstonequernsto", handle: "wappen.example", hue: 137, agents: 24 }, - { did: "did:plc:sneckquernstonequernston", handle: "sneck.example", hue: 268, agents: 6 }, - ]; - - const ratio = Math.min(2, window.devicePixelRatio || 1); - - function board(width, height) { - const canvas = document.createElement("canvas"); - canvas.width = width * ratio; - canvas.height = height * ratio; - canvas.style.width = `${width}px`; - canvas.style.height = `${height}px`; - const paint = canvas.getContext("2d"); - paint.scale(ratio, ratio); - paint.fillStyle = "#0b0d11"; - paint.fillRect(0, 0, width, height); - return { canvas, paint }; - } - - function cell(canvas, title, note) { - const figure = document.createElement("figure"); - figure.className = "cell"; - figure.style.margin = "0"; - const caption = document.createElement("figcaption"); - caption.textContent = title; - const small = document.createElement("small"); - small.textContent = note; - caption.append(small); - figure.append(canvas, caption); - return figure; - } - - /** A few agents around a hub, so a treatment is judged in context. */ - function branches(paint, cx, cy, hue, count) { - const golden = 2.399963229728653; - for (let i = 0; i < count; i += 1) { - const angle = i * golden; - const reach = 46 + (i % 3) * 9; - const x = cx + Math.cos(angle) * reach; - const y = cy + Math.sin(angle) * reach; - paint.strokeStyle = `hsla(${hue}, 45%, 68%, 0.3)`; - paint.lineWidth = 1.4; - paint.beginPath(); - paint.moveTo(cx + Math.cos(angle) * 20, cy + Math.sin(angle) * 20); - paint.quadraticCurveTo( - (cx + x) / 2 - Math.sin(angle) * 10, - (cy + y) / 2 + Math.cos(angle) * 10, - x, - y - ); - paint.stroke(); - paint.fillStyle = `hsla(${hue}, 55%, 60%, 0.95)`; - paint.beginPath(); - paint.arc(x, y, i % 3 === 0 ? 5 : 3.4, 0, Math.PI * 2); - paint.fill(); - if (i % 4 === 1) { - paint.font = "12px ui-monospace, monospace"; - paint.textAlign = "center"; - paint.fillStyle = "#f2f5f9"; - paint.fillText("\u{1F9ED}", x, y - 12); - } - } - } - - function label(paint, x, y, text, hue) { - paint.font = "12px ui-monospace, monospace"; - paint.textAlign = "center"; - paint.textBaseline = "middle"; - const width = paint.measureText(text).width; - paint.fillStyle = "rgba(11, 13, 17, 0.65)"; - paint.fillRect(x - width / 2 - 5, y - 8, width + 10, 16); - paint.fillStyle = `hsla(${hue}, 65%, 82%, 0.95)`; - paint.fillText(text, x, y); - } - - // --- picture styles ------------------------------------------------------- - - const styleNotes = { - blocks: "hard edges, survives the smallest sizes", - orb: "reads as a photograph, which is what replaces it", - initials: "says something true about the account", - bands: "quiet; hard to tell apart across a crowd", - }; - - const styles = document.getElementById("styles"); - for (const style of Avatar.styles) { - const { canvas, paint } = board(230, 96); - OPERATORS.forEach((operator, index) => { - const size = [40, 30, 22][index]; - const x = 36 + index * 70; - const y = 44; - const picture = Avatar.make(operator.did, operator.handle, size, style); - paint.drawImage(picture, x - size / 2, y - size / 2, size, size); - paint.font = "11px ui-monospace, monospace"; - paint.textAlign = "center"; - paint.fillStyle = "#78828f"; - paint.fillText(`${size}px`, x, y + size / 2 + 14); - }); - styles.append(cell(canvas, style, styleNotes[style])); - } - - // --- framings ------------------------------------------------------------- - - const FRAMINGS = [ - { - title: "bare", - note: "picture only, no ring", - draw(paint, cx, cy, operator) { - const size = 34; - paint.drawImage( - Avatar.make(operator.did, operator.handle, size), - cx - size / 2, - cy - size / 2, - size, - size - ); - }, - }, - { - title: "owner ring", - note: "the colour every agent below is drawn in", - draw(paint, cx, cy, operator) { - const size = 34; - paint.drawImage( - Avatar.make(operator.did, operator.handle, size), - cx - size / 2, - cy - size / 2, - size, - size - ); - paint.strokeStyle = `hsl(${operator.hue}, 62%, 70%)`; - paint.lineWidth = 2.5; - paint.beginPath(); - paint.arc(cx, cy, size / 2 + 2, 0, Math.PI * 2); - paint.stroke(); - }, - }, - { - title: "load arc", - note: "arc length is how many agents they run", - draw(paint, cx, cy, operator) { - const size = 34; - paint.drawImage( - Avatar.make(operator.did, operator.handle, size), - cx - size / 2, - cy - size / 2, - size, - size - ); - paint.strokeStyle = "rgba(120, 130, 145, 0.35)"; - paint.lineWidth = 3; - paint.beginPath(); - paint.arc(cx, cy, size / 2 + 4, 0, Math.PI * 2); - paint.stroke(); - const share = Math.min(1, operator.agents / 90); - paint.strokeStyle = `hsl(${operator.hue}, 68%, 68%)`; - paint.lineCap = "round"; - paint.beginPath(); - paint.arc(cx, cy, size / 2 + 4, -Math.PI / 2, -Math.PI / 2 + share * Math.PI * 2); - paint.stroke(); - paint.lineCap = "butt"; - }, - }, - { - title: "hexagon", - note: "a human is not the same shape as an agent", - draw(paint, cx, cy, operator) { - const size = 36; - const radius = size / 2; - paint.save(); - paint.beginPath(); - for (let i = 0; i < 6; i += 1) { - const angle = (Math.PI / 3) * i - Math.PI / 2; - const x = cx + Math.cos(angle) * radius; - const y = cy + Math.sin(angle) * radius; - if (i === 0) paint.moveTo(x, y); - else paint.lineTo(x, y); - } - paint.closePath(); - paint.clip(); - paint.drawImage( - Avatar.make(operator.did, operator.handle, size * 1.16), - cx - size * 0.58, - cy - size * 0.58, - size * 1.16, - size * 1.16 - ); - paint.restore(); - paint.strokeStyle = `hsl(${operator.hue}, 62%, 70%)`; - paint.lineWidth = 2; - paint.stroke(); - }, - }, - { - title: "halo", - note: "lit from behind, so it sits above the branches", - draw(paint, cx, cy, operator) { - const size = 34; - const glow = paint.createRadialGradient(cx, cy, size / 2, cx, cy, size * 1.5); - glow.addColorStop(0, `hsla(${operator.hue}, 70%, 62%, 0.4)`); - glow.addColorStop(1, `hsla(${operator.hue}, 70%, 62%, 0)`); - paint.fillStyle = glow; - paint.beginPath(); - paint.arc(cx, cy, size * 1.5, 0, Math.PI * 2); - paint.fill(); - paint.drawImage( - Avatar.make(operator.did, operator.handle, size), - cx - size / 2, - cy - size / 2, - size, - size - ); - paint.strokeStyle = `hsla(${operator.hue}, 62%, 78%, 0.9)`; - paint.lineWidth = 1.5; - paint.beginPath(); - paint.arc(cx, cy, size / 2 + 1.5, 0, Math.PI * 2); - paint.stroke(); - }, - }, - { - title: "counted", - note: "the number is there without being looked up", - draw(paint, cx, cy, operator) { - const size = 34; - paint.drawImage( - Avatar.make(operator.did, operator.handle, size), - cx - size / 2, - cy - size / 2, - size, - size - ); - paint.strokeStyle = `hsl(${operator.hue}, 62%, 70%)`; - paint.lineWidth = 2.5; - paint.beginPath(); - paint.arc(cx, cy, size / 2 + 2, 0, Math.PI * 2); - paint.stroke(); - const text = String(operator.agents); - paint.font = "10px ui-monospace, monospace"; - const width = paint.measureText(text).width; - const bx = cx + size / 2 - 2; - const by = cy + size / 2 - 2; - paint.fillStyle = "#0b0d11"; - paint.beginPath(); - paint.arc(bx, by, width / 2 + 5, 0, Math.PI * 2); - paint.fill(); - paint.strokeStyle = `hsla(${operator.hue}, 62%, 70%, 0.8)`; - paint.lineWidth = 1; - paint.stroke(); - paint.fillStyle = `hsl(${operator.hue}, 60%, 84%)`; - paint.textAlign = "center"; - paint.textBaseline = "middle"; - paint.fillText(text, bx, by); - }, - }, - ]; - - // --- effort --------------------------------------------------------------- - // - // The harness reports what a turn actually cost. Five levels, and the - // question is which visual channel carries them without a legend. - - const LEVELS = ["low", "medium", "high", "xhigh", "max"]; - const SCALE = { low: 0.7, medium: 1, high: 1.32, xhigh: 1.68, max: 2.1 }; - - const EFFORTS = [ - { - title: "size", - note: "reads at any zoom; needs the branch to carry subtree size", - draw(paint, x, y, level, hue) { - paint.fillStyle = `hsl(${hue}, 55%, 62%)`; - paint.beginPath(); - paint.arc(x, y, 5 * SCALE[level], 0, Math.PI * 2); - paint.fill(); - }, - }, - { - title: "brightness", - note: "cheap, but competes with how recently it spoke", - draw(paint, x, y, level, hue) { - const step = LEVELS.indexOf(level) / (LEVELS.length - 1); - paint.fillStyle = `hsl(${hue}, ${40 + step * 40}%, ${34 + step * 40}%)`; - paint.beginPath(); - paint.arc(x, y, 6, 0, Math.PI * 2); - paint.fill(); - }, - }, - { - title: "ring", - note: "thickness of a halo; survives the emoji mode", - draw(paint, x, y, level, hue) { - paint.fillStyle = `hsl(${hue}, 50%, 58%)`; - paint.beginPath(); - paint.arc(x, y, 5, 0, Math.PI * 2); - paint.fill(); - paint.strokeStyle = `hsla(${hue}, 65%, 74%, 0.9)`; - paint.lineWidth = 0.6 + LEVELS.indexOf(level) * 0.9; - paint.beginPath(); - paint.arc(x, y, 9, 0, Math.PI * 2); - paint.stroke(); - }, - }, - { - title: "pips", - note: "exact and countable, illegible below full zoom", - draw(paint, x, y, level, hue) { - paint.fillStyle = `hsl(${hue}, 50%, 58%)`; - paint.beginPath(); - paint.arc(x, y, 5, 0, Math.PI * 2); - paint.fill(); - const count = LEVELS.indexOf(level) + 1; - for (let i = 0; i < count; i += 1) { - const angle = -Math.PI / 2 + (i - (count - 1) / 2) * 0.42; - paint.fillStyle = `hsl(${hue}, 70%, 80%)`; - paint.beginPath(); - paint.arc(x + Math.cos(angle) * 11, y + Math.sin(angle) * 11, 1.6, 0, Math.PI * 2); - paint.fill(); - } - }, - }, - { - title: "glow", - note: "beautiful in a crowd, unreadable one at a time", - draw(paint, x, y, level, hue) { - const reach = 8 + LEVELS.indexOf(level) * 7; - const glow = paint.createRadialGradient(x, y, 0, x, y, reach); - glow.addColorStop(0, `hsla(${hue}, 80%, 68%, 0.75)`); - glow.addColorStop(1, `hsla(${hue}, 80%, 68%, 0)`); - paint.fillStyle = glow; - paint.beginPath(); - paint.arc(x, y, reach, 0, Math.PI * 2); - paint.fill(); - paint.fillStyle = `hsl(${hue}, 55%, 62%)`; - paint.beginPath(); - paint.arc(x, y, 4.5, 0, Math.PI * 2); - paint.fill(); - }, - }, - { - title: "size, as emoji", - note: "what the same channel does once agents are their last word", - draw(paint, x, y, level, hue) { - paint.strokeStyle = `hsla(${hue}, 60%, 66%, 0.85)`; - paint.lineWidth = 1.4; - paint.beginPath(); - paint.arc(x, y, 8.5 * (0.72 + 0.28 * SCALE[level]), 0, Math.PI * 2); - paint.stroke(); - paint.fillStyle = "#f2f5f9"; - paint.textAlign = "center"; - paint.textBaseline = "middle"; - paint.font = `${(11 + SCALE[level] * 3.5).toFixed(1)}px ui-monospace, monospace`; - paint.fillText("\u{1F9ED}", x, y); - }, - }, - ]; - - const efforts = document.getElementById("efforts"); - for (const effort of EFFORTS) { - const { canvas, paint } = board(300, 96); - LEVELS.forEach((level, index) => { - const x = 42 + index * 54; - effort.draw(paint, x, 40, level, OPERATORS[0].hue); - paint.font = "10px ui-monospace, monospace"; - paint.textAlign = "center"; - paint.textBaseline = "alphabetic"; - paint.fillStyle = "#78828f"; - paint.fillText(level, x, 74); - }); - efforts.append(cell(canvas, effort.title, effort.note)); - } - - const framings = document.getElementById("framings"); - for (const framing of FRAMINGS) { - const { canvas, paint } = board(300, 200); - const operator = OPERATORS[0]; - branches(paint, 150, 100, operator.hue, 9); - framing.draw(paint, 150, 100, operator); - label(paint, 150, 100 + 34, operator.handle, operator.hue); - framings.append(cell(canvas, framing.title, framing.note)); - } -})(); diff --git a/web/style.css b/web/style.css deleted file mode 100644 index 5e3342fa..00000000 --- a/web/style.css +++ /dev/null @@ -1,221 +0,0 @@ -/* The appview canvas. Dark by default: this is a wall display, and the - interesting thing on it is a few hundred small bright points. */ - -:root { - --ink: #dfe4ec; - --ink-dim: #78828f; - --ground: #0b0d11; - --panel: rgba(14, 17, 22, 0.82); - --line: rgba(120, 130, 145, 0.22); - --accent: #8fd0a0; - --warn: #e0a08f; - --mono: ui-monospace, "SF Mono", "JetBrains Mono", "DejaVu Sans Mono", monospace; -} - -* { box-sizing: border-box; } - -html, body { - margin: 0; - height: 100%; - overflow: hidden; - background: var(--ground); - color: var(--ink); - font: 13px/1.5 var(--mono); -} - -#stage { - position: fixed; - inset: 0; - display: block; - cursor: grab; -} -#stage.dragging { cursor: grabbing; } - -/* ---- top bar ---- */ - -.bar { - position: fixed; - inset: 12px 12px auto 12px; - display: flex; - flex-wrap: wrap; - align-items: center; - gap: 10px; - padding: 8px 12px; - background: var(--panel); - border: 1px solid var(--line); - border-radius: 8px; - backdrop-filter: blur(6px); -} - -.mark { color: var(--accent); letter-spacing: 0.04em; } -.source { - padding: 1px 7px; - border: 1px solid var(--line); - border-radius: 999px; - color: var(--ink-dim); -} -.source.live { color: var(--accent); border-color: rgba(143, 208, 160, 0.45); } -.sep { width: 1px; height: 18px; background: var(--line); } - -.field { display: flex; align-items: center; gap: 6px; color: var(--ink-dim); } - -input[type="search"] { - width: 180px; - padding: 3px 7px; - color: var(--ink); - background: rgba(255, 255, 255, 0.04); - border: 1px solid var(--line); - border-radius: 5px; - font: inherit; -} -input[type="search"]:focus { outline: 1px solid var(--accent); } -input[type="range"] { width: 110px; accent-color: var(--accent); } -output { min-width: 42px; color: var(--ink); } - -button { - padding: 3px 10px; - color: var(--ink-dim); - background: rgba(255, 255, 255, 0.03); - border: 1px solid var(--line); - border-radius: 5px; - font: inherit; - cursor: pointer; -} -button:hover { color: var(--ink); } -button.on { color: var(--accent); border-color: rgba(143, 208, 160, 0.4); } - -/* ---- side panel ---- */ - -.panel { - position: fixed; - top: 70px; - right: 12px; - bottom: 46px; - width: 320px; - display: flex; - flex-direction: column; - gap: 12px; - padding: 12px; - background: var(--panel); - border: 1px solid var(--line); - border-radius: 8px; - backdrop-filter: blur(6px); -} - -.stats { display: grid; grid-template-columns: 1fr 1fr; gap: 5px 12px; margin: 0; } -.stats div { display: flex; justify-content: space-between; gap: 8px; } -.stats dt { color: var(--ink-dim); } -.stats dd { margin: 0; font-variant-numeric: tabular-nums; } - -.panel-title { - margin: 0; - font-size: 13px; - font-weight: normal; - color: var(--ink-dim); - border-top: 1px solid var(--line); - padding-top: 10px; -} - -.focus { - padding: 8px; - border: 1px solid rgba(143, 208, 160, 0.35); - border-radius: 6px; - background: rgba(143, 208, 160, 0.06); -} -.focus-head { display: flex; justify-content: space-between; align-items: baseline; gap: 8px; } -.focus-detail { margin: 6px 0 0; color: var(--ink-dim); word-break: break-all; } - -.panel-title .muted { color: var(--ink-dim); float: right; } - -.clusters { - margin: 0; - padding: 0; - list-style: none; - display: flex; - flex-direction: column; - gap: 2px; - max-height: 168px; - overflow: hidden; -} -.clusters li { - display: flex; - align-items: baseline; - gap: 7px; - white-space: nowrap; - cursor: pointer; -} -.clusters li:hover .cluster-label { color: var(--ink); } -.cluster-swatch { width: 8px; height: 8px; border-radius: 2px; flex: none; } -.cluster-label { color: var(--ink-dim); overflow: hidden; text-overflow: ellipsis; } -.cluster-size { margin-left: auto; color: var(--ink); font-variant-numeric: tabular-nums; } -.clusters li.on .cluster-label { color: var(--accent); } - -.log { - flex: 1; - overflow: hidden; - margin: 0; - padding: 0; - list-style: none; - display: flex; - flex-direction: column; - gap: 3px; -} -.log li { - /* Never shrink: a fixed number of lines in a fixed height would otherwise - compress below the line height and print on top of each other. */ - flex: none; - display: flex; - gap: 7px; - align-items: baseline; - white-space: nowrap; - overflow: hidden; - animation: slide 0.25s ease-out; -} -.log .who { color: var(--ink); flex: none; } -.log .what { color: var(--ink-dim); overflow: hidden; text-overflow: ellipsis; } -.log .kind-provisioned .who { color: var(--accent); } -.log .kind-deleted .who { color: var(--warn); } - -@keyframes slide { from { opacity: 0; transform: translateY(-4px); } } - -/* ---- tooltip and legend ---- */ - -.tooltip { - position: fixed; - z-index: 5; - max-width: 300px; - padding: 7px 9px; - background: rgba(8, 10, 13, 0.94); - border: 1px solid var(--line); - border-radius: 6px; - pointer-events: none; - word-break: break-all; -} -.tooltip b { font-weight: normal; color: var(--accent); } -.tooltip span { color: var(--ink-dim); } - -.legend { - position: fixed; - left: 12px; - bottom: 12px; - display: flex; - align-items: center; - gap: 14px; - padding: 6px 12px; - color: var(--ink-dim); - background: var(--panel); - border: 1px solid var(--line); - border-radius: 8px; -} -.legend span { display: flex; align-items: center; gap: 6px; } -.dot { width: 9px; height: 9px; border-radius: 50%; display: inline-block; } -.dot-root { background: #c9cfd8; width: 11px; height: 11px; } -.dot-child { background: #8a93a0; width: 7px; height: 7px; } -.dot-hub { background: transparent; border: 2px solid #c9cfd8; width: 13px; height: 13px; } -.dot-pinned { background: transparent; border: 1.5px solid #e6d79a; } -.hint { opacity: 0.65; } - -@media (max-width: 1100px) { - .panel { display: none; } - .legend { display: none; } -} diff --git a/web/work.js b/web/work.js deleted file mode 100644 index 7dcc0c49..00000000 --- a/web/work.js +++ /dev/null @@ -1,439 +0,0 @@ -/* Finding agents that are doing the same work. - * - * The question this answers: out of several hundred agents on several dozen - * servers, which ones are working on the same thing right now? Nobody tells - * the appview that. All it has is a stream of short status lines. - * - * # Why this algorithm - * - * The constraint is a CPU and a browser tab: no matrix library, no embedding - * model, no quadratic pass over the population on every event. What fits is - * the classic near-duplicate detection stack, which is streaming and - * near-linear: - * - * 1. **Shingle** each line into tokens, stopwords dropped, and fold it into - * the agent's running profile: every existing weight decays a little, and - * the words just used gain. What an agent is working on is the exponential - * moving average of what it has been saying, not its last few lines. Say - * the same thing ten times and the profile is sharp and stable; start - * saying something else and it drifts across to the new subject over a - * handful of updates, which is the behaviour a person watching would call - * correct. - * 2. **Weight** each token by inverse document frequency, kept as a running - * count. Without this the phrases every job shares — the scaffolding words - * a template puts around the interesting nouns — dominate the fingerprint - * and every agent looks like every other. IDF is what makes the rare word - * the one that decides. - * 3. **SimHash** the weighted profile into one 32-bit fingerprint, and - * **band** it into four 8-bit keys. This is locality-sensitive hashing, and - * its job here is recall, not judgement: two agents that collide in a band - * are worth comparing properly, and everything else can be skipped without - * being looked at. That is what keeps the whole thing off the n² path as - * the population grows. - * 4. **Score the candidates by cosine similarity** over the profiles - * themselves, held as sorted typed arrays of interned word ids so a - * comparison is a two-pointer merge over two small `Int32Array`s rather - * than a walk of a hash map. A fingerprint is a lossy summary and thirty-two bits is a - * blunt instrument — real topics sit closer together than random noise but - * not by much, so a Hamming threshold either misses most of them or merges - * unrelated ones. The profiles are a few dozen weighted words each, so - * comparing them exactly costs almost nothing once the candidate set is - * small. - * 5. **Assign to leaders**, not to each other. Each cluster is represented by - * one profile, and an agent joins the most similar leader above the - * threshold or becomes a leader itself. - * - * The last step is the one worth defending. Joining agents to *each other* and - * taking connected components is the obvious move and it collapses: single - * linkage chains A to B to C until one cluster holds most of the population, - * however tight the threshold. Leaders cost the same to compute, hold their - * identity from frame to frame so the colours do not flicker, and cannot - * chain. - * - * The cost per scrobble is a tokenize plus four map writes. A full clustering - * pass is one walk of the population against a handful of leaders, which is - * why it can run twice a second over the whole population without being - * noticed. - * - * # What it is not - * - * SimHash measures word overlap, not meaning. Two agents describing the same - * work in different vocabularies will not be grouped, and that is the honest - * limit of a method with no model behind it. The upgrade path — the same LSH - * structure over embedding vectors, keyed by random hyperplane signs — is the - * same code with a different fingerprint function. - */ - -const Work = (() => { - "use strict"; - - /** Words that say nothing about which job an agent is on. */ - const STOPWORDS = new Set([ - "the", "a", "an", "and", "or", "of", "to", "in", "on", "at", "for", "with", - "after", "before", "left", "lands", "refused", "re", "is", "it", "this", - "that", "from", "into", "over", "under", "then", "now", "still", "again", - ]); - - /** How many bands the fingerprint is split into for the index. */ - const BANDS = 4; - /** Bits per band. Four bands of eight is one 32-bit fingerprint. */ - const BAND_BITS = 8; - /** Profiles at least this alike are the same work. */ - const MIN_SIMILARITY = 0.45; - /** Leaders compared against when the band index turns nothing up. */ - const FALLBACK_LEADERS = 160; - /** - * How much of an agent's profile survives each new line. - * - * At 0.82 a word said once is down to a tenth of its weight after twelve - * updates, so a fingerprint has a memory of roughly the last dozen lines - * without being pinned to any of them. Lower and clusters flicker; higher - * and an agent that has moved on keeps its old company for too long. - */ - const DECAY = 0.82; - /** Weights below this are dropped, so a profile stays a handful of words. */ - const PRUNE = 0.05; - /** A cluster smaller than this is not a group, it is a coincidence. */ - const MIN_CLUSTER = 3; - - /** Splits a status line into the words worth hashing. */ - function tokenize(text) { - const out = []; - for (const raw of text.toLowerCase().split(/[^a-z0-9]+/)) { - if (raw.length < 3 || STOPWORDS.has(raw)) continue; - // Numbers are counters -- "17 left" -- and say nothing about the job. - if (/^\d+$/.test(raw)) continue; - out.push(raw); - } - return out; - } - - /** FNV-1a. Cheap, well spread, and no dependency. */ - function hash32(word) { - let hash = 0x811c9dc5; - for (let i = 0; i < word.length; i += 1) { - hash ^= word.charCodeAt(i); - hash = Math.imul(hash, 0x01000193); - } - return hash >>> 0; - } - - /** - * SimHash of a token list. - * - * Each token votes on all 32 bits with its weight; the sign of each column - * becomes the fingerprint bit. Two texts built from the same rare words - * agree on most of their columns, however the words are ordered. - * - * @param {string[]} tokens - * @param {(token: string) => number} weigh how much one token's vote counts - */ - function simhash(tokens, weigh = () => 1) { - const columns = new Float64Array(32); - for (const token of tokens) { - const hash = hash32(token); - const weight = weigh(token); - for (let bit = 0; bit < 32; bit += 1) { - columns[bit] += (hash >>> bit) & 1 ? weight : -weight; - } - } - let fingerprint = 0; - for (let bit = 0; bit < 32; bit += 1) { - if (columns[bit] > 0) fingerprint |= 1 << bit; - } - return fingerprint >>> 0; - } - - /** - * Cosine similarity of two weighted profiles. - * - * Both are sorted by word id, so this is the merge step of a merge sort: - * one pass, no hashing, no allocation, and it stops as soon as either side - * runs out. A profile is a few dozen words, so a comparison is a few dozen - * integer compares and a handful of multiplications. - */ - function similarity(a, b) { - const left = a.ids; - const right = b.ids; - let i = 0; - let j = 0; - let dot = 0; - while (i < left.length && j < right.length) { - const x = left[i]; - const y = right[j]; - if (x === y) { - dot += a.values[i] * b.values[j]; - i += 1; - j += 1; - } else if (x < y) { - i += 1; - } else { - j += 1; - } - } - return dot / (a.norm * b.norm); - } - - /** Population count of the difference: how far apart two fingerprints are. */ - function hamming(a, b) { - let x = (a ^ b) >>> 0; - x = x - ((x >>> 1) & 0x55555555); - x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); - x = (x + (x >>> 4)) & 0x0f0f0f0f; - return (Math.imul(x, 0x01010101) >>> 24); - } - - /** The streaming index: what each agent is working on, and who agrees. */ - class WorkIndex { - constructor() { - /** did -> { lines, tokens, fingerprint, keys } */ - this.agents = new Map(); - /** band key -> Set of dids */ - this.buckets = new Map(); - /** token -> how many lines it has appeared in */ - this.frequency = new Map(); - /** - * Words, interned. - * - * Every profile is compared against several leaders twice a second, and - * a string-keyed map turns that into hundreds of thousands of hash - * lookups. Interning once, at the point a word is first seen, turns the - * same comparison into integer arithmetic over sorted arrays. - */ - this.tokenIds = new Map(); - this.tokenNames = []; - /** How many lines have been seen at all. */ - this.lines = 0; - /** Cluster leaders, carried between passes so colours hold still. */ - this.leaders = []; - this.nextLeaderId = 0; - } - - /** The id for a word, assigning one if this is the first sighting. */ - intern(token) { - let id = this.tokenIds.get(token); - if (id === undefined) { - id = this.tokenNames.length; - this.tokenIds.set(token, id); - this.tokenNames.push(token); - } - return id; - } - - /** - * How much one token's vote counts. - * - * Inverse document frequency: a word in every third line is worth almost - * nothing, a word in one line in fifty decides the fingerprint. - */ - weight(token) { - const seen = this.frequency.get(token) || 0; - return Math.log((this.lines + 1) / (seen + 1)) + 0.2; - } - - /** Records one status line against an agent. */ - observe(did, text) { - let entry = this.agents.get(did); - if (!entry) { - entry = { profile: new Map(), fingerprint: 0, keys: [] }; - this.agents.set(did, entry); - } - const line = tokenize(text); - - this.lines += 1; - for (const token of new Set(line)) { - this.frequency.set(token, (this.frequency.get(token) || 0) + 1); - } - - // Everything the agent has said before fades by a fixed fraction, and - // what it just said is added on top. - for (const [token, weight] of entry.profile) { - const faded = weight * DECAY; - if (faded < PRUNE) entry.profile.delete(token); - else entry.profile.set(token, faded); - } - for (const token of line) { - entry.profile.set(token, (entry.profile.get(token) || 0) + 1); - } - - // The profile is re-weighted by inverse document frequency once, here, - // so neither the fingerprint nor a similarity has to do it again, and - // laid out sorted so two of them can be merged. - const words = [...entry.profile.keys()].sort(); - entry.ids = new Int32Array(words.length); - entry.values = new Float64Array(words.length); - let square = 0; - words.forEach((token, index) => { - const value = entry.profile.get(token) * this.weight(token); - entry.ids[index] = this.intern(token); - entry.values[index] = value; - square += value * value; - }); - // Sorted by id, which is not the same order as sorted by spelling. - const order = [...entry.ids.keys()].sort((a, b) => entry.ids[a] - entry.ids[b]); - const ids = Int32Array.from(order, (index) => entry.ids[index]); - const values = Float64Array.from(order, (index) => entry.values[index]); - entry.ids = ids; - entry.values = values; - entry.norm = Math.sqrt(square) || 1; - - const fingerprint = simhash(words, (token) => entry.profile.get(token) * this.weight(token)); - if (fingerprint === entry.fingerprint && entry.keys.length > 0) return; - - this.unindex(did, entry); - entry.fingerprint = fingerprint; - entry.keys = []; - for (let band = 0; band < BANDS; band += 1) { - const slice = (fingerprint >>> (band * BAND_BITS)) & ((1 << BAND_BITS) - 1); - const key = `${band}:${slice}`; - entry.keys.push(key); - let bucket = this.buckets.get(key); - if (!bucket) { - bucket = new Set(); - this.buckets.set(key, bucket); - } - bucket.add(did); - } - } - - unindex(did, entry) { - for (const key of entry.keys) { - const bucket = this.buckets.get(key); - if (!bucket) continue; - bucket.delete(did); - if (bucket.size === 0) this.buckets.delete(key); - } - entry.keys = []; - } - - /** Drops an agent that has ended. */ - forget(did) { - const entry = this.agents.get(did); - if (!entry) return; - this.unindex(did, entry); - this.agents.delete(did); - } - - /** - * Groups the population into work clusters. - * - * Each agent joins the closest leader within the threshold, or becomes a - * leader. Leaders survive between passes, so an agent that keeps saying - * the same thing keeps its cluster — and therefore its colour — instead - * of being renumbered every time the view redraws. - */ - cluster() { - // Candidate leaders are looked up through the band index first, so this - // stays sub-linear in the number of leaders as the population grows. - const byKey = new Map(); - for (const leader of this.leaders) { - leader.members = []; - for (const key of leader.keys) { - let list = byKey.get(key); - if (!list) { - list = []; - byKey.set(key, list); - } - list.push(leader); - } - } - - // Sorted, so the pass does not depend on Map insertion order and a - // reload of the same data produces the same clusters. - const dids = [...this.agents.keys()].sort(); - for (const did of dids) { - const entry = this.agents.get(did); - let best = null; - let bestScore = MIN_SIMILARITY; - const seen = new Set(); - for (const key of entry.keys) { - for (const leader of byKey.get(key) || []) { - if (seen.has(leader)) continue; - seen.add(leader); - const score = similarity(entry, leader.entry); - if (score > bestScore) { - bestScore = score; - best = leader; - } - } - } - // The bands are a recall device and they do miss: a profile can be - // plainly about the same subject as a leader and share no eight-bit - // slice with it. When they turn nothing up, the leaders are few enough - // to look at directly rather than open a new cluster on no evidence. - if (!best && this.leaders.length <= FALLBACK_LEADERS) { - for (const leader of this.leaders) { - if (seen.has(leader)) continue; - const score = similarity(entry, leader.entry); - if (score > bestScore) { - bestScore = score; - best = leader; - } - } - } - if (!best) { - best = { - id: this.nextLeaderId, - entry, - fingerprint: entry.fingerprint, - keys: entry.keys.slice(), - members: [], - }; - this.nextLeaderId += 1; - this.leaders.push(best); - for (const key of best.keys) { - let list = byKey.get(key); - if (!list) { - list = []; - byKey.set(key, list); - } - list.push(best); - } - } - best.members.push(did); - } - - this.leaders = this.leaders.filter((leader) => leader.members.length > 0); - - const clusters = this.leaders - .filter((leader) => leader.members.length >= MIN_CLUSTER) - .map((leader) => ({ - id: leader.id, - members: leader.members, - size: leader.members.length, - label: this.label(leader.members), - })); - clusters.sort((a, b) => b.size - a.size); - - const of = new Map(); - for (const cluster of clusters) { - for (const did of cluster.members) of.set(did, cluster.id); - } - return { clusters, of }; - } - - /** - * Names a cluster after the words its members are weighted toward. - * - * Weighted by both the profile and the inverse document frequency, so the - * name is what this group says and other groups do not. - */ - label(members) { - const counts = new Map(); - for (const did of members) { - const entry = this.agents.get(did); - if (!entry) continue; - for (const [token, weight] of entry.profile) { - counts.set(token, (counts.get(token) || 0) + weight * this.weight(token)); - } - } - return [...counts.entries()] - .sort((a, b) => b[1] - a[1]) - .slice(0, 2) - .map(([token]) => token) - .join(" · "); - } - } - - return { WorkIndex, tokenize, simhash, hamming, MIN_CLUSTER }; -})(); -- 2.51.2