From cda4f25e642eed9e232af9ca58164954f9bdb2a0 Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Fri, 2 Oct 2026 03:57:47 +0900 Subject: [PATCH] bobbin: remove `bobbin-knot-ingest` crate Signed-off-by: Seongmin Lee --- Cargo.lock | 31 - Cargo.toml | 1 - bobbin/crates/bobbin-sim/src/runtime.rs | 2 - bobbin/crates/bobbin/Cargo.toml | 1 - bobbin/crates/bobbin/src/config.rs | 20 - bobbin/crates/bobbin/src/main.rs | 49 +- bobbin/crates/ingest/Cargo.toml | 1 - bobbin/crates/ingest/examples/smoke.rs | 2 - bobbin/crates/ingest/src/lib.rs | 258 ----- bobbin/crates/knot-ingest/Cargo.toml | 36 - bobbin/crates/knot-ingest/src/client.rs | 615 ---------- bobbin/crates/knot-ingest/src/firehose.rs | 659 ----------- bobbin/crates/knot-ingest/src/gate.rs | 272 ----- bobbin/crates/knot-ingest/src/legacy.rs | 128 --- bobbin/crates/knot-ingest/src/lib.rs | 15 - bobbin/crates/knot-ingest/src/orchestrator.rs | 864 -------------- bobbin/crates/knot-ingest/src/registry.rs | 129 --- bobbin/crates/knot-ingest/src/roster.rs | 602 ---------- bobbin/crates/knot-ingest/src/stream.rs | 1007 ----------------- bobbin/crates/xrpc/tests/await_record_e2e.rs | 2 - bobbin/example.toml | 17 - nix/Cargo.nix | 398 +++---- 22 files changed, 176 insertions(+), 4933 deletions(-) delete mode 100644 bobbin/crates/knot-ingest/Cargo.toml delete mode 100644 bobbin/crates/knot-ingest/src/client.rs delete mode 100644 bobbin/crates/knot-ingest/src/firehose.rs delete mode 100644 bobbin/crates/knot-ingest/src/gate.rs delete mode 100644 bobbin/crates/knot-ingest/src/legacy.rs delete mode 100644 bobbin/crates/knot-ingest/src/lib.rs delete mode 100644 bobbin/crates/knot-ingest/src/orchestrator.rs delete mode 100644 bobbin/crates/knot-ingest/src/registry.rs delete mode 100644 bobbin/crates/knot-ingest/src/roster.rs delete mode 100644 bobbin/crates/knot-ingest/src/stream.rs diff --git a/Cargo.lock b/Cargo.lock index ef3e723f3..e032e551b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -754,7 +754,6 @@ dependencies = [ "base64", "bobbin-edge-index", "bobbin-ingest", - "bobbin-knot-ingest", "bobbin-knot-proxy", "bobbin-record-lru", "bobbin-resolver", @@ -827,7 +826,6 @@ name = "bobbin-ingest" version = "0.0.1" dependencies = [ "bobbin-edge-index", - "bobbin-knot-ingest", "bobbin-record-lru", "bobbin-resolver", "bobbin-runtime", @@ -856,35 +854,6 @@ dependencies = [ "wiremock", ] -[[package]] -name = "bobbin-knot-ingest" -version = "0.0.1" -dependencies = [ - "bobbin-edge-index", - "bobbin-knot-proxy", - "bobbin-runtime", - "bobbin-types", - "bytes", - "chrono", - "cid", - "futures", - "http", - "jacquard-api", - "jacquard-common", - "knot-capability", - "parking_lot", - "reqwest 0.13.1", - "serde", - "serde_ipld_dagcbor", - "serde_json", - "thiserror 2.0.18", - "tokio", - "tokio-util", - "tracing", - "url", - "wiremock", -] - [[package]] name = "bobbin-knot-proxy" version = "0.0.1" diff --git a/Cargo.toml b/Cargo.toml index 016c9badc..3be4fec86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,6 @@ bobbin-resolver = { path = "bobbin/crates/resolver" } bobbin-slingshot-client = { path = "bobbin/crates/slingshot-client" } bobbin-record-lru = { path = "bobbin/crates/record-lru" } bobbin-knot-proxy = { path = "bobbin/crates/knot-proxy" } -bobbin-knot-ingest = { path = "bobbin/crates/knot-ingest" } bobbin-runtime = { path = "bobbin/crates/runtime" } bobbin-search = { path = "bobbin/crates/search" } bobbin-sim = { path = "bobbin/crates/bobbin-sim" } diff --git a/bobbin/crates/bobbin-sim/src/runtime.rs b/bobbin/crates/bobbin-sim/src/runtime.rs index ec384c10c..358e2a411 100644 --- a/bobbin/crates/bobbin-sim/src/runtime.rs +++ b/bobbin/crates/bobbin-sim/src/runtime.rs @@ -126,8 +126,6 @@ impl Sim { disconnects: Some(disconnects.clone()), warming_shadow: Some(warming_shadow.clone()), warming_buffer: warming_buffer_enabled.then(|| warming_buffer.clone()), - knot_registry: None, - knot_gate: None, settlements: None, analytics: None, }; diff --git a/bobbin/crates/bobbin/Cargo.toml b/bobbin/crates/bobbin/Cargo.toml index 4a23d0e66..5c69204b9 100644 --- a/bobbin/crates/bobbin/Cargo.toml +++ b/bobbin/crates/bobbin/Cargo.toml @@ -12,7 +12,6 @@ path = "src/main.rs" [dependencies] bobbin-edge-index = { workspace = true } bobbin-ingest = { workspace = true } -bobbin-knot-ingest = { workspace = true } bobbin-knot-proxy = { workspace = true } bobbin-record-lru = { workspace = true } bobbin-resolver = { workspace = true } diff --git a/bobbin/crates/bobbin/src/config.rs b/bobbin/crates/bobbin/src/config.rs index a7dd47b0b..28429efa4 100644 --- a/bobbin/crates/bobbin/src/config.rs +++ b/bobbin/crates/bobbin/src/config.rs @@ -35,8 +35,6 @@ const KNOWN_KEYS: &[&str] = &[ "record_cache.lru_bytes", "search.heap_bytes", "codesearch.zoekt_url", - "knot.allow_private", - "knot.require_https", "mirror.url", "pocket.url", "log.format", @@ -70,8 +68,6 @@ const KNOWN_ENVS: &[&str] = &[ "BOBBIN_RECORD_LRU_BYTES", "BOBBIN_SEARCH_HEAP_BYTES", "BOBBIN_CODESEARCH_ZOEKT_URL", - "BOBBIN_KNOT_ALLOW_PRIVATE", - "BOBBIN_KNOT_REQUIRE_HTTPS", "BOBBIN_MIRROR_URL", "BOBBIN_POCKET_URL", "BOBBIN_LOG_FORMAT", @@ -113,9 +109,6 @@ pub struct BobbinConfig { #[config(nested)] pub codesearch: CodeSearchConfig, - #[config(nested)] - pub knot: KnotConfig, - #[config(nested)] pub mirror: MirrorConfig, @@ -311,19 +304,6 @@ pub struct CodeSearchConfig { pub zoekt_url: Option, } -#[derive(Debug, Config)] -pub struct KnotConfig { - /// Whether to allow the knot proxy to dial private/loopback addresses. Off in - /// production - on for local testing against a knotserver on localhost. - #[config(env = "BOBBIN_KNOT_ALLOW_PRIVATE", default = false)] - pub allow_private: bool, - - /// Require https on knot hosts. Disable only when proxying to a local - /// knot for development. - #[config(env = "BOBBIN_KNOT_REQUIRE_HTTPS", default = true)] - pub require_https: bool, -} - #[derive(Debug, Config)] pub struct MirrorConfig { /// Origin of the mirror (gitmirror) XRPC server. diff --git a/bobbin/crates/bobbin/src/main.rs b/bobbin/crates/bobbin/src/main.rs index 71e53a16d..56fcdd995 100644 --- a/bobbin/crates/bobbin/src/main.rs +++ b/bobbin/crates/bobbin/src/main.rs @@ -12,13 +12,11 @@ use bobbin_edge_index::{ use bobbin_ingest::{ IngestConfig, IngestRuntime, RepoIdResolver, WarmingBuffer, run as run_ingest, }; -use bobbin_knot_ingest::{CapabilityGate, KnotClient, KnotRegistry, Orchestrator}; -use bobbin_knot_proxy::{MirrorProxy, classify_ip}; +use bobbin_knot_proxy::MirrorProxy; use bobbin_record_lru::{CacheCapacity, LruRecordStore, RecordStore}; use bobbin_resolver::{HydrantClient, IdentityResolver}; use bobbin_runtime::{ - Clock, GuardedWs, MemoryBudget, NetworkError, OsEntropy, ReqwestHttp, RuntimeHasher, - SystemClock, TungsteniteWs, WsTls, WsTransport, + Clock, MemoryBudget, OsEntropy, ReqwestHttp, RuntimeHasher, SystemClock, TungsteniteWs, WsTls, }; use bobbin_search::{ActorIndex, SearchIndex, SearchReader}; use bobbin_slingshot_client::SlingshotClient; @@ -259,7 +257,6 @@ async fn run(cfg: BobbinConfig) -> anyhow::Result<()> { limiter: limiter.clone(), budget, }); - let knot_registry = Arc::new(KnotRegistry::new()); let mirror = cfg .mirror .url @@ -361,31 +358,6 @@ async fn run(cfg: BobbinConfig) -> anyhow::Result<()> { }); let ingest_coverage = coverage.clone(); - let knot_acl_dev = !cfg.knot.require_https; - let knot_allow_private = cfg.knot.allow_private; - let knot_client = KnotClient::with_default_http(knot_allow_private)?; - let knot_gate = Arc::new(CapabilityGate::new( - knot_client.clone(), - clock.clone(), - knot_acl_dev, - knot_allow_private, - )); - let knot_ws: Arc = if knot_allow_private { - ws.clone() - } else { - GuardedWs::shared( - Arc::new(|addrs: &[SocketAddr]| { - match addrs.iter().find_map(|sa| classify_ip(&sa.ip())) { - Some(reason) => Err(NetworkError::Connect(format!( - "knot firehose resolves to {reason} address space" - ))), - None => Ok(()), - } - }), - ws_tls, - ) - }; - let ingest_runtime = IngestRuntime { store: edges.clone(), issue_states: issue_states.clone(), @@ -404,29 +376,12 @@ async fn run(cfg: BobbinConfig) -> anyhow::Result<()> { disconnects: None, warming_shadow: None, warming_buffer: Some(warming_buffer), - knot_registry: Some(knot_registry.clone()), - knot_gate: Some(knot_gate.clone()), settlements: Some(settlements.clone()), analytics, }; let mut ingest_handle = tokio::spawn(run_ingest(ingest_cfg, ingest_runtime)); let _identity_warmer = tokio::spawn(identity.clone().run_warming(cancel.clone())); - let knot_orchestrator = Orchestrator { - client: Arc::new(knot_client), - gate: knot_gate, - registry: knot_registry, - store: edges.clone(), - invites: invites.clone(), - coverage: coverage.clone(), - ws: knot_ws, - clock: clock.clone(), - dev: knot_acl_dev, - allow_private: knot_allow_private, - cancel: cancel.clone(), - }; - let _knot_acl_handle = tokio::spawn(knot_orchestrator.run()); - let _adaptive_watcher = budget.zip(limiter.as_ref()).map(|(b, l)| { mem::spawn_adaptive_watcher( l.clone(), diff --git a/bobbin/crates/ingest/Cargo.toml b/bobbin/crates/ingest/Cargo.toml index 7a89925ba..94365ebae 100644 --- a/bobbin/crates/ingest/Cargo.toml +++ b/bobbin/crates/ingest/Cargo.toml @@ -8,7 +8,6 @@ rust-version.workspace = true [dependencies] bobbin-types = { workspace = true } bobbin-edge-index = { workspace = true } -bobbin-knot-ingest = { workspace = true } bobbin-record-lru = { workspace = true } bobbin-resolver = { workspace = true } bobbin-runtime = { workspace = true } diff --git a/bobbin/crates/ingest/examples/smoke.rs b/bobbin/crates/ingest/examples/smoke.rs index 94ec1db61..7588c82d6 100644 --- a/bobbin/crates/ingest/examples/smoke.rs +++ b/bobbin/crates/ingest/examples/smoke.rs @@ -52,8 +52,6 @@ async fn main() { disconnects: None, warming_shadow: None, warming_buffer: None, - knot_registry: None, - knot_gate: None, settlements: None, analytics: None, }; diff --git a/bobbin/crates/ingest/src/lib.rs b/bobbin/crates/ingest/src/lib.rs index 1ce3d7431..c5ab5a5a6 100644 --- a/bobbin/crates/ingest/src/lib.rs +++ b/bobbin/crates/ingest/src/lib.rs @@ -9,7 +9,6 @@ use bobbin_edge_index::{ PromotionSignal, PullStatusKind, RecordOutcome, Rejection, Settlement, Settlements, StateIndex, StreamHealth, delete_record_indexes, upsert_record_indexes, }; -use bobbin_knot_ingest::{CapabilityGate, KnotRegistry}; use bobbin_record_lru::RecordStore; use bobbin_resolver::{ HydrantClient, IdentityResolver, NormalizeRepoRefs, RepoClaim, decode_canon_or_upgrade_bytes, @@ -21,7 +20,6 @@ use bobbin_runtime::{ }; use bobbin_types::edges::{Edge, ExtractError, Record}; use bobbin_types::ids::{RepoIdent, SubjectRef}; -use bobbin_types::knot_acl::KnotHostKey; use bobbin_types::record::RecordBody; use bobbin_types::search::{SearchSink, SearchableRecord}; use bobbin_types::sh_tangled::repo::Repo as RepoRecord; @@ -260,8 +258,6 @@ pub struct IngestRuntime { pub disconnects: Option>, pub warming_shadow: Option>, pub warming_buffer: Option>, - pub knot_registry: Option>, - pub knot_gate: Option>, pub settlements: Option>, pub analytics: Option>, } @@ -286,8 +282,6 @@ impl Clone for IngestRuntime { disconnects: self.disconnects.clone(), warming_shadow: self.warming_shadow.clone(), warming_buffer: self.warming_buffer.clone(), - knot_registry: self.knot_registry.clone(), - knot_gate: self.knot_gate.clone(), settlements: self.settlements.clone(), analytics: self.analytics.clone(), } @@ -307,8 +301,6 @@ impl IngestRuntime { search: &self.search, shadow: self.warming_shadow.as_deref(), buffer: self.warming_buffer.as_deref(), - knot_registry: self.knot_registry.as_deref(), - knot_gate: self.knot_gate.as_deref(), settlements: self.settlements.as_deref(), analytics: self.analytics.as_deref(), } @@ -326,8 +318,6 @@ struct PipelineCtx<'a, S: SearchSink + 'static> { search: &'a S, shadow: Option<&'a WarmingShadowBuffer>, buffer: Option<&'a WarmingBuffer>, - knot_registry: Option<&'a KnotRegistry>, - knot_gate: Option<&'a CapabilityGate>, settlements: Option<&'a Settlements>, analytics: Option<&'a Analytics>, } @@ -986,7 +976,6 @@ enum PendingOp { }, } -const KNOT_AUTHORITATIVE: &str = "the knot is authoritative for this record"; const STALE_RENAME_ALIAS: &str = "a newer sh.tangled.repo record owns this repoDid"; struct Prepared { @@ -1126,13 +1115,6 @@ async fn claim_repo( evict_from_buffer(ctx.buffer, &prior_uri).await; } finalize_drained(ctx, take_observed(ctx.buffer, &ident).await).await; - if let Some(registry) = ctx.knot_registry { - let host = KnotHostKey::new(repo.knot.as_ref()); - match repo.repo_did.clone() { - Some(repo_did) => registry.observe_repo(&host, repo_did), - None => registry.observe_host(&host), - } - } claim } @@ -1319,17 +1301,6 @@ async fn prepare_record( return rejected(source, cid, Rejection::Undecodable, Some(&e.to_string())); } }; - match acl_disposition(&parsed, ctx.knot_gate, ctx.knot_registry) { - AclDisposition::NativeSkip => { - return PendingOp::NotIndexed { - source, - nsid, - cid, - detail: KNOT_AUTHORITATIVE, - }; - } - AclDisposition::Other => {} - } let edges = match parsed.extract_edges(&source) { Ok(es) => es, Err(e) => { @@ -1375,34 +1346,6 @@ fn rejected( } } -enum AclDisposition { - Other, - NativeSkip, -} - -fn acl_disposition( - parsed: &Record, - gate: Option<&CapabilityGate>, - registry: Option<&KnotRegistry>, -) -> AclDisposition { - let Some(gate) = gate else { - return AclDisposition::Other; - }; - match parsed { - Record::Collaborator(collaborator) => { - let native = registry - .and_then(|registry| registry.host_of_repo(&collaborator.repo)) - .is_some_and(|host| gate.is_native(&host)); - if native { - AclDisposition::NativeSkip - } else { - AclDisposition::Other - } - } - _ => AclDisposition::Other, - } -} - fn fallback_rfc3339(rkey: &Rkey, rev: &Tid) -> Option { let tid = Tid::new(rkey.as_ref()).ok().unwrap_or_else(|| rev.clone()); Some(tid_time(&tid)?.to_rfc3339_opts(chrono::SecondsFormat::Micros, true)) @@ -1783,8 +1726,6 @@ async fn handle_frame( search, shadow: None, buffer: None, - knot_registry: None, - knot_gate: None, settlements: None, analytics: None, }; @@ -2092,105 +2033,6 @@ mod tests { assert_eq!(cov.snapshot().events_processed(), 1); } - #[tokio::test] - async fn native_knot_collaborator_skipped_legacy_indexed() { - use bobbin_knot_ingest::{CapabilityGate, KnotClient, KnotRegistry}; - use wiremock::matchers::{method, path}; - use wiremock::{Mock, MockServer, ResponseTemplate}; - - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.knot.version")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "version": "1.1.0", - "capabilities": ["knot-acl"] - }))) - .mount(&server) - .await; - let url = url::Url::parse(&server.uri()).unwrap(); - let native_host = format!("{}:{}", url.host_str().unwrap(), url.port().unwrap()); - - let gate = CapabilityGate::new( - KnotClient::with_default_http(true).unwrap(), - Arc::new(SystemClock::new()), - true, - true, - ); - assert!(gate.admit(&KnotHostKey::new(&native_host)).await.is_some()); - - let registry = KnotRegistry::new(); - let (store, issue_states, pull_statuses, cov, resolver) = fresh(); - let identity = detached_identity(); - let ctx = PipelineCtx { - resolver: &resolver, - identity: &identity, - store: &store, - issue_states: &issue_states, - pull_statuses: &pull_statuses, - coverage: &cov, - records: &NoopRecordStore, - search: &NoopSearchSink, - shadow: None, - buffer: None, - knot_registry: Some(®istry), - knot_gate: Some(&gate), - settlements: None, - analytics: None, - }; - - let collaborator_frame = |id: u64, rkey: &str, repo: &str| { - parse_frame(json!({ - "id": id, - "type": "record", - "record": { - "live": false, - "did": "did:plc:akshay", - "rev": fresh_tid().as_str(), - "collection": "sh.tangled.repo.collaborator", - "rkey": rkey, - "action": "create", - "record": { - "$type": "sh.tangled.repo.collaborator", - "subject": "did:plc:boltless", - "repo": repo, - "createdAt": "2026-06-01T00:00:00Z" - } - } - })) - }; - - registry.observe_repo( - &KnotHostKey::new(&native_host), - Did::new_owned("did:plc:scallop").unwrap(), - ); - registry.observe_repo( - &KnotHostKey::new("legacy.knot"), - Did::new_owned("did:plc:whelk").unwrap(), - ); - - let native = prepare_frame( - collaborator_frame(1, "aaaaaaaaaaaaz", "did:plc:scallop"), - &ctx, - now(), - ) - .await; - assert!( - matches!(native.op, PendingOp::NotIndexed { .. }), - "collaborator record for a repo on a native knot must be dropped" - ); - - let legacy = prepare_frame( - collaborator_frame(2, "bbbbbbbbbbbbz", "did:plc:whelk"), - &ctx, - now(), - ) - .await; - assert!( - matches!(legacy.op, PendingOp::Upsert(_)), - "collaborator record for a repo on a legacy knot must be ingested" - ); - } - #[tokio::test] async fn create_then_delete_round_trips_a_star() { let (store, issue_states, pull_statuses, cov, resolver) = fresh(); @@ -2356,8 +2198,6 @@ mod tests { search: &search, shadow: None, buffer: None, - knot_registry: None, - knot_gate: None, settlements: None, analytics: None, }; @@ -3041,8 +2881,6 @@ mod tests { search: &search, shadow: None, buffer: None, - knot_registry: None, - knot_gate: None, settlements: None, analytics: None, }; @@ -3107,8 +2945,6 @@ mod tests { search: &search, shadow: None, buffer: None, - knot_registry: None, - knot_gate: None, settlements: None, analytics: None, }; @@ -3185,8 +3021,6 @@ mod tests { search: &search, shadow: None, buffer: None, - knot_registry: None, - knot_gate: None, settlements: None, analytics: None, }; @@ -3259,8 +3093,6 @@ mod tests { search: &search, shadow: None, buffer: None, - knot_registry: None, - knot_gate: None, settlements: None, analytics: None, }; @@ -3356,8 +3188,6 @@ mod tests { disconnects: None, warming_shadow: None, warming_buffer: None, - knot_registry: None, - knot_gate: None, settlements: None, analytics: None, } @@ -3678,88 +3508,6 @@ mod tests { }); } - #[tokio::test] - async fn star_after_observed_repo_keys_on_repo_did() { - let (store, issue_states, pull_statuses, cov, resolver) = fresh(); - let repo: HydrantFrame = parse_frame(json!({ - "id": 1, - "type": "record", - "record": { - "live": false, - "did": "did:plc:nel", - "rev": fresh_tid().as_str(), - "collection": "sh.tangled.repo", - "rkey": "abcabcabcabcz", - "action": "create", - "record": { - "$type": "sh.tangled.repo", - "createdAt": "2026-05-01T00:00:00Z", - "knot": "oyster.cafe", - "name": "abalone", - "repoDid": "did:plc:abalone" - } - } - })); - handle_frame( - repo, - &store, - &issue_states, - &pull_statuses, - &cov, - &NoopSearchSink, - &NoopRecordStore, - &resolver, - &sys_clock(), - now(), - ) - .await; - - let star: HydrantFrame = parse_frame(json!({ - "id": 2, - "type": "record", - "record": { - "live": false, - "did": "did:plc:olaren", - "rev": fresh_tid().as_str(), - "collection": "sh.tangled.feed.star", - "rkey": "abcabcabcabcz", - "action": "create", - "record": { - "$type": "sh.tangled.feed.star", - "createdAt": "2026-05-01T00:00:00Z", - "subject": "at://did:plc:nel/sh.tangled.repo/abcabcabcabcz" - } - } - })); - handle_frame( - star, - &store, - &issue_states, - &pull_statuses, - &cov, - &NoopSearchSink, - &NoopRecordStore, - &resolver, - &sys_clock(), - now(), - ) - .await; - - let nsid = Nsid::new_static("sh.tangled.feed.star").unwrap(); - let repo_keyed = bobbin_types::ids::EdgeKey::new(nsid.clone(), did_subj("did:plc:abalone")); - let owner_keyed = bobbin_types::ids::EdgeKey::new(nsid, did_subj("did:plc:nel")); - assert_eq!( - store.count(&repo_keyed), - 1, - "star should be keyed on repoDID once the repo is observed" - ); - assert_eq!( - store.count(&owner_keyed), - 0, - "owner DID should not collect the edge" - ); - } - #[tokio::test] async fn out_of_order_repo_preps_cannot_steal_a_rename() { let (store, issue_states, pull_statuses, cov, resolver) = fresh(); @@ -3777,8 +3525,6 @@ mod tests { search: &search, shadow: None, buffer: None, - knot_registry: None, - knot_gate: None, settlements: None, analytics: None, }; @@ -3849,8 +3595,6 @@ mod tests { search: &search, shadow: None, buffer: None, - knot_registry: None, - knot_gate: None, settlements: None, analytics: None, }; @@ -4672,8 +4416,6 @@ mod tests { disconnects: None, warming_shadow: None, warming_buffer: None, - knot_registry: None, - knot_gate: None, settlements: None, analytics: None, }; diff --git a/bobbin/crates/knot-ingest/Cargo.toml b/bobbin/crates/knot-ingest/Cargo.toml deleted file mode 100644 index 0dfe4c4f7..000000000 --- a/bobbin/crates/knot-ingest/Cargo.toml +++ /dev/null @@ -1,36 +0,0 @@ -[package] -name = "bobbin-knot-ingest" -version.workspace = true -edition.workspace = true -license.workspace = true -rust-version.workspace = true - -[dependencies] -bobbin-edge-index = { workspace = true } -bobbin-knot-proxy = { workspace = true } -bobbin-runtime = { workspace = true } -bobbin-types = { workspace = true } -knot-capability = { workspace = true } -jacquard-api = { workspace = true, features = ["com_atproto", "streaming"] } -jacquard-common = { workspace = true } -chrono = { workspace = true } - -bytes = { workspace = true } -cid = { workspace = true } -futures = { workspace = true } -http = { workspace = true } -reqwest = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -serde_ipld_dagcbor = { workspace = true } -thiserror = { workspace = true } -tokio = { workspace = true } -tokio-util = { workspace = true } -tracing = { workspace = true } -parking_lot = { workspace = true } -url = { workspace = true } - - -[dev-dependencies] -tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] } -wiremock = { workspace = true } diff --git a/bobbin/crates/knot-ingest/src/client.rs b/bobbin/crates/knot-ingest/src/client.rs deleted file mode 100644 index 91295bfde..000000000 --- a/bobbin/crates/knot-ingest/src/client.rs +++ /dev/null @@ -1,615 +0,0 @@ -use std::collections::HashSet; -use std::future::Future; -use std::pin::Pin; -use std::sync::Arc; -use std::time::Duration; - -use bobbin_knot_proxy::{KnotHost, KnotHostError, PrivateAddressFilter, PrivateHostReason}; -use bobbin_runtime::{ - HttpRequest, HttpResponseHead, HttpTransport, NetworkError, ReqwestHttp, UnixMicros, -}; -use bytes::{Bytes, BytesMut}; -use futures::TryStreamExt; -use http::{HeaderMap, StatusCode}; -use jacquard_common::DefaultStr; -use jacquard_common::types::datetime::Datetime; -use jacquard_common::types::did::Did; -use jacquard_common::types::nsid::Nsid; -use knot_capability::Capability; -use serde::{Deserialize, Deserializer, de::DeserializeOwned}; -use thiserror::Error; -use url::Url; - -const USER_AGENT: &str = concat!("bobbin/", env!("CARGO_PKG_VERSION")); -const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); -const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); -const MAX_BODY_BYTES: u64 = 4 * 1024 * 1024; -const LIST_PAGE_LIMIT: i64 = 1000; -const MAX_LIST_PAGES: usize = 256; - -const VERSION_NSID: &str = "sh.tangled.knot.version"; -const LIST_COLLABORATORS_NSID: &str = "sh.tangled.repo.listCollaborators"; -const LIST_COLLABORATOR_INVITES_NSID: &str = "sh.tangled.repo.listCollaboratorInvites"; - -#[derive(Clone)] -pub struct KnotClient { - http: Arc, -} - -pub(crate) fn knot_stamp<'de, D: Deserializer<'de>>(de: D) -> Result { - let at = Datetime::deserialize(de)?; - u64::try_from(at.timestamp_micros()) - .map(UnixMicros::new) - .map_err(|_| serde::de::Error::custom("stamp before 1970 won't fit unix micros")) -} - -#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AclEntry { - pub subject: Did, - #[serde(deserialize_with = "knot_stamp")] - pub created_at: UnixMicros, -} - -#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct InviteEntry { - pub subject: Did, - pub added_by: Did, - #[serde(deserialize_with = "knot_stamp")] - pub created_at: UnixMicros, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum Completeness { - Complete, - Truncated, -} - -impl Completeness { - pub fn and(self, other: Self) -> Self { - match (self, other) { - (Self::Complete, Self::Complete) => Self::Complete, - _ => Self::Truncated, - } - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct Listing { - pub entries: Vec, - pub completeness: Completeness, -} - -pub type AclListing = Listing; -pub type InviteListing = Listing; - -#[derive(Debug, Error)] -pub enum KnotClientError { - #[error("knot host: {0}")] - Host(#[from] KnotHostError), - #[error("blocked: knot {host} resolves to {reason} address space")] - PrivateHost { - host: String, - reason: PrivateHostReason, - }, - #[error("http client build: {0}")] - Build(String), - #[error("network: {0}")] - Network(#[from] NetworkError), - #[error("xrpc not found")] - NotFound, - #[error("upstream returned status {0}")] - Upstream(StatusCode), - #[error("response body exceeded {limit} bytes")] - BodyTooLarge { limit: u64 }, - #[error("decode response: {0}")] - Decode(#[from] serde_json::Error), -} - -pub fn knot_endpoint( - host: &str, - dev: bool, - allow_private: bool, -) -> Result { - let raw = if dev { - format!("http://{host}") - } else { - host.to_owned() - }; - let knot = KnotHost::parse(&raw)?; - if !allow_private && let Some(reason) = knot.private_literal_reason() { - return Err(KnotClientError::PrivateHost { - host: host.to_owned(), - reason, - }); - } - Ok(knot) -} - -fn default_http_client(allow_private: bool) -> Result { - reqwest::Client::builder() - .user_agent(USER_AGENT) - .timeout(REQUEST_TIMEOUT) - .connect_timeout(CONNECT_TIMEOUT) - .redirect(reqwest::redirect::Policy::none()) - .dns_resolver(Arc::new(PrivateAddressFilter::new(allow_private))) - .build() -} - -fn nsid(s: &'static str) -> Nsid { - Nsid::new_static(s).expect("static nsid literal must validate") -} - -pub(crate) fn authority(host: &KnotHost) -> String { - let url = host.url(); - match (url.host_str(), url.port()) { - (Some(h), Some(p)) => format!("{h}:{p}"), - (Some(h), None) => h.to_owned(), - (None, _) => String::new(), - } -} - -impl KnotClient { - pub fn new(http: Arc) -> Self { - Self { http } - } - - pub fn with_default_http(allow_private: bool) -> Result { - let client = default_http_client(allow_private) - .map_err(|e| KnotClientError::Build(e.to_string()))?; - Ok(Self::new(ReqwestHttp::shared(client))) - } - - pub async fn capabilities( - &self, - host: &KnotHost, - ) -> Result, KnotClientError> { - let mut url = host.xrpc_url(&nsid(VERSION_NSID)); - url.set_query(None); - let bytes = self.get_json(url).await?; - let resp: VersionWire = serde_json::from_slice(&bytes)?; - Ok(resp - .capabilities - .unwrap_or_default() - .into_iter() - .filter_map(|token| Capability::parse(&token)) - .collect()) - } - - pub async fn list_collaborators( - &self, - host: &KnotHost, - repo: &Did, - ) -> Result { - self.list(host, LIST_COLLABORATORS_NSID, repo).await - } - - pub async fn list_collaborator_invites( - &self, - host: &KnotHost, - repo: &Did, - ) -> Result { - self.list(host, LIST_COLLABORATOR_INVITES_NSID, repo).await - } - - async fn list( - &self, - host: &KnotHost, - endpoint: &'static str, - subject: &Did, - ) -> Result, KnotClientError> { - self.drain( - host, - endpoint, - subject.as_ref().to_owned(), - None, - 0, - Vec::new(), - ) - .await - } - - fn drain<'a, T: DeserializeOwned + Send + 'a>( - &'a self, - host: &'a KnotHost, - endpoint: &'static str, - subject: String, - cursor: Option, - page: usize, - mut acc: Vec, - ) -> Pin, KnotClientError>> + Send + 'a>> { - Box::pin(async move { - if page >= MAX_LIST_PAGES { - tracing::warn!( - host = %authority(host), - endpoint, - pages = page, - "knot list truncated at page cap" - ); - return Ok(Listing { - entries: acc, - completeness: Completeness::Truncated, - }); - } - let resp = self - .fetch_page(host, endpoint, &subject, cursor.as_deref()) - .await?; - acc.extend(resp.items); - match resp.cursor.filter(|c| !c.is_empty()) { - Some(next) => { - self.drain(host, endpoint, subject, Some(next), page + 1, acc) - .await - } - None => Ok(Listing { - entries: acc, - completeness: Completeness::Complete, - }), - } - }) - } - - async fn fetch_page( - &self, - host: &KnotHost, - endpoint: &'static str, - subject: &str, - cursor: Option<&str>, - ) -> Result, KnotClientError> { - let mut url = host.xrpc_url(&nsid(endpoint)); - { - let mut q = url.query_pairs_mut(); - q.clear(); - q.append_pair("subject", subject); - q.append_pair("limit", &LIST_PAGE_LIMIT.to_string()); - if let Some(c) = cursor { - q.append_pair("cursor", c); - } - } - let bytes = self.get_json(url).await?; - Ok(serde_json::from_slice(&bytes)?) - } - - async fn get_json(&self, url: Url) -> Result { - let resp = self - .http - .execute(HttpRequest { - url, - headers: HeaderMap::new(), - }) - .await?; - match resp.status { - StatusCode::OK => read_bounded(resp).await, - StatusCode::NOT_FOUND => Err(KnotClientError::NotFound), - other => Err(KnotClientError::Upstream(other)), - } - } -} - -#[derive(Deserialize)] -struct VersionWire { - #[serde(default)] - capabilities: Option>, -} - -#[derive(Deserialize)] -#[serde(bound = "T: DeserializeOwned")] -struct ListWire { - #[serde(default = "Vec::new")] - items: Vec, - #[serde(default)] - cursor: Option, -} - -async fn read_bounded(resp: HttpResponseHead) -> Result { - if resp.content_length.is_some_and(|len| len > MAX_BODY_BYTES) { - return Err(KnotClientError::BodyTooLarge { - limit: MAX_BODY_BYTES, - }); - } - let buf = resp - .body - .map_err(KnotClientError::Network) - .try_fold(BytesMut::new(), |mut acc, chunk| async move { - if (acc.len() as u64).saturating_add(chunk.len() as u64) > MAX_BODY_BYTES { - return Err(KnotClientError::BodyTooLarge { - limit: MAX_BODY_BYTES, - }); - } - acc.extend_from_slice(&chunk); - Ok(acc) - }) - .await?; - Ok(buf.freeze()) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - use wiremock::matchers::{method, path, query_param}; - use wiremock::{Mock, MockServer, ResponseTemplate}; - - fn did(s: &str) -> Did { - Did::new_owned(s).unwrap() - } - - fn stamp(iso: &str) -> UnixMicros { - knot_stamp(serde_json::Value::String(iso.to_owned())).unwrap() - } - - fn client() -> KnotClient { - KnotClient::new(ReqwestHttp::shared(default_http_client(true).unwrap())) - } - - fn endpoint(server: &MockServer) -> KnotHost { - KnotHost::parse(&server.uri()).unwrap() - } - - #[tokio::test] - async fn capabilities_returns_declared_tokens() { - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.knot.version")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "version": "1.0.0 (deadbeef)", - "capabilities": ["knot-acl", "shuck-oysters", "repo-did-input"] - }))) - .mount(&server) - .await; - - let caps = client().capabilities(&endpoint(&server)).await.unwrap(); - assert_eq!( - caps, - HashSet::from([Capability::KnotAcl, Capability::RepoDidInput]), - "a token we never heard of is dropped instead of poisoning the set" - ); - } - - #[tokio::test] - async fn capabilities_empty_when_field_absent() { - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.knot.version")) - .respond_with( - ResponseTemplate::new(200).set_body_json(json!({ "version": "1.0.0 (cafe)" })), - ) - .mount(&server) - .await; - - let caps = client().capabilities(&endpoint(&server)).await.unwrap(); - assert_eq!( - caps, - HashSet::new(), - "a knot that declares no capabilities serves none" - ); - } - - #[tokio::test] - async fn list_collaborators_drains_single_page() { - let server = MockServer::start().await; - let host = endpoint(&server); - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.repo.listCollaborators")) - .and(query_param("subject", "did:plc:scallop")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "items": [ - {"subject": "did:plc:boltless", "addedBy": "did:plc:akshay", "createdAt": "2026-06-01T00:00:00Z"}, - {"subject": "did:plc:akshay", "addedBy": "did:plc:akshay", "createdAt": "2026-06-02T12:00:00Z"} - ] - }))) - .mount(&server) - .await; - - let repo = did("did:plc:scallop"); - let listing = client().list_collaborators(&host, &repo).await.unwrap(); - assert_eq!(listing.completeness, Completeness::Complete); - assert_eq!( - listing.entries, - vec![ - AclEntry { - subject: did("did:plc:boltless"), - created_at: stamp("2026-06-01T00:00:00Z"), - }, - AclEntry { - subject: did("did:plc:akshay"), - created_at: stamp("2026-06-02T12:00:00Z"), - }, - ] - ); - } - - #[tokio::test] - async fn list_collaborators_drains_multiple_pages() { - let server = MockServer::start().await; - let host = endpoint(&server); - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.repo.listCollaborators")) - .and(query_param("cursor", "p2")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "items": [{"subject": "did:plc:akshay", "addedBy": "did:plc:akshay", "createdAt": "2026-06-02T00:00:00Z"}] - }))) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.repo.listCollaborators")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "items": [{"subject": "did:plc:boltless", "addedBy": "did:plc:akshay", "createdAt": "2026-06-01T00:00:00Z"}], - "cursor": "p2" - }))) - .mount(&server) - .await; - - let repo = did("did:plc:scallop"); - let listing = client().list_collaborators(&host, &repo).await.unwrap(); - assert_eq!(listing.completeness, Completeness::Complete); - let subjects: Vec<_> = listing.entries.into_iter().map(|m| m.subject).collect(); - assert_eq!( - subjects, - vec![did("did:plc:boltless"), did("did:plc:akshay")] - ); - } - - #[tokio::test] - async fn list_collaborators_uses_repo_did_subject() { - let server = MockServer::start().await; - let host = endpoint(&server); - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.repo.listCollaborators")) - .and(query_param("subject", "did:plc:scallop")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "items": [{"subject": "did:plc:olaren", "addedBy": "did:plc:boltless", "createdAt": "2026-06-03T00:00:00Z"}] - }))) - .mount(&server) - .await; - - let listing = client() - .list_collaborators(&host, &did("did:plc:scallop")) - .await - .unwrap(); - assert_eq!(listing.completeness, Completeness::Complete); - assert_eq!(listing.entries.len(), 1); - assert_eq!(listing.entries[0].subject, did("did:plc:olaren")); - } - - #[tokio::test] - async fn an_invite_listing_decodes_at_the_repo_subject() { - let server = MockServer::start().await; - let host = endpoint(&server); - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.repo.listCollaboratorInvites")) - .and(query_param("subject", "did:plc:scallop")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "items": [{"subject": "did:plc:olaren", "addedBy": "did:plc:boltless", "createdAt": "2026-06-05T00:00:00Z"}] - }))) - .mount(&server) - .await; - - let collaborators = client() - .list_collaborator_invites(&host, &did("did:plc:scallop")) - .await - .unwrap(); - assert_eq!(collaborators.completeness, Completeness::Complete); - assert_eq!( - collaborators.entries, - vec![InviteEntry { - subject: did("did:plc:olaren"), - added_by: did("did:plc:boltless"), - created_at: stamp("2026-06-05T00:00:00Z"), - }] - ); - } - - #[tokio::test] - async fn invite_listing_rejects_offer_missing_its_inviter() { - let server = MockServer::start().await; - let host = endpoint(&server); - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.repo.listCollaboratorInvites")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "items": [{"subject": "did:plc:limpet", "createdAt": "2026-06-04T00:00:00Z"}] - }))) - .mount(&server) - .await; - - let rejected = client() - .list_collaborator_invites(&host, &did("did:plc:scallop")) - .await - .is_err(); - assert!( - rejected, - "reject here, since rows can't record who invited without this field" - ); - } - - #[tokio::test] - async fn drain_reports_truncation_at_page_cap() { - let server = MockServer::start().await; - let host = endpoint(&server); - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.repo.listCollaborators")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "items": [{"subject": "did:plc:boltless", "addedBy": "did:plc:akshay", "createdAt": "2026-06-01T00:00:00Z"}], - "cursor": "more" - }))) - .mount(&server) - .await; - - let repo = did("did:plc:scallop"); - let listing = client().list_collaborators(&host, &repo).await.unwrap(); - assert_eq!( - listing.completeness, - Completeness::Truncated, - "a never-terminating cursor must surface as a truncated listing" - ); - assert_eq!(listing.entries.len(), MAX_LIST_PAGES); - } - - #[tokio::test] - async fn maps_404_to_not_found() { - let server = MockServer::start().await; - Mock::given(method("GET")) - .respond_with(ResponseTemplate::new(404)) - .mount(&server) - .await; - - let err = client() - .capabilities(&endpoint(&server)) - .await - .expect_err("404 must surface"); - assert!(matches!(err, KnotClientError::NotFound)); - } - - #[tokio::test] - async fn maps_5xx_to_upstream() { - let server = MockServer::start().await; - Mock::given(method("GET")) - .respond_with(ResponseTemplate::new(503)) - .mount(&server) - .await; - - let err = client() - .capabilities(&endpoint(&server)) - .await - .expect_err("5xx must surface"); - match err { - KnotClientError::Upstream(s) => assert_eq!(s.as_u16(), 503), - other => panic!("wrong variant: {other:?}"), - } - } - - #[tokio::test] - async fn does_not_follow_redirects() { - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.knot.version")) - .respond_with(ResponseTemplate::new(301).insert_header( - "location", - "https://kt.tngl.oyster.cafe/xrpc/sh.tangled.knot.version", - )) - .mount(&server) - .await; - - let err = client() - .capabilities(&endpoint(&server)) - .await - .expect_err("a redirect must surface as an error, not be followed to another knot"); - match err { - KnotClientError::Upstream(s) => assert_eq!(s.as_u16(), 301), - other => panic!("expected Upstream(301), got {other:?}"), - } - } - - #[test] - fn knot_endpoint_rejects_private_host() { - let err = - knot_endpoint("127.0.0.1:9", true, false).expect_err("private host must be refused"); - assert!(matches!(err, KnotClientError::PrivateHost { .. })); - } - - #[test] - fn knot_endpoint_allows_private_when_permitted() { - let knot = knot_endpoint("127.0.0.1:9", true, true).expect("private host allowed"); - assert_eq!(knot.url().scheme(), "http"); - } -} diff --git a/bobbin/crates/knot-ingest/src/firehose.rs b/bobbin/crates/knot-ingest/src/firehose.rs deleted file mode 100644 index 2706c1ab3..000000000 --- a/bobbin/crates/knot-ingest/src/firehose.rs +++ /dev/null @@ -1,659 +0,0 @@ -use std::collections::HashMap; -use std::convert::Infallible; -use std::io::Cursor as IoCursor; - -use bobbin_edge_index::Offer; -use bobbin_runtime::UnixMicros; -use bytes::Bytes; -use jacquard_api::com_atproto::sync::subscribe_repos; -use jacquard_common::DefaultStr; -use jacquard_common::types::cid::Cid; -use jacquard_common::types::did::Did; -use jacquard_common::types::nsid::Nsid; -use jacquard_common::types::recordkey::Rkey; -use serde::Deserialize; -use thiserror::Error; - -const TYPE_COMMIT: &str = "#commit"; -const TYPE_INFO: &str = "#info"; - -const MAX_CBOR_DEPTH: usize = 64; - -#[derive(Debug, Error)] -pub enum FirehoseError { - #[error("frame header didn't decode: {0}")] - Header(#[from] serde_ipld_dagcbor::DecodeError), - #[error("frame body didn't decode: {0}")] - Body(#[from] serde_ipld_dagcbor::DecodeError), - #[error("frame ended mid-value")] - Truncated, - #[error("cbor nesting deeper than the decoder's bound")] - TooDeep, - #[error("malformed cbor at byte {0}")] - Malformed(usize), - #[error("frame names neither an op or a type")] - NoTypeOrError, - #[error("frame car didn't parse: {0}")] - Car(#[from] cid::Error), -} - -#[derive(Clone, Debug, PartialEq)] -pub enum Frame { - Commit(Commit), - Info { name: String }, - Error { name: String, message: String }, - Other, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct Commit { - pub repo: Did, - pub seq: i64, - pub rev: String, - pub records: Vec, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct RecordOp { - pub action: OpAction, - pub collection: Nsid, - pub rkey: Rkey, - pub record: Option, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum OpAction { - Create, - Update, - Delete, -} - -impl OpAction { - pub fn is_delete(self) -> bool { - matches!(self, Self::Delete) - } -} - -pub const COLLABORATOR_INVITE_COLLECTION: &str = "sh.tangled.repo.collaboratorInvite"; - -pub fn is_collaborator_invite(collection: &Nsid) -> bool { - collection.as_str() == COLLABORATOR_INVITE_COLLECTION -} - -#[derive(Clone, Debug, PartialEq)] -pub struct InviteOp { - pub repo: Did, - pub invitee: Did, - pub offer: Option, -} - -#[derive(Deserialize)] -struct InviteWire { - #[serde(rename = "createdAt", deserialize_with = "crate::client::knot_stamp")] - created_at: UnixMicros, - #[serde(rename = "x-tngl-editor")] - editor: Did, -} - -pub fn invite_op_of(op: &RecordOp, repo: &Did) -> Option { - let invitee = Did::::new_owned(op.rkey.as_str()).ok()?; - let offer = if op.action.is_delete() { - None - } else { - let wire: InviteWire = serde_ipld_dagcbor::from_slice(op.record.as_deref()?).ok()?; - Some(Offer { - invited_by: wire.editor, - offered_at: wire.created_at, - }) - }; - Some(InviteOp { - repo: repo.clone(), - invitee, - offer, - }) -} - -#[derive(Deserialize)] -struct Header { - op: Option, - t: Option, -} - -#[derive(Deserialize)] -struct ErrorBody { - error: String, - message: String, -} - -#[derive(Deserialize)] -struct InfoBody { - name: String, -} - -pub fn decode_frame(data: &[u8]) -> Result { - scan_depth(data)?; - let mut reader = IoCursor::new(data); - let head: Header = serde_ipld_dagcbor::de::from_reader_once(&mut reader)?; - let body = &data[reader.position() as usize..]; - match (head.op, head.t.as_deref()) { - (Some(op), _) if op < 0 => { - let body: ErrorBody = serde_ipld_dagcbor::from_slice(body)?; - Ok(Frame::Error { - name: body.error, - message: body.message, - }) - } - (_, Some(TYPE_COMMIT)) => { - let message: subscribe_repos::Commit = - serde_ipld_dagcbor::from_slice(body)?; - Ok(Frame::Commit(commit_of(message))) - } - (_, Some(TYPE_INFO)) => Ok(Frame::Info { - name: serde_ipld_dagcbor::from_slice::(body)?.name, - }), - (_, Some(_)) => Ok(Frame::Other), - _ => Err(FirehoseError::NoTypeOrError), - } -} - -pub fn is_outdated_cursor(frame: &Frame) -> bool { - matches!(frame, Frame::Info { name } if name == "OutdatedCursor") -} - -pub fn is_future_cursor(frame: &Frame) -> bool { - matches!(frame, Frame::Error { name, .. } if name == "FutureCursor") -} - -fn commit_of(message: subscribe_repos::Commit) -> Commit { - let mut commit = Commit { - repo: message.repo, - seq: message.seq, - rev: message.rev.to_string(), - records: Vec::new(), - }; - if message.ops.is_empty() { - return commit; - } - let car = match car_blocks(&message.blocks) { - Ok(car) => car, - Err(err) => { - tracing::warn!(repo = %commit.repo, error = %err, "frame car unreadable, dropping the ops"); - return commit; - } - }; - let repo = commit.repo.as_ref().to_owned(); - commit.records = message - .ops - .into_iter() - .filter_map(|op| record_of(&repo, op, &car)) - .collect(); - commit -} - -fn record_of( - repo: &str, - op: subscribe_repos::RepoOp, - car: &HashMap, -) -> Option { - let action = match op.action { - subscribe_repos::RepoOpAction::Create => OpAction::Create, - subscribe_repos::RepoOpAction::Update => OpAction::Update, - subscribe_repos::RepoOpAction::Delete => OpAction::Delete, - subscribe_repos::RepoOpAction::Other(_) => return None, - }; - let (collection, rkey) = op.path.split_once('/')?; - let collection = Nsid::::new_owned(collection).ok()?; - let rkey = Rkey::::new_owned(rkey).ok()?; - if !action.is_delete() && op.cid.is_none() { - return None; - } - let record = if action.is_delete() { - None - } else { - match op.cid.as_ref().and_then(|link| car.get(&link.0)) { - Some(bytes) => Some(Bytes::copy_from_slice(bytes)), - None => { - tracing::warn!(repo, rkey = %rkey, "op record missing from the frame car, dropping the op"); - return None; - } - } - }; - Some(RecordOp { - action, - collection, - rkey, - record, - }) -} - -fn car_blocks(car: &[u8]) -> Result, FirehoseError> { - let mut wire = Wire { data: car, pos: 0 }; - let header_len = wire.varint()?; - if header_len > 1024 * 1024 { - return Err(FirehoseError::Malformed(wire.pos)); - } - wire.take(header_len as usize)?; - let mut blocks = HashMap::new(); - while wire.pos < wire.data.len() { - let len = wire.varint()? as usize; - let mut blob = wire.take(len)?; - let cid = cid::Cid::read_bytes(&mut blob)?; - blocks.insert(Cid::from(cid), blob); - } - Ok(blocks) -} - -fn scan_depth(data: &[u8]) -> Result<(), FirehoseError> { - let mut wire = Wire { data, pos: 0 }; - while wire.pos < wire.data.len() { - wire.skip(0)?; - } - Ok(()) -} - -struct Wire<'a> { - data: &'a [u8], - pos: usize, -} - -impl<'a> Wire<'a> { - fn take(&mut self, n: usize) -> Result<&'a [u8], FirehoseError> { - let rest = self.data.get(self.pos..).ok_or(FirehoseError::Truncated)?; - let out = rest.get(..n).ok_or(FirehoseError::Truncated)?; - self.pos += n; - Ok(out) - } - - fn head(&mut self) -> Result<(u8, u64), FirehoseError> { - let byte = self.take(1)?[0]; - let major = byte >> 5; - let arg = match byte & 0x1f { - info @ 0..=23 => u64::from(info), - 24 => u64::from(self.take(1)?[0]), - 25 => u64::from(u16::from_be_bytes(self.take(2)?.try_into().unwrap())), - 26 => u64::from(u32::from_be_bytes(self.take(4)?.try_into().unwrap())), - 27 => u64::from_be_bytes(self.take(8)?.try_into().unwrap()), - _ => return Err(FirehoseError::Malformed(self.pos)), - }; - Ok((major, arg)) - } - - fn skip(&mut self, depth: usize) -> Result<(), FirehoseError> { - if depth > MAX_CBOR_DEPTH { - return Err(FirehoseError::TooDeep); - } - let (major, arg) = self.head()?; - match major { - 0 | 1 => Ok(()), - 2 | 3 => self.take(arg as usize).map(|_| ()), - 4 => (0..arg).try_for_each(|_| self.skip(depth + 1)), - 5 => (0..arg).try_for_each(|_| { - self.skip(depth + 1)?; - self.skip(depth + 1) - }), - 6 => self.skip(depth + 1), - 7 => { - if arg >= 32 { - return Err(FirehoseError::Malformed(self.pos)); - } - Ok(()) - } - _ => Ok(()), - } - } - - fn varint(&mut self) -> Result { - let mut out: u64 = 0; - let mut shift = 0u32; - loop { - let byte = self.take(1)?[0]; - out |= u64::from(byte & 0x7f) << shift; - if byte & 0x80 == 0 { - return Ok(out); - } - shift += 7; - if shift > 63 { - return Err(FirehoseError::Malformed(self.pos)); - } - } - } -} - -#[cfg(test)] -mod tests { - use super::testcbor::*; - use super::*; - - #[test] - fn commit_frame_resolves_record_bytes_from_the_car() { - let record = unhex(FROZEN_PUSHED_REF); - let bytes = commit_frame( - "did:plc:scallop", - 1_770_000_000_000_000, - vec![git_ref_op( - "create", - "refs~2fheads~2fmain", - Some(TEST_CID.to_vec()), - )], - car(&[&record]), - ); - let Frame::Commit(commit) = decode_frame(&bytes).unwrap() else { - panic!("expected a commit frame"); - }; - assert_eq!(commit.repo.as_ref(), "did:plc:scallop"); - assert_eq!(commit.seq, 1_770_000_000_000_000); - assert_eq!(commit.records.len(), 1); - let op = &commit.records[0]; - assert_eq!(op.action, OpAction::Create); - assert_eq!(op.collection.as_str(), GIT_REF_COLLECTION); - assert_eq!(op.rkey.as_str(), "refs~2fheads~2fmain"); - assert_eq!(op.record.as_deref(), Some(record.as_slice())); - } - - #[test] - fn delete_op_decodes_with_empty_record() { - let bytes = commit_frame( - "did:plc:scallop", - 7, - vec![git_ref_op("delete", "refs~2ftags~2fv1", None)], - car(&[]), - ); - let Frame::Commit(commit) = decode_frame(&bytes).unwrap() else { - panic!("expected a commit frame"); - }; - assert_eq!(commit.records.len(), 1); - assert_eq!(commit.records[0].action, OpAction::Delete); - assert_eq!(commit.records[0].record, None); - } - - #[test] - fn only_ops_whose_record_is_in_the_car_survive_decode() { - let other_cid = vec![0x01, 0x55, 0x00, 0x01, 0xff]; - let bytes = commit_frame( - "did:plc:scallop", - 9, - vec![git_ref_op("update", "refs~2fheads~2fdev", Some(other_cid))], - car(&[&unhex(FROZEN_BARE_REF)]), - ); - let Frame::Commit(commit) = decode_frame(&bytes).unwrap() else { - panic!("expected a commit frame"); - }; - assert!(commit.records.is_empty()); - } - - #[test] - fn ops_whose_path_is_not_a_collection_and_record_key_are_dropped() { - let record = unhex(FROZEN_BARE_REF); - let cases = [ - ("collection that is not an nsid", "not an nsid", "3mug"), - ( - "record key with a slash in it", - COLLABORATOR_INVITE_COLLECTION, - "did:plc:limpet/extra", - ), - ]; - - cases.into_iter().for_each(|(case, collection, rkey)| { - let bytes = commit_frame( - "did:plc:scallop", - 11, - vec![invite_op(collection, rkey, Some(TEST_CID.to_vec()))], - car(&[&record]), - ); - let Frame::Commit(commit) = decode_frame(&bytes).unwrap() else { - panic!("expected a commit frame"); - }; - assert!(commit.records.is_empty(), "an op with a {case} survived"); - }); - } - - #[test] - fn error_frame_body_is_decoded_after_the_header() { - let mut bytes = encode_one(&Val::Map(vec![(t("op"), Val::Nint(0))])); - bytes.extend(encode_one(&Val::Map(vec![ - (t("error"), t("FutureCursor")), - (t("message"), t("the cursor is ahead of this stream")), - ]))); - let frame = decode_frame(&bytes).unwrap(); - assert!(is_future_cursor(&frame)); - assert!(!is_outdated_cursor(&frame)); - } - - #[test] - fn outdated_cursor_info_frame_is_recognized() { - let mut bytes = encode_one(&Val::Map(vec![(t("t"), t("#info"))])); - bytes.extend(encode_one(&Val::Map(vec![ - (t("name"), t("OutdatedCursor")), - (t("live"), Val::Bool(false)), - ]))); - let frame = decode_frame(&bytes).unwrap(); - assert!(is_outdated_cursor(&frame)); - assert!(!is_future_cursor(&frame)); - } - - #[test] - fn non_map_frame_is_rejected() { - assert!(decode_frame(&encode_one(&Val::Array(vec![]))).is_err()); - } - - #[test] - fn truncated_frame_is_rejected() { - let full = encode_one(&Val::Map(vec![(t("t"), t("#commit"))])); - assert!(matches!( - decode_frame(&full[..full.len() - 1]), - Err(FirehoseError::Truncated) - )); - } - - #[test] - fn deeply_nested_unknown_fields_hit_the_depth_bound() { - fn nest(depth: usize) -> Val { - (0..depth).fold(Val::Text(String::new()), |val, _| Val::Array(vec![val])) - } - let bytes = encode_one(&Val::Map(vec![(t("x"), nest(MAX_CBOR_DEPTH + 8))])); - assert!(matches!(decode_frame(&bytes), Err(FirehoseError::TooDeep))); - - let mut bytes = encode_one(&Val::Map(vec![(t("t"), t("#mystery"))])); - bytes.extend(encode_one(&nest(MAX_CBOR_DEPTH - 4))); - assert_eq!(decode_frame(&bytes).unwrap(), Frame::Other); - } - - #[test] - fn malformed_invite_never_decodes_to_offer() { - let collection = COLLABORATOR_INVITE_COLLECTION; - let full = invite_record(collection, "2026-06-01T00:00:00Z", "did:plc:akshay"); - let repo = Did::::new_owned("did:plc:scallop").unwrap(); - let cases = [ - ("record key that isn't a did", "3lkm2xqbolt2s", full.clone()), - ( - "body cut short of its last field", - "did:plc:limpet", - full[..full.len() - 4].to_vec(), - ), - ]; - - cases.into_iter().for_each(|(case, rkey, body)| { - let op = RecordOp { - action: OpAction::Create, - collection: Nsid::::new_owned(collection).unwrap(), - rkey: Rkey::::new_owned(rkey).unwrap(), - record: Some(Bytes::from(body)), - }; - assert!( - invite_op_of(&op, &repo).is_none(), - "invite with {case} still decoded" - ); - }); - } -} - -#[cfg(test)] -pub(crate) mod testcbor { - - const TAG_CID: u64 = 42; - pub(crate) const GIT_REF_COLLECTION: &str = "sh.tangled.git.ref"; - - pub(crate) const FROZEN_PUSHED_REF: &str = "a4637368617828616261626162616261626162616261626162616261626162616261626162616261626162616261626524747970657273682e74616e676c65642e6769742e7265666d782d746e676c2d656469746f726e6469643a706c633a6c696d70657473782d746e676c2d707573682d6f7074696f6e738167736b69702d6369"; - pub(crate) const FROZEN_BARE_REF: &str = "a2637368617840636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646524747970657273682e74616e676c65642e6769742e726566"; - - #[derive(Clone)] - pub(crate) enum Val { - Text(String), - Bytes(Vec), - Uint(u64), - Nint(u64), - Bool(bool), - Array(Vec), - Map(Vec<(Val, Val)>), - CidLink(Vec), - } - - pub(crate) fn t(s: &str) -> Val { - Val::Text(s.to_owned()) - } - - fn head(major: u8, arg: u64, out: &mut Vec) { - match arg { - 24..=255 => { - out.push(major << 5 | 24); - out.push(arg as u8); - } - 256..=65535 => { - out.push(major << 5 | 25); - out.extend_from_slice(&(arg as u16).to_be_bytes()); - } - 65536..=4294967295 => { - out.push(major << 5 | 26); - out.extend_from_slice(&(arg as u32).to_be_bytes()); - } - _ => { - out.push(major << 5 | 27); - out.extend_from_slice(&arg.to_be_bytes()); - } - } - } - - fn encode(val: &Val, out: &mut Vec) { - match val { - Val::Text(s) => { - head(3, s.len() as u64, out); - out.extend_from_slice(s.as_bytes()); - } - Val::Bytes(b) => { - head(2, b.len() as u64, out); - out.extend_from_slice(b); - } - Val::Uint(n) => head(0, *n, out), - Val::Nint(n) => head(1, *n, out), - Val::Bool(b) => out.push(if *b { 0xf5 } else { 0xf4 }), - Val::Array(items) => { - head(4, items.len() as u64, out); - items.iter().for_each(|item| encode(item, out)); - } - Val::Map(pairs) => { - head(5, pairs.len() as u64, out); - pairs.iter().for_each(|(key, value)| { - encode(key, out); - encode(value, out); - }); - } - Val::CidLink(cid) => { - head(6, TAG_CID, out); - head(2, cid.len() as u64 + 1, out); - out.push(0); - out.extend_from_slice(cid); - } - } - } - - pub(crate) fn encode_one(val: &Val) -> Vec { - let mut out = Vec::new(); - encode(val, &mut out); - out - } - - pub(crate) fn unhex(s: &str) -> Vec { - (0..s.len()) - .step_by(2) - .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) - .collect() - } - - pub(crate) const TEST_CID: &[u8] = &[0x01, 0x55, 0x00, 0x04, 0xde, 0xad, 0xbe, 0xef]; - - fn varint(mut value: usize, out: &mut Vec) { - loop { - let byte = (value & 0x7f) as u8; - value >>= 7; - if value == 0 { - out.push(byte); - return; - } - out.push(byte | 0x80); - } - } - - pub(crate) fn car(records: &[&[u8]]) -> Vec { - let header = encode_one(&Val::Map(vec![ - (t("roots"), Val::Array(vec![])), - (t("version"), Val::Uint(1)), - ])); - let mut out = Vec::new(); - varint(header.len(), &mut out); - out.extend_from_slice(&header); - for record in records { - varint(TEST_CID.len() + record.len(), &mut out); - out.extend_from_slice(TEST_CID); - out.extend_from_slice(record); - } - out - } - - pub(crate) fn commit_frame(repo: &str, seq: i64, ops: Vec, car_bytes: Vec) -> Vec { - let mut bytes = encode_one(&Val::Map(vec![(t("t"), t("#commit"))])); - let mut payload = encode_one(&Val::Map(vec![ - (t("repo"), t(repo)), - (t("seq"), Val::Uint(seq as u64)), - (t("rev"), t("3zzzzzzzzzzzz")), - (t("time"), t("2026-08-27T00:00:00Z")), - (t("tooBig"), Val::Bool(false)), - (t("rebase"), Val::Bool(false)), - (t("blobs"), Val::Array(vec![])), - (t("commit"), Val::CidLink(TEST_CID.to_vec())), - (t("ops"), Val::Array(ops)), - (t("blocks"), Val::Bytes(car_bytes)), - ])); - bytes.append(&mut payload); - bytes - } - - pub(crate) fn git_ref_op(action: &str, rkey: &str, cid: Option>) -> Val { - let mut pairs = vec![ - (t("action"), t(action)), - (t("path"), t(&format!("{GIT_REF_COLLECTION}/{rkey}"))), - ]; - if let Some(cid) = cid { - pairs.push((t("cid"), Val::CidLink(cid))); - } - Val::Map(pairs) - } - - pub(crate) fn invite_op(collection: &str, rkey: &str, cid: Option>) -> Val { - let mut pairs = vec![ - (t("action"), t("create")), - (t("path"), t(&format!("{collection}/{rkey}"))), - ]; - if let Some(cid) = cid { - pairs.push((t("cid"), Val::CidLink(cid))); - } - Val::Map(pairs) - } - - pub(crate) fn invite_record(collection: &str, created_at: &str, editor: &str) -> Vec { - encode_one(&Val::Map(vec![ - (t("$type"), t(collection)), - (t("createdAt"), t(created_at)), - (t("x-tngl-editor"), t(editor)), - ])) - } -} diff --git a/bobbin/crates/knot-ingest/src/gate.rs b/bobbin/crates/knot-ingest/src/gate.rs deleted file mode 100644 index bd967e3cc..000000000 --- a/bobbin/crates/knot-ingest/src/gate.rs +++ /dev/null @@ -1,272 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; - -use parking_lot::Mutex; - -use bobbin_runtime::Clock; -use bobbin_types::knot_acl::KnotHostKey; -use knot_capability::Capability; -use tokio::time::Instant; - -use crate::client::{KnotClient, KnotClientError, knot_endpoint}; -use crate::stream::Feed; - -const NO_ACL_REPROBE_INTERVAL: Duration = Duration::from_secs(300); -const ERROR_REPROBE_INTERVAL: Duration = Duration::from_secs(60); - -struct ProbeRecord { - at: Instant, - retry_after: Duration, -} - -pub struct CapabilityGate { - client: KnotClient, - clock: Arc, - dev: bool, - allow_private: bool, - admitted: Mutex>, - last_probe: Mutex>, -} - -impl CapabilityGate { - pub fn new(client: KnotClient, clock: Arc, dev: bool, allow_private: bool) -> Self { - Self { - client, - clock, - dev, - allow_private, - admitted: Mutex::new(HashMap::new()), - last_probe: Mutex::new(HashMap::new()), - } - } - - pub fn is_native(&self, host: &KnotHostKey) -> bool { - self.feed(host).is_some() - } - - fn feed(&self, host: &KnotHostKey) -> Option { - self.admitted.lock().get(host).copied() - } - - pub async fn admit(&self, host: &KnotHostKey) -> Option { - if let Some(feed) = self.feed(host) { - return Some(feed); - } - let now = self.clock.now_instant(); - if self.throttled(host, now) { - return None; - } - match self.probe(host).await { - Ok(Some(feed)) => { - self.admitted.lock().insert(host.clone(), feed); - Some(feed) - } - Ok(None) => { - self.mark(host, now, NO_ACL_REPROBE_INTERVAL); - None - } - Err(err) => { - tracing::warn!(host = %host, error = %err, "knot capability probe failed"); - self.mark(host, now, ERROR_REPROBE_INTERVAL); - None - } - } - } - - async fn probe(&self, host: &KnotHostKey) -> Result, KnotClientError> { - let endpoint = knot_endpoint(host.as_str(), self.dev, self.allow_private)?; - let caps = self.client.capabilities(&endpoint).await?; - Ok(caps.contains(&Capability::KnotAcl).then(|| { - if caps.contains(&Capability::AtprotoFirehose) { - Feed::Atproto - } else { - Feed::Legacy - } - })) - } - - fn throttled(&self, host: &KnotHostKey, now: Instant) -> bool { - self.last_probe - .lock() - .get(host) - .is_some_and(|rec| now.saturating_duration_since(rec.at) < rec.retry_after) - } - - fn mark(&self, host: &KnotHostKey, now: Instant, retry_after: Duration) { - self.last_probe.lock().insert( - host.clone(), - ProbeRecord { - at: now, - retry_after, - }, - ); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::atomic::{AtomicU64, Ordering}; - - use bobbin_runtime::{ReqwestHttp, SleepFuture, UnixMicros}; - use serde_json::json; - use wiremock::matchers::{method, path}; - use wiremock::{Mock, MockServer, ResponseTemplate}; - - struct ManualClock { - base: Instant, - offset_micros: AtomicU64, - } - - impl ManualClock { - fn new() -> Self { - Self { - base: Instant::now(), - offset_micros: AtomicU64::new(0), - } - } - - fn advance(&self, by: Duration) { - self.offset_micros - .fetch_add(by.as_micros() as u64, Ordering::SeqCst); - } - } - - impl Clock for ManualClock { - fn now_unix_micros(&self) -> UnixMicros { - UnixMicros::new(self.offset_micros.load(Ordering::SeqCst)) - } - fn now_instant(&self) -> Instant { - self.base + Duration::from_micros(self.offset_micros.load(Ordering::SeqCst)) - } - fn sleep(&self, _: Duration) -> SleepFuture { - Box::pin(async {}) - } - fn sleep_until(&self, _: Instant) -> SleepFuture { - Box::pin(async {}) - } - } - - fn gate(server: &MockServer, clock: Arc) -> (CapabilityGate, KnotHostKey) { - let client = KnotClient::new(ReqwestHttp::shared(reqwest::Client::new())); - let url = url::Url::parse(&server.uri()).unwrap(); - let host = format!("{}:{}", url.host_str().unwrap(), url.port().unwrap()); - ( - CapabilityGate::new(client, clock, true, true), - KnotHostKey::new(&host), - ) - } - - async fn mount_version(server: &MockServer, caps: serde_json::Value, expect: u64) { - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.knot.version")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({ "version": "1.0.0 (cafe)", "capabilities": caps })), - ) - .expect(expect) - .mount(server) - .await; - } - - #[tokio::test] - async fn mounted_capabilities_pick_which_feed() { - for (caps, wanted) in [ - (json!(["knot-acl"]), Some(Feed::Legacy)), - (json!(["knot-acl", "atproto-firehose"]), Some(Feed::Atproto)), - (json!([]), None), - ] { - let mounted = caps.to_string(); - let server = MockServer::start().await; - mount_version(&server, caps, 1).await; - let (gate, host) = gate(&server, Arc::new(ManualClock::new())); - - assert_eq!(gate.admit(&host).await, wanted, "knot serving {mounted}"); - assert_eq!( - gate.feed(&host), - wanted, - "{mounted} latched the admitted feed" - ); - assert_eq!( - gate.is_native(&host), - wanted.is_some(), - "{mounted} counts as native" - ); - } - } - - #[tokio::test] - async fn admission_is_latched_and_survives_probe_error() { - let server = MockServer::start().await; - mount_version(&server, json!(["knot-acl"]), 1).await; - let clock = Arc::new(ManualClock::new()); - let (gate, host) = gate(&server, clock.clone()); - assert_eq!(gate.admit(&host).await, Some(Feed::Legacy)); - - server.reset().await; - clock.advance(NO_ACL_REPROBE_INTERVAL + Duration::from_secs(1)); - assert_eq!( - gate.admit(&host).await, - Some(Feed::Legacy), - "latch never re-probes" - ); - assert!(gate.is_native(&host)); - } - - #[tokio::test] - async fn unadmitted_throttled_then_reprobed_after_interval() { - let server = MockServer::start().await; - mount_version(&server, json!([]), 2).await; - let clock = Arc::new(ManualClock::new()); - let (gate, host) = gate(&server, clock.clone()); - assert_eq!(gate.admit(&host).await, None); - clock.advance(Duration::from_secs(60)); - assert_eq!(gate.admit(&host).await, None); - clock.advance(NO_ACL_REPROBE_INTERVAL); - assert_eq!(gate.admit(&host).await, None); - } - - #[tokio::test] - async fn acl_upgrade_is_detected_on_reprobe() { - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.knot.version")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "version": "1.0.0" }))) - .up_to_n_times(1) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.knot.version")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({ "version": "1.1.0", "capabilities": ["knot-acl"] })), - ) - .mount(&server) - .await; - let clock = Arc::new(ManualClock::new()); - let (gate, host) = gate(&server, clock.clone()); - assert_eq!(gate.admit(&host).await, None); - clock.advance(NO_ACL_REPROBE_INTERVAL + Duration::from_secs(1)); - assert_eq!(gate.admit(&host).await, Some(Feed::Legacy)); - assert!(gate.is_native(&host)); - } - - #[tokio::test] - async fn probe_error_throttled_briefly_then_reprobed() { - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.knot.version")) - .respond_with(ResponseTemplate::new(503)) - .expect(2) - .mount(&server) - .await; - let clock = Arc::new(ManualClock::new()); - let (gate, host) = gate(&server, clock.clone()); - assert_eq!(gate.admit(&host).await, None); - clock.advance(Duration::from_secs(1)); - assert_eq!(gate.admit(&host).await, None, "error reprobe is throttled"); - clock.advance(ERROR_REPROBE_INTERVAL); - assert_eq!(gate.admit(&host).await, None); - } -} diff --git a/bobbin/crates/knot-ingest/src/legacy.rs b/bobbin/crates/knot-ingest/src/legacy.rs deleted file mode 100644 index 95b61453c..000000000 --- a/bobbin/crates/knot-ingest/src/legacy.rs +++ /dev/null @@ -1,128 +0,0 @@ -use jacquard_common::DefaultStr; -use jacquard_common::types::did::Did; -use jacquard_common::types::nsid::Nsid; -use serde::Deserialize; - -use crate::stream::{Cursor, FeedHandler, Outcome}; - -const COLLABORATOR_UPDATE_NSID: &str = "sh.tangled.repo.collaboratorUpdate"; - -#[derive(Deserialize)] -struct FrameWire { - nsid: Nsid, - event: EventWire, - created: i64, -} - -#[derive(Deserialize)] -struct EventWire { - repo: Option>, -} - -pub(crate) fn dispatch(text: &str, handler: &dyn FeedHandler, cursor: &mut Cursor) -> Outcome { - let Ok(frame) = serde_json::from_str::(text) else { - return Outcome::Idle; - }; - if frame.created <= 0 { - return Outcome::Idle; - } - cursor.advance(frame.created); - if frame.nsid.as_str() == COLLABORATOR_UPDATE_NSID - && let Some(repo) = frame.event.repo - { - handler.repo_roster_changed(&repo); - } - Outcome::Progressed -} - -#[cfg(test)] -mod tests { - use parking_lot::Mutex; - - use super::*; - use crate::firehose::InviteOp; - use crate::stream::Feed; - - #[derive(Default)] - struct Recorder(Mutex>); - - impl FeedHandler for Recorder { - fn roster_changed(&self) { - self.0.lock().push("roster".to_owned()); - } - - fn repo_roster_changed(&self, repo: &Did) { - self.0.lock().push(repo.as_ref().to_owned()); - } - - fn invite_op(&self, op: InviteOp) { - self.0.lock().push(op.invitee.as_ref().to_owned()); - } - } - - const COLLABORATOR: &str = r#"{"rkey":"3mug","nsid":"sh.tangled.repo.collaboratorUpdate", - "event":{"op":"remove","subject":"did:plc:limpet","repo":"did:plc:scallop"},"created":6521339806120}"#; - const COLLABORATOR_OLD: &str = r#"{"rkey":"3mug","nsid":"sh.tangled.repo.collaboratorUpdate", - "event":{"op":"remove","subject":"did:plc:limpet","repo":"did:plc:scallop"},"created":7}"#; - const COLLABORATOR_UNDATED: &str = r#"{"rkey":"3mug","nsid":"sh.tangled.repo.collaboratorUpdate", - "event":{"op":"remove","subject":"did:plc:limpet","repo":"did:plc:scallop"},"created":0}"#; - const COLLABORATOR_NO_REPO: &str = r#"{"rkey":"3mug","nsid":"sh.tangled.repo.collaboratorUpdate", - "event":{"op":"remove","subject":"did:plc:limpet"},"created":7}"#; - const REF: &str = r#"{"rkey":"3mug","nsid":"sh.tangled.git.refUpdate", - "event":{"ref":"refs/heads/main"},"created":9}"#; - const GARBLED: &str = r#"{"nsid":"sh.tangled.repo.collaboratorUpdate"}"#; - const NOT_AN_NSID: &str = r#"{"rkey":"3mug","nsid":"not an nsid at all", - "event":{"op":"add","subject":"did:plc:limpet"},"created":7}"#; - - #[test] - fn each_legacy_frame_moves_cursor_and_calls_handler() { - let cases: [(&str, &str, i64, bool, i64, &[&str]); 7] = [ - ( - "collaborator update", - COLLABORATOR, - 0, - true, - 6521339806120, - &["did:plc:scallop"], - ), - ("unrelated ref update", REF, 4, true, 9, &[]), - ( - "older collaborator update", - COLLABORATOR_OLD, - 100, - true, - 100, - &["did:plc:scallop"], - ), - ( - "collaborator update naming no repo", - COLLABORATOR_NO_REPO, - 0, - true, - 7, - &[], - ), - ("garbled frame", GARBLED, 3, false, 3, &[]), - ("undated frame", COLLABORATOR_UNDATED, 5, false, 5, &[]), - ("frame with a junk nsid", NOT_AN_NSID, 3, false, 3, &[]), - ]; - - cases - .into_iter() - .for_each(|(case, frame, start, news, wanted, calls)| { - let handler = Recorder::default(); - let mut cursor = Cursor::at(Feed::Legacy, start); - - let outcome = dispatch(frame, &handler, &mut cursor); - - assert_eq!( - matches!(outcome, Outcome::Progressed), - news, - "whether {case} progressed" - ); - assert_eq!(cursor.seq(), wanted, "cursor after {case}"); - assert_eq!(cursor.feed(), Feed::Legacy, "feed after {case}"); - assert_eq!(*handler.0.lock(), calls, "handler calls from {case}"); - }); - } -} diff --git a/bobbin/crates/knot-ingest/src/lib.rs b/bobbin/crates/knot-ingest/src/lib.rs deleted file mode 100644 index bf864b3e0..000000000 --- a/bobbin/crates/knot-ingest/src/lib.rs +++ /dev/null @@ -1,15 +0,0 @@ -pub mod client; -pub mod firehose; -pub mod gate; -pub mod legacy; -pub mod orchestrator; -pub mod registry; -pub mod roster; -pub mod stream; - -pub use client::{AclEntry, AclListing, Completeness, KnotClient, KnotClientError, knot_endpoint}; -pub use gate::CapabilityGate; -pub use orchestrator::Orchestrator; -pub use registry::KnotRegistry; -pub use roster::Roster; -pub use stream::{Feed, StreamConfig, run_stream}; diff --git a/bobbin/crates/knot-ingest/src/orchestrator.rs b/bobbin/crates/knot-ingest/src/orchestrator.rs deleted file mode 100644 index 3a2a8a623..000000000 --- a/bobbin/crates/knot-ingest/src/orchestrator.rs +++ /dev/null @@ -1,864 +0,0 @@ -use parking_lot::Mutex; -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; -use std::time::Duration; - -use bobbin_edge_index::{CoverageWatch, EdgeStore, InviteIndex}; -use bobbin_knot_proxy::KnotHost; -use bobbin_runtime::{Clock, WsTransport}; -use bobbin_types::knot_acl::{KnotHostKey, host_to_knot_did}; -use futures::StreamExt; -use jacquard_common::DefaultStr; -use jacquard_common::types::did::Did; -use tokio_util::sync::CancellationToken; - -use tokio::sync::mpsc; - -use crate::client::{ - AclEntry, Completeness, InviteEntry, KnotClient, KnotClientError, Listing, knot_endpoint, -}; -use crate::firehose::InviteOp; -use crate::gate::CapabilityGate; -use crate::registry::KnotRegistry; -use crate::roster::{AclKind, Applied, Assertion, Roster, Source}; -use crate::stream::{Feed, FeedHandler, StreamConfig, run_stream}; - -const POLL_INTERVAL: Duration = Duration::from_secs(30); -const RECONCILE_INTERVAL: Duration = Duration::from_secs(300); -const PROBE_CONCURRENCY: usize = 16; -const NUDGE_QUEUE_CAPACITY: usize = 64; -const NUDGE_DEBOUNCE: Duration = Duration::from_secs(3); - -pub struct Orchestrator { - pub client: Arc, - pub gate: Arc, - pub registry: Arc, - pub store: Arc, - pub invites: Arc, - pub coverage: Arc, - pub ws: Arc, - pub clock: Arc, - pub dev: bool, - pub allow_private: bool, - pub cancel: CancellationToken, -} - -impl Orchestrator { - pub async fn run(self) { - let mut subscribed: HashMap = HashMap::new(); - let mut unspawnable: HashSet = HashSet::new(); - loop { - if self.cancel.is_cancelled() { - break; - } - let discovered = self.coverage.snapshot().is_ready(); - self.discover(&mut subscribed, &mut unspawnable).await; - if discovered { - self.invites.all_knots_discovered(); - } - tokio::select! { - _ = self.cancel.cancelled() => break, - _ = self.clock.sleep(POLL_INTERVAL) => {} - } - } - } - - async fn discover( - &self, - subscribed: &mut HashMap, - unspawnable: &mut HashSet, - ) { - let candidates: Vec = self - .registry - .hosts() - .into_iter() - .filter(|host| !subscribed.contains_key(host) && !unspawnable.contains(host)) - .collect(); - candidates - .iter() - .filter_map(|host| host_to_knot_did(host.as_str())) - .for_each(|knot| self.invites.awaiting(knot)); - let approved: Vec<(KnotHostKey, Feed)> = futures::stream::iter(candidates) - .map(|host| async move { self.gate.admit(&host).await.map(|feed| (host, feed)) }) - .buffer_unordered(PROBE_CONCURRENCY) - .filter_map(|approved| async move { approved }) - .collect() - .await; - approved - .into_iter() - .for_each(|(host, feed)| match self.spawn(&host, feed) { - Some(token) => { - subscribed.insert(host, token); - } - None => { - tracing::warn!(host = %host, "skipping unspawnable knot endpoint"); - unspawnable.insert(host); - } - }); - } - - fn spawn(&self, host: &KnotHostKey, feed: Feed) -> Option { - let endpoint = knot_endpoint(host.as_str(), self.dev, self.allow_private).ok()?; - let knot_did = host_to_knot_did(host.as_str())?; - let roster = Arc::new(Mutex::new(Roster::new( - self.store.clone(), - self.invites.clone(), - knot_did.clone(), - self.registry.clone(), - host.clone(), - ))); - let token = self.cancel.child_token(); - - let stream_cfg = StreamConfig { - ws: self.ws.clone(), - clock: self.clock.clone(), - cancel: token.clone(), - }; - let (nudge_tx, nudge_rx) = mpsc::channel(NUDGE_QUEUE_CAPACITY); - let handlers = KnotHandlers { - host: host.clone(), - acl: nudge_tx, - roster: roster.clone(), - }; - let stream_endpoint = endpoint.clone(); - let stream_cursor = crate::stream::Cursor::start(feed); - tokio::spawn(async move { - run_stream(&stream_cfg, &stream_endpoint, stream_cursor, &handlers).await; - }); - - let client = self.client.clone(); - let registry = self.registry.clone(); - let invites = self.invites.clone(); - let clock = self.clock.clone(); - let host_owned = host.clone(); - let reconcile_token = token.clone(); - tokio::spawn(async move { - let ctx = ReconcileCtx { - client: &client, - endpoint: &endpoint, - knot: &knot_did, - host: &host_owned, - registry: ®istry, - invites: &invites, - roster: &roster, - }; - reconcile_loop(&ctx, &*clock, &reconcile_token, nudge_rx).await; - }); - - Some(token) - } -} - -#[derive(Clone, Debug, PartialEq)] -enum Nudge { - Full, - Repo(Did), -} - -struct KnotHandlers { - host: KnotHostKey, - acl: mpsc::Sender, - roster: Arc>, -} - -impl KnotHandlers { - fn nudge(&self, nudge: Nudge) { - match self.acl.try_send(nudge) { - Ok(()) => {} - Err(mpsc::error::TrySendError::Full(_)) => { - tracing::warn!( - host = %self.host, - "reconcile nudge queue full; the periodic reconcile will cover it" - ); - } - Err(mpsc::error::TrySendError::Closed(_)) => {} - } - } -} - -impl FeedHandler for KnotHandlers { - fn roster_changed(&self) { - self.nudge(Nudge::Full); - } - - fn repo_roster_changed(&self, repo: &Did) { - self.nudge(Nudge::Repo(repo.clone())); - } - - fn invite_op(&self, op: InviteOp) { - let InviteOp { - repo, - invitee, - offer, - } = op; - let mut roster = self.roster.lock(); - match offer { - Some(offer) => roster.apply( - Assertion::Invite { - repo: &repo, - invited_by: offer.invited_by, - }, - invitee, - offer.offered_at, - Source::Firehose, - ), - None => roster.retire(AclKind::invite(&repo), invitee), - } - } -} - -struct ReconcileCtx<'a> { - client: &'a KnotClient, - endpoint: &'a KnotHost, - knot: &'a Did, - host: &'a KnotHostKey, - registry: &'a KnotRegistry, - invites: &'a InviteIndex, - roster: &'a Mutex, -} - -async fn reconcile_loop( - ctx: &ReconcileCtx<'_>, - clock: &dyn Clock, - cancel: &CancellationToken, - mut nudges: mpsc::Receiver, -) { - reconcile_once(ctx).await; - loop { - if cancel.is_cancelled() { - return; - } - tokio::select! { - _ = cancel.cancelled() => return, - _ = clock.sleep(RECONCILE_INTERVAL) => { - reconcile_once(ctx).await; - } - nudge = nudges.recv() => match nudge { - None => return, - Some(first) => { - let batch = collect_batch(first, &mut nudges, clock, cancel).await; - if cancel.is_cancelled() { - return; - } - apply_batch(ctx, batch).await; - } - }, - } - } -} - -struct NudgeBatch { - full: bool, - repos: HashSet>, -} - -async fn collect_batch( - first: Nudge, - nudges: &mut mpsc::Receiver, - clock: &dyn Clock, - cancel: &CancellationToken, -) -> NudgeBatch { - let mut batch = NudgeBatch { - full: false, - repos: HashSet::new(), - }; - match first { - Nudge::Full => batch.full = true, - Nudge::Repo(repo) => { - batch.repos.insert(repo); - } - } - let debounce = clock.sleep(NUDGE_DEBOUNCE); - tokio::pin!(debounce); - loop { - tokio::select! { - _ = cancel.cancelled() => return batch, - _ = &mut debounce => return batch, - nudge = nudges.recv() => match nudge { - None => return batch, - Some(Nudge::Full) => batch.full = true, - Some(Nudge::Repo(repo)) => { - batch.repos.insert(repo); - } - }, - } - } -} -async fn apply_batch(ctx: &ReconcileCtx<'_>, batch: NudgeBatch) { - if batch.full { - reconcile_once(ctx).await; - return; - } - let horizon = ctx.roster.lock().applied(); - futures::stream::iter(batch.repos) - .for_each(|repo| async move { - reconcile_repo_once(ctx, &repo, horizon).await; - }) - .await; -} - -async fn reconcile_once(ctx: &ReconcileCtx<'_>) { - let horizon = ctx.roster.lock().applied(); - - let repo_offers = futures::stream::iter(ctx.registry.repos(ctx.host)) - .then(|repo| async move { reconcile_repo_once(ctx, &repo, horizon).await }) - .fold( - Completeness::Complete, - |all, one| async move { all.and(one) }, - ) - .await; - - if repo_offers == Completeness::Complete { - ctx.invites.backfilled(ctx.knot); - } - - ctx.roster.lock().purge_legacy(); -} - -async fn reconcile_repo_once( - ctx: &ReconcileCtx<'_>, - repo: &Did, - horizon: Applied, -) -> Completeness { - let collaborators = ctx.client.list_collaborators(ctx.endpoint, repo).await; - reconcile_listing(ctx, repo, collaborators, horizon); - - let invited = ctx - .client - .list_collaborator_invites(ctx.endpoint, repo) - .await; - reconcile_listing(ctx, repo, invited, horizon) -} - -trait Listed { - fn kind(repo: &Did) -> AclKind<'_>; - fn subject(&self) -> &Did; - fn apply_to(self, roster: &mut Roster, repo: &Did, source: Source); -} - -impl Listed for AclEntry { - fn kind(repo: &Did) -> AclKind<'_> { - AclKind::acl(repo) - } - - fn subject(&self) -> &Did { - &self.subject - } - - fn apply_to(self, roster: &mut Roster, repo: &Did, source: Source) { - roster.apply(Assertion::Acl(repo), self.subject, self.created_at, source); - } -} - -impl Listed for InviteEntry { - fn kind(repo: &Did) -> AclKind<'_> { - AclKind::invite(repo) - } - - fn subject(&self) -> &Did { - &self.subject - } - - fn apply_to(self, roster: &mut Roster, repo: &Did, source: Source) { - roster.apply( - Assertion::Invite { - repo, - invited_by: self.added_by, - }, - self.subject, - self.created_at, - source, - ); - } -} - -fn reconcile_listing( - ctx: &ReconcileCtx<'_>, - repo: &Did, - listing: Result, KnotClientError>, - horizon: Applied, -) -> Completeness { - let kind = T::kind(repo); - let Listing { - entries, - completeness, - } = match listing { - Ok(listing) => listing, - Err(KnotClientError::NotFound) if kind.is_invite() => return Completeness::Complete, - Err(err) => { - warn_unreadable(ctx, kind, &err); - return Completeness::Truncated; - } - }; - let present: HashSet> = entries - .iter() - .map(|entry| entry.subject().clone()) - .collect(); - let mut guard = ctx.roster.lock(); - entries - .into_iter() - .for_each(|entry| entry.apply_to(&mut guard, repo, Source::Listing(horizon))); - if completeness == Completeness::Complete { - guard.reap(kind, &present, horizon); - } - completeness -} - -fn warn_unreadable(ctx: &ReconcileCtx<'_>, kind: AclKind<'_>, err: &KnotClientError) { - tracing::warn!( - host = %ctx.host, - repo = %kind.repo.as_ref(), - listing = kind.listing(), - error = %err, - "knot reconcile failed" - ); -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::client::authority; - use bobbin_edge_index::Offer; - use bobbin_runtime::{ReqwestHttp, RuntimeHasher, SystemClock, UnixMicros}; - use bobbin_types::edges::Edge; - use bobbin_types::ids::{EdgeKey, SubjectRef, nsid_static}; - use jacquard_common::types::string::AtUri; - use serde_json::json; - use wiremock::matchers::{method, path, query_param}; - use wiremock::{Mock, MockServer, ResponseTemplate}; - - fn did(s: &str) -> Did { - Did::new_owned(s).unwrap() - } - - fn collaborator_count(store: &EdgeStore, repo: &str) -> u64 { - store.count(&EdgeKey::new( - nsid_static("sh.tangled.repo.collaborator"), - SubjectRef::Did(did(repo)), - )) - } - - struct Harness { - server: MockServer, - endpoint: KnotHost, - knot: Did, - host: KnotHostKey, - registry: Arc, - store: Arc, - invites: Arc, - roster: Arc>, - client: KnotClient, - } - - impl Harness { - fn ctx(&self) -> ReconcileCtx<'_> { - ReconcileCtx { - client: &self.client, - endpoint: &self.endpoint, - knot: &self.knot, - host: &self.host, - registry: &self.registry, - invites: &self.invites, - roster: &self.roster, - } - } - } - - async fn harness() -> Harness { - let server = MockServer::start().await; - let endpoint = KnotHost::parse(&server.uri()).unwrap(); - let host = KnotHostKey::new(&authority(&endpoint)); - let registry = Arc::new(KnotRegistry::new()); - let store = Arc::new(EdgeStore::new(RuntimeHasher::default())); - let invites = Arc::new(InviteIndex::new()); - let knot_did = host_to_knot_did(host.as_str()).unwrap(); - let roster = Arc::new(Mutex::new(Roster::new( - store.clone(), - invites.clone(), - knot_did.clone(), - registry.clone(), - host.clone(), - ))); - let client = KnotClient::new(ReqwestHttp::shared(reqwest::Client::new())); - Harness { - server, - endpoint, - knot: knot_did, - host, - registry, - store, - invites, - roster, - client, - } - } - - const COLLABORATOR_INVITES: &str = "sh.tangled.repo.listCollaboratorInvites"; - const COLLABORATORS: &str = "sh.tangled.repo.listCollaborators"; - - async fn serve(h: &Harness, endpoint: &str, response: ResponseTemplate) { - Mock::given(method("GET")) - .and(path(format!("/xrpc/{endpoint}"))) - .respond_with(response) - .mount(&h.server) - .await; - } - - fn offers(invitees: &[&str]) -> serde_json::Value { - json!({ - "items": invitees - .iter() - .map(|invitee| json!({ - "subject": invitee, - "addedBy": "did:plc:akshay", - "createdAt": "2026-06-04T00:00:00Z" - })) - .collect::>() - }) - } - - #[tokio::test] - async fn reconcile_backfills_collaborators() { - let h = harness().await; - - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.repo.listCollaborators")) - .and(query_param("subject", "did:plc:scallop")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "items": [ - {"subject": "did:plc:olaren", "addedBy": "did:plc:boltless", "createdAt": "2026-06-03T00:00:00Z"} - ] - }))) - .mount(&h.server) - .await; - - h.registry.observe_repo(&h.host, did("did:plc:scallop")); - - reconcile_once(&h.ctx()).await; - - assert_eq!(collaborator_count(&h.store, "did:plc:scallop"), 1); - } - - #[tokio::test] - async fn reconcile_backfills_outstanding_collaborator_invites() { - let h = harness().await; - h.registry.observe_repo(&h.host, did("did:plc:scallop")); - serve( - &h, - COLLABORATOR_INVITES, - ResponseTemplate::new(200).set_body_json(offers(&["did:plc:limpet", "did:plc:olaren"])), - ) - .await; - - reconcile_once(&h.ctx()).await; - - assert_eq!(h.invites.outstanding(&did("did:plc:olaren")).len(), 1); - let offer = h.invites.outstanding(&did("did:plc:limpet")).remove(0).1; - assert_eq!(offer.invited_by, did("did:plc:akshay")); - assert_eq!(offer.offered_at, UnixMicros::new(1_780_531_200_000_000)); - } - - #[tokio::test] - async fn completed_backfill_stops_knot_being_pending() { - let h = harness().await; - h.registry.observe_repo(&h.host, did("did:plc:scallop")); - h.invites.awaiting(h.knot.clone()); - h.invites.all_knots_discovered(); - assert_eq!(h.invites.pending(), vec![h.knot.clone()]); - - serve( - &h, - COLLABORATOR_INVITES, - ResponseTemplate::new(200).set_body_json(offers(&[])), - ) - .await; - reconcile_once(&h.ctx()).await; - - assert!(h.invites.pending().is_empty()); - } - - #[tokio::test] - async fn knot_without_invite_listings_stops_being_pending() { - let h = harness().await; - h.registry.observe_repo(&h.host, did("did:plc:scallop")); - h.invites.awaiting(h.knot.clone()); - h.invites.all_knots_discovered(); - - serve(&h, COLLABORATOR_INVITES, ResponseTemplate::new(404)).await; - reconcile_once(&h.ctx()).await; - - assert!( - h.invites.pending().is_empty(), - "knots predating invite listings won't list offers or stream them, so bobbin has read all it can" - ); - } - - #[tokio::test] - async fn failed_backfill_leaves_only_its_own_knot_pending() { - let h = harness().await; - h.registry.observe_repo(&h.host, did("did:plc:scallop")); - h.invites.awaiting(h.knot.clone()); - h.invites.awaiting(did("did:web:knot.nel.pet")); - h.invites.backfilled(&did("did:web:knot.nel.pet")); - h.invites.all_knots_discovered(); - - serve(&h, COLLABORATOR_INVITES, ResponseTemplate::new(500)).await; - reconcile_once(&h.ctx()).await; - - assert_eq!( - h.invites.pending(), - vec![h.knot.clone()], - "bobbin puts unreadable knots into pending, and answers anyway" - ); - } - - #[tokio::test] - async fn reconcile_reaps_departed_collaborator() { - let h = harness().await; - let repo = did("did:plc:scallop"); - h.registry.observe_repo(&h.host, repo.clone()); - - serve( - &h, - COLLABORATORS, - ResponseTemplate::new(200).set_body_json(json!({ - "items": [ - {"subject": "did:plc:boltless", "addedBy": "did:plc:akshay", "createdAt": "2026-06-01T00:00:00Z"}, - {"subject": "did:plc:akshay", "addedBy": "did:plc:akshay", "createdAt": "2026-06-02T00:00:00Z"} - ] - })), - ) - .await; - reconcile_once(&h.ctx()).await; - assert_eq!(collaborator_count(&h.store, "did:plc:scallop"), 2); - - h.server.reset().await; - serve( - &h, - COLLABORATORS, - ResponseTemplate::new(200).set_body_json(json!({ - "items": [ - {"subject": "did:plc:akshay", "addedBy": "did:plc:akshay", "createdAt": "2026-06-02T00:00:00Z"} - ] - })), - ) - .await; - reconcile_once(&h.ctx()).await; - - assert_eq!( - collaborator_count(&h.store, "did:plc:scallop"), - 1, - "a collaborator dropped from the authoritative snapshot is reaped on reconcile" - ); - } - - #[tokio::test] - async fn reconcile_skips_reap_when_collaborator_list_truncated() { - let h = harness().await; - let repo = did("did:plc:scallop"); - h.registry.observe_repo(&h.host, repo.clone()); - - serve( - &h, - COLLABORATORS, - ResponseTemplate::new(200).set_body_json(json!({ - "items": [{"subject": "did:plc:boltless", "addedBy": "did:plc:akshay", "createdAt": "2026-06-01T00:00:00Z"}], - "cursor": "more" - })), - ) - .await; - - { - let mut roster = h.roster.lock(); - let taken = roster.applied(); - roster.apply( - Assertion::Acl(&repo), - did("did:plc:akshay"), - UnixMicros::new(1_000_000), - Source::Listing(taken), - ); - } - assert_eq!(collaborator_count(&h.store, "did:plc:scallop"), 1); - - reconcile_once(&h.ctx()).await; - - assert_eq!( - collaborator_count(&h.store, "did:plc:scallop"), - 2, - "a truncated snapshot must not reap collaborators it could not enumerate" - ); - } - - fn seed_legacy_edge(store: &EdgeStore, kind: &'static str, subject: &str, source: &str) { - let source = AtUri::new_owned(source).unwrap(); - store.upsert_source( - &source, - vec![Edge { - kind: nsid_static(kind), - subject: SubjectRef::Did(did(subject)), - source: source.clone(), - sort_micros: 0, - }], - ); - } - - #[tokio::test] - async fn reconcile_purges_seeded_legacy_acl() { - let h = harness().await; - - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.repo.listCollaborators")) - .and(query_param("subject", "did:plc:scallop")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "items": [ - {"subject": "did:plc:olaren", "addedBy": "did:plc:boltless", "createdAt": "2026-06-03T00:00:00Z"} - ] - }))) - .mount(&h.server) - .await; - - let repo = did("did:plc:scallop"); - h.registry.observe_repo(&h.host, repo.clone()); - - seed_legacy_edge( - &h.store, - "sh.tangled.repo.collaborator", - "did:plc:scallop", - "at://did:plc:akshay/sh.tangled.repo.collaborator/r2", - ); - - reconcile_once(&h.ctx()).await; - - assert_eq!( - collaborator_count(&h.store, "did:plc:scallop"), - 1, - "stale legacy collaborator purged, knot-owned collaborator kept" - ); - } - - fn test_handlers(roster: Arc>, acl: mpsc::Sender) -> KnotHandlers { - KnotHandlers { - host: KnotHostKey::new("oyster.cafe"), - acl, - roster, - } - } - - #[tokio::test] - async fn handlers_nudge_full_for_roster_changes() { - let h = harness().await; - let (tx, mut rx) = mpsc::channel(4); - let handlers = test_handlers(h.roster.clone(), tx); - handlers.roster_changed(); - handlers.roster_changed(); - assert!(matches!(rx.try_recv(), Ok(Nudge::Full))); - assert!(matches!(rx.try_recv(), Ok(Nudge::Full))); - assert!(rx.try_recv().is_err()); - } - - #[tokio::test] - async fn handlers_nudge_repo_for_repo_roster_changes() { - let h = harness().await; - let (tx, mut rx) = mpsc::channel(4); - let handlers = test_handlers(h.roster.clone(), tx); - handlers.repo_roster_changed(&did("did:plc:scallop")); - assert_eq!(rx.try_recv(), Ok(Nudge::Repo(did("did:plc:scallop"))),); - } - - #[tokio::test] - async fn handlers_apply_firehose_invite_straight_to_roster() { - let h = harness().await; - let repo = did("did:plc:scallop"); - h.registry.observe_repo(&h.host, repo.clone()); - let (tx, _rx) = mpsc::channel(4); - let handlers = test_handlers(h.roster.clone(), tx); - let invitee = did("did:plc:limpet"); - - handlers.invite_op(InviteOp { - repo: repo.clone(), - invitee: invitee.clone(), - offer: Some(Offer { - invited_by: did("did:plc:akshay"), - offered_at: UnixMicros::new(1_780_272_000_000_000), - }), - }); - assert_eq!(h.invites.outstanding(&invitee).len(), 1); - - handlers.invite_op(InviteOp { - repo: repo.clone(), - invitee: invitee.clone(), - offer: None, - }); - assert!( - h.invites.outstanding(&invitee).is_empty(), - "firehose delete withdraws offer it created" - ); - } - - #[tokio::test(start_paused = true)] - async fn collaborator_nudge_reconciles_only_that_repo() { - let Harness { - server, - endpoint, - knot, - host, - registry, - store, - invites, - roster, - client, - } = harness().await; - let other = KnotHostKey::new("elsewhere.cafe"); - - Mock::given(method("GET")) - .and(path("/xrpc/sh.tangled.repo.listCollaborators")) - .and(query_param("subject", "did:plc:scallop")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "items": [ - {"subject": "did:plc:olaren", "addedBy": "did:plc:boltless", "createdAt": "2026-06-03T00:00:00Z"} - ] - }))) - .mount(&server) - .await; - - registry.observe_repo(&host, did("did:plc:scallop")); - let clock = Arc::new(SystemClock::new()); - let cancel = CancellationToken::new(); - let (tx, rx) = mpsc::channel(8); - - let loop_host = host.clone(); - let loop_registry = registry.clone(); - let loop_roster = roster.clone(); - let loop_invites = invites.clone(); - let loop_cancel = cancel.clone(); - let task = tokio::spawn(async move { - let ctx = ReconcileCtx { - client: &client, - endpoint: &endpoint, - knot: &knot, - host: &loop_host, - registry: &loop_registry, - invites: &loop_invites, - roster: &loop_roster, - }; - reconcile_loop(&ctx, clock.as_ref(), &loop_cancel, rx).await; - }); - - tx.send(Nudge::Repo(did("did:plc:scallop"))).await.unwrap(); - for _ in 0..60 { - tokio::time::advance(Duration::from_millis(200)).await; - if collaborator_count(&store, "did:plc:scallop") == 1 { - break; - } - } - cancel.cancel(); - task.await.unwrap(); - - assert_eq!( - collaborator_count(&store, "did:plc:scallop"), - 1, - "a collaborator invite nudges a reconcile that folds the knot's authoritative list" - ); - assert_eq!(other, KnotHostKey::new("elsewhere.cafe")); - } -} diff --git a/bobbin/crates/knot-ingest/src/registry.rs b/bobbin/crates/knot-ingest/src/registry.rs deleted file mode 100644 index 7b6543a46..000000000 --- a/bobbin/crates/knot-ingest/src/registry.rs +++ /dev/null @@ -1,129 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::sync::Mutex; - -use bobbin_types::knot_acl::KnotHostKey; -use jacquard_common::DefaultStr; -use jacquard_common::types::did::Did; - -#[derive(Default)] -struct Inner { - hosts: HashSet, - repos: HashMap>>, - repo_host: HashMap, KnotHostKey>, -} - -#[derive(Default)] -pub struct KnotRegistry { - inner: Mutex, -} - -impl KnotRegistry { - pub fn new() -> Self { - Self::default() - } - - pub fn observe_host(&self, host: &KnotHostKey) { - self.inner.lock().unwrap().hosts.insert(host.clone()); - } - - pub fn observe_repo(&self, host: &KnotHostKey, repo: Did) { - let mut inner = self.inner.lock().unwrap(); - inner.hosts.insert(host.clone()); - inner - .repos - .entry(host.clone()) - .or_default() - .insert(repo.clone()); - inner.repo_host.insert(repo, host.clone()); - } - - pub fn hosts(&self) -> Vec { - self.inner.lock().unwrap().hosts.iter().cloned().collect() - } - - pub fn repos(&self, host: &KnotHostKey) -> Vec> { - self.inner - .lock() - .unwrap() - .repos - .get(host) - .map(|set| set.iter().cloned().collect()) - .unwrap_or_default() - } - - pub fn repo_on_host(&self, host: &KnotHostKey, repo: &Did) -> bool { - self.inner - .lock() - .unwrap() - .repos - .get(host) - .is_some_and(|set| set.contains(repo)) - } - - pub fn host_of_repo(&self, repo: &Did) -> Option { - self.inner.lock().unwrap().repo_host.get(repo).cloned() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn did(s: &str) -> Did { - Did::new_owned(s).unwrap() - } - - fn host(s: &str) -> KnotHostKey { - KnotHostKey::new(s) - } - - #[test] - fn observe_repo_registers_host_and_repo() { - let registry = KnotRegistry::new(); - registry.observe_repo(&host("oyster.cafe"), did("did:plc:scallop")); - registry.observe_repo(&host("oyster.cafe"), did("did:plc:limpet")); - - assert_eq!(registry.hosts(), vec![host("oyster.cafe")]); - let mut repos = registry.repos(&host("oyster.cafe")); - repos.sort_by(|a, b| a.as_ref().cmp(b.as_ref())); - assert_eq!(repos, vec![did("did:plc:limpet"), did("did:plc:scallop")]); - } - - #[test] - fn observe_repo_dedups() { - let registry = KnotRegistry::new(); - registry.observe_repo(&host("nel.pet"), did("did:plc:whelk")); - registry.observe_repo(&host("nel.pet"), did("did:plc:whelk")); - assert_eq!(registry.repos(&host("nel.pet")), vec![did("did:plc:whelk")]); - } - - #[test] - fn observe_host_without_repos() { - let registry = KnotRegistry::new(); - registry.observe_host(&host("oyster.cafe")); - assert_eq!(registry.hosts(), vec![host("oyster.cafe")]); - assert!(registry.repos(&host("oyster.cafe")).is_empty()); - } - - #[test] - fn host_lookups_are_case_insensitive() { - let registry = KnotRegistry::new(); - registry.observe_repo(&host("KT.Oyster.Cafe"), did("did:plc:scallop")); - assert!(registry.repo_on_host(&host("kt.oyster.cafe"), &did("did:plc:scallop"))); - assert_eq!( - registry.host_of_repo(&did("did:plc:scallop")), - Some(host("kt.oyster.cafe")) - ); - } - - #[test] - fn host_of_repo_resolves_owning_knot() { - let registry = KnotRegistry::new(); - registry.observe_repo(&host("oyster.cafe"), did("did:plc:scallop")); - assert_eq!( - registry.host_of_repo(&did("did:plc:scallop")), - Some(host("oyster.cafe")) - ); - assert_eq!(registry.host_of_repo(&did("did:plc:limpet")), None); - } -} diff --git a/bobbin/crates/knot-ingest/src/roster.rs b/bobbin/crates/knot-ingest/src/roster.rs deleted file mode 100644 index b359bd848..000000000 --- a/bobbin/crates/knot-ingest/src/roster.rs +++ /dev/null @@ -1,602 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; - -use bobbin_edge_index::{EdgeStore, InviteIndex, InviteScope, Offer}; -use bobbin_runtime::UnixMicros; -use bobbin_types::ids::{EdgeKey, SubjectRef, nsid_static}; -use bobbin_types::knot_acl::{self, KnotHostKey}; -use jacquard_common::DefaultStr; -use jacquard_common::types::did::Did; - -use crate::registry::KnotRegistry; - -const REPO_COLLABORATOR_KIND: &str = "sh.tangled.repo.collaborator"; - -#[derive(Clone, Copy, Eq, Hash, PartialEq)] -enum Facet { - Acl, - Invite, -} - -#[derive(Clone, Eq, Hash, PartialEq)] -struct DedupKey { - facet: Facet, - repo: Did, - subject: Did, -} - -#[derive(Clone, Copy)] -pub struct AclKind<'a> { - facet: Facet, - pub repo: &'a Did, -} - -impl<'a> AclKind<'a> { - pub(crate) fn acl(repo: &'a Did) -> Self { - Self { - facet: Facet::Acl, - repo, - } - } - - pub(crate) fn invite(repo: &'a Did) -> Self { - Self { - facet: Facet::Invite, - repo, - } - } - - pub(crate) fn is_invite(self) -> bool { - self.facet == Facet::Invite - } - - pub(crate) fn listing(self) -> &'static str { - match self.facet { - Facet::Acl => "collaborators", - Facet::Invite => "collaborator invites", - } - } - - fn dedup(self, subject: Did) -> DedupKey { - DedupKey { - facet: self.facet, - repo: self.repo.clone(), - subject, - } - } -} - -pub enum Assertion<'a> { - Acl(&'a Did), - Invite { - repo: &'a Did, - invited_by: Did, - }, -} - -impl<'a> Assertion<'a> { - fn kind(&self) -> AclKind<'a> { - match self { - Self::Acl(repo) => AclKind::acl(repo), - Self::Invite { repo, .. } => AclKind::invite(repo), - } - } -} - -#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)] -pub struct Applied(u64); - -#[derive(Clone, Copy, Eq, PartialEq)] -pub enum Source { - Firehose, - Listing(Applied), -} - -struct SeenState { - at: UnixMicros, - present: bool, - applied: Applied, -} - -pub struct Roster { - store: Arc, - invites: Arc, - knot: Did, - registry: Arc, - host: KnotHostKey, - seen: HashMap, - applied: Applied, -} - -impl Roster { - pub fn new( - store: Arc, - invites: Arc, - knot: Did, - registry: Arc, - host: KnotHostKey, - ) -> Self { - Self { - store, - invites, - knot, - registry, - host, - seen: HashMap::new(), - applied: Applied::default(), - } - } - - fn invite_scope(&self, repo: &Did) -> InviteScope { - InviteScope { - knot: self.knot.clone(), - repo: repo.clone(), - } - } - - pub fn apply( - &mut self, - assertion: Assertion<'_>, - subject: Did, - at: UnixMicros, - source: Source, - ) { - let kind = assertion.kind(); - if !self.registry.repo_on_host(&self.host, kind.repo) { - return; - } - if !self.advance(kind.dedup(subject.clone()), at, source) { - return; - } - match assertion { - Assertion::Acl(repo) => knot_acl::collaborator_upsert(repo, &subject, at.raw()) - .into_iter() - .for_each(|(record, edges)| self.store.upsert_source(&record, edges)), - Assertion::Invite { repo, invited_by } => self.invites.offer( - subject, - self.invite_scope(repo), - Offer { - invited_by, - offered_at: at, - }, - ), - } - } - - pub fn applied(&self) -> Applied { - self.applied - } - - fn step(&mut self) -> Applied { - self.applied = Applied(self.applied.0.saturating_add(1)); - self.applied - } - - pub fn reap( - &mut self, - kind: AclKind<'_>, - present: &HashSet>, - horizon: Applied, - ) { - let stale: Vec> = self - .seen - .iter() - .filter(|(key, state)| { - key.facet == kind.facet - && &key.repo == kind.repo - && state.present - && state.applied <= horizon - && !present.contains(&key.subject) - }) - .map(|(key, _)| key.subject.clone()) - .collect(); - stale - .into_iter() - .for_each(|subject| self.retire(kind, subject)); - } - - pub fn purge_legacy(&self) { - self.registry - .repos(&self.host) - .into_iter() - .for_each(|repo| { - let key = EdgeKey::new(nsid_static(REPO_COLLABORATOR_KIND), SubjectRef::Did(repo)); - self.store - .sources_for(&key) - .into_iter() - .filter(|source| knot_acl::decode_knot_owned_source(source).is_none()) - .for_each(|source| self.store.remove_source(&source)); - }); - } - - pub fn retire(&mut self, kind: AclKind<'_>, subject: Did) { - match kind.facet { - Facet::Acl => knot_acl::collaborator_source(kind.repo, &subject) - .iter() - .for_each(|record| self.store.remove_source(record)), - Facet::Invite => self - .invites - .withdraw(&subject, &self.invite_scope(kind.repo)), - } - let applied = self.step(); - if let Some(state) = self.seen.get_mut(&kind.dedup(subject)) { - state.present = false; - state.applied = applied; - } - } - - fn advance(&mut self, key: DedupKey, at: UnixMicros, source: Source) -> bool { - let seen = self - .seen - .get(&key) - .map(|state| (state.at, state.present, state.applied)); - let stale = match (seen, source) { - (Some((previous, present, applied)), Source::Listing(taken)) => { - applied > taken || (present && at <= previous) - } - _ => false, - }; - if stale { - return false; - } - let applied = self.step(); - let at = seen.map_or(at, |(previous, _, _)| at.max(previous)); - self.seen.insert( - key, - SeenState { - at, - present: true, - applied, - }, - ); - true - } -} - -#[cfg(test)] -mod tests { - use super::*; - use bobbin_runtime::RuntimeHasher; - use bobbin_types::edges::Edge; - use bobbin_types::ids::{EdgeKey, SubjectRef, nsid_static}; - use jacquard_common::types::string::AtUri; - - fn store() -> Arc { - Arc::new(EdgeStore::new(RuntimeHasher::default())) - } - - fn did(s: &str) -> Did { - Did::new_owned(s).unwrap() - } - - fn at(s: &str) -> AtUri { - AtUri::new_owned(s).unwrap() - } - - fn host() -> KnotHostKey { - KnotHostKey::new("oyster.cafe") - } - - fn add_legacy_edge( - store: &EdgeStore, - kind: &'static str, - subject: &Did, - source: &AtUri, - ) { - store.upsert_source( - source, - vec![Edge { - kind: nsid_static(kind), - subject: SubjectRef::Did(subject.clone()), - source: source.clone(), - sort_micros: 0, - }], - ); - } - - fn knot() -> Did { - knot_acl::host_to_knot_did("oyster.cafe").unwrap() - } - - fn collaborator_count(store: &EdgeStore, repo: &Did) -> u64 { - store.count(&EdgeKey::new( - nsid_static("sh.tangled.repo.collaborator"), - SubjectRef::Did(repo.clone()), - )) - } - - fn empty_registry() -> Arc { - Arc::new(KnotRegistry::new()) - } - - fn invites() -> Arc { - Arc::new(InviteIndex::new()) - } - - fn bare_roster(store: Arc) -> Roster { - Roster::new(store, invites(), knot(), empty_registry(), host()) - } - - fn repo_roster( - store: Arc, - index: Arc, - repo: &Did, - ) -> Roster { - let registry = empty_registry(); - registry.observe_repo(&host(), repo.clone()); - Roster::new(store, index, knot(), registry, host()) - } - - fn indexed_roster(index: Arc, repo: &Did) -> Roster { - repo_roster(store(), index, repo) - } - - fn subjects(items: &[&Did]) -> HashSet> { - items.iter().map(|d| (*d).clone()).collect() - } - - fn collab(repo: &Did) -> Assertion<'_> { - Assertion::Acl(repo) - } - - fn offer_of<'a>(repo: &'a Did, admin: &str) -> Assertion<'a> { - Assertion::Invite { - repo, - invited_by: did(admin), - } - } - - impl Roster { - fn listed(&mut self, assertion: Assertion<'_>, subject: &Did, micros: u64) { - let source = Source::Listing(self.applied()); - self.apply(assertion, subject.clone(), UnixMicros::new(micros), source); - } - - fn firehosed(&mut self, assertion: Assertion<'_>, subject: &Did, micros: u64) { - self.apply( - assertion, - subject.clone(), - UnixMicros::new(micros), - Source::Firehose, - ); - } - } - - #[test] - fn late_add_leaves_newer_row_standing() { - let store = store(); - let repo = did("did:plc:scallop"); - let mut roster = repo_roster(store.clone(), invites(), &repo); - let who = did("did:plc:boltless"); - roster.listed(collab(&repo), &who, 5); - roster.listed(collab(&repo), &who, 1); - assert_eq!(collaborator_count(&store, &repo), 1); - } - - #[test] - fn duplicate_cursor_is_idempotent() { - let store = store(); - let repo = did("did:plc:scallop"); - let mut roster = repo_roster(store.clone(), invites(), &repo); - let who = did("did:plc:akshay"); - roster.listed(collab(&repo), &who, 10); - roster.listed(collab(&repo), &who, 10); - assert_eq!(collaborator_count(&store, &repo), 1); - } - - #[test] - fn collaborator_on_hosted_repo_is_indexed() { - let store = store(); - let repo = did("did:plc:scallop"); - let mut roster = repo_roster(store.clone(), invites(), &repo); - roster.listed(collab(&repo), &did("did:plc:olaren"), 7); - assert_eq!(collaborator_count(&store, &repo), 1); - } - - #[test] - fn reap_removes_departed_collaborator() { - let store = store(); - let repo = did("did:plc:scallop"); - let mut roster = repo_roster(store.clone(), invites(), &repo); - let stayed = did("did:plc:akshay"); - let left = did("did:plc:boltless"); - roster.listed(collab(&repo), &stayed, 10); - roster.listed(collab(&repo), &left, 20); - assert_eq!(collaborator_count(&store, &repo), 2); - - let horizon = roster.applied(); - roster.reap(AclKind::acl(&repo), &subjects(&[&stayed]), horizon); - - assert_eq!( - collaborator_count(&store, &repo), - 1, - "a collaborator absent from the authoritative snapshot is reaped" - ); - } - - #[test] - fn reap_skips_collaborator_added_after_horizon() { - let store = store(); - let repo = did("did:plc:scallop"); - let mut roster = repo_roster(store.clone(), invites(), &repo); - let horizon = roster.applied(); - roster.listed(collab(&repo), &did("did:plc:boltless"), 100); - - roster.reap(AclKind::acl(&repo), &subjects(&[]), horizon); - - assert_eq!( - collaborator_count(&store, &repo), - 1, - "a collaborator added after the snapshot horizon must survive the reap" - ); - } - - #[test] - fn purge_legacy_strips_pds_collaborator_keeps_knot_owned() { - let store = store(); - let repo = did("did:plc:scallop"); - let mut roster = repo_roster(store.clone(), invites(), &repo); - - roster.listed(collab(&repo), &did("did:plc:olaren"), 7); - add_legacy_edge( - &store, - "sh.tangled.repo.collaborator", - &repo, - &at("at://did:plc:akshay/sh.tangled.repo.collaborator/r1"), - ); - assert_eq!(collaborator_count(&store, &repo), 2); - - roster.purge_legacy(); - assert_eq!( - collaborator_count(&store, &repo), - 1, - "only the knot-owned collaborator survives the purge" - ); - } - - #[test] - fn collaborator_for_unhosted_repo_is_dropped() { - let store = store(); - let repo = did("did:plc:scallop"); - let mut roster = bare_roster(store.clone()); - roster.listed(collab(&repo), &did("did:plc:olaren"), 7); - assert_eq!( - collaborator_count(&store, &repo), - 0, - "a knot cannot assert collaborators on a repo it does not host" - ); - } - - #[test] - fn offer_reaches_index_scoped_to_its_repo_and_leaves_when_retired() { - let index = invites(); - let store = store(); - let one = did("did:plc:scallop"); - let two = did("did:plc:whelk"); - let registry = empty_registry(); - registry.observe_repo(&host(), one.clone()); - registry.observe_repo(&host(), two.clone()); - let mut roster = Roster::new(store, index.clone(), knot(), registry, host()); - let here = did("did:plc:limpet"); - let there = did("did:plc:olaren"); - - roster.listed(offer_of(&one, "did:plc:akshay"), &here, 7_000_000); - roster.listed(offer_of(&two, "did:plc:boltless"), &there, 9_000_000); - - let offered = index.outstanding(&here); - assert_eq!(offered.len(), 1); - assert_eq!( - offered[0].0, - InviteScope { - knot: knot(), - repo: one.clone() - } - ); - assert_eq!(offered[0].1.invited_by, did("did:plc:akshay")); - assert_eq!(offered[0].1.offered_at, UnixMicros::new(7_000_000)); - - roster.retire(AclKind::invite(&one), here.clone()); - assert!(index.outstanding(&here).is_empty()); - assert_eq!( - index.outstanding(&there).len(), - 1, - "withdrawing one repo's offer leaves the other standing" - ); - } - - #[test] - fn reap_drops_offer_knot_stopped_listing_and_takes_it_back_when_it_returns() { - let index = invites(); - let repo = did("did:plc:scallop"); - let mut roster = indexed_roster(index.clone(), &repo); - let stayed = did("did:plc:limpet"); - let withdrawn = did("did:plc:boltless"); - let moment = 8_000_000; - roster.listed(offer_of(&repo, "did:plc:akshay"), &stayed, 7_000_000); - roster.listed(offer_of(&repo, "did:plc:akshay"), &withdrawn, moment); - - let horizon = roster.applied(); - roster.reap(AclKind::invite(&repo), &subjects(&[&stayed]), horizon); - - assert_eq!(index.outstanding(&stayed).len(), 1); - assert!( - index.outstanding(&withdrawn).is_empty(), - "knot's filtered listing wins over firehose record it no longer reports" - ); - - roster.listed(offer_of(&repo, "did:plc:akshay"), &withdrawn, moment); - - assert_eq!( - index.outstanding(&withdrawn).len(), - 1, - "listCollaboratorInvites hides blocked invitees, and lifting the block returns offers at timestamps they always had" - ); - } - - #[test] - fn listing_fetched_before_withdrawal_leaves_offer_withdrawn() { - let index = invites(); - let repo = did("did:plc:scallop"); - let mut roster = indexed_roster(index.clone(), &repo); - let invitee = did("did:plc:limpet"); - let moment = 8_000_000; - roster.listed(offer_of(&repo, "did:plc:akshay"), &invitee, moment); - let fetched = roster.applied(); - roster.retire(AclKind::invite(&repo), invitee.clone()); - - roster.apply( - offer_of(&repo, "did:plc:akshay"), - invitee.clone(), - UnixMicros::new(moment), - Source::Listing(fetched), - ); - - assert!( - index.outstanding(&invitee).is_empty(), - "firehose delivered delete after this page was fetched, so page shows knot that has already moved on" - ); - } - - #[test] - fn firehose_reoffer_in_same_second_reaches_invitee() { - let index = invites(); - let repo = did("did:plc:scallop"); - let mut roster = indexed_roster(index.clone(), &repo); - let invitee = did("did:plc:limpet"); - let moment = 8_000_000; - - roster.firehosed(offer_of(&repo, "did:plc:akshay"), &invitee, moment); - roster.retire(AclKind::invite(&repo), invitee.clone()); - roster.firehosed(offer_of(&repo, "did:plc:olaren"), &invitee, moment); - - let outstanding = index.outstanding(&invitee); - assert_eq!( - outstanding.len(), - 1, - "knots stamp offers to the second, so withdraw and re-offer inside one second spell one cursor; firehose order settles which won" - ); - assert_eq!(outstanding[0].1.invited_by, did("did:plc:olaren")); - } - - #[test] - fn offer_applied_mid_fetch_survives_snapshot_that_missed_it() { - let index = invites(); - let repo = did("did:plc:scallop"); - let mut roster = indexed_roster(index.clone(), &repo); - let listed = did("did:plc:limpet"); - let raced = did("did:plc:olaren"); - let moment = 8_000_000; - - roster.listed(offer_of(&repo, "did:plc:akshay"), &listed, moment); - let horizon = roster.applied(); - - roster.firehosed(offer_of(&repo, "did:plc:akshay"), &raced, moment); - roster.reap(AclKind::invite(&repo), &subjects(&[&listed]), horizon); - - assert_eq!( - index.outstanding(&raced).len(), - 1, - "admins inviting two people inside one second stamp both offers alike, and the second reached bobbin after this page was fetched" - ); - } -} diff --git a/bobbin/crates/knot-ingest/src/stream.rs b/bobbin/crates/knot-ingest/src/stream.rs deleted file mode 100644 index f6731bc7c..000000000 --- a/bobbin/crates/knot-ingest/src/stream.rs +++ /dev/null @@ -1,1007 +0,0 @@ -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::Duration; - -use bobbin_knot_proxy::KnotHost; -use bobbin_runtime::{Clock, WsConn, WsMessage, WsTransport}; -use jacquard_common::DefaultStr; -use jacquard_common::types::did::Did; -use tokio_util::sync::CancellationToken; -use url::Url; - -use crate::client::authority; -use crate::firehose::{self, Frame, InviteOp}; -use crate::legacy; - -const SUBSCRIBE_REPOS_PATH: &str = "xrpc/com.atproto.sync.subscribeRepos"; -const EVENTS_PATH: &str = "events"; -const RECONNECT_INITIAL: Duration = Duration::from_secs(1); -const RECONNECT_MAX: Duration = Duration::from_secs(60); -const HEALTHY_SESSION_MIN: Duration = Duration::from_secs(15); -const LIVENESS_TIMEOUT: Duration = Duration::from_secs(90); - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum Feed { - Atproto, - Legacy, -} - -impl Feed { - fn spec(self) -> (Self, &'static str, &'static str) { - match self { - Self::Atproto => (Self::Legacy, SUBSCRIBE_REPOS_PATH, "atproto firehose"), - Self::Legacy => (Self::Atproto, EVENTS_PATH, "knot event stream"), - } - } - - fn other(self) -> Self { - self.spec().0 - } - - fn label(self) -> &'static str { - self.spec().2 - } - - pub fn live(self, clock: &dyn Clock) -> Cursor { - match self { - Self::Atproto => Cursor::start(self), - Self::Legacy => Cursor::at( - self, - i64::try_from(clock.now_unix_micros().raw().saturating_mul(1_000)) - .unwrap_or(i64::MAX), - ), - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct Cursor { - feed: Feed, - seq: i64, -} - -impl Cursor { - pub const fn start(feed: Feed) -> Self { - Self { feed, seq: 0 } - } - - pub(crate) const fn at(feed: Feed, seq: i64) -> Self { - Self { feed, seq } - } - - pub const fn feed(self) -> Feed { - self.feed - } - - pub const fn seq(self) -> i64 { - self.seq - } - - pub fn advance(&mut self, seq: i64) { - self.seq = self.seq.max(seq); - } - - fn switched(self) -> Self { - Self::start(self.feed.other()) - } - - fn read(&mut self, message: &WsMessage, handler: &dyn FeedHandler) -> Outcome { - match (self.feed, message) { - (Feed::Atproto, WsMessage::Binary(bytes)) => dispatch(bytes, handler, self), - (Feed::Legacy, WsMessage::Text(text)) => legacy::dispatch(text, handler, self), - (_, WsMessage::Text(_) | WsMessage::Binary(_)) => Outcome::Mismatch, - _ => Outcome::Idle, - } - } -} - -#[derive(Clone)] -pub struct StreamConfig { - pub ws: Arc, - pub clock: Arc, - pub cancel: CancellationToken, -} - -pub trait FeedHandler: Send + Sync + 'static { - fn roster_changed(&self); - fn repo_roster_changed(&self, repo: &Did); - fn invite_op(&self, op: InviteOp); -} - -#[derive(Debug)] -enum SessionEnd { - Cancelled, - Closed { progressed: bool }, - ConnectFailed, - Mismatch, - Resync, - Unroutable, -} - -pub(crate) enum Outcome { - Progressed, - Idle, - Mismatch, - Resync, -} - -pub async fn run_stream( - cfg: &StreamConfig, - host: &KnotHost, - mut cursor: Cursor, - handler: &dyn FeedHandler, -) { - let mut proven = false; - let mut backoff = RECONNECT_INITIAL; - loop { - if cfg.cancel.is_cancelled() { - return; - } - let started = cfg.clock.now_instant(); - let mut position = session_cursor(cursor, cfg.clock.as_ref()); - let end = run_session(cfg, host, handler, &mut position).await; - cursor = position; - proven |= feed_confirmed(&end); - if wrong_feed(&end, proven) { - let feed = cursor.feed(); - let (knot, tried, next) = (authority(host), feed.label(), feed.other().label()); - match &end { - SessionEnd::Mismatch => { - tracing::warn!(host = %knot, tried, next, "switching feeds, since this knot sent the other one's frames") - } - _ => { - tracing::info!(host = %knot, tried, next, "this knot won't serve the feed we asked for, so we try the other") - } - } - (cursor, proven) = (cursor.switched(), false); - } - match end { - SessionEnd::Cancelled | SessionEnd::Unroutable => return, - SessionEnd::Resync => { - cursor = Cursor::start(cursor.feed()); - backoff = RECONNECT_INITIAL; - } - SessionEnd::ConnectFailed - | SessionEnd::Mismatch - | SessionEnd::Closed { progressed: false } => {} - SessionEnd::Closed { progressed: true } => { - let elapsed = cfg.clock.now_instant().saturating_duration_since(started); - if session_was_healthy(&end, elapsed) { - backoff = RECONNECT_INITIAL; - } - } - } - let delay = jitter_delay(backoff, cfg.clock.now_unix_micros().raw()); - tokio::select! { - _ = cfg.cancel.cancelled() => return, - _ = cfg.clock.sleep(delay) => {} - } - backoff = (backoff * 2).min(RECONNECT_MAX); - } -} - -fn feed_confirmed(end: &SessionEnd) -> bool { - matches!(end, SessionEnd::Resync | SessionEnd::Closed { .. }) -} - -fn wrong_feed(end: &SessionEnd, proven: bool) -> bool { - matches!(end, SessionEnd::Mismatch) || matches!(end, SessionEnd::ConnectFailed if !proven) -} - -fn session_cursor(cursor: Cursor, clock: &dyn Clock) -> Cursor { - match (cursor.feed(), cursor.seq()) { - (feed @ Feed::Legacy, 0) => feed.live(clock), - _ => cursor, - } -} - -fn session_was_healthy(end: &SessionEnd, elapsed: Duration) -> bool { - matches!(end, SessionEnd::Closed { progressed: true }) && elapsed >= HEALTHY_SESSION_MIN -} - -fn jitter_delay(base: Duration, entropy: u64) -> Duration { - let frac = (entropy % 1024) as f64 / 1024.0; - base.mul_f64(0.5 + 0.5 * frac) -} - -async fn run_session( - cfg: &StreamConfig, - host: &KnotHost, - handler: &dyn FeedHandler, - cursor: &mut Cursor, -) -> SessionEnd { - let feed = cursor.feed(); - let Some(url) = feed_url(host, *cursor) else { - tracing::warn!(host = %authority(host), feed = feed.label(), cursor = cursor.seq(), "knot's own hostname won't form a websocket url"); - return SessionEnd::Unroutable; - }; - let conn = tokio::select! { - _ = cfg.cancel.cancelled() => return SessionEnd::Cancelled, - res = cfg.ws.connect(url) => match res { - Ok(conn) => conn, - Err(err) => { - tracing::warn!(host = %authority(host), feed = feed.label(), error = %err, "couldn't open knot stream"); - return SessionEnd::ConnectFailed; - } - }, - }; - let WsConn { - mut sink, - mut stream, - } = conn; - let mut progressed = false; - loop { - let message = tokio::select! { - _ = cfg.cancel.cancelled() => return SessionEnd::Cancelled, - _ = cfg.clock.sleep(LIVENESS_TIMEOUT) => { - tracing::warn!(host = %authority(host), feed = feed.label(), "nothing came down the stream, reconnecting"); - return SessionEnd::Closed { progressed }; - } - message = stream.next() => message, - }; - match message { - None => return SessionEnd::Closed { progressed }, - Some(Ok(WsMessage::Ping(payload))) => { - let _ = sink.send(WsMessage::Pong(payload)).await; - } - Some(Ok(WsMessage::Close { .. })) => return SessionEnd::Closed { progressed }, - Some(Ok(message)) => match cursor.read(&message, handler) { - Outcome::Resync => return SessionEnd::Resync, - Outcome::Mismatch => return SessionEnd::Mismatch, - Outcome::Progressed => progressed = true, - Outcome::Idle => {} - }, - Some(Err(err)) => { - tracing::warn!(host = %authority(host), feed = feed.label(), error = %err, "read failed mid-stream"); - return SessionEnd::Closed { progressed }; - } - } - } -} - -fn dispatch(data: &[u8], handler: &dyn FeedHandler, cursor: &mut Cursor) -> Outcome { - let frame = match firehose::decode_frame(data) { - Ok(frame) => frame, - Err(err) => { - log_undecodable(&err); - return Outcome::Idle; - } - }; - match frame { - Frame::Commit(commit) => { - cursor.advance(commit.seq); - dispatch_commit(&commit, handler); - Outcome::Progressed - } - _ if firehose::is_outdated_cursor(&frame) => { - handler.roster_changed(); - Outcome::Resync - } - _ if firehose::is_future_cursor(&frame) => { - tracing::warn!("firehose cursor is ahead of the knot, resuming live"); - Outcome::Resync - } - Frame::Error { name, message } => { - tracing::warn!(%name, %message, "firehose error frame"); - Outcome::Idle - } - _ => Outcome::Idle, - } -} - -fn log_undecodable(err: &firehose::FirehoseError) { - static LOGGED: AtomicBool = AtomicBool::new(false); - if LOGGED.swap(true, Ordering::Relaxed) { - tracing::debug!(error = %err, "undecodable firehose frame"); - } else { - tracing::warn!(error = %err, "undecodable firehose frame, further failures log at debug"); - } -} - -fn dispatch_commit(commit: &firehose::Commit, handler: &dyn FeedHandler) { - commit - .records - .iter() - .filter(|op| firehose::is_collaborator_invite(&op.collection)) - .for_each(|op| { - handler.repo_roster_changed(&commit.repo); - match firehose::invite_op_of(op, &commit.repo) { - Some(invite) => handler.invite_op(invite), - None => tracing::warn!( - repo = %commit.repo.as_ref(), - rkey = %op.rkey, - "invite record didn't decode, periodic reconcile will cover it" - ), - } - }); -} - -fn feed_url(host: &KnotHost, cursor: Cursor) -> Option { - let scheme = if host.url().scheme() == "https" { - "wss" - } else { - "ws" - }; - let (_, path, _) = cursor.feed().spec(); - let base = format!("{scheme}://{}/{path}", authority(host)); - let full = match cursor.seq() { - 0 => base, - seq => format!("{base}?cursor={seq}"), - }; - Url::parse(&full).ok() -} - -#[cfg(test)] -mod tests { - use std::collections::VecDeque; - use std::sync::atomic::{AtomicUsize, Ordering}; - - use bobbin_edge_index::Offer; - use bobbin_runtime::{ - NetworkError, SimClock, SystemClock, UnixMicros, WsConnectFuture, WsMessageFuture, - WsSendFuture, WsSink, WsStream, - }; - use bytes::Bytes; - use jacquard_common::types::nsid::Nsid; - use jacquard_common::types::recordkey::Rkey; - use parking_lot::Mutex; - - use super::*; - use crate::firehose::OpAction; - use crate::firehose::testcbor::*; - - #[derive(Clone, Debug, PartialEq)] - enum Entry { - Full, - Repo(Did), - Offer(InviteOp), - } - - #[derive(Default)] - struct RecordingHandler { - entries: Mutex>, - } - - impl RecordingHandler { - fn push(&self, entry: Entry) { - self.entries.lock().push(entry); - } - - fn entries(&self) -> Vec { - self.entries.lock().clone() - } - } - - impl FeedHandler for RecordingHandler { - fn roster_changed(&self) { - self.push(Entry::Full); - } - - fn repo_roster_changed(&self, repo: &Did) { - self.push(Entry::Repo(repo.clone())); - } - - fn invite_op(&self, op: InviteOp) { - self.push(Entry::Offer(op)); - } - } - - struct ScriptStream { - msgs: VecDeque, - } - impl WsStream for ScriptStream { - fn next<'a>(&'a mut self) -> WsMessageFuture<'a> { - Box::pin(async move { self.msgs.pop_front().map(Ok) }) - } - } - - struct RecordSink { - sent: Arc>>, - } - impl WsSink for RecordSink { - fn send<'a>(&'a mut self, message: WsMessage) -> WsSendFuture<'a> { - let sent = self.sent.clone(); - Box::pin(async move { - sent.lock().push(message); - Ok(()) - }) - } - } - - struct ScriptWs { - sessions: Mutex>>, - sent: Arc>>, - urls: Mutex>, - connects: AtomicUsize, - fail: bool, - fail_first: usize, - } - - impl ScriptWs { - fn new(sessions: Vec>) -> Self { - Self { - sessions: Mutex::new(sessions), - sent: Arc::new(Mutex::new(Vec::new())), - urls: Mutex::new(Vec::new()), - connects: AtomicUsize::new(0), - fail: false, - fail_first: 0, - } - } - - fn connect_count(&self) -> usize { - self.connects.load(Ordering::SeqCst) - } - } - - impl WsTransport for ScriptWs { - fn connect(&self, url: Url) -> WsConnectFuture { - let attempt = self.connects.fetch_add(1, Ordering::SeqCst); - self.urls.lock().push(url.to_string()); - if self.fail || attempt < self.fail_first { - return Box::pin(async { - Err(NetworkError::Connect("scripted failure".to_owned())) - }); - } - let mut sessions = self.sessions.lock(); - let msgs = if sessions.is_empty() { - VecDeque::new() - } else { - sessions.remove(0) - }; - drop(sessions); - let sent = self.sent.clone(); - Box::pin(async move { - Ok(WsConn { - sink: Box::new(RecordSink { sent }), - stream: Box::new(ScriptStream { msgs }), - }) - }) - } - } - - fn cfg(ws: Arc, cancel: CancellationToken) -> StreamConfig { - StreamConfig { - ws, - clock: Arc::new(SystemClock::new()), - cancel, - } - } - - fn session(frames: Vec) -> VecDeque { - VecDeque::from(frames) - } - - fn invite_commit(repo: &str, seq: i64, collection: &str, rkey: &str) -> WsMessage { - let record = unhex(FROZEN_BARE_REF); - WsMessage::Binary( - commit_frame( - repo, - seq, - vec![invite_op(collection, rkey, Some(TEST_CID.to_vec()))], - car(&[&record]), - ) - .into(), - ) - } - - fn did(s: &str) -> Did { - Did::new_owned(s).unwrap() - } - - type Script = Vec>; - - const FIREHOSE_PATH: &str = "xrpc/com.atproto.sync.subscribeRepos"; - const EVENTS_URL: &str = "ws://oyster.cafe/events"; - const COLLABORATOR_UPDATE: &str = r#"{"rkey":"3mug","nsid":"sh.tangled.repo.collaboratorUpdate", - "event":{"op":"add","subject":"did:plc:limpet","repo":"did:plc:scallop"},"created":11}"#; - const STRAY_REF: &str = r#"{"nsid":"sh.tangled.git.refUpdate","created":9}"#; - - fn text(json: &str) -> WsMessage { - WsMessage::Text(json.to_owned()) - } - - async fn drive(refusals: usize, sessions: Script) -> (Vec, Vec) { - let ws = Arc::new(ScriptWs { - fail_first: refusals, - ..ScriptWs::new(sessions) - }); - let handler = RecordingHandler::default(); - let config = cfg(ws.clone(), CancellationToken::new()); - let host = KnotHost::parse("http://oyster.cafe").unwrap(); - let stream = run_stream(&config, &host, Cursor::start(Feed::Atproto), &handler); - let _ = tokio::time::timeout(Duration::from_secs(30), stream).await; - (handler.entries(), ws.urls.lock().clone()) - } - - #[tokio::test] - async fn session_dispatches_invites_refs_pongs_and_advances_cursor() { - let handler = RecordingHandler::default(); - let repo = "did:plc:scallop"; - let ref_record = unhex(FROZEN_PUSHED_REF); - let frames = session(vec![ - invite_commit( - "did:plc:limpet", - 100, - firehose::COLLABORATOR_INVITE_COLLECTION, - "did:plc:boltless", - ), - invite_commit( - repo, - 200, - firehose::COLLABORATOR_INVITE_COLLECTION, - "did:plc:olaren", - ), - WsMessage::Ping(Bytes::from_static(b"ka")), - WsMessage::Binary( - commit_frame( - repo, - 300, - vec![git_ref_op( - "create", - "refs~2fheads~2fmain", - Some(TEST_CID.to_vec()), - )], - car(&[&ref_record]), - ) - .into(), - ), - WsMessage::Binary( - commit_frame( - repo, - 400, - vec![git_ref_op("delete", "refs~2ftags~2fv1", None)], - car(&[]), - ) - .into(), - ), - WsMessage::Close { - code: 1000, - reason: String::new(), - }, - ]); - let script = ScriptWs::new(vec![frames]); - let sent = script.sent.clone(); - let ws: Arc = Arc::new(script); - let config = cfg(ws, CancellationToken::new()); - let host = KnotHost::parse("http://oyster.cafe").unwrap(); - let mut cursor = Cursor::start(Feed::Atproto); - - let end = run_session(&config, &host, &handler, &mut cursor).await; - - assert!(matches!(end, SessionEnd::Closed { progressed: true })); - assert_eq!(cursor.seq(), 400); - assert_eq!( - handler.entries(), - vec![Entry::Repo(did("did:plc:limpet")), Entry::Repo(did(repo))] - ); - let sent = sent.lock(); - assert_eq!(sent.len(), 1); - assert!(matches!(&sent[0], WsMessage::Pong(p) if p.as_ref() == b"ka")); - } - - fn invite_record_commit( - repo: &str, - action: OpAction, - collection: &str, - rkey: &str, - record: Option>, - ) -> firehose::Commit { - firehose::Commit { - repo: did(repo), - seq: 12, - rev: "3zzzzzzzzzzzz".to_owned(), - records: vec![firehose::RecordOp { - action, - collection: Nsid::::new_owned(collection).unwrap(), - rkey: Rkey::::new_owned(rkey).unwrap(), - record: record.map(Bytes::from), - }], - } - } - - #[test] - fn invite_create_both_nudges_roster_and_delivers_offer() { - let handler = RecordingHandler::default(); - let collection = firehose::COLLABORATOR_INVITE_COLLECTION; - let commit = invite_record_commit( - "did:plc:scallop", - OpAction::Create, - collection, - "did:plc:limpet", - Some(invite_record( - collection, - "2026-06-01T00:00:00Z", - "did:plc:akshay", - )), - ); - - dispatch_commit(&commit, &handler); - - assert_eq!( - handler.entries(), - vec![ - Entry::Repo(did("did:plc:scallop")), - Entry::Offer(InviteOp { - repo: did("did:plc:scallop"), - invitee: did("did:plc:limpet"), - offer: Some(Offer { - invited_by: did("did:plc:akshay"), - offered_at: UnixMicros::new(1_780_272_000_000_000), - }), - }), - ], - "one record is both roster signal and standing offer" - ); - } - - #[test] - fn invite_delete_delivers_op_without_offer() { - let handler = RecordingHandler::default(); - let commit = invite_record_commit( - "did:plc:scallop", - OpAction::Delete, - firehose::COLLABORATOR_INVITE_COLLECTION, - "did:plc:olaren", - None, - ); - - dispatch_commit(&commit, &handler); - - assert_eq!( - handler.entries(), - vec![ - Entry::Repo(did("did:plc:scallop")), - Entry::Offer(InviteOp { - repo: did("did:plc:scallop"), - invitee: did("did:plc:olaren"), - offer: None, - }), - ] - ); - } - - #[tokio::test] - async fn outdated_cursor_info_ends_the_session_in_resync() { - let handler = RecordingHandler::default(); - let mut frames = encode_one(&Val::Map(vec![(t("t"), t("#info"))])); - frames.extend(encode_one(&Val::Map(vec![( - t("name"), - t("OutdatedCursor"), - )]))); - let ws: Arc = Arc::new(ScriptWs::new(vec![session(vec![ - WsMessage::Binary(frames.into()), - WsMessage::Close { - code: 1000, - reason: String::new(), - }, - ])])); - let config = cfg(ws, CancellationToken::new()); - let host = KnotHost::parse("http://oyster.cafe").unwrap(); - let mut cursor = Cursor::at(Feed::Atproto, 42); - - let end = run_session(&config, &host, &handler, &mut cursor).await; - - assert!(matches!(end, SessionEnd::Resync)); - assert_eq!(handler.entries(), vec![Entry::Full]); - } - - #[tokio::test] - async fn future_cursor_error_frame_resumes_live() { - let handler = RecordingHandler::default(); - let mut error = encode_one(&Val::Map(vec![(t("op"), Val::Nint(0))])); - error.extend(encode_one(&Val::Map(vec![ - (t("error"), t("FutureCursor")), - (t("message"), t("the cursor is ahead of this stream")), - ]))); - let ws: Arc = Arc::new(ScriptWs::new(vec![session(vec![ - WsMessage::Binary(error.into()), - WsMessage::Close { - code: 1000, - reason: String::new(), - }, - ])])); - let config = cfg(ws, CancellationToken::new()); - let host = KnotHost::parse("http://oyster.cafe").unwrap(); - let mut cursor = Cursor::at(Feed::Atproto, 42); - - let end = run_session(&config, &host, &handler, &mut cursor).await; - - assert!(matches!(end, SessionEnd::Resync)); - assert!(handler.entries().is_empty()); - } - - #[tokio::test] - async fn undecodable_frames_are_skipped_and_stream_keeps_delivering() { - let handler = RecordingHandler::default(); - let ws: Arc = Arc::new(ScriptWs::new(vec![session(vec![ - WsMessage::Binary(Bytes::from_static(b"\xff\xffnot cbor")), - invite_commit( - "did:plc:limpet", - 500, - firehose::COLLABORATOR_INVITE_COLLECTION, - "did:plc:boltless", - ), - WsMessage::Close { - code: 1000, - reason: String::new(), - }, - ])])); - let config = cfg(ws, CancellationToken::new()); - let host = KnotHost::parse("http://oyster.cafe").unwrap(); - let mut cursor = Cursor::start(Feed::Atproto); - - let end = run_session(&config, &host, &handler, &mut cursor).await; - - assert!(matches!(end, SessionEnd::Closed { progressed: true })); - assert_eq!(cursor.seq(), 500); - assert_eq!(handler.entries(), vec![Entry::Repo(did("did:plc:limpet"))]); - } - - #[tokio::test(start_paused = true)] - async fn run_stream_resets_cursor_and_reconnects_on_outdated_cursor() { - let handler = Arc::new(RecordingHandler::default()); - let mut frames = encode_one(&Val::Map(vec![(t("t"), t("#info"))])); - frames.extend(encode_one(&Val::Map(vec![( - t("name"), - t("OutdatedCursor"), - )]))); - let ws = Arc::new(ScriptWs::new(vec![ - session(vec![WsMessage::Binary(frames.into())]), - session(vec![invite_commit( - "did:plc:limpet", - 700, - firehose::COLLABORATOR_INVITE_COLLECTION, - "did:plc:boltless", - )]), - session(vec![]), - ])); - let cancel = CancellationToken::new(); - let config = cfg(ws.clone(), cancel.clone()); - let host = KnotHost::parse("http://oyster.cafe").unwrap(); - let task_handler = handler.clone(); - let task = tokio::spawn(async move { - run_stream( - &config, - &host, - Cursor::at(Feed::Atproto, 900), - task_handler.as_ref(), - ) - .await; - }); - for _ in 0..30 { - tokio::time::advance(Duration::from_secs(1)).await; - if handler.entries().len() >= 2 { - break; - } - } - cancel.cancel(); - task.await.unwrap(); - - assert_eq!( - handler.entries(), - vec![Entry::Full, Entry::Repo(did("did:plc:limpet"))], - "an outdated cursor resyncs to live and keeps delivering after the reconnect" - ); - assert!( - ws.connect_count() >= 2, - "the stream must dial again after a resync" - ); - } - - #[tokio::test(start_paused = true)] - async fn refused_firehose_falls_back_to_knot_event_stream() { - let sessions = vec![session(vec![text(COLLABORATOR_UPDATE)]), session(vec![])]; - - let (entries, urls) = drive(1, sessions).await; - - assert_eq!( - entries, - vec![Entry::Repo(did("did:plc:scallop"))], - "legacy frame arrives after the refusal" - ); - assert!(urls[0].ends_with(FIREHOSE_PATH), "firehose is tried first"); - let live = urls[1].strip_prefix(&format!("{EVENTS_URL}?cursor=")); - let started_live = live.is_some_and(|at| at.parse::().is_ok_and(|nanos| nanos > 0)); - assert!( - started_live, - "fresh legacy subscription starts at the present" - ); - } - - #[tokio::test(start_paused = true)] - async fn text_frame_on_firehose_switches_to_legacy_feed() { - let stray = session(vec![text(STRAY_REF)]); - let collaborator = session(vec![text(COLLABORATOR_UPDATE)]); - - let (entries, urls) = drive(0, vec![stray, collaborator]).await; - - assert_eq!( - entries, - vec![Entry::Repo(did("did:plc:scallop"))], - "legacy frame shows up once we switch" - ); - assert!( - urls[1].starts_with(EVENTS_URL), - "legacy path follows, got {}", - urls[1] - ); - } - - #[test] - fn wrong_kind_frame_reads_as_mismatch() { - let handler = RecordingHandler::default(); - - [ - (Cursor::start(Feed::Atproto), text("{}")), - (Cursor::start(Feed::Legacy), WsMessage::Binary(Bytes::new())), - ] - .into_iter() - .for_each(|(mut cursor, frame)| { - assert!(matches!(cursor.read(&frame, &handler), Outcome::Mismatch)); - assert_eq!(cursor.seq(), 0, "mismatched frame leaves the cursor alone"); - }); - } - - #[test] - fn only_fresh_legacy_subscription_starts_at_present() { - let clock = SystemClock::new(); - let live = session_cursor(Cursor::start(Feed::Legacy), &clock); - assert_eq!(live.feed(), Feed::Legacy); - assert!(live.seq() > 0, "fresh legacy feed resumes from now"); - - [ - Cursor::at(Feed::Legacy, 1788245839553422000), - Cursor::start(Feed::Atproto), - Cursor::at(Feed::Atproto, 42), - ] - .into_iter() - .for_each(|at| { - assert_eq!(session_cursor(at, &clock), at, "resumed as it stands"); - }); - } - - #[test] - fn switching_feeds_never_reuses_the_other_clocks_number() { - let legacy_nanos = 1788245839553422000; - let clock = SimClock::at(UnixMicros::new(1_788_245_839_553_422)); - - let from_legacy = Cursor::at(Feed::Legacy, legacy_nanos).switched(); - assert_eq!(from_legacy.feed(), Feed::Atproto); - assert_eq!( - from_legacy.seq(), - 0, - "a wall clock reading is not a firehose sequence number" - ); - - let from_atproto = Cursor::at(Feed::Atproto, 42).switched(); - assert_eq!(from_atproto.feed(), Feed::Legacy); - assert_eq!( - from_atproto.seq(), - 0, - "a firehose sequence number is not a wall clock reading" - ); - assert!( - session_cursor(from_atproto, &clock).seq() >= legacy_nanos, - "the legacy feed opens at the present instead" - ); - } - - #[test] - fn session_end_says_whether_feed_is_proven_or_wrong() { - [ - (SessionEnd::Resync, true, false), - (SessionEnd::Closed { progressed: true }, true, false), - (SessionEnd::Closed { progressed: false }, true, false), - (SessionEnd::ConnectFailed, false, true), - (SessionEnd::Mismatch, false, true), - (SessionEnd::Unroutable, false, false), - ] - .into_iter() - .for_each(|(end, proves, wrong)| { - assert_eq!( - feed_confirmed(&end), - proves, - "{end:?} proves the feed: {proves}" - ); - assert_eq!( - wrong_feed(&end, false), - wrong, - "{end:?} is the wrong feed: {wrong}" - ); - }); - - assert!( - !wrong_feed(&SessionEnd::ConnectFailed, true), - "proven feed is kept" - ); - } - - #[tokio::test] - async fn pre_cancelled_stream_returns_without_connecting() { - let handler = RecordingHandler::default(); - let ws: Arc = Arc::new(ScriptWs { - fail: true, - ..ScriptWs::new(Vec::new()) - }); - let cancel = CancellationToken::new(); - cancel.cancel(); - let config = cfg(ws, cancel); - let host = KnotHost::parse("http://oyster.cafe").unwrap(); - - run_stream(&config, &host, Cursor::start(Feed::Atproto), &handler).await; - } - - #[test] - fn each_feed_url_has_scheme_path_and_cursor() { - let plain = KnotHost::parse("http://oyster.cafe").unwrap(); - let secure = KnotHost::parse("https://nel.pet").unwrap(); - - [ - ( - &plain, - Cursor::start(Feed::Atproto), - format!("ws://oyster.cafe/{FIREHOSE_PATH}"), - ), - ( - &plain, - Cursor::at(Feed::Atproto, 42), - format!("ws://oyster.cafe/{FIREHOSE_PATH}?cursor=42"), - ), - (&plain, Cursor::start(Feed::Legacy), EVENTS_URL.to_owned()), - ( - &secure, - Cursor::at(Feed::Atproto, 7), - format!("wss://nel.pet/{FIREHOSE_PATH}?cursor=7"), - ), - ( - &secure, - Cursor::at(Feed::Legacy, 7), - "wss://nel.pet/events?cursor=7".to_owned(), - ), - ] - .into_iter() - .for_each(|(host, cursor, wanted)| { - assert_eq!(feed_url(host, cursor).unwrap().as_str(), wanted); - }); - } - - #[test] - fn only_a_long_lived_session_resets_backoff() { - assert!(session_was_healthy( - &SessionEnd::Closed { progressed: true }, - HEALTHY_SESSION_MIN - )); - assert!( - !session_was_healthy( - &SessionEnd::Closed { progressed: true }, - HEALTHY_SESSION_MIN - Duration::from_millis(1) - ), - "a knot that flaps faster than the healthy floor must keep backing off" - ); - assert!(!session_was_healthy( - &SessionEnd::Closed { progressed: false }, - Duration::from_secs(3600) - )); - assert!(!session_was_healthy( - &SessionEnd::ConnectFailed, - Duration::from_secs(3600) - )); - assert!(!session_was_healthy( - &SessionEnd::Resync, - Duration::from_secs(3600) - )); - } - - #[test] - fn jitter_delay_stays_within_half_to_full_window() { - let base = Duration::from_secs(8); - [0u64, 1, 511, 512, 1023, 1024, u64::MAX] - .into_iter() - .for_each(|entropy| { - let delay = jitter_delay(base, entropy); - assert!(delay >= base / 2, "delay {delay:?} below half of {base:?}"); - assert!(delay <= base, "delay {delay:?} above {base:?}"); - }); - } -} diff --git a/bobbin/crates/xrpc/tests/await_record_e2e.rs b/bobbin/crates/xrpc/tests/await_record_e2e.rs index a083dd9b9..4cdedfc03 100644 --- a/bobbin/crates/xrpc/tests/await_record_e2e.rs +++ b/bobbin/crates/xrpc/tests/await_record_e2e.rs @@ -107,8 +107,6 @@ async fn a_record_off_the_stream_answers_an_await() { disconnects: None, warming_shadow: None, warming_buffer: None, - knot_registry: None, - knot_gate: None, settlements: Some(settlements.clone()), analytics: None, }; diff --git a/bobbin/example.toml b/bobbin/example.toml index ffaa91965..0a0869892 100644 --- a/bobbin/example.toml +++ b/bobbin/example.toml @@ -175,23 +175,6 @@ # Can also be specified via environment variable `BOBBIN_CODESEARCH_ZOEKT_URL`. #zoekt_url = -[knot] -# Whether to allow the knot proxy to dial private/loopback addresses. Off in -# production - on for local testing against a knotserver on localhost. -# -# Can also be specified via environment variable `BOBBIN_KNOT_ALLOW_PRIVATE`. -# -# Default value: false -#allow_private = false - -# Require https on knot hosts. Disable only when proxying to a local -# knot for development. -# -# Can also be specified via environment variable `BOBBIN_KNOT_REQUIRE_HTTPS`. -# -# Default value: true -#require_https = true - [mirror] # Origin of the mirror (gitmirror) XRPC server. # diff --git a/nix/Cargo.nix b/nix/Cargo.nix index 68bce476e..83fd243c9 100644 --- a/nix/Cargo.nix +++ b/nix/Cargo.nix @@ -83,16 +83,6 @@ rec { # File a bug if you depend on any for non-debug work! debug = internal.debugCrate { inherit packageId; }; }; - "bobbin-knot-ingest" = rec { - packageId = "bobbin-knot-ingest"; - build = internal.buildRustCrateWithFeatures { - packageId = "bobbin-knot-ingest"; - }; - - # Debug support which might change between releases. - # File a bug if you depend on any for non-debug work! - debug = internal.debugCrate { inherit packageId; }; - }; "bobbin-knot-proxy" = rec { packageId = "bobbin-knot-proxy"; build = internal.buildRustCrateWithFeatures { @@ -2930,6 +2920,10 @@ rec { packageId = "axum"; features = [ "macros" ]; } + { + name = "base64"; + packageId = "base64"; + } { name = "bobbin-edge-index"; packageId = "bobbin-edge-index"; @@ -2938,10 +2932,6 @@ rec { name = "bobbin-ingest"; packageId = "bobbin-ingest"; } - { - name = "bobbin-knot-ingest"; - packageId = "bobbin-knot-ingest"; - } { name = "bobbin-knot-proxy"; packageId = "bobbin-knot-proxy"; @@ -2994,6 +2984,30 @@ rec { packageId = "jacquard-identity"; features = [ "cache" ]; } + { + name = "opentelemetry"; + packageId = "opentelemetry"; + usesDefaultFeatures = false; + features = [ "logs" "metrics" ]; + } + { + name = "opentelemetry-appender-tracing"; + packageId = "opentelemetry-appender-tracing"; + usesDefaultFeatures = false; + features = [ "experimental_span_attributes" ]; + } + { + name = "opentelemetry-otlp"; + packageId = "opentelemetry-otlp"; + usesDefaultFeatures = false; + features = [ "http-proto" "reqwest-blocking-client" "logs" "metrics" "internal-logs" ]; + } + { + name = "opentelemetry_sdk"; + packageId = "opentelemetry_sdk"; + usesDefaultFeatures = false; + features = [ "logs" "metrics" ]; + } { name = "rustls"; packageId = "rustls"; @@ -3058,11 +3072,22 @@ rec { } ]; devDependencies = [ + { + name = "opentelemetry_sdk"; + packageId = "opentelemetry_sdk"; + usesDefaultFeatures = false; + features = [ "logs" "metrics" "testing" ]; + } { name = "tokio"; packageId = "tokio"; features = [ "macros" "rt-multi-thread" "time" "signal" "io-util" "net" "sync" "test-util" ]; } + { + name = "tower"; + packageId = "tower"; + features = [ "util" "limit" "load-shed" ]; + } ]; }; @@ -3204,10 +3229,6 @@ rec { name = "bobbin-edge-index"; packageId = "bobbin-edge-index"; } - { - name = "bobbin-knot-ingest"; - packageId = "bobbin-knot-ingest"; - } { name = "bobbin-record-lru"; packageId = "bobbin-record-lru"; @@ -3245,6 +3266,18 @@ rec { name = "jacquard-common"; packageId = "jacquard-common"; } + { + name = "opentelemetry"; + packageId = "opentelemetry"; + usesDefaultFeatures = false; + features = [ "logs" "metrics" ]; + } + { + name = "reqwest"; + packageId = "reqwest 0.13.1"; + usesDefaultFeatures = false; + features = [ "rustls" "http2" "json" "gzip" "stream" ]; + } { name = "scc"; packageId = "scc"; @@ -3286,6 +3319,11 @@ rec { packageId = "url"; features = [ "serde" ]; } + { + name = "uuid"; + packageId = "uuid"; + features = [ "v5" ]; + } ]; devDependencies = [ { @@ -3294,117 +3332,20 @@ rec { usesDefaultFeatures = false; features = [ "cargo_bench_support" ]; } - { - name = "tokio"; - packageId = "tokio"; - features = [ "macros" "rt-multi-thread" "time" "signal" "io-util" "net" "sync" "test-util" ]; - } - { - name = "tokio-util"; - packageId = "tokio-util"; - features = [ "rt" ]; - } - { - name = "tracing-subscriber"; - packageId = "tracing-subscriber"; - features = [ "env-filter" "fmt" "json" ]; - } - { - name = "wiremock"; - packageId = "wiremock"; - } - ]; - - }; - "bobbin-knot-ingest" = rec { - crateName = "bobbin-knot-ingest"; - version = "0.0.1"; - edition = "2024"; - src = lib.cleanSourceWith { filter = sourceFilter; src = ../bobbin/crates/knot-ingest; }; - libName = "bobbin_knot_ingest"; - dependencies = [ - { - name = "bobbin-edge-index"; - packageId = "bobbin-edge-index"; - } - { - name = "bobbin-knot-proxy"; - packageId = "bobbin-knot-proxy"; - } - { - name = "bobbin-runtime"; - packageId = "bobbin-runtime"; - } - { - name = "bobbin-types"; - packageId = "bobbin-types"; - } - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "chrono"; - packageId = "chrono"; - features = [ "serde" ]; - } - { - name = "cid"; - packageId = "cid"; - } - { - name = "futures"; - packageId = "futures"; - } { name = "http"; packageId = "http"; } { - name = "jacquard-api"; - packageId = "jacquard-api"; - features = [ "com_atproto" "streaming" ]; - } - { - name = "jacquard-common"; - packageId = "jacquard-common"; - } - { - name = "knot-capability"; - packageId = "knot-capability"; - } - { - name = "parking_lot"; - packageId = "parking_lot"; - } - { - name = "reqwest"; - packageId = "reqwest 0.13.1"; + name = "opentelemetry_sdk"; + packageId = "opentelemetry_sdk"; usesDefaultFeatures = false; - features = [ "rustls" "http2" "json" "gzip" "stream" ]; - } - { - name = "serde"; - packageId = "serde"; - features = [ "derive" ]; - } - { - name = "serde_ipld_dagcbor"; - packageId = "serde_ipld_dagcbor"; - } - { - name = "serde_json"; - packageId = "serde_json"; - features = [ "raw_value" ]; - } - { - name = "thiserror"; - packageId = "thiserror 2.0.18"; + features = [ "logs" "metrics" "testing" ]; } { name = "tokio"; packageId = "tokio"; - features = [ "macros" "rt-multi-thread" "time" "signal" "io-util" "net" "sync" ]; + features = [ "macros" "rt-multi-thread" "time" "signal" "io-util" "net" "sync" "test-util" ]; } { name = "tokio-util"; @@ -3412,20 +3353,9 @@ rec { features = [ "rt" ]; } { - name = "tracing"; - packageId = "tracing"; - } - { - name = "url"; - packageId = "url"; - features = [ "serde" ]; - } - ]; - devDependencies = [ - { - name = "tokio"; - packageId = "tokio"; - features = [ "macros" "rt-multi-thread" "time" "signal" "io-util" "net" "sync" "macros" "rt-multi-thread" "test-util" ]; + name = "tracing-subscriber"; + packageId = "tracing-subscriber"; + features = [ "env-filter" "fmt" "json" ]; } { name = "wiremock"; @@ -9871,7 +9801,7 @@ rec { name = "opentelemetry"; packageId = "opentelemetry"; usesDefaultFeatures = false; - features = [ "logs" ]; + features = [ "logs" "metrics" ]; } { name = "opentelemetry-appender-tracing"; @@ -9883,13 +9813,13 @@ rec { name = "opentelemetry-otlp"; packageId = "opentelemetry-otlp"; usesDefaultFeatures = false; - features = [ "http-proto" "reqwest-blocking-client" "logs" ]; + features = [ "http-proto" "reqwest-blocking-client" "logs" "metrics" ]; } { name = "opentelemetry_sdk"; packageId = "opentelemetry_sdk"; usesDefaultFeatures = false; - features = [ "logs" ]; + features = [ "logs" "metrics" ]; } { name = "reqwest"; @@ -22814,6 +22744,12 @@ rec { optional = true; usesDefaultFeatures = false; } + { + name = "tracing"; + packageId = "tracing"; + optional = true; + usesDefaultFeatures = false; + } ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" "futures" ]; @@ -22828,7 +22764,7 @@ rec { "trace" = [ "futures" "thiserror" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "futures" "futures-core" "futures-sink" "logs" "metrics" "pin-project-lite" "thiserror" "trace" ]; + resolvedDefaultFeatures = [ "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "testing" "thiserror" "trace" "tracing" ]; }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; @@ -23025,7 +22961,7 @@ rec { "zstd-http" = [ "zstd" ]; "zstd-tonic" = [ "tonic/zstd" ]; }; - resolvedDefaultFeatures = [ "http" "http-proto" "httpdate" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "trace" ]; + resolvedDefaultFeatures = [ "http" "http-proto" "httpdate" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "trace" ]; }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; @@ -23134,6 +23070,20 @@ rec { optional = true; usesDefaultFeatures = false; } + { + name = "tokio"; + packageId = "tokio"; + optional = true; + usesDefaultFeatures = false; + } + ]; + devDependencies = [ + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + features = [ "macros" "rt-multi-thread" ]; + } ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" ]; @@ -23163,7 +23113,7 @@ rec { "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" "dep:futures-channel" "dep:futures-executor" "dep:futures-util" "dep:thiserror" ]; "url" = [ "dep:url" ]; }; - resolvedDefaultFeatures = [ "logs" "metrics" "percent-encoding" "rand" "trace" ]; + resolvedDefaultFeatures = [ "internal-logs" "logs" "metrics" "percent-encoding" "rand" "testing" "tokio" "trace" ]; }; "ordered-float 2.10.1" = rec { crateName = "ordered-float"; @@ -30054,6 +30004,19 @@ rec { "zeroize" = [ "dep:zeroize" ]; }; }; + "sha1_smol" = rec { + crateName = "sha1_smol"; + version = "1.0.1"; + edition = "2018"; + sha256 = "0pbh2xjfnzgblws3hims0ib5bphv7r5rfdpizyh51vnzvnribymv"; + authors = [ + "Armin Ronacher " + ]; + features = { + "serde" = [ "dep:serde" ]; + "std" = [ "alloc" ]; + }; + }; "sha2 0.10.9" = rec { crateName = "sha2"; version = "0.10.9"; @@ -34776,6 +34739,12 @@ rec { optional = true; usesDefaultFeatures = false; } + { + name = "sha1_smol"; + packageId = "sha1_smol"; + optional = true; + usesDefaultFeatures = false; + } { name = "wasm-bindgen"; packageId = "wasm-bindgen"; @@ -34816,7 +34785,7 @@ rec { "v7" = [ "rng" ]; "zerocopy" = [ "dep:zerocopy" ]; }; - resolvedDefaultFeatures = [ "default" "rng" "serde" "std" "v4" ]; + resolvedDefaultFeatures = [ "default" "rng" "serde" "sha1" "std" "v4" "v5" ]; }; "valuable" = rec { crateName = "valuable"; @@ -39705,15 +39674,7 @@ rec { ) crateConfigs; target = makeTarget pkgs.stdenv.hostPlatform; - # Build-time dependency graph (for proc-macros and build - # dependencies). When not cross-compiling it equals the host - # graph, so reuse `self`; otherwise build it for - # `pkgs.buildPackages`. - build = - if pkgs.stdenv.buildPlatform.config == pkgs.stdenv.hostPlatform.config then - self - else - mkBuiltByPackageIdByPkgs pkgs.buildPackages; + build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; }; in self; @@ -39729,35 +39690,38 @@ rec { devDependencies = lib.optionals (runTests && packageId == rootPackageId) ( crateConfig'.devDependencies or [ ] ); - # Enabled (platform- and feature-filtered) dependency lists, reused - # for both derivation wiring and the crate renames below. - enabledDependencies = filterEnabledDependencies { + dependencies = dependencyDerivations { inherit features; inherit (self) target; + buildByPackageId = + depPackageId: + # proc_macro crates must be compiled for the build architecture + if crateConfigs.${depPackageId}.procMacro or false then + self.build.crates.${depPackageId} + else + self.crates.${depPackageId}; dependencies = (crateConfig.dependencies or [ ]) ++ devDependencies; }; - enabledBuildDependencies = filterEnabledDependencies { + buildDependencies = dependencyDerivations { inherit features; inherit (self.build) target; + buildByPackageId = depPackageId: self.build.crates.${depPackageId}; dependencies = crateConfig.buildDependencies or [ ]; }; - dependencies = map - ( - dependency: - # proc_macro crates must be compiled for the build architecture - if crateConfigs.${dependency.packageId}.procMacro or false then - self.build.crates.${dependency.packageId} - else - self.crates.${dependency.packageId} - ) - enabledDependencies; - buildDependencies = map - (dependency: self.build.crates.${dependency.packageId}) - enabledBuildDependencies; - # Order (build dependencies, then normal dependencies) feeds the - # crateRenames grouping below. dependenciesWithRenames = - lib.filter (d: d ? "rename") (enabledBuildDependencies ++ enabledDependencies); + let + buildDeps = filterEnabledDependencies { + inherit features; + inherit (self) target; + dependencies = crateConfig.dependencies or [ ] ++ devDependencies; + }; + hostDeps = filterEnabledDependencies { + inherit features; + inherit (self.build) target; + dependencies = crateConfig.buildDependencies or [ ]; + }; + in + lib.filter (d: d ? "rename") (hostDeps ++ buildDeps); # Crate renames have the form: # # { @@ -39929,17 +39893,6 @@ rec { corresponding feature sets are merged. Features in rust are additive. */ mergePackageFeatures = - args: builtins.mapAttrs (_packageId: builtins.attrNames) (mergePackageFeaturesImpl args); - - /* - Core of the feature-resolution fixpoint. The cache (`featuresByPackageId`) - maps each packageId to a feature *set* (an attrset `feature -> 1`) rather - than a sorted list, so the fold merges with `//` and detects convergence - with attrset equality instead of re-concatenating and re-sorting the - accumulated feature list on every step. `mergePackageFeatures` projects the - result back to canonical sorted lists. - */ - mergePackageFeaturesImpl = { crateConfigs ? crates , packageId , rootPackageId ? packageId @@ -39988,15 +39941,14 @@ rec { cache: { packageId, features }: let - cacheFeatures = cache.${packageId} or { }; - # `features` is the (small) incoming list; merge it into the set. - combinedFeatures = cacheFeatures // listToSet features; + cacheFeatures = cache.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ features); in - if cache ? ${packageId} && cacheFeatures == combinedFeatures then + if cache ? ${packageId} && cache.${packageId} == combinedFeatures then cache else - mergePackageFeaturesImpl { - features = builtins.attrNames combinedFeatures; + mergePackageFeatures { + features = combinedFeatures; featuresByPackageId = cache; inherit crateConfigs @@ -40009,8 +39961,8 @@ rec { ); cacheWithSelf = let - cacheFeatures = featuresByPackageId.${packageId} or { }; - combinedFeatures = cacheFeatures // listToSet enabledFeatures; + cacheFeatures = featuresByPackageId.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); in featuresByPackageId // { @@ -40037,31 +39989,27 @@ rec { assert (builtins.isList features); assert (builtins.isAttrs target); - let - # Identical for every dep in this call; build the predicate arg once. - targetArgs = { inherit features target; }; - in lib.filter ( dep: - (dep.target or (features: true)) targetArgs + let + targetFunc = dep.target or (features: true); + in + targetFunc { inherit features target; } && (!(dep.optional or false) || builtins.any (doesFeatureEnableDependency dep) features) ) dependencies; # Returns whether the given feature should enable the given dependency. doesFeatureEnableDependency = - dependency: - # Callers partially apply this once per dependency, then test every feature, - # so hoist the dep-invariant strings out of the per-feature comparison. + dependency: feature: let name = dependency.rename or dependency.name; - depName = "dep:" + name; prefix = "${name}/"; len = builtins.stringLength prefix; + startsWithPrefix = builtins.substring 0 len feature == prefix; in - feature: - feature == name || feature == depName || builtins.substring 0 len feature == prefix; + feature == name || feature == "dep:" + name || startsWithPrefix; /* Returns the expanded features for the given inputFeatures by applying the @@ -40075,24 +40023,29 @@ rec { assert (builtins.isAttrs featureMap); assert (builtins.isList inputFeatures); let - # Transitive closure of `inputFeatures` under `featureMap`. `seen` - # tracks already-expanded features so each is visited at most once; - # this also breaks feature cycles (issue #209). expandFeaturesNoCycle = - seen: features: - builtins.foldl' - ( - acc: feature: - if acc ? ${feature} then - acc - else - expandFeaturesNoCycle (acc // { ${feature} = 1; }) (featureMap.${feature} or [ ]) - ) - seen - features; - seen = expandFeaturesNoCycle { } inputFeatures; + oldSeen: inputFeatures: + if inputFeatures != [ ] then + let + # The feature we're currently expanding. + feature = builtins.head inputFeatures; + # All the features we've seen/expanded so far, including the one + # we're currently processing. + seen = oldSeen // { + ${feature} = 1; + }; + # Expand the feature but be careful to not re-introduce a feature + # that we've already seen: this can easily cause a cycle, see issue + # #209. + enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); + in + [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) + # No more features left, nothing to expand to. + else + [ ]; + outFeatures = expandFeaturesNoCycle { } inputFeatures; in - sortedUnique (builtins.attrNames seen); + sortedUnique outFeatures; /* This function adds optional dependencies as features if they are enabled @@ -40151,19 +40104,16 @@ rec { in defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - # Builds a feature set (attrset `feature -> 1`) from a list of feature names. - listToSet = - features: - builtins.listToAttrs (map (feature: { name = feature; value = 1; }) features); - # Sorts and removes duplicates from a list of strings. sortedUnique = features: assert (builtins.isList features); - # attrNames returns keys in ascending string order, so the result is - # sorted and deduplicated. The feature-merge fixpoint relies on this - # canonical order to detect convergence. - builtins.attrNames (listToSet features); + assert (builtins.all builtins.isString features); + let + outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; + outFeaturesUnique = builtins.attrNames outFeaturesSet; + in + builtins.sort (a: b: a < b) outFeaturesUnique; deprecationWarning = message: value: -- 2.54.0