From 52af26604dcbb37fd0aa413c39c0926cd1ddb1c9 Mon Sep 17 00:00:00 2001 From: Lewis Date: Thu, 28 May 2026 06:32:03 +0000 Subject: [PATCH] bobbin: inline bobbin into monorepo! Lewis: May this revision serve well! --- .gitignore | 3 +-- Cargo.lock | 5072 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- bobbin/containerfiles/bobbin.Containerfile | 17 +++++++++++++++++ bobbin/containerfiles/hydrant.Containerfile | 14 ++++++++++++++ bobbin/containerfiles/slingshot.Containerfile | 17 +++++++++++++++++ bobbin/crates/bobbin-sim/Cargo.toml | 38 ++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/src/determinism.rs | 249 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/src/lib.rs | 13 +++++++++++++ bobbin/crates/bobbin-sim/src/main.rs | 382 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/src/report.rs | 37 +++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/src/runtime.rs | 189 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/src/trace_capture.rs | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/src/workload.rs | 35 +++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/src/workloads/cancel_mid_hydration.rs | 285 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/src/workloads/cold_start_under_live_load.rs | 220 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/src/workloads/concurrent_reads_during_replay.rs | 275 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/src/workloads/frame_burst.rs | 190 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/src/workloads/hydrant_disconnect_barrage.rs | 257 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/src/workloads/mod.rs | 16 ++++++++++++++++ bobbin/crates/bobbin-sim/src/workloads/slingshot_flap.rs | 418 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/src/workloads/util.rs | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/tests/determinism_leak.rs | 236 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/tests/reconnect_replay.rs | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/tests/warming_buffer.rs | 246 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin-sim/tests/warming_shadow.rs | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin/Cargo.toml | 40 ++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin/src/config.rs | 471 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin/src/main.rs | 485 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin/src/mem/cgroup.rs | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin/src/mem/mod.rs | 9 +++++++++ bobbin/crates/bobbin/src/mem/report.rs | 186 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin/src/mem/sizing.rs | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/bobbin/src/mem/watcher.rs | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/edge-index/Cargo.toml | 17 +++++++++++++++++ bobbin/crates/edge-index/src/coverage.rs | 267 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/edge-index/src/lib.rs | 1563 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/edge-index/src/state_index.rs | 453 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/edge-index/tests/bucket_scaling.rs | 231 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/ingest/Cargo.toml | 39 +++++++++++++++++++++++++++++++++++++++ bobbin/crates/ingest/benches/json_decode.rs | 245 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/ingest/examples/smoke.rs | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/ingest/src/frame.rs | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/ingest/src/lib.rs | 3112 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/ingest/src/resolver.rs | 1 + bobbin/crates/ingest/src/shadow.rs | 172 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/ingest/src/warming.rs | 526 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/knot-proxy/Cargo.toml | 22 ++++++++++++++++++++++ bobbin/crates/knot-proxy/src/breaker.rs | 272 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/knot-proxy/src/dns.rs | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/knot-proxy/src/host.rs | 403 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/knot-proxy/src/lib.rs | 779 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/record-lru/Cargo.toml | 16 ++++++++++++++++ bobbin/crates/record-lru/src/lib.rs | 281 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/record-lru/src/record.dict | 0 bobbin/crates/resolver/Cargo.toml | 23 +++++++++++++++++++++++ bobbin/crates/resolver/src/legacy_upgrade.rs | 983 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/resolver/src/lib.rs | 1034 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/resolver/src/normalize.rs | 155 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/runtime/Cargo.toml | 21 +++++++++++++++++++++ bobbin/crates/runtime/src/clock.rs | 234 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/runtime/src/entropy.rs | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/runtime/src/hasher.rs | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/runtime/src/lib.rs | 20 ++++++++++++++++++++ bobbin/crates/runtime/src/mem.rs | 12 ++++++++++++ bobbin/crates/runtime/src/mem_network.rs | 359 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/runtime/src/network.rs | 230 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/search/Cargo.toml | 19 +++++++++++++++++++ bobbin/crates/search/src/lib.rs | 792 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/slingshot-client/Cargo.toml | 25 +++++++++++++++++++++++++ bobbin/crates/slingshot-client/src/lib.rs | 603 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/Cargo.toml | 30 ++++++++++++++++++++++++++++++ bobbin/crates/types/build.rs | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/feed/listReactions.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/feed/listReactionsBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/feed/listStars.json | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/feed/listStarsBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/git/listRefUpdates.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/git/listRefUpdatesBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/graph/listFollows.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/graph/listFollowsBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/graph/listVouches.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/graph/listVouchesBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/knot/listKnots.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/knot/listMembers.json | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/knot/listMembersBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/label/listDefinitions.json | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/label/listOps.json | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/label/listOpsBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/pipeline/listPipelines.json | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/pipeline/listPipelinesBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/pipeline/listStatuses.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/pipeline/listStatusesBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/publicKey/listKeys.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/issue/listComments.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/issue/listCommentsBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/issue/listStates.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/issue/listStatesBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/listArtifacts.json | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/listArtifactsBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/listCollaborators.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/listCollaboratorsBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/listIssues.json | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/listIssuesBy.json | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/listPulls.json | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/listPullsBy.json | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/listRepos.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/pull/listComments.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/pull/listCommentsBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/pull/listStatuses.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/repo/pull/listStatusesBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/spindle/listMembers.json | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/spindle/listMembersBy.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/spindle/listSpindles.json | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/lexicons/string/listStrings.json | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/src/edges.rs | 1064 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/src/ids.rs | 143 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/src/legacy.rs | 255 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/types/src/lib.rs | 23 +++++++++++++++++++++++ bobbin/crates/types/src/record.rs | 21 +++++++++++++++++++++ bobbin/crates/types/src/search.rs | 443 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/xrpc/Cargo.toml | 36 ++++++++++++++++++++++++++++++++++++ bobbin/crates/xrpc/src/backpressure.rs | 171 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/xrpc/src/filter.rs | 139 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/xrpc/src/lib.rs | 2699 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/xrpc/tests/aggregation.rs | 2057 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/xrpc/tests/bulk.rs | 486 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/xrpc/tests/cold_start.rs | 639 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/xrpc/tests/coverage.rs | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/xrpc/tests/extended.rs | 873 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/xrpc/tests/knot_proxy.rs | 982 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/crates/xrpc/tests/search.rs | 832 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bobbin/default.nix | 36 ++++++++++++++++++++++++++++++++++++ bobbin/example.toml | 163 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 134 file(s) changed, 37271 insertion(s)(+), 3 deletion(s)(-) diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -17,9 +17,8 @@ .env *.rdb .envrc **/*.bleve -sites/target/ sites/.wrangler/ -/target +target/ **/src/_lex/ # Created if following hacking.md genjwks.out diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,5072 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "bitpacking" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" +dependencies = [ + "crunchy", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bobbin" +version = "0.0.1" +dependencies = [ + "anyhow", + "axum", + "bobbin-edge-index", + "bobbin-ingest", + "bobbin-knot-proxy", + "bobbin-record-lru", + "bobbin-runtime", + "bobbin-search", + "bobbin-slingshot-client", + "bobbin-xrpc", + "clap", + "confique", + "futures", + "serde", + "socket2", + "thiserror 2.0.18", + "tikv-jemalloc-ctl", + "tikv-jemallocator", + "tokio", + "tokio-util", + "toml", + "tracing", + "tracing-subscriber", + "url", +] + +[[package]] +name = "bobbin-edge-index" +version = "0.0.1" +dependencies = [ + "bobbin-runtime", + "bobbin-types", + "either", + "jacquard-common", + "lasso", + "scc", + "smallvec", + "thiserror 2.0.18", + "tokio", +] + +[[package]] +name = "bobbin-ingest" +version = "0.0.1" +dependencies = [ + "bobbin-edge-index", + "bobbin-record-lru", + "bobbin-resolver", + "bobbin-runtime", + "bobbin-slingshot-client", + "bobbin-types", + "bytes", + "chrono", + "criterion", + "futures", + "jacquard-common", + "scc", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "tracing-subscriber", + "url", + "wiremock", +] + +[[package]] +name = "bobbin-knot-proxy" +version = "0.0.1" +dependencies = [ + "bobbin-runtime", + "bytes", + "futures", + "http", + "jacquard-common", + "reqwest", + "scc", + "thiserror 2.0.18", + "tokio", + "url", + "wiremock", +] + +[[package]] +name = "bobbin-record-lru" +version = "0.0.1" +dependencies = [ + "bobbin-types", + "bytes", + "jacquard-common", + "quick_cache", + "zstd", +] + +[[package]] +name = "bobbin-resolver" +version = "0.0.1" +dependencies = [ + "bobbin-runtime", + "bobbin-slingshot-client", + "bobbin-types", + "jacquard-common", + "scc", + "serde_json", + "tokio", + "tracing", + "url", + "wiremock", +] + +[[package]] +name = "bobbin-runtime" +version = "0.0.1" +dependencies = [ + "ahash", + "bytes", + "futures", + "getrandom 0.3.4", + "http", + "reqwest", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite 0.29.0", + "url", +] + +[[package]] +name = "bobbin-search" +version = "0.0.1" +dependencies = [ + "bobbin-runtime", + "bobbin-types", + "jacquard-common", + "tantivy", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "bobbin-sim" +version = "0.0.1" +dependencies = [ + "bobbin-edge-index", + "bobbin-ingest", + "bobbin-record-lru", + "bobbin-runtime", + "bobbin-search", + "bobbin-slingshot-client", + "bobbin-types", + "bytes", + "clap", + "futures", + "http", + "jacquard-common", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", + "url", +] + +[[package]] +name = "bobbin-slingshot-client" +version = "0.0.1" +dependencies = [ + "bobbin-runtime", + "bobbin-types", + "bytes", + "cid", + "futures", + "http", + "jacquard-common", + "reqwest", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "url", + "wiremock", +] + +[[package]] +name = "bobbin-types" +version = "0.0.1" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "cid", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "bobbin-xrpc" +version = "0.0.1" +dependencies = [ + "axum", + "bobbin-edge-index", + "bobbin-knot-proxy", + "bobbin-record-lru", + "bobbin-resolver", + "bobbin-runtime", + "bobbin-search", + "bobbin-slingshot-client", + "bobbin-types", + "chrono", + "futures", + "http", + "jacquard-common", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tower", + "tower-http", + "tracing", + "url", + "wiremock", +] + +[[package]] +name = "bon" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cc" +version = "1.2.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "confique" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06b4f5ec222421e22bb0a8cbaa36b1d2b50fd45cdd30c915ded34108da78b29f" +dependencies = [ + "confique-macro", + "serde", + "toml", +] + +[[package]] +name = "confique-macro" +version = "0.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4d1754680cd218e7bcb4c960cc9bae3444b5197d64563dccccfdf83cab9e1a7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "page_size", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools 0.13.0", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "datasketches" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", + "serde_core", +] + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastdivide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f04ae4152da20c76fe800fa48659201d5cf627c5149ca0b707b69d7eef6cf9" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + +[[package]] +name = "http" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb92f162bf56536459fc83c79b974bb12837acfed43d6bc370a7916d0ae15ecc" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.7", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jacquard-common" +version = "0.12.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e830579811d60e29209c9466d034225d5e045ecdc2b3c55282709bd07da97869" +dependencies = [ + "base64", + "bon", + "bytes", + "chrono", + "ciborium", + "ciborium-io", + "cid", + "fluent-uri", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hashbrown 0.15.5", + "http", + "ipld-core", + "k256", + "maitake-sync", + "miette", + "multibase", + "multihash", + "n0-future", + "ouroboros", + "oxilangtag", + "p256", + "phf", + "postcard", + "rand 0.9.4", + "regex", + "regex-automata", + "regex-lite", + "reqwest", + "rustversion", + "serde", + "serde_bytes", + "serde_html_form", + "serde_ipld_dagcbor", + "serde_json", + "signature", + "smol_str", + "spin 0.10.0", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite-wasm", + "tokio-util", + "trait-variant", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-derive" +version = "0.12.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f83b8049e4e7916e0f6764c3deaf5e55a7ffbab26c379415e9b1d4d645d957" +dependencies = [ + "heck 0.5.0", + "jacquard-lexicon", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jacquard-lexicon" +version = "0.12.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64935ef85dd24f60f467082c21ad52f739a02dd402a2adf40e5794e3de949e1f" +dependencies = [ + "cid", + "dashmap", + "heck 0.5.0", + "inventory", + "jacquard-common", + "miette", + "multihash", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "serde_path_to_error", + "serde_repr", + "serde_with", + "sha2", + "syn", + "thiserror 2.0.18", + "unicode-segmentation", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "sha2", +] + +[[package]] +name = "lasso" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e14eda50a3494b3bf7b9ce51c52434a761e383d7238ce1dd5dcec2fbc13e9fb" +dependencies = [ + "dashmap", + "hashbrown 0.14.5", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" + +[[package]] +name = "maitake-sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" +dependencies = [ + "cordyceps", + "loom", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "measure_time" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" +dependencies = [ + "log", +] + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multibase" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" +dependencies = [ + "base-x", + "base256emoji", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint", +] + +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn", +] + +[[package]] +name = "ownedbytes" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "version_check", + "yansi", +] + +[[package]] +name = "quick_cache" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c821816e9b928e20e92ed59bb3ac4aab321d16ca2316871c9fe7ca739cd477" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.16.1", + "parking_lot", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 1.0.7", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "saa" +version = "5.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f5acb362a0e75c2a963532fa7fabf13dff81626dc494df16488d30befcbea0" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scc" +version = "3.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bcd12b6caff5213cc3c03123cde8c3db5e413008a63b0c0ba35e6275825ea92" +dependencies = [ + "saa", + "sdd", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdd" +version = "4.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f0e40a01b94e35d1dacbcfbe5bfd3d31e37d9590b2e6d86a82b0e87bd4f551" +dependencies = [ + "saa", +] + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_html_form" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "serde_core", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "sketches-ddsketch" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e40b6cf54d988dc1a2223531b969c9a9e30906ad90ef64890c27b4bfbb46ea" +dependencies = [ + "serde", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tantivy" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edde6a10743fff00a4e1a8c9ef020bf5f3cbad301b7d2d39f2b07f123c4eac07" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64", + "bitpacking", + "bon", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "datasketches", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "htmlescape", + "itertools 0.14.0", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror 2.0.18", + "time", + "typetag", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools 0.14.0", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" +dependencies = [ + "fnv", + "nom", + "ordered-float", + "serde", + "serde_json", +] + +[[package]] +name = "tantivy-sstable" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" +dependencies = [ + "futures-util", + "itertools 0.14.0", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951" +dependencies = [ + "murmurhash32", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98" +dependencies = [ + "serde", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tikv-jemalloc-ctl" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a184c43b8ab2f41df2733b55556e3f5f632f4aeaa205b1bb018f574b7f5f142" +dependencies = [ + "libc", + "paste", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2825c78386b4ae0314074867860ba9577875de945f05992c38815cbec327f0" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite 0.24.0", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite 0.29.0", + "webpki-roots 0.26.11", +] + +[[package]] +name = "tokio-tungstenite-wasm" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" +dependencies = [ + "futures-channel", + "futures-util", + "http", + "httparse", + "js-sys", + "rustls", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite 0.24.0", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow 1.0.3", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.4", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 2.0.18", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + +[[package]] +name = "typetag" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5a897b12c6c1151ad0b138b8db50252dc301f93bc3b027db05eec82aeed298c" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf808357c6ed7e13ba0f3277ec8d8f21b2d501274895104263985330c726c1c5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.7", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" + +[[package]] +name = "winnow" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck 0.5.0", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bce33a6288fa3f072a8c2c7d0f2fdbb90e28298f0135c1f99b96c3db2efcc60b" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd425244944f4ab65ccff928e7323354c5a018c75838362fdce749dfad2ee1e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,10 @@ [workspace] resolver = "2" -members = [] +members = ["bobbin/crates/*"] exclude = ["sites"] [workspace.package] +version = "0.0.1" edition = "2024" rust-version = "1.95" license = "MIT" @@ -24,6 +25,69 @@ dbg_macro = "warn" todo = "warn" print_stdout = "deny" print_stderr = "warn" + +[workspace.dependencies] +bobbin-types = { path = "bobbin/crates/types" } +bobbin-edge-index = { path = "bobbin/crates/edge-index" } +bobbin-ingest = { path = "bobbin/crates/ingest" } +bobbin-resolver = { path = "bobbin/crates/resolver" } +bobbin-slingshot-client = { path = "bobbin/crates/slingshot-client" } +bobbin-record-lru = { path = "bobbin/crates/record-lru" } +bobbin-knot-proxy = { path = "bobbin/crates/knot-proxy" } +bobbin-runtime = { path = "bobbin/crates/runtime" } +bobbin-search = { path = "bobbin/crates/search" } +bobbin-sim = { path = "bobbin/crates/bobbin-sim" } +bobbin-xrpc = { path = "bobbin/crates/xrpc" } + +jacquard-common = "0.12.0-beta.2" +jacquard-derive = "0.12.0-beta.2" +jacquard-lexicon = { version = "0.12.0-beta.2", default-features = false } + +anyhow = "1" +miette = "7" +walkdir = "2" + +tokio = { version = "1.52", features = ["macros", "rt-multi-thread", "time", "signal", "io-util", "sync"] } +tokio-util = "0.7" +tokio-stream = "0.1" +tokio-tungstenite = { version = "0.29", features = ["rustls-tls-webpki-roots"] } +futures = "0.3" +either = "1" + +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["raw_value"] } +serde_ipld_dagcbor = "0.6" + +chrono = { version = "0.4", features = ["serde"] } +cid = "0.11" +url = { version = "2", features = ["serde"] } +bytes = "1" + +scc = "3" +roaring = "0.11" +lasso = { version = "0.7", features = ["multi-threaded"] } +quick_cache = "0.6" +getrandom = "0.3" +ahash = { version = "0.8", default-features = false, features = ["std"] } + +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2", "json", "gzip", "stream"] } +axum = "0.8" +tower = { version = "0.5", features = ["util"] } +tower-http = { version = "0.6", features = ["trace"] } +http = "1" + +wiremock = "0.6" + +tantivy = "0.26" + +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "json"] } + +thiserror = "2" + +confique = { version = "0.4", default-features = false, features = ["toml"] } +clap = { version = "4", features = ["derive", "env"] } +toml = { version = "0.9", default-features = false, features = ["parse"] } [profile.release] lto = "fat" diff --git a/bobbin/containerfiles/bobbin.Containerfile b/bobbin/containerfiles/bobbin.Containerfile new file mode 100644 --- /dev/null +++ b/bobbin/containerfiles/bobbin.Containerfile @@ -0,0 +1,17 @@ +FROM docker.io/library/rust:1-alpine3.23 AS builder +RUN apk add --no-cache build-base musl-dev cmake perl pkgconfig +ARG BOBBIN_PROFILE=release +WORKDIR /src +COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ +COPY lexicons ./lexicons +COPY bobbin ./bobbin +RUN cargo build --profile ${BOBBIN_PROFILE} --bin bobbin --package bobbin +RUN if [ "${BOBBIN_PROFILE}" = "release" ]; then strip target/${BOBBIN_PROFILE}/bobbin; fi + +FROM docker.io/library/alpine:3.23 +ARG BOBBIN_PROFILE=release +RUN apk add --no-cache ca-certificates +COPY --from=builder /src/target/${BOBBIN_PROFILE}/bobbin /usr/local/bin/bobbin +ENV BOBBIN_BIND=0.0.0.0:8090 +EXPOSE 8090 +ENTRYPOINT ["/usr/local/bin/bobbin"] diff --git a/bobbin/containerfiles/hydrant.Containerfile b/bobbin/containerfiles/hydrant.Containerfile new file mode 100644 --- /dev/null +++ b/bobbin/containerfiles/hydrant.Containerfile @@ -0,0 +1,14 @@ +FROM docker.io/library/rust:1-alpine3.23 AS builder +RUN apk add --no-cache build-base musl-dev cmake perl pkgconfig +WORKDIR /src +COPY . ./ +RUN rm -f .cargo/config.toml +RUN cargo build --release --bin hydrant +RUN strip target/release/hydrant + +FROM docker.io/library/alpine:3.23 +RUN apk add --no-cache ca-certificates +COPY --from=builder /src/target/release/hydrant /usr/local/bin/hydrant +ENV HYDRANT_DATABASE_PATH=/var/lib/hydrant +EXPOSE 3000 +ENTRYPOINT ["/usr/local/bin/hydrant"] diff --git a/bobbin/containerfiles/slingshot.Containerfile b/bobbin/containerfiles/slingshot.Containerfile new file mode 100644 --- /dev/null +++ b/bobbin/containerfiles/slingshot.Containerfile @@ -0,0 +1,17 @@ +FROM docker.io/library/rust:1-alpine3.23 AS builder +RUN apk add --no-cache build-base musl-dev cmake perl pkgconfig +WORKDIR /src +COPY . ./ +RUN rm -f .cargo/config.toml +RUN cargo build --release --bin slingshot --package slingshot +RUN strip target/release/slingshot + +FROM docker.io/library/alpine:3.23 +RUN apk add --no-cache ca-certificates +WORKDIR /app +COPY --from=builder /src/target/release/slingshot /usr/local/bin/slingshot +COPY --from=builder /src/slingshot/static /app/static +ENV SLINGSHOT_CACHE_DIR=/var/lib/slingshot +ENV SLINGSHOT_BIND=[::]:8080 +EXPOSE 8080 +ENTRYPOINT ["/usr/local/bin/slingshot"] diff --git a/bobbin/crates/bobbin-sim/Cargo.toml b/bobbin/crates/bobbin-sim/Cargo.toml new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "bobbin-sim" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lib] +path = "src/lib.rs" + +[[bin]] +name = "bobbin-sim" +path = "src/main.rs" + +[dependencies] +bobbin-edge-index = { workspace = true } +bobbin-ingest = { workspace = true } +bobbin-record-lru = { workspace = true } +bobbin-runtime = { workspace = true } +bobbin-search = { workspace = true } +bobbin-slingshot-client = { workspace = true } +bobbin-types = { workspace = true } + +bytes = { workspace = true } +clap = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +jacquard-common = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } +tokio-util = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +url = { workspace = true } + +[dev-dependencies] diff --git a/bobbin/crates/bobbin-sim/src/determinism.rs b/bobbin/crates/bobbin-sim/src/determinism.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/determinism.rs @@ -0,0 +1,249 @@ +use std::num::NonZeroUsize; +use std::time::Duration; + +use bobbin_ingest::{DisconnectSnapshot, WarmingBufferSnapshot, WarmingShadowSnapshot}; +use tokio::runtime::Builder as TokioBuilder; +use tracing_subscriber::Registry; +use tracing_subscriber::layer::SubscriberExt; + +use crate::report::{SimOutcome, SimReport}; +use crate::runtime::{Sim, SimConfig}; +use crate::trace_capture::TraceCapture; +use crate::workload::Workload; + +#[derive(Clone, Debug)] +pub struct LeakRunConfig { + pub seed: u64, + pub parallelism: NonZeroUsize, + pub max_virtual_runtime: Duration, + pub mem_ws_capacity: usize, + pub warming_buffer_enabled: bool, +} + +impl LeakRunConfig { + pub fn new( + seed: u64, + parallelism: NonZeroUsize, + max_virtual_runtime: Duration, + mem_ws_capacity: usize, + ) -> Self { + Self { + seed, + parallelism, + max_virtual_runtime, + mem_ws_capacity, + warming_buffer_enabled: true, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum LeakOutcome { + Match, + OutcomeMismatch { + first: SimOutcome, + second: SimOutcome, + }, + EventCountMismatch { + first: u64, + second: u64, + }, + EdgeCountMismatch { + first: u64, + second: u64, + }, + LastCursorMismatch { + first: u64, + second: u64, + }, + ResolverHitsMismatch { + first: u64, + second: u64, + }, + ResolverMissesMismatch { + first: u64, + second: u64, + }, + ConsumerTooSlowMismatch { + first: u64, + second: u64, + }, + DisconnectCountMismatch { + first: u64, + second: u64, + }, + LastDisconnectMismatch { + first: Option, + second: Option, + }, + WarmingShadowMismatch { + first: WarmingShadowSnapshot, + second: WarmingShadowSnapshot, + }, + WarmingBufferMismatch { + first: WarmingBufferSnapshot, + second: WarmingBufferSnapshot, + }, + TraceLengthMismatch { + first: usize, + second: usize, + }, + TraceLineMismatch { + index: usize, + first: String, + second: String, + }, +} + +#[derive(Debug)] +pub struct LeakRunResult { + pub config: LeakRunConfig, + pub workload: &'static str, + pub outcome: LeakOutcome, + pub first_report: SimReport, + pub second_report: SimReport, +} + +impl LeakRunResult { + pub fn passed(&self) -> bool { + matches!(self.outcome, LeakOutcome::Match) + } +} + +pub fn run_leak_check(config: LeakRunConfig, workload_factory: F) -> LeakRunResult +where + F: Fn() -> Box, +{ + let workload_name = { + let probe = workload_factory(); + probe.name() + }; + + let (first_report, first_lines) = run_once(&config, workload_factory()); + let (second_report, second_lines) = run_once(&config, workload_factory()); + + let outcome = compare_runs(&first_report, &first_lines, &second_report, &second_lines); + + LeakRunResult { + config, + workload: workload_name, + outcome, + first_report, + second_report, + } +} + +fn run_once(config: &LeakRunConfig, workload: Box) -> (SimReport, Vec) { + let capture = TraceCapture::new(); + let subscriber = Registry::default().with(capture.layer()); + + let mut sim_config = SimConfig::new(config.seed); + sim_config.max_virtual_runtime = config.max_virtual_runtime; + sim_config.parallelism = config.parallelism; + sim_config.mem_ws_capacity = config.mem_ws_capacity; + sim_config.warming_buffer_enabled = config.warming_buffer_enabled; + + let runtime = TokioBuilder::new_current_thread() + .enable_all() + .start_paused(true) + .build() + .expect("build current_thread runtime with paused time"); + + let report = tracing::subscriber::with_default(subscriber, || { + runtime.block_on(Sim::new(sim_config, workload).run()) + }); + drop(runtime); + + let lines = capture.into_lines(); + (report, lines) +} + +fn compare_runs( + a: &SimReport, + a_lines: &[String], + b: &SimReport, + b_lines: &[String], +) -> LeakOutcome { + if a.outcome != b.outcome { + return LeakOutcome::OutcomeMismatch { + first: a.outcome, + second: b.outcome, + }; + } + if a.events_processed != b.events_processed { + return LeakOutcome::EventCountMismatch { + first: a.events_processed, + second: b.events_processed, + }; + } + if a.edge_count != b.edge_count { + return LeakOutcome::EdgeCountMismatch { + first: a.edge_count, + second: b.edge_count, + }; + } + if a.last_cursor != b.last_cursor { + return LeakOutcome::LastCursorMismatch { + first: a.last_cursor, + second: b.last_cursor, + }; + } + if a.resolver_hits != b.resolver_hits { + return LeakOutcome::ResolverHitsMismatch { + first: a.resolver_hits, + second: b.resolver_hits, + }; + } + if a.resolver_misses != b.resolver_misses { + return LeakOutcome::ResolverMissesMismatch { + first: a.resolver_misses, + second: b.resolver_misses, + }; + } + if a.consumer_too_slow_count != b.consumer_too_slow_count { + return LeakOutcome::ConsumerTooSlowMismatch { + first: a.consumer_too_slow_count, + second: b.consumer_too_slow_count, + }; + } + if a.disconnect_count != b.disconnect_count { + return LeakOutcome::DisconnectCountMismatch { + first: a.disconnect_count, + second: b.disconnect_count, + }; + } + if a.last_disconnect != b.last_disconnect { + return LeakOutcome::LastDisconnectMismatch { + first: a.last_disconnect.clone(), + second: b.last_disconnect.clone(), + }; + } + if a.warming_shadow != b.warming_shadow { + return LeakOutcome::WarmingShadowMismatch { + first: a.warming_shadow, + second: b.warming_shadow, + }; + } + if a.warming_buffer != b.warming_buffer { + return LeakOutcome::WarmingBufferMismatch { + first: a.warming_buffer, + second: b.warming_buffer, + }; + } + if a_lines.len() != b_lines.len() { + return LeakOutcome::TraceLengthMismatch { + first: a_lines.len(), + second: b_lines.len(), + }; + } + for (i, (x, y)) in a_lines.iter().zip(b_lines.iter()).enumerate() { + if x != y { + return LeakOutcome::TraceLineMismatch { + index: i, + first: x.clone(), + second: y.clone(), + }; + } + } + LeakOutcome::Match +} diff --git a/bobbin/crates/bobbin-sim/src/lib.rs b/bobbin/crates/bobbin-sim/src/lib.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/lib.rs @@ -0,0 +1,13 @@ +mod determinism; +mod report; +mod runtime; +mod trace_capture; +mod workload; + +pub mod workloads; + +pub use determinism::{LeakOutcome, LeakRunConfig, LeakRunResult, run_leak_check}; +pub use report::{SimOutcome, SimReport}; +pub use runtime::{Sim, SimConfig}; +pub use trace_capture::{StageLayer, TraceCapture}; +pub use workload::{Workload, WorkloadCtx, WorkloadHooks}; diff --git a/bobbin/crates/bobbin-sim/src/main.rs b/bobbin/crates/bobbin-sim/src/main.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/main.rs @@ -0,0 +1,382 @@ +use std::num::NonZeroUsize; +use std::path::PathBuf; +use std::process::ExitCode; +use std::time::Duration; + +use bobbin_sim::workloads::{ + CancelMidHydration, CancelMidHydrationConfig, ColdStartUnderLiveLoad, + ColdStartUnderLiveLoadConfig, ConcurrentReadsDuringReplay, ConcurrentReadsDuringReplayConfig, + FrameBurst, HydrantDisconnectBarrage, HydrantDisconnectBarrageConfig, SlingshotFlap, + SlingshotFlapConfig, +}; +use bobbin_sim::{ + LeakOutcome, LeakRunConfig, Sim, SimConfig, SimOutcome, SimReport, Workload, run_leak_check, +}; +use clap::{Parser, ValueEnum}; + +#[derive(Parser, Debug)] +#[command(name = "bobbin-sim", version)] +struct Cli { + #[arg(long, default_value_t = 0)] + seed: u64, + #[arg(long, value_enum, default_value_t = WorkloadName::FrameBurst)] + workload: WorkloadName, + #[arg(long, default_value_t = 64)] + frames: usize, + #[arg(long, default_value_t = 60)] + max_virtual_seconds: u64, + #[arg(long)] + no_brownout: bool, + #[arg(long, default_value_t = 200)] + cross_did_stars: usize, + #[arg(long, default_value_t = 1_000)] + brownout_start_ms: u64, + #[arg(long, default_value_t = 5_000)] + brownout_duration_ms: u64, + #[arg(long, default_value_t = 200)] + brownout_latency_ms: u64, + #[arg(long, default_value_t = 2)] + normal_latency_ms: u64, + #[arg(long, default_value_t = 16)] + parallelism: usize, + #[arg(long, default_value_t = 4096)] + mem_ws_capacity: usize, + #[arg(long, default_value_t = 30_000)] + hydrant_send_timeout_ms: u64, + #[arg(long, default_value_t = 0)] + hydrant_frame_pace_us: u64, + #[arg(long, default_value_t = 1)] + seeds: u64, + #[arg(long)] + quarantine_out: Option, + #[arg(long)] + leak_check: bool, +} + +#[derive(Clone, Copy, Debug, ValueEnum, Eq, PartialEq)] +enum WorkloadName { + FrameBurst, + SlingshotFlap, + CancelMidHydration, + HydrantDisconnectBarrage, + ColdStartUnderLiveLoad, + ConcurrentReadsDuringReplay, +} + +fn main() -> ExitCode { + let cli = Cli::parse(); + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn,bobbin_sim=info")), + ) + .with_writer(std::io::stderr) + .init(); + + if cli.seeds <= 1 && !cli.leak_check { + let report = run_single(&cli, cli.seed); + println!( + "{}", + serde_json::to_string_pretty(&report_as_json(&report)).unwrap() + ); + exit_code_from(report.outcome) + } else if cli.leak_check && cli.seeds <= 1 { + let result = run_leak(&cli, cli.seed); + let json = leak_result_as_json(&result); + println!("{}", serde_json::to_string_pretty(&json).unwrap()); + if matches!(result.outcome, LeakOutcome::Match) + && matches!(result.first_report.outcome, SimOutcome::Passed) + { + ExitCode::from(0) + } else { + ExitCode::from(1) + } + } else { + run_sweep(&cli) + } +} + +fn run_sweep(cli: &Cli) -> ExitCode { + let mut quarantined: Vec = Vec::new(); + let mut passed = 0u64; + let total = cli.seeds; + for offset in 0..total { + let seed = cli.seed.wrapping_add(offset); + let outcome_record = if cli.leak_check { + let result = run_leak(cli, seed); + let leak_match = matches!(result.outcome, LeakOutcome::Match); + let workload_passed = matches!(result.first_report.outcome, SimOutcome::Passed); + let ok = leak_match && workload_passed; + if ok { + passed += 1; + None + } else { + Some(leak_result_as_json(&result)) + } + } else { + let report = run_single(cli, seed); + if matches!(report.outcome, SimOutcome::Passed) { + passed += 1; + None + } else { + Some(report_as_json(&report)) + } + }; + if let Some(rec) = outcome_record { + quarantined.push(rec); + } + } + let summary = serde_json::json!({ + "workload": format!("{:?}", cli.workload), + "seeds_run": total, + "passed": passed, + "failed": total - passed, + "leak_check": cli.leak_check, + "parallelism": cli.parallelism, + "quarantine_count": quarantined.len(), + }); + println!("{}", serde_json::to_string_pretty(&summary).unwrap()); + if let Some(path) = &cli.quarantine_out { + let payload = serde_json::json!({ + "summary": summary, + "failures": quarantined, + }); + std::fs::write(path, serde_json::to_vec_pretty(&payload).unwrap()) + .expect("write quarantine output"); + eprintln!("quarantine written to {}", path.display()); + } + if quarantined.is_empty() { + ExitCode::from(0) + } else { + ExitCode::from(1) + } +} + +fn run_single(cli: &Cli, seed: u64) -> SimReport { + let workload = build_workload(cli); + let mut config = SimConfig::new(seed); + config.max_virtual_runtime = Duration::from_secs(cli.max_virtual_seconds); + config.parallelism = NonZeroUsize::new(cli.parallelism).expect("parallelism > 0"); + config.mem_ws_capacity = cli.mem_ws_capacity; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .start_paused(true) + .build() + .expect("build current_thread runtime with paused time"); + runtime.block_on(Sim::new(config, workload).run()) +} + +fn run_leak(cli: &Cli, seed: u64) -> bobbin_sim::LeakRunResult { + let leak_config = LeakRunConfig { + seed, + parallelism: NonZeroUsize::new(cli.parallelism).expect("parallelism > 0"), + max_virtual_runtime: Duration::from_secs(cli.max_virtual_seconds), + mem_ws_capacity: cli.mem_ws_capacity, + warming_buffer_enabled: true, + }; + let cli_snapshot = CliSnapshot::from(cli); + let factory = move || -> Box { build_workload_from(&cli_snapshot) }; + run_leak_check(leak_config, factory) +} + +#[derive(Clone)] +struct CliSnapshot { + workload: WorkloadName, + frames: usize, + cross_did_stars: usize, + normal_latency_ms: u64, + brownout_start_ms: u64, + brownout_duration_ms: u64, + brownout_latency_ms: u64, + brownout_enabled: bool, + hydrant_send_timeout_ms: u64, + hydrant_frame_pace_us: u64, +} + +impl From<&Cli> for CliSnapshot { + fn from(cli: &Cli) -> Self { + Self { + workload: cli.workload, + frames: cli.frames, + cross_did_stars: cli.cross_did_stars, + normal_latency_ms: cli.normal_latency_ms, + brownout_start_ms: cli.brownout_start_ms, + brownout_duration_ms: cli.brownout_duration_ms, + brownout_latency_ms: cli.brownout_latency_ms, + brownout_enabled: !cli.no_brownout, + hydrant_send_timeout_ms: cli.hydrant_send_timeout_ms, + hydrant_frame_pace_us: cli.hydrant_frame_pace_us, + } + } +} + +fn build_workload(cli: &Cli) -> Box { + build_workload_from(&CliSnapshot::from(cli)) +} + +fn build_workload_from(snap: &CliSnapshot) -> Box { + match snap.workload { + WorkloadName::FrameBurst => Box::new(FrameBurst::new(snap.frames)), + WorkloadName::SlingshotFlap => Box::new(SlingshotFlap::new(SlingshotFlapConfig { + cross_did_stars: snap.cross_did_stars, + normal_latency_ms: snap.normal_latency_ms, + brownout_start_ms: snap.brownout_start_ms, + brownout_duration_ms: snap.brownout_duration_ms, + brownout_latency_ms: snap.brownout_latency_ms, + brownout_enabled: snap.brownout_enabled, + hydrant_send_timeout_ms: snap.hydrant_send_timeout_ms, + hydrant_frame_pace_us: snap.hydrant_frame_pace_us, + omit_target_repos: false, + emit_live_promotion_frame: false, + })), + WorkloadName::CancelMidHydration => { + Box::new(CancelMidHydration::new(CancelMidHydrationConfig::default())) + } + WorkloadName::HydrantDisconnectBarrage => Box::new(HydrantDisconnectBarrage::new( + HydrantDisconnectBarrageConfig::default(), + )), + WorkloadName::ColdStartUnderLiveLoad => Box::new(ColdStartUnderLiveLoad::new( + ColdStartUnderLiveLoadConfig::default(), + )), + WorkloadName::ConcurrentReadsDuringReplay => Box::new(ConcurrentReadsDuringReplay::new( + ConcurrentReadsDuringReplayConfig::default(), + )), + } +} + +fn report_as_json(r: &SimReport) -> serde_json::Value { + serde_json::json!({ + "workload": r.workload, + "seed": r.seed, + "outcome": format!("{:?}", r.outcome), + "virtual_runtime_micros": r.virtual_runtime.as_micros() as u64, + "virtual_clock_end_unix_micros": r.virtual_clock_end.raw(), + "events_processed": r.events_processed, + "last_cursor": r.last_cursor, + "edge_count": r.edge_count, + "resolver_hits": r.resolver_hits, + "resolver_misses": r.resolver_misses, + "consumer_too_slow_count": r.consumer_too_slow_count, + "disconnect_count": r.disconnect_count, + "last_disconnect": r.last_disconnect.as_ref().map(|d| serde_json::json!({ + "kind": format!("{:?}", d.kind), + "message": d.message, + "at_unix_micros": d.at_unix_micros.raw(), + "last_cursor": d.last_cursor.raw(), + })), + "warming_shadow": { + "enqueued_total": r.warming_shadow.enqueued_total, + "drained_via_observe_total": r.warming_shadow.drained_via_observe_total, + "max_concurrent": r.warming_shadow.max_concurrent, + "residual": r.warming_shadow.residual, + "distinct_keys_seen": r.warming_shadow.distinct_keys_seen, + }, + "warming_buffer": { + "enqueued_total": r.warming_buffer.enqueued_total, + "drained_observe_total": r.warming_buffer.drained_observe_total, + "drained_promote_total": r.warming_buffer.drained_promote_total, + "evicted_total": r.warming_buffer.evicted_total, + "rejected_after_seal": r.warming_buffer.rejected_after_seal, + "distinct_keys_seen": r.warming_buffer.distinct_keys_seen, + "current_entries": r.warming_buffer.current_entries, + "max_concurrent_entries": r.warming_buffer.max_concurrent_entries, + "dep_enqueued_total": r.warming_buffer.dep_enqueued_total, + "dep_drained_observe_total": r.warming_buffer.dep_drained_observe_total, + }, + "failure_reason": r.failure_reason, + }) +} + +fn leak_result_as_json(r: &bobbin_sim::LeakRunResult) -> serde_json::Value { + serde_json::json!({ + "workload": r.workload, + "seed": r.config.seed, + "parallelism": r.config.parallelism.get(), + "leak_outcome": leak_outcome_as_json(&r.outcome), + "first": report_as_json(&r.first_report), + "second": report_as_json(&r.second_report), + }) +} + +fn leak_outcome_as_json(outcome: &LeakOutcome) -> serde_json::Value { + match outcome { + LeakOutcome::Match => serde_json::json!({ "kind": "match" }), + LeakOutcome::OutcomeMismatch { first, second } => serde_json::json!({ + "kind": "outcome_mismatch", + "first": format!("{first:?}"), + "second": format!("{second:?}"), + }), + LeakOutcome::EventCountMismatch { first, second } => serde_json::json!({ + "kind": "event_count_mismatch", + "first": first, + "second": second, + }), + LeakOutcome::EdgeCountMismatch { first, second } => serde_json::json!({ + "kind": "edge_count_mismatch", + "first": first, + "second": second, + }), + LeakOutcome::LastCursorMismatch { first, second } => serde_json::json!({ + "kind": "last_cursor_mismatch", + "first": first, + "second": second, + }), + LeakOutcome::ResolverHitsMismatch { first, second } => serde_json::json!({ + "kind": "resolver_hits_mismatch", + "first": first, + "second": second, + }), + LeakOutcome::ResolverMissesMismatch { first, second } => serde_json::json!({ + "kind": "resolver_misses_mismatch", + "first": first, + "second": second, + }), + LeakOutcome::ConsumerTooSlowMismatch { first, second } => serde_json::json!({ + "kind": "consumer_too_slow_mismatch", + "first": first, + "second": second, + }), + LeakOutcome::DisconnectCountMismatch { first, second } => serde_json::json!({ + "kind": "disconnect_count_mismatch", + "first": first, + "second": second, + }), + LeakOutcome::LastDisconnectMismatch { first, second } => serde_json::json!({ + "kind": "last_disconnect_mismatch", + "first": first.as_ref().map(|d| format!("{:?}: {}", d.kind, d.message)), + "second": second.as_ref().map(|d| format!("{:?}: {}", d.kind, d.message)), + }), + LeakOutcome::WarmingShadowMismatch { first, second } => serde_json::json!({ + "kind": "warming_shadow_mismatch", + "first": format!("{first:?}"), + "second": format!("{second:?}"), + }), + LeakOutcome::WarmingBufferMismatch { first, second } => serde_json::json!({ + "kind": "warming_buffer_mismatch", + "first": format!("{first:?}"), + "second": format!("{second:?}"), + }), + LeakOutcome::TraceLengthMismatch { first, second } => serde_json::json!({ + "kind": "trace_length_mismatch", + "first": first, + "second": second, + }), + LeakOutcome::TraceLineMismatch { + index, + first, + second, + } => serde_json::json!({ + "kind": "trace_line_mismatch", + "index": index, + "first": first, + "second": second, + }), + } +} + +fn exit_code_from(outcome: SimOutcome) -> ExitCode { + match outcome { + SimOutcome::Passed => ExitCode::from(0), + SimOutcome::Failed | SimOutcome::TimedOut => ExitCode::from(1), + } +} diff --git a/bobbin/crates/bobbin-sim/src/report.rs b/bobbin/crates/bobbin-sim/src/report.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/report.rs @@ -0,0 +1,37 @@ +use std::time::Duration; + +use bobbin_ingest::{DisconnectSnapshot, WarmingBufferSnapshot, WarmingShadowSnapshot}; +use bobbin_runtime::UnixMicros; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SimOutcome { + Passed, + Failed, + TimedOut, +} + +#[derive(Clone, Debug)] +pub struct SimReport { + pub workload: &'static str, + pub seed: u64, + pub outcome: SimOutcome, + pub virtual_runtime: Duration, + pub virtual_clock_end: UnixMicros, + pub events_processed: u64, + pub last_cursor: u64, + pub edge_count: u64, + pub resolver_hits: u64, + pub resolver_misses: u64, + pub consumer_too_slow_count: u64, + pub disconnect_count: u64, + pub last_disconnect: Option, + pub warming_shadow: WarmingShadowSnapshot, + pub warming_buffer: WarmingBufferSnapshot, + pub failure_reason: Option, +} + +impl SimReport { + pub fn passed(self) -> bool { + matches!(self.outcome, SimOutcome::Passed) + } +} diff --git a/bobbin/crates/bobbin-sim/src/runtime.rs b/bobbin/crates/bobbin-sim/src/runtime.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/runtime.rs @@ -0,0 +1,189 @@ +use std::num::NonZeroUsize; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use bobbin_edge_index::{CoverageWatch, EdgeStore}; +use bobbin_ingest::{ + DEFAULT_INGEST_PARALLELISM, DisconnectSink, IngestConfig, IngestRuntime, RepoIdResolver, + WarmingBuffer, WarmingShadowBuffer, run as run_ingest, +}; +use bobbin_record_lru::NoopRecordStore; +use bobbin_runtime::{ + Clock, DEFAULT_MEM_WS_CAPACITY, MemHttpTransport, MemWsTransport, RuntimeHasher, SeededEntropy, + SimClock, UnixMicros, +}; +use bobbin_slingshot_client::SlingshotClient; +use bobbin_types::search::NoopSearchSink; +use tokio_util::sync::CancellationToken; +use url::Url; + +use crate::report::{SimOutcome, SimReport}; +use crate::workload::{Workload, WorkloadCtx}; + +#[derive(Clone, Debug)] +pub struct SimConfig { + pub seed: u64, + pub base_unix: UnixMicros, + pub max_virtual_runtime: Duration, + pub parallelism: NonZeroUsize, + pub hydrant_base: Url, + pub slingshot_base: Url, + pub mem_ws_capacity: usize, + pub warming_buffer_enabled: bool, +} + +impl SimConfig { + pub fn new(seed: u64) -> Self { + Self { + seed, + base_unix: UnixMicros::new(1_700_000_000_000_000), + max_virtual_runtime: Duration::from_secs(60), + parallelism: DEFAULT_INGEST_PARALLELISM, + hydrant_base: Url::parse("ws://hydrant.sim/").unwrap(), + slingshot_base: Url::parse("http://slingshot.sim/").unwrap(), + mem_ws_capacity: DEFAULT_MEM_WS_CAPACITY, + warming_buffer_enabled: true, + } + } +} + +pub struct Sim { + config: SimConfig, + workload: Box, +} + +impl Sim { + pub fn new(config: SimConfig, workload: Box) -> Self { + Self { config, workload } + } + + pub async fn run(self) -> SimReport { + let SimConfig { + seed, + base_unix, + max_virtual_runtime, + parallelism, + hydrant_base, + slingshot_base, + mem_ws_capacity, + warming_buffer_enabled, + } = self.config; + + let entropy = Arc::new(SeededEntropy::new(seed)); + let hasher = RuntimeHasher::from_entropy(&*entropy); + let clock: Arc = Arc::new(SimClock::at(base_unix)); + + let store = Arc::new(EdgeStore::new(hasher.clone())); + let coverage = Arc::new(CoverageWatch::new()); + let records = Arc::new(NoopRecordStore); + let cancel = CancellationToken::new(); + let consumer_too_slow_count = Arc::new(AtomicU64::new(0)); + let disconnects = Arc::new(DisconnectSink::new()); + let warming_shadow = Arc::new(WarmingShadowBuffer::new(hasher.clone())); + let warming_buffer = Arc::new(WarmingBuffer::new(hasher.clone())); + + let workload_name = self.workload.name(); + let ctx = WorkloadCtx { + seed, + clock: clock.clone(), + entropy: entropy.clone(), + store: store.clone(), + coverage: coverage.clone(), + records: records.clone(), + cancel: cancel.clone(), + consumer_too_slow_count: consumer_too_slow_count.clone(), + }; + let hooks = self.workload.build(ctx); + + let slingshot_http = MemHttpTransport::shared(hooks.slingshot.clone(), clock.clone()); + let slingshot_client = SlingshotClient::new(slingshot_base, slingshot_http) + .expect("slingshot base url is valid"); + let resolver = Arc::new(RepoIdResolver::with_slingshot( + slingshot_client, + clock.clone(), + hasher.clone(), + )); + + let mem_ws = MemWsTransport::shared_with_capacity(hooks.hydrant.clone(), mem_ws_capacity); + + let ingest_runtime: IngestRuntime = IngestRuntime { + store: store.clone(), + issue_states: Arc::new(bobbin_edge_index::StateIndex::new(hasher.clone())), + pull_statuses: Arc::new(bobbin_edge_index::StateIndex::new(hasher.clone())), + coverage: coverage.clone(), + search: Arc::new(NoopSearchSink), + records: records.clone() as Arc, + resolver: resolver.clone(), + clock: clock.clone(), + entropy: entropy.clone(), + ws: mem_ws, + cancel: cancel.clone(), + disconnects: Some(disconnects.clone()), + warming_shadow: Some(warming_shadow.clone()), + warming_buffer: warming_buffer_enabled.then(|| warming_buffer.clone()), + }; + let ingest_config = IngestConfig { + hydrant_base, + start_cursor: bobbin_edge_index::HydrantCursor::new(0), + parallelism, + }; + let mut ingest_handle = tokio::spawn(async move { + let _ = run_ingest(ingest_config, ingest_runtime).await; + }); + + let script = hooks.script; + let initial_report = tokio::select! { + biased; + _ = clock.sleep(max_virtual_runtime) => SimReport { + workload: workload_name, + seed, + outcome: SimOutcome::TimedOut, + virtual_runtime: max_virtual_runtime, + virtual_clock_end: clock.now_unix_micros(), + events_processed: coverage.snapshot().events_processed(), + last_cursor: coverage.snapshot().last_cursor().raw(), + edge_count: store.key_count() as u64, + resolver_hits: 0, + resolver_misses: 0, + consumer_too_slow_count: consumer_too_slow_count.load(Ordering::Relaxed), + disconnect_count: disconnects.count(), + last_disconnect: disconnects.snapshot(), + warming_shadow: warming_shadow.snapshot(), + warming_buffer: warming_buffer.snapshot(), + failure_reason: Some(format!( + "max_virtual_runtime {max_virtual_runtime:?} exhausted", + )), + }, + r = script => r, + }; + + cancel.cancel(); + let drain_deadline = clock.sleep(Duration::from_secs(30)); + tokio::pin!(drain_deadline); + tokio::select! { + _ = &mut drain_deadline => { + ingest_handle.abort(); + let _ = ingest_handle.await; + } + res = &mut ingest_handle => { + let _ = res; + } + } + + let stats = resolver.stats(); + SimReport { + edge_count: store.key_count() as u64, + events_processed: coverage.snapshot().events_processed(), + last_cursor: coverage.snapshot().last_cursor().raw(), + resolver_hits: stats.hits, + resolver_misses: stats.miss_count(), + consumer_too_slow_count: consumer_too_slow_count.load(Ordering::Relaxed), + disconnect_count: disconnects.count(), + last_disconnect: disconnects.snapshot(), + warming_shadow: warming_shadow.snapshot(), + warming_buffer: warming_buffer.snapshot(), + ..initial_report + } + } +} diff --git a/bobbin/crates/bobbin-sim/src/trace_capture.rs b/bobbin/crates/bobbin-sim/src/trace_capture.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/trace_capture.rs @@ -0,0 +1,98 @@ +use std::sync::Arc; +use std::sync::Mutex; + +use jacquard_common::DefaultStr; +use jacquard_common::types::nsid::Nsid; +use tracing::field::{Field, Visit}; +use tracing::{Event, Subscriber}; +use tracing_subscriber::Layer; +use tracing_subscriber::layer::Context; +use tracing_subscriber::registry::LookupSpan; + +#[derive(Clone, Default)] +pub struct TraceCapture { + inner: Arc>>, +} + +impl TraceCapture { + pub fn new() -> Self { + Self::default() + } + + pub fn lines(&self) -> Vec { + self.inner.lock().unwrap().clone() + } + + pub fn into_lines(self) -> Vec { + std::mem::take(&mut *self.inner.lock().unwrap()) + } + + pub fn layer(&self) -> StageLayer { + StageLayer { + sink: self.inner.clone(), + } + } +} + +pub struct StageLayer { + sink: Arc>>, +} + +impl Layer for StageLayer +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + let meta = event.metadata(); + if meta.target() != "bobbin_ingest::stage" { + return; + } + let mut visitor = StageVisitor::default(); + event.record(&mut visitor); + let line = format!( + "cursor={} regime={} nsid={} edge_count={} parallelism={}", + visitor.cursor.unwrap_or(u64::MAX), + visitor.regime.as_deref().unwrap_or(""), + visitor.nsid.as_ref().map(Nsid::as_ref).unwrap_or(""), + visitor.edge_count.unwrap_or(0), + visitor.parallelism.unwrap_or(0), + ); + self.sink.lock().unwrap().push(line); + } +} + +#[derive(Default)] +struct StageVisitor { + cursor: Option, + regime: Option, + nsid: Option>, + edge_count: Option, + parallelism: Option, +} + +impl Visit for StageVisitor { + fn record_u64(&mut self, field: &Field, value: u64) { + match field.name() { + "cursor" => self.cursor = Some(value), + "edge_count" => self.edge_count = Some(value), + "parallelism" => self.parallelism = Some(value as usize), + _ => {} + } + } + + fn record_i64(&mut self, field: &Field, value: i64) { + if field.name() == "cursor" { + self.cursor = Some(value as u64); + } + } + + fn record_str(&mut self, field: &Field, value: &str) { + match field.name() { + "regime" => self.regime = Some(value.to_owned()), + "nsid" => self.nsid = Nsid::new_owned(value).ok(), + _ => {} + } + } + + fn record_debug(&mut self, _field: &Field, _value: &dyn std::fmt::Debug) {} +} diff --git a/bobbin/crates/bobbin-sim/src/workload.rs b/bobbin/crates/bobbin-sim/src/workload.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/workload.rs @@ -0,0 +1,35 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::AtomicU64; + +use bobbin_edge_index::{CoverageWatch, EdgeStore}; +use bobbin_record_lru::NoopRecordStore; +use bobbin_runtime::{Clock, Entropy, MemHttpResponder, MemWsResponder}; +use tokio_util::sync::CancellationToken; + +use crate::report::SimReport; + +pub struct WorkloadCtx { + pub seed: u64, + pub clock: Arc, + pub entropy: Arc, + pub store: Arc, + pub coverage: Arc, + pub records: Arc, + pub cancel: CancellationToken, + pub consumer_too_slow_count: Arc, +} + +pub type WorkloadScript = Pin + Send + 'static>>; + +pub struct WorkloadHooks { + pub slingshot: Arc, + pub hydrant: Arc, + pub script: WorkloadScript, +} + +pub trait Workload: Send + 'static { + fn name(&self) -> &'static str; + fn build(self: Box, ctx: WorkloadCtx) -> WorkloadHooks; +} diff --git a/bobbin/crates/bobbin-sim/src/workloads/cancel_mid_hydration.rs b/bobbin/crates/bobbin-sim/src/workloads/cancel_mid_hydration.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/workloads/cancel_mid_hydration.rs @@ -0,0 +1,285 @@ +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use bobbin_runtime::{ + Clock, HttpRequest, MemHttpBody, MemHttpResponder, MemHttpResponse, MemWsResponder, + MemWsServerFuture, WsMessage, +}; +use bytes::Bytes; +use http::StatusCode; +use jacquard_common::DefaultStr; +use jacquard_common::types::nsid::Nsid; +use tokio::sync::mpsc; +use url::Url; + +use crate::report::{SimOutcome, SimReport}; +use crate::workload::{Workload, WorkloadCtx, WorkloadHooks}; +use crate::workloads::util::{format_rkey, format_tid, parse_repo_lookup}; + +const NAME: &str = "cancel-mid-hydration"; +const REPO_COLLECTION: &str = "sh.tangled.repo"; + +fn repo_collection_nsid() -> Nsid { + Nsid::new_static(REPO_COLLECTION).expect("REPO_COLLECTION literal is a valid NSID") +} + +#[derive(Clone, Debug)] +pub struct CancelMidHydrationConfig { + pub frames: usize, + pub cancel_at_events: u64, + pub slingshot_latency_ms: u64, + pub frame_pace_us: u64, +} + +impl Default for CancelMidHydrationConfig { + fn default() -> Self { + Self { + frames: 200, + cancel_at_events: 50, + slingshot_latency_ms: 50, + frame_pace_us: 100, + } + } +} + +pub struct CancelMidHydration { + config: CancelMidHydrationConfig, +} + +impl CancelMidHydration { + pub fn new(config: CancelMidHydrationConfig) -> Self { + Self { config } + } +} + +impl Workload for CancelMidHydration { + fn name(&self) -> &'static str { + NAME + } + + fn build(self: Box, ctx: WorkloadCtx) -> WorkloadHooks { + let cfg = self.config.clone(); + let WorkloadCtx { + seed, + clock, + coverage, + store, + cancel, + .. + } = ctx; + + let frames = build_frames(cfg.frames); + let slingshot_calls = Arc::new(AtomicU64::new(0)); + + let hydrant: Arc = Arc::new(PacedHydrant { + frames: Mutex::new(Some(frames)), + frame_pace: Duration::from_micros(cfg.frame_pace_us), + clock: clock.clone(), + }); + let slingshot: Arc = Arc::new(LatentSlingshot { + latency: Duration::from_millis(cfg.slingshot_latency_ms), + slingshot_calls: slingshot_calls.clone(), + }); + + let cancel_at = cfg.cancel_at_events; + let frames_total = cfg.frames as u64; + let slingshot_calls_probe = slingshot_calls.clone(); + let script = Box::pin(async move { + let started = clock.now_unix_micros(); + let mut rx = coverage.subscribe(); + let cancel_outcome = loop { + let snap = *rx.borrow_and_update(); + if snap.events_processed() >= cancel_at { + break SimOutcome::Passed; + } + tokio::select! { + res = rx.changed() => match res { + Ok(()) => continue, + Err(_) => break SimOutcome::Failed, + }, + _ = cancel.cancelled() => break SimOutcome::Failed, + } + }; + + cancel.cancel(); + clock.sleep(Duration::from_millis(100)).await; + let events_after_observe = coverage.snapshot().events_processed(); + let slingshot_after_observe = slingshot_calls_probe.load(Ordering::Relaxed); + + clock.sleep(Duration::from_secs(2)).await; + let snap = coverage.snapshot(); + let virtual_runtime = + Duration::from_micros(clock.now_unix_micros().raw().saturating_sub(started.raw())); + let slingshot_after_drain = slingshot_calls_probe.load(Ordering::Relaxed); + let post_observe_events = snap.events_processed().saturating_sub(events_after_observe); + let post_observe_slingshot = + slingshot_after_drain.saturating_sub(slingshot_after_observe); + let failed = !matches!(cancel_outcome, SimOutcome::Passed) + || post_observe_slingshot != 0 + || post_observe_events != 0 + || snap.events_processed() > frames_total + || snap.events_processed() < cancel_at; + let outcome = if failed { + SimOutcome::Failed + } else { + SimOutcome::Passed + }; + let failure_reason = match outcome { + SimOutcome::Passed => None, + SimOutcome::Failed => Some(format!( + "cancel-mid-hydration anomaly: events={} cancel_at={} \ + events_after_observe={} post_observe_events={} \ + slingshot_after_observe={} slingshot_after_drain={} \ + post_observe_slingshot={}", + snap.events_processed(), + cancel_at, + events_after_observe, + post_observe_events, + slingshot_after_observe, + slingshot_after_drain, + post_observe_slingshot, + )), + SimOutcome::TimedOut => Some("timed out".into()), + }; + SimReport { + workload: NAME, + seed, + outcome, + virtual_runtime, + virtual_clock_end: clock.now_unix_micros(), + events_processed: snap.events_processed(), + last_cursor: snap.last_cursor().raw(), + edge_count: store.key_count() as u64, + resolver_hits: 0, + resolver_misses: 0, + consumer_too_slow_count: 0, + disconnect_count: 0, + last_disconnect: None, + warming_shadow: Default::default(), + warming_buffer: Default::default(), + failure_reason, + } + }); + + WorkloadHooks { + slingshot, + hydrant, + script, + } + } +} + +fn build_frames(count: usize) -> Vec { + (0..count) + .map(|i| { + let star_did = format!("did:plc:starer-{i}"); + let target_owner = format!("did:plc:owner-{i}"); + let target_rkey = format_rkey(i); + serde_json::json!({ + "id": (i + 1) as u64, + "type": "record", + "record": { + "live": false, + "did": star_did, + "rev": format_tid(i), + "collection": "sh.tangled.feed.star", + "rkey": format_rkey(i + 1_000_000), + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subject": format!("at://{target_owner}/{REPO_COLLECTION}/{target_rkey}") + } + } + }) + .to_string() + }) + .collect() +} + +struct PacedHydrant { + frames: Mutex>>, + frame_pace: Duration, + clock: Arc, +} + +impl MemWsResponder for PacedHydrant { + fn spawn_server( + &self, + _: Url, + mut recv: mpsc::UnboundedReceiver, + send: mpsc::Sender, + ) -> MemWsServerFuture { + let frames = self.frames.lock().unwrap().take().unwrap_or_default(); + let frame_pace = self.frame_pace; + let clock = self.clock.clone(); + let pong_send = send.clone(); + Box::pin(async move { + let pong_loop = async move { + loop { + match recv.recv().await { + Some(WsMessage::Ping(payload)) => { + if pong_send.send(WsMessage::Pong(payload)).await.is_err() { + return; + } + } + Some(WsMessage::Close { .. }) | None => return, + Some(_) => {} + } + } + }; + let frame_emit = async move { + for text in frames { + if !frame_pace.is_zero() { + clock.sleep(frame_pace).await; + } + if send.send(WsMessage::Text(text)).await.is_err() { + return; + } + } + }; + let _ = tokio::join!(pong_loop, frame_emit); + }) + } +} + +struct LatentSlingshot { + latency: Duration, + slingshot_calls: Arc, +} + +impl MemHttpResponder for LatentSlingshot { + fn respond(&self, request: &HttpRequest) -> MemHttpResponse { + self.slingshot_calls.fetch_add(1, Ordering::Relaxed); + let owner_rkey = parse_repo_lookup(&request.url, &repo_collection_nsid()); + let body = match owner_rkey { + Some((owner, rkey)) => { + let repo_did = owner.as_ref().replace("owner-", "repo-"); + serde_json::json!({ + "uri": format!("at://{}/{REPO_COLLECTION}/{}", owner.as_ref(), rkey.as_ref()), + "cid": "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i", + "value": { + "$type": REPO_COLLECTION, + "createdAt": "2026-05-01T00:00:00Z", + "knot": "oyster.cafe", + "name": "scallop", + "repoDid": repo_did, + } + }) + .to_string() + } + None => { + return MemHttpResponse { + latency: self.latency, + result: Ok(MemHttpBody::status_only(StatusCode::NOT_FOUND)), + }; + } + }; + MemHttpResponse { + latency: self.latency, + result: Ok(MemHttpBody::ok_json(Bytes::from(body))), + } + } +} diff --git a/bobbin/crates/bobbin-sim/src/workloads/cold_start_under_live_load.rs b/bobbin/crates/bobbin-sim/src/workloads/cold_start_under_live_load.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/workloads/cold_start_under_live_load.rs @@ -0,0 +1,220 @@ +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Duration; + +use bobbin_runtime::{Clock, MemHttpResponder, MemWsResponder, MemWsServerFuture, WsMessage}; +use tokio::sync::mpsc; +use url::Url; + +use crate::report::{SimOutcome, SimReport}; +use crate::workload::{Workload, WorkloadCtx, WorkloadHooks}; +use crate::workloads::util::{AssertNoSlingshot, format_rkey, format_tid}; + +const NAME: &str = "cold-start-under-live-load"; + +#[derive(Clone, Debug)] +pub struct ColdStartUnderLiveLoadConfig { + pub replay_frames: usize, + pub live_frames: usize, + pub live_pace_us: u64, + pub replay_pace_us: u64, +} + +impl Default for ColdStartUnderLiveLoadConfig { + fn default() -> Self { + Self { + replay_frames: 200, + live_frames: 50, + live_pace_us: 1_000, + replay_pace_us: 0, + } + } +} + +pub struct ColdStartUnderLiveLoad { + config: ColdStartUnderLiveLoadConfig, +} + +impl ColdStartUnderLiveLoad { + pub fn new(config: ColdStartUnderLiveLoadConfig) -> Self { + Self { config } + } +} + +impl Workload for ColdStartUnderLiveLoad { + fn name(&self) -> &'static str { + NAME + } + + fn build(self: Box, ctx: WorkloadCtx) -> WorkloadHooks { + let cfg = self.config.clone(); + let WorkloadCtx { + seed, + clock, + coverage, + store, + cancel, + .. + } = ctx; + + let total_frames = (cfg.replay_frames + cfg.live_frames) as u64; + let frame_log = build_frame_log(cfg.replay_frames, cfg.live_frames); + + let hydrant: Arc = Arc::new(LiveLoadHydrant { + frames: Mutex::new(Some(frame_log)), + replay_pace: Duration::from_micros(cfg.replay_pace_us), + live_pace: Duration::from_micros(cfg.live_pace_us), + replay_count: cfg.replay_frames, + clock: clock.clone(), + }); + let slingshot_probe = AssertNoSlingshot::new(); + let slingshot: Arc = Arc::new(slingshot_probe.clone()); + + let script = Box::pin(async move { + let started = clock.now_unix_micros(); + let mut rx = coverage.subscribe(); + let outcome = loop { + let snap = *rx.borrow_and_update(); + if snap.events_processed() >= total_frames { + break SimOutcome::Passed; + } + tokio::select! { + res = rx.changed() => match res { + Ok(()) => continue, + Err(_) => break SimOutcome::Failed, + }, + _ = cancel.cancelled() => break SimOutcome::Failed, + } + }; + let snap = coverage.snapshot(); + let virtual_runtime = + Duration::from_micros(clock.now_unix_micros().raw().saturating_sub(started.raw())); + let stray_slingshot = slingshot_probe.calls(); + let outcome = match outcome { + SimOutcome::Passed if stray_slingshot != 0 => SimOutcome::Failed, + other => other, + }; + let failure_reason = match outcome { + SimOutcome::Passed => None, + SimOutcome::Failed => Some(format!( + "events={} target={} last_cursor={} stray_slingshot={}", + snap.events_processed(), + total_frames, + snap.last_cursor().raw(), + stray_slingshot, + )), + SimOutcome::TimedOut => Some("timed out".into()), + }; + SimReport { + workload: NAME, + seed, + outcome, + virtual_runtime, + virtual_clock_end: clock.now_unix_micros(), + events_processed: snap.events_processed(), + last_cursor: snap.last_cursor().raw(), + edge_count: store.key_count() as u64, + resolver_hits: 0, + resolver_misses: 0, + consumer_too_slow_count: 0, + disconnect_count: 0, + last_disconnect: None, + warming_shadow: Default::default(), + warming_buffer: Default::default(), + failure_reason, + } + }); + + WorkloadHooks { + slingshot, + hydrant, + script, + } + } +} + +fn build_frame_log(replay: usize, live: usize) -> Vec<(bool, String)> { + let total = replay + live; + (0..total) + .map(|i| { + let id = (i + 1) as u64; + let is_live = i >= replay; + let owner = format!("did:plc:owner-{i}"); + let body = serde_json::json!({ + "id": id, + "type": "record", + "record": { + "live": is_live, + "did": owner, + "rev": format_tid(i), + "collection": "sh.tangled.repo", + "rkey": format_rkey(i), + "action": "create", + "record": { + "$type": "sh.tangled.repo", + "createdAt": "2026-05-01T00:00:00Z", + "knot": "oyster.cafe", + "name": format!("repo-{i}"), + "repoDid": format!("did:plc:repo-{i}") + } + } + }) + .to_string(); + (is_live, body) + }) + .collect() +} + +struct LiveLoadHydrant { + frames: Mutex>>, + replay_pace: Duration, + live_pace: Duration, + replay_count: usize, + clock: Arc, +} + +impl MemWsResponder for LiveLoadHydrant { + fn spawn_server( + &self, + _: Url, + mut recv: mpsc::UnboundedReceiver, + send: mpsc::Sender, + ) -> MemWsServerFuture { + let frames = self.frames.lock().unwrap().take().unwrap_or_default(); + let replay_pace = self.replay_pace; + let live_pace = self.live_pace; + let replay_count = self.replay_count; + let clock = self.clock.clone(); + let pong_send = send.clone(); + Box::pin(async move { + let pong_loop = async move { + loop { + match recv.recv().await { + Some(WsMessage::Ping(payload)) => { + if pong_send.send(WsMessage::Pong(payload)).await.is_err() { + return; + } + } + Some(WsMessage::Close { .. }) | None => return, + Some(_) => {} + } + } + }; + let frame_emit = async move { + for (i, (is_live, text)) in frames.into_iter().enumerate() { + let pace = if is_live { live_pace } else { replay_pace }; + if !pace.is_zero() { + clock.sleep(pace).await; + } + if send.send(WsMessage::Text(text)).await.is_err() { + return; + } + if i + 1 == replay_count { + clock.sleep(Duration::from_millis(1)).await; + } + } + }; + let _ = tokio::join!(pong_loop, frame_emit); + }) + } +} diff --git a/bobbin/crates/bobbin-sim/src/workloads/concurrent_reads_during_replay.rs b/bobbin/crates/bobbin-sim/src/workloads/concurrent_reads_during_replay.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/workloads/concurrent_reads_during_replay.rs @@ -0,0 +1,275 @@ +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use bobbin_edge_index::{PageCursor, PageLimit, SortDir}; +use bobbin_runtime::{Clock, MemHttpResponder, MemWsResponder, MemWsServerFuture, WsMessage}; +use bobbin_types::ids::{EdgeKey, SubjectRef}; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::nsid::Nsid; +use tokio::sync::mpsc; +use url::Url; + +use crate::report::{SimOutcome, SimReport}; +use crate::workload::{Workload, WorkloadCtx, WorkloadHooks}; +use crate::workloads::util::{AssertNoSlingshot, format_rkey, format_tid}; + +const NAME: &str = "concurrent-reads-during-replay"; + +fn target_did() -> Did { + Did::new_static("did:plc:lyna").expect("literal is a valid DID") +} + +#[derive(Clone, Debug)] +pub struct ConcurrentReadsDuringReplayConfig { + pub follow_frames: usize, + pub frame_pace_us: u64, + pub read_pace_us: u64, +} + +impl Default for ConcurrentReadsDuringReplayConfig { + fn default() -> Self { + Self { + follow_frames: 200, + frame_pace_us: 100, + read_pace_us: 50, + } + } +} + +pub struct ConcurrentReadsDuringReplay { + config: ConcurrentReadsDuringReplayConfig, +} + +impl ConcurrentReadsDuringReplay { + pub fn new(config: ConcurrentReadsDuringReplayConfig) -> Self { + Self { config } + } +} + +impl Workload for ConcurrentReadsDuringReplay { + fn name(&self) -> &'static str { + NAME + } + + fn build(self: Box, ctx: WorkloadCtx) -> WorkloadHooks { + let cfg = self.config.clone(); + let WorkloadCtx { + seed, + clock, + coverage, + store, + cancel, + .. + } = ctx; + + let total = cfg.follow_frames as u64; + let frame_log = build_frame_log(cfg.follow_frames); + + let hydrant: Arc = Arc::new(FollowHydrant { + frames: Mutex::new(Some(frame_log)), + frame_pace: Duration::from_micros(cfg.frame_pace_us), + clock: clock.clone(), + }); + let slingshot_probe = AssertNoSlingshot::new(); + let slingshot: Arc = Arc::new(slingshot_probe.clone()); + + let read_clock = clock.clone(); + let read_store = store.clone(); + let read_cancel = cancel.clone(); + let read_pace = Duration::from_micros(cfg.read_pace_us); + let monotonicity_violations = Arc::new(AtomicU64::new(0)); + let max_observed = Arc::new(AtomicU64::new(0)); + let read_iterations = Arc::new(AtomicU64::new(0)); + let monotonicity_violations_w = monotonicity_violations.clone(); + let max_observed_w = max_observed.clone(); + let read_iterations_w = read_iterations.clone(); + + let read_key = EdgeKey::new( + Nsid::new_static("sh.tangled.graph.follow").unwrap(), + SubjectRef::Did(target_did()), + ); + let final_key = read_key.clone(); + + let reader_task = tokio::spawn(async move { + let mut last_count: u64 = 0; + let mut last_list_len: usize = 0; + loop { + tokio::select! { + biased; + _ = read_cancel.cancelled() => return, + _ = read_clock.sleep(read_pace) => {} + } + let count = read_store.count(&read_key); + let listed = read_store.list( + &read_key, + PageCursor::Start, + PageLimit::new(50).unwrap(), + SortDir::Asc, + ); + if count < last_count || listed.items.len() < last_list_len { + monotonicity_violations_w.fetch_add(1, Ordering::Relaxed); + } + last_count = count.max(last_count); + last_list_len = listed.items.len().max(last_list_len); + max_observed_w.store(last_count, Ordering::Relaxed); + read_iterations_w.fetch_add(1, Ordering::Relaxed); + } + }); + + let script = Box::pin(async move { + let started = clock.now_unix_micros(); + let mut rx = coverage.subscribe(); + let outcome = loop { + let snap = *rx.borrow_and_update(); + if snap.events_processed() >= total { + break SimOutcome::Passed; + } + tokio::select! { + res = rx.changed() => match res { + Ok(()) => continue, + Err(_) => break SimOutcome::Failed, + }, + _ = cancel.cancelled() => break SimOutcome::Failed, + } + }; + + cancel.cancel(); + let _ = reader_task.await; + + let snap = coverage.snapshot(); + let virtual_runtime = + Duration::from_micros(clock.now_unix_micros().raw().saturating_sub(started.raw())); + let violations = monotonicity_violations.load(Ordering::Relaxed); + let observed = max_observed.load(Ordering::Relaxed); + let iters = read_iterations.load(Ordering::Relaxed); + let final_count = store.count(&final_key); + let stray_slingshot = slingshot_probe.calls(); + let outcome = match outcome { + SimOutcome::Passed + if violations == 0 && final_count == total && stray_slingshot == 0 => + { + SimOutcome::Passed + } + SimOutcome::Passed => SimOutcome::Failed, + other => other, + }; + let failure_reason = match outcome { + SimOutcome::Passed => None, + SimOutcome::Failed => Some(format!( + "events={} target={} reads={} max_observed_count={} final_count={} \ + monotonicity_violations={} stray_slingshot={}", + snap.events_processed(), + total, + iters, + observed, + final_count, + violations, + stray_slingshot, + )), + SimOutcome::TimedOut => Some("timed out".into()), + }; + SimReport { + workload: NAME, + seed, + outcome, + virtual_runtime, + virtual_clock_end: clock.now_unix_micros(), + events_processed: snap.events_processed(), + last_cursor: snap.last_cursor().raw(), + edge_count: store.key_count() as u64, + resolver_hits: 0, + resolver_misses: 0, + consumer_too_slow_count: 0, + disconnect_count: 0, + last_disconnect: None, + warming_shadow: Default::default(), + warming_buffer: Default::default(), + failure_reason, + } + }); + + WorkloadHooks { + slingshot, + hydrant, + script, + } + } +} + +fn build_frame_log(count: usize) -> Vec { + let target = target_did(); + (0..count) + .map(|i| { + let id = (i + 1) as u64; + let actor = + Did::::new_owned(format!("did:plc:nautilus-{i}")).expect("valid DID"); + serde_json::json!({ + "id": id, + "type": "record", + "record": { + "live": false, + "did": actor.as_ref(), + "rev": format_tid(i), + "collection": "sh.tangled.graph.follow", + "rkey": format_rkey(i), + "action": "create", + "record": { + "$type": "sh.tangled.graph.follow", + "createdAt": "2026-05-01T00:00:00Z", + "subject": target.as_ref(), + } + } + }) + .to_string() + }) + .collect() +} + +struct FollowHydrant { + frames: Mutex>>, + frame_pace: Duration, + clock: Arc, +} + +impl MemWsResponder for FollowHydrant { + fn spawn_server( + &self, + _: Url, + mut recv: mpsc::UnboundedReceiver, + send: mpsc::Sender, + ) -> MemWsServerFuture { + let frames = self.frames.lock().unwrap().take().unwrap_or_default(); + let frame_pace = self.frame_pace; + let clock = self.clock.clone(); + let pong_send = send.clone(); + Box::pin(async move { + let pong_loop = async move { + loop { + match recv.recv().await { + Some(WsMessage::Ping(payload)) => { + if pong_send.send(WsMessage::Pong(payload)).await.is_err() { + return; + } + } + Some(WsMessage::Close { .. }) | None => return, + Some(_) => {} + } + } + }; + let frame_emit = async move { + for text in frames { + if !frame_pace.is_zero() { + clock.sleep(frame_pace).await; + } + if send.send(WsMessage::Text(text)).await.is_err() { + return; + } + } + }; + let _ = tokio::join!(pong_loop, frame_emit); + }) + } +} diff --git a/bobbin/crates/bobbin-sim/src/workloads/frame_burst.rs b/bobbin/crates/bobbin-sim/src/workloads/frame_burst.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/workloads/frame_burst.rs @@ -0,0 +1,190 @@ +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use bobbin_runtime::{MemHttpResponder, MemWsResponder, MemWsServerFuture, WsMessage}; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use tokio::sync::mpsc; +use url::Url; + +use crate::report::{SimOutcome, SimReport}; +use crate::workload::{Workload, WorkloadCtx, WorkloadHooks}; +use crate::workloads::util::{AssertNoSlingshot, format_rkey, format_tid}; + +const NAME: &str = "frame-burst"; + +fn authority_did() -> Did { + Did::new_static("did:plc:squid").expect("literal is a valid DID") +} + +pub struct FrameBurst { + frames: usize, +} + +impl FrameBurst { + pub fn new(frames: usize) -> Self { + Self { frames } + } +} + +impl Workload for FrameBurst { + fn name(&self) -> &'static str { + NAME + } + + fn build(self: Box, ctx: WorkloadCtx) -> WorkloadHooks { + let count = self.frames; + let frames = build_frame_script(count); + let session_count = Arc::new(AtomicU64::new(0)); + + let hydrant: Arc = Arc::new(BurstHydrant { + frames: Mutex::new(Some(frames)), + session_count: session_count.clone(), + }); + let slingshot_probe = AssertNoSlingshot::new(); + let slingshot: Arc = Arc::new(slingshot_probe.clone()); + + let WorkloadCtx { + seed, + clock, + coverage, + store, + cancel, + .. + } = ctx; + let target = count as u64; + let sessions = session_count.clone(); + + let script = Box::pin(async move { + let mut rx = coverage.subscribe(); + let started = clock.now_unix_micros(); + let outcome = loop { + let snap = *rx.borrow_and_update(); + if snap.events_processed() >= target { + break SimOutcome::Passed; + } + tokio::select! { + res = rx.changed() => match res { + Ok(()) => continue, + Err(_) => break SimOutcome::Failed, + }, + _ = cancel.cancelled() => break SimOutcome::Failed, + } + }; + let snap = coverage.snapshot(); + let virtual_runtime = + Duration::from_micros(clock.now_unix_micros().raw().saturating_sub(started.raw())); + let stray_slingshot = slingshot_probe.calls(); + let session_total = sessions.load(Ordering::Relaxed); + let outcome = match outcome { + SimOutcome::Passed if stray_slingshot == 0 && session_total == 1 => { + SimOutcome::Passed + } + SimOutcome::Passed => SimOutcome::Failed, + other => other, + }; + let failure_reason = match outcome { + SimOutcome::Passed => None, + SimOutcome::Failed => Some(format!( + "events={}/{} stray_slingshot={} sessions={}", + snap.events_processed(), + target, + stray_slingshot, + session_total, + )), + SimOutcome::TimedOut => Some("timed out".into()), + }; + SimReport { + workload: NAME, + seed, + outcome, + virtual_runtime, + virtual_clock_end: clock.now_unix_micros(), + events_processed: snap.events_processed(), + last_cursor: snap.last_cursor().raw(), + edge_count: store.key_count() as u64, + resolver_hits: 0, + resolver_misses: 0, + consumer_too_slow_count: 0, + disconnect_count: 0, + last_disconnect: None, + warming_shadow: Default::default(), + warming_buffer: Default::default(), + failure_reason, + } + }); + + WorkloadHooks { + slingshot, + hydrant, + script, + } + } +} + +fn build_frame_script(count: usize) -> Vec { + let authority = authority_did(); + (0..count) + .map(|i| { + let repo_did = + Did::::new_owned(format!("did:plc:squid-{i}")).expect("valid DID"); + serde_json::json!({ + "id": (i + 1) as u64, + "type": "record", + "record": { + "live": false, + "did": authority.as_ref(), + "rev": format_tid(i), + "collection": "sh.tangled.repo", + "rkey": format_rkey(i), + "action": "create", + "record": { + "$type": "sh.tangled.repo", + "createdAt": "2026-05-01T00:00:00Z", + "knot": "oyster.cafe", + "name": format!("repo-{i}"), + "repoDid": repo_did.as_ref() + } + } + }) + .to_string() + }) + .collect() +} + +struct BurstHydrant { + frames: Mutex>>, + session_count: Arc, +} + +impl MemWsResponder for BurstHydrant { + fn spawn_server( + &self, + _: Url, + mut recv: mpsc::UnboundedReceiver, + send: mpsc::Sender, + ) -> MemWsServerFuture { + let frames = self.frames.lock().unwrap().take().unwrap_or_default(); + self.session_count.fetch_add(1, Ordering::Relaxed); + Box::pin(async move { + for text in frames { + if send.send(WsMessage::Text(text)).await.is_err() { + return; + } + } + loop { + match recv.recv().await { + Some(WsMessage::Ping(payload)) => { + if send.send(WsMessage::Pong(payload)).await.is_err() { + return; + } + } + Some(WsMessage::Close { .. }) | None => return, + Some(_) => {} + } + } + }) + } +} diff --git a/bobbin/crates/bobbin-sim/src/workloads/hydrant_disconnect_barrage.rs b/bobbin/crates/bobbin-sim/src/workloads/hydrant_disconnect_barrage.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/workloads/hydrant_disconnect_barrage.rs @@ -0,0 +1,257 @@ +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use bobbin_runtime::{MemHttpResponder, MemWsResponder, MemWsServerFuture, WsMessage}; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use tokio::sync::mpsc; +use url::Url; + +use crate::report::{SimOutcome, SimReport}; +use crate::workload::{Workload, WorkloadCtx, WorkloadHooks}; +use crate::workloads::util::{AssertNoSlingshot, format_rkey, format_tid}; + +const NAME: &str = "hydrant-disconnect-barrage"; + +fn authority_did() -> Did { + Did::new_static("did:plc:squid").expect("literal is a valid DID") +} + +#[derive(Clone, Debug)] +pub struct HydrantDisconnectBarrageConfig { + pub frames: usize, + pub disconnect_after_frames_per_session: Vec, +} + +impl Default for HydrantDisconnectBarrageConfig { + fn default() -> Self { + Self { + frames: 256, + disconnect_after_frames_per_session: vec![50, 80, 60], + } + } +} + +pub struct HydrantDisconnectBarrage { + config: HydrantDisconnectBarrageConfig, +} + +impl HydrantDisconnectBarrage { + pub fn new(config: HydrantDisconnectBarrageConfig) -> Self { + Self { config } + } +} + +impl Workload for HydrantDisconnectBarrage { + fn name(&self) -> &'static str { + NAME + } + + fn build(self: Box, ctx: WorkloadCtx) -> WorkloadHooks { + let cfg = self.config.clone(); + let WorkloadCtx { + seed, + clock, + coverage, + store, + cancel, + .. + } = ctx; + + let frames_total = cfg.frames as u64; + let frame_log = build_frame_log(cfg.frames); + let drop_schedule = Arc::new(Mutex::new(cfg.disconnect_after_frames_per_session.clone())); + let session_count = Arc::new(AtomicU64::new(0)); + let disconnect_count = Arc::new(AtomicU64::new(0)); + + let hydrant: Arc = Arc::new(BarrageHydrant { + frame_log: Arc::new(frame_log), + drop_schedule, + session_count: session_count.clone(), + disconnect_count: disconnect_count.clone(), + }); + let slingshot_probe = AssertNoSlingshot::new(); + let slingshot: Arc = Arc::new(slingshot_probe.clone()); + + let disconnects = disconnect_count.clone(); + let sessions = session_count.clone(); + let script = Box::pin(async move { + let started = clock.now_unix_micros(); + let mut rx = coverage.subscribe(); + let outcome = loop { + let snap = *rx.borrow_and_update(); + if snap.events_processed() == frames_total { + break SimOutcome::Passed; + } + if snap.events_processed() > frames_total { + break SimOutcome::Failed; + } + tokio::select! { + res = rx.changed() => match res { + Ok(()) => continue, + Err(_) => break SimOutcome::Failed, + }, + _ = cancel.cancelled() => break SimOutcome::Failed, + } + }; + let snap = coverage.snapshot(); + let virtual_runtime = + Duration::from_micros(clock.now_unix_micros().raw().saturating_sub(started.raw())); + let dc = disconnects.load(Ordering::Relaxed); + let ss = sessions.load(Ordering::Relaxed); + let stray_slingshot = slingshot_probe.calls(); + let outcome = match outcome { + SimOutcome::Passed if dc == 0 => SimOutcome::Failed, + SimOutcome::Passed if stray_slingshot != 0 => SimOutcome::Failed, + other => other, + }; + let failure_reason = match outcome { + SimOutcome::Passed => None, + SimOutcome::Failed => Some(format!( + "events={} target={} disconnects={} sessions={} stray_slingshot={}", + snap.events_processed(), + frames_total, + dc, + ss, + stray_slingshot, + )), + SimOutcome::TimedOut => Some("timed out".into()), + }; + SimReport { + workload: NAME, + seed, + outcome, + virtual_runtime, + virtual_clock_end: clock.now_unix_micros(), + events_processed: snap.events_processed(), + last_cursor: snap.last_cursor().raw(), + edge_count: store.key_count() as u64, + resolver_hits: 0, + resolver_misses: 0, + consumer_too_slow_count: 0, + disconnect_count: 0, + last_disconnect: None, + warming_shadow: Default::default(), + warming_buffer: Default::default(), + failure_reason, + } + }); + + WorkloadHooks { + slingshot, + hydrant, + script, + } + } +} + +fn build_frame_log(count: usize) -> Vec<(u64, String)> { + let authority = authority_did(); + (0..count) + .map(|i| { + let id = (i + 1) as u64; + let repo_did = + Did::::new_owned(format!("did:plc:squid-{i}")).expect("valid DID"); + let body = serde_json::json!({ + "id": id, + "type": "record", + "record": { + "live": false, + "did": authority.as_ref(), + "rev": format_tid(i), + "collection": "sh.tangled.repo", + "rkey": format_rkey(i), + "action": "create", + "record": { + "$type": "sh.tangled.repo", + "createdAt": "2026-05-01T00:00:00Z", + "knot": "oyster.cafe", + "name": format!("repo-{i}"), + "repoDid": repo_did.as_ref() + } + } + }) + .to_string(); + (id, body) + }) + .collect() +} + +struct BarrageHydrant { + frame_log: Arc>, + drop_schedule: Arc>>, + session_count: Arc, + disconnect_count: Arc, +} + +impl MemWsResponder for BarrageHydrant { + fn spawn_server( + &self, + url: Url, + mut recv: mpsc::UnboundedReceiver, + send: mpsc::Sender, + ) -> MemWsServerFuture { + let cursor = parse_cursor(&url).unwrap_or(0); + let frame_log = self.frame_log.clone(); + let drop_after = { + let mut guard = self.drop_schedule.lock().unwrap(); + if guard.is_empty() { + usize::MAX + } else { + guard.remove(0) + } + }; + self.session_count.fetch_add(1, Ordering::Relaxed); + let disconnects = self.disconnect_count.clone(); + let pong_send = send.clone(); + Box::pin(async move { + let pong_loop = async move { + loop { + match recv.recv().await { + Some(WsMessage::Ping(payload)) => { + if pong_send.send(WsMessage::Pong(payload)).await.is_err() { + return; + } + } + Some(WsMessage::Close { .. }) | None => return, + Some(_) => {} + } + } + }; + let frame_emit = async move { + let mut emitted = 0usize; + for (id, body) in frame_log.iter() { + if *id < cursor { + continue; + } + if send.send(WsMessage::Text(body.clone())).await.is_err() { + return; + } + emitted += 1; + if emitted >= drop_after { + disconnects.fetch_add(1, Ordering::Relaxed); + let _ = send + .send(WsMessage::Close { + code: 1011, + reason: "scripted disconnect".into(), + }) + .await; + return; + } + } + }; + let _ = tokio::join!(pong_loop, frame_emit); + }) + } +} + +fn parse_cursor(url: &Url) -> Option { + for (k, v) in url.query_pairs() { + if k == "cursor" { + return v.parse().ok(); + } + } + None +} diff --git a/bobbin/crates/bobbin-sim/src/workloads/mod.rs b/bobbin/crates/bobbin-sim/src/workloads/mod.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/workloads/mod.rs @@ -0,0 +1,16 @@ +pub mod cancel_mid_hydration; +pub mod cold_start_under_live_load; +pub mod concurrent_reads_during_replay; +pub mod frame_burst; +pub mod hydrant_disconnect_barrage; +pub mod slingshot_flap; +pub mod util; + +pub use cancel_mid_hydration::{CancelMidHydration, CancelMidHydrationConfig}; +pub use cold_start_under_live_load::{ColdStartUnderLiveLoad, ColdStartUnderLiveLoadConfig}; +pub use concurrent_reads_during_replay::{ + ConcurrentReadsDuringReplay, ConcurrentReadsDuringReplayConfig, +}; +pub use frame_burst::FrameBurst; +pub use hydrant_disconnect_barrage::{HydrantDisconnectBarrage, HydrantDisconnectBarrageConfig}; +pub use slingshot_flap::{SlingshotFlap, SlingshotFlapConfig}; diff --git a/bobbin/crates/bobbin-sim/src/workloads/slingshot_flap.rs b/bobbin/crates/bobbin-sim/src/workloads/slingshot_flap.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/workloads/slingshot_flap.rs @@ -0,0 +1,418 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use bobbin_runtime::{ + Clock, HttpRequest, MemHttpBody, MemHttpResponder, MemHttpResponse, MemWsResponder, + MemWsServerFuture, NetworkError, UnixMicros, WsMessage, +}; +use bytes::Bytes; +use http::StatusCode; +use jacquard_common::DefaultStr; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::tid::Tid; +use tokio::sync::mpsc; +use url::Url; + +use crate::report::{SimOutcome, SimReport}; +use crate::workload::{Workload, WorkloadCtx, WorkloadHooks}; +use crate::workloads::util::{format_rkey, format_tid, parse_repo_lookup}; + +const NAME: &str = "slingshot-flap"; +const REPO_COLLECTION: &str = "sh.tangled.repo"; + +fn repo_collection_nsid() -> Nsid { + Nsid::new_static(REPO_COLLECTION).expect("REPO_COLLECTION literal is a valid NSID") +} + +#[derive(Clone, Debug)] +pub struct SlingshotFlapConfig { + pub cross_did_stars: usize, + pub normal_latency_ms: u64, + pub brownout_start_ms: u64, + pub brownout_duration_ms: u64, + pub brownout_latency_ms: u64, + pub brownout_enabled: bool, + pub hydrant_send_timeout_ms: u64, + pub hydrant_frame_pace_us: u64, + pub omit_target_repos: bool, + pub emit_live_promotion_frame: bool, +} + +impl Default for SlingshotFlapConfig { + fn default() -> Self { + Self { + cross_did_stars: 200, + normal_latency_ms: 2, + brownout_start_ms: 1_000, + brownout_duration_ms: 5_000, + brownout_latency_ms: 200, + brownout_enabled: true, + hydrant_send_timeout_ms: 30_000, + hydrant_frame_pace_us: 0, + omit_target_repos: false, + emit_live_promotion_frame: false, + } + } +} + +pub struct SlingshotFlap { + config: SlingshotFlapConfig, +} + +impl SlingshotFlap { + pub fn new(config: SlingshotFlapConfig) -> Self { + Self { config } + } +} + +impl Workload for SlingshotFlap { + fn name(&self) -> &'static str { + NAME + } + + fn build(self: Box, ctx: WorkloadCtx) -> WorkloadHooks { + let cfg = self.config.clone(); + let WorkloadCtx { + seed, + clock, + coverage, + store, + cancel, + consumer_too_slow_count, + .. + } = ctx; + + let star_count = cfg.cross_did_stars; + let repo_count = if cfg.omit_target_repos { 0 } else { star_count }; + let live_count = if cfg.emit_live_promotion_frame { 1 } else { 0 }; + let total_frames = (star_count + repo_count + live_count) as u64; + let target_owner_count = star_count; + + let started_unix_pre = clock.now_unix_micros(); + let frame_script = Arc::new(build_frame_script(&cfg, started_unix_pre)); + let session_count = Arc::new(AtomicU64::new(0)); + let hydrant: Arc = Arc::new(FlapHydrant { + frames: frame_script, + send_timeout: Duration::from_millis(cfg.hydrant_send_timeout_ms), + frame_pace: Duration::from_micros(cfg.hydrant_frame_pace_us), + consumer_too_slow: consumer_too_slow_count.clone(), + session_count: session_count.clone(), + clock: clock.clone(), + }); + + let started_unix = started_unix_pre; + let slingshot: Arc = Arc::new(FlapSlingshot { + cfg: cfg.clone(), + clock: clock.clone(), + started_unix, + }); + + let cts_counter = consumer_too_slow_count.clone(); + let sessions_counter = session_count.clone(); + let script = Box::pin(async move { + let started = started_unix; + + let mut rx = coverage.subscribe(); + let outcome = loop { + let snap = *rx.borrow_and_update(); + if snap.events_processed() >= total_frames { + break SimOutcome::Passed; + } + if cts_counter.load(Ordering::Relaxed) > 0 { + break SimOutcome::Failed; + } + tokio::select! { + res = rx.changed() => match res { + Ok(()) => continue, + Err(_) => break SimOutcome::Failed, + }, + _ = cancel.cancelled() => break SimOutcome::Failed, + } + }; + let snap = coverage.snapshot(); + let virtual_runtime = + Duration::from_micros(clock.now_unix_micros().raw().saturating_sub(started.raw())); + let cts = cts_counter.load(Ordering::Relaxed); + let sessions = sessions_counter.load(Ordering::Relaxed); + let failure_reason = match outcome { + SimOutcome::Passed => None, + SimOutcome::Failed if cts > 0 => Some(format!( + "ConsumerTooSlow fired {cts} times across {sessions} sessions; processed {}/{} events before disconnect", + snap.events_processed(), + total_frames, + )), + SimOutcome::Failed => Some(format!( + "only {}/{} events processed across {sessions} sessions (expected {} owners staged)", + snap.events_processed(), + total_frames, + target_owner_count, + )), + SimOutcome::TimedOut => Some("timed out".into()), + }; + SimReport { + workload: NAME, + seed, + outcome, + virtual_runtime, + virtual_clock_end: clock.now_unix_micros(), + events_processed: snap.events_processed(), + last_cursor: snap.last_cursor().raw(), + edge_count: store.key_count() as u64, + resolver_hits: 0, + resolver_misses: 0, + consumer_too_slow_count: cts, + disconnect_count: 0, + last_disconnect: None, + warming_shadow: Default::default(), + warming_buffer: Default::default(), + failure_reason, + } + }); + + WorkloadHooks { + slingshot, + hydrant, + script, + } + } +} + +fn build_frame_script(cfg: &SlingshotFlapConfig, started_unix: UnixMicros) -> Vec<(u64, String)> { + let mut frames = Vec::with_capacity(cfg.cross_did_stars * 2 + 1); + for i in 0..cfg.cross_did_stars { + let id = (i + 1) as u64; + let star_did = format!("did:plc:starer-{i}"); + let target_owner = format!("did:plc:owner-{i}"); + let target_rkey = format_rkey(i); + let body = serde_json::json!({ + "id": id, + "type": "record", + "record": { + "live": false, + "did": star_did, + "rev": format_tid(i), + "collection": "sh.tangled.feed.star", + "rkey": format_rkey(i + 1_000_000), + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subject": format!("at://{target_owner}/{REPO_COLLECTION}/{target_rkey}") + } + } + }) + .to_string(); + frames.push((id, body)); + } + if !cfg.omit_target_repos { + for i in 0..cfg.cross_did_stars { + let id = (cfg.cross_did_stars + i + 1) as u64; + let owner = format!("did:plc:owner-{i}"); + let repo_did = format!("did:plc:repo-{i}"); + let body = serde_json::json!({ + "id": id, + "type": "record", + "record": { + "live": false, + "did": owner, + "rev": format_tid(cfg.cross_did_stars + i), + "collection": REPO_COLLECTION, + "rkey": format_rkey(i), + "action": "create", + "record": { + "$type": REPO_COLLECTION, + "createdAt": "2026-05-01T00:00:00Z", + "knot": "oyster.cafe", + "name": format!("repo-{i}"), + "repoDid": repo_did, + } + } + }) + .to_string(); + frames.push((id, body)); + } + } + if cfg.emit_live_promotion_frame { + let prior = cfg.cross_did_stars + + if cfg.omit_target_repos { + 0 + } else { + cfg.cross_did_stars + }; + let id = (prior + 1) as u64; + let promoter_name = format!("periwinkle-{prior}"); + let promoter_did = format!("did:plc:{promoter_name}"); + let promoter_rkey = format_rkey(prior + 1); + let live_rev = Tid::from_time(started_unix.raw(), 0); + let body = serde_json::json!({ + "id": id, + "type": "record", + "record": { + "live": true, + "did": promoter_did, + "rev": live_rev.as_str(), + "collection": REPO_COLLECTION, + "rkey": promoter_rkey, + "action": "create", + "record": { + "$type": REPO_COLLECTION, + "createdAt": "2026-05-01T00:00:00Z", + "knot": "oyster.cafe", + "name": promoter_name, + "repoDid": promoter_did, + } + } + }) + .to_string(); + frames.push((id, body)); + } + frames +} + +fn parse_cursor(url: &Url) -> u64 { + url.query_pairs() + .find_map(|(k, v)| (k == "cursor").then(|| v.parse().ok()).flatten()) + .unwrap_or(0) +} + +struct FlapHydrant { + frames: Arc>, + send_timeout: Duration, + frame_pace: Duration, + consumer_too_slow: Arc, + session_count: Arc, + clock: Arc, +} + +impl MemWsResponder for FlapHydrant { + fn spawn_server( + &self, + url: Url, + mut recv: mpsc::UnboundedReceiver, + send: mpsc::Sender, + ) -> MemWsServerFuture { + let cursor = parse_cursor(&url); + let frames = self.frames.clone(); + let send_timeout = self.send_timeout; + let frame_pace = self.frame_pace; + let consumer_too_slow = self.consumer_too_slow.clone(); + let clock = self.clock.clone(); + self.session_count.fetch_add(1, Ordering::Relaxed); + let pong_send = send.clone(); + Box::pin(async move { + let pong_loop = async move { + loop { + match recv.recv().await { + Some(WsMessage::Ping(payload)) => { + if pong_send.send(WsMessage::Pong(payload)).await.is_err() { + return; + } + } + Some(WsMessage::Close { .. }) | None => return, + Some(_) => {} + } + } + }; + let clock_for_emit = clock.clone(); + let frame_emit = async move { + for (id, text) in frames.iter() { + if *id < cursor { + continue; + } + if !frame_pace.is_zero() { + clock_for_emit.sleep(frame_pace).await; + } + let res = tokio::time::timeout( + send_timeout, + send.send(WsMessage::Text(text.clone())), + ) + .await; + match res { + Ok(Ok(())) => continue, + Ok(Err(_)) => return, + Err(_) => { + consumer_too_slow.fetch_add(1, Ordering::Relaxed); + let err_frame = serde_json::json!({ + "type": "error", + "error": "ConsumerTooSlow", + "message": format!( + "sim hydrant send_timeout {send_timeout:?} exhausted", + ), + }) + .to_string(); + let _ = tokio::time::timeout( + Duration::from_secs(1), + send.send(WsMessage::Text(err_frame)), + ) + .await; + let _ = tokio::time::timeout( + Duration::from_secs(1), + send.send(WsMessage::Close { + code: 1011, + reason: "ConsumerTooSlow".into(), + }), + ) + .await; + return; + } + } + } + }; + let _ = tokio::join!(pong_loop, frame_emit); + }) + } +} + +struct FlapSlingshot { + cfg: SlingshotFlapConfig, + clock: Arc, + started_unix: UnixMicros, +} + +impl MemHttpResponder for FlapSlingshot { + fn respond(&self, request: &HttpRequest) -> MemHttpResponse { + let now = self.clock.now_unix_micros().raw(); + let elapsed_ms = now.saturating_sub(self.started_unix.raw()) / 1000; + let in_brownout = self.cfg.brownout_enabled + && elapsed_ms >= self.cfg.brownout_start_ms + && elapsed_ms < self.cfg.brownout_start_ms + self.cfg.brownout_duration_ms; + + if in_brownout { + return MemHttpResponse { + latency: Duration::from_millis(self.cfg.brownout_latency_ms), + result: Err(NetworkError::Transport("slingshot brownout".into())), + }; + } + + let owner_rkey = parse_repo_lookup(&request.url, &repo_collection_nsid()); + let body = match owner_rkey { + Some((owner, rkey)) => { + let repo_did = owner.as_ref().replace("owner-", "repo-"); + serde_json::json!({ + "uri": format!("at://{}/{REPO_COLLECTION}/{}", owner.as_ref(), rkey.as_ref()), + "cid": "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i", + "value": { + "$type": REPO_COLLECTION, + "createdAt": "2026-05-01T00:00:00Z", + "knot": "oyster.cafe", + "name": "scallop", + "repoDid": repo_did, + } + }) + .to_string() + } + None => { + return MemHttpResponse { + latency: Duration::from_millis(self.cfg.normal_latency_ms), + result: Ok(MemHttpBody::status_only(StatusCode::NOT_FOUND)), + }; + } + }; + + MemHttpResponse { + latency: Duration::from_millis(self.cfg.normal_latency_ms), + result: Ok(MemHttpBody::ok_json(Bytes::from(body))), + } + } +} diff --git a/bobbin/crates/bobbin-sim/src/workloads/util.rs b/bobbin/crates/bobbin-sim/src/workloads/util.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/src/workloads/util.rs @@ -0,0 +1,80 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use bobbin_runtime::{HttpRequest, MemHttpBody, MemHttpResponder, MemHttpResponse}; +use http::StatusCode; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::recordkey::Rkey; +use url::Url; + +const ALPHABET_RKEY: &[u8] = b"abcdefghijklmnopqrstuvwxyz234567"; +const ALPHABET_TID: &[u8] = b"234567abcdefghijklmnopqrstuvwxyz"; +const ENCODED_LEN: usize = 13; + +pub fn format_rkey(i: usize) -> String { + encode_padded(i.saturating_add(1), ALPHABET_RKEY, b'a') +} + +pub fn format_tid(i: usize) -> String { + encode_padded(i.saturating_add(1), ALPHABET_TID, b'2') +} + +fn encode_padded(mut idx: usize, alphabet: &[u8], pad: u8) -> String { + let mut buf = [pad; ENCODED_LEN]; + let mut pos = buf.len(); + while idx > 0 && pos > 0 { + pos -= 1; + buf[pos] = alphabet[idx % alphabet.len()]; + idx /= alphabet.len(); + } + String::from_utf8(buf.to_vec()).unwrap() +} + +pub fn parse_repo_lookup( + url: &Url, + expected_collection: &Nsid, +) -> Option<(Did, Rkey)> { + let mut repo: Option> = None; + let mut collection: Option> = None; + let mut rkey: Option> = None; + for (k, v) in url.query_pairs() { + match k.as_ref() { + "repo" => repo = Did::new_owned(&v).ok(), + "collection" => collection = Nsid::new_owned(&v).ok(), + "rkey" => rkey = Rkey::new_owned(&v).ok(), + _ => {} + } + } + if collection.as_ref()? != expected_collection { + return None; + } + Some((repo?, rkey?)) +} + +#[derive(Clone, Default)] +pub struct AssertNoSlingshot { + calls: Arc, +} + +impl AssertNoSlingshot { + pub fn new() -> Self { + Self::default() + } + + pub fn calls(&self) -> u64 { + self.calls.load(Ordering::Relaxed) + } +} + +impl MemHttpResponder for AssertNoSlingshot { + fn respond(&self, _: &HttpRequest) -> MemHttpResponse { + self.calls.fetch_add(1, Ordering::Relaxed); + MemHttpResponse { + latency: Duration::ZERO, + result: Ok(MemHttpBody::status_only(StatusCode::NOT_FOUND)), + } + } +} diff --git a/bobbin/crates/bobbin-sim/tests/determinism_leak.rs b/bobbin/crates/bobbin-sim/tests/determinism_leak.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/tests/determinism_leak.rs @@ -0,0 +1,236 @@ +use std::num::NonZeroUsize; +use std::time::Duration; + +use bobbin_sim::workloads::{ + CancelMidHydration, CancelMidHydrationConfig, ColdStartUnderLiveLoad, + ColdStartUnderLiveLoadConfig, ConcurrentReadsDuringReplay, ConcurrentReadsDuringReplayConfig, + FrameBurst, HydrantDisconnectBarrage, HydrantDisconnectBarrageConfig, SlingshotFlap, + SlingshotFlapConfig, +}; +use bobbin_sim::{LeakOutcome, LeakRunConfig, Workload, run_leak_check}; + +const PARALLELISM_SWEEP: &[usize] = &[1, 4, 16, 64]; +const SEEDS: &[u64] = &[1, 7, 42, 1337]; + +#[test] +fn frame_burst_is_byte_deterministic_across_parallelism_sweep() { + for &par in PARALLELISM_SWEEP { + for &seed in SEEDS { + let parallelism = NonZeroUsize::new(par).unwrap(); + let config = LeakRunConfig { + seed, + parallelism, + max_virtual_runtime: Duration::from_secs(30), + mem_ws_capacity: 4096, + warming_buffer_enabled: true, + }; + let factory = || -> Box { Box::new(FrameBurst::new(64)) }; + let result = run_leak_check(config.clone(), factory); + assert!( + result.passed(), + "frame-burst leak at par={par} seed={seed}: {:?}", + result.outcome, + ); + match &result.outcome { + LeakOutcome::Match => {} + other => panic!("non-match outcome despite passed(): {other:?}"), + } + assert_eq!( + result.first_report.events_processed, 64, + "frame-burst should process all 64 frames at par={par} seed={seed}", + ); + } + } +} + +#[test] +fn cancel_mid_hydration_is_byte_deterministic() { + for &par in PARALLELISM_SWEEP { + for &seed in SEEDS { + let parallelism = NonZeroUsize::new(par).unwrap(); + let config = LeakRunConfig { + seed, + parallelism, + max_virtual_runtime: Duration::from_secs(60), + mem_ws_capacity: 4096, + warming_buffer_enabled: true, + }; + let factory = || -> Box { + Box::new(CancelMidHydration::new(CancelMidHydrationConfig { + frames: 200, + cancel_at_events: 50, + slingshot_latency_ms: 50, + frame_pace_us: 100, + })) + }; + let result = run_leak_check(config.clone(), factory); + assert!( + result.passed(), + "cancel-mid-hydration leak at par={par} seed={seed}: {:?}", + result.outcome, + ); + match &result.outcome { + LeakOutcome::Match => {} + other => panic!("non-match outcome despite passed(): {other:?}"), + } + } + } +} + +#[test] +fn hydrant_disconnect_barrage_is_byte_deterministic() { + for &par in PARALLELISM_SWEEP { + for &seed in SEEDS { + let parallelism = NonZeroUsize::new(par).unwrap(); + let config = LeakRunConfig { + seed, + parallelism, + max_virtual_runtime: Duration::from_secs(60), + mem_ws_capacity: 4096, + warming_buffer_enabled: true, + }; + let factory = || -> Box { + Box::new(HydrantDisconnectBarrage::new( + HydrantDisconnectBarrageConfig { + frames: 256, + disconnect_after_frames_per_session: vec![50, 80, 60], + }, + )) + }; + let result = run_leak_check(config.clone(), factory); + assert!( + result.passed(), + "hydrant-disconnect-barrage leak at par={par} seed={seed}: {:?}", + result.outcome, + ); + assert_eq!( + result.first_report.events_processed, 256, + "expected all 256 frames processed", + ); + } + } +} + +#[test] +fn cold_start_under_live_load_is_byte_deterministic() { + for &par in PARALLELISM_SWEEP { + for &seed in SEEDS { + let parallelism = NonZeroUsize::new(par).unwrap(); + let config = LeakRunConfig { + seed, + parallelism, + max_virtual_runtime: Duration::from_secs(60), + mem_ws_capacity: 4096, + warming_buffer_enabled: true, + }; + let factory = || -> Box { + Box::new(ColdStartUnderLiveLoad::new(ColdStartUnderLiveLoadConfig { + replay_frames: 200, + live_frames: 50, + live_pace_us: 1_000, + replay_pace_us: 0, + })) + }; + let result = run_leak_check(config.clone(), factory); + assert!( + result.passed(), + "cold-start-under-live-load leak at par={par} seed={seed}: {:?}", + result.outcome, + ); + assert_eq!( + result.first_report.events_processed, 250, + "expected 250 (replay+live) frames", + ); + assert_eq!( + result.first_report.edge_count, 250, + "diversified owners should yield 250 distinct edge keys", + ); + } + } +} + +#[test] +fn concurrent_reads_during_replay_is_byte_deterministic() { + for &par in PARALLELISM_SWEEP { + for &seed in SEEDS { + let parallelism = NonZeroUsize::new(par).unwrap(); + let config = LeakRunConfig { + seed, + parallelism, + max_virtual_runtime: Duration::from_secs(60), + mem_ws_capacity: 4096, + warming_buffer_enabled: true, + }; + let factory = || -> Box { + Box::new(ConcurrentReadsDuringReplay::new( + ConcurrentReadsDuringReplayConfig { + follow_frames: 200, + frame_pace_us: 100, + read_pace_us: 1_000, + }, + )) + }; + let result = run_leak_check(config.clone(), factory); + assert!( + result.passed(), + "concurrent-reads-during-replay leak at par={par} seed={seed}: {:?}", + result.outcome, + ); + assert_eq!( + result.first_report.events_processed, 200, + "expected 200 follow frames processed", + ); + } + } +} + +#[test] +fn slingshot_flap_short_brownout_is_byte_deterministic() { + for &par in &[4usize, 16, 64] { + for &seed in &[7u64, 99] { + let parallelism = NonZeroUsize::new(par).unwrap(); + let config = LeakRunConfig { + seed, + parallelism, + max_virtual_runtime: Duration::from_secs(30), + mem_ws_capacity: 4096, + warming_buffer_enabled: true, + }; + let factory = || -> Box { + Box::new(SlingshotFlap::new(SlingshotFlapConfig { + cross_did_stars: 50, + normal_latency_ms: 2, + brownout_start_ms: 0, + brownout_duration_ms: 100, + brownout_latency_ms: 50, + brownout_enabled: true, + hydrant_send_timeout_ms: 30_000, + hydrant_frame_pace_us: 0, + omit_target_repos: false, + emit_live_promotion_frame: false, + })) + }; + let result = run_leak_check(config.clone(), factory); + assert!( + result.passed(), + "slingshot-flap leak at par={par} seed={seed}: {:?}", + result.outcome, + ); + assert_eq!( + result.first_report.events_processed, 100, + "expected 100 (50 stars + 50 repos) frames processed", + ); + assert!( + result.first_report.warming_buffer.enqueued_total > 0, + "expected the buffer path to exercise during warming at par={par} seed={seed}, got snapshot={:?}", + result.first_report.warming_buffer, + ); + assert_eq!( + result.first_report.warming_buffer.enqueued_total, + result.first_report.warming_buffer.drained_observe_total, + "every parked star must drain via observe at par={par} seed={seed}, got snapshot={:?}", + result.first_report.warming_buffer, + ); + } + } +} diff --git a/bobbin/crates/bobbin-sim/tests/reconnect_replay.rs b/bobbin/crates/bobbin-sim/tests/reconnect_replay.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/tests/reconnect_replay.rs @@ -0,0 +1,71 @@ +use std::num::NonZeroUsize; +use std::time::Duration; + +use bobbin_ingest::DisconnectKind; +use bobbin_sim::workloads::{SlingshotFlap, SlingshotFlapConfig}; +use bobbin_sim::{Sim, SimConfig, SimOutcome}; +use tokio::runtime::Builder as TokioBuilder; + +#[test] +fn slingshot_flap_resumes_via_replay_after_pong_timeout() { + let cfg = SlingshotFlapConfig { + cross_did_stars: 5000, + normal_latency_ms: 2, + brownout_start_ms: 0, + brownout_duration_ms: 60_000, + brownout_latency_ms: 200, + brownout_enabled: true, + hydrant_send_timeout_ms: 30_000, + hydrant_frame_pace_us: 1_000, + omit_target_repos: false, + emit_live_promotion_frame: false, + }; + let mut sim_config = SimConfig::new(7); + sim_config.parallelism = NonZeroUsize::new(4).unwrap(); + sim_config.max_virtual_runtime = Duration::from_secs(600); + sim_config.warming_buffer_enabled = false; + + let workload = Box::new(SlingshotFlap::new(cfg.clone())); + let runtime = TokioBuilder::new_current_thread() + .enable_all() + .start_paused(true) + .build() + .expect("build current_thread runtime"); + let report = runtime.block_on(Sim::new(sim_config, workload).run()); + + let total_frames = (cfg.cross_did_stars as u64) * 2; + + assert_eq!( + report.outcome, + SimOutcome::Passed, + "expected eventual recovery via replay after disconnect, got {:?} reason={:?}", + report.outcome, + report.failure_reason, + ); + assert_eq!( + report.events_processed, total_frames, + "expected all {total_frames} frames processed after reconnect, got {}", + report.events_processed, + ); + assert!( + report.disconnect_count >= 1, + "expected at least one disconnect during sustained brownout, got {}", + report.disconnect_count, + ); + let last = report + .last_disconnect + .as_ref() + .expect("last_disconnect populated when disconnect_count > 0"); + assert_eq!( + last.kind, + DisconnectKind::PongTimeout, + "expected PongTimeout under sustained brownout (paced upstream + N=4 + 200ms slingshot RTT), got {:?}: {}", + last.kind, + last.message, + ); + assert!( + last.last_cursor.raw() > 0 && last.last_cursor.raw() < total_frames, + "expected disconnect mid-stream, got last_cursor={}", + last.last_cursor.raw(), + ); +} diff --git a/bobbin/crates/bobbin-sim/tests/warming_buffer.rs b/bobbin/crates/bobbin-sim/tests/warming_buffer.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/tests/warming_buffer.rs @@ -0,0 +1,246 @@ +use std::num::NonZeroUsize; +use std::time::Duration; + +use bobbin_sim::workloads::{SlingshotFlap, SlingshotFlapConfig}; +use bobbin_sim::{Sim, SimConfig, SimOutcome}; +use tokio::runtime::Builder as TokioBuilder; + +#[test] +fn lever_b_absorbs_sustained_brownout_without_disconnect() { + let cfg = SlingshotFlapConfig { + cross_did_stars: 5000, + normal_latency_ms: 2, + brownout_start_ms: 0, + brownout_duration_ms: 60_000, + brownout_latency_ms: 200, + brownout_enabled: true, + hydrant_send_timeout_ms: 30_000, + hydrant_frame_pace_us: 1_000, + omit_target_repos: false, + emit_live_promotion_frame: false, + }; + let mut sim_config = SimConfig::new(7); + sim_config.parallelism = NonZeroUsize::new(4).unwrap(); + sim_config.max_virtual_runtime = Duration::from_secs(600); + + let workload = Box::new(SlingshotFlap::new(cfg.clone())); + let runtime = TokioBuilder::new_current_thread() + .enable_all() + .start_paused(true) + .build() + .expect("build current_thread runtime"); + let report = runtime.block_on(Sim::new(sim_config, workload).run()); + + let total_frames = (cfg.cross_did_stars as u64) * 2; + assert_eq!( + report.outcome, + SimOutcome::Passed, + "Lever B should keep slingshot off the hot path during cold replay, got {:?} reason={:?}", + report.outcome, + report.failure_reason, + ); + assert_eq!(report.events_processed, total_frames); + assert_eq!( + report.disconnect_count, 0, + "expected zero disconnects under sustained brownout once Lever B parks the cohort, got {} (last={:?})", + report.disconnect_count, report.last_disconnect, + ); + + let b = report.warming_buffer; + assert_eq!( + b.enqueued_total, cfg.cross_did_stars as u64, + "every cross-DID star must park exactly once: {b:?}", + ); + assert_eq!( + b.drained_observe_total, cfg.cross_did_stars as u64, + "every observed repo must drain its parked star: {b:?}", + ); + assert_eq!(b.drained_promote_total, 0, "no residual at promote: {b:?}"); + assert_eq!(b.current_entries, 0, "buffer empty after drain: {b:?}"); + assert_eq!(b.evicted_total, 0, "no evictions in this workload: {b:?}"); + assert!( + b.max_concurrent_entries > 0 && b.max_concurrent_entries <= cfg.cross_did_stars as u64, + "peak depth bounded by cohort size, got {} for {} stars", + b.max_concurrent_entries, + cfg.cross_did_stars, + ); +} + +#[test] +fn shadow_and_buffer_observe_the_same_population() { + let cfg = SlingshotFlapConfig { + cross_did_stars: 2000, + normal_latency_ms: 2, + brownout_start_ms: 0, + brownout_duration_ms: 0, + brownout_latency_ms: 0, + brownout_enabled: false, + hydrant_send_timeout_ms: 30_000, + hydrant_frame_pace_us: 1_000, + omit_target_repos: false, + emit_live_promotion_frame: false, + }; + let mut sim_config = SimConfig::new(7); + sim_config.parallelism = NonZeroUsize::new(16).unwrap(); + sim_config.max_virtual_runtime = Duration::from_secs(120); + + let workload = Box::new(SlingshotFlap::new(cfg.clone())); + let runtime = TokioBuilder::new_current_thread() + .enable_all() + .start_paused(true) + .build() + .expect("build current_thread runtime"); + let report = runtime.block_on(Sim::new(sim_config, workload).run()); + + assert_eq!( + report.outcome, + SimOutcome::Passed, + "{:?}", + report.failure_reason + ); + + let s = report.warming_shadow; + let b = report.warming_buffer; + assert_eq!( + s.enqueued_total, b.dep_enqueued_total, + "shadow note_unresolved per dep must equal buffer dep_enqueued_total: shadow={s:?} buffer={b:?}", + ); + assert_eq!( + s.drained_via_observe_total, b.dep_drained_observe_total, + "shadow note_observed per dep must equal buffer dep_drained_observe_total: shadow={s:?} buffer={b:?}", + ); + assert_eq!( + s.distinct_keys_seen, b.distinct_keys_seen, + "shadow and buffer must see the same distinct (owner, rkey) keys: shadow={s:?} buffer={b:?}", + ); + assert_eq!( + s.residual, + b.dep_enqueued_total - b.dep_drained_observe_total, + "shadow residual must equal buffer un-observed deps: shadow={s:?} buffer={b:?}", + ); +} + +#[test] +fn warming_to_ready_promote_drains_residual_via_parallel_slingshot_wave() { + let cfg = SlingshotFlapConfig { + cross_did_stars: 200, + normal_latency_ms: 2, + brownout_start_ms: 0, + brownout_duration_ms: 0, + brownout_latency_ms: 0, + brownout_enabled: false, + hydrant_send_timeout_ms: 30_000, + hydrant_frame_pace_us: 1_000, + omit_target_repos: true, + emit_live_promotion_frame: true, + }; + let mut sim_config = SimConfig::new(7); + sim_config.parallelism = NonZeroUsize::new(16).unwrap(); + sim_config.max_virtual_runtime = Duration::from_secs(120); + + let workload = Box::new(SlingshotFlap::new(cfg.clone())); + let runtime = TokioBuilder::new_current_thread() + .enable_all() + .start_paused(true) + .build() + .expect("build current_thread runtime"); + let report = runtime.block_on(Sim::new(sim_config, workload).run()); + + assert_eq!( + report.outcome, + SimOutcome::Passed, + "promote-flush workload must complete, got {:?} reason={:?}", + report.outcome, + report.failure_reason, + ); + + let b = report.warming_buffer; + let stars = cfg.cross_did_stars as u64; + assert_eq!( + b.enqueued_total, stars, + "every cross-DID star must park (no repos arrive): {b:?}", + ); + assert_eq!( + b.drained_observe_total, 0, + "no repos arrive on the stream so observe-drain stays at zero: {b:?}", + ); + assert_eq!( + b.drained_promote_total, stars, + "Warming->Ready promote must drain every parked entry: {b:?}", + ); + assert_eq!( + b.current_entries, 0, + "buffer must be empty after promote drain: {b:?}", + ); + assert!( + report.resolver_hits + report.resolver_misses >= stars, + "promote wave must fan out one slingshot resolve per distinct dep, got hits={} misses={} for {stars} stars", + report.resolver_hits, + report.resolver_misses, + ); + assert_eq!( + report.edge_count, + stars * 2 + 1, + "each drained star contributes a primary edge plus a sh.tangled.feed.star.by mirror edge; promoter repo adds one. got {} for {stars} stars", + report.edge_count, + ); +} + +#[test] +fn buffer_enabled_and_disabled_produce_identical_edge_index() { + let cfg = SlingshotFlapConfig { + cross_did_stars: 200, + normal_latency_ms: 2, + brownout_start_ms: 0, + brownout_duration_ms: 0, + brownout_latency_ms: 0, + brownout_enabled: false, + hydrant_send_timeout_ms: 30_000, + hydrant_frame_pace_us: 1_000, + omit_target_repos: false, + emit_live_promotion_frame: false, + }; + + let run_with_buffer = |enabled: bool| { + let mut sim_config = SimConfig::new(7); + sim_config.parallelism = NonZeroUsize::new(16).unwrap(); + sim_config.max_virtual_runtime = Duration::from_secs(120); + sim_config.warming_buffer_enabled = enabled; + let workload = Box::new(SlingshotFlap::new(cfg.clone())); + let runtime = TokioBuilder::new_current_thread() + .enable_all() + .start_paused(true) + .build() + .expect("build current_thread runtime"); + runtime.block_on(Sim::new(sim_config, workload).run()) + }; + + let with_buffer = run_with_buffer(true); + let without_buffer = run_with_buffer(false); + + assert_eq!( + with_buffer.outcome, + SimOutcome::Passed, + "{:?}", + with_buffer.failure_reason + ); + assert_eq!( + without_buffer.outcome, + SimOutcome::Passed, + "{:?}", + without_buffer.failure_reason + ); + assert_eq!( + with_buffer.events_processed, without_buffer.events_processed, + "events_processed must match across buffer modes", + ); + assert_eq!( + with_buffer.edge_count, without_buffer.edge_count, + "edge_count must match: lever B is correctness-preserving (with={} without={})", + with_buffer.edge_count, without_buffer.edge_count, + ); + assert_eq!( + with_buffer.last_cursor, without_buffer.last_cursor, + "final cursor must match across buffer modes", + ); +} diff --git a/bobbin/crates/bobbin-sim/tests/warming_shadow.rs b/bobbin/crates/bobbin-sim/tests/warming_shadow.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin-sim/tests/warming_shadow.rs @@ -0,0 +1,68 @@ +use std::num::NonZeroUsize; +use std::time::Duration; + +use bobbin_sim::workloads::{SlingshotFlap, SlingshotFlapConfig}; +use bobbin_sim::{Sim, SimConfig, SimOutcome}; +use tokio::runtime::Builder as TokioBuilder; + +#[test] +fn shadow_buffer_bounded_by_cross_did_attachment_cohort_and_drains_to_zero() { + let cfg = SlingshotFlapConfig { + cross_did_stars: 5000, + normal_latency_ms: 2, + brownout_start_ms: 0, + brownout_duration_ms: 0, + brownout_latency_ms: 0, + brownout_enabled: false, + hydrant_send_timeout_ms: 30_000, + hydrant_frame_pace_us: 1_000, + omit_target_repos: false, + emit_live_promotion_frame: false, + }; + let mut sim_config = SimConfig::new(7); + sim_config.parallelism = NonZeroUsize::new(16).unwrap(); + sim_config.max_virtual_runtime = Duration::from_secs(120); + + let workload = Box::new(SlingshotFlap::new(cfg.clone())); + let runtime = TokioBuilder::new_current_thread() + .enable_all() + .start_paused(true) + .build() + .expect("build current_thread runtime"); + let report = runtime.block_on(Sim::new(sim_config, workload).run()); + + let stars = cfg.cross_did_stars as u64; + let total = stars * 2; + + assert_eq!( + report.outcome, + SimOutcome::Passed, + "expected clean drain with brownout disabled, got {:?} reason={:?}", + report.outcome, + report.failure_reason, + ); + assert_eq!(report.events_processed, total); + + let s = report.warming_shadow; + assert_eq!( + s.enqueued_total, stars, + "every cross-DID star must enqueue exactly once: {s:?}", + ); + assert_eq!( + s.drained_via_observe_total, stars, + "every observe must drain its corresponding enqueue (stars send their repos in this workload): {s:?}", + ); + assert_eq!( + s.distinct_keys_seen, stars, + "one (owner, rkey) pair per repo: {s:?}", + ); + assert_eq!( + s.residual, 0, + "no entry must remain pending after observe arrives: {s:?}", + ); + assert!( + s.max_concurrent > 0 && s.max_concurrent <= stars, + "peak depth must be bounded by the cross-DID attachment cohort, got max_concurrent={} for {stars} stars: {s:?}", + s.max_concurrent, + ); +} diff --git a/bobbin/crates/bobbin/Cargo.toml b/bobbin/crates/bobbin/Cargo.toml new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "bobbin" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[[bin]] +name = "bobbin" +path = "src/main.rs" + +[dependencies] +bobbin-edge-index = { workspace = true } +bobbin-ingest = { workspace = true } +bobbin-knot-proxy = { workspace = true } +bobbin-record-lru = { workspace = true } +bobbin-runtime = { workspace = true } +bobbin-search = { workspace = true } +bobbin-slingshot-client = { workspace = true } +bobbin-xrpc = { workspace = true } + +axum = { workspace = true } +serde = { workspace = true, features = ["derive"] } +futures = { workspace = true } +socket2 = "0.6" +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "signal"] } +tokio-util = { workspace = true } +url = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +anyhow = { workspace = true } +clap = { workspace = true } +confique = { workspace = true } +thiserror = { workspace = true } +toml = { workspace = true } +tikv-jemallocator = { version = "0.7", features = ["unprefixed_malloc_on_supported_platforms", "profiling"] } +tikv-jemalloc-ctl = { version = "0.7", features = ["stats", "profiling"] } + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/bobbin/crates/bobbin/src/config.rs b/bobbin/crates/bobbin/src/config.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin/src/config.rs @@ -0,0 +1,471 @@ +use std::collections::HashSet; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use anyhow::{Context, anyhow}; +use confique::Config; +use url::Url; + +const SYSTEM_CONFIG_PATH: &str = "/etc/bobbin/config.toml"; +const ENV_PREFIX: &str = "BOBBIN_"; + +const KNOWN_KEYS: &[&str] = &[ + "server.binds", + "server.shutdown_grace_secs", + "server.debug_bind", + "hydrant.url", + "hydrant.start_cursor", + "ingest.parallelism", + "backpressure.per_request_anon_bytes", + "backpressure.adjust_interval_ms", + "backpressure.relieve_below_ratio", + "backpressure.tighten_above_ratio", + "backpressure.reserved_index_bytes", + "slingshot.url", + "record_cache.lru_bytes", + "search.heap_bytes", + "knot.allow_private", + "knot.require_https", + "log.format", + "log.filter", +]; + +const KNOWN_ENVS: &[&str] = &[ + "BOBBIN_CONFIG", + "BOBBIN_BIND", + "BOBBIN_SHUTDOWN_GRACE_SECS", + "BOBBIN_DEBUG_BIND", + "BOBBIN_HYDRANT_URL", + "BOBBIN_START_CURSOR", + "BOBBIN_INGEST_PARALLELISM", + "BOBBIN_BACKPRESSURE_PER_REQUEST_ANON_BYTES", + "BOBBIN_BACKPRESSURE_ADJUST_INTERVAL_MS", + "BOBBIN_BACKPRESSURE_RELIEVE_BELOW_RATIO", + "BOBBIN_BACKPRESSURE_TIGHTEN_ABOVE_RATIO", + "BOBBIN_BACKPRESSURE_RESERVED_INDEX_BYTES", + "BOBBIN_SLINGSHOT_URL", + "BOBBIN_RECORD_LRU_BYTES", + "BOBBIN_SEARCH_HEAP_BYTES", + "BOBBIN_KNOT_ALLOW_PRIVATE", + "BOBBIN_KNOT_REQUIRE_HTTPS", + "BOBBIN_LOG_FORMAT", + "BOBBIN_LOG", +]; + +#[derive(Debug, Config)] +pub struct BobbinConfig { + #[config(nested)] + pub server: ServerConfig, + + #[config(nested)] + pub hydrant: HydrantConfig, + + #[config(nested)] + pub ingest: IngestConfig, + + #[config(nested)] + pub backpressure: BackpressureConfig, + + #[config(nested)] + pub slingshot: SlingshotConfig, + + #[config(nested)] + pub record_cache: RecordCacheConfig, + + #[config(nested)] + pub search: SearchConfig, + + #[config(nested)] + pub knot: KnotConfig, + + #[config(nested)] + pub log: LogConfig, +} + +#[derive(Debug, Config)] +pub struct ServerConfig { + /// Addresses the XRPC server listens on. When using as an env var, comma-separated. + #[config( + env = "BOBBIN_BIND", + parse_env = parse_binds, + default = ["127.0.0.1:8090", "[::1]:8090"] + )] + pub binds: Vec, + + /// The amount of time in seconds to allow in-flight requests to drain after sigterm + /// before forcing the listener closed. + #[config(env = "BOBBIN_SHUTDOWN_GRACE_SECS", default = 30)] + pub shutdown_grace_secs: u64, + + /// Address for the `/debug/mem` and `/debug/heap` introspection endpoints, + /// for example `127.0.0.1:8091`. Empty disables them so the debug surface is + /// never reachable on the public listener. Bind to loopback only. + #[config(env = "BOBBIN_DEBUG_BIND", default = "")] + pub debug_bind: String, +} + +#[derive(Debug, thiserror::Error)] +pub enum BindParseError { + #[error("BOBBIN_BIND must list at least one address")] + Empty, + #[error("invalid bind entry `{0}`: {1}")] + Invalid(String, std::net::AddrParseError), +} + +fn parse_binds(raw: &str) -> Result, BindParseError> { + let addrs: Vec = raw + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(|s| SocketAddr::from_str(s).map_err(|e| BindParseError::Invalid(s.to_owned(), e))) + .collect::>()?; + if addrs.is_empty() { + return Err(BindParseError::Empty); + } + Ok(addrs) +} + +#[derive(Debug, Config)] +pub struct HydrantConfig { + /// Base URL of the hydrant instance - the cursor-replayable /stream lives + /// under this. Use `ws://` or `wss://` - `http://` and `https://` + /// are rewritten to the corresponding ws scheme at connection time! + #[config(env = "BOBBIN_HYDRANT_URL", default = "http://127.0.0.1:13010")] + pub url: Url, + + /// The cursor to request on first connect. Reconnects ignore this and resume + /// strictly after the last cursor seen internally. + #[config(env = "BOBBIN_START_CURSOR", default = 0)] + pub start_cursor: u64, +} + +#[derive(Debug, Config)] +pub struct IngestConfig { + /// Concurrent in-flight resolves during ingest. One slot per slingshot rtt. + /// Default 16 sits at the measured throughput + /// knee for cold replay against a healthy hydrant. The committer stays + /// serial so that like, cursor and Spur allocation order are preserved. + #[config(env = "BOBBIN_INGEST_PARALLELISM", default = 16)] + pub parallelism: usize, +} + +#[derive(Debug, Config)] +pub struct BackpressureConfig { + /// Estimated peak transient anonymous bytes a single hydrating request holds: + /// roughly the page of decoded records plus its serialized copy. Combined with + /// the detected cgroup memory limit to size the in-flight cap on heavy + /// endpoints. Has no effect when no cgroup memory limit is present, the heavy + /// path stays unbounded exactly as before. + #[config( + env = "BOBBIN_BACKPRESSURE_PER_REQUEST_ANON_BYTES", + default = 2_097_152 + )] + pub per_request_anon_bytes: u64, + + /// How often the adaptive watcher samples memory.current/memory.max and + /// adjusts the in-flight cap. Ignored when no cgroup memory limit is present. + #[config(env = "BOBBIN_BACKPRESSURE_ADJUST_INTERVAL_MS", default = 500)] + pub adjust_interval_ms: u64, + + /// memory.current/memory.max ratio below which the watcher additively raises + /// the in-flight cap back toward its ceiling. Sits below the memory.high + /// throttle band so the kernel, not this loop, handles mild pressure. + #[config(env = "BOBBIN_BACKPRESSURE_RELIEVE_BELOW_RATIO", default = 0.85)] + pub relieve_below_ratio: f64, + + /// memory.current/memory.max ratio above which the watcher multiplicatively + /// cuts the in-flight cap and purges jemalloc arenas. Set above the memory.high + /// watermark so this loop acts as the anti-OOM backstop, not a duplicate throttle. + #[config(env = "BOBBIN_BACKPRESSURE_TIGHTEN_ABOVE_RATIO", default = 0.92)] + pub tighten_above_ratio: f64, + + /// Bytes held back for the edge index, state indexes, and other derived + /// state that grows after startup. Folded into the heavy-request reserve so + /// the static in-flight cap leaves the index room to fill. Has no effect when + /// no cgroup memory limit is present. + #[config(env = "BOBBIN_BACKPRESSURE_RESERVED_INDEX_BYTES", default = 67_108_864)] + pub reserved_index_bytes: u64, +} + +#[derive(Debug, Config)] +pub struct SlingshotConfig { + /// Base URL of a slingshot instance. Used for record bodies and identity. + #[config(env = "BOBBIN_SLINGSHOT_URL", default = "http://127.0.0.1:13011")] + pub url: Url, +} + +#[derive(Debug, Config)] +pub struct RecordCacheConfig { + /// Bound of bytes on the in-process record LRU. Records evict on a weighted + /// LRU policy keyed on URI plus payload length. + #[config(env = "BOBBIN_RECORD_LRU_BYTES", default = 67_108_864)] + pub lru_bytes: u64, +} + +#[derive(Debug, Config)] +pub struct SearchConfig { + /// The heap size in bytes for the in-mem tantivy writer. Larger values + /// trade RAM for fewer segment merges - the index itself lives in + /// `RamDirectory` and is rebuilt from hydrant replay on every restart. + #[config(env = "BOBBIN_SEARCH_HEAP_BYTES", default = 50_000_000)] + pub heap_bytes: u64, +} + +#[derive(Debug, Config)] +pub struct KnotConfig { + /// Whether to allow the knot proxy to dial private/loopback addresses. Off in + /// production - on for local testing against a knotserver on localhost. + #[config(env = "BOBBIN_KNOT_ALLOW_PRIVATE", default = false)] + pub allow_private: bool, + + /// Require https on knot hosts. Disable only when proxying to a local + /// knot for development. + #[config(env = "BOBBIN_KNOT_REQUIRE_HTTPS", default = true)] + pub require_https: bool, +} + +#[derive(Debug, Config)] +pub struct LogConfig { + /// Log emitter format. `text` produces human-readable output for local + /// development. `json` emits one structured object per line for log + /// shippers in production. + #[config(env = "BOBBIN_LOG_FORMAT", default = "text")] + pub format: String, + + /// `tracing-subscriber` env-filter directive. Defaults to `info` across + /// every span - override with `BOBBIN_LOG=bobbin_xrpc=debug,info` etc. + #[config(env = "BOBBIN_LOG", default = "info")] + pub filter: String, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LogFormat { + Text, + Json, +} + +impl std::str::FromStr for LogFormat { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "text" => Ok(Self::Text), + "json" => Ok(Self::Json), + other => Err(format!( + "log format must be `text` or `json`, got `{other}`" + )), + } + } +} + +pub fn load(path: Option<&PathBuf>) -> anyhow::Result { + check_envs(std::env::vars().map(|(k, _)| k))?; + if let Some(p) = path { + check_keys(p)?; + } + check_keys(Path::new(SYSTEM_CONFIG_PATH))?; + let mut builder = BobbinConfig::builder().env(); + if let Some(p) = path { + builder = builder.file(p); + } + builder + .file(SYSTEM_CONFIG_PATH) + .load() + .context("load configuration") +} + +pub fn template() -> String { + confique::toml::template::(confique::toml::FormatOptions::default()) +} + +fn check_keys(path: &Path) -> anyhow::Result<()> { + let bytes = match std::fs::read_to_string(path) { + Ok(b) => b, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(anyhow!("read {}: {e}", path.display())), + }; + let value: toml::Value = + toml::from_str(&bytes).map_err(|e| anyhow!("parse {}: {e}", path.display()))?; + let known: HashSet<&str> = KNOWN_KEYS.iter().copied().collect(); + let unknown: Vec = collect_paths(&value, "") + .into_iter() + .filter(|p| !known.contains(p.as_str())) + .collect(); + if unknown.is_empty() { + Ok(()) + } else { + Err(anyhow!( + "unknown keys in {}: {}\nrun `bobbin config-template` for the canonical schema", + path.display(), + unknown.join(", "), + )) + } +} + +fn check_envs(iter: I) -> anyhow::Result<()> +where + I: IntoIterator, + S: AsRef, +{ + let known: HashSet<&str> = KNOWN_ENVS.iter().copied().collect(); + let unknown: Vec = iter + .into_iter() + .filter_map(|var| { + let var = var.as_ref(); + (var.starts_with(ENV_PREFIX) && !known.contains(var)).then(|| var.to_owned()) + }) + .collect(); + if unknown.is_empty() { + Ok(()) + } else { + Err(anyhow!( + "unknown {ENV_PREFIX}* environment variables: {}\nrun `bobbin config-template` for the canonical schema", + unknown.join(", "), + )) + } +} + +fn collect_paths(value: &toml::Value, prefix: &str) -> Vec { + match value { + toml::Value::Table(table) => table + .iter() + .flat_map(|(key, child)| { + let path = if prefix.is_empty() { + key.clone() + } else { + format!("{prefix}.{key}") + }; + match child { + toml::Value::Table(_) => collect_paths(child, &path), + _ => vec![path], + } + }) + .collect(), + _ => Vec::new(), + } +} + +#[cfg(test)] +fn template_paths(template: &str) -> Vec { + template + .lines() + .scan(String::new(), |section, line| { + let trimmed = line.trim_start(); + if let Some(name) = trimmed + .strip_prefix('[') + .and_then(|rest| rest.trim_end().strip_suffix(']')) + { + *section = name.to_owned(); + return Some(None); + } + let path = trimmed + .strip_prefix('#') + .and_then(|rest| rest.split_once('=')) + .map(|(key, _)| key.trim()) + .filter(|key| { + !key.is_empty() && key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + }) + .map(|key| format!("{section}.{key}")); + Some(path) + }) + .flatten() + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn write(name: &str, body: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "bobbin-config-test-{}-{}", + std::process::id(), + name + )); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("config.toml"); + std::fs::write(&path, body).unwrap(); + path + } + + #[test] + fn known_keys_exactly_match_template_paths() { + let template = template(); + let from_template: HashSet = template_paths(&template).into_iter().collect(); + let known: HashSet = KNOWN_KEYS.iter().map(|s| (*s).to_owned()).collect(); + assert_eq!( + from_template, known, + "KNOWN_KEYS must equal the set of paths the confique template generates", + ); + } + + #[test] + fn template_paths_extracts_commented_leaf_assignments_under_each_section() { + let raw = "[server]\n# Doc comment with = inside.\n#binds = [\"127.0.0.1:8090\"]\n#shutdown_grace_secs = 30\n\n[log]\n#format = \"text\"\n"; + let mut paths = template_paths(raw); + paths.sort(); + assert_eq!( + paths, + vec![ + "log.format".to_owned(), + "server.binds".to_owned(), + "server.shutdown_grace_secs".to_owned(), + ], + ); + } + + #[test] + fn unknown_section_rejected() { + let path = write( + "unknown_section", + "[server]\nbinds = [\"127.0.0.1:9000\"]\n\n[mystery]\nfoo = 1\n", + ); + let err = check_keys(&path).expect_err("must reject"); + assert!(err.to_string().contains("mystery.foo"), "got {err}"); + } + + #[test] + fn unknown_field_rejected() { + let path = write("typo", "[server]\nbidns = [\"127.0.0.1:9000\"]\n"); + let err = check_keys(&path).expect_err("must reject"); + assert!(err.to_string().contains("server.bidns"), "got {err}"); + } + + #[test] + fn missing_file_passes_silently() { + check_keys(Path::new("/definitely/does/not/exist.toml")).expect("no file is fine"); + } + + #[test] + fn full_known_config_passes() { + let path = write("ok", &template()); + check_keys(&path).expect("template must validate against itself"); + } + + #[test] + fn known_envs_carry_the_env_prefix() { + KNOWN_ENVS.iter().for_each(|name| { + assert!( + name.starts_with(ENV_PREFIX), + "KNOWN_ENVS entry {name:?} must start with {ENV_PREFIX:?}" + ); + }); + } + + #[test] + fn unknown_bobbin_env_rejected() { + let err = check_envs(["BOBBIN_BIDNS"]).expect_err("typo must surface"); + assert!(err.to_string().contains("BOBBIN_BIDNS"), "got {err}"); + } + + #[test] + fn known_bobbin_env_passes() { + check_envs(["BOBBIN_BIND", "BOBBIN_LOG"]).expect("known names must pass"); + } + + #[test] + fn non_bobbin_env_ignored() { + check_envs(["PATH", "HOME", "RUST_LOG"]).expect("only BOBBIN_* names are validated"); + } +} diff --git a/bobbin/crates/bobbin/src/main.rs b/bobbin/crates/bobbin/src/main.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin/src/main.rs @@ -0,0 +1,485 @@ +use std::net::SocketAddr; +use std::num::NonZeroUsize; +use std::path::PathBuf; +use std::process::ExitCode; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, anyhow}; +use bobbin_edge_index::{CoverageWatch, EdgeStore, HydrantCursor, StateIndex}; +use bobbin_ingest::{ + IngestConfig, IngestRuntime, RepoIdResolver, WarmingBuffer, run as run_ingest, +}; +use bobbin_knot_proxy::{KnotHttpConfig, KnotProxy, KnotProxyConfig}; +use bobbin_record_lru::{CacheCapacity, LruRecordStore, RecordStore}; +use bobbin_runtime::{Clock, MemoryBudget, OsEntropy, RuntimeHasher, SystemClock, TungsteniteWs}; +use bobbin_search::{SearchIndex, SearchReader}; +use bobbin_slingshot_client::SlingshotClient; +use bobbin_xrpc::{ + AppState, HeavyLimiter, MaxInFlight, PerRequestAnonBytes, ReservedFloor, router, +}; +use clap::{Parser, Subcommand}; +use tokio::signal::unix::{SignalKind, signal}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; +use tracing::level_filters::LevelFilter; +use tracing_subscriber::EnvFilter; + +mod config; +mod mem; + +use config::{BobbinConfig, LogFormat}; + +const BASE_RESIDENT_BYTES: u64 = 48 * 1024 * 1024; + +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +#[used] +#[unsafe(no_mangle)] +pub static malloc_conf: Option<&'static core::ffi::c_char> = Some(unsafe { + &*c"narenas:8,dirty_decay_ms:3000,muzzy_decay_ms:3000,background_thread:true".as_ptr() +}); + +#[derive(Parser)] +#[command(name = "bobbin", about = "Read-only AppView for Tangled records")] +struct Cli { + /// Path to a TOML config file. Environment variables override file values + /// for any `BOBBIN_*` setting; `/etc/bobbin/config.toml` is consulted as a + /// final fallback so distro packaging can drop a default in place. + #[arg(short, long, value_name = "FILE", env = "BOBBIN_CONFIG")] + config: Option, + + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +enum Command { + /// Print a fully-commented TOML template to stdout. Use this to seed + /// `config.toml` for a fresh deploy. + ConfigTemplate, + /// Load and validate the configuration without starting the server. + Validate, +} + +#[tokio::main] +async fn main() -> ExitCode { + let cli = Cli::parse(); + + if let Some(Command::ConfigTemplate) = cli.command { + print!("{}", config::template()); + return ExitCode::SUCCESS; + } + + let cfg = match config::load(cli.config.as_ref()) { + Ok(c) => c, + Err(e) => { + eprintln!("failed to load configuration: {e:#}"); + return ExitCode::FAILURE; + } + }; + + if matches!(cli.command, Some(Command::Validate)) { + if let Err(e) = init_tracing(&cfg, Arc::new(SystemClock::new())) { + eprintln!("failed to install tracing subscriber: {e}"); + return ExitCode::FAILURE; + } + println!("configuration is valid"); + return ExitCode::SUCCESS; + } + + match run(cfg).await { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + tracing::error!(error = ?e, "fatal"); + ExitCode::FAILURE + } + } +} + +struct ClockTimer(Arc); + +impl tracing_subscriber::fmt::time::FormatTime for ClockTimer { + fn format_time(&self, w: &mut tracing_subscriber::fmt::format::Writer<'_>) -> std::fmt::Result { + write!(w, "{}", self.0.now_unix_micros().raw()) + } +} + +fn init_tracing(cfg: &BobbinConfig, clock: Arc) -> Result<(), String> { + let combined = format!("{},{}", LevelFilter::INFO, cfg.log.filter); + let format: LogFormat = cfg.log.format.parse()?; + let timer = ClockTimer(clock); + match format { + LogFormat::Text => { + let filter = EnvFilter::try_new(&combined) + .map_err(|e| format!("invalid log filter `{}`: {e}", cfg.log.filter))?; + tracing_subscriber::fmt() + .with_env_filter(filter) + .with_timer(timer) + .try_init() + .map_err(|e| e.to_string()) + } + LogFormat::Json => { + let filter = EnvFilter::try_new(&combined) + .map_err(|e| format!("invalid log filter `{}`: {e}", cfg.log.filter))?; + tracing_subscriber::fmt() + .with_env_filter(filter) + .json() + .with_timer(timer) + .try_init() + .map_err(|e| e.to_string()) + } + } +} + +async fn run(cfg: BobbinConfig) -> anyhow::Result<()> { + let clock: Arc = Arc::new(SystemClock::new()); + init_tracing(&cfg, clock.clone()) + .map_err(|e| anyhow!("failed to install tracing subscriber: {e}"))?; + + let (budget, budget_source) = mem::detect_budget(); + tracing::info!( + source = ?budget_source, + budget_bytes = budget.map(MemoryBudget::bytes), + "memory budget detected" + ); + if let Some(b) = budget { + mem::try_set_high(b); + } + let lru_cap = mem::lru_bytes(budget, cfg.record_cache.lru_bytes); + let search_heap_cap = mem::search_heap_bytes(budget, cfg.search.heap_bytes); + if budget.is_some() { + tracing::info!( + lru_bytes = lru_cap, + search_heap_bytes = search_heap_cap, + "constrained cache sizing" + ); + } + let limiter = budget.map(|b| { + let reserved = ReservedFloor::new( + BASE_RESIDENT_BYTES + .saturating_add(cfg.backpressure.reserved_index_bytes) + .saturating_add(search_heap_cap) + .saturating_add(lru_cap), + ); + let per_request = PerRequestAnonBytes::new(cfg.backpressure.per_request_anon_bytes); + let max = MaxInFlight::from_budget(b, reserved, per_request); + tracing::info!(max_in_flight = max.get(), "heavy-request concurrency cap"); + Arc::new(HeavyLimiter::new(max)) + }); + + let entropy = Arc::new(OsEntropy); + let hasher = RuntimeHasher::from_entropy(&*entropy); + let ws = TungsteniteWs::shared(); + + let records: Arc = + Arc::new(LruRecordStore::new(CacheCapacity::from_bytes(lru_cap))); + let slingshot = SlingshotClient::with_default_http(cfg.slingshot.url.clone())?; + let resolver = Arc::new(RepoIdResolver::with_slingshot( + slingshot.clone(), + clock.clone(), + hasher.clone(), + )); + let edges = Arc::new(EdgeStore::new(hasher.clone())); + let issue_states = Arc::new(StateIndex::new(hasher.clone())); + let pull_statuses = Arc::new(StateIndex::new(hasher.clone())); + let coverage = Arc::new(CoverageWatch::new()); + let warming_buffer = Arc::new(WarmingBuffer::new(hasher.clone())); + let knots = Arc::new(KnotProxy::new( + KnotProxyConfig { + allow_private_hosts: cfg.knot.allow_private, + require_https: cfg.knot.require_https, + ..KnotProxyConfig::default() + }, + KnotHttpConfig::default(), + clock.clone(), + hasher, + )?); + let search_heap = usize::try_from(search_heap_cap) + .with_context(|| format!("search heap {search_heap_cap} exceeds usize"))?; + let search = Arc::new(SearchIndex::new(search_heap, clock.clone())?); + + let configured_parallelism = NonZeroUsize::new(cfg.ingest.parallelism) + .ok_or_else(|| anyhow!("ingest.parallelism must be at least 1"))?; + let parallelism = mem::ingest_parallelism(budget, configured_parallelism); + tracing::info!( + configured = configured_parallelism.get(), + effective = parallelism.get(), + "ingest parallelism" + ); + let ingest_cfg = IngestConfig { + hydrant_base: cfg.hydrant.url.clone(), + start_cursor: HydrantCursor::new(cfg.hydrant.start_cursor), + parallelism, + }; + let cancel = CancellationToken::new(); + let ingest_coverage = coverage.clone(); + let ingest_runtime = IngestRuntime { + store: edges.clone(), + issue_states: issue_states.clone(), + pull_statuses: pull_statuses.clone(), + coverage: coverage.clone(), + search: search.clone(), + records: records.clone(), + resolver: resolver.clone(), + clock: clock.clone(), + entropy, + ws, + cancel: cancel.clone(), + disconnects: None, + warming_shadow: None, + warming_buffer: Some(warming_buffer), + }; + let mut ingest_handle = tokio::spawn(run_ingest(ingest_cfg, ingest_runtime)); + + let _adaptive_watcher = budget.zip(limiter.as_ref()).map(|(b, l)| { + mem::spawn_adaptive_watcher( + l.clone(), + clock.clone(), + b, + mem::AdaptiveThresholds { + interval: Duration::from_millis(cfg.backpressure.adjust_interval_ms), + relieve_below_ratio: cfg.backpressure.relieve_below_ratio, + tighten_above_ratio: cfg.backpressure.tighten_above_ratio, + }, + cancel.clone(), + ) + }); + + let debug_bind: Option = if cfg.server.debug_bind.is_empty() { + None + } else { + Some(cfg.server.debug_bind.parse().with_context(|| { + format!("invalid server.debug_bind `{}`", cfg.server.debug_bind) + })?) + }; + let mem_probe = debug_bind.is_some().then(|| mem::MemProbe { + edges: edges.clone(), + search: search.clone(), + records: records.clone(), + issue_states: issue_states.clone(), + pull_statuses: pull_statuses.clone(), + }); + let state = AppState::new( + records, + slingshot, + edges, + issue_states, + pull_statuses, + coverage, + knots, + search as Arc, + resolver, + ) + .with_limiter(limiter); + let app = router(state); + + let _debug_server = match (debug_bind, mem_probe) { + (Some(addr), Some(probe)) => { + let debug_app = mem::debug_router(probe); + let debug_cancel = cancel.clone(); + Some(tokio::spawn(async move { + match bind_listener(addr) { + Ok(listener) => { + tracing::info!(%addr, "debug endpoints bound, keep this loopback-only"); + let shutdown = async move { debug_cancel.cancelled().await }; + if let Err(e) = axum::serve(listener, debug_app) + .with_graceful_shutdown(shutdown) + .await + { + tracing::warn!(error = %e, "debug endpoint server exited with error"); + } + } + Err(e) => { + tracing::warn!(%addr, error = %e, "could not bind debug endpoints"); + } + } + })) + } + _ => None, + }; + + let binds = cfg.server.binds.clone(); + let hydrant_url = cfg.hydrant.url.as_str().to_owned(); + let slingshot_url = cfg.slingshot.url.as_str().to_owned(); + let grace = Duration::from_secs(cfg.server.shutdown_grace_secs); + let bind_display = binds + .iter() + .map(SocketAddr::to_string) + .collect::>() + .join(","); + tracing::info!(binds = %bind_display, %hydrant_url, %slingshot_url, "bobbin listening"); + + let signal_cancel = cancel.clone(); + let mut server_handle = tokio::spawn(serve_all(binds, app, signal_cancel)); + + tokio::select! { + res = &mut server_handle => { + cancel.cancel(); + let cursor = ingest_coverage.snapshot().last_cursor().raw(); + tracing::info!(grace_secs = grace.as_secs(), cursor, "draining ingest"); + drain_with_grace("ingest", grace, &mut ingest_handle, clock.as_ref()).await; + match res { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err(anyhow::Error::from(e)).context("axum server failed"), + Err(join) => Err(anyhow!("server task panicked: {join}")), + } + } + res = &mut ingest_handle => { + cancel.cancel(); + let cursor = ingest_coverage.snapshot().last_cursor().raw(); + tracing::info!(grace_secs = grace.as_secs(), cursor, "draining server"); + drain_with_grace("server", grace, &mut server_handle, clock.as_ref()).await; + match res { + Ok(Ok(())) => Err(anyhow!("ingest run loop exited; loop is supposed to be infinite")), + Ok(Err(e)) => Err(anyhow::Error::from(e)).context("ingest exited"), + Err(join) => Err(anyhow!("ingest task panicked: {join}")), + } + } + } +} + +async fn drain_with_grace( + label: &'static str, + grace: Duration, + handle: &mut JoinHandle, + clock: &dyn Clock, +) { + tokio::select! { + _ = &mut *handle => {} + _ = clock.sleep(grace) => { + tracing::warn!( + grace_secs = grace.as_secs(), + label, + "task did not stop within grace, aborting" + ); + handle.abort(); + } + } +} + +async fn serve_all( + binds: Vec, + app: axum::Router, + cancel: CancellationToken, +) -> std::io::Result<()> { + let listeners = futures::future::try_join_all(binds.into_iter().map(|addr| async move { + let listener = bind_listener(addr)?; + tracing::info!(%addr, "bobbin listener bound"); + Ok::<_, std::io::Error>(listener) + })) + .await?; + + let trigger = cancel.clone(); + let signal_task = tokio::spawn(async move { + wait_for_shutdown().await; + tracing::info!("shutdown signal received, draining server"); + trigger.cancel(); + }); + + let services = listeners.into_iter().map(|listener| { + let app = app.clone(); + let cancel = cancel.clone(); + async move { + axum::serve(listener, app) + .with_graceful_shutdown(async move { cancel.cancelled().await }) + .await + } + }); + + let result = futures::future::try_join_all(services).await.map(|_| ()); + signal_task.abort(); + result +} + +fn bind_listener(addr: SocketAddr) -> std::io::Result { + let domain = match addr { + SocketAddr::V4(_) => socket2::Domain::IPV4, + SocketAddr::V6(_) => socket2::Domain::IPV6, + }; + let socket = socket2::Socket::new(domain, socket2::Type::STREAM, Some(socket2::Protocol::TCP))?; + if matches!(addr, SocketAddr::V6(_)) { + socket.set_only_v6(true)?; + } + socket.set_reuse_address(true)?; + socket.set_nonblocking(true)?; + socket.bind(&addr.into())?; + socket.listen(1024)?; + let std_listener: std::net::TcpListener = socket.into(); + tokio::net::TcpListener::from_std(std_listener) +} + +async fn wait_for_shutdown() { + let ctrl_c = tokio::signal::ctrl_c(); + let mut sigterm = match signal(SignalKind::terminate()) { + Ok(s) => s, + Err(e) => { + tracing::warn!( + ?e, + "could not install SIGTERM handler, shutdown will only honor ctrl-c" + ); + ctrl_c.await.ok(); + return; + } + }; + tokio::select! { + _ = ctrl_c => {} + _ = sigterm.recv() => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn drain_returns_immediately_when_task_already_done() { + let mut handle = tokio::spawn(async { 7u32 }); + tokio::time::advance(Duration::from_millis(1)).await; + let start = tokio::time::Instant::now(); + drain_with_grace( + "test", + Duration::from_secs(60), + &mut handle, + &SystemClock::new(), + ) + .await; + assert!(start.elapsed() < Duration::from_millis(10)); + } + + #[tokio::test(start_paused = true)] + async fn drain_aborts_runaway_task_after_grace() { + let mut handle = tokio::spawn(async { + std::future::pending::<()>().await; + }); + let grace = Duration::from_secs(5); + let start = tokio::time::Instant::now(); + drain_with_grace("test", grace, &mut handle, &SystemClock::new()).await; + assert!(start.elapsed() >= grace); + let outcome = handle.await; + assert!(outcome.is_err() && outcome.unwrap_err().is_cancelled()); + } + + #[test] + fn typo_filter_keeps_info_default_for_other_targets() { + let filter = format!("{},{}", LevelFilter::INFO, "blah_invalid"); + let parsed = EnvFilter::try_new(&filter).expect("filter parses"); + let rendered = parsed.to_string(); + assert!( + rendered.contains("info"), + "expected info default, got {rendered}" + ); + assert!( + rendered.contains("blah_invalid"), + "expected user override, got {rendered}", + ); + } + + #[test] + fn explicit_user_level_overrides_info_default() { + let filter = format!("{},{}", LevelFilter::INFO, "warn"); + let parsed = EnvFilter::try_new(&filter).expect("filter parses"); + assert_eq!(parsed.to_string(), "warn"); + } +} diff --git a/bobbin/crates/bobbin/src/mem/cgroup.rs b/bobbin/crates/bobbin/src/mem/cgroup.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin/src/mem/cgroup.rs @@ -0,0 +1,116 @@ +use bobbin_runtime::MemoryBudget; + +const MEMORY_MAX_PATH: &str = "/sys/fs/cgroup/memory.max"; +const MEMORY_HIGH_PATH: &str = "/sys/fs/cgroup/memory.high"; +const MEMORY_CURRENT_PATH: &str = "/sys/fs/cgroup/memory.current"; +const HIGH_WATERMARK_NUMERATOR: u64 = 8; +const HIGH_WATERMARK_DENOMINATOR: u64 = 10; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum BudgetSource { + CgroupV2, + Unconstrained, + NotCgroupV2, + MalformedLimit, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ParsedMax { + Unlimited, + Bytes(MemoryBudget), + Malformed, +} + +pub fn detect_budget() -> (Option, BudgetSource) { + match std::fs::read_to_string(MEMORY_MAX_PATH) { + Ok(raw) => match parse_memory_max(&raw) { + ParsedMax::Bytes(budget) => (Some(budget), BudgetSource::CgroupV2), + ParsedMax::Unlimited => (None, BudgetSource::Unconstrained), + ParsedMax::Malformed => (None, BudgetSource::MalformedLimit), + }, + Err(_) => (None, BudgetSource::NotCgroupV2), + } +} + +fn parse_memory_max(raw: &str) -> ParsedMax { + match raw.trim() { + "max" => ParsedMax::Unlimited, + bytes => bytes + .parse::() + .map(|n| ParsedMax::Bytes(MemoryBudget::new(n))) + .unwrap_or(ParsedMax::Malformed), + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct HighWatermark(u64); + +impl HighWatermark { + pub fn from_budget(budget: MemoryBudget) -> Self { + Self(budget.bytes() / HIGH_WATERMARK_DENOMINATOR * HIGH_WATERMARK_NUMERATOR) + } + + pub const fn bytes(self) -> u64 { + self.0 + } +} + +pub fn try_set_high(budget: MemoryBudget) { + let high = HighWatermark::from_budget(budget); + match std::fs::write(MEMORY_HIGH_PATH, high.bytes().to_string()) { + Ok(()) => tracing::info!( + high_bytes = high.bytes(), + "set memory.high throttle watermark" + ), + Err(e) => tracing::warn!( + error = %e, + high_bytes = high.bytes(), + "could not self-write memory.high, set it via compose or ops" + ), + } +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Ord, PartialOrd)] +pub struct MemoryCurrent(u64); + +impl MemoryCurrent { + pub const fn bytes(self) -> u64 { + self.0 + } +} + +pub fn read_current() -> Option { + std::fs::read_to_string(MEMORY_CURRENT_PATH) + .ok() + .and_then(|raw| raw.trim().parse::().ok()) + .map(MemoryCurrent) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unlimited_max_reads_as_no_budget() { + assert_eq!(parse_memory_max("max\n"), ParsedMax::Unlimited); + } + + #[test] + fn numeric_limit_reads_as_budget() { + assert_eq!( + parse_memory_max("104857600\n"), + ParsedMax::Bytes(MemoryBudget::new(104_857_600)), + ); + } + + #[test] + fn garbage_reads_as_malformed() { + assert_eq!(parse_memory_max("not-a-number"), ParsedMax::Malformed); + } + + #[test] + fn high_watermark_is_eighty_percent_of_budget() { + let high = HighWatermark::from_budget(MemoryBudget::new(100 * 1024 * 1024)); + assert_eq!(high.bytes(), 80 * 1024 * 1024); + } +} diff --git a/bobbin/crates/bobbin/src/mem/mod.rs b/bobbin/crates/bobbin/src/mem/mod.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin/src/mem/mod.rs @@ -0,0 +1,9 @@ +mod cgroup; +mod report; +mod sizing; +mod watcher; + +pub use cgroup::{detect_budget, try_set_high}; +pub use report::{MemProbe, debug_router}; +pub use sizing::{ingest_parallelism, lru_bytes, search_heap_bytes}; +pub use watcher::{AdaptiveThresholds, spawn as spawn_adaptive_watcher}; diff --git a/bobbin/crates/bobbin/src/mem/report.rs b/bobbin/crates/bobbin/src/mem/report.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin/src/mem/report.rs @@ -0,0 +1,186 @@ +use std::sync::Arc; + +use axum::extract::State; +use axum::http::StatusCode; +use axum::routing::get; +use axum::{Json, Router}; +use bobbin_edge_index::{EdgeStore, IssueStateKind, PullStatusKind, StateIndex}; +use bobbin_record_lru::RecordStore; +use bobbin_search::SearchIndex; +use serde::Serialize; +use tikv_jemalloc_ctl::{epoch, stats}; + +#[derive(Clone)] +pub struct MemProbe { + pub edges: Arc, + pub search: Arc, + pub records: Arc, + pub issue_states: Arc>, + pub pull_statuses: Arc>, +} + +pub fn debug_router(probe: MemProbe) -> Router { + Router::new() + .route("/debug/mem", get(mem_report)) + .route("/debug/heap", get(heap_dump)) + .with_state(probe) +} + +#[derive(Serialize)] +struct Jemalloc { + allocated: u64, + active: u64, + resident: u64, + retained: u64, + mapped: u64, +} + +#[derive(Serialize)] +struct Tantivy { + num_docs: u64, + num_segments: u64, + total_bytes: u64, +} + +#[derive(Serialize)] +struct HistBin { + le: u64, + count: u64, +} + +#[derive(Serialize)] +struct Edges { + key_count: u64, + source_count: u64, + source_interner_bytes: u64, + did_interner_bytes: u64, + collection_interner_bytes: u64, + key_interner_bytes: u64, + edges_total: u64, + author_refs_total: u64, + reverse_entries: u64, + reverse_cap: u64, + forward_struct_bytes: u64, + reverse_struct_bytes: u64, + bucket_struct_bytes: u64, + max_bucket: u64, + bucket_histogram: Vec, +} + +#[derive(Serialize)] +struct StateIdx { + issue_entities: u64, + issue_sources: u64, + pull_entities: u64, + pull_sources: u64, +} + +#[derive(Serialize)] +struct Lru { + weight: u64, + len: u64, + capacity: u64, +} + +#[derive(Serialize)] +struct Derived { + known_bytes: u64, + allocated_minus_known: u64, +} + +#[derive(Serialize)] +struct MemSnapshot { + jemalloc: Jemalloc, + tantivy: Tantivy, + edges: Edges, + state: StateIdx, + lru: Lru, + derived: Derived, +} + +async fn mem_report(State(p): State) -> Json { + let _ = epoch::advance(); + let allocated = stats::allocated::read().unwrap_or(0) as u64; + let active = stats::active::read().unwrap_or(0) as u64; + let resident = stats::resident::read().unwrap_or(0) as u64; + let retained = stats::retained::read().unwrap_or(0) as u64; + let mapped = stats::mapped::read().unwrap_or(0) as u64; + + let su = p.search.space_usage(); + let er = p.edges.mem_report(); + let lru = p.records.cache_stats().unwrap_or_default(); + + let known_bytes = su.total_bytes + + er.source_interner_bytes + + er.did_interner_bytes + + er.collection_interner_bytes + + er.key_interner_bytes + + er.forward_struct_bytes + + er.reverse_struct_bytes + + lru.weight; + + Json(MemSnapshot { + jemalloc: Jemalloc { + allocated, + active, + resident, + retained, + mapped, + }, + tantivy: Tantivy { + num_docs: su.num_docs, + num_segments: su.num_segments, + total_bytes: su.total_bytes, + }, + edges: Edges { + key_count: er.key_count, + source_count: er.source_count, + source_interner_bytes: er.source_interner_bytes, + did_interner_bytes: er.did_interner_bytes, + collection_interner_bytes: er.collection_interner_bytes, + key_interner_bytes: er.key_interner_bytes, + edges_total: er.edges_total, + author_refs_total: er.author_refs_total, + reverse_entries: er.reverse_entries, + reverse_cap: er.reverse_cap, + forward_struct_bytes: er.forward_struct_bytes, + reverse_struct_bytes: er.reverse_struct_bytes, + bucket_struct_bytes: er.bucket_struct_bytes, + max_bucket: er.max_bucket, + bucket_histogram: er + .bucket_histogram() + .map(|(le, count)| HistBin { le, count }) + .collect(), + }, + state: StateIdx { + issue_entities: p.issue_states.entity_count() as u64, + issue_sources: p.issue_states.source_count() as u64, + pull_entities: p.pull_statuses.entity_count() as u64, + pull_sources: p.pull_statuses.source_count() as u64, + }, + lru: Lru { + weight: lru.weight, + len: lru.len, + capacity: lru.capacity, + }, + derived: Derived { + known_bytes, + allocated_minus_known: allocated.saturating_sub(known_bytes), + }, + }) +} + +async fn heap_dump() -> (StatusCode, String) { + let dump = unsafe { + tikv_jemalloc_ctl::raw::write(b"prof.dump\0", core::ptr::null::()) + }; + match dump { + Ok(()) => (StatusCode::OK, "dumped to prof_prefix".to_owned()), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + format!( + "prof.dump failed, is the binary built with profiling and run with prof:true: {e}" + ), + ), + } +} diff --git a/bobbin/crates/bobbin/src/mem/sizing.rs b/bobbin/crates/bobbin/src/mem/sizing.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin/src/mem/sizing.rs @@ -0,0 +1,100 @@ +use std::num::NonZeroUsize; + +use bobbin_runtime::MemoryBudget; + +const INGEST_BUDGET_PER_SLOT: u64 = 8 * 1024 * 1024; +const SEARCH_HEAP_BUDGET_PCT: u64 = 15; +const LRU_BUDGET_PCT: u64 = 20; +const MIN_SEARCH_HEAP_BYTES: u64 = 15 * 1024 * 1024; +const MIN_LRU_BYTES: u64 = 4 * 1024 * 1024; + +pub fn ingest_parallelism(budget: Option, configured: NonZeroUsize) -> NonZeroUsize { + match budget { + None => configured, + Some(b) => { + let slots = (b.bytes() / INGEST_BUDGET_PER_SLOT).max(1); + let ceiling = usize::try_from(slots).unwrap_or(usize::MAX); + NonZeroUsize::new(ceiling.min(configured.get())).unwrap_or(configured) + } + } +} + +fn clamp_to_budget_fraction( + budget: Option, + configured: u64, + percent: u64, + minimum: u64, +) -> u64 { + match budget { + None => configured, + Some(b) => configured + .min((b.bytes() / 100 * percent).max(minimum)) + .min(b.bytes()), + } +} + +pub fn search_heap_bytes(budget: Option, configured: u64) -> u64 { + clamp_to_budget_fraction( + budget, + configured, + SEARCH_HEAP_BUDGET_PCT, + MIN_SEARCH_HEAP_BYTES, + ) +} + +pub fn lru_bytes(budget: Option, configured: u64) -> u64 { + clamp_to_budget_fraction(budget, configured, LRU_BUDGET_PCT, MIN_LRU_BYTES) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn nz(n: usize) -> NonZeroUsize { + NonZeroUsize::new(n).unwrap() + } + + #[test] + fn unconstrained_keeps_configured() { + assert_eq!(ingest_parallelism(None, nz(16)), nz(16)); + } + + #[test] + fn constrained_clamps_below_configured() { + let budget = Some(MemoryBudget::new(50 * 1024 * 1024)); + assert_eq!(ingest_parallelism(budget, nz(16)), nz(6)); + } + + #[test] + fn large_budget_never_raises_above_configured() { + let budget = Some(MemoryBudget::new(4 * 1024 * 1024 * 1024)); + assert_eq!(ingest_parallelism(budget, nz(16)), nz(16)); + } + + #[test] + fn caches_unclamped_when_unconstrained() { + assert_eq!(search_heap_bytes(None, 50_000_000), 50_000_000); + assert_eq!(lru_bytes(None, 67_108_864), 67_108_864); + } + + #[test] + fn caches_shrink_to_budget_fraction_when_constrained() { + let budget = Some(MemoryBudget::new(100 * 1024 * 1024)); + assert_eq!(search_heap_bytes(budget, 50_000_000), 15 * 1024 * 1024); + assert_eq!(lru_bytes(budget, 67_108_864), 20 * 1024 * 1024); + } + + #[test] + fn caches_honor_minimum_floor_on_tiny_budget() { + let budget = Some(MemoryBudget::new(16 * 1024 * 1024)); + assert_eq!(search_heap_bytes(budget, 50_000_000), MIN_SEARCH_HEAP_BYTES); + assert_eq!(lru_bytes(budget, 67_108_864), MIN_LRU_BYTES); + } + + #[test] + fn caches_never_raise_above_configured() { + let budget = Some(MemoryBudget::new(4 * 1024 * 1024 * 1024)); + assert_eq!(search_heap_bytes(budget, 50_000_000), 50_000_000); + assert_eq!(lru_bytes(budget, 67_108_864), 67_108_864); + } +} diff --git a/bobbin/crates/bobbin/src/mem/watcher.rs b/bobbin/crates/bobbin/src/mem/watcher.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/bobbin/src/mem/watcher.rs @@ -0,0 +1,103 @@ +use std::sync::Arc; +use std::time::Duration; + +use bobbin_runtime::{Clock, MemoryBudget}; +use bobbin_xrpc::{HeavyLimiter, PressureVerdict}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +use super::cgroup; + +pub struct AdaptiveThresholds { + pub interval: Duration, + pub relieve_below_ratio: f64, + pub tighten_above_ratio: f64, +} + +fn verdict(used_ratio: f64, thresholds: &AdaptiveThresholds) -> PressureVerdict { + if used_ratio > thresholds.tighten_above_ratio { + PressureVerdict::Tighten + } else if used_ratio < thresholds.relieve_below_ratio { + PressureVerdict::Relieve + } else { + PressureVerdict::Hold + } +} + +const PURGE_ALL_ARENAS: &[u8] = b"arena.4096.purge\0"; + +fn purge_arenas() { + if let Err(e) = unsafe { tikv_jemalloc_ctl::raw::write(PURGE_ALL_ARENAS, ()) } { + tracing::warn!(error = %e, "jemalloc arena purge failed"); + } +} + +pub fn spawn( + limiter: Arc, + clock: Arc, + budget: MemoryBudget, + thresholds: AdaptiveThresholds, + cancel: CancellationToken, +) -> JoinHandle<()> { + tokio::spawn(async move { + let max = budget.bytes().max(1) as f64; + loop { + tokio::select! { + _ = cancel.cancelled() => break, + _ = clock.sleep(thresholds.interval) => {} + } + let Some(current) = cgroup::read_current() else { + continue; + }; + let used_ratio = current.bytes() as f64 / max; + let decision = verdict(used_ratio, &thresholds); + limiter.adjust(decision); + if matches!(decision, PressureVerdict::Tighten) { + purge_arenas(); + } + tracing::debug!( + used_ratio, + limit = limiter.limit(), + ?decision, + "adaptive concurrency tick" + ); + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn thresholds() -> AdaptiveThresholds { + AdaptiveThresholds { + interval: Duration::from_millis(500), + relieve_below_ratio: 0.65, + tighten_above_ratio: 0.80, + } + } + + #[test] + fn over_tighten_line_tightens() { + assert!(matches!( + verdict(0.92, &thresholds()), + PressureVerdict::Tighten + )); + } + + #[test] + fn under_relieve_line_relieves() { + assert!(matches!( + verdict(0.40, &thresholds()), + PressureVerdict::Relieve + )); + } + + #[test] + fn between_lines_holds() { + assert!(matches!( + verdict(0.72, &thresholds()), + PressureVerdict::Hold + )); + } +} diff --git a/bobbin/crates/edge-index/Cargo.toml b/bobbin/crates/edge-index/Cargo.toml new file mode 100644 --- /dev/null +++ b/bobbin/crates/edge-index/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "bobbin-edge-index" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-runtime = { workspace = true } +bobbin-types = { workspace = true } +either = { workspace = true } +jacquard-common = { workspace = true } +lasso = { workspace = true } +scc = { workspace = true } +smallvec = "1" +thiserror = { workspace = true } +tokio = { workspace = true } diff --git a/bobbin/crates/edge-index/src/coverage.rs b/bobbin/crates/edge-index/src/coverage.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/edge-index/src/coverage.rs @@ -0,0 +1,267 @@ +use tokio::sync::watch; + +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct HydrantCursor(u64); + +impl HydrantCursor { + pub const fn new(id: u64) -> Self { + Self(id) + } + + pub const fn raw(self) -> u64 { + self.0 + } +} + +impl From for HydrantCursor { + fn from(id: u64) -> Self { + Self(id) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Coverage { + Warming { + events_processed: u64, + last_cursor: HydrantCursor, + }, + Ready { + events_processed: u64, + last_cursor: HydrantCursor, + }, +} + +impl Default for Coverage { + fn default() -> Self { + Self::Warming { + events_processed: 0, + last_cursor: HydrantCursor::default(), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct PromotionSignal { + pub rev_micros: Option, + pub now_micros: u64, + pub skew_micros: u64, +} + +impl PromotionSignal { + pub fn caught_up(self) -> bool { + let Some(rev) = self.rev_micros else { + return false; + }; + self.now_micros.abs_diff(rev) <= self.skew_micros + } +} + +impl Coverage { + pub const fn is_ready(self) -> bool { + matches!(self, Self::Ready { .. }) + } + + pub const fn events_processed(self) -> u64 { + match self { + Self::Warming { + events_processed, .. + } + | Self::Ready { + events_processed, .. + } => events_processed, + } + } + + pub const fn last_cursor(self) -> HydrantCursor { + match self { + Self::Warming { last_cursor, .. } | Self::Ready { last_cursor, .. } => last_cursor, + } + } + + pub fn advance(self, cursor: HydrantCursor) -> Self { + debug_assert!( + cursor.raw() >= self.last_cursor().raw(), + "hydrant cursor regressed: {} -> {}", + self.last_cursor().raw(), + cursor.raw(), + ); + let processed = self.events_processed().saturating_add(1); + match self { + Self::Warming { .. } => Self::Warming { + events_processed: processed, + last_cursor: cursor, + }, + Self::Ready { .. } => Self::Ready { + events_processed: processed, + last_cursor: cursor, + }, + } + } + + pub fn maybe_promote(self, signal: PromotionSignal) -> Self { + match self { + Self::Ready { .. } => self, + Self::Warming { + events_processed, + last_cursor, + } if signal.caught_up() => Self::Ready { + events_processed, + last_cursor, + }, + warming => warming, + } + } + + pub const fn force_ready(self) -> Self { + match self { + Self::Ready { .. } => self, + Self::Warming { + events_processed, + last_cursor, + } => Self::Ready { + events_processed, + last_cursor, + }, + } + } +} + +#[derive(Debug)] +pub struct CoverageWatch { + tx: watch::Sender, +} + +impl Default for CoverageWatch { + fn default() -> Self { + Self::new() + } +} + +impl CoverageWatch { + pub fn new() -> Self { + let (tx, _) = watch::channel(Coverage::default()); + Self { tx } + } + + pub fn snapshot(&self) -> Coverage { + *self.tx.borrow() + } + + pub fn subscribe(&self) -> watch::Receiver { + self.tx.subscribe() + } + + pub fn update(&self, transform: F) + where + F: FnOnce(Coverage) -> Coverage, + { + self.tx.send_if_modified(|c| { + let next = transform(*c); + if *c == next { + false + } else { + *c = next; + true + } + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SKEW: u64 = 60_000_000; + + fn signal(rev: u64, now: u64) -> PromotionSignal { + PromotionSignal { + rev_micros: Some(rev), + now_micros: now, + skew_micros: SKEW, + } + } + + #[test] + fn defaults_to_warming_at_zero() { + let c = Coverage::default(); + assert!(!c.is_ready()); + assert_eq!(c.events_processed(), 0); + assert_eq!(c.last_cursor(), HydrantCursor::default()); + } + + #[test] + fn advance_increments_and_tracks_cursor() { + let c = Coverage::default() + .advance(HydrantCursor::new(7)) + .advance(HydrantCursor::new(9)); + assert_eq!(c.events_processed(), 2); + assert_eq!(c.last_cursor(), HydrantCursor::new(9)); + assert!(!c.is_ready()); + } + + #[test] + fn promotion_requires_recent_rev() { + let now = 1_000_000_000; + let recent = now - SKEW / 2; + let stale = now - SKEW * 10; + + let c = Coverage::default().advance(HydrantCursor::new(1)); + assert!(!c.maybe_promote(signal(stale, now)).is_ready()); + assert!(c.maybe_promote(signal(recent, now)).is_ready()); + } + + #[test] + fn promotion_preserves_counters() { + let now = 1_000_000_000; + let c = Coverage::default() + .advance(HydrantCursor::new(3)) + .maybe_promote(signal(now, now)) + .advance(HydrantCursor::new(4)); + assert!(c.is_ready()); + assert_eq!(c.events_processed(), 2); + assert_eq!(c.last_cursor(), HydrantCursor::new(4)); + } + + #[test] + fn ready_is_sticky() { + let now = 1_000_000_000; + let stale = now - SKEW * 10; + let c = Coverage::default() + .advance(HydrantCursor::new(1)) + .maybe_promote(signal(now, now)) + .advance(HydrantCursor::new(2)) + .maybe_promote(signal(stale, now)); + assert!(c.is_ready()); + } + + #[test] + fn future_rev_within_skew_promotes() { + let now = 1_000_000_000; + let near_future = now + SKEW / 2; + let c = Coverage::default() + .advance(HydrantCursor::new(1)) + .maybe_promote(signal(near_future, now)); + assert!(c.is_ready()); + } + + #[test] + fn future_rev_beyond_skew_does_not_promote() { + let now = 1_000_000_000; + let far_future = now + SKEW * 10; + let c = Coverage::default() + .advance(HydrantCursor::new(1)) + .maybe_promote(signal(far_future, now)); + assert!(!c.is_ready()); + } + + #[test] + fn missing_rev_does_not_promote() { + let c = Coverage::default().advance(HydrantCursor::new(1)); + let s = PromotionSignal { + rev_micros: None, + now_micros: 1_000_000_000, + skew_micros: SKEW, + }; + assert!(!c.maybe_promote(s).is_ready()); + } +} diff --git a/bobbin/crates/edge-index/src/lib.rs b/bobbin/crates/edge-index/src/lib.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/edge-index/src/lib.rs @@ -0,0 +1,1563 @@ +use std::collections::{BTreeSet, HashMap}; +use std::marker::PhantomData; +use std::num::NonZeroU32; +use std::ops::{Bound, ControlFlow}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::{Arc, Mutex}; + +const FILTER_SCAN_MULTIPLIER: usize = 64; +const FILTER_SCAN_FLOOR: usize = 512; + +struct ScanState { + matched: Vec<(BucketKey, AtUri)>, + last_scanned: Option, + scanned: usize, +} + +impl ScanState { + fn with_capacity(cap: usize) -> Self { + Self { + matched: Vec::with_capacity(cap), + last_scanned: None, + scanned: 0, + } + } +} + +use bobbin_runtime::RuntimeHasher; +use bobbin_types::edges::Edge; +use bobbin_types::ids::EdgeKey; +use either::Either; +use jacquard_common::DefaultStr; +use jacquard_common::types::string::AtUri; +use lasso::{Key, Spur, ThreadedRodeo}; +use scc::HashMap as SccMap; +use scc::hash_map::Entry; +use smallvec::SmallVec; +use thiserror::Error; + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Ord, PartialOrd)] +struct SortMicros(u64); + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Ord, PartialOrd)] +struct BucketKey { + micros: SortMicros, + source: SourceId, +} + +impl BucketKey { + fn new(micros: u64, source: SourceId) -> Self { + Self { + micros: SortMicros(micros), + source, + } + } + + fn token(self) -> PageToken { + PageToken::new(self.micros.0, self.source.index()) + } + + fn from_token(tok: PageToken) -> Self { + Self { + micros: SortMicros(tok.micros), + source: SourceId::from_raw(tok.source), + } + } +} + +pub mod coverage; +pub mod state_index; +pub use coverage::{Coverage, CoverageWatch, HydrantCursor, PromotionSignal}; +pub use state_index::{ + ApplyOutcome, IssueStateKind, PullStatusKind, StateIndex, StateKind, apply_record_state, +}; + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct SourceTag; +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +struct AuthorTag; +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +struct CollectionTag; + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct Interned(u32, PhantomData); + +impl Interned { + fn from_spur(spur: Spur) -> Self { + Self(spur.into_usize() as u32, PhantomData) + } + + fn from_raw(raw: u32) -> Self { + Self(raw, PhantomData) + } + + fn index(self) -> u32 { + self.0 + } + + fn to_spur(self) -> Option { + Spur::try_from_usize(self.0 as usize) + } +} + +pub type SourceId = Interned; +type AuthorId = Interned; +type CollectionId = Interned; + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct PageToken { + micros: u64, + source: u32, +} + +impl PageToken { + pub fn new(micros: u64, source: u32) -> Self { + Self { micros, source } + } + + pub fn micros(self) -> u64 { + self.micros + } + + pub fn source(self) -> u32 { + self.source + } + + pub fn encode_token(self) -> String { + let mut bytes = [0u8; 12]; + bytes[..8].copy_from_slice(&self.micros.to_be_bytes()); + bytes[8..].copy_from_slice(&self.source.to_be_bytes()); + encode_hex(&bytes) + } + + pub fn decode_token(token: &str) -> Result { + let bytes: [u8; 12] = decode_hex_array(token).ok_or(CursorParseError::Malformed)?; + let micros = u64::from_be_bytes(bytes[..8].try_into().unwrap()); + let source = u32::from_be_bytes(bytes[8..].try_into().unwrap()); + Ok(Self { micros, source }) + } +} + +fn encode_hex(bytes: &[u8]) -> String { + bytes + .iter() + .fold(String::with_capacity(bytes.len() * 2), |mut acc, b| { + acc.push(char::from_digit((b >> 4) as u32, 16).unwrap()); + acc.push(char::from_digit((b & 0x0f) as u32, 16).unwrap()); + acc + }) +} + +fn decode_hex_array(token: &str) -> Option<[u8; N]> { + if token.len() != N * 2 { + return None; + } + let parsed: Vec = token + .as_bytes() + .chunks_exact(2) + .map(|pair| { + let hi = (pair[0] as char).to_digit(16)?; + let lo = (pair[1] as char).to_digit(16)?; + Some(((hi << 4) | lo) as u8) + }) + .collect::>>()?; + parsed.try_into().ok() +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PageCursor { + Start, + After(PageToken), +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum SortDir { + Asc, + #[default] + Desc, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)] +pub enum CursorParseError { + #[error("cursor token must be a valid TID")] + Malformed, +} + +impl PageCursor { + pub fn from_token(raw: Option<&str>) -> Result { + raw.map_or(Ok(Self::Start), |t| { + PageToken::decode_token(t).map(Self::After) + }) + } +} + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct PageLimit(u32); + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)] +pub enum PageLimitError { + #[error("page limit {value} below minimum {min}")] + TooSmall { value: u32, min: u32 }, + #[error("page limit {value} above maximum {max}")] + TooLarge { value: u32, max: u32 }, +} + +impl PageLimit { + pub const MIN: u32 = 1; + pub const MAX: u32 = 1000; + + pub fn new(value: u32) -> Result { + match value { + v if v < Self::MIN => Err(PageLimitError::TooSmall { + value: v, + min: Self::MIN, + }), + v if v > Self::MAX => Err(PageLimitError::TooLarge { + value: v, + max: Self::MAX, + }), + v => Ok(Self(v)), + } + } + + pub const fn get(self) -> u32 { + self.0 + } +} + +#[derive(Debug)] +pub struct EdgePage { + pub items: Vec>, + pub next: Option, +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct EdgeMemReport { + pub key_count: u64, + pub source_count: u64, + pub source_interner_bytes: u64, + pub did_interner_bytes: u64, + pub collection_interner_bytes: u64, + pub key_interner_bytes: u64, + pub edges_total: u64, + pub author_refs_total: u64, + pub reverse_entries: u64, + pub reverse_cap: u64, + pub forward_struct_bytes: u64, + pub reverse_struct_bytes: u64, + pub bucket_struct_bytes: u64, + pub max_bucket: u64, + pub bucket_size_classes: [u64; BUCKET_CLASS_COUNT], +} + +const BUCKET_CLASS_BOUNDS: [u64; 16] = [ + 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 8192, 32768, 131072, u64::MAX, +]; +const BUCKET_CLASS_COUNT: usize = BUCKET_CLASS_BOUNDS.len(); + +fn bucket_class(n: u64) -> usize { + BUCKET_CLASS_BOUNDS + .iter() + .position(|&bound| n <= bound) + .unwrap_or(BUCKET_CLASS_COUNT - 1) +} + +impl EdgeMemReport { + pub fn bucket_histogram(&self) -> impl Iterator { + BUCKET_CLASS_BOUNDS.into_iter().zip(self.bucket_size_classes) + } +} + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +struct EdgeKeyId(u32); + +fn bump_author(authors: &mut HashMap, author: AuthorId) { + authors + .entry(author) + .and_modify(|c| *c = c.saturating_add(1)) + .or_insert(NonZeroU32::MIN); +} + +fn drop_author(authors: &mut HashMap, author: AuthorId) { + match authors.get(&author).map(|c| c.get() - 1) { + Some(0) | None => { + authors.remove(&author); + } + Some(next) => { + authors.insert(author, NonZeroU32::new(next).unwrap()); + } + } +} + +struct LargeBucket { + keys: BTreeSet, + authors: HashMap, +} + +const BUCKET_PROMOTE_AT: usize = 256; +const SOURCE_BYTES: u64 = 16; + +enum Sources { + Small(SmallVec<[BucketKey; 2]>), + Large(Box), +} + +impl Default for Sources { + fn default() -> Self { + Self::Small(SmallVec::new()) + } +} + +impl Sources { + fn len(&self) -> usize { + match self { + Self::Small(v) => v.len(), + Self::Large(big) => big.keys.len(), + } + } + + fn is_empty(&self) -> bool { + self.len() == 0 + } + + fn insert(&mut self, key: BucketKey, author: Option) -> bool { + match self { + Self::Small(v) => match v.binary_search(&key) { + Ok(_) => false, + Err(pos) => { + v.insert(pos, key); + true + } + }, + Self::Large(big) => { + let inserted = big.keys.insert(key); + if inserted && let Some(a) = author { + bump_author(&mut big.authors, a); + } + inserted + } + } + } + + fn remove(&mut self, key: &BucketKey, author: Option) { + match self { + Self::Small(v) => { + if let Ok(pos) = v.binary_search(key) { + v.remove(pos); + } + } + Self::Large(big) => { + if big.keys.remove(key) && let Some(a) = author { + drop_author(&mut big.authors, a); + } + } + } + } + + fn directed( + &self, + cursor: PageCursor, + dir: SortDir, + ) -> Box + '_> { + match self { + Self::Small(v) => Box::new(directed_slice(v, cursor, dir)), + Self::Large(big) => Box::new(directed_tree(&big.keys, cursor, dir)), + } + } + + fn heap_bytes(&self) -> u64 { + const BTREE_BYTES_PER_KEY: u64 = 32; + const HASHMAP_FIXED: u64 = 48; + const HASHMAP_PER_CAP: u64 = 9; + match self { + Self::Small(v) if v.spilled() => v.capacity() as u64 * SOURCE_BYTES, + Self::Small(_) => 0, + Self::Large(big) => { + std::mem::size_of::() as u64 + + big.keys.len() as u64 * BTREE_BYTES_PER_KEY + + HASHMAP_FIXED + + big.authors.capacity() as u64 * HASHMAP_PER_CAP + } + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ReverseEntry { + key_id: EdgeKeyId, + sort_micros: u64, +} + +pub struct EdgeStore { + source_interner: Arc>, + did_interner: Arc>, + collection_interner: Arc>, + key_ids: SccMap, + next_key_id: AtomicU32, + forward: SccMap, + reverse: SccMap, RuntimeHasher>, + hasher: RuntimeHasher, + writer: Mutex<()>, +} + +impl EdgeStore { + pub fn new(hasher: RuntimeHasher) -> Self { + Self { + source_interner: Arc::new(ThreadedRodeo::with_hasher(hasher.clone())), + did_interner: Arc::new(ThreadedRodeo::with_hasher(hasher.clone())), + collection_interner: Arc::new(ThreadedRodeo::with_hasher(hasher.clone())), + key_ids: SccMap::with_hasher(hasher.clone()), + next_key_id: AtomicU32::new(0), + forward: SccMap::with_hasher(hasher.clone()), + reverse: SccMap::with_hasher(hasher.clone()), + hasher, + writer: Mutex::new(()), + } + } + + fn intern_key(&self, key: EdgeKey) -> EdgeKeyId { + match self.key_ids.entry_sync(key) { + Entry::Occupied(e) => *e.get(), + Entry::Vacant(e) => { + let id = EdgeKeyId(self.next_key_id.fetch_add(1, Ordering::Relaxed)); + e.insert_entry(id); + id + } + } + } + + fn lookup_key(&self, key: &EdgeKey) -> Option { + self.key_ids.read_sync(key, |_, id| *id) + } + + pub fn add(&self, edge: Edge) { + let _w = self + .writer + .lock() + .expect("edge-store writer mutex poisoned"); + self.add_locked(edge); + } + + pub fn intern_source(&self, source: &AtUri) -> SourceId { + let author = self.intern_author(source); + SourceId::from_spur( + self.source_interner + .get_or_intern(self.source_key(source, author)), + ) + } + + pub fn upsert_source(&self, source: &AtUri, edges: Vec) { + let _w = self + .writer + .lock() + .expect("edge-store writer mutex poisoned"); + self.clear_source_locked(source); + edges.into_iter().for_each(|e| self.add_locked(e)); + } + + pub fn remove_source(&self, source: &AtUri) { + let _w = self + .writer + .lock() + .expect("edge-store writer mutex poisoned"); + self.clear_source_locked(source); + } + + fn add_locked(&self, edge: Edge) { + let author = self.intern_author(&edge.source); + let source_key = self.source_key(&edge.source, author); + let id = SourceId::from_spur(self.source_interner.get_or_intern(source_key)); + let sort_micros = edge.sort_micros; + let key_id = self.intern_key(EdgeKey::new(edge.kind, edge.subject)); + + let key = BucketKey::new(sort_micros, id); + let mut entry = self.forward.entry_sync(key_id).or_default(); + let inserted = entry.get_mut().insert(key, author); + let promote_keys = match entry.get() { + Sources::Small(v) if v.len() > BUCKET_PROMOTE_AT => { + Some(v.iter().copied().collect::>()) + } + _ => None, + }; + drop(entry); + + if let Some(keys) = promote_keys { + let authors = self.build_author_map(&keys); + let large = LargeBucket { + keys: keys.into_iter().collect(), + authors, + }; + if let Entry::Occupied(mut e) = self.forward.entry_sync(key_id) { + *e.get_mut() = Sources::Large(Box::new(large)); + } + } + + if inserted { + let mut rev = self.reverse.entry_sync(id).or_default(); + rev.get_mut().push(ReverseEntry { + key_id, + sort_micros, + }); + } + } + + fn clear_source_locked(&self, source: &AtUri) { + let author = source_authority_did(source) + .and_then(|s| self.did_interner.get(s)) + .map(AuthorId::from_spur); + let Some(source_spur) = self.source_interner.get(self.source_key(source, author)) else { + return; + }; + let id = SourceId::from_spur(source_spur); + let Some((_, entries)) = self.reverse.remove_sync(&id) else { + return; + }; + entries.into_iter().for_each( + |ReverseEntry { + key_id, + sort_micros, + }| { + self.forward.update_sync(&key_id, |_, sources| { + sources.remove(&BucketKey::new(sort_micros, id), author); + }); + self.forward + .remove_if_sync(&key_id, |sources| sources.is_empty()); + }, + ); + } + + fn intern_author(&self, source: &AtUri) -> Option { + let did = source_authority_did(source)?; + Some(AuthorId::from_spur(self.did_interner.get_or_intern(did))) + } + + fn source_key(&self, source: &AtUri, author: Option) -> String { + match (split_record_uri(source.as_ref()), author) { + (Some((_, collection, rkey)), Some(author)) => { + let collection = + CollectionId::from_spur(self.collection_interner.get_or_intern(collection)); + format!("{}/{}/{}", author.index(), collection.index(), rkey) + } + _ => source.as_ref().to_owned(), + } + } + + fn decode_source(&self, stored: &str) -> Option { + if stored.starts_with("at://") { + return Some(stored.to_owned()); + } + let mut parts = stored.splitn(3, '/'); + let author = AuthorId::from_raw(parts.next()?.parse().ok()?); + let collection = CollectionId::from_raw(parts.next()?.parse().ok()?); + let rkey = parts.next()?; + let did = self.did_interner.try_resolve(&author.to_spur()?)?; + let collection = self + .collection_interner + .try_resolve(&collection.to_spur()?)?; + Some(format!("at://{did}/{collection}/{rkey}")) + } + + fn author_of_stored(&self, stored: &str) -> Option { + match stored.strip_prefix("at://") { + Some(rest) => { + let authority = rest.split('/').next().unwrap_or(rest); + authority + .starts_with("did:") + .then(|| self.did_interner.get(authority).map(AuthorId::from_spur)) + .flatten() + } + None => stored + .split('/') + .next()? + .parse::() + .ok() + .map(AuthorId::from_raw), + } + } + + fn author_of(&self, source: SourceId) -> Option { + let spur = source.to_spur()?; + let stored = self.source_interner.try_resolve(&spur)?; + self.author_of_stored(stored) + } + + fn distinct_authors_small(&self, keys: &[BucketKey]) -> u64 { + keys.iter() + .filter_map(|key| self.author_of(key.source)) + .collect::>() + .len() as u64 + } + + fn build_author_map(&self, keys: &[BucketKey]) -> HashMap { + keys.iter().fold( + HashMap::with_hasher(self.hasher.clone()), + |mut authors, key| { + if let Some(a) = self.author_of(key.source) { + bump_author(&mut authors, a); + } + authors + }, + ) + } + + pub fn count(&self, key: &EdgeKey) -> u64 { + self.lookup_key(key) + .and_then(|id| { + self.forward + .read_sync(&id, |_, sources| sources.len() as u64) + }) + .unwrap_or(0) + } + + pub fn count_distinct_authors(&self, key: &EdgeKey) -> u64 { + self.lookup_key(key) + .and_then(|id| { + self.forward.read_sync(&id, |_, sources| match sources { + Sources::Large(big) => big.authors.len() as u64, + Sources::Small(v) => self.distinct_authors_small(v), + }) + }) + .unwrap_or(0) + } + + pub fn list( + &self, + key: &EdgeKey, + cursor: PageCursor, + limit: PageLimit, + dir: SortDir, + ) -> EdgePage { + let limit_usize = limit.get() as usize; + self.lookup_key(key) + .and_then(|id| { + self.forward.read_sync(&id, |_, sources| { + let iter = sources.directed(cursor, dir); + let entries: Vec = iter.take(limit_usize + 1).collect(); + let has_more = entries.len() > limit_usize; + let page = &entries[..entries.len().min(limit_usize)]; + let items = page + .iter() + .filter_map(|&key| { + let spur = key.source.to_spur()?; + let stored = self.source_interner.try_resolve(&spur)?; + AtUri::new_owned(self.decode_source(stored)?).ok() + }) + .collect(); + let next = has_more + .then(|| page.last().copied()) + .flatten() + .map(BucketKey::token); + EdgePage { items, next } + }) + }) + .unwrap_or(EdgePage { + items: Vec::new(), + next: None, + }) + } + + pub fn list_filtered( + &self, + key: &EdgeKey, + cursor: PageCursor, + limit: PageLimit, + dir: SortDir, + predicate: F, + ) -> EdgePage + where + F: Fn(&AtUri) -> bool, + { + let limit_usize = limit.get() as usize; + let scan_cap = limit_usize + .saturating_mul(FILTER_SCAN_MULTIPLIER) + .max(FILTER_SCAN_FLOOR); + self.lookup_key(key) + .and_then(|id| { + self.forward.read_sync(&id, |_, sources| { + let init = ScanState::with_capacity(limit_usize + 1); + let outcome = sources.directed(cursor, dir).try_fold( + init, + |mut state, key| { + if state.scanned >= scan_cap && state.matched.len() <= limit_usize { + return ControlFlow::Break(state); + } + state.scanned += 1; + state.last_scanned = Some(key); + if let Some(spur) = key.source.to_spur() + && let Some(stored) = self.source_interner.try_resolve(&spur) + && let Some(decoded) = self.decode_source(stored) + && let Ok(uri) = AtUri::new_owned(decoded) + && predicate(&uri) + { + state.matched.push((key, uri)); + if state.matched.len() > limit_usize { + return ControlFlow::Break(state); + } + } + ControlFlow::Continue(state) + }, + ); + let (state, bucket_exhausted) = match outcome { + ControlFlow::Continue(s) => (s, true), + ControlFlow::Break(s) => (s, false), + }; + let has_more_matches = state.matched.len() > limit_usize; + let visible_len = state.matched.len().min(limit_usize); + let next = if has_more_matches && visible_len > 0 { + Some(state.matched[visible_len - 1].0.token()) + } else if !bucket_exhausted { + state.last_scanned.map(BucketKey::token) + } else { + None + }; + let items = state + .matched + .into_iter() + .take(visible_len) + .map(|(_, uri)| uri) + .collect(); + EdgePage { items, next } + }) + }) + .unwrap_or(EdgePage { + items: Vec::new(), + next: None, + }) + } + + pub fn key_count(&self) -> usize { + self.forward.len() + } + + pub fn source_count(&self) -> usize { + self.reverse.len() + } + + pub fn mem_report(&self) -> EdgeMemReport { + const SCC_SLOT: u64 = 32; + let edge_key = std::mem::size_of::() as u64; + let rev_entry = std::mem::size_of::() as u64; + let rev_smallvec = std::mem::size_of::>() as u64; + let bucket_struct = std::mem::size_of::() as u64; + + let mut edges_total = 0u64; + let mut author_refs_total = 0u64; + let mut forward_struct_bytes = 0u64; + let mut max_bucket = 0u64; + let mut bucket_size_classes = [0u64; BUCKET_CLASS_COUNT]; + self.forward.iter_sync(|_, sources| { + let bucket_len = sources.len() as u64; + edges_total += bucket_len; + max_bucket = max_bucket.max(bucket_len); + bucket_size_classes[bucket_class(bucket_len)] += 1; + if let Sources::Large(big) = sources { + author_refs_total += big.authors.len() as u64; + } + forward_struct_bytes += SCC_SLOT + bucket_struct + sources.heap_bytes(); + true + }); + let key_interner_bytes = self.key_ids.len() as u64 + * (edge_key + std::mem::size_of::() as u64 + SCC_SLOT); + + let mut reverse_entries = 0u64; + let mut reverse_cap = 0u64; + let mut reverse_struct_bytes = 0u64; + self.reverse.iter_sync(|_, refs| { + let cap = refs.capacity() as u64; + reverse_entries += refs.len() as u64; + reverse_cap += cap; + let heap = if refs.spilled() { cap * rev_entry } else { 0 }; + reverse_struct_bytes += SCC_SLOT + rev_smallvec + heap; + true + }); + + EdgeMemReport { + key_count: self.forward.len() as u64, + source_count: self.reverse.len() as u64, + source_interner_bytes: self.source_interner.current_memory_usage() as u64, + did_interner_bytes: self.did_interner.current_memory_usage() as u64, + collection_interner_bytes: self.collection_interner.current_memory_usage() as u64, + key_interner_bytes, + edges_total, + author_refs_total, + reverse_entries, + reverse_cap, + forward_struct_bytes, + reverse_struct_bytes, + bucket_struct_bytes: bucket_struct, + max_bucket, + bucket_size_classes, + } + } +} + +fn directed_slice( + sources: &[BucketKey], + cursor: PageCursor, + dir: SortDir, +) -> impl Iterator + '_ { + match dir { + SortDir::Asc => { + let start = match cursor { + PageCursor::Start => 0, + PageCursor::After(tok) => { + sources.partition_point(|k| *k <= BucketKey::from_token(tok)) + } + }; + Either::Left(sources[start..].iter().copied()) + } + SortDir::Desc => { + let end = match cursor { + PageCursor::Start => sources.len(), + PageCursor::After(tok) => { + sources.partition_point(|k| *k < BucketKey::from_token(tok)) + } + }; + Either::Right(sources[..end].iter().rev().copied()) + } + } +} + +fn directed_tree( + keys: &BTreeSet, + cursor: PageCursor, + dir: SortDir, +) -> impl Iterator + '_ { + match dir { + SortDir::Asc => { + let lower = match cursor { + PageCursor::Start => Bound::Unbounded, + PageCursor::After(tok) => Bound::Excluded(BucketKey::from_token(tok)), + }; + Either::Left(keys.range((lower, Bound::Unbounded)).copied()) + } + SortDir::Desc => { + let upper = match cursor { + PageCursor::Start => Bound::Unbounded, + PageCursor::After(tok) => Bound::Excluded(BucketKey::from_token(tok)), + }; + Either::Right(keys.range((Bound::Unbounded, upper)).rev().copied()) + } + } +} + +fn split_record_uri(source: &str) -> Option<(&str, &str, &str)> { + let rest = source.strip_prefix("at://")?; + let mut parts = rest.split('/'); + let authority = parts.next()?; + let collection = parts.next()?; + let rkey = parts.next()?; + if parts.next().is_some() { + return None; + } + (authority.starts_with("did:") && !collection.is_empty() && !rkey.is_empty()) + .then_some((authority, collection, rkey)) +} + +fn source_authority_did(source: &AtUri) -> Option<&str> { + let rest = source.as_ref().strip_prefix("at://")?; + let end = rest.find('/').unwrap_or(rest.len()); + let candidate = &rest[..end]; + candidate.starts_with("did:").then_some(candidate) +} + +#[cfg(test)] +mod tests { + use super::*; + use bobbin_types::ids::SubjectRef; + use jacquard_common::types::did::Did; + use jacquard_common::types::nsid::Nsid; + use jacquard_common::types::string::AtUri; + + fn store() -> EdgeStore { + EdgeStore::new(RuntimeHasher::default()) + } + + fn nsid(s: &'static str) -> Nsid { + Nsid::new_static(s).unwrap() + } + + fn at(s: &str) -> AtUri { + AtUri::new_owned(s).unwrap() + } + + fn did(s: &str) -> Did { + Did::new_owned(s).unwrap() + } + + fn did_subj(s: &str) -> SubjectRef { + SubjectRef::Did(did(s)) + } + + fn limit(n: u32) -> PageLimit { + PageLimit::new(n).unwrap() + } + + const NAMES: [&str; 5] = ["nel", "olaren", "teq", "lyna", "bailey"]; + + fn star_edge(source: AtUri, subject: Did) -> Edge { + star_edge_at(source, subject, 0) + } + + fn star_edge_at(source: AtUri, subject: Did, sort_micros: u64) -> Edge { + Edge { + kind: nsid("sh.tangled.feed.star"), + subject: SubjectRef::Did(subject), + source, + sort_micros, + } + } + + fn shuffled_micros(i: usize) -> u64 { + (i as u64).wrapping_mul(2_654_435_761) % 64 + } + + fn source_uri(i: usize) -> String { + format!( + "at://did:plc:{}/sh.tangled.feed.star/r{i}", + NAMES[i % NAMES.len()] + ) + } + + fn fill_subject(store: &EdgeStore, n: usize) -> EdgeKey { + let subject = did("did:plc:squid"); + (0..n).for_each(|i| { + store.add(star_edge_at( + at(&source_uri(i)), + subject.clone(), + shuffled_micros(i), + )); + }); + EdgeKey::new(nsid("sh.tangled.feed.star"), did_subj("did:plc:squid")) + } + + fn reference(kept: impl Iterator) -> Vec { + let mut rows: Vec<(u64, usize)> = kept.map(|i| (shuffled_micros(i), i)).collect(); + rows.sort_unstable(); + rows.into_iter().map(|(_, i)| source_uri(i)).collect() + } + + fn paginate_all(store: &EdgeStore, key: &EdgeKey, dir: SortDir, page: u32) -> Vec { + std::iter::successors( + Some(store.list(key, PageCursor::Start, limit(page), dir)), + |prev| { + prev.next + .map(|tok| store.list(key, PageCursor::After(tok), limit(page), dir)) + }, + ) + .flat_map(|p| { + p.items + .into_iter() + .map(|u| u.as_ref().to_owned()) + .collect::>() + }) + .collect() + } + + #[test] + fn pagination_matches_reference_across_small_and_large() { + [64usize, 1000].into_iter().for_each(|n| { + let store = store(); + let key = fill_subject(&store, n); + assert_eq!(store.count(&key), n as u64, "count mismatch at n={n}"); + assert_eq!( + store.count_distinct_authors(&key), + NAMES.len() as u64, + "distinct authors at n={n}" + ); + + let asc = reference(0..n); + let desc: Vec = asc.iter().rev().cloned().collect(); + + [3u32, 7, 50].into_iter().for_each(|page| { + assert_eq!( + paginate_all(&store, &key, SortDir::Asc, page), + asc, + "asc mismatch n={n} page={page}" + ); + assert_eq!( + paginate_all(&store, &key, SortDir::Desc, page), + desc, + "desc mismatch n={n} page={page}" + ); + }); + }); + } + + #[test] + fn remove_from_large_bucket_keeps_pagination_exact() { + let store = store(); + let n = 1000usize; + let key = fill_subject(&store, n); + (0..n).step_by(3).for_each(|i| { + store.remove_source(&at(&source_uri(i))); + }); + let expected = reference((0..n).filter(|i| i % 3 != 0)); + assert_eq!(store.count(&key), expected.len() as u64); + assert_eq!( + store.count_distinct_authors(&key), + NAMES.len() as u64, + "every author keeps sources after partial removal" + ); + assert_eq!(paginate_all(&store, &key, SortDir::Asc, 7), expected); + assert_eq!( + paginate_all(&store, &key, SortDir::Desc, 11), + expected.iter().rev().cloned().collect::>() + ); + } + + #[test] + fn add_then_count() { + let store = store(); + let key = EdgeKey::new(nsid("sh.tangled.feed.star"), did_subj("did:plc:abalone")); + + store.add(star_edge( + at("at://did:plc:nel/sh.tangled.feed.star/r1"), + did("did:plc:abalone"), + )); + store.add(star_edge( + at("at://did:plc:olaren/sh.tangled.feed.star/r2"), + did("did:plc:abalone"), + )); + store.add(star_edge( + at("at://did:plc:nel/sh.tangled.feed.star/r3"), + did("did:plc:abalone"), + )); + + assert_eq!(store.count(&key), 3); + assert_eq!(store.count_distinct_authors(&key), 2); + } + + #[test] + fn duplicate_add_is_idempotent() { + let store = store(); + let key = EdgeKey::new(nsid("sh.tangled.feed.star"), did_subj("did:plc:abalone")); + let edge = star_edge( + at("at://did:plc:nel/sh.tangled.feed.star/r1"), + did("did:plc:abalone"), + ); + store.add(edge.clone()); + store.add(edge); + assert_eq!(store.count(&key), 1); + assert_eq!(store.count_distinct_authors(&key), 1); + } + + #[test] + fn remove_source_clears_all_keys_for_that_source() { + let store = store(); + let star_key = EdgeKey::new(nsid("sh.tangled.feed.star"), did_subj("did:plc:abalone")); + let follow_key = EdgeKey::new(nsid("sh.tangled.graph.follow"), did_subj("did:plc:lyna")); + let source = "at://did:plc:nel/sh.tangled.feed.star/r1"; + + store.add(Edge { + kind: nsid("sh.tangled.feed.star"), + subject: did_subj("did:plc:abalone"), + source: at(source), + sort_micros: 0, + }); + store.add(Edge { + kind: nsid("sh.tangled.graph.follow"), + subject: did_subj("did:plc:lyna"), + source: at(source), + sort_micros: 0, + }); + + assert_eq!(store.count(&star_key), 1); + assert_eq!(store.count(&follow_key), 1); + + store.remove_source(&at(source)); + assert_eq!(store.count(&star_key), 0); + assert_eq!(store.count(&follow_key), 0); + } + + #[test] + fn upsert_source_replaces_old_edges() { + let store = store(); + let source = at("at://did:plc:teq/sh.tangled.feed.star/r1"); + let old_subject = did_subj("did:plc:abalone"); + let new_subject = did_subj("did:plc:uni"); + let kind = nsid("sh.tangled.feed.star"); + + store.upsert_source( + &source, + vec![Edge { + kind: kind.clone(), + subject: old_subject.clone(), + source: source.clone(), + sort_micros: 0, + }], + ); + assert_eq!( + store.count(&EdgeKey::new(kind.clone(), old_subject.clone())), + 1 + ); + + store.upsert_source( + &source, + vec![Edge { + kind: kind.clone(), + subject: new_subject.clone(), + source: source.clone(), + sort_micros: 0, + }], + ); + assert_eq!(store.count(&EdgeKey::new(kind.clone(), old_subject)), 0); + assert_eq!(store.count(&EdgeKey::new(kind, new_subject)), 1); + } + + #[test] + fn list_pages_in_sort_order() { + let store = store(); + let key = EdgeKey::new(nsid("sh.tangled.feed.star"), did_subj("did:plc:abalone")); + (0..5).for_each(|i| { + store.add(star_edge_at( + at(&format!( + "at://did:plc:{}/sh.tangled.feed.star/r{i}", + NAMES[i] + )), + did("did:plc:abalone"), + 1_000_000 + i as u64 * 1_000_000, + )); + }); + + let page1 = store.list(&key, PageCursor::Start, limit(2), SortDir::Asc); + assert_eq!(page1.items.len(), 2); + let cursor = PageCursor::After(page1.next.expect("has next")); + + let page2 = store.list(&key, cursor, limit(2), SortDir::Asc); + assert_eq!(page2.items.len(), 2); + + let cursor2 = PageCursor::After(page2.next.expect("has next")); + let page3 = store.list(&key, cursor2, limit(2), SortDir::Asc); + assert_eq!(page3.items.len(), 1); + assert!( + page3.next.is_none(), + "final partial page should signal exhaustion" + ); + } + + #[test] + fn list_exact_fill_signals_exhaustion() { + let store = store(); + let key = EdgeKey::new(nsid("sh.tangled.feed.star"), did_subj("did:plc:abalone")); + (0..2).for_each(|i| { + store.add(star_edge( + at(&format!( + "at://did:plc:{}/sh.tangled.feed.star/r{i}", + NAMES[i] + )), + did("did:plc:abalone"), + )); + }); + + let page = store.list(&key, PageCursor::Start, limit(2), SortDir::Asc); + assert_eq!(page.items.len(), 2); + assert!(page.next.is_none(), "exact-fill page must not promise more"); + } + + #[test] + fn list_on_unknown_key_is_empty() { + let store = store(); + let page = store.list( + &EdgeKey::new(nsid("sh.tangled.feed.star"), did_subj("did:plc:periwinkle")), + PageCursor::Start, + limit(10), + SortDir::Asc, + ); + assert!(page.items.is_empty()); + assert!(page.next.is_none()); + } + + #[test] + fn distinct_authors_decreases_when_last_source_from_author_removed() { + let store = store(); + let key = EdgeKey::new(nsid("sh.tangled.feed.star"), did_subj("did:plc:abalone")); + let s1 = at("at://did:plc:nel/sh.tangled.feed.star/r1"); + let s2 = at("at://did:plc:nel/sh.tangled.feed.star/r2"); + let s3 = at("at://did:plc:olaren/sh.tangled.feed.star/r3"); + + store.add(star_edge(s1.clone(), did("did:plc:abalone"))); + store.add(star_edge(s2.clone(), did("did:plc:abalone"))); + store.add(star_edge(s3, did("did:plc:abalone"))); + assert_eq!(store.count_distinct_authors(&key), 2); + + store.remove_source(&s1); + assert_eq!(store.count_distinct_authors(&key), 2, "user1 still has s2"); + + store.remove_source(&s2); + assert_eq!(store.count_distinct_authors(&key), 1, "user1 fully gone"); + } + + #[test] + fn page_limit_rejects_zero_and_oversize() { + assert!(matches!( + PageLimit::new(0), + Err(PageLimitError::TooSmall { value: 0, min: 1 }) + )); + assert!(matches!( + PageLimit::new(PageLimit::MAX + 1), + Err(PageLimitError::TooLarge { .. }) + )); + assert_eq!(PageLimit::new(50).unwrap().get(), 50); + } + + #[test] + fn cursor_token_round_trip() { + let original = PageToken::new(1_730_000_000_000_000, 0x1234_abcd); + let token = original.encode_token(); + assert_eq!(token.len(), 24, "12-byte cursor encodes to 24 hex chars"); + assert_eq!(PageToken::decode_token(&token).unwrap(), original); + } + + #[test] + fn from_token_none_yields_start() { + assert_eq!(PageCursor::from_token(None).unwrap(), PageCursor::Start); + } + + #[test] + fn from_token_some_yields_after() { + let original = PageToken::new(1_730_000_000_000_000, 42); + let token = original.encode_token(); + assert_eq!( + PageCursor::from_token(Some(&token)).unwrap(), + PageCursor::After(original), + ); + } + + #[test] + fn cursor_decode_rejects_malformed() { + let bad = [ + "", + "deadbeef", + "no-hex!!aaaaaaaaaaaaaaaa", + "12345", + "this-string-is-way-too-long-to-be-a-valid-cursor", + ]; + bad.into_iter().for_each(|s| { + assert!( + matches!(PageToken::decode_token(s), Err(CursorParseError::Malformed)), + "expected malformed for {s:?}", + ); + }); + } + + #[test] + fn cursor_token_is_hex_shape() { + let token = PageToken::new(1_730_000_000_000_000, 7).encode_token(); + assert_eq!(token.len(), 24); + assert!(token.chars().all(|c| c.is_ascii_hexdigit())); + } + + #[test] + fn list_does_not_drop_entries_at_same_sort_micros_across_pages() { + let store = store(); + let subject_did = did("did:plc:limpet"); + let key = EdgeKey::new( + nsid("sh.tangled.feed.star"), + SubjectRef::Did(subject_did.clone()), + ); + (0..4).for_each(|i| { + store.add(star_edge_at( + at(&format!( + "at://did:plc:{}/sh.tangled.feed.star/r{i}", + NAMES[i] + )), + subject_did.clone(), + 42, + )); + }); + + let page1 = store.list(&key, PageCursor::Start, limit(2), SortDir::Asc); + assert_eq!(page1.items.len(), 2); + let token = page1.next.expect("cursor must continue across ties"); + + let page2 = store.list(&key, PageCursor::After(token), limit(2), SortDir::Asc); + assert_eq!( + page2.items.len(), + 2, + "remaining ties must surface on next page" + ); + assert!(page2.next.is_none()); + let combined: std::collections::HashSet = page1 + .items + .iter() + .chain(page2.items.iter()) + .map(|u| u.as_ref().to_owned()) + .collect(); + assert_eq!(combined.len(), 4, "every tied entry visible exactly once"); + } + + #[test] + fn list_filtered_narrows_by_predicate_and_paginates() { + let store = store(); + let subject_did = did("did:plc:limpet"); + let key = EdgeKey::new( + nsid("sh.tangled.repo.issue"), + SubjectRef::Did(subject_did.clone()), + ); + let by_nel = (0..3).map(|i| { + star_edge_at( + at(&format!("at://did:plc:nel/sh.tangled.repo.issue/r{i}")), + subject_did.clone(), + 100 + i as u64, + ) + }); + let by_olaren = (0..2).map(|i| { + star_edge_at( + at(&format!("at://did:plc:olaren/sh.tangled.repo.issue/o{i}")), + subject_did.clone(), + 500 + i as u64, + ) + }); + by_nel + .chain(by_olaren) + .map(|mut e| { + e.kind = nsid("sh.tangled.repo.issue"); + e + }) + .for_each(|e| store.add(e)); + + let only_nel = |u: &AtUri| u.as_ref().starts_with("at://did:plc:nel/"); + let page1 = store.list_filtered(&key, PageCursor::Start, limit(2), SortDir::Asc, only_nel); + assert_eq!(page1.items.len(), 2); + assert!(page1.next.is_some(), "cursor must allow more nel matches"); + + let page2 = store.list_filtered( + &key, + PageCursor::After(page1.next.unwrap()), + limit(2), + SortDir::Asc, + only_nel, + ); + assert_eq!(page2.items.len(), 1, "only one nel issue left"); + assert!(page2.next.is_none(), "tail page must not promise more"); + } + + #[test] + fn list_descending_returns_newest_first() { + let store = store(); + let subject_did = did("did:plc:limpet"); + let key = EdgeKey::new( + nsid("sh.tangled.feed.star"), + SubjectRef::Did(subject_did.clone()), + ); + (0..5).for_each(|i| { + store.add(star_edge_at( + at(&format!( + "at://did:plc:{}/sh.tangled.feed.star/r{i}", + NAMES[i] + )), + subject_did.clone(), + 1_000_000 + i as u64 * 1_000_000, + )); + }); + + let asc = store.list(&key, PageCursor::Start, limit(5), SortDir::Asc); + let desc = store.list(&key, PageCursor::Start, limit(5), SortDir::Desc); + assert_eq!(asc.items.len(), 5); + assert_eq!(desc.items.len(), 5); + let asc_uris: Vec<_> = asc.items.iter().map(|u| u.as_ref().to_owned()).collect(); + let mut reversed = asc_uris.clone(); + reversed.reverse(); + let desc_uris: Vec<_> = desc.items.iter().map(|u| u.as_ref().to_owned()).collect(); + assert_eq!(desc_uris, reversed, "desc must be exact reverse of asc"); + } + + #[test] + fn list_descending_paginates_with_cursor() { + let store = store(); + let subject_did = did("did:plc:whelk"); + let key = EdgeKey::new( + nsid("sh.tangled.feed.star"), + SubjectRef::Did(subject_did.clone()), + ); + (0..5).for_each(|i| { + store.add(star_edge_at( + at(&format!( + "at://did:plc:{}/sh.tangled.feed.star/r{i}", + NAMES[i] + )), + subject_did.clone(), + 1_000_000 + i as u64 * 1_000_000, + )); + }); + + let page1 = store.list(&key, PageCursor::Start, limit(2), SortDir::Desc); + assert_eq!(page1.items.len(), 2); + let cursor = PageCursor::After(page1.next.expect("desc page1 must continue")); + let page2 = store.list(&key, cursor, limit(2), SortDir::Desc); + assert_eq!(page2.items.len(), 2); + let cursor2 = PageCursor::After(page2.next.expect("desc page2 must continue")); + let page3 = store.list(&key, cursor2, limit(2), SortDir::Desc); + assert_eq!(page3.items.len(), 1); + assert!(page3.next.is_none()); + + let combined: std::collections::HashSet = page1 + .items + .iter() + .chain(page2.items.iter()) + .chain(page3.items.iter()) + .map(|u| u.as_ref().to_owned()) + .collect(); + assert_eq!( + combined.len(), + 5, + "desc pagination visits each item exactly once" + ); + } + + #[test] + fn list_filtered_descending_respects_predicate() { + let store = store(); + let subject_did = did("did:plc:scallop"); + let key = EdgeKey::new( + nsid("sh.tangled.repo.issue"), + SubjectRef::Did(subject_did.clone()), + ); + let by_nel = (0..3).map(|i| { + star_edge_at( + at(&format!("at://did:plc:nel/sh.tangled.repo.issue/r{i}")), + subject_did.clone(), + 100 + i as u64, + ) + }); + let by_olaren = (0..2).map(|i| { + star_edge_at( + at(&format!("at://did:plc:olaren/sh.tangled.repo.issue/o{i}")), + subject_did.clone(), + 500 + i as u64, + ) + }); + by_nel + .chain(by_olaren) + .map(|mut e| { + e.kind = nsid("sh.tangled.repo.issue"); + e + }) + .for_each(|e| store.add(e)); + + let only_nel = |u: &AtUri| u.as_ref().starts_with("at://did:plc:nel/"); + let page = store.list_filtered(&key, PageCursor::Start, limit(5), SortDir::Desc, only_nel); + assert_eq!(page.items.len(), 3, "all three nel issues visible"); + let last = page.items.last().unwrap().as_ref(); + let first = page.items.first().unwrap().as_ref(); + assert!( + first > last, + "desc order: first item rkey must be greater than last (got first={first}, last={last})", + ); + } + + #[test] + fn non_did_source_round_trips_via_raw_fallback() { + let store = store(); + let subject = did("did:plc:limpet"); + let key = EdgeKey::new( + nsid("sh.tangled.feed.star"), + SubjectRef::Did(subject.clone()), + ); + let did_source = at("at://did:plc:nel/sh.tangled.feed.star/r1"); + let handle_source = at("at://witchcraft.systems/sh.tangled.feed.star/r2"); + store.add(star_edge_at(did_source.clone(), subject.clone(), 1)); + store.add(star_edge_at(handle_source.clone(), subject.clone(), 2)); + + let page = store.list(&key, PageCursor::Start, limit(10), SortDir::Asc); + let got: std::collections::HashSet = + page.items.iter().map(|u| u.as_ref().to_owned()).collect(); + assert!( + got.contains(did_source.as_ref()), + "did source must decode exactly" + ); + assert!( + got.contains(handle_source.as_ref()), + "non-did authority must round-trip through the raw fallback" + ); + + store.remove_source(&handle_source); + assert_eq!( + store.count(&key), + 1, + "raw-keyed source removable by its uri" + ); + } + + #[test] + fn distinct_collections_decode_with_their_own_collection() { + let store = store(); + let subject = did("did:plc:limpet"); + let key = EdgeKey::new( + nsid("sh.tangled.feed.star"), + SubjectRef::Did(subject.clone()), + ); + let star_src = at("at://did:plc:nel/sh.tangled.feed.star/aaa"); + let issue_src = at("at://did:plc:nel/sh.tangled.repo.issue/bbb"); + store.add(Edge { + kind: nsid("sh.tangled.feed.star"), + subject: SubjectRef::Did(subject.clone()), + source: star_src.clone(), + sort_micros: 1, + }); + store.add(Edge { + kind: nsid("sh.tangled.feed.star"), + subject: SubjectRef::Did(subject.clone()), + source: issue_src.clone(), + sort_micros: 2, + }); + + let page = store.list(&key, PageCursor::Start, limit(10), SortDir::Asc); + let got: std::collections::HashSet = + page.items.iter().map(|u| u.as_ref().to_owned()).collect(); + assert!( + got.contains(star_src.as_ref()), + "star-collection source decodes exactly" + ); + assert!( + got.contains(issue_src.as_ref()), + "issue-collection source must keep its own collection, not borrow the star one" + ); + } + + #[test] + fn author_refs_spill_preserves_distinct_count() { + let store = store(); + let subject = did("did:plc:scallop"); + let key = EdgeKey::new( + nsid("sh.tangled.feed.star"), + SubjectRef::Did(subject.clone()), + ); + (0..5).for_each(|i| { + store.add(star_edge_at( + at(&format!( + "at://did:plc:{}/sh.tangled.feed.star/r{i}", + NAMES[i] + )), + subject.clone(), + i as u64, + )); + }); + assert_eq!( + store.count_distinct_authors(&key), + 5, + "five authors exceed the inline cap and spill to the map" + ); + + store.add(star_edge_at( + at("at://did:plc:nel/sh.tangled.feed.star/r99"), + subject.clone(), + 99, + )); + assert_eq!( + store.count_distinct_authors(&key), + 5, + "second nel source adds no author" + ); + + store.remove_source(&at("at://did:plc:nel/sh.tangled.feed.star/r0")); + assert_eq!( + store.count_distinct_authors(&key), + 5, + "nel still present via r99" + ); + store.remove_source(&at("at://did:plc:nel/sh.tangled.feed.star/r99")); + assert_eq!(store.count_distinct_authors(&key), 4, "nel fully removed"); + } +} diff --git a/bobbin/crates/edge-index/src/state_index.rs b/bobbin/crates/edge-index/src/state_index.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/edge-index/src/state_index.rs @@ -0,0 +1,453 @@ +use std::collections::BTreeMap; +use std::sync::Mutex; + +use bobbin_runtime::RuntimeHasher; +use bobbin_types::edges::Record; +use bobbin_types::sh_tangled::repo::issue::state::StateState; +use bobbin_types::sh_tangled::repo::pull::status::StatusStatus; +use jacquard_common::DefaultStr; +use jacquard_common::types::string::AtUri; +use scc::HashMap as SccMap; +use scc::hash_map::Entry; + +pub trait StateKind: Copy + Eq + std::fmt::Debug + Send + Sync + 'static { + fn wire(self) -> &'static str; +} + +#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] +pub enum IssueStateKind { + #[default] + Open, + Closed, +} + +impl StateKind for IssueStateKind { + fn wire(self) -> &'static str { + match self { + Self::Open => "open", + Self::Closed => "closed", + } + } +} + +#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] +pub enum PullStatusKind { + #[default] + Open, + Closed, + Merged, +} + +impl StateKind for PullStatusKind { + fn wire(self) -> &'static str { + match self { + Self::Open => "open", + Self::Closed => "closed", + Self::Merged => "merged", + } + } +} + +#[derive(Clone, Debug)] +struct ReverseRef { + entity: AtUri, + sort_micros: u64, + kind: V, +} + +#[derive(Clone, Debug)] +struct SortableSource(AtUri); + +impl PartialEq for SortableSource { + fn eq(&self, other: &Self) -> bool { + self.0.as_ref() == other.0.as_ref() + } +} + +impl Eq for SortableSource {} + +impl PartialOrd for SortableSource { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for SortableSource { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.0.as_ref().cmp(other.0.as_ref()) + } +} + +type ForwardKey = (u64, SortableSource); + +pub struct StateIndex { + forward: SccMap, BTreeMap, RuntimeHasher>, + reverse: SccMap, ReverseRef, RuntimeHasher>, + writer: Mutex<()>, +} + +impl StateIndex { + pub fn new(hasher: RuntimeHasher) -> Self { + Self { + forward: SccMap::with_hasher(hasher.clone()), + reverse: SccMap::with_hasher(hasher), + writer: Mutex::new(()), + } + } + + pub fn upsert( + &self, + source: AtUri, + entity: AtUri, + sort_micros: u64, + kind: V, + ) { + let _w = self + .writer + .lock() + .expect("state-index writer mutex poisoned"); + let rev = self.reverse.entry_sync(source.clone()); + let prior = match &rev { + Entry::Occupied(occ) => Some((occ.get().entity.clone(), occ.get().sort_micros)), + Entry::Vacant(_) => None, + }; + if let Some((prior_entity, prior_micros)) = prior.as_ref() + && (prior_entity != &entity || *prior_micros != sort_micros) + { + self.remove_forward(prior_entity, *prior_micros, &source); + } + self.insert_forward(entity.clone(), sort_micros, kind, source); + match rev { + Entry::Occupied(mut occ) => { + let slot = occ.get_mut(); + slot.entity = entity; + slot.sort_micros = sort_micros; + slot.kind = kind; + } + Entry::Vacant(vac) => { + vac.insert_entry(ReverseRef { + entity, + sort_micros, + kind, + }); + } + } + } + + pub fn remove_source(&self, source: &AtUri) { + let _w = self + .writer + .lock() + .expect("state-index writer mutex poisoned"); + let Entry::Occupied(occ) = self.reverse.entry_sync(source.clone()) else { + return; + }; + let entity = occ.get().entity.clone(); + let sort_micros = occ.get().sort_micros; + let _ = occ.remove(); + self.remove_forward(&entity, sort_micros, source); + } + + pub fn remove_entity(&self, entity: &AtUri) { + let _w = self + .writer + .lock() + .expect("state-index writer mutex poisoned"); + let Some((_, set)) = self.forward.remove_sync(entity) else { + return; + }; + set.into_iter().for_each(|((_, src), _)| { + let Entry::Occupied(occ) = self.reverse.entry_sync(src.0.clone()) else { + return; + }; + if occ.get().entity == *entity { + let _ = occ.remove(); + } + }); + } + + fn insert_forward( + &self, + entity: AtUri, + sort_micros: u64, + kind: V, + source: AtUri, + ) { + let mut entry = self.forward.entry_sync(entity).or_default(); + entry + .get_mut() + .insert((sort_micros, SortableSource(source)), kind); + } + + fn remove_forward( + &self, + entity: &AtUri, + sort_micros: u64, + source: &AtUri, + ) { + let Entry::Occupied(mut entry) = self.forward.entry_sync(entity.clone()) else { + return; + }; + entry + .get_mut() + .remove(&(sort_micros, SortableSource(source.clone()))); + if entry.get().is_empty() { + let _ = entry.remove(); + } + } + + pub fn latest(&self, entity: &AtUri) -> Option<(V, u64)> { + self.latest_by(entity, |_| true) + } + + pub fn latest_by(&self, entity: &AtUri, accept: F) -> Option<(V, u64)> + where + F: Fn(&AtUri) -> bool, + { + self.forward + .read_sync(entity, |_, set| { + set.iter() + .rev() + .find_map(|((m, src), k)| accept(&src.0).then_some((*k, *m))) + }) + .flatten() + } + + pub fn entity_count(&self) -> usize { + self.forward.len() + } + + pub fn source_count(&self) -> usize { + self.reverse.len() + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ApplyOutcome { + Applied, + UnknownVariant, + NotStateRecord, + Removed, +} + +fn issue_kind_from(s: &StateState) -> Option { + match s { + StateState::ShTangledRepoIssueStateOpen => Some(IssueStateKind::Open), + StateState::ShTangledRepoIssueStateClosed => Some(IssueStateKind::Closed), + StateState::Other(_) => None, + } +} + +fn pull_kind_from(s: &StatusStatus) -> Option { + match s { + StatusStatus::ShTangledRepoPullStatusOpen => Some(PullStatusKind::Open), + StatusStatus::ShTangledRepoPullStatusClosed => Some(PullStatusKind::Closed), + StatusStatus::ShTangledRepoPullStatusMerged => Some(PullStatusKind::Merged), + StatusStatus::Other(_) => None, + } +} + +pub fn apply_record_state( + issue_idx: &StateIndex, + pull_idx: &StateIndex, + source: &AtUri, + record: &Record, +) -> ApplyOutcome { + let sort_micros = record.sort_micros_for(source); + match record { + Record::IssueState(r) => match issue_kind_from(&r.state) { + Some(kind) => { + issue_idx.upsert(source.clone(), r.issue.clone(), sort_micros, kind); + ApplyOutcome::Applied + } + None => ApplyOutcome::UnknownVariant, + }, + Record::PullStatus(r) => match pull_kind_from(&r.status) { + Some(kind) => { + pull_idx.upsert(source.clone(), r.pull.clone(), sort_micros, kind); + ApplyOutcome::Applied + } + None => ApplyOutcome::UnknownVariant, + }, + _ => ApplyOutcome::NotStateRecord, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn idx() -> StateIndex { + StateIndex::new(RuntimeHasher::default()) + } + + fn at(s: &str) -> AtUri { + AtUri::new_owned(s).unwrap() + } + + #[test] + fn latest_returns_largest_sort_micros() { + let s = idx(); + let issue = at("at://did:plc:limpet/sh.tangled.repo.issue/i1"); + s.upsert( + at("at://did:plc:nel/sh.tangled.repo.issue.state/s1"), + issue.clone(), + 100, + IssueStateKind::Open, + ); + s.upsert( + at("at://did:plc:nel/sh.tangled.repo.issue.state/s2"), + issue.clone(), + 200, + IssueStateKind::Closed, + ); + assert_eq!(s.latest(&issue), Some((IssueStateKind::Closed, 200))); + } + + #[test] + fn remove_source_drops_entry_and_recovers_prior() { + let s = idx(); + let issue = at("at://did:plc:limpet/sh.tangled.repo.issue/i1"); + let s1 = at("at://did:plc:nel/sh.tangled.repo.issue.state/s1"); + let s2 = at("at://did:plc:nel/sh.tangled.repo.issue.state/s2"); + s.upsert(s1.clone(), issue.clone(), 100, IssueStateKind::Open); + s.upsert(s2.clone(), issue.clone(), 200, IssueStateKind::Closed); + s.remove_source(&s2); + assert_eq!(s.latest(&issue), Some((IssueStateKind::Open, 100))); + s.remove_source(&s1); + assert!(s.latest(&issue).is_none()); + assert_eq!(s.entity_count(), 0); + assert_eq!(s.source_count(), 0); + } + + #[test] + fn upsert_same_source_replaces() { + let s = idx(); + let issue = at("at://did:plc:limpet/sh.tangled.repo.issue/i1"); + let src = at("at://did:plc:nel/sh.tangled.repo.issue.state/s1"); + s.upsert(src.clone(), issue.clone(), 100, IssueStateKind::Open); + s.upsert(src.clone(), issue.clone(), 100, IssueStateKind::Closed); + assert_eq!(s.latest(&issue), Some((IssueStateKind::Closed, 100))); + assert_eq!(s.source_count(), 1); + } + + #[test] + fn distinct_sources_with_identical_micros_and_kind_survive_individual_removal() { + let s = idx(); + let issue = at("at://did:plc:limpet/sh.tangled.repo.issue/i1"); + let s1 = at("at://did:plc:nel/sh.tangled.repo.issue.state/s1"); + let s2 = at("at://did:plc:olaren/sh.tangled.repo.issue.state/s2"); + s.upsert(s1.clone(), issue.clone(), 100, IssueStateKind::Open); + s.upsert(s2.clone(), issue.clone(), 100, IssueStateKind::Open); + s.remove_source(&s1); + assert_eq!( + s.latest(&issue), + Some((IssueStateKind::Open, 100)), + "removing one source must not wipe the other's matching state" + ); + s.remove_source(&s2); + assert!(s.latest(&issue).is_none()); + } + + #[test] + fn unknown_variant_is_reported() { + use bobbin_types::sh_tangled::repo::issue::state::State as IssueStateRec; + use jacquard_common::deps::smol_str::SmolStr; + let issue_idx = idx(); + let pull_idx = StateIndex::::new(RuntimeHasher::default()); + let rec = Record::IssueState(IssueStateRec { + issue: at("at://did:plc:limpet/sh.tangled.repo.issue/i1"), + state: StateState::Other(SmolStr::new_static("sh.tangled.repo.issue.state.reopened")), + extra_data: None, + }); + let outcome = apply_record_state( + &issue_idx, + &pull_idx, + &at("at://did:plc:nel/sh.tangled.repo.issue.state/s1"), + &rec, + ); + assert_eq!(outcome, ApplyOutcome::UnknownVariant); + assert_eq!(issue_idx.entity_count(), 0); + } + + #[test] + fn latest_by_filters_unauthorized_sources() { + let s = idx(); + let issue = at("at://did:plc:limpet/sh.tangled.repo.issue/i1"); + let owner_state = at("at://did:plc:limpet/sh.tangled.repo.issue.state/legit"); + let attacker_state = at("at://did:plc:nautilus/sh.tangled.repo.issue.state/spoof"); + s.upsert( + owner_state.clone(), + issue.clone(), + 100, + IssueStateKind::Open, + ); + s.upsert( + attacker_state.clone(), + issue.clone(), + 500, + IssueStateKind::Closed, + ); + let only_owner = |src: &AtUri| src.as_ref().starts_with("at://did:plc:limpet/"); + assert_eq!( + s.latest_by(&issue, only_owner), + Some((IssueStateKind::Open, 100)), + "spoofed attacker state must be ignored", + ); + assert_eq!( + s.latest(&issue), + Some((IssueStateKind::Closed, 500)), + "unfiltered latest still surfaces the spoof for sanity", + ); + } + + #[test] + fn remove_entity_drops_forward_and_reverse() { + let s = idx(); + let issue = at("at://did:plc:limpet/sh.tangled.repo.issue/i1"); + let s1 = at("at://did:plc:nel/sh.tangled.repo.issue.state/s1"); + let s2 = at("at://did:plc:olaren/sh.tangled.repo.issue.state/s2"); + s.upsert(s1.clone(), issue.clone(), 100, IssueStateKind::Open); + s.upsert(s2.clone(), issue.clone(), 200, IssueStateKind::Closed); + s.remove_entity(&issue); + assert!(s.latest(&issue).is_none()); + assert_eq!(s.entity_count(), 0); + assert_eq!( + s.source_count(), + 0, + "reverse entries pointing at the dead entity must clear too", + ); + } + + #[test] + fn remove_entity_does_not_touch_reverse_pointing_elsewhere() { + let s = idx(); + let dead = at("at://did:plc:limpet/sh.tangled.repo.issue/dead"); + let alive = at("at://did:plc:limpet/sh.tangled.repo.issue/alive"); + let src = at("at://did:plc:nel/sh.tangled.repo.issue.state/s1"); + s.upsert(src.clone(), dead.clone(), 100, IssueStateKind::Open); + s.upsert(src.clone(), alive.clone(), 200, IssueStateKind::Closed); + s.remove_entity(&dead); + assert_eq!( + s.latest(&alive), + Some((IssueStateKind::Closed, 200)), + "removing the dead entity must not touch state for the live one", + ); + assert_eq!(s.source_count(), 1); + } + + #[test] + fn same_micros_distinct_kind_picks_by_source_not_kind() { + let s = StateIndex::::new(RuntimeHasher::default()); + let pull = at("at://did:plc:limpet/sh.tangled.repo.pull/p1"); + let early = at("at://did:plc:nel/sh.tangled.repo.pull.status/aaa"); + let later = at("at://did:plc:nel/sh.tangled.repo.pull.status/zzz"); + s.upsert(early.clone(), pull.clone(), 1000, PullStatusKind::Merged); + s.upsert(later.clone(), pull.clone(), 1000, PullStatusKind::Open); + assert_eq!( + s.latest(&pull), + Some((PullStatusKind::Open, 1000)), + "tied micros must use source ordering as the tiebreak", + ); + } +} diff --git a/bobbin/crates/edge-index/tests/bucket_scaling.rs b/bobbin/crates/edge-index/tests/bucket_scaling.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/edge-index/tests/bucket_scaling.rs @@ -0,0 +1,231 @@ +use std::cell::Cell; +use std::cmp::Ordering; +use std::collections::BTreeSet; +use std::ops::Bound; + +type Key = (u64, u32); + +fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +fn adversarial_keys(n: u32) -> Vec { + (0..n) + .scan(0xD1CE_5EED_u64, |state, i| Some((splitmix64(state), i))) + .collect() +} + +fn vec_shift_work(keys: &[Key]) -> u128 { + let mut sorted: Vec = Vec::with_capacity(keys.len()); + keys.iter().fold(0u128, |moves, &key| { + match sorted.binary_search(&key) { + Ok(_) => moves, + Err(pos) => { + let displaced = (sorted.len() - pos) as u128; + sorted.insert(pos, key); + moves + displaced + } + } + }) +} + +thread_local! { + static COMPARES: Cell = Cell::new(0); +} + +#[derive(PartialEq, Eq)] +struct CountedKey(Key); + +impl PartialOrd for CountedKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for CountedKey { + fn cmp(&self, other: &Self) -> Ordering { + COMPARES.with(|c| c.set(c.get() + 1)); + self.0.cmp(&other.0) + } +} + +fn btree_compare_work(keys: &[Key]) -> u128 { + COMPARES.with(|c| c.set(0)); + let mut tree: BTreeSet = BTreeSet::new(); + keys.iter().for_each(|&key| { + tree.insert(CountedKey(key)); + }); + COMPARES.with(Cell::get) +} + +fn doubling_ratios(work: &[u128]) -> Vec { + work.windows(2) + .map(|w| w[1] as f64 / w[0] as f64) + .collect() +} + +#[test] +fn sorted_vec_build_is_quadratic_while_btree_is_quasilinear() { + let sizes = [2_048u32, 4_096, 8_192, 16_384]; + let measured: Vec<(u128, u128)> = sizes + .iter() + .map(|&n| { + let keys = adversarial_keys(n); + (vec_shift_work(&keys), btree_compare_work(&keys)) + }) + .collect(); + + let vec_work: Vec = measured.iter().map(|m| m.0).collect(); + let btree_work: Vec = measured.iter().map(|m| m.1).collect(); + + eprintln!("{:<8} {:>16} {:>16}", "N", "vec_shifts", "btree_compares"); + sizes.iter().enumerate().for_each(|(i, n)| { + eprintln!("{:<8} {:>16} {:>16}", n, vec_work[i], btree_work[i]); + }); + + let vec_ratios = doubling_ratios(&vec_work); + let btree_ratios = doubling_ratios(&btree_work); + eprintln!("vec doubling ratios: {vec_ratios:?}"); + eprintln!("btree doubling ratios: {btree_ratios:?}"); + + vec_ratios.iter().for_each(|&ratio| { + assert!( + (3.5..=4.5).contains(&ratio), + "sorted-vec build must quadruple per doubling of N, proving O(n^2), got {ratio:.2}x" + ); + }); + + btree_ratios.iter().for_each(|&ratio| { + assert!( + ratio < 3.0, + "btree build must stay near a doubling per doubling of N, proving sub-quadratic, got {ratio:.2}x" + ); + }); + + let advantage: Vec = (0..sizes.len()) + .map(|i| vec_work[i] as f64 / btree_work[i] as f64) + .collect(); + advantage.windows(2).for_each(|w| { + assert!( + w[1] > w[0], + "btree's advantage over the sorted vec must widen as N grows, got {advantage:?}" + ); + }); +} + +fn vec_remove_work(keys: &[Key]) -> u128 { + let mut sorted: Vec = keys.to_vec(); + sorted.sort_unstable(); + keys.iter().fold(0u128, |moves, key| { + match sorted.binary_search(key) { + Ok(pos) => { + let displaced = (sorted.len() - pos - 1) as u128; + sorted.remove(pos); + moves + displaced + } + Err(_) => moves, + } + }) +} + +fn btree_remove_work(keys: &[Key]) -> u128 { + let mut tree: BTreeSet = keys.iter().map(|&k| CountedKey(k)).collect(); + COMPARES.with(|c| c.set(0)); + keys.iter().for_each(|&k| { + tree.remove(&CountedKey(k)); + }); + COMPARES.with(Cell::get) +} + +fn sample_cursors(keys: &[Key]) -> Vec { + let mut sorted = keys.to_vec(); + sorted.sort_unstable(); + let step = (sorted.len() / 256).max(1); + sorted.iter().step_by(step).copied().collect() +} + +fn vec_seek_work(keys: &[Key], cursors: &[Key]) -> u128 { + let mut sorted: Vec = keys.iter().map(|&k| CountedKey(k)).collect(); + sorted.sort_unstable(); + COMPARES.with(|c| c.set(0)); + cursors.iter().for_each(|&cur| { + let _ = sorted.partition_point(|k| *k <= CountedKey(cur)); + }); + COMPARES.with(Cell::get) +} + +fn btree_seek_work(keys: &[Key], cursors: &[Key]) -> u128 { + let tree: BTreeSet = keys.iter().map(|&k| CountedKey(k)).collect(); + COMPARES.with(|c| c.set(0)); + cursors.iter().for_each(|&cur| { + let _ = tree + .range((Bound::Excluded(CountedKey(cur)), Bound::Unbounded)) + .next(); + }); + COMPARES.with(Cell::get) +} + +#[test] +fn sorted_vec_teardown_is_quadratic_while_btree_is_quasilinear() { + let sizes = [2_048u32, 4_096, 8_192, 16_384]; + let measured: Vec<(u128, u128)> = sizes + .iter() + .map(|&n| { + let keys = adversarial_keys(n); + (vec_remove_work(&keys), btree_remove_work(&keys)) + }) + .collect(); + + let vec_ratios = doubling_ratios(&measured.iter().map(|m| m.0).collect::>()); + let btree_ratios = doubling_ratios(&measured.iter().map(|m| m.1).collect::>()); + eprintln!("teardown vec doubling ratios: {vec_ratios:?}"); + eprintln!("teardown btree doubling ratios: {btree_ratios:?}"); + + vec_ratios.iter().for_each(|&ratio| { + assert!( + (3.5..=4.5).contains(&ratio), + "sorted-vec teardown must quadruple per doubling, proving O(n^2), got {ratio:.2}x" + ); + }); + btree_ratios.iter().for_each(|&ratio| { + assert!( + ratio < 3.0, + "btree teardown must stay sub-quadratic, got {ratio:.2}x" + ); + }); +} + +#[test] +fn cursor_seek_is_sublinear_for_both_representations() { + let sizes = [2_048u32, 4_096, 8_192, 16_384]; + let measured: Vec<(u128, u128)> = sizes + .iter() + .map(|&n| { + let keys = adversarial_keys(n); + let cursors = sample_cursors(&keys); + ( + vec_seek_work(&keys, &cursors), + btree_seek_work(&keys, &cursors), + ) + }) + .collect(); + + let vec_ratios = doubling_ratios(&measured.iter().map(|m| m.0).collect::>()); + let btree_ratios = doubling_ratios(&measured.iter().map(|m| m.1).collect::>()); + eprintln!("seek vec doubling ratios: {vec_ratios:?}"); + eprintln!("seek btree doubling ratios: {btree_ratios:?}"); + + vec_ratios + .iter() + .chain(btree_ratios.iter()) + .for_each(|&ratio| { + assert!( + ratio < 1.8, + "a fixed cursor sample must seek in logarithmic work as N grows, got {ratio:.2}x" + ); + }); +} diff --git a/bobbin/crates/ingest/Cargo.toml b/bobbin/crates/ingest/Cargo.toml new file mode 100644 --- /dev/null +++ b/bobbin/crates/ingest/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "bobbin-ingest" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-types = { workspace = true } +bobbin-edge-index = { workspace = true } +bobbin-record-lru = { workspace = true } +bobbin-resolver = { workspace = true } +bobbin-runtime = { workspace = true } +bobbin-slingshot-client = { workspace = true } +jacquard-common = { workspace = true } + +bytes = { workspace = true } +chrono = { workspace = true } +futures = { workspace = true } +scc = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +tokio-util = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +tracing-subscriber = { workspace = true } +tokio-util = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } +wiremock = { workspace = true } +criterion = { version = "0.8", default-features = false, features = ["cargo_bench_support"] } + +[[bench]] +name = "json_decode" +harness = false diff --git a/bobbin/crates/ingest/benches/json_decode.rs b/bobbin/crates/ingest/benches/json_decode.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/ingest/benches/json_decode.rs @@ -0,0 +1,245 @@ +use std::hint::black_box; + +use bobbin_types::edges::Record; +use criterion::{Criterion, criterion_group, criterion_main}; +use jacquard_common::DefaultStr; +use jacquard_common::types::nsid::Nsid; +use serde::Deserialize; +use serde_json::value::RawValue; + +#[derive(Deserialize)] +struct ValueFrame { + record: ValueRecordFrame, +} + +#[derive(Deserialize)] +struct ValueRecordFrame { + collection: Nsid, + #[serde(default)] + record: Option, +} + +#[derive(Deserialize)] +struct BorrowedRawFrame<'a> { + #[serde(borrow)] + record: BorrowedRawRecordFrame<'a>, +} + +#[derive(Deserialize)] +struct BorrowedRawRecordFrame<'a> { + collection: Nsid, + #[serde(borrow, default)] + record: Option<&'a RawValue>, +} + +#[derive(Deserialize)] +struct OwnedRawFrame { + record: OwnedRawRecordFrame, +} + +#[derive(Deserialize)] +struct OwnedRawRecordFrame { + collection: Nsid, + #[serde(default)] + record: Option>, +} + +fn current_path(text: &str) -> Record { + let frame: ValueFrame = serde_json::from_str(text).expect("frame"); + let value = frame.record.record.expect("upsert body"); + let bytes = serde_json::to_vec(&value).expect("re-serialize Value"); + Record::from_json_bytes(&frame.record.collection, &bytes).expect("decode record") +} + +fn raw_value_borrowed(text: &str) -> Record { + let frame: BorrowedRawFrame<'_> = serde_json::from_str(text).expect("frame"); + let raw = frame.record.record.expect("upsert body"); + Record::from_json_bytes(&frame.record.collection, raw.get().as_bytes()).expect("decode record") +} + +fn raw_value_owned(text: &str) -> Record { + let frame: OwnedRawFrame = serde_json::from_str(text).expect("frame"); + let raw = frame.record.record.expect("upsert body"); + Record::from_json_bytes(&frame.record.collection, raw.get().as_bytes()).expect("decode record") +} + +fn floor_record_only(collection: &Nsid, record_bytes: &[u8]) -> Record { + Record::from_json_bytes(collection, record_bytes).expect("decode record") +} + +fn star_text() -> &'static str { + r#"{ + "id": 1, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": "3lq2zk5wqsh2k", + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subject": "at://did:plc:abalone/sh.tangled.repo/3lq2zk5wq0000" + } + } + }"# +} + +fn repo_text() -> &'static str { + r#"{ + "id": 2, + "type": "record", + "record": { + "live": false, + "did": "did:plc:nel", + "rev": "3lq2zk5wqsh2l", + "collection": "sh.tangled.repo", + "rkey": "3lq2zk5wq0001", + "action": "create", + "record": { + "$type": "sh.tangled.repo", + "createdAt": "2026-05-01T00:00:00Z", + "knot": "knot.witchcraft.systems", + "name": "limpet", + "description": "demonstration repository for the bench corpus", + "owner": "did:plc:nel", + "repoDid": "did:plc:limpet", + "labels": [ + "at://did:plc:periwinkle/sh.tangled.label.definition/3lq2zk5wq0010", + "at://did:plc:periwinkle/sh.tangled.label.definition/3lq2zk5wq0011" + ] + } + } + }"# +} + +fn issue_text() -> &'static str { + r#"{ + "id": 3, + "type": "record", + "record": { + "live": false, + "did": "did:plc:teq", + "rev": "3lq2zk5wqsh2m", + "collection": "sh.tangled.repo.issue", + "rkey": "3lq2zk5wq0100", + "action": "create", + "record": { + "$type": "sh.tangled.repo.issue", + "createdAt": "2026-05-01T00:00:00Z", + "title": "ingest: single-pass JSON decode follow-up corpus entry", + "body": "Long body to give the bench a realistic decode cost. Repeats: blahhhhh meow meow aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "repo": "at://did:plc:limpet", + "repoDid": "did:plc:limpet", + "mentions": [ + "did:plc:nel", + "did:plc:olaren", + "did:plc:bailey" + ], + "references": [ + "at://did:plc:limpet/sh.tangled.repo.issue/3lq2zk5wq0099", + "at://did:plc:limpet/sh.tangled.repo.pull/3lq2zk5wq0098" + ] + } + } + }"# +} + +fn pull_comment_text() -> &'static str { + r#"{ + "id": 4, + "type": "record", + "record": { + "live": false, + "did": "did:plc:lyna", + "rev": "3lq2zk5wqsh2n", + "collection": "sh.tangled.repo.pull.comment", + "rkey": "3lq2zk5wq0200", + "action": "create", + "record": { + "$type": "sh.tangled.repo.pull.comment", + "createdAt": "2026-05-01T00:00:00Z", + "body": "lgtm i thinks!!!! but please verify the cursor invariant under buffered(N) before landing. :3", + "pull": "at://did:plc:limpet/sh.tangled.repo.pull/3lq2zk5wq0098", + "owner": "did:plc:lyna" + } + } + }"# +} + +fn follow_text() -> &'static str { + r#"{ + "id": 5, + "type": "record", + "record": { + "live": false, + "did": "did:plc:bailey", + "rev": "3lq2zk5wqsh2o", + "collection": "sh.tangled.graph.follow", + "rkey": "3lq2zk5wq0300", + "action": "create", + "record": { + "$type": "sh.tangled.graph.follow", + "createdAt": "2026-05-01T00:00:00Z", + "subject": "did:plc:nel" + } + } + }"# +} + +fn extract_record_slice(text: &str) -> (Nsid, Vec) { + let frame: ValueFrame = serde_json::from_str(text).expect("frame"); + let value = frame.record.record.expect("upsert body"); + let bytes = serde_json::to_vec(&value).expect("re-serialize Value"); + (frame.record.collection, bytes) +} + +fn bench_decode(c: &mut Criterion) { + let corpus: &[(&str, &str)] = &[ + ("star", star_text()), + ("repo", repo_text()), + ("issue", issue_text()), + ("pull_comment", pull_comment_text()), + ("follow", follow_text()), + ]; + for (name, text) in corpus { + let mut group = c.benchmark_group(format!("decode/{name}")); + let (collection, record_bytes) = extract_record_slice(text); + + group.bench_function("current_path", |b| { + b.iter(|| { + let r = current_path(black_box(text)); + black_box(r); + }); + }); + + group.bench_function("raw_value_borrowed", |b| { + b.iter(|| { + let r = raw_value_borrowed(black_box(text)); + black_box(r); + }); + }); + + group.bench_function("raw_value_owned", |b| { + b.iter(|| { + let r = raw_value_owned(black_box(text)); + black_box(r); + }); + }); + + group.bench_function("floor_record_only", |b| { + let bytes = record_bytes.as_slice(); + b.iter(|| { + let r = floor_record_only(black_box(&collection), black_box(bytes)); + black_box(r); + }); + }); + + group.finish(); + } +} + +criterion_group!(benches, bench_decode); +criterion_main!(benches); diff --git a/bobbin/crates/ingest/examples/smoke.rs b/bobbin/crates/ingest/examples/smoke.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/ingest/examples/smoke.rs @@ -0,0 +1,85 @@ +use std::sync::Arc; +use std::time::Duration; + +use bobbin_edge_index::{CoverageWatch, EdgeStore, StateIndex}; +use bobbin_ingest::{IngestConfig, IngestRuntime, RepoIdResolver, run}; +use bobbin_record_lru::{NoopRecordStore, RecordStore}; +use bobbin_runtime::{OsEntropy, RuntimeHasher, SystemClock, TungsteniteWs}; +use bobbin_types::search::NoopSearchSink; +use futures::stream::{self, StreamExt}; +use tokio_util::sync::CancellationToken; +use url::Url; + +const MIN_EVENTS_PER_RUN: u64 = 100; + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + tracing_subscriber::fmt::try_init().ok(); + + let endpoint = std::env::args() + .nth(1) + .unwrap_or_else(|| "ws://127.0.0.1:3010".to_owned()); + let ticks: u32 = std::env::args() + .nth(2) + .and_then(|s| s.parse().ok()) + .unwrap_or(6); + + let url = Url::parse(&endpoint).expect("valid hydrant base url"); + let hasher = RuntimeHasher::from_entropy(&OsEntropy); + let store = Arc::new(EdgeStore::new(hasher.clone())); + let coverage = Arc::new(CoverageWatch::new()); + + let cfg = IngestConfig::new(url); + let cancel = CancellationToken::new(); + let runtime = IngestRuntime { + store: store.clone(), + issue_states: Arc::new(StateIndex::new(hasher.clone())), + pull_statuses: Arc::new(StateIndex::new(hasher.clone())), + coverage: coverage.clone(), + search: Arc::new(NoopSearchSink), + records: Arc::new(NoopRecordStore) as Arc, + resolver: Arc::new(RepoIdResolver::detached(hasher)), + clock: Arc::new(SystemClock::new()), + entropy: Arc::new(OsEntropy), + ws: TungsteniteWs::shared(), + cancel: cancel.clone(), + disconnects: None, + warming_shadow: None, + warming_buffer: None, + }; + let task = tokio::spawn(async move { + let _ = run(cfg, runtime).await; + }); + + stream::iter(0..ticks) + .for_each(|i| { + let store = store.clone(); + let coverage = coverage.clone(); + async move { + tokio::time::sleep(Duration::from_secs(5)).await; + let snap = coverage.snapshot(); + println!( + "[{i:02}] coverage={snap:?} edge_keys={} sources={}", + store.key_count(), + store.source_count() + ); + } + }) + .await; + + cancel.cancel(); + let _ = task.await; + + let final_snap = coverage.snapshot(); + let events = final_snap.events_processed(); + println!( + "done: events={events} edge_keys={} sources={} ready={}", + store.key_count(), + store.source_count(), + final_snap.is_ready(), + ); + assert!( + events >= MIN_EVENTS_PER_RUN, + "smoke received {events} events from {endpoint} across {ticks} five-second ticks, below threshold of {MIN_EVENTS_PER_RUN}; hydrant may be silent or unreachable", + ); +} diff --git a/bobbin/crates/ingest/src/frame.rs b/bobbin/crates/ingest/src/frame.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/ingest/src/frame.rs @@ -0,0 +1,84 @@ +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::recordkey::Rkey; +use jacquard_common::types::string::Cid; +use jacquard_common::types::tid::Tid; +use serde::Deserialize; +use serde_json::value::RawValue; + +#[derive(Debug, Deserialize)] +pub struct HydrantFrame { + pub id: u64, + #[serde(rename = "type")] + pub kind: FrameKind, + #[serde(default)] + pub record: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FrameKind { + Record, + Identity, + Account, + Other, +} + +impl<'de> Deserialize<'de> for FrameKind { + fn deserialize(d: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = >::deserialize(d)?; + Ok(match s.as_ref() { + "record" => Self::Record, + "identity" => Self::Identity, + "account" => Self::Account, + _ => Self::Other, + }) + } +} + +#[derive(Clone, Debug, Deserialize)] +pub struct HydrantStreamErrorFrame { + pub error: String, + #[serde(default)] + pub message: Option, +} + +#[derive(Debug, Deserialize)] +pub struct RecordFrame { + pub live: bool, + pub did: Did, + pub rev: Tid, + pub collection: Nsid, + pub rkey: Rkey, + pub action: RecordAction, + #[serde(default)] + pub record: Option>, + #[serde(default)] + pub cid: Option>, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecordAction { + Create, + Update, + Delete, + Other, +} + +impl<'de> Deserialize<'de> for RecordAction { + fn deserialize(d: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = >::deserialize(d)?; + Ok(match s.as_ref() { + "create" => Self::Create, + "update" => Self::Update, + "delete" => Self::Delete, + _ => Self::Other, + }) + } +} diff --git a/bobbin/crates/ingest/src/lib.rs b/bobbin/crates/ingest/src/lib.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/ingest/src/lib.rs @@ -0,0 +1,3112 @@ +use std::collections::{HashSet, VecDeque}; +use std::num::NonZeroUsize; +use std::sync::Arc; +use std::time::Duration; + +use bobbin_edge_index::{ + ApplyOutcome, Coverage, CoverageWatch, EdgeStore, HydrantCursor, IssueStateKind, + PromotionSignal, PullStatusKind, StateIndex, apply_record_state, +}; +use bobbin_record_lru::RecordStore; +use bobbin_resolver::{NormalizeRepoRefs, decode_canon_or_upgrade_bytes, synthesize_created_at}; +use bobbin_runtime::{ + Clock, Entropy, NetworkError, RuntimeHasher, UnixMicros, WsConn, WsMessage, WsStream, + WsTransport, +}; +use bobbin_types::edges::{Edge, ExtractError, Record}; +use bobbin_types::ids::{RepoIdent, SubjectRef}; +use bobbin_types::record::RecordBody; +use bobbin_types::search::{SearchSink, SearchableRecord}; +use bytes::Bytes; +use futures::StreamExt; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::ident::AtIdentifier; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::recordkey::Rkey; +use jacquard_common::types::string::{AtStrError, AtUri, Cid}; +use thiserror::Error; +use tokio::time::Instant; +use tokio_stream::wrappers::ReceiverStream; +use tokio_util::sync::CancellationToken; +use tracing::{debug, info, warn}; +use url::Url; + +mod frame; +mod resolver; +mod shadow; +mod warming; +use frame::HydrantStreamErrorFrame; +pub use frame::{FrameKind, HydrantFrame, RecordAction, RecordFrame}; +pub use resolver::{RepoIdResolver, Resolution}; +pub use shadow::{WarmingShadowBuffer, WarmingShadowSnapshot}; +pub use warming::{ParkedUpsert, WarmingBuffer, WarmingBufferSnapshot}; + +const TANGLED_PREFIX: &str = "sh.tangled."; +const RECONNECT_INITIAL_DELAY: Duration = Duration::from_millis(500); +const RECONNECT_MAX_DELAY: Duration = Duration::from_secs(30); +const PING_INTERVAL: Duration = Duration::from_secs(20); +const PONG_TIMEOUT: Duration = Duration::from_secs(15); +const READY_SKEW: Duration = Duration::from_secs(60); +const FRAME_CHANNEL_DEPTH: usize = 256; +const CONTROL_CHANNEL_DEPTH: usize = 16; +const READER_HOLD_LIMIT: usize = 64; +const SEND_TIMEOUT: Duration = Duration::from_secs(10); +const NORMAL_CLOSE: u16 = 1000; +const METRICS_DUMP_INTERVAL: Duration = Duration::from_secs(10); +const WARMING_FLUSH_PARALLELISM: usize = 64; +pub const DEFAULT_INGEST_PARALLELISM: NonZeroUsize = match NonZeroUsize::new(16) { + Some(n) => n, + None => unreachable!(), +}; + +#[derive(Clone, Debug)] +pub struct IngestConfig { + pub hydrant_base: Url, + pub start_cursor: HydrantCursor, + pub parallelism: NonZeroUsize, +} + +impl IngestConfig { + pub fn new(hydrant_base: Url) -> Self { + Self { + hydrant_base, + start_cursor: HydrantCursor::new(0), + parallelism: DEFAULT_INGEST_PARALLELISM, + } + } + + fn stream_url(&self, cursor: HydrantCursor) -> Result { + let mut url = self.hydrant_base.clone(); + match url.scheme() { + "http" => url + .set_scheme("ws") + .map_err(|_| IngestError::Url("set ws scheme"))?, + "https" => url + .set_scheme("wss") + .map_err(|_| IngestError::Url("set wss scheme"))?, + "ws" | "wss" => {} + other => return Err(IngestError::UnknownScheme(other.to_owned())), + } + url.set_path("/stream"); + url.query_pairs_mut() + .clear() + .append_pair("cursor", &cursor.raw().to_string()); + Ok(url) + } +} + +#[derive(Debug, Error)] +pub enum IngestError { + #[error("invalid hydrant url: {0}")] + Url(&'static str), + #[error("unsupported url scheme: {0}")] + UnknownScheme(String), + #[error("network: {0}")] + Network(#[from] NetworkError), + #[error("frame decode: {0}")] + Decode(#[from] serde_json::Error), + #[error("invalid at-uri synthesized from frame: {0}")] + InvalidAtUri(#[from] AtStrError), + #[error("record extraction: {0}")] + Extract(#[from] ExtractError), + #[error("hydrant did not respond to ping within {0:?}")] + PongTimeout(Duration), + #[error("websocket send blocked for at least {0:?}, treating link as dead")] + SendTimeout(Duration), + #[error("hydrant disconnected because bobbin's stream consumer fell behind: {message}")] + ConsumerTooSlow { message: String }, + #[error("hydrant signaled stream error {code}: {message}")] + HydrantStream { code: String, message: String }, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum DisconnectKind { + Url, + UnknownScheme, + Network, + Decode, + InvalidAtUri, + Extract, + PongTimeout, + SendTimeout, + ConsumerTooSlow, + HydrantStream, +} + +impl DisconnectKind { + pub fn from_error(err: &IngestError) -> Self { + match err { + IngestError::Url(_) => Self::Url, + IngestError::UnknownScheme(_) => Self::UnknownScheme, + IngestError::Network(_) => Self::Network, + IngestError::Decode(_) => Self::Decode, + IngestError::InvalidAtUri(_) => Self::InvalidAtUri, + IngestError::Extract(_) => Self::Extract, + IngestError::PongTimeout(_) => Self::PongTimeout, + IngestError::SendTimeout(_) => Self::SendTimeout, + IngestError::ConsumerTooSlow { .. } => Self::ConsumerTooSlow, + IngestError::HydrantStream { .. } => Self::HydrantStream, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DisconnectSnapshot { + pub kind: DisconnectKind, + pub message: String, + pub at_unix_micros: UnixMicros, + pub last_cursor: HydrantCursor, +} + +#[derive(Default)] +pub struct DisconnectSink { + last: std::sync::Mutex>, + count: std::sync::atomic::AtomicU64, +} + +impl DisconnectSink { + pub fn new() -> Self { + Self::default() + } + + pub fn record(&self, snap: DisconnectSnapshot) { + *self.last.lock().expect("disconnect sink mutex poisoned") = Some(snap); + self.count + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + + pub fn snapshot(&self) -> Option { + self.last + .lock() + .expect("disconnect sink mutex poisoned") + .clone() + } + + pub fn count(&self) -> u64 { + self.count.load(std::sync::atomic::Ordering::Relaxed) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum SessionOutcome { + Progressed, + Empty, +} + +#[derive(Debug)] +struct SessionEnd { + outcome: SessionOutcome, + error: Option, +} + +pub struct IngestRuntime { + pub store: Arc, + pub issue_states: Arc>, + pub pull_statuses: Arc>, + pub coverage: Arc, + pub search: Arc, + pub records: Arc, + pub resolver: Arc, + pub clock: Arc, + pub entropy: Arc, + pub ws: Arc, + pub cancel: CancellationToken, + pub disconnects: Option>, + pub warming_shadow: Option>, + pub warming_buffer: Option>, +} + +impl Clone for IngestRuntime { + fn clone(&self) -> Self { + Self { + store: self.store.clone(), + issue_states: self.issue_states.clone(), + pull_statuses: self.pull_statuses.clone(), + coverage: self.coverage.clone(), + search: self.search.clone(), + records: self.records.clone(), + resolver: self.resolver.clone(), + clock: self.clock.clone(), + entropy: self.entropy.clone(), + ws: self.ws.clone(), + cancel: self.cancel.clone(), + disconnects: self.disconnects.clone(), + warming_shadow: self.warming_shadow.clone(), + warming_buffer: self.warming_buffer.clone(), + } + } +} + +impl IngestRuntime { + fn pipeline_ctx(&self) -> PipelineCtx<'_, S> { + PipelineCtx { + resolver: &self.resolver, + store: &self.store, + issue_states: &self.issue_states, + pull_statuses: &self.pull_statuses, + coverage: &self.coverage, + records: &*self.records, + search: &self.search, + shadow: self.warming_shadow.as_deref(), + buffer: self.warming_buffer.as_deref(), + } + } +} + +struct PipelineCtx<'a, S: SearchSink + 'static> { + resolver: &'a RepoIdResolver, + store: &'a EdgeStore, + issue_states: &'a StateIndex, + pull_statuses: &'a StateIndex, + coverage: &'a CoverageWatch, + records: &'a dyn RecordStore, + search: &'a S, + shadow: Option<&'a WarmingShadowBuffer>, + buffer: Option<&'a WarmingBuffer>, +} + +pub async fn run( + config: IngestConfig, + runtime: IngestRuntime, +) -> Result<(), IngestError> { + let metrics_dumper = spawn_metrics_dumper(&runtime); + let idle_promoter = spawn_idle_promoter(&runtime); + let warming_flusher = spawn_warming_flusher(&runtime); + let result = run_inner(config, &runtime).await; + if let Err(join) = metrics_dumper.await { + warn!(?join, "metrics dumper task panicked"); + } + if let Err(join) = idle_promoter.await { + warn!(?join, "idle promoter task panicked"); + } + if let Some(handle) = warming_flusher + && let Err(join) = handle.await + { + warn!(?join, "warming flusher task panicked"); + } + result +} + +async fn run_inner( + config: IngestConfig, + runtime: &IngestRuntime, +) -> Result<(), IngestError> { + let mut backoff = RECONNECT_INITIAL_DELAY; + loop { + let cursor = next_connect_cursor(runtime.coverage.snapshot(), config.start_cursor); + let SessionEnd { outcome, error } = run_session(&config, cursor, runtime).await; + if runtime.cancel.is_cancelled() { + info!( + last_cursor = runtime.coverage.snapshot().last_cursor().raw(), + "ingest stopped after shutdown signal" + ); + return Ok(()); + } + match (outcome, &error) { + (SessionOutcome::Progressed, None) => { + info!("hydrant stream closed after delivering frames, reconnecting") + } + (SessionOutcome::Empty, None) => { + warn!("hydrant stream closed without delivering frames") + } + (_, Some(err)) => warn!(?err, "hydrant stream errored"), + } + if let (Some(sink), Some(err)) = (runtime.disconnects.as_ref(), error.as_ref()) { + sink.record(DisconnectSnapshot { + kind: DisconnectKind::from_error(err), + message: err.to_string(), + at_unix_micros: runtime.clock.now_unix_micros(), + last_cursor: runtime.coverage.snapshot().last_cursor(), + }); + } + let made_progress = matches!(outcome, SessionOutcome::Progressed); + if made_progress { + backoff = RECONNECT_INITIAL_DELAY; + } else { + tokio::select! { + biased; + _ = runtime.cancel.cancelled() => return Ok(()), + _ = runtime.clock.sleep(jittered(backoff, &*runtime.entropy)) => {} + } + backoff = (backoff * 2).min(RECONNECT_MAX_DELAY); + } + } +} + +fn spawn_warming_flusher( + runtime: &IngestRuntime, +) -> Option> { + runtime.warming_buffer.as_ref()?; + let rt = runtime.clone(); + Some(tokio::spawn(async move { + let buffer = rt + .warming_buffer + .as_deref() + .expect("warming flusher only spawns when buffer is set"); + let mut rx = rt.coverage.subscribe(); + let reason = loop { + if rx.borrow_and_update().is_ready() { + break FlushReason::Ready; + } + tokio::select! { + biased; + _ = rt.cancel.cancelled() => break FlushReason::Cancelled, + res = rx.changed() => match res { + Ok(()) => continue, + Err(_) => break FlushReason::CoverageDropped, + }, + } + }; + flush_warming_buffer(&rt, buffer, reason).await; + })) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum FlushReason { + Ready, + Cancelled, + CoverageDropped, +} + +impl FlushReason { + fn as_str(self) -> &'static str { + match self { + FlushReason::Ready => "ready", + FlushReason::Cancelled => "cancelled", + FlushReason::CoverageDropped => "coverage_dropped", + } + } +} + +async fn flush_warming_buffer( + runtime: &IngestRuntime, + buffer: &WarmingBuffer, + reason: FlushReason, +) { + let drained = buffer.drain_for_promote().await; + if drained.is_empty() { + return; + } + if reason != FlushReason::Ready { + info!( + target: "bobbin_ingest::warming", + abandoned_entries = drained.len(), + reason = reason.as_str(), + "abandoning parked items on non-ready flush", + ); + return; + } + let hasher = buffer.hasher().clone(); + let unique: HashSet = drained + .iter() + .flat_map(|(_, deps)| deps.iter().cloned()) + .fold(HashSet::with_hasher(hasher), |mut acc, dep| { + acc.insert(dep); + acc + }); + if !unique.is_empty() { + let resolver = runtime.resolver.clone(); + let _: Vec<()> = futures::stream::iter(unique) + .map(|key| { + let resolver = resolver.clone(); + async move { + let _ = resolver.resolve(&key.owner, &key.rkey).await; + } + }) + .buffer_unordered(WARMING_FLUSH_PARALLELISM) + .collect() + .await; + } + let upserts: Vec = drained.into_iter().map(|(u, _)| u).collect(); + let count = upserts.len(); + let ctx = runtime.pipeline_ctx(); + finalize_drained(&ctx, upserts).await; + info!( + target: "bobbin_ingest::warming", + flushed_entries = count, + reason = reason.as_str(), + "drained warming buffer", + ); +} + +const IDLE_PROMOTE_WINDOW: Duration = Duration::from_secs(15); +const IDLE_PROMOTE_MIN_EVENTS: u64 = 256; + +fn spawn_idle_promoter( + runtime: &IngestRuntime, +) -> tokio::task::JoinHandle<()> { + let rt = runtime.clone(); + tokio::spawn(async move { + let mut prev = rt.coverage.snapshot().events_processed(); + loop { + tokio::select! { + biased; + _ = rt.cancel.cancelled() => return, + _ = rt.clock.sleep(IDLE_PROMOTE_WINDOW) => {} + } + let snap = rt.coverage.snapshot(); + if snap.is_ready() { + return; + } + let processed = snap.events_processed(); + if processed >= IDLE_PROMOTE_MIN_EVENTS && processed == prev { + rt.coverage.update(|c| c.force_ready()); + info!( + target: "bobbin_ingest::coverage", + events_processed = processed, + last_cursor = snap.last_cursor().raw(), + "stream idle, promoting coverage to ready", + ); + return; + } + prev = processed; + } + }) +} + +fn spawn_metrics_dumper( + runtime: &IngestRuntime, +) -> tokio::task::JoinHandle<()> { + let rt = runtime.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + biased; + _ = rt.cancel.cancelled() => break, + _ = rt.clock.sleep(METRICS_DUMP_INTERVAL) => { + let s = rt.resolver.stats(); + info!( + target: "bobbin_ingest::metrics", + resolver_hits = s.hits, + resolver_misses_mapped = s.misses_mapped, + resolver_misses_no_repo_did = s.misses_no_repo_did, + resolver_misses_unresolvable = s.misses_unresolvable, + resolver_misses_transient = s.misses_transient, + resolver_misses_no_client = s.misses_no_client, + resolver_miss_latency_micros_avg = s.miss_latency_micros_avg().unwrap_or(0), + resolver_miss_latency_micros_max = s.miss_latency_micros_max, + resolver_total = s.total(), + "resolver stats", + ); + } + } + } + }) +} + +fn next_connect_cursor(snapshot: Coverage, start: HydrantCursor) -> HydrantCursor { + if snapshot.events_processed() == 0 { + start + } else { + HydrantCursor::new(snapshot.last_cursor().raw().saturating_add(1)) + } +} + +fn jittered(base: Duration, entropy: &dyn Entropy) -> Duration { + let base_ms = u64::try_from(base.as_millis()).unwrap_or(u64::MAX); + let cap_ms = (base_ms / 4).max(1); + base + Duration::from_millis(entropy.next_u64() % cap_ms) +} + +async fn run_session( + config: &IngestConfig, + cursor: HydrantCursor, + runtime: &IngestRuntime, +) -> SessionEnd { + let url = match config.stream_url(cursor) { + Ok(u) => u, + Err(e) => { + return SessionEnd { + outcome: SessionOutcome::Empty, + error: Some(e), + }; + } + }; + info!(%url, "connecting to hydrant /stream"); + let connect = tokio::select! { + biased; + _ = runtime.cancel.cancelled() => { + return SessionEnd { outcome: SessionOutcome::Empty, error: None }; + } + res = runtime.ws.connect(url) => res, + }; + let WsConn { + sink: mut ws_sink, + stream: ws_stream, + } = match connect { + Ok(c) => c, + Err(e) => { + return SessionEnd { + outcome: SessionOutcome::Empty, + error: Some(IngestError::Network(e)), + }; + } + }; + + let (frame_tx, frame_rx) = tokio::sync::mpsc::channel::(FRAME_CHANNEL_DEPTH); + let parallelism = config.parallelism.get(); + let processor_runtime = runtime.clone(); + let processor = tokio::spawn(async move { + let cancel = processor_runtime.cancel.clone(); + let prep_rt = processor_runtime.clone(); + let resolve_rt = processor_runtime.clone(); + let commit_rt = processor_runtime; + + let pipeline = ReceiverStream::new(frame_rx) + .map(move |frame| prep_stage(frame, prep_rt.clone())) + .buffered(parallelism) + .map(move |staged| resolve_stage(staged, resolve_rt.clone())) + .buffered(parallelism) + .for_each(move |staged| commit_stage(staged, commit_rt.clone(), parallelism)); + + tokio::select! { + biased; + _ = cancel.cancelled() => {}, + _ = pipeline => {}, + } + }); + + let (control_tx, mut control_rx) = tokio::sync::mpsc::channel::(CONTROL_CHANNEL_DEPTH); + let session_cancel = runtime.cancel.child_token(); + let reader_cancel = session_cancel.clone(); + let reader = tokio::spawn(reader_loop(ws_stream, frame_tx, control_tx, reader_cancel)); + + let mut next_ping = runtime.clock.now_instant() + PING_INTERVAL; + let mut pong_deadline: Option = None; + + let writer_error: Option = loop { + tokio::select! { + biased; + _ = runtime.cancel.cancelled() => { + let _ = timed_send( + &mut ws_sink, + WsMessage::Close { code: NORMAL_CLOSE, reason: "bobbin shutdown".to_owned() }, + ).await; + break None; + } + _ = runtime.clock.sleep_until(next_ping) => { + next_ping = runtime.clock.now_instant() + PING_INTERVAL; + if pong_deadline.is_none() { + if let Err(e) = timed_send(&mut ws_sink, WsMessage::Ping(Bytes::new())).await { + break Some(e); + } + pong_deadline = Some(runtime.clock.now_instant() + PONG_TIMEOUT); + } + } + _ = wait_until(pong_deadline, runtime.clock.as_ref()) => { + break Some(IngestError::PongTimeout(PONG_TIMEOUT)); + } + evt = control_rx.recv() => { + let Some(evt) = evt else { break None; }; + match evt { + WsEvent::IncomingPing(payload) => { + if let Err(e) = timed_send(&mut ws_sink, WsMessage::Pong(payload)).await { + break Some(e); + } + } + WsEvent::IncomingPong => { + pong_deadline = None; + } + } + } + } + }; + + session_cancel.cancel(); + drop(ws_sink); + drop(control_rx); + + let reader_end = reader.await.unwrap_or(SessionEnd { + outcome: SessionOutcome::Empty, + error: None, + }); + if let Err(join) = processor.await { + warn!(?join, "frame processor task panicked"); + } + SessionEnd { + outcome: reader_end.outcome, + error: writer_error.or(reader_end.error), + } +} + +async fn timed_send( + sink: &mut Box, + msg: WsMessage, +) -> Result<(), IngestError> { + match tokio::time::timeout(SEND_TIMEOUT, sink.send(msg)).await { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err(IngestError::Network(e)), + Err(_elapsed) => Err(IngestError::SendTimeout(SEND_TIMEOUT)), + } +} + +async fn reader_loop( + mut ws_stream: Box, + frame_tx: tokio::sync::mpsc::Sender, + control_tx: tokio::sync::mpsc::Sender, + cancel: CancellationToken, +) -> SessionEnd { + let mut outcome = SessionOutcome::Empty; + let mut held: VecDeque = VecDeque::new(); + + let error: Option = loop { + let step = if held.is_empty() { + tokio::select! { + biased; + _ = cancel.cancelled() => ReaderStep::Cancelled, + msg = ws_stream.next() => ReaderStep::WsMessage(msg), + } + } else if held.len() < READER_HOLD_LIMIT { + tokio::select! { + biased; + _ = cancel.cancelled() => ReaderStep::Cancelled, + permit_result = frame_tx.reserve() => match permit_result { + Ok(permit) => { + let frame = held + .pop_front() + .expect("non-empty held when reserve succeeds"); + permit.send(frame); + ReaderStep::Sent + } + Err(_) => ReaderStep::FrameSinkClosed, + }, + msg = ws_stream.next() => ReaderStep::WsMessage(msg), + } + } else { + tokio::select! { + biased; + _ = cancel.cancelled() => ReaderStep::Cancelled, + permit_result = frame_tx.reserve() => match permit_result { + Ok(permit) => { + let frame = held + .pop_front() + .expect("held at limit when reserve succeeds"); + permit.send(frame); + ReaderStep::Sent + } + Err(_) => ReaderStep::FrameSinkClosed, + }, + } + }; + + match step { + ReaderStep::Cancelled => break None, + ReaderStep::FrameSinkClosed => break None, + ReaderStep::Sent => { + outcome = SessionOutcome::Progressed; + } + ReaderStep::WsMessage(msg) => { + let Some(msg) = msg else { + break None; + }; + let parsed = match msg { + Ok(m) => m, + Err(e) => break Some(IngestError::Network(e)), + }; + match parsed { + WsMessage::Text(text) => { + let frame = match classify_text_frame(&text) { + Ok(f) => f, + Err(e) => break Some(e), + }; + held.push_back(frame); + } + WsMessage::Binary(_) => { + debug!("hydrant sent unexpected binary frame, ignoring"); + } + WsMessage::Ping(payload) => { + if control_tx + .send(WsEvent::IncomingPing(payload)) + .await + .is_err() + { + break None; + } + } + WsMessage::Pong(_) => { + if control_tx.send(WsEvent::IncomingPong).await.is_err() { + break None; + } + } + WsMessage::Close { code, reason } => { + debug!(code, %reason, "hydrant closed stream"); + break None; + } + } + } + } + }; + SessionEnd { outcome, error } +} + +enum ReaderStep { + Cancelled, + FrameSinkClosed, + Sent, + WsMessage(Option>), +} + +fn classify_text_frame(text: &str) -> Result { + #[derive(serde::Deserialize)] + struct PeekType<'a> { + #[serde(rename = "type", borrow)] + kind: Option>, + } + let is_error_frame = serde_json::from_str::(text) + .ok() + .and_then(|p| p.kind) + .as_deref() + == Some("error"); + if is_error_frame { + return match serde_json::from_str::(text) { + Ok(err_frame) => Err(classify_hydrant_error(err_frame)), + Err(decode_err) => Err(IngestError::Decode(decode_err)), + }; + } + serde_json::from_str::(text).map_err(IngestError::Decode) +} + +fn classify_hydrant_error(frame: HydrantStreamErrorFrame) -> IngestError { + let HydrantStreamErrorFrame { error, message } = frame; + let message = message.unwrap_or_default(); + match error.as_str() { + "ConsumerTooSlow" => IngestError::ConsumerTooSlow { message }, + _ => IngestError::HydrantStream { + code: error, + message, + }, + } +} + +#[derive(Debug)] +enum WsEvent { + IncomingPing(Bytes), + IncomingPong, +} + +async fn wait_until(deadline: Option, clock: &dyn Clock) { + match deadline { + Some(d) => clock.sleep_until(d).await, + None => std::future::pending::<()>().await, + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Regime { + Replay, + Live, + NonRecord, +} + +impl Regime { + fn as_str(self) -> &'static str { + match self { + Self::Replay => "replay", + Self::Live => "live", + Self::NonRecord => "non_record", + } + } +} + +struct Pending { + cursor: HydrantCursor, + signal: PromotionSignal, + regime: Regime, + op: PendingOp, +} + +enum PendingOp { + Noop, + ClearCache { + source: AtUri, + }, + Upsert { + source: AtUri, + nsid: Nsid, + parsed: Box, + bytes: Bytes, + cid: Option>, + edges: Vec, + }, + Parked { + nsid: Nsid, + }, + Delete { + source: AtUri, + nsid: Nsid, + }, +} + +struct Prepared { + pending: Pending, + prepare_start: Instant, + prepare_end: Instant, +} + +struct Resolved { + pending: Pending, + prepare_start: Instant, + prepare_end: Instant, + resolve_start: Instant, + resolve_end: Instant, +} + +fn pending_nsid(op: &PendingOp) -> Option<&Nsid> { + match op { + PendingOp::Upsert { nsid, .. } => Some(nsid), + PendingOp::Delete { nsid, .. } => Some(nsid), + PendingOp::Parked { nsid, .. } => Some(nsid), + PendingOp::Noop | PendingOp::ClearCache { .. } => None, + } +} + +fn pending_edge_count(op: &PendingOp) -> u64 { + match op { + PendingOp::Upsert { edges, .. } => edges.len() as u64, + _ => 0, + } +} + +async fn prep_stage( + frame: HydrantFrame, + rt: IngestRuntime, +) -> Prepared { + let now = rt.clock.now_unix_micros(); + let prepare_start = rt.clock.now_instant(); + let ctx = rt.pipeline_ctx(); + let pending = prepare_frame(frame, &ctx, now).await; + let prepare_end = rt.clock.now_instant(); + Prepared { + pending, + prepare_start, + prepare_end, + } +} + +async fn resolve_stage( + staged: Prepared, + rt: IngestRuntime, +) -> Resolved { + let resolve_start = rt.clock.now_instant(); + let ctx = rt.pipeline_ctx(); + let pending = resolve_pending(staged.pending, &ctx).await; + let resolve_end = rt.clock.now_instant(); + Resolved { + pending, + prepare_start: staged.prepare_start, + prepare_end: staged.prepare_end, + resolve_start, + resolve_end, + } +} + +async fn commit_stage( + staged: Resolved, + rt: IngestRuntime, + parallelism: usize, +) { + let nsid = pending_nsid(&staged.pending.op).cloned(); + let edge_count = pending_edge_count(&staged.pending.op); + let regime = staged.pending.regime; + let cursor = staged.pending.cursor.raw(); + let Resolved { + pending, + prepare_start, + prepare_end, + resolve_start, + resolve_end, + } = staged; + let commit_start = rt.clock.now_instant(); + commit_pending( + pending, + &rt.store, + &rt.issue_states, + &rt.pull_statuses, + &rt.coverage, + &*rt.search, + &*rt.records, + &rt.resolver, + ) + .await; + let commit_end = rt.clock.now_instant(); + tracing::trace!( + target: "bobbin_ingest::stage", + cursor, + regime = regime.as_str(), + nsid = %nsid.as_ref().map(Nsid::as_str).unwrap_or(""), + edge_count, + prepare_us = prepare_end.duration_since(prepare_start).as_micros() as u64, + queue_resolve_wait_us = resolve_start.duration_since(prepare_end).as_micros() as u64, + resolve_us = resolve_end.duration_since(resolve_start).as_micros() as u64, + queue_commit_wait_us = commit_start.duration_since(resolve_end).as_micros() as u64, + commit_us = commit_end.duration_since(commit_start).as_micros() as u64, + total_us = commit_end.duration_since(prepare_start).as_micros() as u64, + parallelism, + "pipeline stage timings", + ); +} + +async fn prepare_frame( + frame: HydrantFrame, + ctx: &PipelineCtx<'_, S>, + now: UnixMicros, +) -> Pending { + let cursor = HydrantCursor::new(frame.id); + let signal = promotion_signal(frame.record.as_ref(), now); + let regime = match frame.record.as_ref() { + Some(r) if r.live => Regime::Live, + Some(_) => Regime::Replay, + None => Regime::NonRecord, + }; + let op = match frame.kind { + FrameKind::Record => prepare_record(frame.record, ctx).await, + FrameKind::Identity | FrameKind::Account => PendingOp::Noop, + FrameKind::Other => { + debug!(id = frame.id, "ignoring unknown hydrant frame kind"); + PendingOp::Noop + } + }; + Pending { + cursor, + signal, + regime, + op, + } +} + +async fn prepare_record( + record: Option, + ctx: &PipelineCtx<'_, S>, +) -> PendingOp { + let Some(record) = record else { + debug!("record-typed frame missing payload, skipping"); + return PendingOp::Noop; + }; + if !record.collection.as_ref().starts_with(TANGLED_PREFIX) { + return PendingOp::Noop; + } + let nsid = record.collection.clone(); + let source = match build_source_uri(&record) { + Ok(s) => s, + Err(e) => { + warn!(?e, "invalid frame source, dropping record"); + return PendingOp::Noop; + } + }; + match record.action { + RecordAction::Create | RecordAction::Update => { + evict_from_buffer(ctx.buffer, &source).await; + let Some(raw) = record.record else { + debug!(collection = %nsid, "create/update missing record body, clearing cache"); + return PendingOp::ClearCache { source }; + }; + let raw_bytes = Bytes::copy_from_slice(raw.get().as_bytes()); + let wire_bytes = match fallback_rfc3339(&record.rkey, &record.rev) + .and_then(|fallback| synthesize_created_at(&raw_bytes, &fallback)) + { + Some(patched) => Bytes::from(patched), + None => raw_bytes, + }; + let (parsed, bytes) = match decode_canon_or_upgrade_bytes( + &record.collection, + &wire_bytes, + ctx.resolver, + ) + .await + { + Ok((parsed, canon_bytes)) => { + let bytes = match canon_bytes { + std::borrow::Cow::Borrowed(_) => wire_bytes, + std::borrow::Cow::Owned(v) => Bytes::from(v), + }; + (parsed, bytes) + } + Err(ExtractError::UnknownCollection(name)) => { + debug!(collection = %name, "unknown sh.tangled.* collection, clearing cache"); + return PendingOp::ClearCache { source }; + } + Err(e) => { + warn!(?e, collection = %record.collection, "record decode failed, clearing cache"); + return PendingOp::ClearCache { source }; + } + }; + if let Record::Repo(repo) = &parsed { + if let Some(shadow) = ctx.shadow { + shadow.note_observed(&record.did, &record.rkey).await; + } + let superseded = ctx + .resolver + .observe( + record.did.clone(), + record.rkey.clone(), + repo.repo_did.clone(), + ) + .await; + if let Some(prior) = superseded { + let prior_uri = format!( + "at://{}/sh.tangled.repo/{}", + prior.owner.as_ref(), + prior.rkey.as_ref(), + ); + if let Ok(prior_at_uri) = AtUri::::new_owned(&prior_uri) { + evict_from_buffer(ctx.buffer, &prior_at_uri).await; + ctx.store.remove_source(&prior_at_uri); + ctx.records.remove(&prior_at_uri); + ctx.search.remove(&prior_at_uri).await; + } + } + if let Some(buffer) = ctx.buffer { + let drained = buffer.take_observed(&record.did, &record.rkey).await; + if !drained.is_empty() { + finalize_drained(ctx, drained).await; + } + } + } + let edges = match parsed.extract_edges(&source) { + Ok(es) => es, + Err(e) => { + warn!(?e, "edge extraction failed, clearing cache"); + return PendingOp::ClearCache { source }; + } + }; + let _ = ctx.store.intern_source(&source); + PendingOp::Upsert { + source, + nsid, + parsed: Box::new(parsed), + bytes, + cid: record.cid, + edges, + } + } + RecordAction::Delete => { + evict_from_buffer(ctx.buffer, &source).await; + if nsid.as_ref() == "sh.tangled.repo" { + ctx.resolver.forget(&record.did, &record.rkey).await; + } + PendingOp::Delete { source, nsid } + } + RecordAction::Other => { + debug!(collection = %nsid, "ignoring unknown record action"); + PendingOp::Noop + } + } +} + +fn fallback_rfc3339( + rkey: &Rkey, + rev: &jacquard_common::types::tid::Tid, +) -> Option { + let tid = jacquard_common::types::tid::Tid::new(rkey.as_ref()) + .ok() + .unwrap_or_else(|| rev.clone()); + let micros = i64::try_from(tid.timestamp()).ok()?; + let dt = chrono::DateTime::::from_timestamp_micros(micros)?; + Some(dt.to_rfc3339_opts(chrono::SecondsFormat::Micros, true)) +} + +async fn evict_from_buffer(buffer: Option<&WarmingBuffer>, source: &AtUri) { + if let Some(buffer) = buffer + && !buffer.is_sealed() + { + buffer.evict_source(source).await; + } +} + +async fn resolve_pending( + pending: Pending, + ctx: &PipelineCtx<'_, S>, +) -> Pending { + let Pending { + cursor, + signal, + regime, + op, + } = pending; + let op = match op { + PendingOp::Upsert { + source, + nsid, + parsed, + bytes, + cid, + edges, + } => { + let pieces = UpsertPieces { + source, + nsid, + parsed: *parsed, + bytes, + cid, + edges, + }; + let pieces = match try_park_warming(ctx, cursor, pieces).await { + ParkOutcome::Parked { nsid } => { + return Pending { + cursor, + signal, + regime, + op: PendingOp::Parked { nsid }, + }; + } + ParkOutcome::Passthrough(pieces) => *pieces, + }; + let UpsertPieces { + source, + nsid, + parsed, + bytes, + cid, + edges, + } = pieces; + let edges = normalize_subjects(edges, ctx.resolver, ctx.coverage, ctx.shadow).await; + PendingOp::Upsert { + source, + nsid, + parsed: Box::new(parsed), + bytes, + cid, + edges, + } + } + other => other, + }; + Pending { + cursor, + signal, + regime, + op, + } +} + +struct UpsertPieces { + source: AtUri, + nsid: Nsid, + parsed: Record, + bytes: Bytes, + cid: Option>, + edges: Vec, +} + +impl From for UpsertPieces { + fn from(u: ParkedUpsert) -> Self { + Self { + source: u.source, + nsid: u.nsid, + parsed: u.parsed, + bytes: u.bytes, + cid: u.cid, + edges: u.edges, + } + } +} + +enum ParkOutcome { + Parked { nsid: Nsid }, + Passthrough(Box), +} + +async fn try_park_warming( + ctx: &PipelineCtx<'_, S>, + cursor: HydrantCursor, + pieces: UpsertPieces, +) -> ParkOutcome { + let Some(buffer) = ctx.buffer else { + return ParkOutcome::Passthrough(Box::new(pieces)); + }; + if ctx.coverage.snapshot().is_ready() || buffer.is_sealed() { + return ParkOutcome::Passthrough(Box::new(pieces)); + } + let deps = collect_unresolved_deps(&pieces.edges, ctx.resolver).await; + if deps.is_empty() { + return ParkOutcome::Passthrough(Box::new(pieces)); + } + let nsid = pieces.nsid.clone(); + let upsert = ParkedUpsert { + cursor, + source: pieces.source, + nsid: pieces.nsid, + parsed: pieces.parsed, + bytes: pieces.bytes, + cid: pieces.cid, + edges: pieces.edges, + }; + let deps_for_shadow = ctx.shadow.is_some().then(|| deps.clone()); + match buffer.try_park(upsert, deps).await { + Ok(()) => { + if let Some((shadow, noted)) = ctx.shadow.zip(deps_for_shadow) { + let _ = futures::future::join_all(noted.into_iter().map(|dep| async move { + shadow.note_unresolved(dep.owner, dep.rkey).await; + })) + .await; + } + ParkOutcome::Parked { nsid } + } + Err(returned) => ParkOutcome::Passthrough(Box::new(returned.into())), + } +} + +async fn collect_unresolved_deps(edges: &[Edge], resolver: &RepoIdResolver) -> Vec { + futures::stream::iter(edges) + .fold(Vec::new(), |mut acc, edge| async move { + let Some(uri) = edge.subject.as_uri() else { + return acc; + }; + let Some((owner, rkey)) = parse_repo_subject_uri(uri) else { + return acc; + }; + if resolver.cached_resolution(&owner, &rkey).await.is_some() { + return acc; + } + let candidate = RepoIdent::new(owner, rkey); + if !acc.contains(&candidate) { + acc.push(candidate); + } + acc + }) + .await +} + +async fn finalize_drained( + ctx: &PipelineCtx<'_, S>, + drained: Vec, +) { + for upsert in drained { + let ParkedUpsert { + cursor: _, + source, + nsid: _, + parsed, + bytes, + cid, + edges, + } = upsert; + let edges = normalize_subjects(edges, ctx.resolver, ctx.coverage, None).await; + cache_body(ctx.records, &source, cid, bytes); + ctx.store.upsert_source(&source, edges); + let outcome = apply_record_state(ctx.issue_states, ctx.pull_statuses, &source, &parsed); + log_unknown_state_variant(outcome, &source); + index_search(ctx.search, ctx.resolver, &source, parsed).await; + } +} + +#[allow(clippy::too_many_arguments)] +async fn commit_pending( + pending: Pending, + store: &EdgeStore, + issue_states: &StateIndex, + pull_statuses: &StateIndex, + coverage: &CoverageWatch, + search: &S, + records: &dyn RecordStore, + resolver: &RepoIdResolver, +) { + let Pending { + cursor, + signal, + regime: _, + op, + } = pending; + match op { + PendingOp::Noop | PendingOp::Parked { .. } => {} + PendingOp::ClearCache { source } => records.remove(&source), + PendingOp::Upsert { + source, + nsid: _, + parsed, + bytes, + cid, + edges, + } => { + cache_body(records, &source, cid, bytes); + store.upsert_source(&source, edges); + let outcome = apply_record_state(issue_states, pull_statuses, &source, &parsed); + log_unknown_state_variant(outcome, &source); + index_search(search, resolver, &source, *parsed).await; + } + PendingOp::Delete { source, nsid } => { + store.remove_source(&source); + apply_delete_to_state_index(issue_states, pull_statuses, &source, &nsid); + records.remove(&source); + search.remove(&source).await; + } + } + coverage.update(|c| c.advance(cursor).maybe_promote(signal)); +} + +async fn index_search( + search: &S, + resolver: &RepoIdResolver, + source: &AtUri, + parsed: Record, +) { + let Some(searchable) = SearchableRecord::try_from_record(parsed) else { + return; + }; + let Some(searchable) = searchable.normalize(resolver).await else { + return; + }; + search.upsert(searchable.to_search_doc(source)).await; +} + +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +async fn handle_frame( + frame: HydrantFrame, + store: &EdgeStore, + issue_states: &StateIndex, + pull_statuses: &StateIndex, + coverage: &CoverageWatch, + search: &S, + records: &dyn RecordStore, + resolver: &RepoIdResolver, + clock: &dyn Clock, + now: UnixMicros, +) { + let _ = clock; + let ctx = PipelineCtx { + resolver, + store, + issue_states, + pull_statuses, + coverage, + records, + search, + shadow: None, + buffer: None, + }; + let pending = prepare_frame(frame, &ctx, now).await; + let pending = resolve_pending(pending, &ctx).await; + commit_pending( + pending, + store, + issue_states, + pull_statuses, + coverage, + search, + records, + resolver, + ) + .await; +} + +fn log_unknown_state_variant(outcome: ApplyOutcome, source: &AtUri) { + if matches!(outcome, ApplyOutcome::UnknownVariant) { + warn!( + target: "bobbin_ingest::state_index", + %source, + "state record has unknown wire variant, skipping index update", + ); + } +} + +fn apply_delete_to_state_index( + issue_states: &StateIndex, + pull_statuses: &StateIndex, + source: &AtUri, + nsid: &Nsid, +) { + match nsid.as_ref() { + "sh.tangled.repo.issue" => issue_states.remove_entity(source), + "sh.tangled.repo.pull" => pull_statuses.remove_entity(source), + "sh.tangled.repo.issue.state" => issue_states.remove_source(source), + "sh.tangled.repo.pull.status" => pull_statuses.remove_source(source), + _ => {} + } +} + +fn promotion_signal(record: Option<&RecordFrame>, now: UnixMicros) -> PromotionSignal { + PromotionSignal { + rev_micros: record.map(|r| r.rev.timestamp()), + now_micros: now.raw(), + skew_micros: READY_SKEW.as_micros() as u64, + } +} + +fn cache_body( + records: &dyn RecordStore, + source: &AtUri, + cid: Option>, + bytes: Bytes, +) { + match cid { + Some(cid) => records.put( + source.clone(), + Arc::new(RecordBody { + uri: source.clone(), + cid, + value: bytes, + }), + ), + None => records.remove(source), + } +} + +async fn normalize_subjects( + edges: Vec, + resolver: &RepoIdResolver, + coverage: &CoverageWatch, + shadow: Option<&WarmingShadowBuffer>, +) -> Vec { + let warming = shadow.is_some() && !coverage.snapshot().is_ready(); + futures::stream::iter(edges) + .filter_map(|edge| async move { + let Some(uri) = edge.subject.as_uri() else { + return Some(edge); + }; + let Some((owner, rkey)) = parse_repo_subject_uri(uri) else { + return Some(edge); + }; + if warming + && let Some(shadow) = shadow + && resolver.cached_resolution(&owner, &rkey).await.is_none() + { + shadow + .note_unresolved(owner.clone(), rkey.clone()) + .await; + } + match resolver.resolve(&owner, &rkey).await { + Resolution::Mapped(repo_did) => Some(Edge { + subject: SubjectRef::Did(repo_did), + ..edge + }), + Resolution::NoRepoDid => { + warn!( + target: "bobbin_ingest::normalize", + kind = %edge.kind, + owner = owner.as_ref(), + rkey = rkey.as_ref(), + source = edge.source.as_ref(), + "dropping edge: target repo has no repoDid, no canonical DID subject available", + ); + None + } + Resolution::Unresolvable => { + warn!( + target: "bobbin_ingest::normalize", + kind = %edge.kind, + owner = owner.as_ref(), + rkey = rkey.as_ref(), + source = edge.source.as_ref(), + "dropping edge: repo unresolvable, rkey-form subject will not match bare-DID queries", + ); + None + } + } + }) + .collect() + .await +} + +fn parse_repo_subject_uri(uri: &AtUri) -> Option<(Did, Rkey)> { + let collection = uri.collection()?; + if collection.as_ref() != "sh.tangled.repo" { + return None; + } + let AtIdentifier::Did(authority) = uri.authority() else { + return None; + }; + let rkey = uri.rkey()?; + let owner = Did::new_owned(authority.as_ref()).ok()?; + let rkey = Rkey::new_owned(rkey.as_ref()).ok()?; + Some((owner, rkey)) +} + +fn build_source_uri(r: &RecordFrame) -> Result, IngestError> { + Ok(AtUri::from_parts_owned( + r.did.as_ref(), + r.collection.as_ref(), + r.rkey.as_ref(), + )?) +} + +#[cfg(test)] +mod tests { + use super::*; + use bobbin_edge_index::Coverage; + use bobbin_record_lru::{CacheCapacity, LruRecordStore, NoopRecordStore, RecordStore}; + use bobbin_runtime::{OsEntropy, RuntimeHasher, SystemClock, TungsteniteWs}; + use bobbin_types::search::NoopSearchSink; + use jacquard_common::types::nsid::Nsid; + use jacquard_common::types::tid::Tid; + use serde_json::json; + + const VALID_CID: &str = "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i"; + + fn did_subj(s: &str) -> SubjectRef { + SubjectRef::Did(Did::new_owned(s).unwrap()) + } + + fn uri_subj(s: &str) -> SubjectRef { + SubjectRef::Uri(AtUri::new_owned(s).unwrap()) + } + + fn rkey(s: &str) -> Rkey { + Rkey::new_owned(s).unwrap() + } + + #[allow(clippy::type_complexity)] + fn fresh() -> ( + Arc, + Arc>, + Arc>, + Arc, + Arc, + ) { + ( + Arc::new(EdgeStore::new(RuntimeHasher::default())), + Arc::new(StateIndex::new(RuntimeHasher::default())), + Arc::new(StateIndex::new(RuntimeHasher::default())), + Arc::new(CoverageWatch::new()), + Arc::new(RepoIdResolver::detached(RuntimeHasher::default())), + ) + } + + fn now() -> UnixMicros { + SystemClock::new().now_unix_micros() + } + + fn sys_clock() -> SystemClock { + SystemClock::new() + } + + fn parse_frame(value: serde_json::Value) -> HydrantFrame { + let text = serde_json::to_string(&value).expect("serialize fixture"); + serde_json::from_str(&text).expect("deserialize fixture") + } + + fn fresh_tid() -> Tid { + Tid::now_0() + } + + #[tokio::test] + async fn ignores_non_tangled_collections() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let frame: HydrantFrame = parse_frame(json!({ + "id": 1, + "type": "record", + "record": { + "live": false, + "did": "did:plc:nel", + "rev": fresh_tid().as_str(), + "collection": "app.bsky.feed.post", + "rkey": "abcabcabcabcz", + "action": "create", + "record": {"$type": "app.bsky.feed.post", "text": "hi"} + } + })); + handle_frame( + frame, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + assert_eq!(store.key_count(), 0); + assert_eq!(cov.snapshot().events_processed(), 1); + } + + #[tokio::test] + async fn create_then_delete_round_trips_a_star() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let create: HydrantFrame = parse_frame(json!({ + "id": 10, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subjectDid": "did:plc:abalone" + } + } + })); + handle_frame( + create, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + let key = bobbin_types::ids::EdgeKey::new( + Nsid::new_static("sh.tangled.feed.star").unwrap(), + did_subj("did:plc:abalone"), + ); + assert_eq!(store.count(&key), 1); + + let delete: HydrantFrame = parse_frame(json!({ + "id": 11, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "delete", + "record": null + } + })); + handle_frame( + delete, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + assert_eq!(store.count(&key), 0); + } + + #[tokio::test] + async fn update_replaces_prior_edges() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let mk = |subject_did: &Did, id: u64| -> HydrantFrame { + parse_frame(json!({ + "id": id, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "update", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subjectDid": subject_did.as_ref() + } + } + })) + }; + handle_frame( + mk(&Did::new_owned("did:plc:abalone").unwrap(), 1), + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + handle_frame( + mk(&Did::new_owned("did:plc:uni").unwrap(), 2), + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + + let kind = Nsid::new_static("sh.tangled.feed.star").unwrap(); + let old = bobbin_types::ids::EdgeKey::new(kind.clone(), did_subj("did:plc:abalone")); + let new = bobbin_types::ids::EdgeKey::new(kind, did_subj("did:plc:uni")); + assert_eq!(store.count(&old), 0); + assert_eq!(store.count(&new), 1); + } + + #[tokio::test] + async fn prepare_frame_tags_regime_from_live_flag() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let search = NoopSearchSink; + let records = NoopRecordStore; + let ctx = PipelineCtx { + resolver: &resolver, + store: &store, + issue_states: &issue_states, + pull_statuses: &pull_statuses, + coverage: &cov, + records: &records, + search: &search, + shadow: None, + buffer: None, + }; + let mk = |live: bool| -> HydrantFrame { + parse_frame(json!({ + "id": 1, + "type": "record", + "record": { + "live": live, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subjectDid": "did:plc:abalone" + } + } + })) + }; + let live_pending = prepare_frame(mk(true), &ctx, now()).await; + assert_eq!(live_pending.regime, Regime::Live); + let replay_pending = prepare_frame(mk(false), &ctx, now()).await; + assert_eq!(replay_pending.regime, Regime::Replay); + + let identity: HydrantFrame = parse_frame(json!({ + "id": 9, + "type": "identity", + })); + let id_pending = prepare_frame(identity, &ctx, now()).await; + assert_eq!(id_pending.regime, Regime::NonRecord); + } + + #[tokio::test] + async fn create_with_cid_warms_record_lru() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let lru = LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024)); + let frame: HydrantFrame = parse_frame(json!({ + "id": 1, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "create", + "cid": VALID_CID, + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subjectDid": "did:plc:abalone" + } + } + })); + let source = + AtUri::new_owned("at://did:plc:olaren/sh.tangled.feed.star/abcabcabcabcz").unwrap(); + handle_frame( + frame, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &lru, + &resolver, + &sys_clock(), + now(), + ) + .await; + let cached = lru.get(&source).expect("hydrant cid must seed the lru"); + assert_eq!(cached.cid.as_ref(), VALID_CID); + let parsed: serde_json::Value = serde_json::from_slice(&cached.value).unwrap(); + assert_eq!( + parsed["subject"]["did"], "did:plc:abalone", + "legacy wire is upgraded to canon shape before caching so downstream readers see canonical fields" + ); + } + + #[tokio::test] + async fn create_without_cid_clears_record_lru() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let lru = LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024)); + let source = + AtUri::new_owned("at://did:plc:olaren/sh.tangled.feed.star/abcabcabcabcz").unwrap(); + let cid: Cid = VALID_CID.parse().unwrap(); + lru.put( + source.clone(), + Arc::new(RecordBody { + uri: source.clone(), + cid, + value: bytes::Bytes::from_static(b"{\"stale\":true}"), + }), + ); + let frame: HydrantFrame = parse_frame(json!({ + "id": 1, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "update", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subjectDid": "did:plc:abalone" + } + } + })); + handle_frame( + frame, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &lru, + &resolver, + &sys_clock(), + now(), + ) + .await; + assert!( + lru.get(&source).is_none(), + "missing cid means we cannot trust the body, so the lru must be cleared", + ); + } + + #[tokio::test] + async fn delete_evicts_record_lru() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let lru = LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024)); + let source = + AtUri::new_owned("at://did:plc:olaren/sh.tangled.feed.star/abcabcabcabcz").unwrap(); + let cid: Cid = VALID_CID.parse().unwrap(); + lru.put( + source.clone(), + Arc::new(RecordBody { + uri: source.clone(), + cid, + value: bytes::Bytes::from_static(b"{}"), + }), + ); + let frame: HydrantFrame = parse_frame(json!({ + "id": 1, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "delete", + "record": null + } + })); + handle_frame( + frame, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &lru, + &resolver, + &sys_clock(), + now(), + ) + .await; + assert!(lru.get(&source).is_none()); + } + + #[tokio::test] + async fn live_recent_event_promotes_coverage_to_ready() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + assert!(!cov.snapshot().is_ready()); + let frame: HydrantFrame = parse_frame(json!({ + "id": 99, + "type": "record", + "record": { + "live": true, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subjectDid": "did:plc:abalone" + } + } + })); + handle_frame( + frame, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + assert!(cov.snapshot().is_ready()); + assert_eq!(cov.snapshot().last_cursor(), HydrantCursor::new(99)); + } + + #[tokio::test] + async fn live_but_stale_rev_does_not_promote() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let stale_tid = Tid::from_time(1_000_000, 0); + let frame: HydrantFrame = parse_frame(json!({ + "id": 7, + "type": "record", + "record": { + "live": true, + "did": "did:plc:olaren", + "rev": stale_tid.as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subjectDid": "did:plc:abalone" + } + } + })); + handle_frame( + frame, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + assert!(!cov.snapshot().is_ready()); + assert!(matches!(cov.snapshot(), Coverage::Warming { .. })); + } + + #[test] + fn classify_consumer_too_slow_frame_returns_typed_variant() { + let text = r#"{"type":"error","error":"ConsumerTooSlow","message":"stream socket send blocked for at least 30 seconds"}"#; + match classify_text_frame(text) { + Err(IngestError::ConsumerTooSlow { message }) => { + assert!( + message.contains("30 seconds"), + "message field preserved verbatim, got: {message}" + ); + } + other => panic!("expected ConsumerTooSlow variant, got: {other:?}"), + } + } + + #[test] + fn classify_unknown_hydrant_error_falls_back_to_generic_variant() { + let text = r#"{"type":"error","error":"NewFutureCode","message":"some new failure mode"}"#; + match classify_text_frame(text) { + Err(IngestError::HydrantStream { code, message }) => { + assert_eq!(code, "NewFutureCode"); + assert_eq!(message, "some new failure mode"); + } + other => panic!("expected HydrantStream variant, got: {other:?}"), + } + } + + #[test] + fn classify_error_frame_without_message_uses_empty_string() { + let text = r#"{"type":"error","error":"ConsumerTooSlow"}"#; + match classify_text_frame(text) { + Err(IngestError::ConsumerTooSlow { message }) => assert!(message.is_empty()), + other => panic!("expected ConsumerTooSlow with empty message, got: {other:?}"), + } + } + + #[test] + fn classify_normal_record_frame_unchanged() { + let text = r#"{"id":42,"type":"record"}"#; + let frame = classify_text_frame(text).expect("normal record frame must parse"); + assert_eq!(frame.id, 42); + assert_eq!(frame.kind, FrameKind::Record); + } + + #[test] + fn classify_garbage_object_returns_decode_error() { + let text = r#"{"random":"object","without":"required fields"}"#; + match classify_text_frame(text) { + Err(IngestError::Decode(_)) => {} + other => panic!("expected Decode error, got: {other:?}"), + } + } + + struct ScriptedWsStream { + messages: std::collections::VecDeque>, + } + + impl WsStream for ScriptedWsStream { + fn next<'a>(&'a mut self) -> bobbin_runtime::WsMessageFuture<'a> { + let msg = self.messages.pop_front(); + Box::pin(async move { msg }) + } + } + + #[tokio::test] + async fn reader_loop_surfaces_consumer_too_slow_from_error_frame() { + let mut messages = std::collections::VecDeque::new(); + messages.push_back(Ok(WsMessage::Text( + r#"{"type":"error","error":"ConsumerTooSlow","message":"stream delivery blocked"}"# + .to_owned(), + ))); + let stream: Box = Box::new(ScriptedWsStream { messages }); + let (frame_tx, _frame_rx) = tokio::sync::mpsc::channel::(FRAME_CHANNEL_DEPTH); + let (control_tx, _control_rx) = + tokio::sync::mpsc::channel::(CONTROL_CHANNEL_DEPTH); + let cancel = CancellationToken::new(); + + let end = reader_loop(stream, frame_tx, control_tx, cancel).await; + + match end.error { + Some(IngestError::ConsumerTooSlow { message }) => { + assert_eq!(message, "stream delivery blocked"); + } + other => panic!("expected ConsumerTooSlow SessionEnd error, got: {other:?}"), + } + assert_eq!(end.outcome, SessionOutcome::Empty); + } + + #[tokio::test] + async fn reader_loop_surfaces_unknown_hydrant_error_distinctly() { + let mut messages = std::collections::VecDeque::new(); + messages.push_back(Ok(WsMessage::Text( + r#"{"type":"error","error":"NewFutureCode","message":"new mode"}"#.to_owned(), + ))); + let stream: Box = Box::new(ScriptedWsStream { messages }); + let (frame_tx, _frame_rx) = tokio::sync::mpsc::channel::(FRAME_CHANNEL_DEPTH); + let (control_tx, _control_rx) = + tokio::sync::mpsc::channel::(CONTROL_CHANNEL_DEPTH); + let cancel = CancellationToken::new(); + + let end = reader_loop(stream, frame_tx, control_tx, cancel).await; + + match end.error { + Some(IngestError::HydrantStream { code, message }) => { + assert_eq!(code, "NewFutureCode"); + assert_eq!(message, "new mode"); + } + other => panic!("expected HydrantStream SessionEnd error, got: {other:?}"), + } + } + + struct ChannelWsStream { + rx: tokio::sync::mpsc::Receiver>, + } + + impl WsStream for ChannelWsStream { + fn next<'a>(&'a mut self) -> bobbin_runtime::WsMessageFuture<'a> { + Box::pin(async move { self.rx.recv().await }) + } + } + + fn star_frame_text(id: u64, rkey: &Rkey) -> String { + json!({ + "id": id, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": rkey.as_ref(), + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subjectDid": "did:plc:abalone" + } + } + }) + .to_string() + } + + #[tokio::test] + async fn pong_forwards_promptly_when_frame_channel_is_full() { + let (ws_tx, ws_rx) = tokio::sync::mpsc::channel::>(8); + let stream: Box = Box::new(ChannelWsStream { rx: ws_rx }); + let (frame_tx, frame_rx) = tokio::sync::mpsc::channel::(1); + let (control_tx, mut control_rx) = + tokio::sync::mpsc::channel::(CONTROL_CHANNEL_DEPTH); + let cancel = CancellationToken::new(); + + let prefill: HydrantFrame = parse_frame(json!({ + "id": 0, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "prefilrkey001", + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subjectDid": "did:plc:abalone" + } + } + })); + frame_tx + .try_send(prefill) + .expect("depth-1 frame channel must accept the prefill"); + + let reader_handle = tokio::spawn(reader_loop( + stream, + frame_tx.clone(), + control_tx.clone(), + cancel.clone(), + )); + + ws_tx + .send(Ok(WsMessage::Text(star_frame_text( + 1, + &rkey("starrkeyaa001"), + )))) + .await + .unwrap(); + ws_tx.send(Ok(WsMessage::Pong(Bytes::new()))).await.unwrap(); + + let pong_event = tokio::time::timeout(Duration::from_millis(500), control_rx.recv()) + .await + .expect("pong must surface inside 500ms even when frame_tx is saturated. A reader blocked on a frame send would never poll the next ws message") + .expect("control_tx was not closed"); + assert!( + matches!(pong_event, WsEvent::IncomingPong), + "first control event must be the pong, not a held text", + ); + + let too_slow = + "{\"type\":\"error\",\"error\":\"ConsumerTooSlow\",\"message\":\"saturated\"}" + .to_string(); + ws_tx.send(Ok(WsMessage::Text(too_slow))).await.unwrap(); + + let end = tokio::time::timeout(Duration::from_secs(1), reader_handle) + .await + .expect("reader must exit promptly once ConsumerTooSlow is read") + .expect("reader task must not panic"); + match end.error { + Some(IngestError::ConsumerTooSlow { message }) => assert_eq!(message, "saturated"), + other => panic!("expected ConsumerTooSlow disconnect, got: {other:?}"), + } + + drop(frame_rx); + } + + #[tokio::test] + async fn reader_drains_held_frames_once_processor_catches_up() { + let (ws_tx, ws_rx) = tokio::sync::mpsc::channel::>(8); + let stream: Box = Box::new(ChannelWsStream { rx: ws_rx }); + let (frame_tx, mut frame_rx) = tokio::sync::mpsc::channel::(1); + let (control_tx, _control_rx) = + tokio::sync::mpsc::channel::(CONTROL_CHANNEL_DEPTH); + let cancel = CancellationToken::new(); + + let prefill: HydrantFrame = parse_frame(json!({ + "id": 0, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "prefilrkey001", + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subjectDid": "did:plc:abalone" + } + } + })); + frame_tx.try_send(prefill).unwrap(); + + let reader_handle = tokio::spawn(reader_loop( + stream, + frame_tx.clone(), + control_tx, + cancel.clone(), + )); + + ws_tx + .send(Ok(WsMessage::Text(star_frame_text( + 1, + &rkey("heldrkeyaa001"), + )))) + .await + .unwrap(); + ws_tx + .send(Ok(WsMessage::Text(star_frame_text( + 2, + &rkey("heldrkeyaa002"), + )))) + .await + .unwrap(); + + let _drained_prefill = frame_rx.recv().await.expect("prefilled frame drains"); + let first = tokio::time::timeout(Duration::from_millis(500), frame_rx.recv()) + .await + .expect("first held frame must reach frame_rx after slot opens") + .expect("frame_tx still open"); + assert_eq!(first.id, 1); + let second = tokio::time::timeout(Duration::from_millis(500), frame_rx.recv()) + .await + .expect("second held frame must reach frame_rx after slot opens") + .expect("frame_tx still open"); + assert_eq!(second.id, 2); + + cancel.cancel(); + let _ = tokio::time::timeout(Duration::from_secs(1), reader_handle) + .await + .expect("reader must stop after cancel"); + } + + #[test] + fn first_connect_uses_configured_start_cursor() { + let start = HydrantCursor::new(42); + assert_eq!(next_connect_cursor(Coverage::default(), start), start); + } + + #[test] + fn reconnect_resumes_strictly_after_last_seen() { + let snap = Coverage::default().advance(HydrantCursor::new(7)); + assert_eq!( + next_connect_cursor(snap, HydrantCursor::new(0)), + HydrantCursor::new(8), + ); + } + + #[test] + fn reconnect_overrides_configured_start() { + let snap = Coverage::default().advance(HydrantCursor::new(100)); + assert_eq!( + next_connect_cursor(snap, HydrantCursor::new(50)), + HydrantCursor::new(101), + ); + } + + #[test] + fn first_connect_uses_start_even_when_first_frame_id_would_be_zero() { + let start = HydrantCursor::new(7); + let snap = Coverage::default(); + assert_eq!(snap.last_cursor(), HydrantCursor::new(0)); + assert_eq!(snap.events_processed(), 0); + assert_eq!(next_connect_cursor(snap, start), start); + } + + #[test] + fn reconnect_after_processing_id_zero_advances_to_one() { + let snap = Coverage::default().advance(HydrantCursor::new(0)); + assert_eq!(snap.events_processed(), 1); + assert_eq!( + next_connect_cursor(snap, HydrantCursor::new(99)), + HydrantCursor::new(1), + "events_processed disambiguates 'never seen' from 'saw id 0'", + ); + } + + #[tokio::test] + async fn identity_frame_advances_cursor_only() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let frame: HydrantFrame = parse_frame(json!({ + "id": 5, + "type": "identity", + "identity": { + "did": "did:plc:olaren", + "handle": "olaren.dev" + } + })); + handle_frame( + frame, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + assert_eq!(store.key_count(), 0); + assert_eq!(cov.snapshot().last_cursor(), HydrantCursor::new(5)); + assert!(!cov.snapshot().is_ready()); + } + + #[tokio::test] + async fn account_frame_advances_cursor_only() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let frame: HydrantFrame = parse_frame(json!({ + "id": 6, + "type": "account", + "account": {"did": "did:plc:olaren", "active": true} + })); + handle_frame( + frame, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + assert_eq!(store.key_count(), 0); + assert_eq!(cov.snapshot().last_cursor(), HydrantCursor::new(6)); + } + + #[tokio::test] + async fn unknown_frame_kind_advances_cursor_without_panic() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let frame: HydrantFrame = parse_frame(json!({"id": 8, "type": "future_event"})); + assert_eq!(frame.kind, FrameKind::Other); + handle_frame( + frame, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + assert_eq!(cov.snapshot().last_cursor(), HydrantCursor::new(8)); + } + + fn fresh_runtime(cancel: CancellationToken) -> IngestRuntime { + IngestRuntime { + store: Arc::new(EdgeStore::new(RuntimeHasher::default())), + issue_states: Arc::new(StateIndex::new(RuntimeHasher::default())), + pull_statuses: Arc::new(StateIndex::new(RuntimeHasher::default())), + coverage: Arc::new(CoverageWatch::new()), + search: Arc::new(NoopSearchSink), + records: Arc::new(NoopRecordStore) as Arc, + resolver: Arc::new(RepoIdResolver::detached(RuntimeHasher::default())), + clock: Arc::new(SystemClock::new()), + entropy: Arc::new(OsEntropy), + ws: TungsteniteWs::shared(), + cancel, + disconnects: None, + warming_shadow: None, + warming_buffer: None, + } + } + + #[tokio::test(start_paused = true)] + async fn cancel_token_short_circuits_reconnect_sleep() { + let cfg = IngestConfig::new(Url::parse("ws://127.0.0.1:1").unwrap()); + let cancel = CancellationToken::new(); + let runtime = fresh_runtime(cancel.clone()); + let task = tokio::spawn(async move { run(cfg, runtime).await }); + tokio::time::advance(Duration::from_millis(10)).await; + cancel.cancel(); + let outcome = tokio::time::timeout(Duration::from_secs(1), task) + .await + .expect("ingest must stop within timeout once cancel fires"); + assert!(matches!(outcome, Ok(Ok(()))), "got {outcome:?}"); + } + + #[test] + fn jittered_stays_within_one_quarter_of_base() { + let base = Duration::from_secs(1); + let cap = base + Duration::from_millis(250); + let entropy = OsEntropy; + (0..50).for_each(|_| { + let j = jittered(base, &entropy); + assert!(j >= base, "jitter must not undershoot"); + assert!(j <= cap, "jitter must not exceed +25%, got {:?}", j); + }); + } + + #[tokio::test] + async fn star_after_observed_repo_keys_on_repo_did() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let repo: HydrantFrame = parse_frame(json!({ + "id": 1, + "type": "record", + "record": { + "live": false, + "did": "did:plc:nel", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.repo", + "rkey": "abcabcabcabcz", + "action": "create", + "record": { + "$type": "sh.tangled.repo", + "createdAt": "2026-05-01T00:00:00Z", + "knot": "oyster.cafe", + "name": "abalone", + "repoDid": "did:plc:abalone" + } + } + })); + handle_frame( + repo, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + + let star: HydrantFrame = parse_frame(json!({ + "id": 2, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subject": "at://did:plc:nel/sh.tangled.repo/abcabcabcabcz" + } + } + })); + handle_frame( + star, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + + let nsid = Nsid::new_static("sh.tangled.feed.star").unwrap(); + let repo_keyed = bobbin_types::ids::EdgeKey::new(nsid.clone(), did_subj("did:plc:abalone")); + let owner_keyed = bobbin_types::ids::EdgeKey::new(nsid, did_subj("did:plc:nel")); + assert_eq!( + store.count(&repo_keyed), + 1, + "star should be keyed on repoDID once the repo is observed" + ); + assert_eq!( + store.count(&owner_keyed), + 0, + "owner DID should not collect the edge" + ); + } + + #[tokio::test] + async fn unresolvable_repo_subject_drops_edge() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let star: HydrantFrame = parse_frame(json!({ + "id": 1, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subject": "at://did:plc:nel/sh.tangled.repo/abcabcabcabcz" + } + } + })); + handle_frame( + star, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + + let nsid = Nsid::new_static("sh.tangled.feed.star").unwrap(); + let owner_keyed = bobbin_types::ids::EdgeKey::new(nsid.clone(), did_subj("did:plc:nel")); + let uri_keyed = bobbin_types::ids::EdgeKey::new( + nsid, + uri_subj("at://did:plc:nel/sh.tangled.repo/abcabcabcabcz"), + ); + assert_eq!( + store.count(&owner_keyed), + 0, + "must not silently misfile under the authoring DID", + ); + assert_eq!( + store.count(&uri_keyed), + 0, + "unresolvable rkey-form subject must drop the edge; keeping it would index against a key that never matches bare-DID queries", + ); + } + + #[tokio::test] + async fn repo_without_repo_did_drops_edge() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let repo: HydrantFrame = parse_frame(json!({ + "id": 1, + "type": "record", + "record": { + "live": false, + "did": "did:plc:nel", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.repo", + "rkey": "abcabcabcabcz", + "action": "create", + "record": { + "$type": "sh.tangled.repo", + "createdAt": "2026-05-01T00:00:00Z", + "knot": "oyster.cafe", + "name": "abalone" + } + } + })); + handle_frame( + repo, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + + let star: HydrantFrame = parse_frame(json!({ + "id": 2, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subject": "at://did:plc:nel/sh.tangled.repo/abcabcabcabcz" + } + } + })); + handle_frame( + star, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + + let nsid = Nsid::new_static("sh.tangled.feed.star").unwrap(); + let uri_keyed = bobbin_types::ids::EdgeKey::new( + nsid.clone(), + uri_subj("at://did:plc:nel/sh.tangled.repo/abcabcabcabcz"), + ); + let owner_keyed = bobbin_types::ids::EdgeKey::new(nsid, did_subj("did:plc:nel")); + assert_eq!( + store.count(&uri_keyed), + 0, + "no canonical DID exists for a repo without repoDID, so the edge must be dropped", + ); + assert_eq!( + store.count(&owner_keyed), + 0, + "the authoring DID is not the canonical repo identity", + ); + } + + #[tokio::test] + async fn explicit_subject_did_skips_normalization() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let star: HydrantFrame = parse_frame(json!({ + "id": 1, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": "abcabcabcabcz", + "action": "create", + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subjectDid": "did:plc:abalone" + } + } + })); + handle_frame( + star, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + let key = bobbin_types::ids::EdgeKey::new( + Nsid::new_static("sh.tangled.feed.star").unwrap(), + did_subj("did:plc:abalone"), + ); + assert_eq!(store.count(&key), 1); + } + + #[tokio::test] + async fn issue_with_repo_uri_resolves_to_repo_did() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + resolver + .observe( + Did::new_owned("did:plc:nel").unwrap(), + Rkey::new_owned("abcabcabcabcz").unwrap(), + Some(Did::new_owned("did:plc:abalone").unwrap()), + ) + .await; + let issue: HydrantFrame = parse_frame(json!({ + "id": 1, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.repo.issue", + "rkey": "abcabcabcabcz", + "action": "create", + "record": { + "$type": "sh.tangled.repo.issue", + "createdAt": "2026-05-01T00:00:00Z", + "title": "bug", + "repo": "at://did:plc:nel/sh.tangled.repo/abcabcabcabcz" + } + } + })); + handle_frame( + issue, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + let key = bobbin_types::ids::EdgeKey::new( + Nsid::new_static("sh.tangled.repo.issue").unwrap(), + did_subj("did:plc:abalone"), + ); + assert_eq!(store.count(&key), 1); + } + + #[derive(Default)] + struct RecordingSearchSink { + docs: tokio::sync::Mutex>, + } + + impl SearchSink for RecordingSearchSink { + async fn upsert(&self, doc: bobbin_types::search::SearchDoc) { + self.docs.lock().await.push(doc); + } + async fn remove(&self, _uri: &AtUri) {} + } + + #[tokio::test] + async fn search_index_hydrates_repo_did_via_resolver() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + resolver + .observe( + Did::new_owned("did:plc:nel").unwrap(), + Rkey::new_owned("abcabcabcabcz").unwrap(), + Some(Did::new_owned("did:plc:abalone").unwrap()), + ) + .await; + let search = RecordingSearchSink::default(); + let issue: HydrantFrame = parse_frame(json!({ + "id": 1, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.repo.issue", + "rkey": "abcabcabcabcz", + "action": "create", + "record": { + "$type": "sh.tangled.repo.issue", + "createdAt": "2026-05-01T00:00:00Z", + "title": "bug", + "repo": "at://did:plc:nel/sh.tangled.repo/abcabcabcabcz" + } + } + })); + handle_frame( + issue, + &store, + &issue_states, + &pull_statuses, + &cov, + &search, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + let docs = search.docs.lock().await; + assert_eq!(docs.len(), 1, "issue should produce one search doc"); + assert_eq!( + docs[0].repo, + Some(Did::new_owned("did:plc:abalone").unwrap()), + "search doc repo field must be resolved from the observed repo, not left as None", + ); + } + + #[tokio::test] + async fn delete_repo_record_evicts_resolver_cache() { + let (store, issue_states, pull_statuses, cov, resolver) = fresh(); + let owner = Did::new_owned("did:plc:nel").unwrap(); + let rkey = Rkey::new_owned("abcabcabcabcz").unwrap(); + resolver + .observe( + owner.clone(), + rkey.clone(), + Some(Did::new_owned("did:plc:abalone").unwrap()), + ) + .await; + assert!( + resolver.cached_resolution(&owner, &rkey).await.is_some(), + "observe must seed the cache", + ); + let delete: HydrantFrame = parse_frame(json!({ + "id": 1, + "type": "record", + "record": { + "live": false, + "did": owner.as_ref(), + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.repo", + "rkey": rkey.as_ref(), + "action": "delete" + } + })); + handle_frame( + delete, + &store, + &issue_states, + &pull_statuses, + &cov, + &NoopSearchSink, + &NoopRecordStore, + &resolver, + &sys_clock(), + now(), + ) + .await; + assert!( + resolver.cached_resolution(&owner, &rkey).await.is_none(), + "deleting the repo record must clear the resolver cache so future observes are not blocked by a stale Authoritative entry", + ); + } + + #[tokio::test] + async fn cancel_short_circuits_a_hung_ws_connect() { + let _listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind sink listener"); + let port = _listener.local_addr().expect("local addr").port(); + let cfg = + IngestConfig::new(Url::parse(&format!("ws://127.0.0.1:{port}")).expect("hydrant url")); + let cancel = CancellationToken::new(); + let runtime = fresh_runtime(cancel.clone()); + let task = tokio::spawn(async move { run(cfg, runtime).await }); + tokio::time::sleep(Duration::from_millis(100)).await; + cancel.cancel(); + let outcome = tokio::time::timeout(Duration::from_secs(2), task) + .await + .expect("cancel must short-circuit the hung ws connect"); + assert!(matches!(outcome, Ok(Ok(()))), "got {outcome:?}"); + } + + struct CloseOnConnectTransport { + used: std::sync::Mutex, + } + impl bobbin_runtime::WsTransport for CloseOnConnectTransport { + fn connect(&self, _url: Url) -> bobbin_runtime::WsConnectFuture { + let mut used = self.used.lock().unwrap(); + if *used { + return Box::pin(async move { + Err(NetworkError::Connect("only one connect allowed".to_owned())) + }); + } + *used = true; + Box::pin(async move { + let mut q = std::collections::VecDeque::new(); + q.push_back(Ok(WsMessage::Close { + code: 1000, + reason: "bye".to_owned(), + })); + let stream: Box = Box::new(ScriptedWsStream { messages: q }); + struct NoopSink; + impl bobbin_runtime::WsSink for NoopSink { + fn send<'a>(&'a mut self, _m: WsMessage) -> bobbin_runtime::WsSendFuture<'a> { + Box::pin(async move { Ok(()) }) + } + } + let sink: Box = Box::new(NoopSink); + Ok(bobbin_runtime::WsConn { sink, stream }) + }) + } + } + + #[tokio::test] + async fn run_session_returns_after_remote_close_when_outer_cancel_unfired() { + let cfg = IngestConfig::new(Url::parse("ws://127.0.0.1:1").unwrap()); + let cancel = CancellationToken::new(); + let mut runtime = fresh_runtime(cancel.clone()); + runtime.ws = Arc::new(CloseOnConnectTransport { + used: std::sync::Mutex::new(false), + }); + + let res = tokio::time::timeout( + Duration::from_secs(3), + run_session(&cfg, HydrantCursor::new(0), &runtime), + ) + .await; + assert!( + res.is_ok(), + "run_session must return after a remote Close even when outer cancel never fires - regression for a session-scoped task hanging on the parent token", + ); + } + + struct HangingSink; + impl bobbin_runtime::WsSink for HangingSink { + fn send<'a>(&'a mut self, _: WsMessage) -> bobbin_runtime::WsSendFuture<'a> { + Box::pin(std::future::pending()) + } + } + + #[tokio::test(start_paused = true)] + async fn timed_send_surfaces_send_timeout_when_sink_pends_forever() { + let mut sink: Box = Box::new(HangingSink); + let task = + tokio::spawn(async move { timed_send(&mut sink, WsMessage::Ping(Bytes::new())).await }); + tokio::time::advance(SEND_TIMEOUT + Duration::from_secs(1)).await; + let result = task.await.expect("task panicked"); + match result { + Err(IngestError::SendTimeout(d)) => assert_eq!(d, SEND_TIMEOUT), + other => panic!( + "expected SendTimeout, got {other:?}. A bare ws_sink.send.await would hang forever on a half-dead socket and starve the writer's pong-deadline arm", + ), + } + } + + struct OkSink; + impl bobbin_runtime::WsSink for OkSink { + fn send<'a>(&'a mut self, _: WsMessage) -> bobbin_runtime::WsSendFuture<'a> { + Box::pin(async move { Ok(()) }) + } + } + + #[tokio::test] + async fn timed_send_returns_ok_when_sink_succeeds_promptly() { + let mut sink: Box = Box::new(OkSink); + let result = timed_send(&mut sink, WsMessage::Ping(Bytes::new())).await; + assert!(matches!(result, Ok(())), "got {result:?}"); + } + + #[test] + fn classify_error_frame_with_id_dispatches_as_error_not_unknown_kind() { + let text = r#"{"id":42,"type":"error","error":"ConsumerTooSlow","message":"slow"}"#; + match classify_text_frame(text) { + Err(IngestError::ConsumerTooSlow { message }) => assert_eq!(message, "slow"), + other => panic!( + "type=\"error\" must dispatch to the error path even when id is present, got: {other:?}" + ), + } + } + + #[tokio::test] + async fn buffered_pipeline_preserves_cursor_order_under_resolve_latency_skew() { + use bobbin_record_lru::RecordStore; + use bobbin_types::record::RecordBody; + use std::sync::Mutex; + + let server = wiremock::MockServer::start().await; + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/xrpc/com.atproto.repo.getRecord")) + .respond_with( + wiremock::ResponseTemplate::new(404).set_delay(Duration::from_millis(150)), + ) + .mount(&server) + .await; + + let client = bobbin_slingshot_client::SlingshotClient::with_default_http( + Url::parse(&server.uri()).unwrap(), + ) + .unwrap(); + let clock: Arc = Arc::new(SystemClock::new()); + let resolver = Arc::new(RepoIdResolver::with_slingshot( + client, + clock.clone(), + RuntimeHasher::default(), + )); + + let owner = Did::new_owned("did:plc:nel").unwrap(); + let abalone = Did::new_owned("did:plc:abalone").unwrap(); + let fast_rkeys: [Rkey; 2] = [rkey("fastrkeyaa01"), rkey("fastrkeyaa02")]; + let slow_rkeys: [Rkey; 2] = [rkey("slowrkeyaa01"), rkey("slowrkeyaa02")]; + for r in &fast_rkeys { + resolver + .observe(owner.clone(), r.clone(), Some(abalone.clone())) + .await; + } + + #[derive(Default)] + struct Capturing { + urls: Mutex>>, + } + impl RecordStore for Capturing { + fn get(&self, _uri: &AtUri) -> Option> { + None + } + fn put(&self, uri: AtUri, _body: Arc) { + self.urls.lock().unwrap().push(uri); + } + fn remove(&self, _uri: &AtUri) {} + } + let capturing = Arc::new(Capturing::default()); + + let runtime: IngestRuntime = IngestRuntime { + store: Arc::new(EdgeStore::new(RuntimeHasher::default())), + issue_states: Arc::new(StateIndex::new(RuntimeHasher::default())), + pull_statuses: Arc::new(StateIndex::new(RuntimeHasher::default())), + coverage: Arc::new(CoverageWatch::new()), + search: Arc::new(NoopSearchSink), + records: capturing.clone() as Arc, + resolver, + clock, + entropy: Arc::new(OsEntropy), + ws: TungsteniteWs::shared(), + cancel: CancellationToken::new(), + disconnects: None, + warming_shadow: None, + warming_buffer: None, + }; + + let parallelism = 4usize; + let (frame_tx, frame_rx) = tokio::sync::mpsc::channel::(64); + let pipeline_rt = runtime.clone(); + let pipeline = tokio::spawn(async move { + let prep_rt = pipeline_rt.clone(); + let resolve_rt = pipeline_rt.clone(); + let commit_rt = pipeline_rt; + ReceiverStream::new(frame_rx) + .then(move |frame| prep_stage(frame, prep_rt.clone())) + .map(move |staged| resolve_stage(staged, resolve_rt.clone())) + .buffered(parallelism) + .for_each(move |staged| commit_stage(staged, commit_rt.clone(), parallelism)) + .await; + }); + + let mk = |id: u64, idx: u64, repo_rkey: &Rkey| -> HydrantFrame { + parse_frame(json!({ + "id": id, + "type": "record", + "record": { + "live": false, + "did": "did:plc:olaren", + "rev": fresh_tid().as_str(), + "collection": "sh.tangled.feed.star", + "rkey": format!("starrkeya{idx:04}"), + "action": "create", + "cid": VALID_CID, + "record": { + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subject": format!("at://did:plc:nel/sh.tangled.repo/{}", repo_rkey.as_ref()), + } + } + })) + }; + + frame_tx.send(mk(1, 1, &slow_rkeys[0])).await.unwrap(); + frame_tx.send(mk(2, 2, &fast_rkeys[0])).await.unwrap(); + frame_tx.send(mk(3, 3, &slow_rkeys[1])).await.unwrap(); + frame_tx.send(mk(4, 4, &fast_rkeys[1])).await.unwrap(); + drop(frame_tx); + + pipeline.await.unwrap(); + + let captured = capturing.urls.lock().unwrap(); + let rkeys: Vec = captured + .iter() + .filter_map(|uri| uri.rkey().map(|r| r.as_ref().to_owned())) + .collect(); + assert_eq!( + rkeys, + vec![ + "starrkeya0001".to_owned(), + "starrkeya0002".to_owned(), + "starrkeya0003".to_owned(), + "starrkeya0004".to_owned(), + ], + "buffered(N) must preserve cursor order even when resolves complete out of order, with ~150ms slow vs cache-hit fast as the latency skew here", + ); + assert_eq!(runtime.coverage.snapshot().last_cursor().raw(), 4); + } +} diff --git a/bobbin/crates/ingest/src/resolver.rs b/bobbin/crates/ingest/src/resolver.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/ingest/src/resolver.rs @@ -0,0 +1,1 @@ +pub use bobbin_resolver::*; diff --git a/bobbin/crates/ingest/src/shadow.rs b/bobbin/crates/ingest/src/shadow.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/ingest/src/shadow.rs @@ -0,0 +1,172 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +use bobbin_runtime::RuntimeHasher; +use bobbin_types::ids::RepoIdent; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::recordkey::Rkey; +use scc::HashMap as SccMap; + +pub struct WarmingShadowBuffer { + pending: SccMap, + enqueued_total: AtomicU64, + drained_via_observe_total: AtomicU64, + max_concurrent: AtomicU64, + current_concurrent: AtomicU64, + distinct_keys_seen: AtomicU64, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct WarmingShadowSnapshot { + pub enqueued_total: u64, + pub drained_via_observe_total: u64, + pub max_concurrent: u64, + pub residual: u64, + pub distinct_keys_seen: u64, +} + +impl WarmingShadowBuffer { + pub fn new(hasher: RuntimeHasher) -> Self { + Self { + pending: SccMap::with_hasher(hasher), + enqueued_total: AtomicU64::new(0), + drained_via_observe_total: AtomicU64::new(0), + max_concurrent: AtomicU64::new(0), + current_concurrent: AtomicU64::new(0), + distinct_keys_seen: AtomicU64::new(0), + } + } + + pub async fn note_unresolved(&self, owner: Did, rkey: Rkey) { + let key = RepoIdent::new(owner, rkey); + let mut entry = self.pending.entry_async(key).await.or_insert_with(|| { + self.distinct_keys_seen.fetch_add(1, Ordering::Relaxed); + 0 + }); + *entry.get_mut() += 1; + self.enqueued_total.fetch_add(1, Ordering::Relaxed); + let cur = self.current_concurrent.fetch_add(1, Ordering::Relaxed) + 1; + self.max_concurrent.fetch_max(cur, Ordering::Relaxed); + } + + pub async fn note_observed(&self, owner: &Did, rkey: &Rkey) { + let key = RepoIdent::new(owner.clone(), rkey.clone()); + let Some(mut entry) = self.pending.get_async(&key).await else { + return; + }; + let count = std::mem::replace(entry.get_mut(), 0); + if count > 0 { + self.drained_via_observe_total + .fetch_add(count, Ordering::Relaxed); + self.current_concurrent.fetch_sub(count, Ordering::Relaxed); + } + } + + pub fn snapshot(&self) -> WarmingShadowSnapshot { + WarmingShadowSnapshot { + enqueued_total: self.enqueued_total.load(Ordering::Relaxed), + drained_via_observe_total: self.drained_via_observe_total.load(Ordering::Relaxed), + max_concurrent: self.max_concurrent.load(Ordering::Relaxed), + residual: self.current_concurrent.load(Ordering::Relaxed), + distinct_keys_seen: self.distinct_keys_seen.load(Ordering::Relaxed), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use jacquard_common::types::did::Did; + use jacquard_common::types::recordkey::Rkey; + + fn d(s: &str) -> Did { + Did::new_owned(s).unwrap() + } + + fn r(s: &str) -> Rkey { + Rkey::new_owned(s).unwrap() + } + + #[tokio::test] + async fn enqueue_then_observe_drains_to_zero() { + let shadow = WarmingShadowBuffer::new(RuntimeHasher::default()); + shadow + .note_unresolved(d("did:plc:nel"), r("abcabcabcabcz")) + .await; + shadow + .note_unresolved(d("did:plc:nel"), r("abcabcabcabcz")) + .await; + shadow + .note_observed(&d("did:plc:nel"), &r("abcabcabcabcz")) + .await; + let s = shadow.snapshot(); + assert_eq!(s.enqueued_total, 2); + assert_eq!(s.drained_via_observe_total, 2); + assert_eq!(s.max_concurrent, 2); + assert_eq!(s.residual, 0); + assert_eq!(s.distinct_keys_seen, 1); + } + + #[tokio::test] + async fn distinct_keys_tracked_separately() { + let shadow = WarmingShadowBuffer::new(RuntimeHasher::default()); + shadow + .note_unresolved(d("did:plc:nel"), r("abcabcabcabcz")) + .await; + shadow + .note_unresolved(d("did:plc:olaren"), r("abcabcabcabd1")) + .await; + let s = shadow.snapshot(); + assert_eq!(s.enqueued_total, 2); + assert_eq!(s.distinct_keys_seen, 2); + assert_eq!(s.max_concurrent, 2); + assert_eq!(s.residual, 2); + } + + #[tokio::test] + async fn observe_without_prior_enqueue_is_a_noop() { + let shadow = WarmingShadowBuffer::new(RuntimeHasher::default()); + shadow + .note_observed(&d("did:plc:nel"), &r("abcabcabcabcz")) + .await; + let s = shadow.snapshot(); + assert_eq!(s.enqueued_total, 0); + assert_eq!(s.drained_via_observe_total, 0); + assert_eq!(s.distinct_keys_seen, 0); + } + + #[tokio::test] + async fn second_observe_after_drain_does_not_double_count() { + let shadow = WarmingShadowBuffer::new(RuntimeHasher::default()); + shadow + .note_unresolved(d("did:plc:nel"), r("abcabcabcabcz")) + .await; + shadow + .note_observed(&d("did:plc:nel"), &r("abcabcabcabcz")) + .await; + shadow + .note_observed(&d("did:plc:nel"), &r("abcabcabcabcz")) + .await; + let s = shadow.snapshot(); + assert_eq!(s.drained_via_observe_total, 1); + assert_eq!(s.residual, 0); + } + + #[tokio::test] + async fn residual_reflects_unobserved_keys() { + let shadow = WarmingShadowBuffer::new(RuntimeHasher::default()); + shadow + .note_unresolved(d("did:plc:nel"), r("abcabcabcabcz")) + .await; + shadow + .note_unresolved(d("did:plc:olaren"), r("abcabcabcabd1")) + .await; + shadow + .note_observed(&d("did:plc:nel"), &r("abcabcabcabcz")) + .await; + let s = shadow.snapshot(); + assert_eq!(s.enqueued_total, 2); + assert_eq!(s.drained_via_observe_total, 1); + assert_eq!(s.residual, 1); + } +} diff --git a/bobbin/crates/ingest/src/warming.rs b/bobbin/crates/ingest/src/warming.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/ingest/src/warming.rs @@ -0,0 +1,526 @@ +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +use bobbin_edge_index::HydrantCursor; +use bobbin_runtime::RuntimeHasher; +use bobbin_types::edges::{Edge, Record}; +use bobbin_types::ids::RepoIdent; +use bytes::Bytes; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::recordkey::Rkey; +use jacquard_common::types::string::{AtUri, Cid}; +use scc::HashMap as SccMap; +use scc::hash_map::Entry; + +pub struct ParkedUpsert { + pub cursor: HydrantCursor, + pub source: AtUri, + pub nsid: Nsid, + pub parsed: Record, + pub bytes: Bytes, + pub cid: Option>, + pub edges: Vec, +} + +struct EntryState { + upsert: ParkedUpsert, + deps: Vec, +} + +type EntryHandle = Arc>>; + +pub struct WarmingBuffer { + by_source: SccMap, EntryHandle, RuntimeHasher>, + by_key: SccMap, RuntimeHasher>, + hasher: RuntimeHasher, + sealed: AtomicBool, + active_parks: AtomicU64, + enqueued_total: AtomicU64, + drained_observe_total: AtomicU64, + drained_promote_total: AtomicU64, + evicted_total: AtomicU64, + rejected_after_seal: AtomicU64, + distinct_keys_seen: AtomicU64, + current_entries: AtomicU64, + max_concurrent_entries: AtomicU64, + dep_enqueued_total: AtomicU64, + dep_drained_observe_total: AtomicU64, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct WarmingBufferSnapshot { + pub enqueued_total: u64, + pub drained_observe_total: u64, + pub drained_promote_total: u64, + pub evicted_total: u64, + pub rejected_after_seal: u64, + pub distinct_keys_seen: u64, + pub current_entries: u64, + pub max_concurrent_entries: u64, + pub dep_enqueued_total: u64, + pub dep_drained_observe_total: u64, +} + +impl WarmingBuffer { + pub fn new(hasher: RuntimeHasher) -> Self { + Self { + by_source: SccMap::with_hasher(hasher.clone()), + by_key: SccMap::with_hasher(hasher.clone()), + hasher, + sealed: AtomicBool::new(false), + active_parks: AtomicU64::new(0), + enqueued_total: AtomicU64::new(0), + drained_observe_total: AtomicU64::new(0), + drained_promote_total: AtomicU64::new(0), + evicted_total: AtomicU64::new(0), + rejected_after_seal: AtomicU64::new(0), + distinct_keys_seen: AtomicU64::new(0), + current_entries: AtomicU64::new(0), + max_concurrent_entries: AtomicU64::new(0), + dep_enqueued_total: AtomicU64::new(0), + dep_drained_observe_total: AtomicU64::new(0), + } + } + + pub fn hasher(&self) -> &RuntimeHasher { + &self.hasher + } + + pub fn is_sealed(&self) -> bool { + self.sealed.load(Ordering::Acquire) + } + + pub async fn try_park( + &self, + upsert: ParkedUpsert, + deps: Vec, + ) -> Result<(), ParkedUpsert> { + self.active_parks.fetch_add(1, Ordering::AcqRel); + let outcome = if self.is_sealed() { + self.rejected_after_seal.fetch_add(1, Ordering::Relaxed); + Err(upsert) + } else { + self.park(upsert, deps).await; + Ok(()) + }; + self.active_parks.fetch_sub(1, Ordering::Release); + outcome + } + + async fn park(&self, upsert: ParkedUpsert, deps: Vec) { + debug_assert!( + !deps.is_empty(), + "park requires at least one unresolved dep" + ); + let dep_count = deps.len() as u64; + let source = upsert.source.clone(); + let handle: EntryHandle = Arc::new(Mutex::new(Some(EntryState { + upsert, + deps: deps.clone(), + }))); + + let prior = match self.by_source.entry_async(source).await { + Entry::Occupied(mut occ) => Some(occ.insert(handle.clone())), + Entry::Vacant(vac) => { + vac.insert_entry(handle.clone()); + None + } + }; + if let Some(prior) = prior { + let prior_deps = self.snapshot_deps(&prior); + if self.deactivate_handle(&prior) { + self.evicted_total.fetch_add(1, Ordering::Relaxed); + self.cleanup_by_key(&prior, &prior_deps).await; + } + } + + let _ = futures::future::join_all(deps.into_iter().map(|dep| { + let h = handle.clone(); + async move { self.register_dep(dep, h).await } + })) + .await; + + self.enqueued_total.fetch_add(1, Ordering::Relaxed); + self.dep_enqueued_total + .fetch_add(dep_count, Ordering::Relaxed); + let cur = self.current_entries.fetch_add(1, Ordering::Relaxed) + 1; + self.max_concurrent_entries + .fetch_max(cur, Ordering::Relaxed); + } + + async fn register_dep(&self, dep: RepoIdent, handle: EntryHandle) { + let was_new = match self.by_key.entry_async(dep).await { + Entry::Occupied(mut occ) => { + occ.get_mut().push(handle); + false + } + Entry::Vacant(vac) => { + vac.insert_entry(vec![handle]); + true + } + }; + if was_new { + self.distinct_keys_seen.fetch_add(1, Ordering::Relaxed); + } + } + + pub async fn evict_source(&self, source: &AtUri) -> bool { + let Some((_, handle)) = self.by_source.remove_async(source).await else { + return false; + }; + let deps = self.snapshot_deps(&handle); + if !self.deactivate_handle(&handle) { + return false; + } + self.evicted_total.fetch_add(1, Ordering::Relaxed); + self.cleanup_by_key(&handle, &deps).await; + true + } + + pub async fn take_observed( + &self, + owner: &Did, + rkey: &Rkey, + ) -> Vec { + let park_key = RepoIdent::new(owner.clone(), rkey.clone()); + let Some((_, handles)) = self.by_key.remove_async(&park_key).await else { + return Vec::new(); + }; + let mut drained: Vec = Vec::new(); + let mut sources_to_remove: Vec> = Vec::new(); + let mut dep_matches: u64 = 0; + handles.into_iter().for_each(|handle| { + let outcome = { + let mut slot = handle.lock().expect("warming buffer entry mutex poisoned"); + match slot.as_mut() { + Some(state) => { + let before = state.deps.len(); + state.deps.retain(|d| d != &park_key); + let after = state.deps.len(); + Some((before > after, after == 0)) + } + None => None, + } + }; + let Some((matched, drain_now)) = outcome else { + return; + }; + if matched { + dep_matches += 1; + } + if drain_now { + let mut slot = handle.lock().expect("warming buffer entry mutex poisoned"); + if let Some(state) = slot.take() { + sources_to_remove.push(state.upsert.source.clone()); + drained.push(state.upsert); + } + } + }); + if dep_matches > 0 { + self.dep_drained_observe_total + .fetch_add(dep_matches, Ordering::Relaxed); + } + let drained_count = drained.len() as u64; + if drained_count > 0 { + self.drained_observe_total + .fetch_add(drained_count, Ordering::Relaxed); + self.current_entries + .fetch_sub(drained_count, Ordering::Relaxed); + let _ = futures::future::join_all( + sources_to_remove + .into_iter() + .map(|source| async move { self.by_source.remove_async(&source).await }), + ) + .await; + } + drained + } + + pub async fn drain_for_promote(&self) -> Vec<(ParkedUpsert, Vec)> { + self.sealed.store(true, Ordering::Release); + while self.active_parks.load(Ordering::Acquire) > 0 { + tokio::task::yield_now().await; + } + let mut drained: Vec<(ParkedUpsert, Vec)> = Vec::new(); + self.by_source + .retain_async(|_source, handle| { + let mut slot = handle.lock().expect("warming buffer entry mutex poisoned"); + if let Some(state) = slot.take() { + drained.push((state.upsert, state.deps)); + } + false + }) + .await; + let count = drained.len() as u64; + if count > 0 { + self.drained_promote_total + .fetch_add(count, Ordering::Relaxed); + self.current_entries.fetch_sub(count, Ordering::Relaxed); + } + self.by_key.clear_async().await; + drained + } + + pub fn snapshot(&self) -> WarmingBufferSnapshot { + WarmingBufferSnapshot { + enqueued_total: self.enqueued_total.load(Ordering::Relaxed), + drained_observe_total: self.drained_observe_total.load(Ordering::Relaxed), + drained_promote_total: self.drained_promote_total.load(Ordering::Relaxed), + evicted_total: self.evicted_total.load(Ordering::Relaxed), + rejected_after_seal: self.rejected_after_seal.load(Ordering::Relaxed), + distinct_keys_seen: self.distinct_keys_seen.load(Ordering::Relaxed), + current_entries: self.current_entries.load(Ordering::Relaxed), + max_concurrent_entries: self.max_concurrent_entries.load(Ordering::Relaxed), + dep_enqueued_total: self.dep_enqueued_total.load(Ordering::Relaxed), + dep_drained_observe_total: self.dep_drained_observe_total.load(Ordering::Relaxed), + } + } + + fn snapshot_deps(&self, handle: &EntryHandle) -> Vec { + let slot = handle.lock().expect("warming buffer entry mutex poisoned"); + slot.as_ref().map(|s| s.deps.clone()).unwrap_or_default() + } + + fn deactivate_handle(&self, handle: &EntryHandle) -> bool { + let mut slot = handle.lock().expect("warming buffer entry mutex poisoned"); + if slot.take().is_some() { + self.current_entries.fetch_sub(1, Ordering::Relaxed); + true + } else { + false + } + } + + async fn cleanup_by_key(&self, handle: &EntryHandle, deps: &[RepoIdent]) { + let _ = futures::future::join_all(deps.iter().map(|dep| async move { + let Some(mut occupied) = self.by_key.get_async(dep).await else { + return; + }; + occupied.get_mut().retain(|h| !Arc::ptr_eq(h, handle)); + if occupied.get().is_empty() { + let _ = occupied.remove_entry(); + } + })) + .await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bobbin_types::ids::nsid_static; + + fn d(s: &str) -> Did { + Did::new_owned(s).unwrap() + } + + fn r(s: &str) -> Rkey { + Rkey::new_owned(s).unwrap() + } + + fn at(s: &str) -> AtUri { + AtUri::new_owned(s).unwrap() + } + + fn make_upsert(source: &str) -> ParkedUpsert { + let source_uri = at(source); + let star_json = br#"{"$type":"sh.tangled.feed.star","createdAt":"2026-05-01T00:00:00Z","subject":{"$type":"sh.tangled.feed.star#repo","did":"did:plc:abalone"}}"#; + let parsed = Record::from_json_bytes(&nsid_static("sh.tangled.feed.star"), star_json) + .expect("star fixture parses"); + ParkedUpsert { + cursor: HydrantCursor::new(1), + source: source_uri, + nsid: nsid_static("sh.tangled.feed.star"), + parsed, + bytes: Bytes::from_static(star_json), + cid: None, + edges: Vec::new(), + } + } + + #[tokio::test] + async fn park_then_observe_drains_to_zero() { + let hasher = RuntimeHasher::default(); + let buf = WarmingBuffer::new(hasher); + let dep = RepoIdent::new(d("did:plc:nel"), r("abcabcabcabcz")); + let upsert = make_upsert("at://did:plc:starer/sh.tangled.feed.star/zzzzzzzzzzzzz"); + buf.park(upsert, vec![dep.clone()]).await; + let drained = buf.take_observed(&dep.owner, &dep.rkey).await; + assert_eq!(drained.len(), 1); + let s = buf.snapshot(); + assert_eq!(s.enqueued_total, 1); + assert_eq!(s.drained_observe_total, 1); + assert_eq!(s.distinct_keys_seen, 1); + assert_eq!(s.current_entries, 0); + assert_eq!(s.max_concurrent_entries, 1); + assert_eq!(s.dep_enqueued_total, 1); + assert_eq!(s.dep_drained_observe_total, 1); + } + + #[tokio::test] + async fn distinct_keys_tracked_separately() { + let hasher = RuntimeHasher::default(); + let buf = WarmingBuffer::new(hasher); + let dep_a = RepoIdent::new(d("did:plc:nel"), r("abcabcabcabcz")); + let dep_b = RepoIdent::new(d("did:plc:olaren"), r("abcabcabcabd1")); + buf.park( + make_upsert("at://did:plc:starer1/sh.tangled.feed.star/aaaaaaaaaaaaa"), + vec![dep_a.clone()], + ) + .await; + buf.park( + make_upsert("at://did:plc:starer2/sh.tangled.feed.star/bbbbbbbbbbbbb"), + vec![dep_b.clone()], + ) + .await; + let s = buf.snapshot(); + assert_eq!(s.enqueued_total, 2); + assert_eq!(s.distinct_keys_seen, 2); + assert_eq!(s.current_entries, 2); + assert_eq!(s.max_concurrent_entries, 2); + assert_eq!(s.dep_enqueued_total, 2); + } + + #[tokio::test] + async fn observe_unrelated_key_is_noop() { + let hasher = RuntimeHasher::default(); + let buf = WarmingBuffer::new(hasher); + buf.park( + make_upsert("at://did:plc:starer/sh.tangled.feed.star/zzzzzzzzzzzzz"), + vec![RepoIdent::new(d("did:plc:nel"), r("abcabcabcabcz"))], + ) + .await; + let drained = buf + .take_observed(&d("did:plc:olaren"), &r("nopenopenopep")) + .await; + assert!(drained.is_empty()); + let s = buf.snapshot(); + assert_eq!(s.current_entries, 1); + assert_eq!(s.drained_observe_total, 0); + assert_eq!(s.dep_drained_observe_total, 0); + } + + #[tokio::test] + async fn evict_on_replacement_clears_prior_entry() { + let hasher = RuntimeHasher::default(); + let buf = WarmingBuffer::new(hasher); + let dep = RepoIdent::new(d("did:plc:nel"), r("abcabcabcabcz")); + let source = "at://did:plc:starer/sh.tangled.feed.star/zzzzzzzzzzzzz"; + buf.park(make_upsert(source), vec![dep.clone()]).await; + let evicted = buf.evict_source(&at(source)).await; + assert!(evicted); + let drained = buf.take_observed(&dep.owner, &dep.rkey).await; + assert!( + drained.is_empty(), + "evicted entry must not surface on observe", + ); + let s = buf.snapshot(); + assert_eq!(s.current_entries, 0); + assert_eq!(s.evicted_total, 1); + } + + #[tokio::test] + async fn evict_clears_by_key_so_observe_walks_no_handles() { + let hasher = RuntimeHasher::default(); + let buf = WarmingBuffer::new(hasher); + let dep_a = RepoIdent::new(d("did:plc:nel"), r("abcabcabcabcz")); + let dep_b = RepoIdent::new(d("did:plc:olaren"), r("abcabcabcabd1")); + let source = "at://did:plc:starer/sh.tangled.feed.star/zzzzzzzzzzzzz"; + buf.park(make_upsert(source), vec![dep_a.clone(), dep_b.clone()]) + .await; + assert!(buf.evict_source(&at(source)).await); + let drained_a = buf.take_observed(&dep_a.owner, &dep_a.rkey).await; + let drained_b = buf.take_observed(&dep_b.owner, &dep_b.rkey).await; + assert!(drained_a.is_empty()); + assert!(drained_b.is_empty()); + let s = buf.snapshot(); + assert_eq!(s.dep_drained_observe_total, 0); + } + + #[tokio::test] + async fn replacement_park_under_same_source_increments_evicted_total() { + let hasher = RuntimeHasher::default(); + let buf = WarmingBuffer::new(hasher); + let dep = RepoIdent::new(d("did:plc:nel"), r("abcabcabcabcz")); + let source = "at://did:plc:starer/sh.tangled.feed.star/zzzzzzzzzzzzz"; + buf.park(make_upsert(source), vec![dep.clone()]).await; + buf.park(make_upsert(source), vec![dep.clone()]).await; + let s = buf.snapshot(); + assert_eq!(s.enqueued_total, 2); + assert_eq!(s.evicted_total, 1); + assert_eq!(s.current_entries, 1); + } + + #[tokio::test] + async fn replacement_park_does_not_double_drain_on_observe() { + let hasher = RuntimeHasher::default(); + let buf = WarmingBuffer::new(hasher); + let dep = RepoIdent::new(d("did:plc:nel"), r("abcabcabcabcz")); + let source = "at://did:plc:starer/sh.tangled.feed.star/zzzzzzzzzzzzz"; + buf.park(make_upsert(source), vec![dep.clone()]).await; + buf.park(make_upsert(source), vec![dep.clone()]).await; + let drained = buf.take_observed(&dep.owner, &dep.rkey).await; + assert_eq!( + drained.len(), + 1, + "evicted handle must not surface alongside the live one", + ); + let s = buf.snapshot(); + assert_eq!(s.drained_observe_total, 1); + assert_eq!(s.current_entries, 0); + } + + #[tokio::test] + async fn drain_for_promote_returns_residual() { + let hasher = RuntimeHasher::default(); + let buf = WarmingBuffer::new(hasher); + let dep = RepoIdent::new(d("did:plc:nel"), r("abcabcabcabcz")); + buf.park( + make_upsert("at://did:plc:starer/sh.tangled.feed.star/zzzzzzzzzzzzz"), + vec![dep], + ) + .await; + let residual = buf.drain_for_promote().await; + assert_eq!(residual.len(), 1); + let s = buf.snapshot(); + assert_eq!(s.drained_promote_total, 1); + assert_eq!(s.current_entries, 0); + } + + #[tokio::test] + async fn try_park_after_drain_for_promote_is_rejected() { + let hasher = RuntimeHasher::default(); + let buf = WarmingBuffer::new(hasher); + let dep = RepoIdent::new(d("did:plc:nel"), r("abcabcabcabcz")); + let _ = buf.drain_for_promote().await; + let upsert = make_upsert("at://did:plc:starer/sh.tangled.feed.star/zzzzzzzzzzzzz"); + let res = buf.try_park(upsert, vec![dep]).await; + assert!(res.is_err(), "post-seal try_park must reject"); + let s = buf.snapshot(); + assert_eq!(s.rejected_after_seal, 1); + assert_eq!(s.enqueued_total, 0); + assert_eq!(s.current_entries, 0); + } + + #[tokio::test] + async fn double_observe_after_drain_does_not_double_count() { + let hasher = RuntimeHasher::default(); + let buf = WarmingBuffer::new(hasher); + let dep = RepoIdent::new(d("did:plc:nel"), r("abcabcabcabcz")); + buf.park( + make_upsert("at://did:plc:starer/sh.tangled.feed.star/zzzzzzzzzzzzz"), + vec![dep.clone()], + ) + .await; + let first = buf.take_observed(&dep.owner, &dep.rkey).await; + let second = buf.take_observed(&dep.owner, &dep.rkey).await; + assert_eq!(first.len(), 1); + assert!(second.is_empty()); + let s = buf.snapshot(); + assert_eq!(s.drained_observe_total, 1); + assert_eq!(s.dep_drained_observe_total, 1); + } +} diff --git a/bobbin/crates/knot-proxy/Cargo.toml b/bobbin/crates/knot-proxy/Cargo.toml new file mode 100644 --- /dev/null +++ b/bobbin/crates/knot-proxy/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "bobbin-knot-proxy" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-runtime = { workspace = true } +bytes = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +jacquard-common = { workspace = true } +reqwest = { workspace = true } +scc = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["net"] } +url = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "time"] } +wiremock = { workspace = true } diff --git a/bobbin/crates/knot-proxy/src/breaker.rs b/bobbin/crates/knot-proxy/src/breaker.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/knot-proxy/src/breaker.rs @@ -0,0 +1,272 @@ +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bobbin_runtime::Clock; +use thiserror::Error; +use tokio::time::Instant; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct FailureThreshold(u32); + +#[derive(Clone, Copy, Debug, Error)] +#[error("failure threshold must be at least 1")] +pub struct ThresholdError; + +impl FailureThreshold { + pub const fn new(n: u32) -> Result { + match n { + 0 => Err(ThresholdError), + other => Ok(Self(other)), + } + } + + pub const fn get(self) -> u32 { + self.0 + } +} + +#[derive(Clone, Copy, Debug)] +enum BreakerState { + Closed { failures: u32 }, + Open { until: Instant }, + HalfOpen, +} + +#[derive(Clone, Copy, Debug, Error)] +#[error("circuit breaker open")] +pub struct CircuitOpen; + +pub struct Breaker { + state: Mutex, + threshold: FailureThreshold, + cooldown: Duration, + clock: Arc, +} + +impl std::fmt::Debug for Breaker { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Breaker") + .field("state", &self.state) + .field("threshold", &self.threshold) + .field("cooldown", &self.cooldown) + .finish_non_exhaustive() + } +} + +impl Breaker { + pub fn new(threshold: FailureThreshold, cooldown: Duration, clock: Arc) -> Self { + Self { + state: Mutex::new(BreakerState::Closed { failures: 0 }), + threshold, + cooldown, + clock, + } + } + + pub fn try_acquire(self: &Arc) -> Result { + self.try_acquire_at(self.clock.now_instant()) + } + + fn try_acquire_at(self: &Arc, now: Instant) -> Result { + let mut state = self.state.lock().expect("breaker mutex poisoned"); + match *state { + BreakerState::Closed { .. } => Ok(BreakerPermit::new(Arc::clone(self))), + BreakerState::Open { until } if now >= until => { + *state = BreakerState::HalfOpen; + Ok(BreakerPermit::new(Arc::clone(self))) + } + BreakerState::Open { .. } | BreakerState::HalfOpen => Err(CircuitOpen), + } + } + + pub fn record_success(&self) { + let mut state = self.state.lock().expect("breaker mutex poisoned"); + *state = BreakerState::Closed { failures: 0 }; + } + + pub fn record_failure(&self) { + self.record_failure_at(self.clock.now_instant()); + } + + fn record_failure_at(&self, now: Instant) { + let mut state = self.state.lock().expect("breaker mutex poisoned"); + let next = match *state { + BreakerState::Closed { failures } => { + let bumped = failures.saturating_add(1); + if bumped >= self.threshold.get() { + BreakerState::Open { + until: now + self.cooldown, + } + } else { + BreakerState::Closed { failures: bumped } + } + } + BreakerState::HalfOpen => BreakerState::Open { + until: now + self.cooldown, + }, + BreakerState::Open { .. } => *state, + }; + *state = next; + } +} + +#[must_use = "permit must outlive the upstream call so failures can be recorded"] +#[derive(Debug)] +pub struct BreakerPermit { + breaker: Arc, + resolved: bool, +} + +impl BreakerPermit { + fn new(breaker: Arc) -> Self { + Self { + breaker, + resolved: false, + } + } + + pub fn record_success(mut self) { + self.resolved = true; + self.breaker.record_success(); + } + + pub fn record_failure(mut self) { + self.resolved = true; + self.breaker.record_failure(); + } +} + +impl Drop for BreakerPermit { + fn drop(&mut self) { + if !self.resolved { + self.breaker.record_success(); + } + } +} + +#[cfg(test)] +impl BreakerPermit { + fn record_failure_at(mut self, now: Instant) { + self.resolved = true; + self.breaker.record_failure_at(now); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bobbin_runtime::SystemClock; + + fn breaker(threshold: u32, cooldown_ms: u64) -> Arc { + Arc::new(Breaker::new( + FailureThreshold::new(threshold).unwrap(), + Duration::from_millis(cooldown_ms), + Arc::new(SystemClock::new()), + )) + } + + #[test] + fn closed_breaker_admits_all() { + let b = breaker(3, 100); + b.try_acquire().unwrap().record_success(); + b.try_acquire().unwrap().record_success(); + b.try_acquire().unwrap().record_success(); + } + + #[test] + fn opens_at_threshold() { + let b = breaker(2, 1_000); + b.record_failure(); + b.record_failure(); + assert!(b.try_acquire().is_err(), "must be open after 2 failures"); + } + + #[test] + fn success_resets_failure_count() { + let b = breaker(2, 1_000); + b.record_failure(); + b.record_success(); + b.record_failure(); + b.try_acquire() + .expect("successful run between failures must reset count") + .record_success(); + } + + #[test] + fn cooldown_admits_one_trial_only() { + let b = breaker(1, 50); + let t0 = Instant::now(); + let after = t0 + Duration::from_millis(60); + b.record_failure_at(t0); + assert!(b.try_acquire_at(t0).is_err()); + let trial = b + .try_acquire_at(after) + .expect("trial admitted after cooldown"); + assert!( + b.try_acquire_at(after).is_err(), + "second concurrent half-open call must be rejected", + ); + trial.record_success(); + } + + #[test] + fn half_open_failure_reopens() { + let b = breaker(1, 50); + let t0 = Instant::now(); + let after = t0 + Duration::from_millis(60); + b.record_failure_at(t0); + b.try_acquire_at(after) + .expect("trial admitted") + .record_failure_at(after); + assert!( + b.try_acquire_at(after).is_err(), + "half-open failure must reopen breaker", + ); + } + + #[test] + fn half_open_success_closes() { + let b = breaker(1, 50); + let t0 = Instant::now(); + let after = t0 + Duration::from_millis(60); + b.record_failure_at(t0); + b.try_acquire_at(after) + .expect("trial admitted") + .record_success(); + b.try_acquire_at(after).expect("closed").record_success(); + b.try_acquire_at(after).expect("closed").record_success(); + } + + #[test] + fn open_failure_does_not_extend_cooldown_indefinitely() { + let b = breaker(1, 50); + b.record_failure(); + let mid = Instant::now(); + b.record_failure(); + b.try_acquire_at(mid + Duration::from_millis(60)) + .expect("second failure while open must not push cooldown out") + .record_success(); + } + + #[test] + fn dropped_permit_counts_as_success() { + let b = breaker(2, 1_000); + b.record_failure(); + drop(b.try_acquire().expect("admitted")); + b.try_acquire() + .expect("dropped permit must reset failure count") + .record_success(); + } + + #[test] + fn dropped_half_open_permit_closes_breaker() { + let b = breaker(1, 50); + let t0 = Instant::now(); + let after = t0 + Duration::from_millis(60); + b.record_failure_at(t0); + drop(b.try_acquire_at(after).expect("trial admitted")); + b.try_acquire_at(after) + .expect("dropped half-open permit must close breaker") + .record_success(); + } +} diff --git a/bobbin/crates/knot-proxy/src/dns.rs b/bobbin/crates/knot-proxy/src/dns.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/knot-proxy/src/dns.rs @@ -0,0 +1,109 @@ +use std::error::Error as StdError; +use std::fmt; +use std::io; +use std::net::SocketAddr; + +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; + +use crate::host::{PrivateHostReason, classify_ip}; + +pub(crate) struct PrivateAddressFilter { + allow_private: bool, +} + +impl PrivateAddressFilter { + pub(crate) fn new(allow_private: bool) -> Self { + Self { allow_private } + } +} + +impl Resolve for PrivateAddressFilter { + fn resolve(&self, name: Name) -> Resolving { + let allow_private = self.allow_private; + let host = name.as_str().to_owned(); + Box::pin(async move { + let resolved: Vec = + tokio::net::lookup_host((host.as_str(), 0)).await?.collect(); + partition_safe(host, allow_private, resolved) + }) + } +} + +fn partition_safe( + host: String, + allow_private: bool, + resolved: Vec, +) -> Result> { + if !allow_private && let Some(reason) = resolved.iter().find_map(|sa| classify_ip(&sa.ip())) { + return Err(Box::new(BlockedAddressError { host, reason })); + } + if resolved.is_empty() { + return Err(Box::new(io::Error::other(format!( + "no resolvable addresses for {host}" + )))); + } + Ok(Box::new(resolved.into_iter())) +} + +#[derive(Debug)] +pub(crate) struct BlockedAddressError { + host: String, + reason: PrivateHostReason, +} + +impl fmt::Display for BlockedAddressError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "dns resolution for {} returned {} address", + self.host, self.reason, + ) + } +} + +impl StdError for BlockedAddressError {} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::{IpAddr, Ipv4Addr}; + + fn sa(ip: &str, port: u16) -> SocketAddr { + SocketAddr::new(ip.parse().unwrap(), port) + } + + #[test] + fn blocks_when_any_resolved_address_is_private_under_strict() { + let mixed = vec![sa("8.8.8.8", 0), sa("127.0.0.1", 0)]; + let res = partition_safe("mixed.example".into(), false, mixed); + assert!(res.is_err(), "any private address must fail strict resolve"); + } + + #[test] + fn allows_all_when_permissive() { + let mixed = vec![sa("8.8.8.8", 0), sa("127.0.0.1", 0)]; + let res = partition_safe("mixed.example".into(), true, mixed).expect("permissive"); + let collected: Vec = res.collect(); + assert_eq!(collected.len(), 2); + } + + #[test] + fn permits_public_only_resolution_under_strict() { + let public = vec![sa("8.8.8.8", 0), sa("1.1.1.1", 0)]; + let res = partition_safe("public.example".into(), false, public).expect("public"); + let collected: Vec = res.collect(); + assert_eq!(collected.len(), 2); + } + + #[test] + fn empty_resolution_is_an_error() { + let res = partition_safe("nx.example".into(), false, vec![]); + assert!(res.is_err(), "empty address list must surface an error"); + } + + #[test] + fn classify_ip_matches_url_classifier() { + assert!(classify_ip(&IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))).is_some()); + assert!(classify_ip(&IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8))).is_none()); + } +} diff --git a/bobbin/crates/knot-proxy/src/host.rs b/bobbin/crates/knot-proxy/src/host.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/knot-proxy/src/host.rs @@ -0,0 +1,403 @@ +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +use jacquard_common::BosStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::nsid::Nsid; +use thiserror::Error; +use url::{Host, Url}; + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct KnotHost(Url); + +#[derive(Clone, Debug, Error)] +pub enum KnotHostError { + #[error("empty host")] + Empty, + #[error("parse: {0}")] + Parse(url::ParseError), + #[error("scheme must be http or https, got {0}")] + BadScheme(String), + #[error("host has no authority")] + NoAuthority, +} + +impl KnotHost { + pub fn parse(raw: &str) -> Result { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Err(KnotHostError::Empty); + } + let candidate = if trimmed.contains("://") { + trimmed.to_owned() + } else { + format!("https://{trimmed}") + }; + let mut url = Url::parse(&candidate).map_err(KnotHostError::Parse)?; + match url.scheme() { + "http" | "https" => {} + other => return Err(KnotHostError::BadScheme(other.to_owned())), + } + if url.host().is_none() { + return Err(KnotHostError::NoAuthority); + } + url.set_path("/"); + url.set_query(None); + url.set_fragment(None); + let _ = url.set_username(""); + let _ = url.set_password(None); + Ok(Self(url)) + } + + pub fn url(&self) -> &Url { + &self.0 + } + + pub fn xrpc_url>(&self, nsid: &Nsid) -> Url { + let mut url = self.0.clone(); + url.set_path(&format!("/xrpc/{}", nsid.as_ref())); + url + } + + pub fn private_literal_reason(&self) -> Option { + match self.0.host()? { + Host::Ipv4(ip) => classify_v4(ip), + Host::Ipv6(ip) => classify_v6(ip), + Host::Domain(name) if is_loopback_domain(name) => Some(PrivateHostReason::Loopback), + Host::Domain(_) => None, + } + } +} + +pub fn classify_ip(ip: &IpAddr) -> Option { + match ip { + IpAddr::V4(v4) => classify_v4(*v4), + IpAddr::V6(v6) => classify_v6(*v6), + } +} + +fn is_loopback_domain(name: &str) -> bool { + name.eq_ignore_ascii_case("localhost") + || name + .rsplit_once('.') + .is_some_and(|(_, label)| label.eq_ignore_ascii_case("localhost")) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PrivateHostReason { + Loopback, + Private, + LinkLocal, + Unspecified, + Multicast, + Broadcast, + Documentation, + UniqueLocal, +} + +impl std::fmt::Display for PrivateHostReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Loopback => "loopback", + Self::Private => "private", + Self::LinkLocal => "link-local", + Self::Unspecified => "unspecified", + Self::Multicast => "multicast", + Self::Broadcast => "broadcast", + Self::Documentation => "documentation", + Self::UniqueLocal => "unique-local", + }) + } +} + +fn classify_v4(ip: Ipv4Addr) -> Option { + if ip.is_loopback() { + Some(PrivateHostReason::Loopback) + } else if ip.is_private() { + Some(PrivateHostReason::Private) + } else if ip.is_link_local() { + Some(PrivateHostReason::LinkLocal) + } else if ip.is_unspecified() { + Some(PrivateHostReason::Unspecified) + } else if ip.is_broadcast() { + Some(PrivateHostReason::Broadcast) + } else if ip.is_multicast() { + Some(PrivateHostReason::Multicast) + } else if ip.is_documentation() { + Some(PrivateHostReason::Documentation) + } else if is_v4_carrier_grade_nat(ip) { + Some(PrivateHostReason::Private) + } else { + None + } +} + +fn is_v4_carrier_grade_nat(ip: Ipv4Addr) -> bool { + let [a, b, ..] = ip.octets(); + a == 100 && (0x40..=0x7f).contains(&b) +} + +fn classify_v6(ip: Ipv6Addr) -> Option { + if ip.is_loopback() { + Some(PrivateHostReason::Loopback) + } else if ip.is_unspecified() { + Some(PrivateHostReason::Unspecified) + } else if ip.is_multicast() { + Some(PrivateHostReason::Multicast) + } else if is_v6_link_local(ip) { + Some(PrivateHostReason::LinkLocal) + } else if is_v6_unique_local(ip) { + Some(PrivateHostReason::UniqueLocal) + } else if let Some(v4) = ip.to_ipv4_mapped() { + classify_v4(v4) + } else { + None + } +} + +fn is_v6_link_local(ip: Ipv6Addr) -> bool { + ip.segments()[0] & 0xffc0 == 0xfe80 +} + +fn is_v6_unique_local(ip: Ipv6Addr) -> bool { + ip.segments()[0] & 0xfe00 == 0xfc00 +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RepoSlug(String); + +#[derive(Clone, Debug, Error)] +pub enum RepoSlugError { + #[error("empty repo name")] + EmptyName, + #[error("repo name contains slash: {0}")] + NameHasSlash(String), +} + +impl RepoSlug { + pub fn new>(did: &Did, name: &str) -> Result { + if name.is_empty() { + return Err(RepoSlugError::EmptyName); + } + if name.contains('/') { + return Err(RepoSlugError::NameHasSlash(name.to_owned())); + } + Ok(Self(format!("{}/{}", did.as_ref(), name))) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use jacquard_common::DefaultStr; + + fn did(s: &'static str) -> Did { + Did::new_static(s).unwrap() + } + + fn nsid(s: &'static str) -> Nsid { + Nsid::new_static(s).unwrap() + } + + #[test] + fn parses_bare_host_as_https() { + let host = KnotHost::parse("oyster.cafe").unwrap(); + assert_eq!(host.url().as_str(), "https://oyster.cafe/"); + } + + #[test] + fn parses_explicit_scheme() { + let host = KnotHost::parse("http://127.0.0.1:5555").unwrap(); + assert_eq!(host.url().as_str(), "http://127.0.0.1:5555/"); + } + + #[test] + fn strips_path_query_fragment() { + let host = KnotHost::parse("https://nel.pet/some/path?x=1#y").unwrap(); + assert_eq!(host.url().as_str(), "https://nel.pet/"); + } + + #[test] + fn strips_userinfo() { + let host = KnotHost::parse("https://attacker:secret@oyster.cafe/").unwrap(); + assert_eq!(host.url().as_str(), "https://oyster.cafe/"); + assert!(host.url().username().is_empty()); + assert!(host.url().password().is_none()); + } + + #[test] + fn strips_userinfo_only_username() { + let host = KnotHost::parse("https://nel@nel.pet").unwrap(); + assert_eq!(host.url().as_str(), "https://nel.pet/"); + assert!(host.url().username().is_empty()); + } + + #[test] + fn rejects_bad_scheme() { + let err = KnotHost::parse("ftp://oyster.cafe").unwrap_err(); + assert!(matches!(err, KnotHostError::BadScheme(s) if s == "ftp")); + } + + #[test] + fn rejects_empty() { + assert!(matches!( + KnotHost::parse(" ").unwrap_err(), + KnotHostError::Empty, + )); + } + + #[test] + fn xrpc_url_appends_nsid() { + let host = KnotHost::parse("oyster.cafe").unwrap(); + let url = host.xrpc_url(&nsid("sh.tangled.repo.blob")); + assert_eq!( + url.as_str(), + "https://oyster.cafe/xrpc/sh.tangled.repo.blob", + ); + } + + #[test] + fn slug_joins_did_and_name() { + let slug = RepoSlug::new(&did("did:plc:squid"), "barnacle").unwrap(); + assert_eq!(slug.as_str(), "did:plc:squid/barnacle"); + } + + #[test] + fn slug_rejects_slash_in_name() { + assert!(matches!( + RepoSlug::new(&did("did:plc:squid"), "bad/name").unwrap_err(), + RepoSlugError::NameHasSlash(s) if s == "bad/name", + )); + } + + #[test] + fn slug_rejects_empty_name() { + assert!(matches!( + RepoSlug::new(&did("did:plc:squid"), "").unwrap_err(), + RepoSlugError::EmptyName, + )); + } + + #[test] + fn flags_loopback_v4() { + let host = KnotHost::parse("http://127.0.0.1").unwrap(); + assert_eq!( + host.private_literal_reason(), + Some(PrivateHostReason::Loopback), + ); + } + + #[test] + fn flags_private_rfc1918() { + for raw in ["http://10.0.0.1", "http://192.168.1.1", "http://172.16.0.1"] { + let host = KnotHost::parse(raw).unwrap(); + assert_eq!( + host.private_literal_reason(), + Some(PrivateHostReason::Private), + "{raw}", + ); + } + } + + #[test] + fn flags_link_local_v4() { + let host = KnotHost::parse("http://169.254.169.254").unwrap(); + assert_eq!( + host.private_literal_reason(), + Some(PrivateHostReason::LinkLocal), + ); + } + + #[test] + fn flags_carrier_grade_nat() { + let host = KnotHost::parse("http://100.64.0.1").unwrap(); + assert_eq!( + host.private_literal_reason(), + Some(PrivateHostReason::Private), + ); + } + + #[test] + fn flags_loopback_v6() { + let host = KnotHost::parse("http://[::1]").unwrap(); + assert_eq!( + host.private_literal_reason(), + Some(PrivateHostReason::Loopback), + ); + } + + #[test] + fn flags_link_local_v6() { + let host = KnotHost::parse("http://[fe80::1]").unwrap(); + assert_eq!( + host.private_literal_reason(), + Some(PrivateHostReason::LinkLocal), + ); + } + + #[test] + fn flags_unique_local_v6() { + let host = KnotHost::parse("http://[fc00::1]").unwrap(); + assert_eq!( + host.private_literal_reason(), + Some(PrivateHostReason::UniqueLocal), + ); + } + + #[test] + fn flags_v4_mapped_v6() { + let host = KnotHost::parse("http://[::ffff:10.0.0.1]").unwrap(); + assert_eq!( + host.private_literal_reason(), + Some(PrivateHostReason::Private), + ); + } + + #[test] + fn allows_public_v4() { + let host = KnotHost::parse("http://93.184.216.34").unwrap(); + assert_eq!(host.private_literal_reason(), None); + } + + #[test] + fn dns_names_are_not_classified() { + let host = KnotHost::parse("https://oyster.cafe").unwrap(); + assert_eq!(host.private_literal_reason(), None); + } + + #[test] + fn flags_localhost_domain() { + let host = KnotHost::parse("http://localhost").unwrap(); + assert_eq!( + host.private_literal_reason(), + Some(PrivateHostReason::Loopback), + ); + } + + #[test] + fn flags_localhost_subdomain() { + let host = KnotHost::parse("http://internal.localhost").unwrap(); + assert_eq!( + host.private_literal_reason(), + Some(PrivateHostReason::Loopback), + ); + } + + #[test] + fn flags_localhost_case_insensitive() { + let host = KnotHost::parse("http://LOCALHOST").unwrap(); + assert_eq!( + host.private_literal_reason(), + Some(PrivateHostReason::Loopback), + ); + } + + #[test] + fn does_not_flag_domain_merely_containing_localhost() { + let host = KnotHost::parse("https://localhost.attacker.example").unwrap(); + assert_eq!(host.private_literal_reason(), None); + } +} diff --git a/bobbin/crates/knot-proxy/src/lib.rs b/bobbin/crates/knot-proxy/src/lib.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/knot-proxy/src/lib.rs @@ -0,0 +1,779 @@ +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use bobbin_runtime::{ + BodyStream as InnerBodyStream, Clock, HttpRequest, HttpResponseHead, HttpTransport, + NetworkError, ReqwestHttp, RuntimeHasher, +}; +use bytes::Bytes; +use futures::Stream; +use http::{HeaderMap, StatusCode}; +use jacquard_common::BosStr; +use jacquard_common::types::nsid::Nsid; +use reqwest::{Client, redirect::Policy}; +use scc::HashMap as SccMap; +use thiserror::Error; +use url::Url; + +mod breaker; +mod dns; +mod host; + +pub use breaker::{Breaker, BreakerPermit, CircuitOpen, FailureThreshold, ThresholdError}; +pub use host::{KnotHost, KnotHostError, PrivateHostReason, RepoSlug, RepoSlugError}; + +const USER_AGENT: &str = concat!("bobbin/", env!("CARGO_PKG_VERSION")); +const HTTPS_SCHEME: &str = "https"; + +#[derive(Clone, Debug)] +pub struct KnotProxyConfig { + pub failure_threshold: FailureThreshold, + pub cooldown: Duration, + pub allow_private_hosts: bool, + pub require_https: bool, +} + +impl Default for KnotProxyConfig { + fn default() -> Self { + Self { + failure_threshold: FailureThreshold::new(5).expect("nonzero literal"), + cooldown: Duration::from_secs(30), + allow_private_hosts: false, + require_https: true, + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct KnotHttpConfig { + pub connect_timeout: Duration, + pub read_timeout: Duration, +} + +impl Default for KnotHttpConfig { + fn default() -> Self { + Self { + connect_timeout: Duration::from_secs(5), + read_timeout: Duration::from_secs(60), + } + } +} + +#[derive(Debug, Error)] +pub enum KnotProxyError { + #[error("circuit breaker open")] + CircuitOpen, + #[error("blocked: host {host} resolves to {reason} address space")] + BlockedHost { + host: String, + reason: PrivateHostReason, + }, + #[error("blocked: knot {host} requires https, got plaintext http")] + PlaintextHttp { host: String }, + #[error("connect failed: {0}")] + Connect(String), + #[error("upstream read timed out: {0}")] + Timeout(String), + #[error("redirect refused: {0}")] + Redirect(String), + #[error("transport: {0}")] + Transport(String), + #[error("upstream returned status {0}")] + Upstream(StatusCode), +} + +pub struct KnotProxy { + http: Arc, + breakers: SccMap, RuntimeHasher>, + threshold: FailureThreshold, + cooldown: Duration, + allow_private_hosts: bool, + require_https: bool, + clock: Arc, +} + +impl KnotProxy { + pub fn new( + config: KnotProxyConfig, + http: KnotHttpConfig, + clock: Arc, + hasher: RuntimeHasher, + ) -> Result { + let resolver = Arc::new(dns::PrivateAddressFilter::new(config.allow_private_hosts)); + let client = Client::builder() + .user_agent(USER_AGENT) + .connect_timeout(http.connect_timeout) + .read_timeout(http.read_timeout) + .redirect(Policy::none()) + .no_gzip() + .no_brotli() + .no_deflate() + .dns_resolver(resolver) + .build()?; + Ok(Self::with_transport( + ReqwestHttp::shared(client), + config, + clock, + hasher, + )) + } + + pub fn with_transport( + http: Arc, + config: KnotProxyConfig, + clock: Arc, + hasher: RuntimeHasher, + ) -> Self { + Self { + http, + breakers: SccMap::with_hasher(hasher), + threshold: config.failure_threshold, + cooldown: config.cooldown, + allow_private_hosts: config.allow_private_hosts, + require_https: config.require_https, + clock, + } + } + + pub fn allows_private_hosts(&self) -> bool { + self.allow_private_hosts + } + + pub fn requires_https(&self) -> bool { + self.require_https + } + + pub async fn forward>( + &self, + host: &KnotHost, + nsid: &Nsid, + query: &[(&str, &str)], + headers: HeaderMap, + ) -> Result { + self.guard_host(host)?; + let breaker = self.breaker_for(host).await; + let permit = breaker + .try_acquire() + .map_err(|_: CircuitOpen| KnotProxyError::CircuitOpen)?; + let url = build_xrpc_url(host, nsid, query); + let outcome = self.http.execute(HttpRequest { url, headers }).await; + classify(outcome, permit) + } + + fn guard_host(&self, host: &KnotHost) -> Result<(), KnotProxyError> { + let host_str = || host.url().host_str().unwrap_or_default().to_owned(); + if self.require_https && host.url().scheme() != HTTPS_SCHEME { + return Err(KnotProxyError::PlaintextHttp { host: host_str() }); + } + if self.allow_private_hosts { + return Ok(()); + } + match host.private_literal_reason() { + None => Ok(()), + Some(reason) => Err(KnotProxyError::BlockedHost { + host: host_str(), + reason, + }), + } + } + + async fn breaker_for(&self, host: &KnotHost) -> Arc { + if let Some(existing) = self.breakers.read_async(host, |_, v| Arc::clone(v)).await { + return existing; + } + let entry = self.breakers.entry_async(host.clone()).await; + Arc::clone( + entry + .or_insert_with(|| { + Arc::new(Breaker::new( + self.threshold, + self.cooldown, + self.clock.clone(), + )) + }) + .get(), + ) + } +} + +pub struct ProxyResponse { + status: StatusCode, + headers: HeaderMap, + body: InnerBodyStream, + permit: BreakerPermit, +} + +impl std::fmt::Debug for ProxyResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProxyResponse") + .field("status", &self.status) + .field("headers", &self.headers) + .finish_non_exhaustive() + } +} + +impl ProxyResponse { + pub fn status(&self) -> StatusCode { + self.status + } + + pub fn headers(&self) -> &HeaderMap { + &self.headers + } + + pub fn into_body_stream(self) -> BodyStream { + BodyStream::new(self.body, self.permit) + } +} + +pub struct BodyStream { + inner: InnerBodyStream, + permit: Option, +} + +impl BodyStream { + fn new(inner: InnerBodyStream, permit: BreakerPermit) -> Self { + Self { + inner, + permit: Some(permit), + } + } + + fn resolve(&mut self, success: bool) { + if let Some(permit) = self.permit.take() { + if success { + permit.record_success(); + } else { + permit.record_failure(); + } + } + } +} + +impl Stream for BodyStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let next = match self.inner.as_mut().poll_next(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(item) => item, + }; + match &next { + Some(Ok(_)) => {} + Some(Err(_)) => self.resolve(false), + None => self.resolve(true), + } + Poll::Ready(next) + } +} + +fn build_xrpc_url>( + host: &KnotHost, + nsid: &Nsid, + query: &[(&str, &str)], +) -> Url { + let mut url = host.xrpc_url(nsid); + { + let mut pairs = url.query_pairs_mut(); + query.iter().for_each(|(k, v)| { + pairs.append_pair(k, v); + }); + } + url +} + +fn classify( + outcome: Result, + permit: BreakerPermit, +) -> Result { + match outcome { + Ok(head) if is_upstream_failure(head.status) => { + let status = head.status; + permit.record_failure(); + Err(KnotProxyError::Upstream(status)) + } + Ok(head) => Ok(ProxyResponse { + status: head.status, + headers: head.headers, + body: head.body, + permit, + }), + Err(err) => { + permit.record_failure(); + Err(map_network(err)) + } + } +} + +fn is_upstream_failure(status: StatusCode) -> bool { + status.is_server_error() || is_unfollowable_redirect(status) +} + +fn is_unfollowable_redirect(status: StatusCode) -> bool { + matches!(status.as_u16(), 301 | 302 | 303 | 307 | 308) +} + +fn map_network(err: NetworkError) -> KnotProxyError { + match err { + NetworkError::Timeout(msg) => KnotProxyError::Timeout(msg), + NetworkError::Connect(msg) => KnotProxyError::Connect(msg), + NetworkError::Redirect(msg) => KnotProxyError::Redirect(msg), + NetworkError::Transport(msg) | NetworkError::Body(msg) | NetworkError::Protocol(msg) => { + KnotProxyError::Transport(msg) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bobbin_runtime::SystemClock; + use futures::stream::TryStreamExt; + use jacquard_common::DefaultStr; + use tokio::io::AsyncWriteExt; + use wiremock::matchers::{method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn nsid(s: &'static str) -> Nsid { + Nsid::new_static(s).unwrap() + } + + pub(crate) fn config_for_test() -> KnotProxyConfig { + KnotProxyConfig { + failure_threshold: FailureThreshold::new(2).unwrap(), + cooldown: Duration::from_millis(80), + allow_private_hosts: true, + require_https: false, + } + } + + pub(crate) fn http_config_for_test() -> KnotHttpConfig { + KnotHttpConfig { + connect_timeout: Duration::from_millis(500), + read_timeout: Duration::from_secs(2), + } + } + + fn proxy_for_test() -> KnotProxy { + KnotProxy::new( + config_for_test(), + http_config_for_test(), + Arc::new(SystemClock::new()), + RuntimeHasher::default(), + ) + .unwrap() + } + + async fn server() -> MockServer { + MockServer::start().await + } + + fn host_of(server: &MockServer) -> KnotHost { + KnotHost::parse(&server.uri()).unwrap() + } + + pub(crate) async fn drain(stream: BodyStream) -> Result { + let chunks: Vec = stream.try_collect().await?; + let total: usize = chunks.iter().map(|b| b.len()).sum(); + let mut buf = bytes::BytesMut::with_capacity(total); + chunks.iter().for_each(|c| buf.extend_from_slice(c)); + Ok(buf.freeze()) + } + + #[tokio::test] + async fn forwards_query_params_and_returns_body() { + let server = server().await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .and(query_param("repo", "did:plc:squid/barnacle")) + .and(query_param("ref", "main")) + .and(query_param("path", "README.md")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/json") + .set_body_string(r#"{"path":"README.md"}"#), + ) + .mount(&server) + .await; + + let proxy = proxy_for_test(); + let resp = proxy + .forward( + &host_of(&server), + &nsid("sh.tangled.repo.blob"), + &[ + ("repo", "did:plc:squid/barnacle"), + ("ref", "main"), + ("path", "README.md"), + ], + HeaderMap::new(), + ) + .await + .expect("happy path"); + assert_eq!(resp.status(), 200); + let body = drain(resp.into_body_stream()).await.unwrap(); + assert_eq!(&body[..], br#"{"path":"README.md"}"#); + } + + #[tokio::test] + async fn five_hundreds_open_breaker() { + let server = server().await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + + let proxy = proxy_for_test(); + let host = host_of(&server); + let r1 = proxy + .forward(&host, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await; + assert!(matches!(r1, Err(KnotProxyError::Upstream(_)))); + let r2 = proxy + .forward(&host, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await; + assert!(matches!(r2, Err(KnotProxyError::Upstream(_)))); + let r3 = proxy + .forward(&host, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await; + assert!(matches!(r3, Err(KnotProxyError::CircuitOpen))); + } + + #[tokio::test] + async fn four_hundreds_do_not_open_breaker() { + let server = server().await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with(ResponseTemplate::new(404).set_body_string("not found")) + .mount(&server) + .await; + + let proxy = proxy_for_test(); + let host = host_of(&server); + let r1 = proxy + .forward(&host, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await; + assert_eq!(r1.unwrap().status(), 404); + let r2 = proxy + .forward(&host, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await; + assert_eq!(r2.unwrap().status(), 404); + let r3 = proxy + .forward(&host, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await; + assert_eq!( + r3.unwrap().status(), + 404, + "client errors must not trip breaker", + ); + } + + #[tokio::test] + async fn breaker_recovers_after_cooldown() { + let server = server().await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with(ResponseTemplate::new(503)) + .up_to_n_times(2) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/json") + .set_body_string("ok"), + ) + .mount(&server) + .await; + + let proxy = proxy_for_test(); + let host = host_of(&server); + let _ = proxy + .forward(&host, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await; + let _ = proxy + .forward(&host, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await; + assert!(matches!( + proxy + .forward(&host, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await, + Err(KnotProxyError::CircuitOpen), + )); + tokio::time::sleep(Duration::from_millis(120)).await; + let recovered = proxy + .forward(&host, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await + .expect("must recover after cooldown"); + assert_eq!(recovered.status(), 200); + let body = drain(recovered.into_body_stream()).await.unwrap(); + assert_eq!(&body[..], b"ok"); + } + + #[tokio::test] + async fn breakers_are_isolated_per_host() { + let bad = server().await; + let good = server().await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with(ResponseTemplate::new(503)) + .mount(&bad) + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/json") + .set_body_string("ok"), + ) + .mount(&good) + .await; + + let proxy = proxy_for_test(); + let bad_host = host_of(&bad); + let good_host = host_of(&good); + let _ = proxy + .forward( + &bad_host, + &nsid("sh.tangled.repo.blob"), + &[], + HeaderMap::new(), + ) + .await; + let _ = proxy + .forward( + &bad_host, + &nsid("sh.tangled.repo.blob"), + &[], + HeaderMap::new(), + ) + .await; + assert!(matches!( + proxy + .forward( + &bad_host, + &nsid("sh.tangled.repo.blob"), + &[], + HeaderMap::new() + ) + .await, + Err(KnotProxyError::CircuitOpen), + )); + let resp = proxy + .forward( + &good_host, + &nsid("sh.tangled.repo.blob"), + &[], + HeaderMap::new(), + ) + .await + .expect("healthy host stays open"); + assert_eq!(resp.status(), 200); + } + + #[tokio::test] + async fn build_xrpc_url_appends_query() { + let host = KnotHost::parse("https://oyster.cafe").unwrap(); + let url = build_xrpc_url( + &host, + &nsid("sh.tangled.repo.tree"), + &[("repo", "did:plc:squid/barnacle"), ("ref", "main")], + ); + assert_eq!( + url.as_str(), + "https://oyster.cafe/xrpc/sh.tangled.repo.tree?repo=did%3Aplc%3Asquid%2Fbarnacle&ref=main", + ); + } + + #[tokio::test] + async fn rejects_private_host_by_default() { + let server = server().await; + let strict = KnotProxyConfig { + allow_private_hosts: false, + ..config_for_test() + }; + let proxy = KnotProxy::new( + strict, + http_config_for_test(), + Arc::new(SystemClock::new()), + RuntimeHasher::default(), + ) + .unwrap(); + let err = proxy + .forward( + &host_of(&server), + &nsid("sh.tangled.repo.blob"), + &[], + HeaderMap::new(), + ) + .await + .expect_err("loopback must be blocked under strict config"); + assert!(matches!(err, KnotProxyError::BlockedHost { .. })); + } + + #[tokio::test] + async fn rejects_plaintext_when_https_required() { + let server = server().await; + let strict = KnotProxyConfig { + require_https: true, + ..config_for_test() + }; + let proxy = KnotProxy::new( + strict, + http_config_for_test(), + Arc::new(SystemClock::new()), + RuntimeHasher::default(), + ) + .unwrap(); + let err = proxy + .forward( + &host_of(&server), + &nsid("sh.tangled.repo.blob"), + &[], + HeaderMap::new(), + ) + .await + .expect_err("plaintext http must be rejected under https-required"); + assert!( + matches!(err, KnotProxyError::PlaintextHttp { .. }), + "got {err:?}", + ); + } + + #[tokio::test] + async fn transport_error_trips_breaker() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + drop(listener); + let dead = KnotHost::parse(&format!("http://{addr}")).unwrap(); + + let proxy = proxy_for_test(); + let r1 = proxy + .forward(&dead, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await; + assert!( + r1.is_err(), + "transport must fail against closed port: {r1:?}" + ); + let r2 = proxy + .forward(&dead, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await; + assert!(r2.is_err(), "second transport must fail: {r2:?}"); + let r3 = proxy + .forward(&dead, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await; + assert!( + matches!(r3, Err(KnotProxyError::CircuitOpen)), + "transport failures must trip breaker, got {r3:?}", + ); + } + + #[tokio::test] + async fn redirects_surface_as_upstream_failure() { + let primary = server().await; + let secondary = server().await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with( + ResponseTemplate::new(302) + .insert_header("location", &format!("{}/secret", secondary.uri())), + ) + .mount(&primary) + .await; + Mock::given(method("GET")) + .and(path("/secret")) + .respond_with(ResponseTemplate::new(200).set_body_string("leaked")) + .mount(&secondary) + .await; + + let proxy = proxy_for_test(); + let err = proxy + .forward( + &host_of(&primary), + &nsid("sh.tangled.repo.blob"), + &[], + HeaderMap::new(), + ) + .await + .expect_err("302 must surface as upstream failure"); + assert!( + matches!(err, KnotProxyError::Upstream(s) if s.as_u16() == 302), + "got {err:?}", + ); + let received = secondary.received_requests().await.unwrap(); + assert!(received.is_empty(), "secondary must never be dialled"); + } + + #[tokio::test] + async fn not_modified_passes_through() { + let server = server().await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with(ResponseTemplate::new(304).insert_header("etag", "\"v1\"")) + .mount(&server) + .await; + let proxy = proxy_for_test(); + let resp = proxy + .forward( + &host_of(&server), + &nsid("sh.tangled.repo.blob"), + &[], + HeaderMap::new(), + ) + .await + .expect("304 is a cache validator, not a redirect"); + assert_eq!(resp.status(), 304); + } + + #[tokio::test] + async fn mid_stream_drop_records_breaker_failure() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + async fn drop_after_partial(mut socket: tokio::net::TcpStream) { + let _ = socket + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 1024\r\nContent-Type: application/octet-stream\r\n\r\nabcd", + ) + .await; + drop(socket); + } + let admit = || async { + let (socket, _) = listener.accept().await.ok()?; + drop_after_partial(socket).await; + Some(()) + }; + admit().await; + admit().await; + }); + + let host = KnotHost::parse(&format!("http://{addr}")).unwrap(); + let proxy = proxy_for_test(); + + let r1 = proxy + .forward(&host, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await + .expect("headers arrive even when body is truncated"); + assert_eq!(r1.status(), 200); + let _ = drain(r1.into_body_stream()).await; + + let r2 = proxy + .forward(&host, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await + .expect("second call still gets headers"); + let _ = drain(r2.into_body_stream()).await; + + let r3 = proxy + .forward(&host, &nsid("sh.tangled.repo.blob"), &[], HeaderMap::new()) + .await; + assert!( + matches!(r3, Err(KnotProxyError::CircuitOpen)), + "two truncated streams must trip the breaker, got {r3:?}", + ); + server.abort(); + } +} diff --git a/bobbin/crates/record-lru/Cargo.toml b/bobbin/crates/record-lru/Cargo.toml new file mode 100644 --- /dev/null +++ b/bobbin/crates/record-lru/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "bobbin-record-lru" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-types = { workspace = true } +jacquard-common = { workspace = true } +bytes = { workspace = true } +quick_cache = { workspace = true } +zstd = "0.13" + +[dev-dependencies] +bytes = { workspace = true } diff --git a/bobbin/crates/record-lru/src/lib.rs b/bobbin/crates/record-lru/src/lib.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/record-lru/src/lib.rs @@ -0,0 +1,281 @@ +use std::cell::RefCell; +use std::sync::Arc; +use std::sync::OnceLock; + +use bobbin_types::record::RecordBody; +use bytes::Bytes; +use jacquard_common::DefaultStr; +use jacquard_common::types::string::{AtUri, Cid}; +use quick_cache::Weighter; +use quick_cache::sync::Cache; +use zstd::bulk::{Compressor, Decompressor}; +use zstd::dict::{DecoderDictionary, EncoderDictionary}; + +static RECORD_DICT: &[u8] = include_bytes!("record.dict"); +const COMPRESS_LEVEL: i32 = 6; +const ENTRY_OVERHEAD: u64 = 64; + +static ENCODER_DICT: OnceLock> = OnceLock::new(); +static DECODER_DICT: OnceLock> = OnceLock::new(); + +fn encoder_dict() -> &'static EncoderDictionary<'static> { + ENCODER_DICT.get_or_init(|| EncoderDictionary::copy(RECORD_DICT, COMPRESS_LEVEL)) +} + +fn decoder_dict() -> &'static DecoderDictionary<'static> { + DECODER_DICT.get_or_init(|| DecoderDictionary::copy(RECORD_DICT)) +} + +thread_local! { + static COMPRESSOR: RefCell> = RefCell::new( + Compressor::with_prepared_dictionary(encoder_dict()).expect("zstd compressor init"), + ); + static DECOMPRESSOR: RefCell> = RefCell::new( + Decompressor::with_prepared_dictionary(decoder_dict()).expect("zstd decompressor init"), + ); +} + +pub trait RecordStore: Send + Sync { + fn get(&self, uri: &AtUri) -> Option>; + fn put(&self, uri: AtUri, body: Arc); + fn remove(&self, uri: &AtUri); + fn cache_stats(&self) -> Option { + None + } +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct CacheStats { + pub weight: u64, + pub len: u64, + pub capacity: u64, +} + +pub struct NoopRecordStore; + +impl RecordStore for NoopRecordStore { + fn get(&self, _uri: &AtUri) -> Option> { + None + } + fn put(&self, _uri: AtUri, _body: Arc) {} + fn remove(&self, _uri: &AtUri) {} +} + +#[derive(Clone, Debug)] +pub struct CacheCapacity { + pub bytes: u64, + pub estimated_items: usize, +} + +impl CacheCapacity { + pub fn from_bytes(bytes: u64) -> Self { + const TYPICAL_RECORD_BYTES: u64 = 512; + let estimated = bytes.div_ceil(TYPICAL_RECORD_BYTES).max(64) as usize; + Self { + bytes, + estimated_items: estimated, + } + } +} + +#[derive(Clone)] +enum Payload { + Raw(Bytes), + Zstd { compressed: Bytes, plain_len: usize }, +} + +#[derive(Clone)] +struct Stored { + cid: Cid, + payload: Payload, +} + +#[derive(Clone)] +struct ByteWeighter; + +impl Weighter, Stored> for ByteWeighter { + fn weight(&self, key: &AtUri, val: &Stored) -> u64 { + let payload = match &val.payload { + Payload::Raw(bytes) => bytes.len(), + Payload::Zstd { compressed, .. } => compressed.len(), + }; + ENTRY_OVERHEAD + + payload as u64 + + key.as_ref().len() as u64 + + val.cid.as_ref().len() as u64 + } +} + +pub struct LruRecordStore { + cache: Cache, Stored, ByteWeighter>, +} + +impl LruRecordStore { + pub fn new(capacity: CacheCapacity) -> Self { + Self { + cache: Cache::with_weighter(capacity.estimated_items, capacity.bytes, ByteWeighter), + } + } +} + +impl RecordStore for LruRecordStore { + fn get(&self, uri: &AtUri) -> Option> { + let stored = self.cache.get(uri)?; + let value = match stored.payload { + Payload::Raw(bytes) => bytes, + Payload::Zstd { + compressed, + plain_len, + } => { + let plain = DECOMPRESSOR + .with_borrow_mut(|d| d.decompress(&compressed, plain_len)) + .ok()?; + Bytes::from(plain) + } + }; + Some(Arc::new(RecordBody { + uri: uri.clone(), + cid: stored.cid, + value, + })) + } + + fn put(&self, uri: AtUri, body: Arc) { + let plain_len = body.value.len(); + let payload = match COMPRESSOR.with_borrow_mut(|c| c.compress(&body.value)) { + Ok(mut compressed) if compressed.len() < plain_len => { + compressed.shrink_to_fit(); + Payload::Zstd { + compressed: Bytes::from(compressed), + plain_len, + } + } + _ => Payload::Raw(body.value.clone()), + }; + self.cache.insert( + uri, + Stored { + cid: body.cid.clone(), + payload, + }, + ); + } + + fn remove(&self, uri: &AtUri) { + self.cache.remove(uri); + } + + fn cache_stats(&self) -> Option { + Some(CacheStats { + weight: self.cache.weight(), + len: self.cache.len() as u64, + capacity: self.cache.capacity(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use jacquard_common::types::string::Cid; + + fn at(s: &str) -> AtUri { + AtUri::new_owned(s).unwrap() + } + + fn body(uri: AtUri, payload: &[u8]) -> Arc { + let cid: Cid = "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i" + .parse() + .unwrap(); + Arc::new(RecordBody { + uri, + cid, + value: Bytes::copy_from_slice(payload), + }) + } + + fn high_entropy(seed: u64, len: usize) -> Vec { + use std::hash::{Hash, Hasher}; + let mut out = Vec::with_capacity(len); + let mut counter = seed; + while out.len() < len { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + counter.hash(&mut hasher); + out.extend_from_slice(&hasher.finish().to_le_bytes()); + counter = counter.wrapping_add(1); + } + out.truncate(len); + out + } + + #[test] + fn put_then_get_round_trips_through_compression() { + let store = LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024)); + let uri = at("at://did:plc:nel/sh.tangled.repo/r1"); + let payload = br#"{"$type":"sh.tangled.repo","name":"oyster","knot":"oyster.cafe"}"#; + let b = body(uri.clone(), payload); + store.put(uri.clone(), b.clone()); + let got = store.get(&uri).expect("hit"); + assert_eq!(got.value, b.value, "decompressed body must equal original"); + assert_eq!(got.cid, b.cid); + } + + #[test] + fn remove_evicts_entry() { + let store = LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024)); + let uri = at("at://did:plc:nel/sh.tangled.repo/r1"); + store.put(uri.clone(), body(uri.clone(), br#"{"v":1}"#)); + assert!(store.get(&uri).is_some()); + store.remove(&uri); + assert!(store.get(&uri).is_none()); + } + + #[test] + fn miss_returns_none() { + let store = LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024)); + assert!( + store + .get(&at("at://did:plc:abalone/sh.tangled.repo/r1")) + .is_none() + ); + } + + #[test] + fn byte_capacity_evicts_incompressible_entries() { + let store = LruRecordStore::new(CacheCapacity::from_bytes(2_048)); + let resident = (0..16) + .map(|i| { + let uri = at(&format!("at://did:plc:olaren/sh.tangled.string/r{i}")); + store.put(uri.clone(), body(uri.clone(), &high_entropy(i, 900))); + uri + }) + .collect::>() + .iter() + .filter(|uri| store.get(uri).is_some()) + .count(); + assert!( + resident < 16, + "byte cap must evict incompressible bodies, observed {resident} of 16 resident" + ); + } + + #[test] + fn capacity_estimates_items_floor() { + let cap = CacheCapacity::from_bytes(0); + assert_eq!(cap.estimated_items, 64, "tiny caches still need a floor"); + } + + #[test] + fn incompressible_body_round_trips_via_raw() { + let store = LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024)); + let uri = at("at://did:plc:olaren/sh.tangled.string/r1"); + let payload = high_entropy(7, 300); + let stored = body(uri.clone(), &payload); + store.put(uri.clone(), stored.clone()); + let got = store.get(&uri).expect("hit"); + assert_eq!( + got.value, stored.value, + "incompressible body must round-trip byte-exact through the raw path" + ); + } +} diff --git a/bobbin/crates/record-lru/src/record.dict b/bobbin/crates/record-lru/src/record.dict new file mode 100644 --- /dev/null +++ b/bobbin/crates/record-lru/src/record.dict diff --git a/bobbin/crates/resolver/Cargo.toml b/bobbin/crates/resolver/Cargo.toml new file mode 100644 --- /dev/null +++ b/bobbin/crates/resolver/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "bobbin-resolver" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-types = { workspace = true } +bobbin-runtime = { workspace = true } +bobbin-slingshot-client = { workspace = true } +jacquard-common = { workspace = true } + +scc = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +wiremock = { workspace = true } +url = { workspace = true } +serde_json = { workspace = true } diff --git a/bobbin/crates/resolver/src/legacy_upgrade.rs b/bobbin/crates/resolver/src/legacy_upgrade.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/resolver/src/legacy_upgrade.rs @@ -0,0 +1,983 @@ +use bobbin_types::edges::{ExtractError, Record}; +use bobbin_types::legacy::{ + LegacyCollaborator, LegacyIssue, LegacyKnotMember, LegacyPublicKey, LegacyPull, LegacyRecord, + LegacyRefUpdate, LegacyRepo, LegacySource, LegacyStar, LegacyTarget, +}; +use bobbin_types::sh_tangled::feed::star::{Repo as StarRepo, Star, StarString, StarSubject}; +use bobbin_types::sh_tangled::git::ref_update::RefUpdate; +use bobbin_types::sh_tangled::knot::member::Member as KnotMember; +use bobbin_types::sh_tangled::public_key::PublicKey; +use bobbin_types::sh_tangled::repo::Repo; +use bobbin_types::sh_tangled::repo::collaborator::Collaborator; +use bobbin_types::sh_tangled::repo::issue::Issue; +use bobbin_types::sh_tangled::repo::pull::{Pull, Round, Source, Target}; +use jacquard_common::types::did::Did; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::string::AtUri; +use jacquard_common::{BosStr, DefaultStr}; + +use crate::normalize::{is_repo_at_uri, resolve_repo_uri}; +use crate::{RepoIdResolver, Resolution}; +use jacquard_common::IntoStatic; +use jacquard_common::types::ident::AtIdentifier; +use jacquard_common::types::recordkey::Rkey; + +#[derive(Debug)] +pub enum DecodedRecord { + Canon(Record), + Legacy(LegacyRecord), +} + +impl DecodedRecord { + pub fn try_decode>( + nsid: &Nsid, + bytes: &[u8], + ) -> Result { + match Record::from_json_bytes(nsid, bytes) { + Ok(record) => Ok(Self::Canon(record)), + Err(canon_err) => { + let normalized = normalize_record_fields(bytes); + let working: &[u8] = normalized.as_deref().unwrap_or(bytes); + if normalized.is_some() + && let Ok(record) = Record::from_json_bytes(nsid, working) + { + return Ok(Self::Canon(record)); + } + if let Some(scrubbed) = scrub_record_bytes(nsid, working) { + if let Ok(record) = Record::from_json_bytes(nsid, &scrubbed) { + return Ok(Self::Canon(record)); + } + if let Ok(legacy) = LegacyRecord::from_json_bytes(nsid, &scrubbed) { + return Ok(Self::Legacy(legacy)); + } + } + match LegacyRecord::from_json_bytes(nsid, working) { + Ok(legacy) => Ok(Self::Legacy(legacy)), + Err(_) => Err(canon_err), + } + } + } + } +} + +pub fn normalize_record_fields(bytes: &[u8]) -> Option> { + let value: serde_json::Value = serde_json::from_slice(bytes).ok()?; + let reserialized = serde_json::to_vec(&value).ok()?; + (reserialized.as_slice() != bytes).then_some(reserialized) +} + +pub fn synthesize_created_at(bytes: &[u8], fallback_rfc3339: &str) -> Option> { + let mut value: serde_json::Value = serde_json::from_slice(bytes).ok()?; + let obj = value.as_object_mut()?; + let needs_fill = match obj.get("createdAt") { + None => true, + Some(serde_json::Value::String(s)) if s.is_empty() => true, + _ => false, + }; + if !needs_fill { + return None; + } + obj.insert( + "createdAt".to_owned(), + serde_json::Value::String(fallback_rfc3339.to_owned()), + ); + serde_json::to_vec(&value).ok() +} + +#[derive(Clone, Copy, Debug)] +enum FieldRule { + DropIfEmptyString, + NullToEmptyArray, +} + +fn scrub_rules(nsid: &str) -> &'static [(&'static str, FieldRule)] { + match nsid { + "sh.tangled.actor.profile" => &[("preferredHandle", FieldRule::DropIfEmptyString)], + "sh.tangled.label.op" => &[ + ("add", FieldRule::NullToEmptyArray), + ("delete", FieldRule::NullToEmptyArray), + ], + "sh.tangled.repo.pull" => &[("rounds", FieldRule::NullToEmptyArray)], + _ => &[], + } +} + +pub fn scrub_record_bytes>( + nsid: &Nsid, + bytes: &[u8], +) -> Option> { + let rules = scrub_rules(nsid.as_ref()); + if rules.is_empty() { + return None; + } + let value: serde_json::Value = serde_json::from_slice(bytes).ok()?; + let mut obj = value.as_object()?.clone(); + let touched: alloc::vec::Vec<(&str, FieldRule)> = rules + .iter() + .filter_map(|(field, rule)| match (rule, obj.get(*field)) { + (FieldRule::DropIfEmptyString, Some(serde_json::Value::String(s))) if s.is_empty() => { + Some((*field, *rule)) + } + (FieldRule::NullToEmptyArray, Some(serde_json::Value::Null)) => Some((*field, *rule)), + _ => None, + }) + .collect(); + if touched.is_empty() { + return None; + } + touched.iter().for_each(|(field, rule)| match rule { + FieldRule::DropIfEmptyString => { + obj.remove(*field); + } + FieldRule::NullToEmptyArray => { + obj.insert( + (*field).to_owned(), + serde_json::Value::Array(alloc::vec::Vec::new()), + ); + } + }); + tracing::debug!(nsid = %nsid.as_ref(), ?touched, "scrubbing fields before record retry"); + serde_json::to_vec(&serde_json::Value::Object(obj)).ok() +} + +async fn upgrade_repo_did( + resolver: &RepoIdResolver, + at_uri: Option>, + explicit_did: Option>, +) -> Option> { + if let Some(d) = explicit_did { + return Some(d); + } + let uri = at_uri?; + resolve_repo_uri(resolver, &uri).await +} + +pub async fn upgrade_wire_bytes>( + nsid: &Nsid, + bytes: &[u8], + resolver: &RepoIdResolver, +) -> Result, ExtractError> { + let legacy = LegacyRecord::from_json_bytes(nsid, bytes)?; + let canon = upgrade(legacy, resolver) + .await + .ok_or_else(|| upgrade_failed(nsid))?; + serialize_canon_variant(&canon).map_err(ExtractError::DecodeJson) +} + +fn upgrade_failed>(nsid: &Nsid) -> ExtractError { + ExtractError::UnknownCollection(alloc::format!("{}: legacy upgrade failed", nsid.as_ref())) +} + +pub async fn decode_canon_or_upgrade>( + nsid: &Nsid, + bytes: &[u8], + resolver: &RepoIdResolver, +) -> Result { + match DecodedRecord::try_decode(nsid, bytes)? { + DecodedRecord::Canon(r) => Ok(r), + DecodedRecord::Legacy(legacy) => upgrade(legacy, resolver) + .await + .ok_or_else(|| upgrade_failed(nsid)), + } +} + +pub async fn decode_canon_or_upgrade_bytes<'a, S: BosStr + AsRef>( + nsid: &Nsid, + bytes: &'a [u8], + resolver: &RepoIdResolver, +) -> Result<(Record, alloc::borrow::Cow<'a, [u8]>), ExtractError> { + let decoded = DecodedRecord::try_decode(nsid, bytes)?; + match decoded { + DecodedRecord::Canon(r) => Ok((r, alloc::borrow::Cow::Borrowed(bytes))), + DecodedRecord::Legacy(legacy) => { + let canon = upgrade(legacy, resolver) + .await + .ok_or_else(|| upgrade_failed(nsid))?; + let canon_bytes = serialize_canon_variant(&canon).map_err(ExtractError::DecodeJson)?; + Ok((canon, alloc::borrow::Cow::Owned(canon_bytes))) + } + } +} + +fn serialize_canon_variant(record: &Record) -> Result, serde_json::Error> { + match record { + Record::Issue(r) => serde_json::to_vec(r), + Record::Pull(r) => serde_json::to_vec(r), + Record::Collaborator(r) => serde_json::to_vec(r), + Record::RefUpdate(r) => serde_json::to_vec(r), + Record::Star(r) => serde_json::to_vec(r), + Record::PublicKey(r) => serde_json::to_vec(r), + Record::Repo(r) => serde_json::to_vec(r), + Record::KnotMember(r) => serde_json::to_vec(r), + _ => unreachable!( + "upgrade only produces Issue/Pull/Collaborator/RefUpdate/Star/PublicKey/Repo/KnotMember" + ), + } +} + +pub async fn upgrade(legacy: LegacyRecord, resolver: &RepoIdResolver) -> Option { + match legacy { + LegacyRecord::Issue(l) => upgrade_issue(l, resolver).await.map(Record::Issue), + LegacyRecord::Pull(l) => upgrade_pull(l, resolver).await.map(Record::Pull), + LegacyRecord::Collaborator(l) => upgrade_collaborator(l, resolver) + .await + .map(Record::Collaborator), + LegacyRecord::RefUpdate(l) => Some(Record::RefUpdate(upgrade_ref_update(l))), + LegacyRecord::Star(l) => upgrade_star(l, resolver).await.map(Record::Star), + LegacyRecord::PublicKey(l) => Some(Record::PublicKey(upgrade_public_key(l))), + LegacyRecord::Repo(l) => Some(Record::Repo(upgrade_repo(l))), + LegacyRecord::KnotMember(l) => Some(Record::KnotMember(upgrade_knot_member(l))), + } +} + +async fn upgrade_issue( + l: LegacyIssue, + resolver: &RepoIdResolver, +) -> Option> { + let repo = upgrade_repo_did(resolver, l.repo, l.repo_did).await?; + Some(Issue { + created_at: l.created_at, + body: l.body, + mentions: l.mentions, + references: l.references, + repo, + title: l.title, + extra_data: l.extra_data, + }) +} + +async fn upgrade_target( + l: LegacyTarget, + resolver: &RepoIdResolver, +) -> Option> { + let repo = upgrade_repo_did(resolver, l.repo, l.repo_did).await?; + Some(Target { + branch: l.branch, + repo, + extra_data: None, + }) +} + +async fn upgrade_source( + l: LegacySource, + resolver: &RepoIdResolver, +) -> Source { + let repo = upgrade_repo_did(resolver, l.repo, l.repo_did).await; + Source { + branch: l.branch, + repo, + extra_data: None, + } +} + +async fn upgrade_pull( + l: LegacyPull, + resolver: &RepoIdResolver, +) -> Option> { + let target = upgrade_target(l.target, resolver).await?; + let source = match l.source { + Some(s) => Some(upgrade_source(s, resolver).await), + None => None, + }; + let rounds = if l.rounds.is_empty() { + l.patch_blob + .map(|patch_blob| { + alloc::vec![Round { + created_at: l.created_at.clone(), + patch_blob, + extra_data: None, + }] + }) + .unwrap_or_default() + } else { + l.rounds + }; + Some(Pull { + created_at: l.created_at, + body: l.body, + dependent_on: l.dependent_on, + mentions: l.mentions, + references: l.references, + rounds, + source, + target, + title: l.title, + extra_data: l.extra_data, + }) +} + +fn upgrade_public_key(l: LegacyPublicKey) -> PublicKey { + PublicKey { + created_at: l.created, + key: l.key, + name: l.name, + extra_data: l.extra_data, + } +} + +fn upgrade_repo(l: LegacyRepo) -> Repo { + let _ = l.owner; + Repo { + created_at: l.added_at, + description: l.description, + knot: l.knot, + labels: None, + name: l.name, + repo_did: None, + source: None, + spindle: None, + topics: None, + website: None, + extra_data: l.extra_data, + } +} + +fn upgrade_knot_member(l: LegacyKnotMember) -> KnotMember { + KnotMember { + created_at: l.added_at, + domain: l.domain, + subject: l.member, + extra_data: l.extra_data, + } +} + +async fn upgrade_collaborator( + l: LegacyCollaborator, + resolver: &RepoIdResolver, +) -> Option> { + let repo = upgrade_repo_did(resolver, l.repo, l.repo_did).await?; + Some(Collaborator { + created_at: l.created_at, + repo, + subject: l.subject, + extra_data: l.extra_data, + }) +} + +fn upgrade_ref_update(l: LegacyRefUpdate) -> RefUpdate { + RefUpdate { + committer_did: l.committer_did, + meta: l.meta, + new_sha: l.new_sha, + old_sha: l.old_sha, + owner_did: l.owner_did, + r#ref: l.r#ref, + repo: l.repo_did, + extra_data: l.extra_data, + } +} + +async fn upgrade_star( + l: LegacyStar, + resolver: &RepoIdResolver, +) -> Option> { + let subject = if let Some(did) = l.subject_did { + StarSubject::Repo(alloc::boxed::Box::new(StarRepo { + did, + extra_data: None, + })) + } else { + let uri = l.subject?; + let resolved = if is_repo_at_uri(&uri) { + cached_repo_did(resolver, &uri).await + } else { + None + }; + match resolved { + Some(did) => StarSubject::Repo(alloc::boxed::Box::new(StarRepo { + did, + extra_data: None, + })), + None => StarSubject::String(alloc::boxed::Box::new(StarString { + uri, + extra_data: None, + })), + } + }; + Some(Star { + created_at: l.created_at, + subject, + extra_data: l.extra_data, + }) +} + +async fn cached_repo_did( + resolver: &RepoIdResolver, + uri: &jacquard_common::types::string::AtUri, +) -> Option> { + let owner = match uri.authority() { + AtIdentifier::Did(d) => d.clone().into_static(), + AtIdentifier::Handle(_) => return None, + }; + let rkey: Rkey = uri.rkey()?.clone().into_static(); + match resolver.cached_resolution(&owner, &rkey).await? { + Resolution::Mapped(did) => Some(did), + Resolution::NoRepoDid | Resolution::Unresolvable => None, + } +} + +extern crate alloc; + +#[cfg(test)] +mod tests { + use super::*; + use crate::RepoIdResolver; + use bobbin_runtime::RuntimeHasher; + use bobbin_types::edges::Record; + use jacquard_common::DefaultStr; + use jacquard_common::types::did::Did; + use jacquard_common::types::recordkey::Rkey; + + fn did(s: &str) -> Did { + Did::new_owned(s).unwrap() + } + + fn rkey(s: &str) -> Rkey { + Rkey::new_owned(s).unwrap() + } + + fn nsid(s: &'static str) -> Nsid { + Nsid::new_static(s).unwrap() + } + + #[test] + fn legacy_decode_routes_through_try_decode_for_known_nsids() { + let json = br#"{"$type":"sh.tangled.repo.issue","repoDid":"did:plc:squid","title":"t","createdAt":"2026-05-01T00:00:00Z"}"#; + let decoded = DecodedRecord::try_decode(&nsid("sh.tangled.repo.issue"), json) + .expect("legacy issue must decode"); + assert!(matches!( + decoded, + DecodedRecord::Legacy(LegacyRecord::Issue(_)) + )); + } + + #[test] + fn canon_decode_wins_when_wire_matches_new_shape() { + let json = br#"{"$type":"sh.tangled.repo.issue","repo":"did:plc:squid","title":"t","createdAt":"2026-05-01T00:00:00Z"}"#; + let decoded = DecodedRecord::try_decode(&nsid("sh.tangled.repo.issue"), json) + .expect("canon issue must decode"); + match decoded { + DecodedRecord::Canon(Record::Issue(i)) => { + assert_eq!(i.repo.as_ref(), "did:plc:squid") + } + other => panic!("expected canon issue, got {other:?}"), + } + } + + #[test] + fn scrub_returns_none_for_unknown_nsid() { + let json = br#"{"$type":"sh.tangled.repo.issue","preferredHandle":""}"#; + assert!(scrub_record_bytes(&nsid("sh.tangled.repo.issue"), json).is_none()); + } + + #[test] + fn scrub_returns_none_when_target_field_is_non_empty() { + let json = + br#"{"$type":"sh.tangled.actor.profile","bluesky":true,"preferredHandle":"nel.pet"}"#; + assert!(scrub_record_bytes(&nsid("sh.tangled.actor.profile"), json).is_none()); + } + + #[test] + fn scrub_returns_none_when_target_field_is_absent() { + let json = br#"{"$type":"sh.tangled.actor.profile","bluesky":true}"#; + assert!(scrub_record_bytes(&nsid("sh.tangled.actor.profile"), json).is_none()); + } + + #[test] + fn scrub_returns_none_for_non_string_value() { + let json = br#"{"$type":"sh.tangled.actor.profile","bluesky":true,"preferredHandle":42}"#; + assert!(scrub_record_bytes(&nsid("sh.tangled.actor.profile"), json).is_none()); + } + + #[test] + fn scrub_returns_none_for_non_object_json() { + assert!(scrub_record_bytes(&nsid("sh.tangled.actor.profile"), b"[]").is_none()); + assert!(scrub_record_bytes(&nsid("sh.tangled.actor.profile"), b"null").is_none()); + assert!(scrub_record_bytes(&nsid("sh.tangled.actor.profile"), b"123").is_none()); + } + + #[test] + fn scrub_returns_none_for_invalid_json() { + assert!(scrub_record_bytes(&nsid("sh.tangled.actor.profile"), b"{not json").is_none()); + } + + #[test] + fn scrub_drops_empty_preferred_handle_and_preserves_other_fields() { + let json = br#"{"$type":"sh.tangled.actor.profile","bluesky":true,"preferredHandle":"","description":"hi"}"#; + let scrubbed = scrub_record_bytes(&nsid("sh.tangled.actor.profile"), json) + .expect("empty preferredHandle must trigger scrub"); + let value: serde_json::Value = serde_json::from_slice(&scrubbed).expect("valid json"); + let obj = value.as_object().expect("object"); + assert!(!obj.contains_key("preferredHandle")); + assert_eq!(obj.get("bluesky"), Some(&serde_json::json!(true))); + assert_eq!(obj.get("description"), Some(&serde_json::json!("hi"))); + assert_eq!( + obj.get("$type"), + Some(&serde_json::json!("sh.tangled.actor.profile")) + ); + } + + #[test] + fn scrub_replaces_null_arrays_with_empty_for_label_op() { + let json = br#"{"$type":"sh.tangled.label.op","add":[{"key":"at://did:plc:limpet/sh.tangled.label.definition/k","value":"v"}],"delete":null,"performedAt":"2026-05-01T00:00:00Z","subject":"at://did:plc:limpet/sh.tangled.repo.issue/3aaa"}"#; + let scrubbed = scrub_record_bytes(&nsid("sh.tangled.label.op"), json) + .expect("null delete must trigger scrub"); + let value: serde_json::Value = serde_json::from_slice(&scrubbed).expect("valid json"); + let obj = value.as_object().expect("object"); + assert_eq!(obj.get("delete"), Some(&serde_json::json!([]))); + assert!(obj.get("add").is_some_and(|v| v.is_array())); + } + + #[test] + fn scrub_passes_through_when_label_op_arrays_are_non_null() { + let json = br#"{"$type":"sh.tangled.label.op","add":[],"delete":[],"performedAt":"2026-05-01T00:00:00Z","subject":"at://did:plc:limpet/sh.tangled.repo.issue/3aaa"}"#; + assert!(scrub_record_bytes(&nsid("sh.tangled.label.op"), json).is_none()); + } + + #[test] + fn try_decode_recovers_label_op_with_null_delete() { + let json = br#"{"$type":"sh.tangled.label.op","add":[{"key":"at://did:plc:limpet/sh.tangled.label.definition/k","value":"v"}],"delete":null,"performedAt":"2026-05-01T00:00:00Z","subject":"at://did:plc:limpet/sh.tangled.repo.issue/3aaa"}"#; + let decoded = DecodedRecord::try_decode(&nsid("sh.tangled.label.op"), json) + .expect("label.op with null delete must scrub-recover"); + assert!(matches!(decoded, DecodedRecord::Canon(Record::LabelOp(_)))); + } + + #[test] + fn try_decode_recovers_profile_with_empty_preferred_handle() { + let json = br#"{"$type":"sh.tangled.actor.profile","bluesky":true,"preferredHandle":"","description":"hi"}"#; + let decoded = DecodedRecord::try_decode(&nsid("sh.tangled.actor.profile"), json) + .expect("profile with empty preferredHandle must scrub-recover"); + match decoded { + DecodedRecord::Canon(Record::Profile(p)) => { + assert!(p.preferred_handle.is_none()); + assert_eq!(p.description.as_deref(), Some("hi")); + } + other => panic!("expected canon profile, got {other:?}"), + } + } + + #[test] + fn legacy_decode_passes_through_for_unaffected_nsids() { + let json = br#"{"$type":"sh.tangled.graph.follow","subject":"did:plc:bailey","createdAt":"2026-05-01T00:00:00Z"}"#; + let decoded = DecodedRecord::try_decode(&nsid("sh.tangled.graph.follow"), json) + .expect("follow has no legacy form, must decode canon"); + assert!(matches!(decoded, DecodedRecord::Canon(Record::Follow(_)))); + } + + #[test] + fn normalize_returns_none_for_invalid_json() { + assert!(normalize_record_fields(b"{not json").is_none()); + } + + #[test] + fn normalize_drops_repeated_dollar_type_key() { + let json = + br#"{"$type":"sh.tangled.repo.pull","title":"t","$type":"sh.tangled.repo.pull"}"#; + let normalized = + normalize_record_fields(json).expect("duplicate key must produce normalized output"); + let value: serde_json::Value = serde_json::from_slice(&normalized).expect("valid json"); + let obj = value.as_object().expect("object"); + assert_eq!(obj.len(), 2); + assert_eq!( + obj.get("$type"), + Some(&serde_json::json!("sh.tangled.repo.pull")) + ); + assert_eq!(obj.get("title"), Some(&serde_json::json!("t"))); + } + + #[test] + fn normalize_keeps_last_value_for_repeated_keys() { + let json = br#"{"$type":"sh.tangled.repo.issue","$type":"sh.tangled.repo.pull"}"#; + let normalized = + normalize_record_fields(json).expect("duplicate key must produce normalized output"); + let value: serde_json::Value = serde_json::from_slice(&normalized).expect("valid json"); + assert_eq!( + value.get("$type"), + Some(&serde_json::json!("sh.tangled.repo.pull")), + ); + } + + #[test] + fn synthesize_fills_empty_created_at_with_fallback() { + let json = br#"{"$type":"sh.tangled.repo.issue","title":"meow","createdAt":""}"#; + let patched = synthesize_created_at(json, "2026-05-01T00:00:00.000000Z") + .expect("empty createdAt must be filled"); + let value: serde_json::Value = serde_json::from_slice(&patched).expect("valid json"); + assert_eq!( + value.get("createdAt"), + Some(&serde_json::json!("2026-05-01T00:00:00.000000Z")), + ); + } + + #[test] + fn synthesize_fills_missing_created_at_with_fallback() { + let json = br#"{"$type":"sh.tangled.repo.issue","title":"meow"}"#; + let patched = synthesize_created_at(json, "2026-05-01T00:00:00.000000Z") + .expect("missing createdAt must be filled"); + let value: serde_json::Value = serde_json::from_slice(&patched).expect("valid json"); + assert_eq!( + value.get("createdAt"), + Some(&serde_json::json!("2026-05-01T00:00:00.000000Z")), + ); + } + + #[test] + fn synthesize_returns_none_when_created_at_already_set() { + let json = br#"{"$type":"sh.tangled.repo.issue","createdAt":"2026-05-01T00:00:00Z"}"#; + assert!(synthesize_created_at(json, "2026-04-01T00:00:00Z").is_none()); + } + + #[test] + fn synthesize_returns_none_for_non_string_created_at() { + let json = br#"{"$type":"sh.tangled.repo.issue","createdAt":null}"#; + assert!(synthesize_created_at(json, "2026-04-01T00:00:00Z").is_none()); + } + + #[tokio::test] + async fn try_decode_recovers_legacy_issue_after_synthesized_created_at() { + let json = br#"{"$type":"sh.tangled.repo.issue","body":"a bug","createdAt":"","repo":"at://did:plc:scallop/sh.tangled.repo/limpet","repoDid":"did:plc:scallop","title":"a bug"}"#; + let patched = synthesize_created_at(json, "2025-08-01T12:00:00.000000Z") + .expect("empty createdAt must be filled"); + let decoded = DecodedRecord::try_decode(&nsid("sh.tangled.repo.issue"), &patched) + .expect("issue must legacy-decode after createdAt fill"); + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let canon = match decoded { + DecodedRecord::Canon(r) => r, + DecodedRecord::Legacy(l) => upgrade(l, &resolver).await.expect("upgrade"), + }; + match canon { + Record::Issue(i) => { + assert_eq!(AsRef::::as_ref(&i.title), "a bug"); + assert_eq!(i.repo.as_ref(), "did:plc:scallop"); + } + other => panic!("expected issue, got {other:?}"), + } + } + + #[tokio::test] + async fn try_decode_recovers_canon_pull_with_duplicate_dollar_type() { + let json = br#"{"$type":"sh.tangled.repo.pull","createdAt":"2026-05-01T00:00:00Z","title":"meow","target":{"branch":"main","repo":"at://did:plc:scallop/sh.tangled.repo/limpet"},"rounds":[],"$type":"sh.tangled.repo.pull"}"#; + let decoded = DecodedRecord::try_decode(&nsid("sh.tangled.repo.pull"), json) + .expect("duplicate $type pull must normalize-recover"); + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + resolver + .observe( + did("did:plc:scallop"), + rkey("limpet"), + Some(did("did:plc:scallop")), + ) + .await; + let canon = match decoded { + DecodedRecord::Canon(r) => r, + DecodedRecord::Legacy(l) => upgrade(l, &resolver).await.expect("upgrade"), + }; + match canon { + Record::Pull(p) => assert_eq!(AsRef::::as_ref(&p.title), "meow"), + other => panic!("expected pull, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_issue_uses_repo_did_directly() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let json = br#"{"$type":"sh.tangled.repo.issue","repoDid":"did:plc:scallop","title":"t","createdAt":"2026-05-01T00:00:00Z"}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.repo.issue"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::Issue(i) => assert_eq!(i.repo, did("did:plc:scallop")), + other => panic!("expected canon issue, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_issue_resolves_repo_uri_via_observed_resolver() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + resolver + .observe(owner.clone(), key.clone(), Some(did("did:plc:scallop"))) + .await; + let json = br#"{"$type":"sh.tangled.repo.issue","repo":"at://did:plc:nel/sh.tangled.repo/abcabcabcabcz","title":"t","createdAt":"2026-05-01T00:00:00Z"}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.repo.issue"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::Issue(i) => assert_eq!(i.repo, did("did:plc:scallop")), + other => panic!("expected canon issue, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_issue_drops_when_resolver_cannot_map() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let json = br#"{"$type":"sh.tangled.repo.issue","repo":"at://did:plc:nel/sh.tangled.repo/abcabcabcabcz","title":"t","createdAt":"2026-05-01T00:00:00Z"}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.repo.issue"), json).expect("decode"); + assert!( + upgrade(legacy, &resolver).await.is_none(), + "no resolver entry and no repoDid means the canon Did cannot be constructed", + ); + } + + #[tokio::test] + async fn upgrade_pull_propagates_target_resolution() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let json = br#"{"$type":"sh.tangled.repo.pull","title":"t","createdAt":"2026-05-01T00:00:00Z","rounds":[],"target":{"branch":"main","repoDid":"did:plc:scallop"}}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.repo.pull"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::Pull(p) => { + assert_eq!(p.target.repo, did("did:plc:scallop")); + assert!(p.source.is_none()); + } + other => panic!("expected canon pull, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_pull_pre_rounds_synthesizes_round_from_top_level_patch_blob() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let json = br#"{"$type":"sh.tangled.repo.pull","title":"t","createdAt":"2026-05-01T00:00:00Z","target":{"branch":"main","repoDid":"did:plc:scallop"},"patchBlob":{"$type":"blob","mimeType":"application/gzip","ref":{"$link":"bafkreibpatvbeajtwzlr4jwr4s2hnwo5l7sgdbfnqu6n7ctd2bcbtluw4a"},"size":920}}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.repo.pull"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::Pull(p) => { + assert_eq!( + p.rounds.len(), + 1, + "pre-rounds wire must yield exactly one synthesized round" + ); + assert_eq!( + p.rounds[0].patch_blob.blob().mime_type.as_ref(), + "application/gzip" + ); + assert_eq!(p.rounds[0].created_at, p.created_at); + } + other => panic!("expected canon pull, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_pull_omits_round_when_neither_rounds_nor_patch_blob_present() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let json = br#"{"$type":"sh.tangled.repo.pull","title":"t","createdAt":"2026-05-01T00:00:00Z","target":{"branch":"main","repoDid":"did:plc:scallop"}}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.repo.pull"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::Pull(p) => assert!(p.rounds.is_empty()), + other => panic!("expected canon pull, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_public_key_renames_created_to_created_at() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let json = br#"{"$type":"sh.tangled.publicKey","created":"2025-04-15T18:35:38Z","key":"ssh-ed25519 AAAA","name":"laptop"}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.publicKey"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::PublicKey(k) => { + assert_eq!(k.created_at.as_str(), "2025-04-15T18:35:38Z"); + assert_eq!(k.key.as_str(), "ssh-ed25519 AAAA"); + assert_eq!(k.name.as_str(), "laptop"); + } + other => panic!("expected canon publicKey, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_repo_renames_added_at_and_drops_owner() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let json = br#"{"$type":"sh.tangled.repo","addedAt":"2025-03-21T10:18:58Z","description":"hi","knot":"knot1.tangled.sh","name":"site","owner":"did:plc:nel"}"#; + let legacy = LegacyRecord::from_json_bytes(&nsid("sh.tangled.repo"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::Repo(r) => { + assert_eq!(r.created_at.as_str(), "2025-03-21T10:18:58Z"); + assert_eq!(r.description.as_deref(), Some("hi")); + assert_eq!(r.knot.as_str(), "knot1.tangled.sh"); + assert_eq!(r.name.as_deref(), Some("site")); + assert!(r.repo_did.is_none(), "legacy repos have no repo_did"); + } + other => panic!("expected canon repo, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_knot_member_renames_added_at_and_member() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let json = br#"{"$type":"sh.tangled.knot.member","addedAt":"2025-03-31T05:14:09Z","domain":"knot.example","member":"did:plc:nel"}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.knot.member"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::KnotMember(m) => { + assert_eq!(m.created_at.as_str(), "2025-03-31T05:14:09Z"); + assert_eq!(m.domain.as_str(), "knot.example"); + assert_eq!(m.subject, did("did:plc:nel")); + } + other => panic!("expected canon knot.member, got {other:?}"), + } + } + + #[tokio::test] + async fn legacy_pull_target_with_empty_repo_did_treats_as_none() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + resolver + .observe( + did("did:plc:nel"), + rkey("abcabcabcabcz"), + Some(did("did:plc:scallop")), + ) + .await; + let json = br#"{"$type":"sh.tangled.repo.pull","title":"t","createdAt":"2026-05-01T00:00:00Z","rounds":[],"target":{"branch":"main","repo":"at://did:plc:nel/sh.tangled.repo/abcabcabcabcz","repoDid":""}}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.repo.pull"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::Pull(p) => assert_eq!(p.target.repo, did("did:plc:scallop")), + other => panic!("expected canon pull, got {other:?}"), + } + } + + #[tokio::test] + async fn try_decode_recovers_publickey_with_created_field() { + let json = br#"{"$type":"sh.tangled.publicKey","created":"2025-04-15T18:35:38Z","key":"k","name":"n"}"#; + let decoded = DecodedRecord::try_decode(&nsid("sh.tangled.publicKey"), json) + .expect("legacy publicKey must decode"); + assert!(matches!( + decoded, + DecodedRecord::Legacy(LegacyRecord::PublicKey(_)) + )); + } + + #[tokio::test] + async fn try_decode_recovers_repo_with_added_at_field() { + let json = br#"{"$type":"sh.tangled.repo","addedAt":"2025-03-21T10:18:58Z","knot":"knot1.tangled.sh","owner":"did:plc:nel"}"#; + let decoded = DecodedRecord::try_decode(&nsid("sh.tangled.repo"), json) + .expect("legacy repo must decode"); + assert!(matches!( + decoded, + DecodedRecord::Legacy(LegacyRecord::Repo(_)) + )); + } + + #[tokio::test] + async fn upgrade_pull_source_repo_resolution_is_independent_of_target() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let json = br#"{"$type":"sh.tangled.repo.pull","title":"t","createdAt":"2026-05-01T00:00:00Z","rounds":[],"target":{"branch":"main","repoDid":"did:plc:scallop"},"source":{"branch":"feat","repo":"at://did:plc:nel/sh.tangled.repo/missing"}}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.repo.pull"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::Pull(p) => { + assert_eq!(p.target.repo, did("did:plc:scallop")); + let source = p.source.expect("source struct retained"); + assert_eq!(source.branch.as_str(), "feat"); + assert!( + source.repo.is_none(), + "unresolvable source repo at-uri leaves the source.repo None rather than dropping the whole pull", + ); + } + other => panic!("expected canon pull, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_ref_update_renames_repo_did_to_repo() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let json = br#"{"$type":"sh.tangled.git.refUpdate","ref":"refs/heads/main","committerDid":"did:plc:olaren","repoDid":"did:plc:scallop","oldSha":"0000000000000000000000000000000000000000","newSha":"1111111111111111111111111111111111111111","meta":{"isDefaultRef":true,"commitCount":{}}}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.git.refUpdate"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::RefUpdate(r) => assert_eq!(r.repo, did("did:plc:scallop")), + other => panic!("expected canon ref update, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_star_prefers_subject_did_over_subject_uri() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let json = br#"{"$type":"sh.tangled.feed.star","createdAt":"2026-05-01T00:00:00Z","subject":"at://did:plc:nel/sh.tangled.string/k1","subjectDid":"did:plc:scallop"}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.feed.star"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::Star(s) => match s.subject { + StarSubject::Repo(r) => assert_eq!(r.did, did("did:plc:scallop")), + StarSubject::String(_) => panic!("subjectDid must win"), + }, + other => panic!("expected canon star, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_star_falls_back_to_string_when_repo_uri_not_in_cache() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let json = br#"{"$type":"sh.tangled.feed.star","createdAt":"2026-05-01T00:00:00Z","subject":"at://did:plc:nel/sh.tangled.repo/abcabcabcabcz"}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.feed.star"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::Star(s) => match s.subject { + StarSubject::String(s) => assert_eq!( + s.uri.as_ref(), + "at://did:plc:nel/sh.tangled.repo/abcabcabcabcz", + "cache-miss on repo uri preserves the uri under the #string variant for later normalization", + ), + StarSubject::Repo(_) => panic!("cold cache must not upgrade to Repo variant"), + }, + other => panic!("expected canon star, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_star_uses_cached_repo_did_when_observed() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + resolver + .observe(owner.clone(), key.clone(), Some(did("did:plc:scallop"))) + .await; + let json = br#"{"$type":"sh.tangled.feed.star","createdAt":"2026-05-01T00:00:00Z","subject":"at://did:plc:nel/sh.tangled.repo/abcabcabcabcz"}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.feed.star"), json).expect("decode"); + let canon = upgrade(legacy, &resolver).await.expect("upgrade"); + match canon { + Record::Star(s) => match s.subject { + StarSubject::Repo(r) => assert_eq!(r.did, did("did:plc:scallop")), + StarSubject::String(_) => panic!("observed cache must upgrade to Repo variant"), + }, + other => panic!("expected canon star, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_collaborator_requires_repo_did() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let with_did = br#"{"$type":"sh.tangled.repo.collaborator","createdAt":"2026-05-01T00:00:00Z","subject":"did:plc:lyna","repoDid":"did:plc:scallop"}"#; + let canon = upgrade( + LegacyRecord::from_json_bytes(&nsid("sh.tangled.repo.collaborator"), with_did) + .expect("decode"), + &resolver, + ) + .await + .expect("upgrade"); + match canon { + Record::Collaborator(c) => assert_eq!(c.repo, did("did:plc:scallop")), + other => panic!("expected canon collaborator, got {other:?}"), + } + + let no_resolution = br#"{"$type":"sh.tangled.repo.collaborator","createdAt":"2026-05-01T00:00:00Z","subject":"did:plc:lyna","repo":"at://did:plc:nel/sh.tangled.repo/abcabcabcabcz"}"#; + let legacy = + LegacyRecord::from_json_bytes(&nsid("sh.tangled.repo.collaborator"), no_resolution) + .expect("decode"); + assert!(upgrade(legacy, &resolver).await.is_none()); + } +} diff --git a/bobbin/crates/resolver/src/lib.rs b/bobbin/crates/resolver/src/lib.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/resolver/src/lib.rs @@ -0,0 +1,1034 @@ +mod legacy_upgrade; +mod normalize; + +pub use legacy_upgrade::{ + DecodedRecord, decode_canon_or_upgrade, decode_canon_or_upgrade_bytes, normalize_record_fields, + scrub_record_bytes, synthesize_created_at, upgrade, upgrade_wire_bytes, +}; +pub use normalize::NormalizeRepoRefs; + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use tokio::time::Instant; + +use bobbin_runtime::{Clock, RuntimeHasher}; +use bobbin_slingshot_client::{SlingshotClient, SlingshotError}; +use bobbin_types::edges::{ExtractError, Record}; +use bobbin_types::ids::{RepoIdent, nsid_static}; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::recordkey::Rkey; +use scc::HashMap as SccMap; +use tokio::sync::OnceCell; +use tracing::warn; + +const REPO_COLLECTION: &str = "sh.tangled.repo"; +const TRANSIENT_TTL: Duration = Duration::from_secs(60); + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Resolution { + Mapped(Did), + NoRepoDid, + Unresolvable, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum AuthoritativeResolution { + Mapped(Did), + NoRepoDid, +} + +impl AuthoritativeResolution { + fn from_repo_did(repo_did: Option>) -> Self { + match repo_did { + Some(did) => Self::Mapped(did), + None => Self::NoRepoDid, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum CacheEntry { + Authoritative(AuthoritativeResolution), + Provisional(Resolution), + Transient { expires_at: Instant }, +} + +impl CacheEntry { + fn into_resolution(self) -> Resolution { + match self { + Self::Authoritative(AuthoritativeResolution::Mapped(did)) => Resolution::Mapped(did), + Self::Authoritative(AuthoritativeResolution::NoRepoDid) => Resolution::NoRepoDid, + Self::Provisional(r) => r, + Self::Transient { .. } => Resolution::Unresolvable, + } + } + + fn is_expired_transient(&self, now: Instant) -> bool { + matches!(self, Self::Transient { expires_at } if *expires_at <= now) + } +} + +#[derive(Default)] +pub struct ResolverStats { + hits: AtomicU64, + misses_mapped: AtomicU64, + misses_no_repo_did: AtomicU64, + misses_unresolvable: AtomicU64, + misses_transient: AtomicU64, + misses_no_client: AtomicU64, + miss_latency_micros_sum: AtomicU64, + miss_latency_micros_max: AtomicU64, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct ResolverStatsSnapshot { + pub hits: u64, + pub misses_mapped: u64, + pub misses_no_repo_did: u64, + pub misses_unresolvable: u64, + pub misses_transient: u64, + pub misses_no_client: u64, + pub miss_latency_micros_sum: u64, + pub miss_latency_micros_max: u64, +} + +impl ResolverStatsSnapshot { + pub fn miss_count(&self) -> u64 { + self.misses_mapped + + self.misses_no_repo_did + + self.misses_unresolvable + + self.misses_transient + + self.misses_no_client + } + + pub fn total(&self) -> u64 { + self.hits + self.miss_count() + } + + pub fn miss_latency_micros_avg(&self) -> Option { + let misses = self.miss_count() - self.misses_no_client; + (misses > 0).then(|| self.miss_latency_micros_sum / misses) + } +} + +#[derive(Clone, Copy)] +enum MissKind { + Mapped, + NoRepoDid, + Unresolvable, + Transient, + NoClient, +} + +impl ResolverStats { + fn record_hit(&self) { + self.hits.fetch_add(1, Ordering::Relaxed); + } + + fn record_miss(&self, kind: MissKind, latency: Option) { + let counter = match kind { + MissKind::Mapped => &self.misses_mapped, + MissKind::NoRepoDid => &self.misses_no_repo_did, + MissKind::Unresolvable => &self.misses_unresolvable, + MissKind::Transient => &self.misses_transient, + MissKind::NoClient => &self.misses_no_client, + }; + counter.fetch_add(1, Ordering::Relaxed); + if let Some(latency) = latency { + let micros = u64::try_from(latency.as_micros()).unwrap_or(u64::MAX); + self.miss_latency_micros_sum + .fetch_add(micros, Ordering::Relaxed); + self.miss_latency_micros_max + .fetch_max(micros, Ordering::Relaxed); + } + } + + pub fn snapshot(&self) -> ResolverStatsSnapshot { + ResolverStatsSnapshot { + hits: self.hits.load(Ordering::Relaxed), + misses_mapped: self.misses_mapped.load(Ordering::Relaxed), + misses_no_repo_did: self.misses_no_repo_did.load(Ordering::Relaxed), + misses_unresolvable: self.misses_unresolvable.load(Ordering::Relaxed), + misses_transient: self.misses_transient.load(Ordering::Relaxed), + misses_no_client: self.misses_no_client.load(Ordering::Relaxed), + miss_latency_micros_sum: self.miss_latency_micros_sum.load(Ordering::Relaxed), + miss_latency_micros_max: self.miss_latency_micros_max.load(Ordering::Relaxed), + } + } +} + +struct SlingshotProbe { + client: SlingshotClient, + clock: Arc, +} + +pub struct RepoIdResolver { + cache: SccMap, + by_repo_did: SccMap, RepoIdent, RuntimeHasher>, + in_flight: SccMap>, RuntimeHasher>, + probe: Option, + stats: ResolverStats, +} + +impl RepoIdResolver { + pub fn with_slingshot( + client: SlingshotClient, + clock: Arc, + hasher: RuntimeHasher, + ) -> Self { + Self { + cache: SccMap::with_hasher(hasher.clone()), + by_repo_did: SccMap::with_hasher(hasher.clone()), + in_flight: SccMap::with_hasher(hasher), + probe: Some(SlingshotProbe { client, clock }), + stats: ResolverStats::default(), + } + } + + pub fn detached(hasher: RuntimeHasher) -> Self { + Self { + cache: SccMap::with_hasher(hasher.clone()), + by_repo_did: SccMap::with_hasher(hasher.clone()), + in_flight: SccMap::with_hasher(hasher), + probe: None, + stats: ResolverStats::default(), + } + } + + pub fn stats(&self) -> ResolverStatsSnapshot { + self.stats.snapshot() + } + + pub async fn cached_resolution( + &self, + owner: &Did, + rkey: &Rkey, + ) -> Option { + let key = RepoIdent::new(owner.clone(), rkey.clone()); + let entry = self.cache.get_async(&key).await?; + let now = self.probe.as_ref().map(|p| p.clock.now_instant()); + if let Some(now) = now + && entry.get().is_expired_transient(now) + { + return None; + } + Some(entry.get().clone().into_resolution()) + } + + pub async fn lookup_by_repo_did(&self, repo_did: &Did) -> Option { + self.by_repo_did + .get_async(repo_did) + .await + .map(|e| e.get().clone()) + } + + pub async fn observe( + &self, + owner: Did, + rkey: Rkey, + repo_did: Option>, + ) -> Option { + let ident = RepoIdent::new(owner, rkey); + let entry = + CacheEntry::Authoritative(AuthoritativeResolution::from_repo_did(repo_did.clone())); + self.cache + .entry_async(ident.clone()) + .await + .and_modify(|existing| *existing = entry.clone()) + .or_insert(entry); + + let repo_did = repo_did?; + let mut prior: Option = None; + self.by_repo_did + .entry_async(repo_did) + .await + .and_modify(|existing| { + if *existing != ident { + prior = Some(existing.clone()); + *existing = ident.clone(); + } + }) + .or_insert(ident); + prior + } + + pub async fn forget(&self, owner: &Did, rkey: &Rkey) { + let ident = RepoIdent::new(owner.clone(), rkey.clone()); + let prior_resolution = self + .cache + .remove_async(&ident) + .await + .map(|(_, entry)| entry.into_resolution()); + if let Some(Resolution::Mapped(repo_did)) = prior_resolution { + self.by_repo_did + .remove_if_async(&repo_did, |existing| *existing == ident) + .await; + } + } + + async fn fill_provisional(&self, key: RepoIdent, resolution: Resolution) { + let entry = CacheEntry::Provisional(resolution); + self.cache + .entry_async(key) + .await + .and_modify(|existing| { + if matches!(existing, CacheEntry::Authoritative(_)) { + return; + } + *existing = entry.clone(); + }) + .or_insert(entry); + } + + async fn fill_transient(&self, key: RepoIdent, expires_at: Instant) { + let entry = CacheEntry::Transient { expires_at }; + self.cache + .entry_async(key) + .await + .and_modify(|existing| { + if matches!(existing, CacheEntry::Authoritative(_)) { + return; + } + *existing = entry.clone(); + }) + .or_insert(entry); + } + + pub async fn resolve(&self, owner: &Did, rkey: &Rkey) -> Resolution { + let key = RepoIdent::new(owner.clone(), rkey.clone()); + + let Some(probe) = self.probe.as_ref() else { + if let Some(entry) = self.cache.get_async(&key).await { + self.stats.record_hit(); + return entry.get().clone().into_resolution(); + } + self.stats.record_miss(MissKind::NoClient, None); + return Resolution::Unresolvable; + }; + + let now = probe.clock.now_instant(); + if let Some(entry) = self.cache.get_async(&key).await + && !entry.get().is_expired_transient(now) + { + self.stats.record_hit(); + return entry.get().clone().into_resolution(); + } + + let cell: Arc> = self + .in_flight + .entry_async(key.clone()) + .await + .or_insert_with(|| Arc::new(OnceCell::new())) + .get() + .clone(); + + let result = cell + .get_or_init(|| async { self.fetch_repo_did(owner, rkey, &key).await }) + .await + .clone(); + + self.in_flight.remove_async(&key).await; + + result + } + + async fn fetch_repo_did( + &self, + owner: &Did, + rkey: &Rkey, + key: &RepoIdent, + ) -> Resolution { + let probe = self + .probe + .as_ref() + .expect("fetch_repo_did is only called when a probe is present"); + let started = probe.clock.now_instant(); + let nsid: Nsid = nsid_static(REPO_COLLECTION); + let provisional = match probe.client.get_record(owner, &nsid, rkey).await { + Ok(body) => match repo_did_from_body(&nsid, &body.value) { + Ok(Some(did)) => Resolution::Mapped(did), + Ok(None) => Resolution::NoRepoDid, + Err(e) => { + warn!( + error = ?e, + owner = owner.as_ref(), + rkey = rkey.as_ref(), + "slingshot returned unparseable repo body, caching as unresolvable", + ); + Resolution::Unresolvable + } + }, + Err(SlingshotError::NotFound) => { + warn!( + owner = owner.as_ref(), + rkey = rkey.as_ref(), + "no repo record on slingshot, caching as unresolvable", + ); + Resolution::Unresolvable + } + Err(ref e) if is_garbage_response(e) => { + warn!( + error = ?e, + owner = owner.as_ref(), + rkey = rkey.as_ref(), + "slingshot returned malformed response, caching as unresolvable", + ); + Resolution::Unresolvable + } + Err(e) => { + warn!( + error = ?e, + owner = owner.as_ref(), + rkey = rkey.as_ref(), + "caching transient slingshot failure for repoDID lookup under short TTL", + ); + let elapsed = probe.clock.now_instant().duration_since(started); + self.stats.record_miss(MissKind::Transient, Some(elapsed)); + let expires_at = probe.clock.now_instant() + TRANSIENT_TTL; + self.fill_transient(key.clone(), expires_at).await; + return Resolution::Unresolvable; + } + }; + let elapsed = probe.clock.now_instant().duration_since(started); + let kind = match &provisional { + Resolution::Mapped(_) => MissKind::Mapped, + Resolution::NoRepoDid => MissKind::NoRepoDid, + Resolution::Unresolvable => MissKind::Unresolvable, + }; + self.stats.record_miss(kind, Some(elapsed)); + self.fill_provisional(key.clone(), provisional.clone()) + .await; + provisional + } +} + +fn is_garbage_response(err: &SlingshotError) -> bool { + matches!( + err, + SlingshotError::Decode(_) + | SlingshotError::MissingField(_) + | SlingshotError::InvalidAtUri(_) + | SlingshotError::InvalidCid(_) + | SlingshotError::UriMismatch { .. }, + ) +} + +fn repo_did_from_body( + nsid: &Nsid, + body: &[u8], +) -> Result>, ExtractError> { + match DecodedRecord::try_decode(nsid, body)? { + DecodedRecord::Canon(Record::Repo(repo)) => Ok(repo.repo_did), + DecodedRecord::Canon(_) | DecodedRecord::Legacy(_) => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bobbin_runtime::SystemClock; + use jacquard_common::types::did::Did; + use jacquard_common::types::recordkey::Rkey; + + fn did(s: &str) -> Did { + Did::new_owned(s).unwrap() + } + + fn rkey(s: &str) -> Rkey { + Rkey::new_owned(s).unwrap() + } + + fn test_clock() -> Arc { + Arc::new(SystemClock::new()) + } + + #[tokio::test] + async fn observation_returns_prior_ident_when_repo_did_moves() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let prior = resolver + .observe( + did("did:plc:nel"), + rkey("3liuighjy2h22"), + Some(did("did:plc:clam")), + ) + .await; + assert!(prior.is_none(), "first observation has no prior"); + + let prior = resolver + .observe(did("did:plc:nel"), rkey("core"), Some(did("did:plc:clam"))) + .await; + assert_eq!( + prior, + Some(RepoIdent::new(did("did:plc:nel"), rkey("3liuighjy2h22"))), + "same repoDID at a new (owner, rkey) returns the prior ident so callers can evict the stale at-uri", + ); + + let prior = resolver + .observe(did("did:plc:nel"), rkey("core"), Some(did("did:plc:clam"))) + .await; + assert!(prior.is_none(), "re-observing the same ident is a no-op"); + } + + #[tokio::test] + async fn observation_without_repo_did_does_not_track_reverse() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let prior = resolver + .observe(did("did:plc:nel"), rkey("abcabcabcabcz"), None) + .await; + assert!(prior.is_none()); + } + + #[tokio::test] + async fn forget_clears_reverse_only_when_still_owned() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + resolver + .observe( + did("did:plc:nel"), + rkey("3liuighjy2h22"), + Some(did("did:plc:clam")), + ) + .await; + resolver + .observe(did("did:plc:nel"), rkey("core"), Some(did("did:plc:clam"))) + .await; + + resolver + .forget(&did("did:plc:nel"), &rkey("3liuighjy2h22")) + .await; + + let prior = resolver + .observe( + did("did:plc:nel"), + rkey("core-renamed"), + Some(did("did:plc:clam")), + ) + .await; + assert_eq!( + prior, + Some(RepoIdent::new(did("did:plc:nel"), rkey("core"))), + "stale at-uri's forget must not displace the live owner of did:plc:clam", + ); + } + + #[tokio::test] + async fn observation_with_repo_did_resolves_mapped() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + resolver + .observe( + did("did:plc:nel"), + rkey("abcabcabcabcz"), + Some(did("did:plc:clam")), + ) + .await; + let got = resolver + .resolve(&did("did:plc:nel"), &rkey("abcabcabcabcz")) + .await; + assert_eq!(got, Resolution::Mapped(did("did:plc:clam"))); + } + + #[tokio::test] + async fn observation_without_repo_did_resolves_no_repo_did() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + resolver + .observe(did("did:plc:nel"), rkey("abcabcabcabcz"), None) + .await; + let got = resolver + .resolve(&did("did:plc:nel"), &rkey("abcabcabcabcz")) + .await; + assert_eq!( + got, + Resolution::NoRepoDid, + "observed but empty repoDID is a definitive answer not a lookup failure", + ); + } + + #[tokio::test] + async fn cache_miss_without_client_is_unresolvable() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let got = resolver + .resolve(&did("did:plc:nel"), &rkey("abcabcabcabcz")) + .await; + assert_eq!(got, Resolution::Unresolvable); + } + + #[tokio::test] + async fn lookup_by_repo_did_finds_observed_ident() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + resolver + .observe( + did("did:plc:nel"), + rkey("abcabcabcabcz"), + Some(did("did:plc:limpet")), + ) + .await; + let got = resolver.lookup_by_repo_did(&did("did:plc:limpet")).await; + assert_eq!( + got, + Some(RepoIdent::new(did("did:plc:nel"), rkey("abcabcabcabcz"))), + ); + } + + #[tokio::test] + async fn lookup_by_repo_did_misses_when_unobserved() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let got = resolver.lookup_by_repo_did(&did("did:plc:limpet")).await; + assert_eq!(got, None); + } + + #[tokio::test] + async fn lookup_by_repo_did_misses_when_repo_did_was_none() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + resolver + .observe(did("did:plc:nel"), rkey("abcabcabcabcz"), None) + .await; + let got = resolver.lookup_by_repo_did(&did("did:plc:limpet")).await; + assert_eq!(got, None); + } + + #[tokio::test] + async fn lookup_by_repo_did_follows_move_to_new_ident() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + resolver + .observe( + did("did:plc:nel"), + rkey("abcabcabcabcz"), + Some(did("did:plc:limpet")), + ) + .await; + resolver + .observe( + did("did:plc:olaren"), + rkey("xyzxyzxyzxyzx"), + Some(did("did:plc:limpet")), + ) + .await; + let got = resolver.lookup_by_repo_did(&did("did:plc:limpet")).await; + assert_eq!( + got, + Some(RepoIdent::new(did("did:plc:olaren"), rkey("xyzxyzxyzxyzx"))), + ); + } + + #[tokio::test] + async fn observation_overwrites_prior_value() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + resolver + .observe( + did("did:plc:nel"), + rkey("abcabcabcabcz"), + Some(did("did:plc:clam")), + ) + .await; + resolver + .observe( + did("did:plc:nel"), + rkey("abcabcabcabcz"), + Some(did("did:plc:uni")), + ) + .await; + let got = resolver + .resolve(&did("did:plc:nel"), &rkey("abcabcabcabcz")) + .await; + assert_eq!(got, Resolution::Mapped(did("did:plc:uni"))); + } + + #[tokio::test] + async fn fill_provisional_does_not_downgrade_authoritative_mapped() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + resolver + .observe(owner.clone(), key.clone(), Some(did("did:plc:clam"))) + .await; + resolver + .fill_provisional( + RepoIdent::new(owner.clone(), key.clone()), + Resolution::Unresolvable, + ) + .await; + let got = resolver.resolve(&owner, &key).await; + assert_eq!( + got, + Resolution::Mapped(did("did:plc:clam")), + "firehose-observed mapping must outrank provisional slingshot info", + ); + } + + #[tokio::test] + async fn fill_provisional_does_not_downgrade_authoritative_no_repo_did() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + resolver.observe(owner.clone(), key.clone(), None).await; + resolver + .fill_provisional( + RepoIdent::new(owner.clone(), key.clone()), + Resolution::Mapped(did("did:plc:clam")), + ) + .await; + let got = resolver.resolve(&owner, &key).await; + assert_eq!( + got, + Resolution::NoRepoDid, + "an authoritative empty observation must outrank provisional slingshot info even when slingshot disagrees", + ); + } + + #[tokio::test] + async fn slingshot_404_caches_as_unresolvable() { + let server = wiremock::MockServer::start().await; + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(wiremock::ResponseTemplate::new(404)) + .expect(1) + .mount(&server) + .await; + + let client = + SlingshotClient::with_default_http(url::Url::parse(&server.uri()).unwrap()).unwrap(); + let resolver = + RepoIdResolver::with_slingshot(client, test_clock(), RuntimeHasher::default()); + + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + let first = resolver.resolve(&owner, &key).await; + let second = resolver.resolve(&owner, &key).await; + assert_eq!(first, Resolution::Unresolvable); + assert_eq!(second, Resolution::Unresolvable); + } + + #[tokio::test] + async fn slingshot_malformed_envelope_caches_as_unresolvable() { + let server = wiremock::MockServer::start().await; + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/xrpc/com.atproto.repo.getRecord")) + .respond_with( + wiremock::ResponseTemplate::new(200) + .insert_header("content-type", "application/json") + .set_body_string("not json"), + ) + .expect(1) + .mount(&server) + .await; + + let client = + SlingshotClient::with_default_http(url::Url::parse(&server.uri()).unwrap()).unwrap(); + let resolver = + RepoIdResolver::with_slingshot(client, test_clock(), RuntimeHasher::default()); + + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + let first = resolver.resolve(&owner, &key).await; + let second = resolver.resolve(&owner, &key).await; + assert_eq!(first, Resolution::Unresolvable); + assert_eq!( + second, + Resolution::Unresolvable, + "garbage envelopes are stable across retries, so caching avoids hammering slingshot", + ); + } + + #[tokio::test] + async fn slingshot_uri_mismatch_caches_as_unresolvable() { + let server = wiremock::MockServer::start().await; + let body = serde_json::json!({ + "uri": "at://did:plc:limpet/sh.tangled.repo/elsewhere", + "cid": "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i", + "value": {"$type": "sh.tangled.repo", "knot": "oyster.cafe", "createdAt": "2026-05-01T00:00:00Z"} + }); + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(body)) + .expect(1) + .mount(&server) + .await; + + let client = + SlingshotClient::with_default_http(url::Url::parse(&server.uri()).unwrap()).unwrap(); + let resolver = + RepoIdResolver::with_slingshot(client, test_clock(), RuntimeHasher::default()); + + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + let first = resolver.resolve(&owner, &key).await; + let second = resolver.resolve(&owner, &key).await; + assert_eq!(first, Resolution::Unresolvable); + assert_eq!(second, Resolution::Unresolvable); + } + + #[tokio::test] + async fn slingshot_legacy_repo_body_resolves_no_repo_did_not_unresolvable() { + let server = wiremock::MockServer::start().await; + let body = serde_json::json!({ + "uri": "at://did:plc:nel/sh.tangled.repo/abcabcabcabcz", + "cid": "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i", + "value": { + "$type": "sh.tangled.repo", + "addedAt": "2025-03-07T21:47:53Z", + "knot": "knot1.tangled.sh", + "name": "scallop", + "owner": "did:plc:nel", + }, + }); + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(body)) + .expect(1) + .mount(&server) + .await; + + let client = + SlingshotClient::with_default_http(url::Url::parse(&server.uri()).unwrap()).unwrap(); + let resolver = + RepoIdResolver::with_slingshot(client, test_clock(), RuntimeHasher::default()); + + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + let got = resolver.resolve(&owner, &key).await; + assert_eq!( + got, + Resolution::NoRepoDid, + "legacy repo wires without a repo_did parse via legacy upgrade and resolve as NoRepoDid, not Unresolvable", + ); + } + + #[tokio::test] + async fn slingshot_unparseable_repo_value_caches_as_unresolvable() { + let server = wiremock::MockServer::start().await; + let body = serde_json::json!({ + "uri": "at://did:plc:nel/sh.tangled.repo/abcabcabcabcz", + "cid": "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i", + "value": {"$type": "sh.tangled.repo"} + }); + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(body)) + .expect(1) + .mount(&server) + .await; + + let client = + SlingshotClient::with_default_http(url::Url::parse(&server.uri()).unwrap()).unwrap(); + let resolver = + RepoIdResolver::with_slingshot(client, test_clock(), RuntimeHasher::default()); + + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + let first = resolver.resolve(&owner, &key).await; + let second = resolver.resolve(&owner, &key).await; + assert_eq!( + first, + Resolution::Unresolvable, + "a repo body that fails lexicon validation must not be conflated with NoRepoDid", + ); + assert_eq!(second, Resolution::Unresolvable); + } + + #[tokio::test] + async fn slingshot_transport_error_caches_with_short_ttl() { + let server = wiremock::MockServer::start().await; + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(wiremock::ResponseTemplate::new(503)) + .expect(1) + .mount(&server) + .await; + + let client = + SlingshotClient::with_default_http(url::Url::parse(&server.uri()).unwrap()).unwrap(); + let resolver = + RepoIdResolver::with_slingshot(client, test_clock(), RuntimeHasher::default()); + + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + let first = resolver.resolve(&owner, &key).await; + let second = resolver.resolve(&owner, &key).await; + assert_eq!(first, Resolution::Unresolvable); + assert_eq!( + second, + Resolution::Unresolvable, + "transient TTL must suppress immediate re-hammering of a sick upstream", + ); + } + + #[tokio::test] + async fn slingshot_transient_recorded_separately_from_unresolvable() { + let server = wiremock::MockServer::start().await; + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(wiremock::ResponseTemplate::new(503)) + .expect(1) + .mount(&server) + .await; + + let client = + SlingshotClient::with_default_http(url::Url::parse(&server.uri()).unwrap()).unwrap(); + let resolver = + RepoIdResolver::with_slingshot(client, test_clock(), RuntimeHasher::default()); + + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + resolver.resolve(&owner, &key).await; + resolver.resolve(&owner, &key).await; + + let snap = resolver.stats(); + assert_eq!( + snap.misses_transient, 1, + "second resolve must hit the short-TTL cache instead of re-firing the transient miss", + ); + assert_eq!(snap.hits, 1, "second call hits cached transient entry"); + assert_eq!( + snap.misses_unresolvable, 0, + "canonical unresolvable counter is reserved for cached terminal answers", + ); + assert!( + snap.miss_latency_micros_sum > 0, + "transient misses still have latency contributions", + ); + assert_eq!(snap.miss_count(), 1); + } + + #[tokio::test] + async fn slingshot_in_flight_requests_coalesce() { + let server = wiremock::MockServer::start().await; + let body = serde_json::json!({ + "uri": "at://did:plc:nel/sh.tangled.repo/abcabcabcabcz", + "cid": "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i", + "value": {"$type": "sh.tangled.repo", "knot": "oyster.cafe", "createdAt": "2026-05-01T00:00:00Z", "repoDid": "did:plc:limpet"} + }); + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/xrpc/com.atproto.repo.getRecord")) + .respond_with( + wiremock::ResponseTemplate::new(200) + .set_body_json(body) + .set_delay(Duration::from_millis(200)), + ) + .expect(1) + .mount(&server) + .await; + + let client = + SlingshotClient::with_default_http(url::Url::parse(&server.uri()).unwrap()).unwrap(); + let resolver = Arc::new(RepoIdResolver::with_slingshot( + client, + test_clock(), + RuntimeHasher::default(), + )); + + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + let r0 = resolver.clone(); + let r1 = resolver.clone(); + let r2 = resolver.clone(); + let o0 = owner.clone(); + let o1 = owner.clone(); + let o2 = owner.clone(); + let k0 = key.clone(); + let k1 = key.clone(); + let k2 = key.clone(); + let (a, b, c) = tokio::join!( + tokio::spawn(async move { r0.resolve(&o0, &k0).await }), + tokio::spawn(async move { r1.resolve(&o1, &k1).await }), + tokio::spawn(async move { r2.resolve(&o2, &k2).await }), + ); + let expected = Resolution::Mapped(did("did:plc:limpet")); + assert_eq!(a.unwrap(), expected); + assert_eq!(b.unwrap(), expected); + assert_eq!(c.unwrap(), expected); + + let snap = resolver.stats(); + assert_eq!( + snap.misses_mapped, 1, + "only the winning task pays the slingshot RTT", + ); + } + + #[tokio::test] + async fn stats_count_hits_misses_and_latency() { + let server = wiremock::MockServer::start().await; + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(wiremock::ResponseTemplate::new(404)) + .mount(&server) + .await; + let client = + SlingshotClient::with_default_http(url::Url::parse(&server.uri()).unwrap()).unwrap(); + let resolver = + RepoIdResolver::with_slingshot(client, test_clock(), RuntimeHasher::default()); + + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + resolver.resolve(&owner, &key).await; + resolver.resolve(&owner, &key).await; + + let snap = resolver.stats(); + assert_eq!( + snap.misses_unresolvable, 1, + "first call is the slingshot miss" + ); + assert_eq!(snap.hits, 1, "second call hits the unresolvable cache"); + assert_eq!(snap.miss_count(), 1); + assert_eq!(snap.total(), 2); + assert!( + snap.miss_latency_micros_sum > 0, + "latency recorded for slingshot miss" + ); + assert!(snap.miss_latency_micros_avg().unwrap() > 0); + } + + #[tokio::test] + async fn stats_no_client_miss_recorded_without_latency() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + resolver + .resolve(&did("did:plc:nel"), &rkey("abcabcabcabcz")) + .await; + let snap = resolver.stats(); + assert_eq!(snap.misses_no_client, 1); + assert_eq!(snap.miss_latency_micros_sum, 0); + assert_eq!(snap.miss_latency_micros_avg(), None); + } + + #[tokio::test] + async fn firehose_observe_can_demote_provisional() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + resolver + .fill_provisional( + RepoIdent::new(owner.clone(), key.clone()), + Resolution::Mapped(did("did:plc:clam")), + ) + .await; + resolver.observe(owner.clone(), key.clone(), None).await; + let got = resolver.resolve(&owner, &key).await; + assert_eq!( + got, + Resolution::NoRepoDid, + "firehose update is canonical and may legitimately remove repoDID", + ); + } + + #[tokio::test] + async fn forget_removes_cache_entry() { + let resolver = RepoIdResolver::detached(RuntimeHasher::default()); + let owner = did("did:plc:nel"); + let key = rkey("abcabcabcabcz"); + resolver + .observe(owner.clone(), key.clone(), Some(did("did:plc:clam"))) + .await; + assert_eq!( + resolver.cached_resolution(&owner, &key).await, + Some(Resolution::Mapped(did("did:plc:clam"))), + ); + resolver.forget(&owner, &key).await; + assert_eq!( + resolver.cached_resolution(&owner, &key).await, + None, + "forget must drop the entry entirely so a subsequent observe can supply fresh state", + ); + } +} diff --git a/bobbin/crates/resolver/src/normalize.rs b/bobbin/crates/resolver/src/normalize.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/resolver/src/normalize.rs @@ -0,0 +1,155 @@ +use crate::{RepoIdResolver, Resolution}; +use bobbin_types::search::SearchableRecord; +use bobbin_types::sh_tangled::repo::artifact::Artifact; +use jacquard_common::DefaultStr; +use jacquard_common::IntoStatic; +use jacquard_common::types::did::Did; +use jacquard_common::types::ident::AtIdentifier; +use jacquard_common::types::recordkey::Rkey; +use jacquard_common::types::string::AtUri; + +pub(crate) const REPO_COLLECTION: &str = "sh.tangled.repo"; + +pub trait NormalizeRepoRefs: Sized { + fn normalize( + self, + resolver: &RepoIdResolver, + ) -> impl std::future::Future> + Send; +} + +pub(crate) fn is_repo_at_uri(uri: &AtUri) -> bool { + uri.collection() + .map(|c| c.as_ref() == REPO_COLLECTION) + .unwrap_or(false) +} + +pub(crate) async fn resolve_repo_uri( + resolver: &RepoIdResolver, + uri: &AtUri, +) -> Option> { + if !is_repo_at_uri(uri) { + return None; + } + let owner = match uri.authority() { + AtIdentifier::Did(d) => d.clone().into_static(), + AtIdentifier::Handle(_) => return None, + }; + let rkey: Rkey = uri.rkey()?.clone().into_static(); + match resolver.resolve(&owner, &rkey).await { + Resolution::Mapped(did) => Some(did), + Resolution::NoRepoDid | Resolution::Unresolvable => None, + } +} + +async fn fill_repo_did( + resolver: &RepoIdResolver, + uri_field: &mut Option>, + did_field: &mut Option>, +) -> bool { + if did_field.is_some() { + *uri_field = None; + return true; + } + let Some(uri) = uri_field.as_ref() else { + return false; + }; + let Some(did) = resolve_repo_uri(resolver, uri).await else { + return false; + }; + *did_field = Some(did); + *uri_field = None; + true +} + +impl NormalizeRepoRefs for Artifact { + async fn normalize(mut self, resolver: &RepoIdResolver) -> Option { + if !fill_repo_did(resolver, &mut self.repo, &mut self.repo_did).await { + return None; + } + Some(self) + } +} + +impl NormalizeRepoRefs for bobbin_types::sh_tangled::pipeline::Pipeline { + async fn normalize(mut self, resolver: &RepoIdResolver) -> Option { + let trig = &mut self.trigger_metadata.repo; + if trig.repo_did.is_some() { + trig.repo = None; + return Some(self); + } + let raw = trig.repo.as_deref()?; + let parsed = AtUri::::new_owned(raw).ok()?; + let did = resolve_repo_uri(resolver, &parsed).await?; + trig.repo_did = Some(did); + trig.repo = None; + Some(self) + } +} + +impl NormalizeRepoRefs for SearchableRecord { + async fn normalize(self, _resolver: &RepoIdResolver) -> Option { + Some(self) + } +} + +macro_rules! identity_normalize { + ($($t:ty),+ $(,)?) => { + $( + impl NormalizeRepoRefs for $t { + async fn normalize(self, _resolver: &RepoIdResolver) -> Option { + Some(self) + } + } + )+ + }; +} + +use bobbin_types::sh_tangled::actor::profile::Profile; +use bobbin_types::sh_tangled::feed::reaction::Reaction; +use bobbin_types::sh_tangled::feed::star::Star; +use bobbin_types::sh_tangled::git::ref_update::RefUpdate; +use bobbin_types::sh_tangled::graph::follow::Follow; +use bobbin_types::sh_tangled::graph::vouch::Vouch; +use bobbin_types::sh_tangled::knot::Knot; +use bobbin_types::sh_tangled::knot::member::Member as KnotMember; +use bobbin_types::sh_tangled::label::definition::Definition as LabelDefinition; +use bobbin_types::sh_tangled::label::op::Op as LabelOp; +use bobbin_types::sh_tangled::pipeline::status::Status as PipelineStatus; +use bobbin_types::sh_tangled::public_key::PublicKey; +use bobbin_types::sh_tangled::repo::Repo; +use bobbin_types::sh_tangled::repo::collaborator::Collaborator; +use bobbin_types::sh_tangled::repo::issue::Issue; +use bobbin_types::sh_tangled::repo::issue::comment::Comment as IssueComment; +use bobbin_types::sh_tangled::repo::issue::state::State as IssueState; +use bobbin_types::sh_tangled::repo::pull::Pull; +use bobbin_types::sh_tangled::repo::pull::comment::Comment as PullComment; +use bobbin_types::sh_tangled::repo::pull::status::Status as PullStatus; +use bobbin_types::sh_tangled::spindle::Spindle; +use bobbin_types::sh_tangled::spindle::member::Member as SpindleMember; +use bobbin_types::sh_tangled::string::TangledString; + +identity_normalize!( + Profile, + Reaction, + Star, + RefUpdate, + Follow, + Vouch, + Knot, + KnotMember, + LabelDefinition, + LabelOp, + PipelineStatus, + PublicKey, + Repo, + Collaborator, + Issue, + IssueComment, + IssueState, + Pull, + PullComment, + PullStatus, + Spindle, + SpindleMember, + TangledString, +); diff --git a/bobbin/crates/runtime/Cargo.toml b/bobbin/crates/runtime/Cargo.toml new file mode 100644 --- /dev/null +++ b/bobbin/crates/runtime/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "bobbin-runtime" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +ahash = { workspace = true } +bytes = { workspace = true } +futures = { workspace = true } +getrandom = { workspace = true } +http = { workspace = true } +reqwest = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-tungstenite = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/bobbin/crates/runtime/src/clock.rs b/bobbin/crates/runtime/src/clock.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/runtime/src/clock.rs @@ -0,0 +1,234 @@ +use std::future::Future; +use std::pin::Pin; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use tokio::time::Instant; + +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct UnixMicros(u64); + +impl UnixMicros { + pub const fn new(value: u64) -> Self { + Self(value) + } + + pub const fn raw(self) -> u64 { + self.0 + } +} + +pub type SleepFuture = Pin + Send + 'static>>; + +pub trait Clock: Send + Sync + 'static { + fn now_unix_micros(&self) -> UnixMicros; + fn now_instant(&self) -> Instant; + fn sleep(&self, duration: Duration) -> SleepFuture; + fn sleep_until(&self, deadline: Instant) -> SleepFuture; +} + +#[derive(Clone, Copy, Debug)] +pub struct SystemClock { + base_unix: UnixMicros, + base_instant: Instant, +} + +impl SystemClock { + pub fn new() -> Self { + let raw = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock before unix epoch") + .as_micros(); + Self { + base_unix: UnixMicros::new(u64::try_from(raw).unwrap_or(u64::MAX)), + base_instant: Instant::now(), + } + } +} + +impl Default for SystemClock { + fn default() -> Self { + Self::new() + } +} + +impl Clock for SystemClock { + fn now_unix_micros(&self) -> UnixMicros { + let elapsed = self + .now_instant() + .saturating_duration_since(self.base_instant); + let micros = u64::try_from(elapsed.as_micros()).unwrap_or(u64::MAX); + UnixMicros::new(self.base_unix.raw().saturating_add(micros)) + } + + fn now_instant(&self) -> Instant { + Instant::now() + } + + fn sleep(&self, duration: Duration) -> SleepFuture { + Box::pin(tokio::time::sleep(duration)) + } + + fn sleep_until(&self, deadline: Instant) -> SleepFuture { + Box::pin(tokio::time::sleep_until(deadline)) + } +} + +#[derive(Clone, Debug)] +pub struct SimClock { + base_unix: UnixMicros, + base_instant: Instant, +} + +impl SimClock { + pub fn at(base_unix: UnixMicros) -> Self { + Self { + base_unix, + base_instant: Instant::now(), + } + } +} + +impl Clock for SimClock { + fn now_unix_micros(&self) -> UnixMicros { + let elapsed = self + .now_instant() + .saturating_duration_since(self.base_instant); + let micros = u64::try_from(elapsed.as_micros()).unwrap_or(u64::MAX); + UnixMicros::new(self.base_unix.raw().saturating_add(micros)) + } + + fn now_instant(&self) -> Instant { + Instant::now() + } + + fn sleep(&self, duration: Duration) -> SleepFuture { + Box::pin(tokio::time::sleep(duration)) + } + + fn sleep_until(&self, deadline: Instant) -> SleepFuture { + Box::pin(tokio::time::sleep_until(deadline)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::atomic::{AtomicU64, Ordering}; + + struct ManualClock { + base: Instant, + unix_micros: AtomicU64, + } + + impl ManualClock { + fn new(unix_micros: u64) -> Self { + Self { + base: Instant::now(), + unix_micros: AtomicU64::new(unix_micros), + } + } + + fn advance(&self, micros: u64) { + self.unix_micros.fetch_add(micros, Ordering::SeqCst); + } + } + + impl Clock for ManualClock { + fn now_unix_micros(&self) -> UnixMicros { + UnixMicros::new(self.unix_micros.load(Ordering::SeqCst)) + } + + fn now_instant(&self) -> Instant { + self.base + Duration::from_micros(self.unix_micros.load(Ordering::SeqCst)) + } + + fn sleep(&self, _: Duration) -> SleepFuture { + Box::pin(std::future::ready(())) + } + + fn sleep_until(&self, _: Instant) -> SleepFuture { + Box::pin(std::future::ready(())) + } + } + + #[tokio::test] + async fn external_clock_works_behind_dyn_arc() { + let clock: Arc = Arc::new(ManualClock::new(42)); + assert_eq!(clock.now_unix_micros().raw(), 42); + let before = clock.now_instant(); + clock.sleep(Duration::from_secs(60)).await; + assert_eq!( + clock.now_instant(), + before, + "manual clock's sleep must not advance virtual time on its own", + ); + } + + #[tokio::test] + async fn manual_clock_advance_moves_both_unix_and_instant() { + let clock = Arc::new(ManualClock::new(1_000)); + let dyn_clock: Arc = clock.clone(); + let unix_before = dyn_clock.now_unix_micros().raw(); + let instant_before = dyn_clock.now_instant(); + clock.advance(500); + assert_eq!(dyn_clock.now_unix_micros().raw(), unix_before + 500); + assert_eq!( + dyn_clock.now_instant(), + instant_before + Duration::from_micros(500), + ); + } + + #[tokio::test(start_paused = true)] + async fn sim_clock_anchors_unix_at_explicit_base_under_paused_time() { + let base = UnixMicros::new(1_700_000_000_000_000); + let clock = SimClock::at(base); + assert_eq!( + clock.now_unix_micros(), + base, + "sim clock unix axis must reflect the explicit base, not wall clock", + ); + let bump = Duration::from_secs(10); + tokio::time::advance(bump).await; + assert_eq!( + clock.now_unix_micros().raw(), + base.raw() + u64::try_from(bump.as_micros()).unwrap(), + "sim clock unix axis must follow tokio virtual advance", + ); + } + + #[tokio::test(start_paused = true)] + async fn sim_clock_two_constructions_with_same_base_align() { + let base = UnixMicros::new(2_000_000_000_000_000); + let a = SimClock::at(base); + let b = SimClock::at(base); + assert_eq!(a.now_unix_micros(), b.now_unix_micros()); + tokio::time::advance(Duration::from_secs(5)).await; + assert_eq!( + a.now_unix_micros(), + b.now_unix_micros(), + "two sim clocks anchored at the same base must agree on virtual time", + ); + } + + #[tokio::test(start_paused = true)] + async fn system_clock_axes_advance_together_under_paused_time() { + let clock = SystemClock::new(); + let unix_before = clock.now_unix_micros().raw(); + let instant_before = clock.now_instant(); + let bump = Duration::from_secs(60); + tokio::time::advance(bump).await; + let unix_after = clock.now_unix_micros().raw(); + let instant_after = clock.now_instant(); + assert_eq!( + instant_after.saturating_duration_since(instant_before), + bump, + "instant axis must reflect virtual advance", + ); + assert_eq!( + unix_after - unix_before, + u64::try_from(bump.as_micros()).unwrap(), + "unix axis must follow the same virtual advance, not wall clock", + ); + } +} diff --git a/bobbin/crates/runtime/src/entropy.rs b/bobbin/crates/runtime/src/entropy.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/runtime/src/entropy.rs @@ -0,0 +1,116 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +pub trait Entropy: Send + Sync + 'static { + fn next_u64(&self) -> u64; +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct OsEntropy; + +impl Entropy for OsEntropy { + fn next_u64(&self) -> u64 { + let mut buf = [0u8; 8]; + getrandom::fill(&mut buf).expect("os entropy source unavailable"); + u64::from_le_bytes(buf) + } +} + +#[derive(Debug)] +pub struct SeededEntropy { + state: AtomicU64, +} + +impl SeededEntropy { + const GOLDEN: u64 = 0x9E37_79B9_7F4A_7C15; + + pub fn new(seed: u64) -> Self { + Self { + state: AtomicU64::new(seed), + } + } +} + +impl Entropy for SeededEntropy { + fn next_u64(&self) -> u64 { + let prev = self.state.fetch_add(Self::GOLDEN, Ordering::Relaxed); + let z = prev.wrapping_add(Self::GOLDEN); + let z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + let z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::atomic::{AtomicU64, Ordering}; + + struct CountingEntropy(AtomicU64); + + impl Entropy for CountingEntropy { + fn next_u64(&self) -> u64 { + self.0.fetch_add(1, Ordering::SeqCst) + } + } + + #[test] + fn external_impl_works_behind_dyn_arc() { + let e: Arc = Arc::new(CountingEntropy(AtomicU64::new(100))); + assert_eq!(e.next_u64(), 100); + assert_eq!(e.next_u64(), 101); + } + + #[test] + fn seeded_entropy_two_constructions_with_same_seed_yield_same_stream() { + let a = SeededEntropy::new(0xCAFE_F00D); + let b = SeededEntropy::new(0xCAFE_F00D); + for _ in 0..32 { + assert_eq!( + a.next_u64(), + b.next_u64(), + "splitmix stream must reproduce exactly across constructions with same seed", + ); + } + } + + #[test] + fn seeded_entropy_different_seeds_diverge() { + let a = SeededEntropy::new(1); + let b = SeededEntropy::new(2); + let mut all_match = true; + for _ in 0..32 { + if a.next_u64() != b.next_u64() { + all_match = false; + break; + } + } + assert!( + !all_match, + "two seeded entropies with distinct seeds must diverge inside 32 draws", + ); + } + + #[test] + fn seeded_entropy_does_not_repeat_inside_short_window() { + let e = SeededEntropy::new(0); + let mut seen = std::collections::HashSet::new(); + for _ in 0..1024 { + assert!( + seen.insert(e.next_u64()), + "splitmix collided inside 1024 draws" + ); + } + } + + #[test] + fn os_entropy_does_not_collide_on_back_to_back_calls() { + let e = OsEntropy; + let a = e.next_u64(); + let b = e.next_u64(); + assert_ne!( + a, b, + "back-to-back os entropy collided, getrandom likely not actually wired up", + ); + } +} diff --git a/bobbin/crates/runtime/src/hasher.rs b/bobbin/crates/runtime/src/hasher.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/runtime/src/hasher.rs @@ -0,0 +1,101 @@ +use std::hash::BuildHasher; + +use ahash::{AHasher, RandomState}; + +use crate::{Entropy, OsEntropy}; + +#[derive(Clone, Debug)] +pub struct RuntimeHasher { + inner: RandomState, +} + +impl RuntimeHasher { + pub fn from_entropy(entropy: &dyn Entropy) -> Self { + Self { + inner: RandomState::with_seeds( + entropy.next_u64(), + entropy.next_u64(), + entropy.next_u64(), + entropy.next_u64(), + ), + } + } + + pub const fn from_seeds(k0: u64, k1: u64, k2: u64, k3: u64) -> Self { + Self { + inner: RandomState::with_seeds(k0, k1, k2, k3), + } + } +} + +impl Default for RuntimeHasher { + fn default() -> Self { + Self::from_entropy(&OsEntropy) + } +} + +impl BuildHasher for RuntimeHasher { + type Hasher = AHasher; + + fn build_hasher(&self) -> Self::Hasher { + self.inner.build_hasher() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::OsEntropy; + use std::collections::HashMap; + use std::sync::Arc; + use std::sync::atomic::{AtomicU64, Ordering}; + + struct CountingEntropy(AtomicU64); + + impl Entropy for CountingEntropy { + fn next_u64(&self) -> u64 { + self.0.fetch_add(1, Ordering::SeqCst) + } + } + + #[test] + fn same_seed_inputs_produce_identical_hashes() { + let a = RuntimeHasher::from_seeds(1, 2, 3, 4); + let b = RuntimeHasher::from_seeds(1, 2, 3, 4); + assert_eq!(a.hash_one("limpet"), b.hash_one("limpet")); + assert_eq!(a.hash_one(42_u64), b.hash_one(42_u64)); + } + + #[test] + fn different_seeds_produce_different_hashes() { + let a = RuntimeHasher::from_seeds(1, 2, 3, 4); + let b = RuntimeHasher::from_seeds(5, 6, 7, 8); + assert_ne!( + a.hash_one("conch"), + b.hash_one("conch"), + "two seedings must not collapse to the same key state", + ); + } + + #[test] + fn from_entropy_consumes_four_words_in_call_order() { + let entropy: Arc = Arc::new(CountingEntropy(AtomicU64::new(100))); + let hasher = RuntimeHasher::from_entropy(&*entropy); + let direct = RuntimeHasher::from_seeds(100, 101, 102, 103); + assert_eq!( + hasher.hash_one("nautilus"), + direct.hash_one("nautilus"), + "from_entropy must seed by draining four u64s in order", + ); + } + + #[test] + fn os_entropy_seeds_a_usable_hashmap() { + let hasher = RuntimeHasher::from_entropy(&OsEntropy); + let mut map: HashMap<&str, u32, RuntimeHasher> = HashMap::with_hasher(hasher); + map.insert("kelp", 1); + map.insert("uni", 2); + assert_eq!(map.get("kelp"), Some(&1)); + assert_eq!(map.get("uni"), Some(&2)); + } +} diff --git a/bobbin/crates/runtime/src/lib.rs b/bobbin/crates/runtime/src/lib.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/runtime/src/lib.rs @@ -0,0 +1,20 @@ +mod clock; +mod entropy; +mod hasher; +mod mem; +mod mem_network; +mod network; + +pub use clock::{Clock, SimClock, SleepFuture, SystemClock, UnixMicros}; +pub use entropy::{Entropy, OsEntropy, SeededEntropy}; +pub use hasher::RuntimeHasher; +pub use mem::MemoryBudget; +pub use mem_network::{ + DEFAULT_MEM_WS_CAPACITY, MemHttpBody, MemHttpResponder, MemHttpResponse, MemHttpTransport, + MemWsResponder, MemWsServerFuture, MemWsTransport, +}; +pub use network::{ + BodyStream, HttpRequest, HttpResponseFuture, HttpResponseHead, HttpResult, HttpTransport, + NetworkError, ReqwestHttp, TungsteniteWs, WsConn, WsConnectFuture, WsMessage, WsMessageFuture, + WsSendFuture, WsSink, WsStream, WsTransport, +}; diff --git a/bobbin/crates/runtime/src/mem.rs b/bobbin/crates/runtime/src/mem.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/runtime/src/mem.rs @@ -0,0 +1,12 @@ +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct MemoryBudget(u64); + +impl MemoryBudget { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn bytes(self) -> u64 { + self.0 + } +} diff --git a/bobbin/crates/runtime/src/mem_network.rs b/bobbin/crates/runtime/src/mem_network.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/runtime/src/mem_network.rs @@ -0,0 +1,359 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; +use futures::stream; +use http::{HeaderMap, StatusCode}; +use tokio::sync::mpsc; +use url::Url; + +use crate::clock::{Clock, SleepFuture}; +use crate::network::{ + BodyStream, HttpRequest, HttpResponseFuture, HttpResponseHead, HttpResult, HttpTransport, + NetworkError, WsConn, WsConnectFuture, WsMessage, WsMessageFuture, WsSendFuture, WsSink, + WsStream, WsTransport, +}; + +#[derive(Debug)] +pub struct MemHttpResponse { + pub latency: Duration, + pub result: Result, +} + +#[derive(Debug)] +pub struct MemHttpBody { + pub status: StatusCode, + pub headers: HeaderMap, + pub body: Bytes, +} + +impl MemHttpBody { + pub fn ok_json(body: Bytes) -> Self { + let mut headers = HeaderMap::new(); + headers.insert( + http::header::CONTENT_TYPE, + "application/json".parse().unwrap(), + ); + Self { + status: StatusCode::OK, + headers, + body, + } + } + + pub fn status_only(status: StatusCode) -> Self { + Self { + status, + headers: HeaderMap::new(), + body: Bytes::new(), + } + } +} + +pub trait MemHttpResponder: Send + Sync + 'static { + fn respond(&self, request: &HttpRequest) -> MemHttpResponse; +} + +#[derive(Clone)] +pub struct MemHttpTransport { + responder: Arc, + clock: Arc, +} + +impl MemHttpTransport { + pub fn new(responder: Arc, clock: Arc) -> Self { + Self { responder, clock } + } + + pub fn shared( + responder: Arc, + clock: Arc, + ) -> Arc { + Arc::new(Self::new(responder, clock)) + } +} + +impl HttpTransport for MemHttpTransport { + fn execute(&self, request: HttpRequest) -> HttpResponseFuture { + let response = self.responder.respond(&request); + let sleep: SleepFuture = self.clock.sleep(response.latency); + Box::pin(async move { + sleep.await; + into_response_head(response.result) + }) + } +} + +fn into_response_head(result: Result) -> HttpResult { + let body = result?; + let content_length = Some(body.body.len() as u64); + let body_stream: BodyStream = Box::pin(stream::once(async move { Ok(body.body) })); + Ok(HttpResponseHead { + status: body.status, + headers: body.headers, + content_length, + body: body_stream, + }) +} + +pub type MemWsServerFuture = Pin + Send + 'static>>; + +pub trait MemWsResponder: Send + Sync + 'static { + fn spawn_server( + &self, + url: Url, + recv: mpsc::UnboundedReceiver, + send: mpsc::Sender, + ) -> MemWsServerFuture; +} + +pub const DEFAULT_MEM_WS_CAPACITY: usize = 4096; + +#[derive(Clone)] +pub struct MemWsTransport { + responder: Arc, + capacity: usize, +} + +impl MemWsTransport { + pub fn new(responder: Arc) -> Self { + Self::with_capacity(responder, DEFAULT_MEM_WS_CAPACITY) + } + + pub fn with_capacity(responder: Arc, capacity: usize) -> Self { + assert!(capacity > 0, "MemWsTransport capacity must be > 0"); + Self { + responder, + capacity, + } + } + + pub fn shared(responder: Arc) -> Arc { + Arc::new(Self::new(responder)) + } + + pub fn shared_with_capacity( + responder: Arc, + capacity: usize, + ) -> Arc { + Arc::new(Self::with_capacity(responder, capacity)) + } +} + +impl WsTransport for MemWsTransport { + fn connect(&self, url: Url) -> WsConnectFuture { + let responder = self.responder.clone(); + let capacity = self.capacity; + Box::pin(async move { + let (c2s_tx, c2s_rx) = mpsc::unbounded_channel(); + let (s2c_tx, s2c_rx) = mpsc::channel(capacity); + let server_future = responder.spawn_server(url, c2s_rx, s2c_tx); + tokio::spawn(server_future); + let sink: Box = Box::new(MemWsSink { sender: c2s_tx }); + let stream: Box = Box::new(MemWsStream { receiver: s2c_rx }); + Ok(WsConn { sink, stream }) + }) + } +} + +struct MemWsSink { + sender: mpsc::UnboundedSender, +} + +impl WsSink for MemWsSink { + fn send<'a>(&'a mut self, message: WsMessage) -> WsSendFuture<'a> { + let result = self + .sender + .send(message) + .map_err(|_| NetworkError::Transport("server side closed channel".into())); + Box::pin(async move { result }) + } +} + +struct MemWsStream { + receiver: mpsc::Receiver, +} + +impl WsStream for MemWsStream { + fn next<'a>(&'a mut self) -> WsMessageFuture<'a> { + Box::pin(async move { self.receiver.recv().await.map(Ok) }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::SimClock; + use crate::UnixMicros; + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::time::Instant; + + struct ScriptedHttp { + responses: Mutex>, + cursor: AtomicUsize, + } + + impl ScriptedHttp { + fn new(responses: Vec) -> Self { + Self { + responses: Mutex::new(responses), + cursor: AtomicUsize::new(0), + } + } + } + + impl MemHttpResponder for ScriptedHttp { + fn respond(&self, _: &HttpRequest) -> MemHttpResponse { + let i = self.cursor.fetch_add(1, Ordering::Relaxed); + let mut guard = self.responses.lock().unwrap(); + std::mem::replace( + &mut guard[i], + MemHttpResponse { + latency: Duration::ZERO, + result: Err(NetworkError::Transport("script consumed".into())), + }, + ) + } + } + + fn ok_response(body: &str, latency_ms: u64) -> MemHttpResponse { + MemHttpResponse { + latency: Duration::from_millis(latency_ms), + result: Ok(MemHttpBody::ok_json(Bytes::from(body.to_owned()))), + } + } + + fn err_response(latency_ms: u64) -> MemHttpResponse { + MemHttpResponse { + latency: Duration::from_millis(latency_ms), + result: Err(NetworkError::Transport("brownout".into())), + } + } + + #[tokio::test(start_paused = true)] + async fn http_returns_scripted_body_after_injected_latency() { + let clock: Arc = Arc::new(SimClock::at(UnixMicros::new(0))); + let responder: Arc = + Arc::new(ScriptedHttp::new(vec![ok_response("{\"hello\":1}", 10)])); + let transport = MemHttpTransport::new(responder, clock); + let request = HttpRequest { + url: Url::parse("http://oyster.cafe/xrpc/x").unwrap(), + headers: HeaderMap::new(), + }; + + let before = Instant::now(); + let resp = transport.execute(request).await.unwrap(); + let elapsed = Instant::now().saturating_duration_since(before); + + assert_eq!(resp.status, StatusCode::OK); + assert_eq!(elapsed, Duration::from_millis(10)); + let chunks = collect_body(resp.body).await; + assert_eq!(chunks.as_ref(), b"{\"hello\":1}"); + } + + #[tokio::test(start_paused = true)] + async fn http_propagates_scripted_errors_with_latency() { + let clock: Arc = Arc::new(SimClock::at(UnixMicros::new(0))); + let responder: Arc = + Arc::new(ScriptedHttp::new(vec![err_response(50)])); + let transport = MemHttpTransport::new(responder, clock); + let request = HttpRequest { + url: Url::parse("http://oyster.cafe/xrpc/x").unwrap(), + headers: HeaderMap::new(), + }; + + let before = Instant::now(); + let resp = transport.execute(request).await; + let elapsed = Instant::now().saturating_duration_since(before); + + assert_eq!(elapsed, Duration::from_millis(50)); + assert!(matches!(resp, Err(NetworkError::Transport(_)))); + } + + async fn collect_body(mut body: BodyStream) -> Bytes { + use futures::StreamExt; + let mut acc = Vec::new(); + while let Some(chunk) = body.next().await { + acc.extend_from_slice(&chunk.unwrap()); + } + Bytes::from(acc) + } + + struct ScriptedWs { + frames: Mutex>, + } + + impl MemWsResponder for ScriptedWs { + fn spawn_server( + &self, + _: Url, + mut _recv: mpsc::UnboundedReceiver, + send: mpsc::Sender, + ) -> MemWsServerFuture { + let frames: Vec = std::mem::take(&mut *self.frames.lock().unwrap()); + Box::pin(async move { + for frame in frames { + if send.send(frame).await.is_err() { + return; + } + } + }) + } + } + + #[tokio::test(start_paused = true)] + async fn ws_delivers_scripted_frames_to_client() { + let responder: Arc = Arc::new(ScriptedWs { + frames: Mutex::new(vec![ + WsMessage::Text("frame-a".into()), + WsMessage::Text("frame-b".into()), + ]), + }); + let transport = MemWsTransport::new(responder); + let mut conn = transport + .connect(Url::parse("ws://oyster.cafe/").unwrap()) + .await + .unwrap(); + let a = conn.stream.next().await.unwrap().unwrap(); + let b = conn.stream.next().await.unwrap().unwrap(); + assert!(matches!(a, WsMessage::Text(t) if t == "frame-a")); + assert!(matches!(b, WsMessage::Text(t) if t == "frame-b")); + } + + struct EchoWs; + + impl MemWsResponder for EchoWs { + fn spawn_server( + &self, + _: Url, + mut recv: mpsc::UnboundedReceiver, + send: mpsc::Sender, + ) -> MemWsServerFuture { + Box::pin(async move { + while let Some(msg) = recv.recv().await { + if send.send(msg).await.is_err() { + return; + } + } + }) + } + } + + #[tokio::test(start_paused = true)] + async fn ws_round_trip_via_server_echo() { + let transport = MemWsTransport::new(Arc::new(EchoWs)); + let mut conn = transport + .connect(Url::parse("ws://oyster.cafe/").unwrap()) + .await + .unwrap(); + conn.sink + .send(WsMessage::Text("ping".into())) + .await + .unwrap(); + let echoed = conn.stream.next().await.unwrap().unwrap(); + assert!(matches!(echoed, WsMessage::Text(t) if t == "ping")); + } +} diff --git a/bobbin/crates/runtime/src/network.rs b/bobbin/crates/runtime/src/network.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/runtime/src/network.rs @@ -0,0 +1,230 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use bytes::Bytes; +use futures::stream::{Stream, StreamExt}; +use http::{HeaderMap, StatusCode}; +use thiserror::Error; +use tokio_tungstenite::tungstenite::{ + Bytes as WsBytes, Message as TungsteniteMessage, protocol::CloseFrame as TungsteniteClose, + protocol::frame::coding::CloseCode as TungsteniteCloseCode, +}; +use url::Url; + +#[derive(Debug, Error)] +pub enum NetworkError { + #[error("connect: {0}")] + Connect(String), + #[error("timeout: {0}")] + Timeout(String), + #[error("redirect: {0}")] + Redirect(String), + #[error("transport: {0}")] + Transport(String), + #[error("body: {0}")] + Body(String), + #[error("protocol: {0}")] + Protocol(String), +} + +pub struct HttpRequest { + pub url: Url, + pub headers: HeaderMap, +} + +pub type BodyStream = Pin> + Send + 'static>>; + +pub struct HttpResponseHead { + pub status: StatusCode, + pub headers: HeaderMap, + pub content_length: Option, + pub body: BodyStream, +} + +pub type HttpResult = Result; +pub type HttpResponseFuture = Pin + Send + 'static>>; + +pub trait HttpTransport: Send + Sync + 'static { + fn execute(&self, request: HttpRequest) -> HttpResponseFuture; +} + +#[derive(Clone, Debug)] +pub struct ReqwestHttp { + client: reqwest::Client, +} + +impl ReqwestHttp { + pub fn new(client: reqwest::Client) -> Self { + Self { client } + } + + pub fn shared(client: reqwest::Client) -> Arc { + Arc::new(Self::new(client)) + } +} + +impl HttpTransport for ReqwestHttp { + fn execute(&self, request: HttpRequest) -> HttpResponseFuture { + let client = self.client.clone(); + Box::pin(async move { + let resp = client + .get(request.url) + .headers(request.headers) + .send() + .await + .map_err(map_reqwest)?; + let status = resp.status(); + let headers = resp.headers().clone(); + let content_length = resp.content_length(); + let body: BodyStream = Box::pin( + resp.bytes_stream() + .map(|chunk| chunk.map_err(|e| NetworkError::Body(e.to_string()))), + ); + Ok(HttpResponseHead { + status, + headers, + content_length, + body, + }) + }) + } +} + +fn map_reqwest(err: reqwest::Error) -> NetworkError { + let msg = err.to_string(); + if err.is_timeout() { + NetworkError::Timeout(msg) + } else if err.is_connect() { + NetworkError::Connect(msg) + } else if err.is_redirect() { + NetworkError::Redirect(msg) + } else { + NetworkError::Transport(msg) + } +} + +#[derive(Clone, Debug)] +pub enum WsMessage { + Text(String), + Binary(Bytes), + Ping(Bytes), + Pong(Bytes), + Close { code: u16, reason: String }, +} + +pub type WsSendFuture<'a> = Pin> + Send + 'a>>; +pub type WsMessageFuture<'a> = + Pin>> + Send + 'a>>; + +pub trait WsSink: Send + 'static { + fn send<'a>(&'a mut self, message: WsMessage) -> WsSendFuture<'a>; +} + +pub trait WsStream: Send + 'static { + fn next<'a>(&'a mut self) -> WsMessageFuture<'a>; +} + +pub struct WsConn { + pub sink: Box, + pub stream: Box, +} + +pub type WsConnectFuture = + Pin> + Send + 'static>>; + +pub trait WsTransport: Send + Sync + 'static { + fn connect(&self, url: Url) -> WsConnectFuture; +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct TungsteniteWs; + +impl TungsteniteWs { + pub fn shared() -> Arc { + Arc::new(Self) + } +} + +impl WsTransport for TungsteniteWs { + fn connect(&self, url: Url) -> WsConnectFuture { + Box::pin(async move { + let url_str = url.as_str().to_owned(); + let (ws, _resp) = tokio_tungstenite::connect_async(&url_str) + .await + .map_err(|e| NetworkError::Connect(e.to_string()))?; + let (sink_inner, stream_inner) = futures::StreamExt::split(ws); + let sink: Box = Box::new(TungsteniteSink { inner: sink_inner }); + let stream: Box = Box::new(TungsteniteStream { + inner: stream_inner, + }); + Ok(WsConn { sink, stream }) + }) + } +} + +type TungsteniteWsStream = + tokio_tungstenite::WebSocketStream>; + +struct TungsteniteSink { + inner: futures::stream::SplitSink, +} + +impl WsSink for TungsteniteSink { + fn send<'a>(&'a mut self, message: WsMessage) -> WsSendFuture<'a> { + Box::pin(async move { + use futures::SinkExt; + self.inner + .send(message_to_tungstenite(message)) + .await + .map_err(|e| NetworkError::Transport(e.to_string())) + }) + } +} + +struct TungsteniteStream { + inner: futures::stream::SplitStream, +} + +impl WsStream for TungsteniteStream { + fn next<'a>(&'a mut self) -> WsMessageFuture<'a> { + Box::pin(async move { + let item = StreamExt::next(&mut self.inner).await?; + Some( + item.map_err(|e| NetworkError::Transport(e.to_string())) + .and_then(message_from_tungstenite), + ) + }) + } +} + +fn message_to_tungstenite(message: WsMessage) -> TungsteniteMessage { + match message { + WsMessage::Text(text) => TungsteniteMessage::Text(text.into()), + WsMessage::Binary(bytes) => TungsteniteMessage::Binary(WsBytes::copy_from_slice(&bytes)), + WsMessage::Ping(bytes) => TungsteniteMessage::Ping(WsBytes::copy_from_slice(&bytes)), + WsMessage::Pong(bytes) => TungsteniteMessage::Pong(WsBytes::copy_from_slice(&bytes)), + WsMessage::Close { code, reason } => TungsteniteMessage::Close(Some(TungsteniteClose { + code: TungsteniteCloseCode::from(code), + reason: reason.into(), + })), + } +} + +fn message_from_tungstenite(message: TungsteniteMessage) -> Result { + match message { + TungsteniteMessage::Text(t) => Ok(WsMessage::Text(t.to_string())), + TungsteniteMessage::Binary(b) => Ok(WsMessage::Binary(Bytes::copy_from_slice(&b))), + TungsteniteMessage::Ping(b) => Ok(WsMessage::Ping(Bytes::copy_from_slice(&b))), + TungsteniteMessage::Pong(b) => Ok(WsMessage::Pong(Bytes::copy_from_slice(&b))), + TungsteniteMessage::Close(close) => { + let (code, reason) = close + .map(|c| (u16::from(c.code), c.reason.to_string())) + .unwrap_or((1000, String::new())); + Ok(WsMessage::Close { code, reason }) + } + TungsteniteMessage::Frame(_) => Err(NetworkError::Protocol( + "tungstenite raw frame surfaced unexpectedly".to_owned(), + )), + } +} diff --git a/bobbin/crates/search/Cargo.toml b/bobbin/crates/search/Cargo.toml new file mode 100644 --- /dev/null +++ b/bobbin/crates/search/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "bobbin-search" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-runtime = { workspace = true } +bobbin-types = { workspace = true } +jacquard-common = { workspace = true } + +tantivy = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["rt", "sync", "time"] } +tracing = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/bobbin/crates/search/src/lib.rs b/bobbin/crates/search/src/lib.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/search/src/lib.rs @@ -0,0 +1,792 @@ +use std::future::Future; +use std::ops::Bound; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use bobbin_runtime::Clock; +use bobbin_types::search::{SearchDoc, SearchSink}; +use jacquard_common::DefaultStr; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::string::{AtUri, Did}; +use tantivy::collector::TopDocs; +use tantivy::query::{ + BooleanQuery, Occur, Query, QueryParser, QueryParserError, RangeQuery, TermQuery, +}; +use tantivy::schema::{ + FAST, Field, INDEXED, IndexRecordOption, STORED, STRING, Schema, TEXT, TextFieldIndexing, + TextOptions, Value, +}; +use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, TantivyError, Term}; +use thiserror::Error; +use tokio::sync::{mpsc, oneshot}; +use tracing::warn; + +pub const DEFAULT_WRITER_HEAP_BYTES: usize = 50_000_000; +const BATCH_SIZE: usize = 200; +const BATCH_INTERVAL: Duration = Duration::from_millis(250); +const TITLE_BOOST: f32 = 4.0; +const OFFSET_TOKEN_LEN: usize = 8; +const WRITE_QUEUE_CAPACITY: usize = 4096; + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct SearchOffset(u32); + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)] +pub enum SearchOffsetError { + #[error("offset token must be {OFFSET_TOKEN_LEN} hex characters")] + Malformed, +} + +impl SearchOffset { + pub const fn new(value: u32) -> Self { + Self(value) + } + + pub const fn raw(self) -> u32 { + self.0 + } + + pub fn encode_token(self) -> String { + format!("{:0width$x}", self.0, width = OFFSET_TOKEN_LEN) + } + + pub fn decode_token(token: &str) -> Result { + if token.len() != OFFSET_TOKEN_LEN || !token.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(SearchOffsetError::Malformed); + } + u32::from_str_radix(token, 16) + .map(Self) + .map_err(|_| SearchOffsetError::Malformed) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SearchCursor { + Start, + At(SearchOffset), +} + +impl SearchCursor { + pub fn from_token(raw: Option<&str>) -> Result { + raw.map_or(Ok(Self::Start), |t| { + SearchOffset::decode_token(t).map(Self::At) + }) + } + + fn offset(self) -> u32 { + match self { + Self::Start => 0, + Self::At(o) => o.raw(), + } + } +} + +#[derive(Debug)] +pub struct SearchHit { + pub uri: AtUri, + pub nsid: Nsid, + pub score: f32, +} + +#[derive(Debug)] +pub struct SearchPage { + pub hits: Vec, + pub next: Option, +} + +#[derive(Debug, Error)] +pub enum SearchError { + #[error("tantivy: {0}")] + Tantivy(#[from] TantivyError), + #[error("query parse: {0}")] + Query(#[from] QueryParserError), + #[error("indexed uri is invalid: {0}")] + InvalidUri(String), + #[error("indexed nsid is invalid: {0}")] + InvalidNsid(String), + #[error("indexed document missing field: {0}")] + MissingField(&'static str), + #[error("search blocking task cancelled: {0}")] + Cancelled(String), +} + +#[derive(Clone, Copy)] +struct Fields { + uri: Field, + nsid: Field, + title: Field, + body: Field, + author: Field, + created_at: Field, + repo: Field, +} + +#[derive(Clone, Debug, Default)] +pub struct SearchFilters { + pub nsid: Option>, + pub author: Option>, + pub repo: Option>, + pub since: Option, + pub until: Option, +} + +impl SearchFilters { + pub fn is_empty(&self) -> bool { + self.nsid.is_none() + && self.author.is_none() + && self.repo.is_none() + && self.since.is_none() + && self.until.is_none() + } +} + +enum WriteOp { + Upsert(SearchDoc), + Remove(AtUri), + Flush(oneshot::Sender<()>), +} + +pub struct SearchIndex { + inner: Arc, +} + +struct Inner { + fields: Fields, + reader: IndexReader, + parser: QueryParser, + tx: mpsc::Sender, +} + +impl SearchIndex { + pub fn new(heap_bytes: usize, clock: Arc) -> Result { + let mut sb = Schema::builder(); + let uri = sb.add_text_field("uri", STRING | STORED); + let nsid = sb.add_text_field("nsid", STRING | STORED); + let title_indexing = TextFieldIndexing::default() + .set_tokenizer("default") + .set_index_option(IndexRecordOption::WithFreqsAndPositions); + let title_opts = TextOptions::default() + .set_indexing_options(title_indexing) + .set_stored(); + let title = sb.add_text_field("title", title_opts); + let body = sb.add_text_field("body", TEXT); + let author = sb.add_text_field("author", STRING | STORED); + let created_at = sb.add_i64_field("created_at", INDEXED | FAST | STORED); + let repo = sb.add_text_field("repo", STRING | STORED); + let schema = sb.build(); + let index = Index::create_in_ram(schema); + let writer: IndexWriter = index.writer(heap_bytes)?; + let reader = index + .reader_builder() + .reload_policy(ReloadPolicy::Manual) + .try_into()?; + let mut parser = QueryParser::for_index(&index, vec![title, body]); + parser.set_field_boost(title, TITLE_BOOST); + let fields = Fields { + uri, + nsid, + title, + body, + author, + created_at, + repo, + }; + let (tx, rx) = mpsc::channel(WRITE_QUEUE_CAPACITY); + let writer_reader = reader.clone(); + tokio::spawn(writer_loop(writer, fields, writer_reader, rx, clock)); + Ok(Self { + inner: Arc::new(Inner { + fields, + reader, + parser, + tx, + }), + }) + } + + pub async fn flush(&self) { + let (done_tx, done_rx) = oneshot::channel(); + if self.inner.tx.send(WriteOp::Flush(done_tx)).await.is_err() { + warn!("search writer channel closed before flush"); + return; + } + if done_rx.await.is_err() { + warn!("search writer dropped before completing flush"); + } + } + + pub async fn search( + &self, + q: &str, + filters: SearchFilters, + cursor: SearchCursor, + limit: u32, + ) -> Result { + let trimmed = q.trim(); + if trimmed.is_empty() { + return Ok(SearchPage { + hits: Vec::new(), + next: None, + }); + } + let inner = self.inner.clone(); + let q_owned = trimmed.to_owned(); + let limit_usize = limit as usize; + let offset = cursor.offset() as usize; + match tokio::task::spawn_blocking(move || { + inner.search_blocking(&q_owned, &filters, offset, limit_usize) + }) + .await + { + Ok(result) => result, + Err(e) if e.is_panic() => std::panic::resume_unwind(e.into_panic()), + Err(e) => Err(SearchError::Cancelled(e.to_string())), + } + } +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct SearchSpaceUsage { + pub num_docs: u64, + pub num_segments: u64, + pub total_bytes: u64, +} + +impl SearchIndex { + pub fn space_usage(&self) -> SearchSpaceUsage { + let searcher = self.inner.reader.searcher(); + let segments = searcher.segment_readers(); + let total_bytes = segments + .iter() + .map(|s| s.space_usage().map(|u| u.total().get_bytes()).unwrap_or(0)) + .sum(); + SearchSpaceUsage { + num_docs: searcher.num_docs(), + num_segments: segments.len() as u64, + total_bytes, + } + } +} + +pub type SearchReadFuture<'a> = + Pin> + Send + 'a>>; + +pub trait SearchReader: Send + Sync + 'static { + fn search<'a>( + &'a self, + query: &'a str, + filters: SearchFilters, + cursor: SearchCursor, + limit: u32, + ) -> SearchReadFuture<'a>; +} + +impl SearchReader for SearchIndex { + fn search<'a>( + &'a self, + query: &'a str, + filters: SearchFilters, + cursor: SearchCursor, + limit: u32, + ) -> SearchReadFuture<'a> { + Box::pin(SearchIndex::search(self, query, filters, cursor, limit)) + } +} + +impl Inner { + fn search_blocking( + &self, + q: &str, + filters: &SearchFilters, + offset: usize, + limit: usize, + ) -> Result { + let parsed = self.parser.parse_query(q)?; + let final_query: Box = if filters.is_empty() { + parsed + } else { + let mut clauses: Vec<(Occur, Box)> = Vec::with_capacity(5); + clauses.push((Occur::Must, parsed)); + if let Some(n) = &filters.nsid { + let term = Term::from_field_text(self.fields.nsid, n.as_ref()); + clauses.push(( + Occur::Must, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)), + )); + } + if let Some(a) = &filters.author { + let term = Term::from_field_text(self.fields.author, a.as_ref()); + clauses.push(( + Occur::Must, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)), + )); + } + if let Some(r) = &filters.repo { + let term = Term::from_field_text(self.fields.repo, r.as_ref()); + clauses.push(( + Occur::Must, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)), + )); + } + if filters.since.is_some() || filters.until.is_some() { + let lower = filters.since.map_or(Bound::Unbounded, |s| { + Bound::Included(Term::from_field_i64(self.fields.created_at, s)) + }); + let upper = filters.until.map_or(Bound::Unbounded, |u| { + Bound::Excluded(Term::from_field_i64(self.fields.created_at, u)) + }); + clauses.push((Occur::Must, Box::new(RangeQuery::new(lower, upper)))); + } + Box::new(BooleanQuery::new(clauses)) + }; + let collector = TopDocs::with_limit(limit + 1) + .and_offset(offset) + .order_by_score(); + let searcher = self.reader.searcher(); + let raw_hits: Vec<(f32, tantivy::DocAddress)> = + searcher.search(&final_query, &collector)?; + let has_more = raw_hits.len() > limit; + let page_slice = &raw_hits[..raw_hits.len().min(limit)]; + let hits = page_slice + .iter() + .map(|(score, addr)| { + let doc: TantivyDocument = searcher.doc(*addr)?; + let uri_str = stored_text(&doc, self.fields.uri, "uri")?; + let nsid_str = stored_text(&doc, self.fields.nsid, "nsid")?; + let uri = AtUri::::new_owned(uri_str.as_str()) + .map_err(|e| SearchError::InvalidUri(format!("{e}: {uri_str}")))?; + let nsid = Nsid::::new_owned(nsid_str.as_str()) + .map_err(|e| SearchError::InvalidNsid(format!("{e}: {nsid_str}")))?; + Ok::<_, SearchError>(SearchHit { + uri, + nsid, + score: *score, + }) + }) + .collect::, _>>()?; + Ok(SearchPage { + hits, + next: next_cursor(offset, limit, has_more), + }) + } +} + +fn next_cursor(offset: usize, limit: usize, has_more: bool) -> Option { + has_more + .then(|| u32::try_from(offset.saturating_add(limit)).ok()) + .flatten() + .map(SearchOffset::new) +} + +async fn writer_loop( + mut writer: IndexWriter, + fields: Fields, + reader: IndexReader, + mut rx: mpsc::Receiver, + clock: Arc, +) { + let mut pending: usize = 0; + let mut deadline = clock.now_instant() + BATCH_INTERVAL; + loop { + let outcome = tokio::select! { + biased; + msg = rx.recv() => RecvOutcome::Message(msg), + _ = clock.sleep_until(deadline) => RecvOutcome::Idle, + }; + match outcome { + RecvOutcome::Message(Some(WriteOp::Upsert(doc))) => { + apply_upsert(&mut writer, fields, doc); + pending += 1; + } + RecvOutcome::Message(Some(WriteOp::Remove(uri))) => { + apply_remove(&mut writer, fields, &uri); + pending += 1; + } + RecvOutcome::Message(Some(WriteOp::Flush(done))) => { + if pending > 0 { + commit_and_reload(&mut writer, &reader); + pending = 0; + } + let _ = done.send(()); + deadline = clock.now_instant() + BATCH_INTERVAL; + continue; + } + RecvOutcome::Message(None) => break, + RecvOutcome::Idle => { + if pending > 0 { + commit_and_reload(&mut writer, &reader); + pending = 0; + } + deadline = clock.now_instant() + BATCH_INTERVAL; + continue; + } + } + if pending >= BATCH_SIZE { + commit_and_reload(&mut writer, &reader); + pending = 0; + } + } + if pending > 0 { + commit_and_reload(&mut writer, &reader); + } +} + +enum RecvOutcome { + Message(Option), + Idle, +} + +fn apply_upsert(writer: &mut IndexWriter, fields: Fields, doc: SearchDoc) { + let uri_term = Term::from_field_text(fields.uri, doc.uri.as_ref()); + writer.delete_term(uri_term); + let mut td = TantivyDocument::default(); + td.add_text(fields.uri, doc.uri.as_ref()); + td.add_text(fields.nsid, doc.nsid.as_ref()); + td.add_text(fields.title, &doc.title); + td.add_text(fields.body, &doc.body); + if let Some(author) = &doc.author { + td.add_text(fields.author, author.as_ref()); + } + if let Some(ts) = doc.created_at { + td.add_i64(fields.created_at, ts); + } + if let Some(repo) = &doc.repo { + td.add_text(fields.repo, repo.as_ref()); + } + if let Err(e) = writer.add_document(td) { + warn!(?e, "search add_document failed"); + } +} + +fn apply_remove(writer: &mut IndexWriter, fields: Fields, uri: &AtUri) { + let term = Term::from_field_text(fields.uri, uri.as_ref()); + writer.delete_term(term); +} + +fn commit_and_reload(writer: &mut IndexWriter, reader: &IndexReader) { + match writer.commit() { + Ok(_) => { + if let Err(e) = reader.reload() { + warn!(?e, "search reader reload failed"); + } + } + Err(e) => warn!(?e, "search index commit failed"), + } +} + +fn stored_text( + doc: &TantivyDocument, + field: Field, + name: &'static str, +) -> Result { + doc.get_first(field) + .and_then(|v| v.as_str().map(|s| s.to_owned())) + .ok_or(SearchError::MissingField(name)) +} + +impl SearchSink for SearchIndex { + async fn upsert(&self, doc: SearchDoc) { + if self.inner.tx.send(WriteOp::Upsert(doc)).await.is_err() { + warn!("search writer channel closed; upsert dropped"); + } + } + + async fn remove(&self, uri: &AtUri) { + if self + .inner + .tx + .send(WriteOp::Remove(uri.clone())) + .await + .is_err() + { + warn!("search writer channel closed; remove dropped"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bobbin_runtime::SystemClock; + use bobbin_types::search::SearchDoc; + use jacquard_common::types::nsid::Nsid as NsidType; + + fn at(s: &str) -> AtUri { + AtUri::new_owned(s).unwrap() + } + + fn nsid(s: &'static str) -> NsidType { + NsidType::new_static(s).unwrap() + } + + fn doc( + uri: AtUri, + nsid: NsidType, + title: &str, + body: &str, + ) -> SearchDoc { + SearchDoc { + uri, + nsid, + title: title.to_owned(), + body: body.to_owned(), + author: None, + created_at: None, + repo: None, + } + } + + fn build() -> SearchIndex { + SearchIndex::new(DEFAULT_WRITER_HEAP_BYTES, Arc::new(SystemClock::new())).unwrap() + } + + #[tokio::test] + async fn upsert_then_query_returns_matching_doc() { + let idx = build(); + idx.upsert(doc( + at("at://did:plc:nel/sh.tangled.repo.issue/r1"), + nsid("sh.tangled.repo.issue"), + "barnacle pagination", + "scroll resets", + )) + .await; + idx.upsert(doc( + at("at://did:plc:teq/sh.tangled.repo.issue/r2"), + nsid("sh.tangled.repo.issue"), + "kelp grew sideways", + "kelp", + )) + .await; + idx.flush().await; + + let page = idx + .search( + "barnacle", + SearchFilters::default(), + SearchCursor::Start, + 10, + ) + .await + .unwrap(); + assert_eq!(page.hits.len(), 1); + assert_eq!( + page.hits[0].uri.as_ref(), + "at://did:plc:nel/sh.tangled.repo.issue/r1" + ); + assert!(page.next.is_none()); + } + + #[tokio::test] + async fn query_filters_by_nsid() { + let idx = build(); + idx.upsert(doc( + at("at://did:plc:nel/sh.tangled.repo.issue/r1"), + nsid("sh.tangled.repo.issue"), + "anemone tide", + "", + )) + .await; + idx.upsert(doc( + at("at://did:plc:teq/sh.tangled.string/k1"), + nsid("sh.tangled.string"), + "anemone.md", + "anemone again", + )) + .await; + idx.flush().await; + + let only_strings = idx + .search( + "anemone", + SearchFilters { + nsid: Some(nsid("sh.tangled.string")), + ..SearchFilters::default() + }, + SearchCursor::Start, + 10, + ) + .await + .unwrap(); + assert_eq!(only_strings.hits.len(), 1); + assert_eq!(only_strings.hits[0].nsid.as_ref(), "sh.tangled.string"); + } + + #[tokio::test] + async fn upsert_replaces_prior_document_for_same_uri() { + let idx = build(); + let uri = at("at://did:plc:nel/sh.tangled.repo.issue/r1"); + idx.upsert(doc( + uri.clone(), + nsid("sh.tangled.repo.issue"), + "abalone", + "", + )) + .await; + idx.upsert(doc(uri, nsid("sh.tangled.repo.issue"), "limpet", "")) + .await; + idx.flush().await; + + let abalone_hits = idx + .search("abalone", SearchFilters::default(), SearchCursor::Start, 10) + .await + .unwrap(); + assert!(abalone_hits.hits.is_empty(), "old title must be evicted"); + + let limpet_hits = idx + .search("limpet", SearchFilters::default(), SearchCursor::Start, 10) + .await + .unwrap(); + assert_eq!(limpet_hits.hits.len(), 1); + } + + #[tokio::test] + async fn remove_drops_document_from_index() { + let idx = build(); + let uri = at("at://did:plc:nel/sh.tangled.repo.issue/r1"); + idx.upsert(doc( + uri.clone(), + nsid("sh.tangled.repo.issue"), + "whelk", + "shell", + )) + .await; + idx.remove(&uri).await; + idx.flush().await; + let hits = idx + .search("whelk", SearchFilters::default(), SearchCursor::Start, 10) + .await + .unwrap(); + assert!(hits.hits.is_empty()); + } + + #[tokio::test] + async fn pagination_advances_cursor() { + let idx = build(); + let names = ["nel", "olaren", "teq", "lyna", "bailey"]; + for (i, owner) in names.iter().enumerate() { + idx.upsert(doc( + at(&format!("at://did:plc:{owner}/sh.tangled.repo.issue/r{i}")), + nsid("sh.tangled.repo.issue"), + "anemone", + "tides", + )) + .await; + } + idx.flush().await; + + let page1 = idx + .search("anemone", SearchFilters::default(), SearchCursor::Start, 2) + .await + .unwrap(); + assert_eq!(page1.hits.len(), 2); + let next = page1.next.expect("more pages"); + let page2 = idx + .search( + "anemone", + SearchFilters::default(), + SearchCursor::At(next), + 2, + ) + .await + .unwrap(); + assert_eq!(page2.hits.len(), 2); + let next2 = page2.next.expect("more pages"); + let page3 = idx + .search( + "anemone", + SearchFilters::default(), + SearchCursor::At(next2), + 2, + ) + .await + .unwrap(); + assert_eq!(page3.hits.len(), 1); + assert!(page3.next.is_none()); + } + + #[tokio::test] + async fn empty_query_short_circuits_to_empty_page() { + let idx = build(); + idx.upsert(doc( + at("at://did:plc:nel/sh.tangled.repo.issue/r1"), + nsid("sh.tangled.repo.issue"), + "abalone", + "", + )) + .await; + idx.flush().await; + let page = idx + .search(" ", SearchFilters::default(), SearchCursor::Start, 10) + .await + .unwrap(); + assert!(page.hits.is_empty()); + assert!(page.next.is_none()); + } + + #[tokio::test] + async fn flush_without_pending_ops_completes() { + let idx = build(); + idx.flush().await; + } + + #[tokio::test] + async fn writer_auto_commits_on_idle_interval() { + let idx = build(); + idx.upsert(doc( + at("at://did:plc:nel/sh.tangled.repo.issue/r1"), + nsid("sh.tangled.repo.issue"), + "auto", + "", + )) + .await; + tokio::time::sleep(BATCH_INTERVAL * 3).await; + let page = idx + .search("auto", SearchFilters::default(), SearchCursor::Start, 10) + .await + .unwrap(); + assert_eq!(page.hits.len(), 1); + } + + #[test] + fn next_cursor_advances_by_limit_when_more_results() { + assert_eq!(next_cursor(0, 10, true), Some(SearchOffset::new(10))); + assert_eq!(next_cursor(10, 25, true), Some(SearchOffset::new(35))); + } + + #[test] + fn next_cursor_is_none_when_no_more_results() { + assert_eq!(next_cursor(0, 10, false), None); + assert_eq!(next_cursor(99, 50, false), None); + } + + #[test] + fn next_cursor_returns_none_on_u32_overflow_rather_than_wrapping() { + let max = u32::MAX as usize; + assert_eq!(next_cursor(max, 1, true), None); + assert_eq!(next_cursor(max - 5, 10, true), None); + assert_eq!( + next_cursor(max - 10, 10, true), + Some(SearchOffset::new(u32::MAX)) + ); + } + + #[test] + fn cursor_token_round_trips() { + let off = SearchOffset::new(0xc0fe); + let token = off.encode_token(); + assert_eq!(token.len(), OFFSET_TOKEN_LEN); + assert_eq!(SearchOffset::decode_token(&token).unwrap(), off); + } + + #[test] + fn cursor_decode_rejects_malformed() { + let bad = ["", "deadbeef0", "no-hex!!", "zzzzzzzz", "1234567"]; + for s in bad { + assert!(matches!( + SearchOffset::decode_token(s), + Err(SearchOffsetError::Malformed), + )); + } + } +} diff --git a/bobbin/crates/slingshot-client/Cargo.toml b/bobbin/crates/slingshot-client/Cargo.toml new file mode 100644 --- /dev/null +++ b/bobbin/crates/slingshot-client/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "bobbin-slingshot-client" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-runtime = { workspace = true } +bobbin-types = { workspace = true } +jacquard-common = { workspace = true } + +bytes = { workspace = true } +cid = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +wiremock = { workspace = true } diff --git a/bobbin/crates/slingshot-client/src/lib.rs b/bobbin/crates/slingshot-client/src/lib.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/slingshot-client/src/lib.rs @@ -0,0 +1,603 @@ +use std::sync::Arc; +use std::time::Duration; + +use bobbin_runtime::{HttpRequest, HttpResponseHead, HttpTransport, NetworkError, ReqwestHttp}; +use bobbin_types::record::RecordBody; +use bytes::{Bytes, BytesMut}; +use cid::Cid as IpldCid; +use futures::TryStreamExt; +use http::{HeaderMap, StatusCode}; +use jacquard_common::BosStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::ident::AtIdentifier; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::recordkey::Rkey; +use jacquard_common::types::string::{AtStrError, AtUri}; +use serde::Deserialize; +use serde_json::value::RawValue; +use thiserror::Error; +use url::Url; + +const USER_AGENT: &str = concat!("bobbin/", env!("CARGO_PKG_VERSION")); +const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); +const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +const GET_RECORD_PATH: &str = "xrpc/com.atproto.repo.getRecord"; +const RESOLVE_MINI_DOC_PATH: &str = "xrpc/com.bad-example.identity.resolveMiniDoc"; +pub const MAX_BODY_BYTES: u64 = 4 * 1024 * 1024; + +#[derive(Clone)] +pub struct SlingshotClient { + http: Arc, + base: Url, +} + +impl std::fmt::Debug for SlingshotClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SlingshotClient") + .field("base", &self.base) + .finish_non_exhaustive() + } +} + +#[derive(Debug, Error)] +pub enum SlingshotError { + #[error("invalid base url scheme: {0}")] + BadScheme(String), + #[error("network: {0}")] + Network(#[from] NetworkError), + #[error("http client build: {0}")] + Build(String), + #[error("record not found")] + NotFound, + #[error("upstream returned status {0}")] + Upstream(StatusCode), + #[error("response body exceeded {limit} bytes")] + BodyTooLarge { limit: u64 }, + #[error("decode response: {0}")] + Decode(#[from] serde_json::Error), + #[error("response missing required field: {0}")] + MissingField(&'static str), + #[error("invalid AT-URI in response: {0}")] + InvalidAtUri(#[from] AtStrError), + #[error("invalid CID in response: {0}")] + InvalidCid(#[from] cid::Error), + #[error("upstream returned uri {got}, expected {expected}")] + UriMismatch { expected: String, got: String }, +} + +pub fn default_http_client() -> Result { + reqwest::Client::builder() + .user_agent(USER_AGENT) + .timeout(REQUEST_TIMEOUT) + .connect_timeout(CONNECT_TIMEOUT) + .build() +} + +impl SlingshotClient { + pub fn new(base: Url, http: Arc) -> Result { + match base.scheme() { + "http" | "https" => {} + other => return Err(SlingshotError::BadScheme(other.to_owned())), + } + let base = ensure_trailing_slash(base); + Ok(Self { http, base }) + } + + pub fn with_default_http(base: Url) -> Result { + let client = default_http_client().map_err(|e| SlingshotError::Build(e.to_string()))?; + Self::new(base, ReqwestHttp::shared(client)) + } + + pub async fn resolve_mini_doc( + &self, + identifier: &AtIdentifier, + ) -> Result + where + S: BosStr, + { + let mut url = self.base.join(RESOLVE_MINI_DOC_PATH).expect( + "base url is hierarchical and RESOLVE_MINI_DOC_PATH is a literal relative path", + ); + url.query_pairs_mut() + .clear() + .append_pair("identifier", identifier.as_str()); + + let resp = self + .http + .execute(HttpRequest { + url, + headers: HeaderMap::new(), + }) + .await?; + match resp.status { + StatusCode::OK => read_bounded(resp).await, + StatusCode::NOT_FOUND => Err(SlingshotError::NotFound), + other => Err(SlingshotError::Upstream(other)), + } + } + + pub async fn get_record( + &self, + repo: &Did, + collection: &Nsid, + rkey: &Rkey, + ) -> Result, SlingshotError> + where + S: BosStr, + { + let mut url = self + .base + .join(GET_RECORD_PATH) + .expect("base url is hierarchical and GET_RECORD_PATH is a literal relative path"); + url.query_pairs_mut() + .clear() + .append_pair("repo", repo.as_ref()) + .append_pair("collection", collection.as_ref()) + .append_pair("rkey", rkey.as_ref()); + + let resp = self + .http + .execute(HttpRequest { + url, + headers: HeaderMap::new(), + }) + .await?; + match resp.status { + StatusCode::OK => { + let bytes = read_bounded(resp).await?; + let body = decode(&bytes)?; + verify_addresses(&body, repo, collection, rkey)?; + Ok(Arc::new(body)) + } + StatusCode::NOT_FOUND => Err(SlingshotError::NotFound), + other => Err(SlingshotError::Upstream(other)), + } + } +} + +fn ensure_trailing_slash(mut base: Url) -> Url { + if !base.path().ends_with('/') { + let with_slash = format!("{}/", base.path()); + base.set_path(&with_slash); + } + base +} + +async fn read_bounded(resp: HttpResponseHead) -> Result { + if resp.content_length.is_some_and(|len| len > MAX_BODY_BYTES) { + return Err(SlingshotError::BodyTooLarge { + limit: MAX_BODY_BYTES, + }); + } + let buf = resp + .body + .map_err(SlingshotError::Network) + .try_fold(BytesMut::new(), |mut acc, chunk| async move { + if (acc.len() as u64).saturating_add(chunk.len() as u64) > MAX_BODY_BYTES { + return Err(SlingshotError::BodyTooLarge { + limit: MAX_BODY_BYTES, + }); + } + acc.extend_from_slice(&chunk); + Ok(acc) + }) + .await?; + Ok(buf.freeze()) +} + +fn decode(bytes: &[u8]) -> Result { + let raw: RawResponse<'_> = serde_json::from_slice(bytes)?; + let value = raw.value.ok_or(SlingshotError::MissingField("value"))?; + IpldCid::try_from(raw.cid)?; + Ok(RecordBody { + uri: AtUri::new_owned(raw.uri)?, + cid: raw.cid.to_owned().into(), + value: Bytes::copy_from_slice(value.get().as_bytes()), + }) +} + +fn verify_addresses( + body: &RecordBody, + repo: &Did, + collection: &Nsid, + rkey: &Rkey, +) -> Result<(), SlingshotError> { + let expected = format!( + "at://{}/{}/{}", + repo.as_ref(), + collection.as_ref(), + rkey.as_ref() + ); + if body.uri.as_ref() == expected { + Ok(()) + } else { + Err(SlingshotError::UriMismatch { + expected, + got: body.uri.as_ref().to_owned(), + }) + } +} + +#[derive(Deserialize)] +struct RawResponse<'a> { + uri: &'a str, + cid: &'a str, + #[serde(default, borrow)] + value: Option<&'a RawValue>, +} + +#[cfg(test)] +mod tests { + use super::*; + use jacquard_common::DefaultStr; + use serde_json::json; + use wiremock::matchers::{method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + const VALID_CID: &str = "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i"; + + fn did(s: &'static str) -> Did { + Did::new_static(s).unwrap() + } + fn nsid(s: &'static str) -> Nsid { + Nsid::new_static(s).unwrap() + } + fn rkey(s: &str) -> Rkey { + Rkey::new_owned(s).unwrap() + } + + async fn server() -> MockServer { + MockServer::start().await + } + + fn client_for(server: &MockServer) -> SlingshotClient { + SlingshotClient::with_default_http(Url::parse(&server.uri()).unwrap()).unwrap() + } + + #[tokio::test] + async fn returns_decoded_record_on_200() { + let server = server().await; + let body = json!({ + "uri": "at://did:plc:abalone/sh.tangled.repo/r1", + "cid": VALID_CID, + "value": { + "$type": "sh.tangled.repo", + "knot": "oyster.cafe", + "createdAt": "2026-05-01T00:00:00Z" + } + }); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", "did:plc:abalone")) + .and(query_param("collection", "sh.tangled.repo")) + .and(query_param("rkey", "r1")) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .mount(&server) + .await; + + let client = client_for(&server); + let resp = client + .get_record( + &did("did:plc:abalone"), + &nsid("sh.tangled.repo"), + &rkey("r1"), + ) + .await + .unwrap(); + assert_eq!(resp.uri.as_ref(), "at://did:plc:abalone/sh.tangled.repo/r1"); + let parsed: serde_json::Value = serde_json::from_slice(&resp.value).unwrap(); + assert_eq!(parsed["knot"], "oyster.cafe"); + } + + #[tokio::test] + async fn preserves_value_bytes_byte_for_byte() { + let server = server().await; + let raw = format!( + r#"{{"uri":"at://did:plc:uni/sh.tangled.repo/r1","cid":"{VALID_CID}","value":{{"z":1,"a":2}}}}"# + ); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/json") + .set_body_string(raw), + ) + .mount(&server) + .await; + + let client = client_for(&server); + let resp = client + .get_record(&did("did:plc:uni"), &nsid("sh.tangled.repo"), &rkey("r1")) + .await + .unwrap(); + assert_eq!(resp.value.as_ref(), br#"{"z":1,"a":2}"#); + } + + #[tokio::test] + async fn maps_404_to_not_found() { + let server = server().await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(ResponseTemplate::new(404).set_body_string("nope")) + .mount(&server) + .await; + + let client = client_for(&server); + let err = client + .get_record( + &did("did:plc:abalone"), + &nsid("sh.tangled.repo"), + &rkey("r1"), + ) + .await + .expect_err("404 must surface"); + assert!(matches!(err, SlingshotError::NotFound)); + } + + #[tokio::test] + async fn maps_400_to_upstream() { + let server = server().await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(ResponseTemplate::new(400).set_body_string("bad")) + .mount(&server) + .await; + + let client = client_for(&server); + let err = client + .get_record( + &did("did:plc:abalone"), + &nsid("sh.tangled.repo"), + &rkey("r1"), + ) + .await + .expect_err("400 must surface as upstream, not not-found"); + match err { + SlingshotError::Upstream(s) => assert_eq!(s.as_u16(), 400), + other => panic!("wrong variant: {other:?}"), + } + } + + #[tokio::test] + async fn maps_5xx_to_upstream() { + let server = server().await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + + let client = client_for(&server); + let err = client + .get_record( + &did("did:plc:abalone"), + &nsid("sh.tangled.repo"), + &rkey("r1"), + ) + .await + .expect_err("5xx must surface"); + match err { + SlingshotError::Upstream(s) => assert_eq!(s.as_u16(), 503), + other => panic!("wrong variant: {other:?}"), + } + } + + #[tokio::test] + async fn rejects_oversize_body_via_content_length() { + let server = server().await; + let payload = vec![b'x'; (MAX_BODY_BYTES + 1) as usize]; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/json") + .set_body_bytes(payload), + ) + .mount(&server) + .await; + + let client = client_for(&server); + let err = client + .get_record( + &did("did:plc:abalone"), + &nsid("sh.tangled.repo"), + &rkey("r1"), + ) + .await + .expect_err("oversize body must be rejected"); + assert!( + matches!(err, SlingshotError::BodyTooLarge { limit } if limit == MAX_BODY_BYTES), + "wrong variant: {err:?}" + ); + } + + #[tokio::test] + async fn rejects_garbage_cid() { + let server = server().await; + let body = json!({ + "uri": "at://did:plc:abalone/sh.tangled.repo/r1", + "cid": "not-a-real-cid", + "value": {"$type": "sh.tangled.repo", "knot": "oyster.cafe", "createdAt": "2026-05-01T00:00:00Z"} + }); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .mount(&server) + .await; + + let client = client_for(&server); + let err = client + .get_record( + &did("did:plc:abalone"), + &nsid("sh.tangled.repo"), + &rkey("r1"), + ) + .await + .expect_err("garbage cid must be rejected"); + assert!( + matches!(err, SlingshotError::InvalidCid(_)), + "wrong variant: {err:?}" + ); + } + + #[tokio::test] + async fn rejects_uri_mismatch() { + let server = server().await; + let body = json!({ + "uri": "at://did:plc:limpet/sh.tangled.repo/elsewhere", + "cid": VALID_CID, + "value": {"$type": "sh.tangled.repo", "knot": "oyster.cafe", "createdAt": "2026-05-01T00:00:00Z"} + }); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .mount(&server) + .await; + + let client = client_for(&server); + let err = client + .get_record( + &did("did:plc:abalone"), + &nsid("sh.tangled.repo"), + &rkey("r1"), + ) + .await + .expect_err("uri mismatch must be rejected"); + match err { + SlingshotError::UriMismatch { expected, got } => { + assert_eq!(expected, "at://did:plc:abalone/sh.tangled.repo/r1"); + assert_eq!(got, "at://did:plc:limpet/sh.tangled.repo/elsewhere"); + } + other => panic!("wrong variant: {other:?}"), + } + } + + #[tokio::test] + async fn rejects_explicit_null_value() { + let server = server().await; + let body = json!({ + "uri": "at://did:plc:abalone/sh.tangled.repo/r1", + "cid": VALID_CID, + "value": null + }); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .mount(&server) + .await; + + let client = client_for(&server); + let err = client + .get_record( + &did("did:plc:abalone"), + &nsid("sh.tangled.repo"), + &rkey("r1"), + ) + .await + .expect_err("explicit null value must be rejected"); + assert!( + matches!(err, SlingshotError::MissingField("value")), + "{err:?}" + ); + } + + #[tokio::test] + async fn rejects_missing_value_field() { + let server = server().await; + let body = json!({ + "uri": "at://did:plc:abalone/sh.tangled.repo/r1", + "cid": VALID_CID + }); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .mount(&server) + .await; + + let client = client_for(&server); + let err = client + .get_record( + &did("did:plc:abalone"), + &nsid("sh.tangled.repo"), + &rkey("r1"), + ) + .await + .expect_err("missing value must be rejected"); + assert!( + matches!(err, SlingshotError::MissingField("value")), + "{err:?}" + ); + } + + #[test] + fn rejects_non_http_scheme() { + let err = SlingshotClient::with_default_http(Url::parse("ftp://nel.pet").unwrap()) + .expect_err("ftp bad"); + assert!(matches!(err, SlingshotError::BadScheme(s) if s == "ftp")); + } + + #[tokio::test] + async fn preserves_operator_configured_base_path() { + let server = server().await; + let body = json!({ + "uri": "at://did:plc:abalone/sh.tangled.repo/r1", + "cid": VALID_CID, + "value": { + "$type": "sh.tangled.repo", + "knot": "oyster.cafe", + "createdAt": "2026-05-01T00:00:00Z" + } + }); + Mock::given(method("GET")) + .and(path("/api/v0/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", "did:plc:abalone")) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .mount(&server) + .await; + + let base = Url::parse(&format!("{}/api/v0", server.uri())).unwrap(); + let client = SlingshotClient::with_default_http(base).unwrap(); + client + .get_record( + &did("did:plc:abalone"), + &nsid("sh.tangled.repo"), + &rkey("r1"), + ) + .await + .expect("path-prefixed base must reach prefixed xrpc endpoint"); + } + + #[tokio::test] + async fn accepts_borrowed_string_newtypes() { + let server = server().await; + let body = json!({ + "uri": "at://did:plc:limpet/sh.tangled.repo/r1", + "cid": VALID_CID, + "value": { + "$type": "sh.tangled.repo", + "knot": "nel.pet", + "createdAt": "2026-05-01T00:00:00Z" + } + }); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .mount(&server) + .await; + + let uri = AtUri::::new_owned("at://did:plc:limpet/sh.tangled.repo/r1").unwrap(); + let collection = uri.collection().unwrap(); + let rkey = uri.rkey().unwrap(); + let did_borrow = match uri.authority() { + jacquard_common::types::ident::AtIdentifier::Did(d) => d, + jacquard_common::types::ident::AtIdentifier::Handle(_) => unreachable!(), + }; + + let client = client_for(&server); + let resp = client + .get_record(&did_borrow, &collection, &rkey) + .await + .unwrap(); + assert_eq!(resp.uri.as_ref(), "at://did:plc:limpet/sh.tangled.repo/r1"); + } +} diff --git a/bobbin/crates/types/Cargo.toml b/bobbin/crates/types/Cargo.toml new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "bobbin-types" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bytes = { workspace = true } +chrono = { workspace = true } +cid = { workspace = true } +jacquard-common = { workspace = true } +jacquard-derive = { workspace = true } +jacquard-lexicon = { workspace = true } +miette = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } + +[build-dependencies] +anyhow = { workspace = true } +jacquard-lexicon = { workspace = true, features = ["codegen"] } +walkdir = { workspace = true } + +[features] +default = ["sh_tangled"] +com_atproto = [] +sh_tangled = ["com_atproto"] +streaming = ["jacquard-common/websocket"] + diff --git a/bobbin/crates/types/build.rs b/bobbin/crates/types/build.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/build.rs @@ -0,0 +1,78 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use jacquard_lexicon::codegen::{CodeGenerator, CodegenMode}; +use jacquard_lexicon::corpus::LexiconCorpus; +use walkdir::WalkDir; + +const DEFAULT_LEXICONS_SUBDIR: &str = "lexicons"; +const STAGED_SUBDIR: &str = "lexicons-staged"; +const GENERATED_SUBDIR: &str = "src/_lex"; +const TEMP_SEGMENT: &str = "temp"; + +fn main() -> Result<()> { + let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR")?); + let workspace_root = manifest_dir + .parent() + .and_then(Path::parent) + .and_then(Path::parent) + .context("resolve workspace root from manifest dir")? + .to_path_buf(); + + let lexicons_dir = std::env::var("BOBBIN_LEXICONS_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| workspace_root.join(DEFAULT_LEXICONS_SUBDIR)); + let additions_dir = manifest_dir.join(DEFAULT_LEXICONS_SUBDIR); + + println!("cargo:rerun-if-env-changed=BOBBIN_LEXICONS_DIR"); + println!("cargo:rerun-if-changed={}", lexicons_dir.display()); + println!("cargo:rerun-if-changed={}", additions_dir.display()); + println!("cargo:rerun-if-changed=build.rs"); + + let out_dir = PathBuf::from(std::env::var("OUT_DIR")?); + + let staged = out_dir.join(STAGED_SUBDIR); + if staged.exists() { + std::fs::remove_dir_all(&staged).context("clean staged lexicons")?; + } + stage_lexicons(&lexicons_dir, &staged)?; + stage_lexicons(&additions_dir, &staged)?; + + let corpus = LexiconCorpus::load_from_dir(&staged) + .map_err(|e| anyhow::anyhow!("load lexicon corpus: {e:?}"))?; + + let generated = manifest_dir.join(GENERATED_SUBDIR); + if generated.exists() { + std::fs::remove_dir_all(&generated).context("clean generated dir")?; + } + std::fs::create_dir_all(&generated).context("create generated dir")?; + + let codegen = CodeGenerator::with_mode(&corpus, "crate", CodegenMode::Pretty); + codegen + .write_to_disk(&generated) + .map_err(|e| anyhow::anyhow!("write generated code: {e:?}"))?; + + Ok(()) +} + +fn stage_lexicons(src: &Path, dst: &Path) -> Result<()> { + WalkDir::new(src) + .into_iter() + .filter_map(std::result::Result::ok) + .filter(|e| e.file_type().is_file()) + .filter(|e| { + e.path() + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("json")) + }) + .filter(|e| !e.path().components().any(|c| c.as_os_str() == TEMP_SEGMENT)) + .try_for_each(|entry| -> Result<()> { + let rel = entry.path().strip_prefix(src).context("strip src prefix")?; + let target = dst.join(rel); + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent).context("create staged parent")?; + } + std::fs::copy(entry.path(), &target).context("copy lexicon file")?; + Ok(()) + }) +} diff --git a/bobbin/crates/types/lexicons/feed/listReactions.json b/bobbin/crates/types/lexicons/feed/listReactions.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/feed/listReactions.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.feed.listReactions", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "at-uri", + "description": "Record AT-URI the reactions target." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.feed.reaction record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/feed/listReactionsBy.json b/bobbin/crates/types/lexicons/feed/listReactionsBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/feed/listReactionsBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.feed.listReactionsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose reaction authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.feed.listReactions#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/feed/listStars.json b/bobbin/crates/types/lexicons/feed/listStars.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/feed/listStars.json @@ -0,0 +1,61 @@ +{ + "lexicon": 1, + "id": "sh.tangled.feed.listStars", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Repo DID to list star edges for." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.feed.star record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/feed/listStarsBy.json b/bobbin/crates/types/lexicons/feed/listStarsBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/feed/listStarsBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.feed.listStarsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose star authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.feed.listStars#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/git/listRefUpdates.json b/bobbin/crates/types/lexicons/git/listRefUpdates.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/git/listRefUpdates.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.git.listRefUpdates", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Repo DID whose ref-update records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.git.refUpdate record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/git/listRefUpdatesBy.json b/bobbin/crates/types/lexicons/git/listRefUpdatesBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/git/listRefUpdatesBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.git.listRefUpdatesBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose ref-update authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.git.listRefUpdates#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/graph/listFollows.json b/bobbin/crates/types/lexicons/graph/listFollows.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/graph/listFollows.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.graph.listFollows", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Followee DID whose inbound follows to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.graph.follow record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/graph/listFollowsBy.json b/bobbin/crates/types/lexicons/graph/listFollowsBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/graph/listFollowsBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.graph.listFollowsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose follow authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.graph.listFollows#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/graph/listVouches.json b/bobbin/crates/types/lexicons/graph/listVouches.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/graph/listVouches.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.graph.listVouches", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Vouchee DID whose inbound vouches to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.graph.vouch record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/graph/listVouchesBy.json b/bobbin/crates/types/lexicons/graph/listVouchesBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/graph/listVouchesBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.graph.listVouchesBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose vouch authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.graph.listVouches#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/knot/listKnots.json b/bobbin/crates/types/lexicons/knot/listKnots.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/knot/listKnots.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.knot.listKnots", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Owner DID whose knot records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.knot record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/knot/listMembers.json b/bobbin/crates/types/lexicons/knot/listMembers.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/knot/listMembers.json @@ -0,0 +1,61 @@ +{ + "lexicon": 1, + "id": "sh.tangled.knot.listMembers", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Knot identifier whose member records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.knot.member record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/knot/listMembersBy.json b/bobbin/crates/types/lexicons/knot/listMembersBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/knot/listMembersBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.knot.listMembersBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose knot-member authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.knot.listMembers#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/label/listDefinitions.json b/bobbin/crates/types/lexicons/label/listDefinitions.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/label/listDefinitions.json @@ -0,0 +1,61 @@ +{ + "lexicon": 1, + "id": "sh.tangled.label.listDefinitions", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Scope identifier whose label definitions to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.label.definition record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/label/listOps.json b/bobbin/crates/types/lexicons/label/listOps.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/label/listOps.json @@ -0,0 +1,61 @@ +{ + "lexicon": 1, + "id": "sh.tangled.label.listOps", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Scope identifier whose label op records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.label.op record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/label/listOpsBy.json b/bobbin/crates/types/lexicons/label/listOpsBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/label/listOpsBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.label.listOpsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose label-op authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.label.listOps#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/pipeline/listPipelines.json b/bobbin/crates/types/lexicons/pipeline/listPipelines.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/pipeline/listPipelines.json @@ -0,0 +1,61 @@ +{ + "lexicon": 1, + "id": "sh.tangled.pipeline.listPipelines", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Repo or spindle identifier whose pipeline records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.pipeline record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/pipeline/listPipelinesBy.json b/bobbin/crates/types/lexicons/pipeline/listPipelinesBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/pipeline/listPipelinesBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.pipeline.listPipelinesBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose pipeline authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.pipeline.listPipelines#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/pipeline/listStatuses.json b/bobbin/crates/types/lexicons/pipeline/listStatuses.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/pipeline/listStatuses.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.pipeline.listStatuses", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "at-uri", + "description": "Pipeline AT-URI whose status records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.pipeline.status record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/pipeline/listStatusesBy.json b/bobbin/crates/types/lexicons/pipeline/listStatusesBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/pipeline/listStatusesBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.pipeline.listStatusesBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose pipeline-status authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.pipeline.listStatuses#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/publicKey/listKeys.json b/bobbin/crates/types/lexicons/publicKey/listKeys.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/publicKey/listKeys.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.publicKey.listKeys", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Owner DID whose public-key records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.publicKey record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/issue/listComments.json b/bobbin/crates/types/lexicons/repo/issue/listComments.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/issue/listComments.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.issue.listComments", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "at-uri", + "description": "Issue AT-URI whose comments to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo.issue.comment record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/issue/listCommentsBy.json b/bobbin/crates/types/lexicons/repo/issue/listCommentsBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/issue/listCommentsBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.issue.listCommentsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose issue-comment authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.repo.issue.listComments#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/issue/listStates.json b/bobbin/crates/types/lexicons/repo/issue/listStates.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/issue/listStates.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.issue.listStates", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "at-uri", + "description": "Issue AT-URI whose state records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo.issue.state record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/issue/listStatesBy.json b/bobbin/crates/types/lexicons/repo/issue/listStatesBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/issue/listStatesBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.issue.listStatesBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose issue-state authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.repo.issue.listStates#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/listArtifacts.json b/bobbin/crates/types/lexicons/repo/listArtifacts.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/listArtifacts.json @@ -0,0 +1,61 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.listArtifacts", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Repo or release identifier whose artifacts to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo.artifact record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/listArtifactsBy.json b/bobbin/crates/types/lexicons/repo/listArtifactsBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/listArtifactsBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.listArtifactsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose artifact authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.repo.listArtifacts#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/listCollaborators.json b/bobbin/crates/types/lexicons/repo/listCollaborators.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/listCollaborators.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.listCollaborators", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Repo DID whose collaborator records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo.collaborator record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/listCollaboratorsBy.json b/bobbin/crates/types/lexicons/repo/listCollaboratorsBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/listCollaboratorsBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.listCollaboratorsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose collaborator authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.repo.listCollaborators#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/listIssues.json b/bobbin/crates/types/lexicons/repo/listIssues.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/listIssues.json @@ -0,0 +1,98 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.listIssues", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Repo DID to list issues for" + }, + "author": { + "type": "string", + "format": "did", + "description": "Restrict to issues authored by this user DID." + }, + "state": { + "type": "string", + "knownValues": ["open", "closed"], + "description": "Restrict to issues whose latest derived state matches." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "ref", + "ref": "#issueListItem" + } + }, + "cursor": { + "type": "string" + } + } + } + } + }, + "issueListItem": { + "type": "object", + "required": ["uri", "value", "state", "commentCount"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo.issue record" + }, + "state": { + "type": "string", + "knownValues": ["open", "closed"], + "description": "Latest derived state." + }, + "stateUpdatedAt": { + "type": "string", + "format": "datetime", + "description": "TID-derived timestamp of the latest state record." + }, + "commentCount": { + "type": "integer", + "minimum": 0, + "description": "Count of sh.tangled.repo.issue.comment records targeting this issue." + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/listIssuesBy.json b/bobbin/crates/types/lexicons/repo/listIssuesBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/listIssuesBy.json @@ -0,0 +1,60 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.listIssuesBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose issue authorings to list" + }, + "state": { + "type": "string", + "knownValues": ["open", "closed"], + "description": "Restrict to issues whose latest derived state matches." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "ref", + "ref": "sh.tangled.repo.listIssues#issueListItem" + } + }, + "cursor": { + "type": "string" + } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/listPulls.json b/bobbin/crates/types/lexicons/repo/listPulls.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/listPulls.json @@ -0,0 +1,98 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.listPulls", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Repo DID to list pulls for" + }, + "author": { + "type": "string", + "format": "did", + "description": "Restrict to pulls authored by this user DID." + }, + "status": { + "type": "string", + "knownValues": ["open", "closed", "merged"], + "description": "Restrict to pulls whose latest derived status matches." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "ref", + "ref": "#pullListItem" + } + }, + "cursor": { + "type": "string" + } + } + } + } + }, + "pullListItem": { + "type": "object", + "required": ["uri", "value", "state", "commentCount"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo.pull record" + }, + "state": { + "type": "string", + "knownValues": ["open", "closed", "merged"], + "description": "Latest derived state." + }, + "stateUpdatedAt": { + "type": "string", + "format": "datetime", + "description": "TID-derived timestamp of the latest pull status record." + }, + "commentCount": { + "type": "integer", + "minimum": 0, + "description": "Count of sh.tangled.repo.pull.comment records targeting this pull." + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/listPullsBy.json b/bobbin/crates/types/lexicons/repo/listPullsBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/listPullsBy.json @@ -0,0 +1,60 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.listPullsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose pull authorings to list" + }, + "status": { + "type": "string", + "knownValues": ["open", "closed", "merged"], + "description": "Restrict to pulls whose latest derived status matches." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "ref", + "ref": "sh.tangled.repo.listPulls#pullListItem" + } + }, + "cursor": { + "type": "string" + } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/listRepos.json b/bobbin/crates/types/lexicons/repo/listRepos.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/listRepos.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.listRepos", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Owner DID whose repo records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/pull/listComments.json b/bobbin/crates/types/lexicons/repo/pull/listComments.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/pull/listComments.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.pull.listComments", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "at-uri", + "description": "Pull AT-URI whose comments to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo.pull.comment record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/pull/listCommentsBy.json b/bobbin/crates/types/lexicons/repo/pull/listCommentsBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/pull/listCommentsBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.pull.listCommentsBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose pull-comment authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.repo.pull.listComments#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/pull/listStatuses.json b/bobbin/crates/types/lexicons/repo/pull/listStatuses.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/pull/listStatuses.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.pull.listStatuses", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "at-uri", + "description": "Pull AT-URI whose status records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.repo.pull.status record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/repo/pull/listStatusesBy.json b/bobbin/crates/types/lexicons/repo/pull/listStatusesBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/repo/pull/listStatusesBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.pull.listStatusesBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose pull-status authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.repo.pull.listStatuses#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/spindle/listMembers.json b/bobbin/crates/types/lexicons/spindle/listMembers.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/spindle/listMembers.json @@ -0,0 +1,61 @@ +{ + "lexicon": 1, + "id": "sh.tangled.spindle.listMembers", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Spindle identifier whose member records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.spindle.member record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/spindle/listMembersBy.json b/bobbin/crates/types/lexicons/spindle/listMembersBy.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/spindle/listMembersBy.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "sh.tangled.spindle.listMembersBy", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Actor DID whose spindle-member authorings to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "sh.tangled.spindle.listMembers#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/spindle/listSpindles.json b/bobbin/crates/types/lexicons/spindle/listSpindles.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/spindle/listSpindles.json @@ -0,0 +1,62 @@ +{ + "lexicon": 1, + "id": "sh.tangled.spindle.listSpindles", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "Owner DID whose spindle records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.spindle record" + } + } + } + } +} diff --git a/bobbin/crates/types/lexicons/string/listStrings.json b/bobbin/crates/types/lexicons/string/listStrings.json new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/lexicons/string/listStrings.json @@ -0,0 +1,61 @@ +{ + "lexicon": 1, + "id": "sh.tangled.string.listStrings", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": ["subject"], + "properties": { + "subject": { + "type": "string", + "description": "Scope identifier whose string records to list." + }, + "cursor": { + "type": "string", + "description": "Pagination cursor" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 50 + }, + "order": { + "type": "string", + "knownValues": ["asc", "desc"], + "default": "desc", + "description": "Sort direction by createdAt." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { "type": "ref", "ref": "#listItem" } + }, + "cursor": { "type": "string" } + } + } + } + }, + "listItem": { + "type": "object", + "required": ["uri", "value"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "value": { + "type": "unknown", + "description": "Embedded sh.tangled.string record" + } + } + } + } +} diff --git a/bobbin/crates/types/src/edges.rs b/bobbin/crates/types/src/edges.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/src/edges.rs @@ -0,0 +1,1064 @@ +use alloc::vec; +use alloc::vec::Vec; + +use jacquard_common::types::did::Did; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::string::{AtStrError, AtUri, Datetime}; +use jacquard_common::types::tid::Tid; +use jacquard_common::{BosStr, DefaultStr}; + +use crate::ids::{SubjectRef, nsid_static}; +use crate::sh_tangled::actor::profile::Profile; +use crate::sh_tangled::feed::reaction::Reaction; +use crate::sh_tangled::feed::star::Star; +use crate::sh_tangled::git::ref_update::RefUpdate; +use crate::sh_tangled::graph::follow::Follow; +use crate::sh_tangled::graph::vouch::Vouch; +use crate::sh_tangled::knot::Knot; +use crate::sh_tangled::knot::member::Member as KnotMemberRecord; +use crate::sh_tangled::label::definition::Definition as LabelDefinitionRecord; +use crate::sh_tangled::label::op::Op as LabelOpRecord; +use crate::sh_tangled::pipeline::Pipeline; +use crate::sh_tangled::pipeline::status::Status as PipelineStatusRecord; +use crate::sh_tangled::public_key::PublicKey; +use crate::sh_tangled::repo::Repo as RepoRecord; +use crate::sh_tangled::repo::artifact::Artifact; +use crate::sh_tangled::repo::collaborator::Collaborator; +use crate::sh_tangled::repo::issue::Issue; +use crate::sh_tangled::repo::issue::comment::Comment as IssueCommentRecord; +use crate::sh_tangled::repo::issue::state::State as IssueStateRecord; +use crate::sh_tangled::repo::pull::Pull; +use crate::sh_tangled::repo::pull::comment::Comment as PullCommentRecord; +use crate::sh_tangled::repo::pull::status::Status as PullStatusRecord; +use crate::sh_tangled::spindle::Spindle; +use crate::sh_tangled::spindle::member::Member as SpindleMemberRecord; +use crate::sh_tangled::string::TangledString; + +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub struct Edge { + pub kind: Nsid, + pub subject: SubjectRef, + pub source: AtUri, + pub sort_micros: u64, +} + +#[derive(Debug, thiserror::Error)] +pub enum ExtractError { + #[error("decode JSON record body: {0}")] + DecodeJson(#[from] serde_json::Error), + #[error("invalid AT-URI synthesized from record subject: {0}")] + InvalidAtUri(#[from] AtStrError), + #[error("unknown collection NSID: {0}")] + UnknownCollection(alloc::string::String), +} + +#[derive(Debug)] +pub enum Record { + Profile(Profile), + Reaction(Reaction), + Star(Star), + RefUpdate(RefUpdate), + Follow(Follow), + Vouch(Vouch), + Knot(Knot), + KnotMember(KnotMemberRecord), + LabelDefinition(LabelDefinitionRecord), + LabelOp(LabelOpRecord), + Pipeline(Pipeline), + PipelineStatus(PipelineStatusRecord), + PublicKey(PublicKey), + Repo(RepoRecord), + Artifact(Artifact), + Collaborator(Collaborator), + Issue(Issue), + IssueComment(IssueCommentRecord), + IssueState(IssueStateRecord), + Pull(Pull), + PullComment(PullCommentRecord), + PullStatus(PullStatusRecord), + Spindle(Spindle), + SpindleMember(SpindleMemberRecord), + TangledString(TangledString), +} + +impl Record { + pub fn from_json_value>( + collection: &Nsid, + value: serde_json::Value, + ) -> Result { + let bytes = serde_json::to_vec(&value)?; + Self::from_json_bytes(collection, &bytes) + } + + pub fn from_json_bytes>( + nsid: &Nsid, + bytes: &[u8], + ) -> Result { + macro_rules! parse { + ($variant:ident) => { + Ok(Self::$variant(serde_json::from_slice(bytes)?)) + }; + } + match nsid.as_ref() { + "sh.tangled.actor.profile" => parse!(Profile), + "sh.tangled.feed.reaction" => parse!(Reaction), + "sh.tangled.feed.star" => parse!(Star), + "sh.tangled.git.refUpdate" => parse!(RefUpdate), + "sh.tangled.graph.follow" => parse!(Follow), + "sh.tangled.graph.vouch" => parse!(Vouch), + "sh.tangled.knot" => parse!(Knot), + "sh.tangled.knot.member" => parse!(KnotMember), + "sh.tangled.label.definition" => parse!(LabelDefinition), + "sh.tangled.label.op" => parse!(LabelOp), + "sh.tangled.pipeline" => parse!(Pipeline), + "sh.tangled.pipeline.status" => parse!(PipelineStatus), + "sh.tangled.publicKey" => parse!(PublicKey), + "sh.tangled.repo" => parse!(Repo), + "sh.tangled.repo.artifact" => parse!(Artifact), + "sh.tangled.repo.collaborator" => parse!(Collaborator), + "sh.tangled.repo.issue" => parse!(Issue), + "sh.tangled.repo.issue.comment" => parse!(IssueComment), + "sh.tangled.repo.issue.state" => parse!(IssueState), + "sh.tangled.repo.pull" => parse!(Pull), + "sh.tangled.repo.pull.comment" => parse!(PullComment), + "sh.tangled.repo.pull.status" => parse!(PullStatus), + "sh.tangled.spindle" => parse!(Spindle), + "sh.tangled.spindle.member" => parse!(SpindleMember), + "sh.tangled.string" => parse!(TangledString), + other => Err(ExtractError::UnknownCollection(other.into())), + } + } + + pub fn collection(&self) -> Nsid { + let s: &'static str = match self { + Self::Profile(_) => "sh.tangled.actor.profile", + Self::Reaction(_) => "sh.tangled.feed.reaction", + Self::Star(_) => "sh.tangled.feed.star", + Self::RefUpdate(_) => "sh.tangled.git.refUpdate", + Self::Follow(_) => "sh.tangled.graph.follow", + Self::Vouch(_) => "sh.tangled.graph.vouch", + Self::Knot(_) => "sh.tangled.knot", + Self::KnotMember(_) => "sh.tangled.knot.member", + Self::LabelDefinition(_) => "sh.tangled.label.definition", + Self::LabelOp(_) => "sh.tangled.label.op", + Self::Pipeline(_) => "sh.tangled.pipeline", + Self::PipelineStatus(_) => "sh.tangled.pipeline.status", + Self::PublicKey(_) => "sh.tangled.publicKey", + Self::Repo(_) => "sh.tangled.repo", + Self::Artifact(_) => "sh.tangled.repo.artifact", + Self::Collaborator(_) => "sh.tangled.repo.collaborator", + Self::Issue(_) => "sh.tangled.repo.issue", + Self::IssueComment(_) => "sh.tangled.repo.issue.comment", + Self::IssueState(_) => "sh.tangled.repo.issue.state", + Self::Pull(_) => "sh.tangled.repo.pull", + Self::PullComment(_) => "sh.tangled.repo.pull.comment", + Self::PullStatus(_) => "sh.tangled.repo.pull.status", + Self::Spindle(_) => "sh.tangled.spindle", + Self::SpindleMember(_) => "sh.tangled.spindle.member", + Self::TangledString(_) => "sh.tangled.string", + }; + nsid_static(s) + } + + pub fn extract_edges(&self, source: &AtUri) -> Result, ExtractError> { + let sort_micros = self.sort_micros_for(source); + let mut primary = self.primary_edges(source)?; + primary.iter_mut().for_each(|e| e.sort_micros = sort_micros); + Ok(append_mirror_edges(primary, source)) + } + + pub fn sort_micros_for(&self, source: &AtUri) -> u64 { + if let Some(dt) = self.created_at() { + let micros = dt.timestamp_micros(); + if micros >= 0 { + return micros as u64; + } + } + if let Some(rkey) = source.rkey() + && let Ok(tid) = Tid::new(rkey.as_ref()) + { + return tid.timestamp(); + } + 0 + } + + fn created_at(&self) -> Option<&Datetime> { + match self { + Self::Profile(_) => None, + Self::Reaction(r) => Some(&r.created_at), + Self::Star(r) => Some(&r.created_at), + Self::RefUpdate(_) => None, + Self::Follow(r) => Some(&r.created_at), + Self::Vouch(r) => Some(&r.created_at), + Self::Knot(r) => Some(&r.created_at), + Self::KnotMember(r) => Some(&r.created_at), + Self::LabelDefinition(r) => Some(&r.created_at), + Self::LabelOp(r) => Some(&r.performed_at), + Self::Pipeline(_) => None, + Self::PipelineStatus(r) => Some(&r.created_at), + Self::PublicKey(r) => Some(&r.created_at), + Self::Repo(r) => Some(&r.created_at), + Self::Artifact(r) => Some(&r.created_at), + Self::Collaborator(r) => Some(&r.created_at), + Self::Issue(r) => Some(&r.created_at), + Self::IssueComment(r) => Some(&r.created_at), + Self::IssueState(_) => None, + Self::Pull(r) => Some(&r.created_at), + Self::PullComment(r) => Some(&r.created_at), + Self::PullStatus(_) => None, + Self::Spindle(r) => Some(&r.created_at), + Self::SpindleMember(r) => Some(&r.created_at), + Self::TangledString(r) => Some(&r.created_at), + } + } + + fn primary_edges(&self, source: &AtUri) -> Result, ExtractError> { + match self { + Self::Star(r) => star_edges(source, r), + Self::Reaction(r) => reaction_edges(source, r), + Self::Follow(r) => follow_edges(source, r), + Self::RefUpdate(r) => ref_update_edges(source, r), + Self::KnotMember(r) => knot_member_edges(source, r), + Self::LabelOp(r) => label_op_edges(source, r), + Self::PipelineStatus(r) => pipeline_status_edges(source, r), + Self::Artifact(r) => artifact_edges(source, r), + Self::Collaborator(r) => collaborator_edges(source, r), + Self::Issue(r) => issue_edges(source, r), + Self::IssueComment(r) => issue_comment_edges(source, r), + Self::IssueState(r) => issue_state_edges(source, r), + Self::Pull(r) => pull_edges(source, r), + Self::PullComment(r) => pull_comment_edges(source, r), + Self::PullStatus(r) => pull_status_edges(source, r), + Self::SpindleMember(r) => spindle_member_edges(source, r), + Self::Pipeline(r) => pipeline_edges(source, r), + Self::Knot(_) => Ok(owner_self_edges("sh.tangled.knot", source)), + Self::LabelDefinition(_) => Ok(owner_self_edges("sh.tangled.label.definition", source)), + Self::PublicKey(_) => Ok(owner_self_edges("sh.tangled.publicKey", source)), + Self::Repo(_) => Ok(owner_self_edges("sh.tangled.repo", source)), + Self::Spindle(_) => Ok(owner_self_edges("sh.tangled.spindle", source)), + Self::TangledString(_) => Ok(owner_self_edges("sh.tangled.string", source)), + Self::Vouch(r) => vouch_edges(source, r), + Self::Profile(_) => Ok(Vec::new()), + } + } +} + +fn repo_subject( + uri: &Option>, + did: &Option>, +) -> Option { + if let Some(d) = did.as_ref() { + return Some(SubjectRef::Did(d.clone())); + } + uri_subject_for_record(uri.as_ref()?) +} + +fn uri_subject_for_record(uri: &AtUri) -> Option { + crate::ids::owner_did_from_aturi(uri)?; + Some(SubjectRef::Uri(uri.clone())) +} + +fn one_edge(kind: &'static str, subject: SubjectRef, source: &AtUri) -> Vec { + vec![Edge { + kind: nsid_static(kind), + subject, + source: source.clone(), + sort_micros: 0, + }] +} + +const MIRROR_KINDS: &[(&str, &str)] = &[ + ("sh.tangled.feed.star", "sh.tangled.feed.star.by"), + ("sh.tangled.feed.reaction", "sh.tangled.feed.reaction.by"), + ("sh.tangled.graph.follow", "sh.tangled.graph.follow.by"), + ("sh.tangled.graph.vouch", "sh.tangled.graph.vouch.by"), + ("sh.tangled.git.refUpdate", "sh.tangled.git.refUpdate.by"), + ("sh.tangled.knot.member", "sh.tangled.knot.member.by"), + ("sh.tangled.label.op", "sh.tangled.label.op.by"), + ("sh.tangled.pipeline", "sh.tangled.pipeline.by"), + ( + "sh.tangled.pipeline.status", + "sh.tangled.pipeline.status.by", + ), + ("sh.tangled.repo.artifact", "sh.tangled.repo.artifact.by"), + ( + "sh.tangled.repo.collaborator", + "sh.tangled.repo.collaborator.by", + ), + ("sh.tangled.repo.issue", "sh.tangled.repo.issue.by"), + ( + "sh.tangled.repo.issue.comment", + "sh.tangled.repo.issue.comment.by", + ), + ( + "sh.tangled.repo.issue.state", + "sh.tangled.repo.issue.state.by", + ), + ("sh.tangled.repo.pull", "sh.tangled.repo.pull.by"), + ( + "sh.tangled.repo.pull.comment", + "sh.tangled.repo.pull.comment.by", + ), + ( + "sh.tangled.repo.pull.status", + "sh.tangled.repo.pull.status.by", + ), + ("sh.tangled.spindle.member", "sh.tangled.spindle.member.by"), +]; + +fn mirror_kind_for(kind: &str) -> Option<&'static str> { + MIRROR_KINDS + .iter() + .find(|(primary, _)| *primary == kind) + .map(|(_, mirror)| *mirror) +} + +fn append_mirror_edges(primary: Vec, source: &AtUri) -> Vec { + let Some(author) = crate::ids::owner_did_from_aturi(source) else { + return primary; + }; + let author_subject = SubjectRef::Did(author); + let mirrors: Vec = primary + .iter() + .filter_map(|edge| { + let mirror_nsid = mirror_kind_for(edge.kind.as_ref())?; + (edge.subject != author_subject).then(|| Edge { + kind: nsid_static(mirror_nsid), + subject: author_subject.clone(), + source: edge.source.clone(), + sort_micros: edge.sort_micros, + }) + }) + .collect(); + primary.into_iter().chain(mirrors).collect() +} + +fn star_edges( + source: &AtUri, + record: &Star, +) -> Result, ExtractError> { + use crate::sh_tangled::feed::star::StarSubject; + let subject = match &record.subject { + StarSubject::Repo(r) => SubjectRef::Did(r.did.clone()), + StarSubject::String(s) => { + let Some(uri_subject) = uri_subject_for_record(&s.uri) else { + return Ok(Vec::new()); + }; + uri_subject + } + }; + Ok(one_edge("sh.tangled.feed.star", subject, source)) +} + +fn reaction_edges( + source: &AtUri, + record: &Reaction, +) -> Result, ExtractError> { + let Some(subject) = uri_subject_for_record(&record.subject) else { + return Ok(Vec::new()); + }; + Ok(one_edge("sh.tangled.feed.reaction", subject, source)) +} + +fn follow_edges( + source: &AtUri, + record: &Follow, +) -> Result, ExtractError> { + Ok(one_edge( + "sh.tangled.graph.follow", + SubjectRef::Did(record.subject.clone()), + source, + )) +} + +fn ref_update_edges( + source: &AtUri, + record: &RefUpdate, +) -> Result, ExtractError> { + Ok(one_edge( + "sh.tangled.git.refUpdate", + SubjectRef::Did(record.repo.clone()), + source, + )) +} + +fn knot_member_edges( + source: &AtUri, + record: &KnotMemberRecord, +) -> Result, ExtractError> { + Ok(one_edge( + "sh.tangled.knot.member", + SubjectRef::Did(record.subject.clone()), + source, + )) +} + +fn label_op_edges( + source: &AtUri, + record: &LabelOpRecord, +) -> Result, ExtractError> { + let Some(subject) = uri_subject_for_record(&record.subject) else { + return Ok(Vec::new()); + }; + Ok(one_edge("sh.tangled.label.op", subject, source)) +} + +fn pipeline_status_edges( + source: &AtUri, + record: &PipelineStatusRecord, +) -> Result, ExtractError> { + let Some(subject) = uri_subject_for_record(&record.pipeline) else { + return Ok(Vec::new()); + }; + Ok(one_edge("sh.tangled.pipeline.status", subject, source)) +} + +fn artifact_edges( + source: &AtUri, + record: &Artifact, +) -> Result, ExtractError> { + let Some(subject) = repo_subject(&record.repo, &record.repo_did) else { + return Ok(Vec::new()); + }; + Ok(one_edge("sh.tangled.repo.artifact", subject, source)) +} + +fn collaborator_edges( + source: &AtUri, + record: &Collaborator, +) -> Result, ExtractError> { + Ok(one_edge( + "sh.tangled.repo.collaborator", + SubjectRef::Did(record.repo.clone()), + source, + )) +} + +fn issue_edges( + source: &AtUri, + record: &Issue, +) -> Result, ExtractError> { + Ok(one_edge( + "sh.tangled.repo.issue", + SubjectRef::Did(record.repo.clone()), + source, + )) +} + +fn issue_comment_edges( + source: &AtUri, + record: &IssueCommentRecord, +) -> Result, ExtractError> { + let Some(subject) = uri_subject_for_record(&record.issue) else { + return Ok(Vec::new()); + }; + Ok(one_edge("sh.tangled.repo.issue.comment", subject, source)) +} + +fn issue_state_edges( + source: &AtUri, + record: &IssueStateRecord, +) -> Result, ExtractError> { + let Some(subject) = uri_subject_for_record(&record.issue) else { + return Ok(Vec::new()); + }; + Ok(one_edge("sh.tangled.repo.issue.state", subject, source)) +} + +fn pull_edges( + source: &AtUri, + record: &Pull, +) -> Result, ExtractError> { + Ok(one_edge( + "sh.tangled.repo.pull", + SubjectRef::Did(record.target.repo.clone()), + source, + )) +} + +fn pull_comment_edges( + source: &AtUri, + record: &PullCommentRecord, +) -> Result, ExtractError> { + let Some(subject) = uri_subject_for_record(&record.pull) else { + return Ok(Vec::new()); + }; + Ok(one_edge("sh.tangled.repo.pull.comment", subject, source)) +} + +fn pull_status_edges( + source: &AtUri, + record: &PullStatusRecord, +) -> Result, ExtractError> { + let Some(subject) = uri_subject_for_record(&record.pull) else { + return Ok(Vec::new()); + }; + Ok(one_edge("sh.tangled.repo.pull.status", subject, source)) +} + +fn vouch_edges( + source: &AtUri, + _record: &Vouch, +) -> Result, ExtractError> { + let Some(rkey) = source.rkey() else { + return Ok(Vec::new()); + }; + let Ok(vouchee) = Did::::new_owned(rkey.as_ref()) else { + return Ok(Vec::new()); + }; + Ok(one_edge( + "sh.tangled.graph.vouch", + SubjectRef::Did(vouchee), + source, + )) +} + +fn spindle_member_edges( + source: &AtUri, + record: &SpindleMemberRecord, +) -> Result, ExtractError> { + Ok(one_edge( + "sh.tangled.spindle.member", + SubjectRef::Did(record.subject.clone()), + source, + )) +} + +fn pipeline_edges( + source: &AtUri, + record: &Pipeline, +) -> Result, ExtractError> { + let trigger_repo = &record.trigger_metadata.repo; + let repo_did = trigger_repo + .repo_did + .as_ref() + .unwrap_or(&trigger_repo.did) + .clone(); + Ok(one_edge( + "sh.tangled.pipeline", + SubjectRef::Did(repo_did), + source, + )) +} + +fn owner_self_edges(kind: &'static str, source: &AtUri) -> Vec { + crate::ids::owner_did_from_aturi(source) + .map(|did| one_edge(kind, SubjectRef::Did(did), source)) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + use jacquard_common::types::nsid::Nsid; + use serde_json::json; + + fn at(s: &str) -> AtUri { + AtUri::new_owned(s).unwrap() + } + + fn did(s: &str) -> Did { + Did::new_owned(s).unwrap() + } + + fn did_subj(s: &str) -> SubjectRef { + SubjectRef::Did(did(s)) + } + + fn uri_subj(s: &str) -> SubjectRef { + SubjectRef::Uri(at(s)) + } + + fn nsid(s: &'static str) -> Nsid { + Nsid::new_static(s).unwrap() + } + + fn extract(collection: &'static str, source: &str, body: serde_json::Value) -> Vec { + let parsed = Record::from_json_value(&nsid(collection), body).expect("parse"); + parsed.primary_edges(&at(source)).expect("extract") + } + + #[test] + fn star_repo_variant_keys_on_did() { + let edges = extract( + "sh.tangled.feed.star", + "at://did:plc:olaren/sh.tangled.feed.star/abcabcabcabcz", + json!({ + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subject": { + "$type": "sh.tangled.feed.star#repo", + "did": "did:plc:abalone" + } + }), + ); + assert_eq!( + edges, + vec![Edge { + kind: nsid("sh.tangled.feed.star"), + subject: did_subj("did:plc:abalone"), + source: at("at://did:plc:olaren/sh.tangled.feed.star/abcabcabcabcz"), + sort_micros: 0, + }] + ); + } + + #[test] + fn star_string_variant_keeps_full_path() { + let edges = extract( + "sh.tangled.feed.star", + "at://did:plc:olaren/sh.tangled.feed.star/abcabcabcabcz", + json!({ + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subject": { + "$type": "sh.tangled.feed.star#string", + "uri": "at://did:plc:teq/sh.tangled.string/k1" + } + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!( + edges[0].subject, + uri_subj("at://did:plc:teq/sh.tangled.string/k1") + ); + } + + #[test] + fn star_string_variant_handle_authority_emits_no_edge() { + let edges = extract( + "sh.tangled.feed.star", + "at://did:plc:olaren/sh.tangled.feed.star/abcabcabcabcz", + json!({ + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subject": { + "$type": "sh.tangled.feed.star#string", + "uri": "at://oyster.cafe/sh.tangled.repo/r1" + } + }), + ); + assert!(edges.is_empty()); + } + + #[test] + fn follow_uses_subject_did() { + let edges = extract( + "sh.tangled.graph.follow", + "at://did:plc:olaren/sh.tangled.graph.follow/abcabcabcabcz", + json!({ + "$type": "sh.tangled.graph.follow", + "createdAt": "2026-05-01T00:00:00Z", + "subject": "did:plc:bailey" + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].kind, nsid("sh.tangled.graph.follow")); + assert_eq!(edges[0].subject, did_subj("did:plc:bailey")); + } + + #[test] + fn issue_keys_on_repo_did() { + let edges = extract( + "sh.tangled.repo.issue", + "at://did:plc:nel/sh.tangled.repo.issue/abcabcabcabcz", + json!({ + "$type": "sh.tangled.repo.issue", + "repo": "did:plc:abalone", + "title": "bug", + "createdAt": "2026-05-01T00:00:00Z" + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].subject, did_subj("did:plc:abalone")); + } + + #[test] + fn issue_comment_uses_issue_uri() { + let edges = extract( + "sh.tangled.repo.issue.comment", + "at://did:plc:nel/sh.tangled.repo.issue.comment/abcabcabcabcz", + json!({ + "$type": "sh.tangled.repo.issue.comment", + "issue": "at://did:plc:nel/sh.tangled.repo.issue/3lk1", + "body": "thoughts", + "createdAt": "2026-05-01T00:00:00Z" + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!( + edges[0].subject, + uri_subj("at://did:plc:nel/sh.tangled.repo.issue/3lk1") + ); + } + + #[test] + fn pull_keys_on_target_repo_did() { + let edges = extract( + "sh.tangled.repo.pull", + "at://did:plc:nel/sh.tangled.repo.pull/abcabcabcabcz", + json!({ + "$type": "sh.tangled.repo.pull", + "title": "feature", + "createdAt": "2026-05-01T00:00:00Z", + "rounds": [], + "target": {"repo": "did:plc:abalone", "branch": "main"} + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].subject, did_subj("did:plc:abalone")); + } + + fn artifact_body( + repo: Option<&AtUri>, + repo_did: Option<&Did>, + ) -> serde_json::Value { + let mut body = json!({ + "$type": "sh.tangled.repo.artifact", + "createdAt": "2026-05-01T00:00:00Z", + "name": "out.bin", + "tag": {"$bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAA="}, + "artifact": { + "$type": "blob", + "ref": {"$link": "bafkreigh2akiscaildc7gnvtklbsfhdgwz72eolmpckbqr5ej26byp3uli"}, + "mimeType": "application/octet-stream", + "size": 12 + } + }); + let obj = body.as_object_mut().expect("artifact_body returns object"); + if let Some(r) = repo { + obj.insert("repo".into(), json!(r.as_ref())); + } + if let Some(d) = repo_did { + obj.insert("repoDid".into(), json!(d.as_ref())); + } + body + } + + #[test] + fn artifact_prefers_repo_did_over_repo_uri() { + let edges = extract( + "sh.tangled.repo.artifact", + "at://did:plc:nel/sh.tangled.repo.artifact/abcabcabcabcz", + artifact_body( + Some(&at("at://did:plc:abalone/sh.tangled.repo/r1")), + Some(&did("did:plc:lyna")), + ), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].subject, did_subj("did:plc:lyna")); + } + + #[test] + fn artifact_did_only_keys_on_did() { + let edges = extract( + "sh.tangled.repo.artifact", + "at://did:plc:nel/sh.tangled.repo.artifact/abcabcabcabcz", + artifact_body(None, Some(&did("did:plc:abalone"))), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].subject, did_subj("did:plc:abalone")); + } + + #[test] + fn artifact_without_repo_emits_no_edge() { + let edges = extract( + "sh.tangled.repo.artifact", + "at://did:plc:nel/sh.tangled.repo.artifact/abcabcabcabcz", + artifact_body(None, None), + ); + assert!(edges.is_empty()); + } + + #[test] + fn artifact_uri_only_is_preserved_for_normalization() { + let edges = extract( + "sh.tangled.repo.artifact", + "at://did:plc:nel/sh.tangled.repo.artifact/abcabcabcabcz", + artifact_body(Some(&at("at://did:plc:abalone/sh.tangled.repo/r1")), None), + ); + assert_eq!(edges.len(), 1); + assert_eq!( + edges[0].subject, + uri_subj("at://did:plc:abalone/sh.tangled.repo/r1"), + ); + } + + #[test] + fn artifact_uri_with_handle_authority_emits_no_edge() { + let edges = extract( + "sh.tangled.repo.artifact", + "at://did:plc:nel/sh.tangled.repo.artifact/abcabcabcabcz", + artifact_body(Some(&at("at://oyster.cafe/sh.tangled.repo/r1")), None), + ); + assert!(edges.is_empty()); + } + + #[test] + fn collaborator_keys_on_repo_did() { + let edges = extract( + "sh.tangled.repo.collaborator", + "at://did:plc:nel/sh.tangled.repo.collaborator/abcabcabcabcz", + json!({ + "$type": "sh.tangled.repo.collaborator", + "repo": "did:plc:abalone", + "subject": "did:plc:lyna", + "createdAt": "2026-05-01T00:00:00Z" + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].kind, nsid("sh.tangled.repo.collaborator")); + assert_eq!(edges[0].subject, did_subj("did:plc:abalone")); + } + + #[test] + fn ref_update_keys_on_repo_did() { + let edges = extract( + "sh.tangled.git.refUpdate", + "at://did:plc:nel/sh.tangled.git.refUpdate/abcabcabcabcz", + json!({ + "$type": "sh.tangled.git.refUpdate", + "ref": "refs/heads/main", + "committerDid": "did:plc:lyna", + "repo": "did:plc:abalone", + "oldSha": "0000000000000000000000000000000000000000", + "newSha": "1111111111111111111111111111111111111111", + "meta": { + "isDefaultRef": true, + "commitCount": {} + } + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].kind, nsid("sh.tangled.git.refUpdate")); + assert_eq!(edges[0].subject, did_subj("did:plc:abalone")); + } + + #[test] + fn label_op_keys_on_subject_at_uri() { + let issue_uri = "at://did:plc:abalone/sh.tangled.repo.issue/i1"; + let edges = extract( + "sh.tangled.label.op", + "at://did:plc:nel/sh.tangled.label.op/abcabcabcabcz", + json!({ + "$type": "sh.tangled.label.op", + "performedAt": "2026-05-01T00:00:00Z", + "subject": issue_uri, + "add": [{ + "key": "at://did:plc:abalone/sh.tangled.label.definition/bug", + "value": "true" + }], + "delete": [] + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].kind, nsid("sh.tangled.label.op")); + assert_eq!(edges[0].subject, uri_subj(issue_uri)); + } + + #[test] + fn label_op_pull_subject_keys_on_pull_uri() { + let pull_uri = "at://did:plc:abalone/sh.tangled.repo.pull/p1"; + let edges = extract( + "sh.tangled.label.op", + "at://did:plc:nel/sh.tangled.label.op/abcabcabcabcz", + json!({ + "$type": "sh.tangled.label.op", + "performedAt": "2026-05-01T00:00:00Z", + "subject": pull_uri, + "add": [], + "delete": [] + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].subject, uri_subj(pull_uri)); + } + + #[test] + fn pipeline_keys_on_repo_did_when_present() { + let edges = extract( + "sh.tangled.pipeline", + "at://did:plc:lyna/sh.tangled.pipeline/pl1", + json!({ + "$type": "sh.tangled.pipeline", + "workflows": [], + "triggerMetadata": { + "kind": "manual", + "repo": { + "did": "did:plc:nel", + "repoDid": "did:plc:abalone", + "knot": "oyster.cafe", + "defaultBranch": "main" + } + } + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].kind, nsid("sh.tangled.pipeline")); + assert_eq!(edges[0].subject, did_subj("did:plc:abalone")); + } + + #[test] + fn pipeline_falls_back_to_owner_did_when_repo_did_absent() { + let edges = extract( + "sh.tangled.pipeline", + "at://did:plc:lyna/sh.tangled.pipeline/pl1", + json!({ + "$type": "sh.tangled.pipeline", + "workflows": [], + "triggerMetadata": { + "kind": "manual", + "repo": { + "did": "did:plc:nel", + "knot": "oyster.cafe", + "defaultBranch": "main" + } + } + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].kind, nsid("sh.tangled.pipeline")); + assert_eq!(edges[0].subject, did_subj("did:plc:nel")); + } + + #[test] + fn pipeline_status_keys_on_pipeline_at_uri() { + let pipeline_uri = "at://did:plc:lyna/sh.tangled.pipeline/pl1"; + let edges = extract( + "sh.tangled.pipeline.status", + "at://did:plc:bailey/sh.tangled.pipeline.status/abcabcabcabcz", + json!({ + "$type": "sh.tangled.pipeline.status", + "createdAt": "2026-05-01T00:00:00Z", + "pipeline": pipeline_uri, + "workflow": pipeline_uri, + "status": "success" + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].kind, nsid("sh.tangled.pipeline.status")); + assert_eq!(edges[0].subject, uri_subj(pipeline_uri)); + } + + #[test] + fn knot_member_keys_on_subject_did() { + let edges = extract( + "sh.tangled.knot.member", + "at://did:plc:teq/sh.tangled.knot.member/abcabcabcabcz", + json!({ + "$type": "sh.tangled.knot.member", + "createdAt": "2026-05-01T00:00:00Z", + "subject": "did:plc:nel", + "domain": "oyster.cafe" + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].kind, nsid("sh.tangled.knot.member")); + assert_eq!(edges[0].subject, did_subj("did:plc:nel")); + } + + #[test] + fn spindle_member_keys_on_subject_did() { + let edges = extract( + "sh.tangled.spindle.member", + "at://did:plc:teq/sh.tangled.spindle.member/abcabcabcabcz", + json!({ + "$type": "sh.tangled.spindle.member", + "createdAt": "2026-05-01T00:00:00Z", + "subject": "did:plc:olaren", + "instance": "spin.nel.pet" + }), + ); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].kind, nsid("sh.tangled.spindle.member")); + assert_eq!(edges[0].subject, did_subj("did:plc:olaren")); + } + + #[test] + fn repo_self_edge_keys_on_owner_did() { + let edges = extract( + "sh.tangled.repo", + "at://did:plc:teq/sh.tangled.repo/r1", + json!({ + "$type": "sh.tangled.repo", + "name": "abalone", + "knot": "oyster.cafe", + "createdAt": "2026-05-01T00:00:00Z" + }), + ); + assert_eq!( + edges, + vec![Edge { + kind: nsid("sh.tangled.repo"), + subject: did_subj("did:plc:teq"), + source: at("at://did:plc:teq/sh.tangled.repo/r1"), + sort_micros: 0, + }] + ); + } + + #[test] + fn knot_spindle_publickey_string_label_all_self_edge_to_owner() { + let cases = [ + ( + "sh.tangled.knot", + json!({"$type": "sh.tangled.knot", "createdAt": "2026-05-01T00:00:00Z"}), + ), + ( + "sh.tangled.spindle", + json!({"$type": "sh.tangled.spindle", "createdAt": "2026-05-01T00:00:00Z"}), + ), + ( + "sh.tangled.publicKey", + json!({"$type": "sh.tangled.publicKey", "createdAt": "2026-05-01T00:00:00Z", "key": "ssh-ed25519 AAA", "name": "laptop"}), + ), + ( + "sh.tangled.string", + json!({"$type": "sh.tangled.string", "createdAt": "2026-05-01T00:00:00Z", "filename": "f.txt", "description": "x", "contents": "hi"}), + ), + ( + "sh.tangled.label.definition", + json!({ + "$type": "sh.tangled.label.definition", + "createdAt": "2026-05-01T00:00:00Z", + "name": "bug", + "valueType": {"type": "boolean", "format": "any"}, + "scope": ["sh.tangled.repo.issue"] + }), + ), + ]; + cases.into_iter().for_each(|(collection, body)| { + let source = format!("at://did:plc:teq/{collection}/r1"); + let edges = extract(collection, &source, body); + assert_eq!(edges.len(), 1, "{collection} must produce one self-edge"); + assert_eq!(edges[0].kind, nsid(collection)); + assert_eq!(edges[0].subject, did_subj("did:plc:teq")); + }); + } + + #[test] + fn profile_and_vouch_emit_no_edges() { + let profile = extract( + "sh.tangled.actor.profile", + "at://did:plc:teq/sh.tangled.actor.profile/self", + json!({"$type": "sh.tangled.actor.profile", "bluesky": false}), + ); + assert!(profile.is_empty()); + let vouch = extract( + "sh.tangled.graph.vouch", + "at://did:plc:teq/sh.tangled.graph.vouch/r1", + json!({"$type": "sh.tangled.graph.vouch", "createdAt": "2026-05-01T00:00:00Z", "kind": "vouch"}), + ); + assert!(vouch.is_empty()); + } + + #[test] + fn unknown_collection_errors_with_unknown_collection() { + let err = Record::from_json_value( + &nsid("app.bsky.feed.post"), + json!({"$type": "app.bsky.feed.post", "text": "hi"}), + ) + .expect_err("foreign collection"); + assert!(matches!(err, ExtractError::UnknownCollection(_))); + } +} diff --git a/bobbin/crates/types/src/ids.rs b/bobbin/crates/types/src/ids.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/src/ids.rs @@ -0,0 +1,143 @@ +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::recordkey::Rkey; +use jacquard_common::types::string::AtUri; + +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub enum SubjectRef { + Did(Did), + Uri(AtUri), +} + +impl SubjectRef { + pub fn from_did(did: Did) -> Self { + Self::Did(did) + } + + pub fn from_uri(uri: AtUri) -> Self { + Self::Uri(uri) + } + + pub fn as_str(&self) -> &str { + match self { + Self::Did(d) => d.as_ref(), + Self::Uri(u) => u.as_ref(), + } + } + + pub fn as_did(&self) -> Option<&Did> { + match self { + Self::Did(d) => Some(d), + Self::Uri(_) => None, + } + } + + pub fn as_uri(&self) -> Option<&AtUri> { + match self { + Self::Uri(u) => Some(u), + Self::Did(_) => None, + } + } +} + +impl From> for SubjectRef { + fn from(did: Did) -> Self { + Self::Did(did) + } +} + +impl From> for SubjectRef { + fn from(uri: AtUri) -> Self { + Self::Uri(uri) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub struct EdgeKey { + pub kind: Nsid, + pub subject: SubjectRef, +} + +impl EdgeKey { + pub fn new(kind: Nsid, subject: SubjectRef) -> Self { + Self { kind, subject } + } +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct RepoIdent { + pub owner: Did, + pub rkey: Rkey, +} + +impl RepoIdent { + pub fn new(owner: Did, rkey: Rkey) -> Self { + Self { owner, rkey } + } +} + +pub fn nsid_static(s: &'static str) -> Nsid { + Nsid::new_static(s).expect("compile-time NSID literal must validate") +} + +pub fn owner_did_from_aturi(uri: &AtUri) -> Option> { + use jacquard_common::IntoStatic; + use jacquard_common::types::ident::AtIdentifier; + match uri.authority() { + AtIdentifier::Did(d) => Some(d.into_static()), + AtIdentifier::Handle(_) => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn did(s: &'static str) -> Did { + Did::new_static(s).unwrap() + } + + fn at(s: &str) -> AtUri { + AtUri::new_owned(s).unwrap() + } + + #[test] + fn owner_did_from_full_uri() { + assert_eq!( + owner_did_from_aturi(&at("at://did:plc:olaren/sh.tangled.feed.star/3lk1")), + Some(did("did:plc:olaren")), + ); + } + + #[test] + fn owner_did_from_did_only_uri() { + assert_eq!( + owner_did_from_aturi(&at("at://did:plc:olaren")), + Some(did("did:plc:olaren")) + ); + } + + #[test] + fn owner_did_rejects_handle_authority() { + assert_eq!( + owner_did_from_aturi(&at("at://oyster.cafe/sh.tangled.repo/r1")), + None + ); + } + + #[test] + fn subject_ref_did_and_uri_hash_distinct() { + use std::collections::HashSet; + let mut s = HashSet::new(); + s.insert(SubjectRef::Did(did("did:plc:squid"))); + s.insert(SubjectRef::Uri( + AtUri::new_owned("at://did:plc:squid").unwrap(), + )); + assert_eq!( + s.len(), + 2, + "did variant and uri variant must hash distinctly" + ); + } +} diff --git a/bobbin/crates/types/src/legacy.rs b/bobbin/crates/types/src/legacy.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/src/legacy.rs @@ -0,0 +1,255 @@ +use alloc::collections::BTreeMap; +use alloc::vec::Vec; + +use jacquard_common::deps::smol_str::SmolStr; +use jacquard_common::types::blob::BlobRef; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::string::{AtUri, Datetime, Did}; +use jacquard_common::types::value::Data; +use jacquard_common::{BosStr, DefaultStr}; +use serde::{Deserialize, Deserializer}; + +use crate::edges::ExtractError; +use crate::sh_tangled::repo::pull::Round as CanonRound; + +fn empty_string_as_none<'de, D, T>(d: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + let value: Option = Deserialize::deserialize(d)?; + match value { + None | Some(serde_json::Value::Null) => Ok(None), + Some(serde_json::Value::String(s)) if s.is_empty() => Ok(None), + Some(other) => T::deserialize(other) + .map(Some) + .map_err(serde::de::Error::custom), + } +} + +#[derive(Debug, Deserialize)] +#[serde( + rename_all = "camelCase", + rename = "sh.tangled.repo.issue", + tag = "$type", + bound(deserialize = "S: Deserialize<'de> + BosStr") +)] +pub struct LegacyIssue { + pub created_at: Datetime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub body: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mentions: Option>>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub references: Option>>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repo: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repo_did: Option>, + pub title: S, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Option>>, +} + +#[derive(Debug, Deserialize)] +#[serde( + rename_all = "camelCase", + bound(deserialize = "S: Deserialize<'de> + BosStr") +)] +pub struct LegacyTarget { + pub branch: S, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repo: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "empty_string_as_none" + )] + pub repo_did: Option>, +} + +#[derive(Debug, Deserialize)] +#[serde( + rename_all = "camelCase", + bound(deserialize = "S: Deserialize<'de> + BosStr") +)] +pub struct LegacySource { + pub branch: S, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repo: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "empty_string_as_none" + )] + pub repo_did: Option>, +} + +#[derive(Debug, Deserialize)] +#[serde( + rename_all = "camelCase", + rename = "sh.tangled.repo.pull", + tag = "$type", + bound(deserialize = "S: Deserialize<'de> + BosStr") +)] +pub struct LegacyPull { + pub created_at: Datetime, + pub title: S, + #[serde(default)] + pub rounds: Vec>, + pub target: LegacyTarget, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub body: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mentions: Option>>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub references: Option>>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dependent_on: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub patch_blob: Option>, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Option>>, +} + +#[derive(Debug, Deserialize)] +#[serde( + rename_all = "camelCase", + rename = "sh.tangled.repo.collaborator", + tag = "$type", + bound(deserialize = "S: Deserialize<'de> + BosStr") +)] +pub struct LegacyCollaborator { + pub created_at: Datetime, + pub subject: Did, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repo: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repo_did: Option>, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Option>>, +} + +#[derive(Debug, Deserialize)] +#[serde( + rename_all = "camelCase", + rename = "sh.tangled.git.refUpdate", + tag = "$type", + bound(deserialize = "S: Deserialize<'de> + BosStr") +)] +pub struct LegacyRefUpdate { + pub committer_did: Did, + pub meta: crate::sh_tangled::git::ref_update::Meta, + pub new_sha: S, + pub old_sha: S, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub owner_did: Option>, + pub r#ref: S, + pub repo_did: Did, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repo_name: Option, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Option>>, +} + +#[derive(Debug, Deserialize)] +#[serde( + rename_all = "camelCase", + rename = "sh.tangled.feed.star", + tag = "$type", + bound(deserialize = "S: Deserialize<'de> + BosStr") +)] +pub struct LegacyStar { + pub created_at: Datetime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subject: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subject_did: Option>, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Option>>, +} + +#[derive(Debug, Deserialize)] +#[serde( + rename_all = "camelCase", + rename = "sh.tangled.publicKey", + tag = "$type", + bound(deserialize = "S: Deserialize<'de> + BosStr") +)] +pub struct LegacyPublicKey { + pub created: Datetime, + pub key: S, + pub name: S, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Option>>, +} + +#[derive(Debug, Deserialize)] +#[serde( + rename_all = "camelCase", + rename = "sh.tangled.repo", + tag = "$type", + bound(deserialize = "S: Deserialize<'de> + BosStr") +)] +pub struct LegacyRepo { + pub added_at: Datetime, + pub knot: S, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub owner: Option>, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Option>>, +} + +#[derive(Debug, Deserialize)] +#[serde( + rename_all = "camelCase", + rename = "sh.tangled.knot.member", + tag = "$type", + bound(deserialize = "S: Deserialize<'de> + BosStr") +)] +pub struct LegacyKnotMember { + pub added_at: Datetime, + pub domain: S, + pub member: Did, + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] + pub extra_data: Option>>, +} + +#[derive(Debug)] +pub enum LegacyRecord { + Issue(LegacyIssue), + Pull(LegacyPull), + Collaborator(LegacyCollaborator), + RefUpdate(LegacyRefUpdate), + Star(LegacyStar), + PublicKey(LegacyPublicKey), + Repo(LegacyRepo), + KnotMember(LegacyKnotMember), +} + +impl LegacyRecord { + pub fn from_json_bytes>( + nsid: &Nsid, + bytes: &[u8], + ) -> Result { + match nsid.as_ref() { + "sh.tangled.repo.issue" => Ok(Self::Issue(serde_json::from_slice(bytes)?)), + "sh.tangled.repo.pull" => Ok(Self::Pull(serde_json::from_slice(bytes)?)), + "sh.tangled.repo.collaborator" => { + Ok(Self::Collaborator(serde_json::from_slice(bytes)?)) + } + "sh.tangled.git.refUpdate" => Ok(Self::RefUpdate(serde_json::from_slice(bytes)?)), + "sh.tangled.feed.star" => Ok(Self::Star(serde_json::from_slice(bytes)?)), + "sh.tangled.publicKey" => Ok(Self::PublicKey(serde_json::from_slice(bytes)?)), + "sh.tangled.repo" => Ok(Self::Repo(serde_json::from_slice(bytes)?)), + "sh.tangled.knot.member" => Ok(Self::KnotMember(serde_json::from_slice(bytes)?)), + other => Err(ExtractError::UnknownCollection(other.into())), + } + } +} diff --git a/bobbin/crates/types/src/lib.rs b/bobbin/crates/types/src/lib.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/src/lib.rs @@ -0,0 +1,23 @@ +extern crate alloc; + +#[path = "_lex/lib.rs"] +#[allow(non_snake_case, unused_imports)] +#[allow( + clippy::absurd_extreme_comparisons, + clippy::collapsible_if, + clippy::manual_strip, + clippy::needless_update, + clippy::new_ret_no_self, + clippy::new_without_default, + clippy::should_implement_trait, + clippy::type_complexity +)] +mod _lex; + +pub use _lex::*; + +pub mod edges; +pub mod ids; +pub mod legacy; +pub mod record; +pub mod search; diff --git a/bobbin/crates/types/src/record.rs b/bobbin/crates/types/src/record.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/src/record.rs @@ -0,0 +1,21 @@ +use bytes::Bytes; +use jacquard_common::DefaultStr; +use jacquard_common::types::string::{AtUri, Cid}; + +#[derive(Clone, Debug)] +pub struct RecordBody { + pub uri: AtUri, + pub cid: Cid, + pub value: Bytes, +} + +impl RecordBody { + const FIXED_OVERHEAD: u64 = 128; + + pub fn weight(&self) -> u64 { + Self::FIXED_OVERHEAD + + self.value.len() as u64 + + self.uri.as_ref().len() as u64 + + self.cid.as_ref().len() as u64 + } +} diff --git a/bobbin/crates/types/src/search.rs b/bobbin/crates/types/src/search.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/types/src/search.rs @@ -0,0 +1,443 @@ +use alloc::string::String; +use alloc::vec::Vec; +use core::future::Future; + +use jacquard_common::types::ident::AtIdentifier; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::string::{AtUri, Did}; +use jacquard_common::{BosStr, DefaultStr, IntoStatic}; +use serde::Serialize; + +use crate::edges::{ExtractError, Record}; +use crate::ids::nsid_static; +use crate::sh_tangled::actor::profile::Profile; +use crate::sh_tangled::label::definition::Definition as LabelDefinitionRecord; +use crate::sh_tangled::repo::Repo as RepoRecord; +use crate::sh_tangled::repo::issue::Issue; +use crate::sh_tangled::repo::issue::comment::Comment as IssueCommentRecord; +use crate::sh_tangled::repo::pull::Pull; +use crate::sh_tangled::repo::pull::comment::Comment as PullCommentRecord; +use crate::sh_tangled::string::TangledString; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SearchDoc { + pub uri: AtUri, + pub nsid: Nsid, + pub title: String, + pub body: String, + pub author: Option>, + pub created_at: Option, + pub repo: Option>, +} + +pub trait SearchSink: Send + Sync { + fn upsert(&self, doc: SearchDoc) -> impl Future + Send; + fn remove(&self, uri: &AtUri) -> impl Future + Send; +} + +pub struct NoopSearchSink; + +impl SearchSink for NoopSearchSink { + async fn upsert(&self, _doc: SearchDoc) {} + async fn remove(&self, _uri: &AtUri) {} +} + +#[derive(Debug, Serialize)] +#[serde(untagged)] +pub enum SearchableRecord { + Profile(Profile), + Repo(RepoRecord), + Issue(Issue), + IssueComment(IssueCommentRecord), + Pull(Pull), + PullComment(PullCommentRecord), + TangledString(TangledString), + LabelDefinition(LabelDefinitionRecord), +} + +impl SearchableRecord { + pub fn try_from_record(record: Record) -> Option { + match record { + Record::Profile(r) => Some(Self::Profile(r)), + Record::Repo(r) => Some(Self::Repo(r)), + Record::Issue(r) => Some(Self::Issue(r)), + Record::IssueComment(r) => Some(Self::IssueComment(r)), + Record::Pull(r) => Some(Self::Pull(r)), + Record::PullComment(r) => Some(Self::PullComment(r)), + Record::TangledString(r) => Some(Self::TangledString(r)), + Record::LabelDefinition(r) => Some(Self::LabelDefinition(r)), + Record::Reaction(_) + | Record::Star(_) + | Record::RefUpdate(_) + | Record::Follow(_) + | Record::Vouch(_) + | Record::Knot(_) + | Record::KnotMember(_) + | Record::LabelOp(_) + | Record::Pipeline(_) + | Record::PipelineStatus(_) + | Record::PublicKey(_) + | Record::Artifact(_) + | Record::Collaborator(_) + | Record::IssueState(_) + | Record::PullStatus(_) + | Record::Spindle(_) + | Record::SpindleMember(_) => None, + } + } + + pub fn from_json_bytes>( + nsid: &Nsid, + bytes: &[u8], + ) -> Result { + let parsed = Record::from_json_bytes(nsid, bytes)?; + Self::try_from_record(parsed) + .ok_or_else(|| ExtractError::UnknownCollection(nsid.as_ref().into())) + } + + pub fn nsid(&self) -> Nsid { + let s: &'static str = match self { + Self::Profile(_) => "sh.tangled.actor.profile", + Self::Repo(_) => "sh.tangled.repo", + Self::Issue(_) => "sh.tangled.repo.issue", + Self::IssueComment(_) => "sh.tangled.repo.issue.comment", + Self::Pull(_) => "sh.tangled.repo.pull", + Self::PullComment(_) => "sh.tangled.repo.pull.comment", + Self::TangledString(_) => "sh.tangled.string", + Self::LabelDefinition(_) => "sh.tangled.label.definition", + }; + nsid_static(s) + } + + pub fn to_search_doc(&self, source: &AtUri) -> SearchDoc { + match self { + Self::Profile(r) => profile_doc(source, r), + Self::Repo(r) => repo_doc(source, r), + Self::Issue(r) => issue_doc(source, r), + Self::IssueComment(r) => issue_comment_doc(source, r), + Self::Pull(r) => pull_doc(source, r), + Self::PullComment(r) => pull_comment_doc(source, r), + Self::TangledString(r) => string_doc(source, r), + Self::LabelDefinition(r) => label_definition_doc(source, r), + } + } +} + +fn author_of(source: &AtUri) -> Option> { + match source.authority() { + AtIdentifier::Did(d) => Some(d.into_static()), + AtIdentifier::Handle(_) => None, + } +} + +fn doc( + source: &AtUri, + nsid: &'static str, + title: &str, + body_parts: Vec, + created_at: Option, + repo: Option>, +) -> SearchDoc { + SearchDoc { + uri: source.clone(), + nsid: nsid_static(nsid), + title: title.to_owned(), + body: body_parts.join(" "), + author: author_of(source), + created_at, + repo, + } +} + +fn profile_doc(source: &AtUri, r: &Profile) -> SearchDoc { + let title = r + .preferred_handle + .as_ref() + .map(|h| h.as_str().to_owned()) + .unwrap_or_default(); + let mut parts: Vec = Vec::new(); + if let Some(d) = &r.description { + parts.push(d.as_str().to_owned()); + } + if let Some(loc) = &r.location { + parts.push(loc.as_str().to_owned()); + } + if let Some(p) = &r.pronouns { + parts.push(p.as_str().to_owned()); + } + doc( + source, + "sh.tangled.actor.profile", + &title, + parts, + None, + None, + ) +} + +fn repo_doc(source: &AtUri, r: &RepoRecord) -> SearchDoc { + let mut parts: Vec = Vec::new(); + if let Some(d) = &r.description { + parts.push(d.as_str().to_owned()); + } + if let Some(topics) = &r.topics { + parts.extend(topics.iter().map(|t| t.as_str().to_owned())); + } + let rkey = source.rkey(); + let title = r + .name + .as_deref() + .or_else(|| rkey.as_ref().map(|k| k.as_str())) + .unwrap_or(""); + doc( + source, + "sh.tangled.repo", + title, + parts, + Some(r.created_at.timestamp()), + r.repo_did.clone(), + ) +} + +fn issue_doc(source: &AtUri, r: &Issue) -> SearchDoc { + let body = r + .body + .as_ref() + .map(|b| Vec::from([b.as_str().to_owned()])) + .unwrap_or_default(); + doc( + source, + "sh.tangled.repo.issue", + r.title.as_str(), + body, + Some(r.created_at.timestamp()), + Some(r.repo.clone()), + ) +} + +fn issue_comment_doc(source: &AtUri, r: &IssueCommentRecord) -> SearchDoc { + doc( + source, + "sh.tangled.repo.issue.comment", + "", + Vec::from([r.body.as_str().to_owned()]), + Some(r.created_at.timestamp()), + None, + ) +} + +fn pull_doc(source: &AtUri, r: &Pull) -> SearchDoc { + let body = r + .body + .as_ref() + .map(|b| Vec::from([b.as_str().to_owned()])) + .unwrap_or_default(); + doc( + source, + "sh.tangled.repo.pull", + r.title.as_str(), + body, + Some(r.created_at.timestamp()), + Some(r.target.repo.clone()), + ) +} + +fn pull_comment_doc(source: &AtUri, r: &PullCommentRecord) -> SearchDoc { + doc( + source, + "sh.tangled.repo.pull.comment", + "", + Vec::from([r.body.as_str().to_owned()]), + Some(r.created_at.timestamp()), + None, + ) +} + +fn string_doc(source: &AtUri, r: &TangledString) -> SearchDoc { + doc( + source, + "sh.tangled.string", + r.filename.as_str(), + Vec::from([ + r.description.as_str().to_owned(), + r.contents.as_str().to_owned(), + ]), + Some(r.created_at.timestamp()), + None, + ) +} + +fn label_definition_doc( + source: &AtUri, + r: &LabelDefinitionRecord, +) -> SearchDoc { + doc( + source, + "sh.tangled.label.definition", + r.name.as_str(), + Vec::new(), + Some(r.created_at.timestamp()), + None, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use jacquard_common::types::nsid::Nsid; + use serde_json::json; + + fn at(s: &str) -> AtUri { + AtUri::new_owned(s).unwrap() + } + + fn nsid(s: &'static str) -> Nsid { + Nsid::new_static(s).unwrap() + } + + fn extract( + collection: &Nsid, + source: &AtUri, + body: serde_json::Value, + ) -> SearchDoc { + let parsed = Record::from_json_value(collection, body).expect("parse"); + let searchable = SearchableRecord::try_from_record(parsed).expect("indexable record"); + searchable.to_search_doc(source) + } + + #[test] + fn issue_titles_and_body_indexable() { + let doc = extract( + &nsid("sh.tangled.repo.issue"), + &at("at://did:plc:nel/sh.tangled.repo.issue/abcabcabcabcz"), + json!({ + "$type": "sh.tangled.repo.issue", + "repo": "did:plc:squid", + "title": "barnacle pagination overflow", + "body": "scrolling resets when the cursor wraps", + "createdAt": "2026-05-01T00:00:00Z" + }), + ); + assert_eq!(doc.title, "barnacle pagination overflow"); + assert_eq!(doc.body, "scrolling resets when the cursor wraps"); + assert_eq!(doc.nsid, nsid("sh.tangled.repo.issue")); + } + + #[test] + fn issue_without_body_yields_empty_body() { + let doc = extract( + &nsid("sh.tangled.repo.issue"), + &at("at://did:plc:nel/sh.tangled.repo.issue/abcabcabcabcz"), + json!({ + "$type": "sh.tangled.repo.issue", + "repo": "did:plc:squid", + "title": "kelp ate my newline", + "createdAt": "2026-05-01T00:00:00Z" + }), + ); + assert_eq!(doc.title, "kelp ate my newline"); + assert!(doc.body.is_empty()); + } + + #[test] + fn repo_indexes_name_description_topics() { + let doc = extract( + &nsid("sh.tangled.repo"), + &at("at://did:plc:teq/sh.tangled.repo/r1"), + json!({ + "$type": "sh.tangled.repo", + "name": "scallop", + "knot": "oyster.cafe", + "description": "shell index for tide pools", + "topics": ["intertidal", "molluscs"], + "createdAt": "2026-05-01T00:00:00Z" + }), + ); + assert_eq!(doc.title, "scallop"); + assert!(doc.body.contains("shell index")); + assert!(doc.body.contains("intertidal")); + assert!(doc.body.contains("molluscs")); + } + + #[test] + fn repo_without_name_field_falls_back_to_rkey() { + let doc = extract( + &nsid("sh.tangled.repo"), + &at("at://did:plc:teq/sh.tangled.repo/limpet"), + json!({ + "$type": "sh.tangled.repo", + "knot": "oyster.cafe", + "createdAt": "2026-05-01T00:00:00Z" + }), + ); + assert_eq!(doc.title, "limpet"); + } + + #[test] + fn non_text_records_have_no_searchable_view() { + let parsed = Record::from_json_value( + &nsid("sh.tangled.feed.star"), + json!({ + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subject": { + "$type": "sh.tangled.feed.star#repo", + "did": "did:plc:squid" + } + }), + ) + .expect("parse"); + assert!(SearchableRecord::try_from_record(parsed).is_none()); + } + + #[test] + fn string_indexes_filename_description_contents() { + let doc = extract( + &nsid("sh.tangled.string"), + &at("at://did:plc:teq/sh.tangled.string/k1"), + json!({ + "$type": "sh.tangled.string", + "filename": "anemone.md", + "description": "field notes on tide pools", + "contents": "the limpet returned at dawn", + "createdAt": "2026-05-01T00:00:00Z" + }), + ); + assert_eq!(doc.title, "anemone.md"); + assert!(doc.body.contains("field notes")); + assert!(doc.body.contains("limpet returned at dawn")); + } + + #[test] + fn from_json_bytes_round_trips_via_serialize() { + let body = json!({ + "$type": "sh.tangled.repo.issue", + "repo": "did:plc:squid", + "title": "uni shell", + "createdAt": "2026-05-01T00:00:00Z" + }); + let bytes = serde_json::to_vec(&body).unwrap(); + let view = + SearchableRecord::from_json_bytes(&nsid("sh.tangled.repo.issue"), &bytes).unwrap(); + assert_eq!(view.nsid(), nsid("sh.tangled.repo.issue")); + let serialized = serde_json::to_value(&view).unwrap(); + assert_eq!(serialized["title"], json!("uni shell")); + assert_eq!(serialized["$type"], json!("sh.tangled.repo.issue")); + } + + #[test] + fn from_json_bytes_rejects_unknown_nsid() { + let body = json!({ + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subject": { + "$type": "sh.tangled.feed.star#repo", + "did": "did:plc:squid" + } + }); + let bytes = serde_json::to_vec(&body).unwrap(); + let err = SearchableRecord::from_json_bytes(&nsid("sh.tangled.feed.star"), &bytes) + .expect_err("non-searchable nsid must be rejected at search hydration"); + assert!(matches!(err, ExtractError::UnknownCollection(_))); + } +} diff --git a/bobbin/crates/xrpc/Cargo.toml b/bobbin/crates/xrpc/Cargo.toml new file mode 100644 --- /dev/null +++ b/bobbin/crates/xrpc/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "bobbin-xrpc" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-types = { workspace = true } +bobbin-edge-index = { workspace = true } +bobbin-record-lru = { workspace = true } +bobbin-resolver = { workspace = true } +bobbin-runtime = { workspace = true } +bobbin-search = { workspace = true } +bobbin-slingshot-client = { workspace = true } +bobbin-knot-proxy = { workspace = true } +jacquard-common = { workspace = true } + +axum = { workspace = true } +chrono = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tower-http = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +bobbin-runtime = { workspace = true } +http = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tower = { workspace = true } +url = { workspace = true } +wiremock = { workspace = true } diff --git a/bobbin/crates/xrpc/src/backpressure.rs b/bobbin/crates/xrpc/src/backpressure.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/xrpc/src/backpressure.rs @@ -0,0 +1,171 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use bobbin_runtime::MemoryBudget; + +use crate::XrpcError; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct PerRequestAnonBytes(u64); + +impl PerRequestAnonBytes { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn bytes(self) -> u64 { + self.0 + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ReservedFloor(u64); + +impl ReservedFloor { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn bytes(self) -> u64 { + self.0 + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct MaxInFlight(usize); + +impl MaxInFlight { + pub fn from_budget( + budget: MemoryBudget, + reserved: ReservedFloor, + per_request: PerRequestAnonBytes, + ) -> Self { + let usable = budget.bytes().saturating_sub(reserved.bytes()); + let per = per_request.bytes().max(1); + let slots = (usable / per).max(1); + Self(usize::try_from(slots).unwrap_or(usize::MAX)) + } + + pub const fn get(self) -> usize { + self.0 + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PressureVerdict { + Tighten, + Hold, + Relieve, +} + +struct LimiterCore { + in_flight: AtomicUsize, + limit: AtomicUsize, + floor: usize, + ceiling: usize, + relieve_step: usize, +} + +pub struct HeavyLimiter { + core: Arc, +} + +pub struct HeavyPermit { + core: Arc, +} + +impl Drop for HeavyPermit { + fn drop(&mut self) { + self.core.in_flight.fetch_sub(1, Ordering::AcqRel); + } +} + +impl HeavyLimiter { + pub fn new(max: MaxInFlight) -> Self { + let ceiling = max.get(); + Self { + core: Arc::new(LimiterCore { + in_flight: AtomicUsize::new(0), + limit: AtomicUsize::new(ceiling), + floor: 1, + ceiling, + relieve_step: (ceiling / 16).max(1), + }), + } + } + + pub fn try_enter(&self) -> Result { + let prev = self.core.in_flight.fetch_add(1, Ordering::AcqRel); + if prev < self.core.limit.load(Ordering::Acquire) { + Ok(HeavyPermit { + core: Arc::clone(&self.core), + }) + } else { + self.core.in_flight.fetch_sub(1, Ordering::AcqRel); + Err(XrpcError::overloaded()) + } + } + + pub fn adjust(&self, verdict: PressureVerdict) { + let cur = self.core.limit.load(Ordering::Acquire); + let next = match verdict { + PressureVerdict::Tighten => (cur / 2).max(self.core.floor), + PressureVerdict::Relieve => (cur + self.core.relieve_step).min(self.core.ceiling), + PressureVerdict::Hold => cur, + }; + self.core.limit.store(next, Ordering::Release); + } + + pub fn limit(&self) -> usize { + self.core.limit.load(Ordering::Acquire) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reserved_floor_above_budget_yields_one_slot() { + let max = MaxInFlight::from_budget( + MemoryBudget::new(100 * 1024 * 1024), + ReservedFloor::new(114 * 1024 * 1024), + PerRequestAnonBytes::new(2 * 1024 * 1024), + ); + assert_eq!(max.get(), 1); + } + + #[test] + fn larger_budget_opens_more_slots() { + let max = MaxInFlight::from_budget( + MemoryBudget::new(500 * 1024 * 1024), + ReservedFloor::new(114 * 1024 * 1024), + PerRequestAnonBytes::new(2 * 1024 * 1024), + ); + assert_eq!(max.get(), (386 * 1024 * 1024) / (2 * 1024 * 1024)); + } + + #[test] + fn limiter_sheds_when_at_limit() { + let limiter = HeavyLimiter::new(MaxInFlight(1)); + let held = limiter.try_enter().expect("first permit enters"); + assert!(matches!(limiter.try_enter(), Err(XrpcError::Overloaded))); + drop(held); + assert!(limiter.try_enter().is_ok()); + } + + #[test] + fn adjust_clamps_between_floor_and_ceiling() { + let limiter = HeavyLimiter::new(MaxInFlight(8)); + limiter.adjust(PressureVerdict::Tighten); + assert_eq!(limiter.limit(), 4); + limiter.adjust(PressureVerdict::Tighten); + assert_eq!(limiter.limit(), 2); + limiter.adjust(PressureVerdict::Tighten); + assert_eq!(limiter.limit(), 1); + limiter.adjust(PressureVerdict::Tighten); + assert_eq!(limiter.limit(), 1); + (0..20).for_each(|_| limiter.adjust(PressureVerdict::Relieve)); + assert_eq!(limiter.limit(), 8); + } +} diff --git a/bobbin/crates/xrpc/src/filter.rs b/bobbin/crates/xrpc/src/filter.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/xrpc/src/filter.rs @@ -0,0 +1,139 @@ +use std::sync::Arc; + +use bobbin_edge_index::{IssueStateKind, PullStatusKind, StateIndex}; +use bobbin_types::ids::SubjectRef; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::string::AtUri; +use serde::Deserialize; + +use crate::AppState; +use crate::{accept_state_source, at_uri_owned_by, source_authority_did}; + +pub type FilterPredicate = Box) -> bool + Send + Sync + 'static>; + +pub trait ListFilter: serde::de::DeserializeOwned + Send + Sync + 'static { + fn predicate(&self, state: &AppState, subject: &SubjectRef) -> FilterPredicate; + fn is_identity(&self) -> bool; +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +pub struct NoFilter; + +impl ListFilter for NoFilter { + fn predicate(&self, _: &AppState, _: &SubjectRef) -> FilterPredicate { + Box::new(|_| true) + } + + fn is_identity(&self) -> bool { + true + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum IssueState { + Open, + Closed, +} + +impl From for IssueStateKind { + fn from(s: IssueState) -> Self { + match s { + IssueState::Open => IssueStateKind::Open, + IssueState::Closed => IssueStateKind::Closed, + } + } +} + +#[derive(Debug, Default, Deserialize)] +pub struct IssueFilter { + pub author: Option>, + pub state: Option, +} + +impl ListFilter for IssueFilter { + fn predicate(&self, state: &AppState, subject: &SubjectRef) -> FilterPredicate { + compose_state_filter::( + self.author.clone(), + self.state.map(Into::into), + state.issue_states.clone(), + subject.as_did().cloned(), + ) + } + + fn is_identity(&self) -> bool { + self.author.is_none() && self.state.is_none() + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum PullStatus { + Open, + Closed, + Merged, +} + +impl From for PullStatusKind { + fn from(s: PullStatus) -> Self { + match s { + PullStatus::Open => PullStatusKind::Open, + PullStatus::Closed => PullStatusKind::Closed, + PullStatus::Merged => PullStatusKind::Merged, + } + } +} + +#[derive(Debug, Default, Deserialize)] +pub struct PullFilter { + pub author: Option>, + pub status: Option, +} + +impl ListFilter for PullFilter { + fn predicate(&self, state: &AppState, subject: &SubjectRef) -> FilterPredicate { + compose_state_filter::( + self.author.clone(), + self.status.map(Into::into), + state.pull_statuses.clone(), + subject.as_did().cloned(), + ) + } + + fn is_identity(&self) -> bool { + self.author.is_none() && self.status.is_none() + } +} + +fn compose_state_filter( + author: Option>, + want: Option, + index: Arc>, + repo_owner: Option>, +) -> FilterPredicate +where + K: bobbin_edge_index::StateKind + Default + 'static, +{ + Box::new(move |uri| { + if let Some(a) = &author + && !at_uri_owned_by(uri, a) + { + return false; + } + let Some(want) = want else { + return true; + }; + let Some(repo_owner) = repo_owner.as_ref() else { + return false; + }; + let entity_author = source_authority_did(uri); + let accept = + |src: &AtUri| accept_state_source(src, entity_author.as_ref(), repo_owner); + let effective = index + .latest_by(uri, accept) + .map(|(k, _)| k) + .unwrap_or_default(); + effective == want + }) +} diff --git a/bobbin/crates/xrpc/src/lib.rs b/bobbin/crates/xrpc/src/lib.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/xrpc/src/lib.rs @@ -0,0 +1,2699 @@ +use std::future::Future; +use std::sync::Arc; + +use bobbin_resolver::{ + NormalizeRepoRefs, decode_canon_or_upgrade, normalize_record_fields, scrub_record_bytes, + upgrade_wire_bytes, +}; + +use axum::{ + Router, + body::Body, + extract::{FromRequestParts, Query, RawQuery, State, rejection::QueryRejection}, + http::{ + HeaderMap, HeaderName, StatusCode, + header::{ + ACCEPT_RANGES, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LENGTH, + CONTENT_RANGE, CONTENT_TYPE, ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_RANGE, + LAST_MODIFIED, RANGE, + }, + request::Parts, + }, + response::{IntoResponse, Json, Response}, + routing::get, +}; +use bobbin_edge_index::{ + Coverage, CoverageWatch, CursorParseError, EdgePage, EdgeStore, IssueStateKind, PageCursor, + PageLimit, PageToken, PullStatusKind, SortDir, StateIndex, StateKind, +}; +use bobbin_knot_proxy::{KnotHost, KnotProxy, KnotProxyError, ProxyResponse, RepoSlug}; +use bobbin_record_lru::RecordStore; +use bobbin_resolver::RepoIdResolver; +use bobbin_search::{ + SearchCursor, SearchError, SearchFilters, SearchHit, SearchOffset, SearchReader, +}; +use bobbin_slingshot_client::{SlingshotClient, SlingshotError}; +use bobbin_types::ids::{EdgeKey, SubjectRef, nsid_static}; +use bobbin_types::record::RecordBody; +use bobbin_types::search::SearchableRecord; +use bobbin_types::sh_tangled::actor::profile::{Profile, ProfileGetRecordOutput, ProfileRecord}; +use bobbin_types::sh_tangled::feed::reaction::{Reaction, ReactionRecord}; +use bobbin_types::sh_tangled::feed::star::{Star, StarRecord}; +use bobbin_types::sh_tangled::git::ref_update::{RefUpdate, RefUpdateRecord}; +use bobbin_types::sh_tangled::graph::follow::{Follow, FollowRecord}; +use bobbin_types::sh_tangled::graph::vouch::{Vouch, VouchRecord}; +use bobbin_types::sh_tangled::knot::member::{ + Member as KnotMember, MemberRecord as KnotMemberRecord, +}; +use bobbin_types::sh_tangled::knot::{Knot, KnotRecord}; +use bobbin_types::sh_tangled::label::definition::{ + Definition as LabelDefinition, DefinitionRecord as LabelDefinitionRecord, +}; +use bobbin_types::sh_tangled::label::op::{Op as LabelOp, OpRecord as LabelOpRecord}; +use bobbin_types::sh_tangled::pipeline::status::{ + Status as PipelineStatus, StatusRecord as PipelineStatusRecord, +}; +use bobbin_types::sh_tangled::pipeline::{Pipeline, PipelineRecord}; +use bobbin_types::sh_tangled::public_key::{PublicKey, PublicKeyRecord}; +use bobbin_types::sh_tangled::repo::artifact::{Artifact, ArtifactRecord}; +use bobbin_types::sh_tangled::repo::collaborator::{Collaborator, CollaboratorRecord}; +use bobbin_types::sh_tangled::repo::issue::comment::{ + Comment as IssueComment, CommentRecord as IssueCommentRecord, +}; +use bobbin_types::sh_tangled::repo::issue::state::{ + State as IssueState, StateRecord as IssueStateRecord, +}; +use bobbin_types::sh_tangled::repo::issue::{Issue, IssueGetRecordOutput, IssueRecord}; +use bobbin_types::sh_tangled::repo::pull::comment::{ + Comment as PullComment, CommentRecord as PullCommentRecord, +}; +use bobbin_types::sh_tangled::repo::pull::status::{ + Status as PullStatus, StatusRecord as PullStatusRecord, +}; +use bobbin_types::sh_tangled::repo::pull::{Pull, PullGetRecordOutput, PullRecord}; +use bobbin_types::sh_tangled::repo::{Repo, RepoGetRecordOutput, RepoRecord}; +use bobbin_types::sh_tangled::spindle::member::{ + Member as SpindleMember, MemberRecord as SpindleMemberRecord, +}; +use bobbin_types::sh_tangled::spindle::{Spindle, SpindleRecord}; +use bobbin_types::sh_tangled::string::{TangledString, TangledStringRecord}; +use futures::Stream; +use futures::stream::{self, StreamExt, TryStreamExt}; +use jacquard_common::types::did::Did; +use jacquard_common::types::ident::AtIdentifier; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::recordkey::Rkey; +use jacquard_common::types::string::{AtUri, Cid}; +use jacquard_common::xrpc::XrpcResp; +use jacquard_common::{DefaultStr, IntoStatic}; +use serde::{Deserialize, Serialize}; +use std::convert::Infallible; +use std::time::Duration; +use thiserror::Error; +use url::form_urlencoded; + +use tower_http::classify::ServerErrorsFailureClass; +use tower_http::trace::{DefaultMakeSpan, OnFailure, OnResponse, TraceLayer}; +use tracing::{Level, Span}; + +mod backpressure; +mod filter; + +pub use backpressure::{ + HeavyLimiter, HeavyPermit, MaxInFlight, PerRequestAnonBytes, PressureVerdict, ReservedFloor, +}; +use filter::{IssueFilter, ListFilter, NoFilter, PullFilter}; + +const DEFAULT_LIMIT: u32 = 50; +const FETCH_CONCURRENCY: usize = 8; + +#[derive(Clone)] +pub struct AppState { + pub records: Arc, + pub slingshot: SlingshotClient, + pub edges: Arc, + pub issue_states: Arc>, + pub pull_statuses: Arc>, + pub coverage: Arc, + pub knots: Arc, + pub search: Arc, + pub resolver: Arc, + pub limiter: Option>, +} + +impl AppState { + #[allow(clippy::too_many_arguments)] + pub fn new( + records: Arc, + slingshot: SlingshotClient, + edges: Arc, + issue_states: Arc>, + pull_statuses: Arc>, + coverage: Arc, + knots: Arc, + search: Arc, + resolver: Arc, + ) -> Self { + Self { + records, + slingshot, + edges, + issue_states, + pull_statuses, + coverage, + knots, + search, + resolver, + limiter: None, + } + } + + pub fn with_limiter(mut self, limiter: Option>) -> Self { + self.limiter = limiter; + self + } + + fn heavy_permit(&self) -> Result, XrpcError> { + self.limiter.as_ref().map(|l| l.try_enter()).transpose() + } +} + +pub fn router(state: AppState) -> Router { + Router::new() + .route("/xrpc/sh.tangled.repo.getRepo", get(get_repo)) + .route("/xrpc/sh.tangled.repo.getRepos", get(get_repos)) + .route( + "/xrpc/sh.tangled.repo.getRepoByRepoDid", + get(get_repo_by_repo_did), + ) + .route("/xrpc/sh.tangled.actor.getProfile", get(get_profile)) + .route("/xrpc/sh.tangled.actor.getProfiles", get(get_profiles)) + .route("/xrpc/sh.tangled.repo.getIssue", get(get_issue)) + .route("/xrpc/sh.tangled.repo.getIssues", get(get_issues)) + .route("/xrpc/sh.tangled.repo.getPull", get(get_pull)) + .route("/xrpc/sh.tangled.repo.getPulls", get(get_pulls)) + .route("/xrpc/sh.tangled.feed.listStars", get(list_stars)) + .route("/xrpc/sh.tangled.feed.countStars", get(count_stars)) + .route("/xrpc/sh.tangled.graph.listFollows", get(list_follows)) + .route("/xrpc/sh.tangled.graph.countFollows", get(count_follows)) + .route("/xrpc/sh.tangled.repo.listIssues", get(list_issues)) + .route("/xrpc/sh.tangled.repo.countIssues", get(count_issues)) + .route("/xrpc/sh.tangled.repo.listPulls", get(list_pulls)) + .route("/xrpc/sh.tangled.repo.countPulls", get(count_pulls)) + .route( + "/xrpc/sh.tangled.repo.issue.listComments", + get(list_issue_comments), + ) + .route( + "/xrpc/sh.tangled.repo.issue.countComments", + get(count_issue_comments), + ) + .route( + "/xrpc/sh.tangled.repo.pull.listComments", + get(list_pull_comments), + ) + .route( + "/xrpc/sh.tangled.repo.pull.countComments", + get(count_pull_comments), + ) + .route("/xrpc/sh.tangled.feed.listReactions", get(list_reactions)) + .route("/xrpc/sh.tangled.feed.countReactions", get(count_reactions)) + .route("/xrpc/sh.tangled.git.listRefUpdates", get(list_ref_updates)) + .route( + "/xrpc/sh.tangled.git.countRefUpdates", + get(count_ref_updates), + ) + .route( + "/xrpc/sh.tangled.repo.listCollaborators", + get(list_collaborators), + ) + .route( + "/xrpc/sh.tangled.repo.countCollaborators", + get(count_collaborators), + ) + .route( + "/xrpc/sh.tangled.repo.issue.listStates", + get(list_issue_states), + ) + .route( + "/xrpc/sh.tangled.repo.issue.countStates", + get(count_issue_states), + ) + .route( + "/xrpc/sh.tangled.repo.pull.listStatuses", + get(list_pull_statuses), + ) + .route( + "/xrpc/sh.tangled.repo.pull.countStatuses", + get(count_pull_statuses), + ) + .route("/xrpc/sh.tangled.repo.listRepos", get(list_repos)) + .route("/xrpc/sh.tangled.repo.countRepos", get(count_repos)) + .route("/xrpc/sh.tangled.knot.listKnots", get(list_knots)) + .route("/xrpc/sh.tangled.knot.countKnots", get(count_knots)) + .route("/xrpc/sh.tangled.spindle.listSpindles", get(list_spindles)) + .route( + "/xrpc/sh.tangled.spindle.countSpindles", + get(count_spindles), + ) + .route("/xrpc/sh.tangled.publicKey.listKeys", get(list_public_keys)) + .route( + "/xrpc/sh.tangled.publicKey.countKeys", + get(count_public_keys), + ) + .route("/xrpc/sh.tangled.graph.listVouches", get(list_vouches)) + .route("/xrpc/sh.tangled.graph.countVouches", get(count_vouches)) + .route("/xrpc/sh.tangled.feed.listStarsBy", get(list_stars_by)) + .route("/xrpc/sh.tangled.feed.countStarsBy", get(count_stars_by)) + .route( + "/xrpc/sh.tangled.feed.listReactionsBy", + get(list_reactions_by), + ) + .route( + "/xrpc/sh.tangled.feed.countReactionsBy", + get(count_reactions_by), + ) + .route("/xrpc/sh.tangled.graph.listFollowsBy", get(list_follows_by)) + .route( + "/xrpc/sh.tangled.graph.countFollowsBy", + get(count_follows_by), + ) + .route("/xrpc/sh.tangled.graph.listVouchesBy", get(list_vouches_by)) + .route( + "/xrpc/sh.tangled.graph.countVouchesBy", + get(count_vouches_by), + ) + .route( + "/xrpc/sh.tangled.git.listRefUpdatesBy", + get(list_ref_updates_by), + ) + .route( + "/xrpc/sh.tangled.git.countRefUpdatesBy", + get(count_ref_updates_by), + ) + .route( + "/xrpc/sh.tangled.knot.listMembersBy", + get(list_knot_members_by), + ) + .route( + "/xrpc/sh.tangled.knot.countMembersBy", + get(count_knot_members_by), + ) + .route("/xrpc/sh.tangled.label.listOpsBy", get(list_label_ops_by)) + .route("/xrpc/sh.tangled.label.countOpsBy", get(count_label_ops_by)) + .route( + "/xrpc/sh.tangled.pipeline.listPipelinesBy", + get(list_pipelines_by), + ) + .route( + "/xrpc/sh.tangled.pipeline.countPipelinesBy", + get(count_pipelines_by), + ) + .route( + "/xrpc/sh.tangled.pipeline.listStatusesBy", + get(list_pipeline_statuses_by), + ) + .route( + "/xrpc/sh.tangled.pipeline.countStatusesBy", + get(count_pipeline_statuses_by), + ) + .route( + "/xrpc/sh.tangled.repo.listArtifactsBy", + get(list_artifacts_by), + ) + .route( + "/xrpc/sh.tangled.repo.countArtifactsBy", + get(count_artifacts_by), + ) + .route( + "/xrpc/sh.tangled.repo.listCollaboratorsBy", + get(list_collaborators_by), + ) + .route( + "/xrpc/sh.tangled.repo.countCollaboratorsBy", + get(count_collaborators_by), + ) + .route("/xrpc/sh.tangled.repo.listIssuesBy", get(list_issues_by)) + .route("/xrpc/sh.tangled.repo.countIssuesBy", get(count_issues_by)) + .route( + "/xrpc/sh.tangled.repo.issue.listCommentsBy", + get(list_issue_comments_by), + ) + .route( + "/xrpc/sh.tangled.repo.issue.countCommentsBy", + get(count_issue_comments_by), + ) + .route( + "/xrpc/sh.tangled.repo.issue.listStatesBy", + get(list_issue_states_by), + ) + .route( + "/xrpc/sh.tangled.repo.issue.countStatesBy", + get(count_issue_states_by), + ) + .route("/xrpc/sh.tangled.repo.listPullsBy", get(list_pulls_by)) + .route("/xrpc/sh.tangled.repo.countPullsBy", get(count_pulls_by)) + .route( + "/xrpc/sh.tangled.repo.pull.listCommentsBy", + get(list_pull_comments_by), + ) + .route( + "/xrpc/sh.tangled.repo.pull.countCommentsBy", + get(count_pull_comments_by), + ) + .route( + "/xrpc/sh.tangled.repo.pull.listStatusesBy", + get(list_pull_statuses_by), + ) + .route( + "/xrpc/sh.tangled.repo.pull.countStatusesBy", + get(count_pull_statuses_by), + ) + .route( + "/xrpc/sh.tangled.spindle.listMembersBy", + get(list_spindle_members_by), + ) + .route( + "/xrpc/sh.tangled.spindle.countMembersBy", + get(count_spindle_members_by), + ) + .route( + "/xrpc/sh.tangled.label.listDefinitions", + get(list_label_definitions), + ) + .route( + "/xrpc/sh.tangled.label.countDefinitions", + get(count_label_definitions), + ) + .route("/xrpc/sh.tangled.label.listOps", get(list_label_ops)) + .route("/xrpc/sh.tangled.label.countOps", get(count_label_ops)) + .route( + "/xrpc/sh.tangled.pipeline.listPipelines", + get(list_pipelines), + ) + .route( + "/xrpc/sh.tangled.pipeline.countPipelines", + get(count_pipelines), + ) + .route( + "/xrpc/sh.tangled.pipeline.listStatuses", + get(list_pipeline_statuses), + ) + .route( + "/xrpc/sh.tangled.pipeline.countStatuses", + get(count_pipeline_statuses), + ) + .route("/xrpc/sh.tangled.repo.listArtifacts", get(list_artifacts)) + .route("/xrpc/sh.tangled.repo.countArtifacts", get(count_artifacts)) + .route("/xrpc/sh.tangled.knot.listMembers", get(list_knot_members)) + .route( + "/xrpc/sh.tangled.knot.countMembers", + get(count_knot_members), + ) + .route( + "/xrpc/sh.tangled.spindle.listMembers", + get(list_spindle_members), + ) + .route( + "/xrpc/sh.tangled.spindle.countMembers", + get(count_spindle_members), + ) + .route("/xrpc/sh.tangled.string.listStrings", get(list_strings)) + .route("/xrpc/sh.tangled.string.countStrings", get(count_strings)) + .route("/xrpc/sh.tangled.search.query", get(search_query)) + .route("/xrpc/sh.tangled.bobbin.getCoverage", get(get_coverage)) + .route( + "/xrpc/com.bad-example.identity.resolveMiniDoc", + get(resolve_mini_doc), + ) + .merge(knot_proxied_routes()) + .layer( + TraceLayer::new_for_http() + .make_span_with(DefaultMakeSpan::new().level(Level::INFO)) + .on_request(()) + .on_response(LatencyFreeTrace) + .on_failure(LatencyFreeTrace), + ) + .with_state(state) +} + +#[derive(Clone, Copy, Debug)] +struct LatencyFreeTrace; + +impl OnResponse for LatencyFreeTrace { + fn on_response(self, response: &Response, _latency: Duration, _span: &Span) { + tracing::event!( + target: "tower_http::trace::on_response", + Level::INFO, + status = response.status().as_u16(), + "request completed", + ); + } +} + +impl OnFailure for LatencyFreeTrace { + fn on_failure(&mut self, error: ServerErrorsFailureClass, _latency: Duration, _span: &Span) { + tracing::event!( + target: "tower_http::trace::on_failure", + Level::WARN, + error = %error, + "request failed", + ); + } +} + +const REPO_PROXIED_NSIDS: &[&str] = &[ + "sh.tangled.repo.archive", + "sh.tangled.repo.blob", + "sh.tangled.repo.branch", + "sh.tangled.repo.branches", + "sh.tangled.repo.compare", + "sh.tangled.repo.describeRepo", + "sh.tangled.repo.diff", + "sh.tangled.repo.getDefaultBranch", + "sh.tangled.repo.languages", + "sh.tangled.repo.listSecrets", + "sh.tangled.repo.log", + "sh.tangled.repo.tag", + "sh.tangled.repo.tags", + "sh.tangled.repo.tree", +]; + +const KNOT_PROXIED_NSIDS: &[&str] = &[ + "sh.tangled.owner", + "sh.tangled.knot.version", + "sh.tangled.knot.listKeys", +]; + +const PASSTHROUGH_HEADERS: &[&HeaderName] = &[ + &CONTENT_TYPE, + &CONTENT_LENGTH, + &CONTENT_ENCODING, + &ETAG, + &CACHE_CONTROL, + &LAST_MODIFIED, + &CONTENT_DISPOSITION, + &ACCEPT_RANGES, + &CONTENT_RANGE, +]; + +const FORWARDED_REQUEST_HEADERS: &[&HeaderName] = + &[&RANGE, &IF_RANGE, &IF_NONE_MATCH, &IF_MODIFIED_SINCE]; + +const KNOT_HOST_PARAM: &str = "knot"; +const REPO_PARAM: &str = "repo"; + +type ProxyParams = Vec<(String, String)>; + +fn knot_proxied_routes() -> Router { + let with_repo = register_proxied(Router::new(), REPO_PROXIED_NSIDS, proxy_repo_handler); + register_proxied(with_repo, KNOT_PROXIED_NSIDS, proxy_knot_handler) +} + +fn register_proxied( + router: Router, + nsids: &[&'static str], + handler: H, +) -> Router +where + H: Fn(AppState, HeaderMap, ProxyParams, Nsid) -> Fut + + Clone + + Send + + Sync + + 'static, + Fut: Future> + Send + 'static, +{ + nsids.iter().fold(router, |router, &nsid_lit| { + let handler = handler.clone(); + let nsid = nsid_static(nsid_lit); + router.route( + &format!("/xrpc/{nsid_lit}"), + get( + move |State(state): State, + headers: HeaderMap, + Query(params): Query| { + handler(state, headers, params, nsid.clone()) + }, + ), + ) + }) +} + +#[derive(Clone, Debug)] +pub enum SubjectQuery { + Did(Did), + Uri(AtUri), +} + +impl<'de> Deserialize<'de> for SubjectQuery { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let raw = String::deserialize(deserializer)?; + if let Ok(did) = Did::::new_owned(&raw) { + return Ok(Self::Did(did)); + } + AtUri::::new_owned(&raw) + .map(Self::Uri) + .map_err(serde::de::Error::custom) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExpectedNsid(Nsid); + +impl ExpectedNsid { + pub fn new(nsid: Nsid) -> Self { + Self(nsid) + } + + pub fn from_static(s: &'static str) -> Self { + Self(nsid_static(s)) + } + + pub fn as_nsid(&self) -> &Nsid { + &self.0 + } + + pub fn as_str(&self) -> &str { + self.0.as_ref() + } +} + +#[derive(Debug, Deserialize)] +struct GetRepoQuery { + repo: AtUri, +} + +#[derive(Debug, Deserialize)] +struct GetRepoByRepoDidQuery { + #[serde(rename = "repoDid")] + repo_did: Did, +} + +#[derive(Debug, Deserialize)] +struct GetProfileQuery { + actor: AtUri, +} + +#[derive(Debug, Deserialize)] +struct GetIssueQuery { + issue: AtUri, +} + +#[derive(Debug, Deserialize)] +struct GetPullQuery { + pull: AtUri, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "lowercase")] +enum Order { + Asc, + #[default] + Desc, +} + +impl From for SortDir { + fn from(o: Order) -> Self { + match o { + Order::Asc => SortDir::Asc, + Order::Desc => SortDir::Desc, + } + } +} + +#[derive(Debug, Deserialize)] +struct TypedListQuery { + subject: SubjectQuery, + cursor: Option, + limit: Option, + #[serde(default)] + order: Order, + #[serde(flatten)] + filter: F, +} + +impl TypedListQuery { + fn dir(&self) -> SortDir { + self.order.into() + } +} + +#[derive(Debug, Deserialize)] +struct CountQuery { + subject: SubjectQuery, +} + +#[derive(Debug, Deserialize)] +struct SearchQueryParams { + q: String, + nsid: Option>, + author: Option>, + repo: Option>, + since: Option, + until: Option, + cursor: Option, + limit: Option, +} + +pub struct XrpcQuery(pub T); + +impl FromRequestParts for XrpcQuery +where + S: Send + Sync, + T: serde::de::DeserializeOwned + Send + 'static, +{ + type Rejection = XrpcError; + + async fn from_request_parts(parts: &mut Parts, state: &S) -> Result { + Query::::from_request_parts(parts, state) + .await + .map(|Query(t)| Self(t)) + .map_err(|rej: QueryRejection| XrpcError::InvalidParams(rej.body_text())) + } +} + +#[derive(Debug, Error)] +pub enum XrpcError { + #[error("invalid request: {0}")] + InvalidParams(String), + #[error("record not found")] + NotFound, + #[error("upstream unavailable: {0}")] + UpstreamUnavailable(String), + #[error("upstream gone: {0}")] + UpstreamGone(String), + #[error("invalid record: {0}")] + InvalidRecord(String), + #[error("internal: {0}")] + Internal(String), + #[error("overloaded, shedding under memory pressure")] + Overloaded, +} + +impl XrpcError { + pub fn overloaded() -> Self { + Self::Overloaded + } +} + +#[derive(Serialize)] +struct ErrorBody { + error: &'static str, + message: String, +} + +impl IntoResponse for XrpcError { + fn into_response(self) -> Response { + let (status, error) = match &self { + Self::InvalidParams(_) => (StatusCode::BAD_REQUEST, "InvalidRequest"), + Self::NotFound => (StatusCode::NOT_FOUND, "RecordNotFound"), + Self::UpstreamUnavailable(_) => (StatusCode::BAD_GATEWAY, "UpstreamFailed"), + Self::UpstreamGone(_) => (StatusCode::BAD_GATEWAY, "UpstreamGone"), + Self::InvalidRecord(_) => (StatusCode::BAD_GATEWAY, "InvalidRecord"), + Self::Internal(_) => (StatusCode::INTERNAL_SERVER_ERROR, "InternalError"), + Self::Overloaded => (StatusCode::SERVICE_UNAVAILABLE, "Overloaded"), + }; + let body = ErrorBody { + error, + message: self.to_string(), + }; + (status, Json(body)).into_response() + } +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct CoverageEnvelope { + ready: bool, + events_processed: u64, + last_cursor: u64, +} + +impl From for CoverageEnvelope { + fn from(c: Coverage) -> Self { + Self { + ready: c.is_ready(), + events_processed: c.events_processed(), + last_cursor: c.last_cursor().raw(), + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct RecordView { + uri: AtUri, + cid: Option>, + value: V, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct StatefulItem { + #[serde(flatten)] + view: RecordView, + state: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + state_updated_at: Option, + comment_count: u64, +} + +fn format_micros(micros: u64) -> String { + let signed = i64::try_from(micros).ok(); + let rfc = signed + .and_then(chrono::DateTime::::from_timestamp_micros) + .map(|dt| dt.to_rfc3339_opts(chrono::SecondsFormat::Micros, true)); + rfc.unwrap_or_else(|| micros.to_string()) +} + +pub(crate) fn source_authority_did(source: &AtUri) -> Option> { + match source.authority() { + AtIdentifier::Did(d) => Some(d.clone().into_static()), + AtIdentifier::Handle(_) => None, + } +} + +fn enrich_issue_view( + state: &AppState, + view: RecordView>, +) -> StatefulItem> { + let issue_author = source_authority_did(&view.uri); + let repo_did = view.value.repo.clone(); + enrich_view( + &state.edges, + nsid_static("sh.tangled.repo.issue.comment"), + &state.issue_states, + view, + move |src| accept_state_source(src, issue_author.as_ref(), &repo_did), + ) +} + +fn enrich_pull_view( + state: &AppState, + view: RecordView>, +) -> StatefulItem> { + let pull_author = source_authority_did(&view.uri); + let target_repo = view.value.target.repo.clone(); + enrich_view( + &state.edges, + nsid_static("sh.tangled.repo.pull.comment"), + &state.pull_statuses, + view, + move |src| accept_state_source(src, pull_author.as_ref(), &target_repo), + ) +} + +pub(crate) fn accept_state_source( + source: &AtUri, + entity_author: Option<&Did>, + repo_owner: &Did, +) -> bool { + let Some(src) = source_authority_did(source) else { + return false; + }; + Some(&src) == entity_author || &src == repo_owner +} + +fn enrich_view( + edges: &EdgeStore, + comment_nsid: Nsid, + states: &StateIndex, + view: RecordView, + accept: F, +) -> StatefulItem +where + K: StateKind + Default, + F: Fn(&AtUri) -> bool, +{ + let comment_count = edges.count(&EdgeKey::new( + comment_nsid, + SubjectRef::Uri(view.uri.clone()), + )); + let (state, state_updated_at) = states + .latest_by(&view.uri, accept) + .map_or((K::default().wire(), None), |(kind, micros)| { + (kind.wire(), Some(format_micros(micros))) + }); + StatefulItem { + view, + state, + state_updated_at, + comment_count, + } +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct CountResponse { + count: u64, + distinct_authors: u64, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct SearchHitView { + uri: AtUri, + cid: Option>, + nsid: Nsid, + score: f32, + value: SearchableRecord, +} + +fn map_slingshot(err: SlingshotError) -> XrpcError { + use SlingshotError as E; + match err { + E::NotFound => XrpcError::NotFound, + e @ (E::Decode(_) + | E::MissingField(_) + | E::InvalidAtUri(_) + | E::InvalidCid(_) + | E::UriMismatch { .. }) => XrpcError::InvalidRecord(e.to_string()), + e @ (E::Network(_) + | E::Build(_) + | E::Upstream(_) + | E::BodyTooLarge { .. } + | E::BadScheme(_)) => XrpcError::UpstreamUnavailable(e.to_string()), + } +} + +fn parse_uri(raw: &str) -> Result, XrpcError> { + AtUri::::new_owned(raw).map_err(|e| XrpcError::InvalidParams(format!("uri: {e}"))) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SubjectShape { + BareDid, + Collection(&'static str), + BareDidOrOneOfCollections(&'static [&'static str]), + OneOfCollections(&'static [&'static str]), + AnyAtUri, +} + +pub trait HasSubject { + const SHAPE: SubjectShape; +} + +pub trait MirrorOf { + type Record: XrpcResp; + const EDGE_KIND: &'static str; + const SHAPE: SubjectShape; +} + +pub struct StarBy; +pub struct ReactionBy; +pub struct FollowBy; +pub struct VouchBy; +pub struct RefUpdateBy; +pub struct KnotMemberBy; +pub struct LabelOpBy; +pub struct PipelineBy; +pub struct PipelineStatusBy; +pub struct ArtifactBy; +pub struct CollaboratorBy; +pub struct IssueBy; +pub struct IssueCommentBy; +pub struct IssueStateBy; +pub struct PullBy; +pub struct PullCommentBy; +pub struct PullStatusBy; +pub struct SpindleMemberBy; + +impl MirrorOf for StarBy { + type Record = StarRecord; + const EDGE_KIND: &'static str = "sh.tangled.feed.star.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for ReactionBy { + type Record = ReactionRecord; + const EDGE_KIND: &'static str = "sh.tangled.feed.reaction.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for FollowBy { + type Record = FollowRecord; + const EDGE_KIND: &'static str = "sh.tangled.graph.follow.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for VouchBy { + type Record = VouchRecord; + const EDGE_KIND: &'static str = "sh.tangled.graph.vouch.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for RefUpdateBy { + type Record = RefUpdateRecord; + const EDGE_KIND: &'static str = "sh.tangled.git.refUpdate.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for KnotMemberBy { + type Record = KnotMemberRecord; + const EDGE_KIND: &'static str = "sh.tangled.knot.member.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for LabelOpBy { + type Record = LabelOpRecord; + const EDGE_KIND: &'static str = "sh.tangled.label.op.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for PipelineBy { + type Record = PipelineRecord; + const EDGE_KIND: &'static str = "sh.tangled.pipeline.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for PipelineStatusBy { + type Record = PipelineStatusRecord; + const EDGE_KIND: &'static str = "sh.tangled.pipeline.status.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for ArtifactBy { + type Record = ArtifactRecord; + const EDGE_KIND: &'static str = "sh.tangled.repo.artifact.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for CollaboratorBy { + type Record = CollaboratorRecord; + const EDGE_KIND: &'static str = "sh.tangled.repo.collaborator.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for IssueBy { + type Record = IssueRecord; + const EDGE_KIND: &'static str = "sh.tangled.repo.issue.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for IssueCommentBy { + type Record = IssueCommentRecord; + const EDGE_KIND: &'static str = "sh.tangled.repo.issue.comment.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for IssueStateBy { + type Record = IssueStateRecord; + const EDGE_KIND: &'static str = "sh.tangled.repo.issue.state.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for PullBy { + type Record = PullRecord; + const EDGE_KIND: &'static str = "sh.tangled.repo.pull.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for PullCommentBy { + type Record = PullCommentRecord; + const EDGE_KIND: &'static str = "sh.tangled.repo.pull.comment.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for PullStatusBy { + type Record = PullStatusRecord; + const EDGE_KIND: &'static str = "sh.tangled.repo.pull.status.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl MirrorOf for SpindleMemberBy { + type Record = SpindleMemberRecord; + const EDGE_KIND: &'static str = "sh.tangled.spindle.member.by"; + const SHAPE: SubjectShape = SubjectShape::BareDid; +} + +impl HasSubject for StarRecord { + const SHAPE: SubjectShape = SubjectShape::BareDidOrOneOfCollections(&["sh.tangled.string"]); +} +impl HasSubject for FollowRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for IssueRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for PullRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for IssueCommentRecord { + const SHAPE: SubjectShape = SubjectShape::Collection("sh.tangled.repo.issue"); +} +impl HasSubject for PullCommentRecord { + const SHAPE: SubjectShape = SubjectShape::Collection("sh.tangled.repo.pull"); +} +impl HasSubject for LabelDefinitionRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for LabelOpRecord { + const SHAPE: SubjectShape = + SubjectShape::OneOfCollections(&["sh.tangled.repo.issue", "sh.tangled.repo.pull"]); +} +impl HasSubject for PipelineRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for PipelineStatusRecord { + const SHAPE: SubjectShape = SubjectShape::Collection("sh.tangled.pipeline"); +} +impl HasSubject for ArtifactRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for KnotMemberRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for SpindleMemberRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for TangledStringRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for ReactionRecord { + const SHAPE: SubjectShape = SubjectShape::AnyAtUri; +} +impl HasSubject for RefUpdateRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for CollaboratorRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for IssueStateRecord { + const SHAPE: SubjectShape = SubjectShape::Collection("sh.tangled.repo.issue"); +} +impl HasSubject for PullStatusRecord { + const SHAPE: SubjectShape = SubjectShape::Collection("sh.tangled.repo.pull"); +} +impl HasSubject for KnotRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for SpindleRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for PublicKeyRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for RepoRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} +impl HasSubject for VouchRecord { + const SHAPE: SubjectShape = SubjectShape::BareDid; +} + +fn parse_subject(raw: &SubjectQuery, shape: SubjectShape) -> Result { + let uri = match raw { + SubjectQuery::Did(did) => { + return match shape { + SubjectShape::BareDid | SubjectShape::BareDidOrOneOfCollections(_) => { + Ok(SubjectRef::Did(did.clone())) + } + SubjectShape::Collection(expected) => Err(XrpcError::InvalidParams(format!( + "subject must be at:///{expected}/, got bare did" + ))), + SubjectShape::OneOfCollections(allowed) => Err(XrpcError::InvalidParams(format!( + "subject must be at://// with nsid in [{}], got bare did", + allowed.join(", "), + ))), + SubjectShape::AnyAtUri => Err(XrpcError::InvalidParams( + "subject must be at-uri form, got bare did".into(), + )), + }; + } + SubjectQuery::Uri(uri) => uri, + }; + if matches!(uri.authority(), AtIdentifier::Handle(_)) { + return Err(XrpcError::InvalidParams( + "subject authority must be a did, not a handle".into(), + )); + } + let Some(collection) = uri.collection() else { + return Err(XrpcError::InvalidParams( + "subject must be a bare did or full at:////".into(), + )); + }; + let c = collection.as_ref(); + match shape { + SubjectShape::BareDid => Err(XrpcError::InvalidParams(format!( + "subject must be a bare did, got at-uri with collection {c}" + ))), + SubjectShape::Collection(expected) if c == expected => { + require_rkey(uri, expected)?; + Ok(SubjectRef::Uri(uri.clone())) + } + SubjectShape::Collection(expected) => Err(XrpcError::InvalidParams(format!( + "subject must be at:///{expected}/, got collection {c}" + ))), + SubjectShape::OneOfCollections(allowed) if allowed.contains(&c) => { + require_rkey(uri, c)?; + Ok(SubjectRef::Uri(uri.clone())) + } + SubjectShape::OneOfCollections(allowed) => Err(XrpcError::InvalidParams(format!( + "subject must be at://// with nsid in [{}], got collection {c}", + allowed.join(", "), + ))), + SubjectShape::BareDidOrOneOfCollections(allowed) if allowed.contains(&c) => { + require_rkey(uri, c)?; + Ok(SubjectRef::Uri(uri.clone())) + } + SubjectShape::BareDidOrOneOfCollections(allowed) => Err(XrpcError::InvalidParams(format!( + "subject must be a bare did or at://// with nsid in [{}], got collection {c}", + allowed.join(", "), + ))), + SubjectShape::AnyAtUri => Ok(SubjectRef::Uri(uri.clone())), + } +} + +fn require_rkey(uri: &AtUri, expected: &str) -> Result<(), XrpcError> { + uri.rkey().map(|_| ()).ok_or_else(|| { + XrpcError::InvalidParams(format!( + "subject must be at:///{expected}/; missing rkey" + )) + }) +} + +fn parse_cursor(raw: Option<&str>) -> Result { + PageCursor::from_token(raw) + .map_err(|e: CursorParseError| XrpcError::InvalidParams(format!("cursor: {e}"))) +} + +fn parse_limit(raw: Option) -> Result { + PageLimit::new(raw.unwrap_or(DEFAULT_LIMIT)) + .map_err(|e| XrpcError::InvalidParams(format!("limit: {e}"))) +} + +pub(crate) fn at_uri_owned_by(uri: &AtUri, author: &Did) -> bool { + match uri.authority() { + AtIdentifier::Did(d) => d.as_ref() == author.as_ref(), + AtIdentifier::Handle(_) => false, + } +} + +async fn resolve_for_view( + state: &AppState, + expected_nsid: &Nsid, + uri: AtUri, +) -> Result, XrpcError> { + let raw = uri.as_ref().to_owned(); + resolve(state, ExpectedNsid::new(expected_nsid.clone()), uri) + .await + .map(|(body, _did)| body) + .map_err(|e| match e { + XrpcError::NotFound => XrpcError::UpstreamGone(raw), + other => other, + }) +} + +async fn resolve( + state: &AppState, + expected: ExpectedNsid, + uri: AtUri, +) -> Result<(Arc, Did), XrpcError> { + let collection = uri + .collection() + .ok_or_else(|| XrpcError::InvalidParams("uri missing collection".into()))?; + if collection.as_ref() != expected.as_str() { + return Err(XrpcError::InvalidParams(format!( + "collection mismatch: expected {}, got {}", + expected.as_str(), + collection.as_ref() + ))); + } + let rkey = uri + .rkey() + .ok_or_else(|| XrpcError::InvalidParams("uri missing rkey".into()))?; + let did_ref = match uri.authority() { + AtIdentifier::Did(d) => d, + AtIdentifier::Handle(_) => { + return Err(XrpcError::InvalidParams( + "uri authority must be a did, not a handle".into(), + )); + } + }; + let did: Did = did_ref.clone().into_static(); + + if let Some(hit) = state.records.get(&uri) { + return Ok((hit, did)); + } + let body = state + .slingshot + .get_record(&did_ref, &collection, &rkey) + .await + .map_err(map_slingshot)?; + verify_type_tag(&body, &expected)?; + state.records.put(uri, body.clone()); + Ok((body, did)) +} + +#[derive(Deserialize)] +struct TypeTag<'a> { + #[serde(rename = "$type", borrow)] + ty: &'a str, +} + +fn verify_type_tag(body: &RecordBody, expected: &ExpectedNsid) -> Result<(), XrpcError> { + let bytes = body.value.as_ref(); + let ty: std::borrow::Cow<'_, str> = match serde_json::from_slice::(bytes) { + Ok(t) => std::borrow::Cow::Borrowed(t.ty), + Err(_) => { + let value: serde_json::Value = serde_json::from_slice(bytes) + .map_err(|e| XrpcError::InvalidRecord(format!("$type peek: {e}")))?; + value + .as_object() + .and_then(|m| m.get("$type")) + .and_then(|v| v.as_str()) + .map(|s| std::borrow::Cow::Owned(s.to_owned())) + .ok_or_else(|| XrpcError::InvalidRecord("$type peek: missing $type field".into()))? + } + }; + if ty.as_ref() != expected.as_str() { + return Err(XrpcError::InvalidRecord(format!( + "$type mismatch: expected {}, got {}", + expected.as_str(), + ty + ))); + } + Ok(()) +} + +async fn deserialize_or_upgrade( + state: &AppState, + nsid: &Nsid, + bytes: &[u8], +) -> Result +where + V: serde::de::DeserializeOwned, +{ + match serde_json::from_slice::(bytes) { + Ok(v) => Ok(v), + Err(canon_err) => { + let normalized = normalize_record_fields(bytes); + let working: &[u8] = normalized.as_deref().unwrap_or(bytes); + if normalized.is_some() + && let Ok(v) = serde_json::from_slice::(working) + { + return Ok(v); + } + let scrubbed = scrub_record_bytes(nsid, working); + let retry_bytes: &[u8] = scrubbed.as_deref().unwrap_or(working); + if scrubbed.is_some() + && let Ok(v) = serde_json::from_slice::(retry_bytes) + { + return Ok(v); + } + match upgrade_wire_bytes(nsid, retry_bytes, &state.resolver).await { + Ok(canon_bytes) => serde_json::from_slice(&canon_bytes) + .map_err(|e| XrpcError::InvalidRecord(e.to_string())), + Err(_) => Err(XrpcError::InvalidRecord(canon_err.to_string())), + } + } + } +} + +async fn fetch_from_uri( + state: &AppState, + uri: AtUri, +) -> Result<(Arc, V), XrpcError> +where + R: XrpcResp, + V: serde::de::DeserializeOwned + NormalizeRepoRefs, +{ + let raw = uri.as_str().to_owned(); + let nsid = nsid_static(R::NSID); + let (body, _did) = resolve(state, ExpectedNsid::new(nsid.clone()), uri).await?; + let value: V = deserialize_or_upgrade(state, &nsid, &body.value).await?; + let value = value + .normalize(&state.resolver) + .await + .ok_or(XrpcError::UpstreamGone(raw))?; + Ok((body, value)) +} + +async fn fetch( + state: &AppState, + uri: &AtUri, +) -> Result<(Arc, V), XrpcError> +where + R: XrpcResp, + V: serde::de::DeserializeOwned + NormalizeRepoRefs, +{ + fetch_from_uri::(state, uri.clone()).await +} + +async fn get_repo( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result>, XrpcError> { + let (body, value) = fetch::>(&state, &q.repo).await?; + Ok(Json(RepoGetRecordOutput { + cid: Some(body.cid.clone()), + uri: body.uri.clone(), + value, + })) +} + +async fn get_repo_by_repo_did( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result>, XrpcError> { + let ident = state + .resolver + .lookup_by_repo_did(&q.repo_did) + .await + .ok_or(XrpcError::NotFound)?; + let uri = AtUri::::from_parts_owned( + ident.owner.as_str(), + RepoRecord::NSID, + ident.rkey.as_str(), + ) + .expect("Did and Rkey newtypes already validated, at-uri assembly cannot fail"); + let (body, value) = fetch_from_uri::>(&state, uri).await?; + Ok(Json(RepoGetRecordOutput { + cid: Some(body.cid.clone()), + uri: body.uri.clone(), + value, + })) +} + +async fn get_profile( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result>, XrpcError> { + let (body, value) = fetch::>(&state, &q.actor).await?; + Ok(Json(ProfileGetRecordOutput { + cid: Some(body.cid.clone()), + uri: body.uri.clone(), + value, + })) +} + +async fn get_issue( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result>, XrpcError> { + let (body, value) = fetch::>(&state, &q.issue).await?; + Ok(Json(IssueGetRecordOutput { + cid: Some(body.cid.clone()), + uri: body.uri.clone(), + value, + })) +} + +async fn get_pull( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result>, XrpcError> { + let (body, value) = fetch::>(&state, &q.pull).await?; + Ok(Json(PullGetRecordOutput { + cid: Some(body.cid.clone()), + uri: body.uri.clone(), + value, + })) +} + +async fn get_repos( + State(state): State, + RawQuery(query): RawQuery, +) -> Result { + let uris = collect_repeated(query.as_deref(), BULK_REPOS_KEY); + bulk_fetch::>(&state, uris).await +} + +async fn get_profiles( + State(state): State, + RawQuery(query): RawQuery, +) -> Result { + let uris = collect_repeated(query.as_deref(), BULK_PROFILES_KEY); + bulk_fetch::>(&state, uris).await +} + +async fn get_issues( + State(state): State, + RawQuery(query): RawQuery, +) -> Result { + let uris = collect_repeated(query.as_deref(), BULK_ISSUES_KEY); + bulk_fetch::>(&state, uris).await +} + +async fn get_pulls( + State(state): State, + RawQuery(query): RawQuery, +) -> Result { + let uris = collect_repeated(query.as_deref(), BULK_PULLS_KEY); + bulk_fetch::>(&state, uris).await +} + +const BULK_REPOS_KEY: &str = "repos"; +const BULK_PROFILES_KEY: &str = "actors"; +const BULK_ISSUES_KEY: &str = "issues"; +const BULK_PULLS_KEY: &str = "pulls"; +const BULK_LIMIT: usize = 50; + +fn collect_repeated(query: Option<&str>, key: &str) -> Vec { + let Some(q) = query else { + return Vec::new(); + }; + form_urlencoded::parse(q.as_bytes()) + .filter_map(|(k, v)| (k == key).then(|| v.into_owned())) + .collect() +} + +async fn hydrate_record_view( + state: &AppState, + nsid: &Nsid, + uri: AtUri, +) -> Result>, XrpcError> +where + V: serde::de::DeserializeOwned + NormalizeRepoRefs, +{ + let body = resolve_for_view(state, nsid, uri).await?; + let value: V = deserialize_or_upgrade::(state, nsid, &body.value).await?; + let Some(value) = value.normalize(&state.resolver).await else { + return Ok(None); + }; + Ok(Some(RecordView { + uri: body.uri.clone(), + cid: Some(body.cid.clone()), + value, + })) +} + +#[derive(Clone, Copy)] +enum HitProvenance { + ClientSupplied, + Indexed, +} + +fn is_index_evictable(err: &XrpcError) -> bool { + matches!( + err, + XrpcError::NotFound + | XrpcError::UpstreamGone(_) + | XrpcError::InvalidRecord(_) + | XrpcError::InvalidParams(_) + ) +} + +fn drop_unhydratable( + provenance: HitProvenance, + nsid: &Nsid, + uri: &AtUri, + result: Result, XrpcError>, +) -> Result, XrpcError> { + match result { + Ok(view) => Ok(view), + Err(err @ (XrpcError::NotFound | XrpcError::UpstreamGone(_))) => { + tracing::debug!( + uri = %uri, + nsid = %nsid.as_ref(), + error = %err, + "dropping gone hit during hydration", + ); + Ok(None) + } + Err(err @ XrpcError::UpstreamUnavailable(_)) => { + tracing::warn!( + uri = %uri, + nsid = %nsid.as_ref(), + error = %err, + "dropping hit, upstream unavailable during hydration", + ); + Ok(None) + } + Err(err @ XrpcError::InvalidRecord(_)) => { + tracing::warn!( + uri = %uri, + nsid = %nsid.as_ref(), + error = %err, + "dropping invalid hit during hydration", + ); + Ok(None) + } + Err(err @ XrpcError::InvalidParams(_)) => match provenance { + HitProvenance::ClientSupplied => Err(err), + HitProvenance::Indexed => { + tracing::warn!( + uri = %uri, + nsid = %nsid.as_ref(), + error = %err, + "dropping malformed indexed hit during hydration", + ); + Ok(None) + } + }, + Err(err @ (XrpcError::Internal(_) | XrpcError::Overloaded)) => Err(err), + } +} + +fn hydrate_stream( + items: impl IntoIterator, + produce: impl FnMut(T) -> Fut, +) -> impl Stream> +where + Fut: Future, XrpcError>>, +{ + stream::iter(items) + .map(produce) + .buffered(FETCH_CONCURRENCY) + .try_filter_map(|view| async move { Ok(view) }) +} + +fn hydrate_record_stream( + state: &AppState, + nsid: Nsid, + uris: Vec>, + provenance: HitProvenance, +) -> impl Stream, XrpcError>> + Send + 'static +where + V: serde::de::DeserializeOwned + Serialize + NormalizeRepoRefs + Send + 'static, +{ + let owned = state.clone(); + hydrate_stream(uris, move |uri| { + let owned = owned.clone(); + let nsid = nsid.clone(); + async move { + let result = hydrate_record_view::(&owned, &nsid, uri.clone()).await; + if matches!(provenance, HitProvenance::Indexed) + && let Err(err) = &result + && is_index_evictable(err) + { + owned.edges.remove_source(&uri); + } + drop_unhydratable(provenance, &nsid, &uri, result) + } + }) +} + +enum PagePhase { + Head, + Body { first: bool }, + Done, +} + +struct PageState { + items: std::pin::Pin>, + phase: PagePhase, + array_key: &'static str, + tail: Vec, + permit: Option, +} + +fn paged_tail(cursor: Option) -> Vec { + let encoded = serde_json::to_string(&cursor).unwrap_or_else(|_| "null".to_owned()); + format!("],\"cursor\":{encoded}}}").into_bytes() +} + +fn unpaged_tail() -> Vec { + b"]}".to_vec() +} + +fn json_stream( + array_key: &'static str, + items: S, + tail: Vec, + permit: Option, +) -> Response +where + V: Serialize + Send + 'static, + S: Stream> + Send + 'static, +{ + let init = PageState { + items: Box::pin(items), + phase: PagePhase::Head, + array_key, + tail, + permit, + }; + let chunks = stream::unfold(init, |mut st| async move { + match st.phase { + PagePhase::Head => { + let head = format!("{{\"{}\":[", st.array_key).into_bytes(); + st.phase = PagePhase::Body { first: true }; + Some((Ok::, Infallible>(head), st)) + } + PagePhase::Body { first } => match st.items.next().await { + Some(Ok(view)) => match serde_json::to_vec(&view) { + Ok(encoded) => { + let mut chunk = Vec::with_capacity(encoded.len() + 1); + if !first { + chunk.push(b','); + } + chunk.extend_from_slice(&encoded); + st.phase = PagePhase::Body { first: false }; + Some((Ok(chunk), st)) + } + Err(e) => { + tracing::warn!(error = %e, "skipping hit, serialize failed mid-stream"); + Some((Ok(Vec::new()), st)) + } + }, + Some(Err(e)) => { + tracing::warn!(error = %e, "ending page early, hydration failed mid-stream"); + let tail = std::mem::take(&mut st.tail); + st.phase = PagePhase::Done; + Some((Ok(tail), st)) + } + None => { + let tail = std::mem::take(&mut st.tail); + st.phase = PagePhase::Done; + Some((Ok(tail), st)) + } + }, + PagePhase::Done => { + drop(st.permit.take()); + None + } + } + }); + ( + [(CONTENT_TYPE, "application/json")], + Body::from_stream(chunks), + ) + .into_response() +} + +async fn bulk_fetch(state: &AppState, uris: Vec) -> Result +where + R: XrpcResp, + V: serde::de::DeserializeOwned + Serialize + NormalizeRepoRefs + Send + 'static, +{ + if uris.is_empty() { + return Err(XrpcError::InvalidParams("at least one uri required".into())); + } + if uris.len() > BULK_LIMIT { + return Err(XrpcError::InvalidParams(format!( + "at most {BULK_LIMIT} uris per request" + ))); + } + let parsed: Vec> = uris + .iter() + .map(|s| parse_uri(s)) + .collect::>()?; + let nsid = nsid_static(R::NSID); + if let Some(bad) = parsed + .iter() + .find(|uri| uri.collection().is_none_or(|c| c.as_ref() != nsid.as_ref())) + { + return Err(XrpcError::InvalidParams(format!( + "uri collection must be {}, got {}", + nsid.as_ref(), + bad.as_ref() + ))); + } + let permit = state.heavy_permit()?; + let views = hydrate_record_stream::(state, nsid, parsed, HitProvenance::ClientSupplied); + Ok(json_stream::, _>( + "items", + views, + unpaged_tail(), + permit, + )) +} + +fn record_edge_page( + state: &AppState, + q: &TypedListQuery, +) -> Result<(EdgePage, Nsid), XrpcError> +where + R: XrpcResp + HasSubject, + F: ListFilter, +{ + let subject = parse_subject(&q.subject, R::SHAPE)?; + let cursor = parse_cursor(q.cursor.as_deref())?; + let limit = parse_limit(q.limit)?; + let dir = q.dir(); + let nsid = nsid_static(R::NSID); + let page = if q.filter.is_identity() { + let key = EdgeKey::new(nsid.clone(), subject); + state.edges.list(&key, cursor, limit, dir) + } else { + let pred = q.filter.predicate(state, &subject); + let key = EdgeKey::new(nsid.clone(), subject); + state.edges.list_filtered(&key, cursor, limit, dir, pred) + }; + Ok((page, nsid)) +} + +async fn list_records( + state: &AppState, + q: TypedListQuery, +) -> Result +where + R: XrpcResp + HasSubject, + V: serde::de::DeserializeOwned + Serialize + NormalizeRepoRefs + Send + 'static, + F: ListFilter, +{ + let (page, nsid) = record_edge_page::(state, &q)?; + let permit = state.heavy_permit()?; + let views = hydrate_record_stream::(state, nsid, page.items, HitProvenance::Indexed); + Ok(json_stream::, _>( + "items", + views, + paged_tail(page.next.map(PageToken::encode_token)), + permit, + )) +} + +fn count_for( + state: &AppState, + q: CountQuery, +) -> Result { + let subject = parse_subject(&q.subject, R::SHAPE)?; + let key = EdgeKey::new(nsid_static(R::NSID), subject); + Ok(CountResponse { + count: state.edges.count(&key), + distinct_authors: state.edges.count_distinct_authors(&key), + }) +} + +fn mirror_edge_page( + state: &AppState, + q: &TypedListQuery, +) -> Result<(EdgePage, Nsid), XrpcError> +where + M: MirrorOf, + F: ListFilter, +{ + let subject = parse_subject(&q.subject, M::SHAPE)?; + let cursor = parse_cursor(q.cursor.as_deref())?; + let limit = parse_limit(q.limit)?; + let dir = q.dir(); + let edge_nsid = nsid_static(M::EDGE_KIND); + let page = if q.filter.is_identity() { + let key = EdgeKey::new(edge_nsid, subject); + state.edges.list(&key, cursor, limit, dir) + } else { + let pred = q.filter.predicate(state, &subject); + let key = EdgeKey::new(edge_nsid, subject); + state.edges.list_filtered(&key, cursor, limit, dir, pred) + }; + let record_nsid = nsid_static(::NSID); + Ok((page, record_nsid)) +} + +async fn list_mirror(state: &AppState, q: TypedListQuery) -> Result +where + M: MirrorOf, + V: serde::de::DeserializeOwned + Serialize + NormalizeRepoRefs + Send + 'static, + F: ListFilter, +{ + let (page, record_nsid) = mirror_edge_page::(state, &q)?; + let permit = state.heavy_permit()?; + let views = hydrate_record_stream::(state, record_nsid, page.items, HitProvenance::Indexed); + Ok(json_stream::, _>( + "items", + views, + paged_tail(page.next.map(PageToken::encode_token)), + permit, + )) +} + +fn count_mirror(state: &AppState, q: CountQuery) -> Result { + let subject = parse_subject(&q.subject, M::SHAPE)?; + let key = EdgeKey::new(nsid_static(M::EDGE_KIND), subject); + Ok(CountResponse { + count: state.edges.count(&key), + distinct_authors: state.edges.count_distinct_authors(&key), + }) +} + +async fn list_stars( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_stars( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_follows( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_follows( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_issues( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + let (page, nsid) = record_edge_page::(&state, &q)?; + let permit = state.heavy_permit()?; + let owned = state.clone(); + let items = hydrate_record_stream::>( + &state, + nsid, + page.items, + HitProvenance::Indexed, + ) + .map(move |view| view.map(|v| enrich_issue_view(&owned, v))); + Ok(json_stream::>, _>( + "items", + items, + paged_tail(page.next.map(PageToken::encode_token)), + permit, + )) +} + +async fn count_issues( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_pulls( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + let (page, nsid) = record_edge_page::(&state, &q)?; + let permit = state.heavy_permit()?; + let owned = state.clone(); + let items = + hydrate_record_stream::>(&state, nsid, page.items, HitProvenance::Indexed) + .map(move |view| view.map(|v| enrich_pull_view(&owned, v))); + Ok(json_stream::>, _>( + "items", + items, + paged_tail(page.next.map(PageToken::encode_token)), + permit, + )) +} + +async fn count_pulls( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_issue_comments( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_issue_comments( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_pull_comments( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_pull_comments( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_reactions( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_reactions( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_ref_updates( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_ref_updates( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_collaborators( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_collaborators( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_issue_states( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_issue_states( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_pull_statuses( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_pull_statuses( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_repos( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_repos( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_knots( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_knots( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_spindles( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_spindles( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_public_keys( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_public_keys( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_vouches( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_vouches( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_stars_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_stars_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_reactions_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_reactions_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_follows_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_follows_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_vouches_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_vouches_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_ref_updates_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_ref_updates_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_knot_members_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_knot_members_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_label_ops_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_label_ops_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_pipelines_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_pipelines_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_pipeline_statuses_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_pipeline_statuses_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_artifacts_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_artifacts_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_collaborators_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_collaborators_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_issues_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + let (page, nsid) = mirror_edge_page::(&state, &q)?; + let permit = state.heavy_permit()?; + let owned = state.clone(); + let items = hydrate_record_stream::>( + &state, + nsid, + page.items, + HitProvenance::Indexed, + ) + .map(move |view| view.map(|v| enrich_issue_view(&owned, v))); + Ok(json_stream::>, _>( + "items", + items, + paged_tail(page.next.map(PageToken::encode_token)), + permit, + )) +} +async fn count_issues_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_issue_comments_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_issue_comments_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_issue_states_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_issue_states_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_pulls_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + let (page, nsid) = mirror_edge_page::(&state, &q)?; + let permit = state.heavy_permit()?; + let owned = state.clone(); + let items = + hydrate_record_stream::>(&state, nsid, page.items, HitProvenance::Indexed) + .map(move |view| view.map(|v| enrich_pull_view(&owned, v))); + Ok(json_stream::>, _>( + "items", + items, + paged_tail(page.next.map(PageToken::encode_token)), + permit, + )) +} +async fn count_pulls_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_pull_comments_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_pull_comments_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_pull_statuses_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_pull_statuses_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_spindle_members_by( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_mirror::, _>(&state, q).await +} +async fn count_spindle_members_by( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_mirror::(&state, q).map(Json) +} + +async fn list_label_definitions( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_label_definitions( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_label_ops( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_label_ops( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_pipelines( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_pipelines( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_pipeline_statuses( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_pipeline_statuses( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_artifacts( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_artifacts( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_knot_members( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_knot_members( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_spindle_members( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_spindle_members( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +async fn list_strings( + State(state): State, + XrpcQuery(q): XrpcQuery>, +) -> Result { + list_records::, _>(&state, q).await +} + +async fn count_strings( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result, XrpcError> { + count_for::(&state, q).map(Json) +} + +#[derive(Deserialize)] +struct ResolveMiniDocParams { + identifier: AtIdentifier, +} + +async fn resolve_mini_doc( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result { + let body = state + .slingshot + .resolve_mini_doc(&q.identifier) + .await + .map_err(map_slingshot)?; + Ok((StatusCode::OK, [(CONTENT_TYPE, "application/json")], body).into_response()) +} + +async fn get_coverage(State(state): State) -> Json { + Json(state.coverage.snapshot().into()) +} + +async fn search_query( + State(state): State, + XrpcQuery(q): XrpcQuery, +) -> Result { + if q.q.trim().is_empty() { + return Err(XrpcError::InvalidParams("q must not be empty".into())); + } + let cursor = SearchCursor::from_token(q.cursor.as_deref()) + .map_err(|e| XrpcError::InvalidParams(format!("cursor: {e}")))?; + let limit = parse_limit(q.limit)?; + let filters = build_search_filters(&q)?; + let permit = state.heavy_permit()?; + let page = state + .search + .search(&q.q, filters, cursor, limit.get()) + .await + .map_err(map_search_err)?; + let next = page.next.map(SearchOffset::encode_token); + let owned = state.clone(); + let hits = hydrate_stream(page.hits, move |hit| { + let owned = owned.clone(); + let uri = hit.uri.clone(); + let nsid = hit.nsid.clone(); + async move { + let result = hydrate_search_hit(&owned, hit).await; + drop_unhydratable(HitProvenance::Indexed, &nsid, &uri, result) + } + }); + Ok(json_stream::( + "hits", + hits, + paged_tail(next), + permit, + )) +} + +fn build_search_filters(q: &SearchQueryParams) -> Result { + let since = q + .since + .as_deref() + .map(parse_rfc3339_seconds) + .transpose() + .map_err(|e| XrpcError::InvalidParams(format!("since: {e}")))?; + let until = q + .until + .as_deref() + .map(parse_rfc3339_seconds) + .transpose() + .map_err(|e| XrpcError::InvalidParams(format!("until: {e}")))?; + if let (Some(s), Some(u)) = (since, until) + && s > u + { + return Err(XrpcError::InvalidParams("since must be <= until".into())); + } + Ok(SearchFilters { + nsid: q.nsid.clone(), + author: q.author.clone(), + repo: q.repo.clone(), + since, + until, + }) +} + +fn parse_rfc3339_seconds(raw: &str) -> Result { + chrono::DateTime::parse_from_rfc3339(raw) + .map(|dt| dt.timestamp()) + .map_err(|e| format!("expected RFC3339, got {raw}: {e}")) +} + +async fn hydrate_search_hit( + state: &AppState, + hit: SearchHit, +) -> Result, XrpcError> { + let SearchHit { uri, nsid, score } = hit; + let body = resolve_for_view(state, &nsid, uri).await?; + let record = decode_canon_or_upgrade(&nsid, &body.value, &state.resolver) + .await + .map_err(|err| XrpcError::InvalidRecord(err.to_string()))?; + let Some(value) = SearchableRecord::try_from_record(record) else { + return Ok(None); + }; + let Some(value) = value.normalize(&state.resolver).await else { + return Ok(None); + }; + Ok(Some(SearchHitView { + uri: body.uri.clone(), + cid: Some(body.cid.clone()), + nsid, + score, + value, + })) +} + +fn map_search_err(err: SearchError) -> XrpcError { + use SearchError as E; + match err { + E::Query(e) => XrpcError::InvalidParams(format!("query: {e}")), + e @ (E::Tantivy(_) + | E::InvalidUri(_) + | E::InvalidNsid(_) + | E::MissingField(_) + | E::Cancelled(_)) => XrpcError::Internal(format!("search: {e}")), + } +} + +fn map_proxy_error(err: KnotProxyError) -> XrpcError { + match err { + KnotProxyError::CircuitOpen => { + XrpcError::UpstreamUnavailable("knot circuit breaker open".into()) + } + KnotProxyError::BlockedHost { host, reason } => { + XrpcError::InvalidRecord(format!("knot host {host} is {reason} address space")) + } + KnotProxyError::PlaintextHttp { host } => { + XrpcError::InvalidRecord(format!("knot host {host} requires https")) + } + KnotProxyError::Connect(e) => XrpcError::UpstreamUnavailable(format!("connect: {e}")), + KnotProxyError::Timeout(e) => { + XrpcError::UpstreamUnavailable(format!("upstream timeout: {e}")) + } + KnotProxyError::Redirect(e) => XrpcError::UpstreamUnavailable(format!("redirect: {e}")), + KnotProxyError::Transport(e) => XrpcError::UpstreamUnavailable(format!("transport: {e}")), + KnotProxyError::Upstream(s) => XrpcError::UpstreamUnavailable(format!("status {s}")), + } +} + +fn validate_client_supplied_knot(state: &AppState, host: &KnotHost) -> Result<(), XrpcError> { + let host_str = || host.url().host_str().unwrap_or_default().to_owned(); + if state.knots.requires_https() && host.url().scheme() != "https" { + return Err(XrpcError::InvalidParams(format!( + "knot host {} must be https", + host_str(), + ))); + } + if state.knots.allows_private_hosts() { + return Ok(()); + } + match host.private_literal_reason() { + None => Ok(()), + Some(reason) => Err(XrpcError::InvalidParams(format!( + "knot host {} blocked: {} address space", + host_str(), + reason, + ))), + } +} + +async fn resolve_knot_target( + state: &AppState, + repo_uri: AtUri, +) -> Result<(KnotHost, RepoSlug), XrpcError> { + let rkey: Option> = repo_uri.rkey().map(|r| r.clone().into_static()); + let (body, did) = resolve(state, ExpectedNsid::from_static(RepoRecord::NSID), repo_uri).await?; + let value: Repo = serde_json::from_slice(&body.value) + .map_err(|e| XrpcError::InvalidRecord(format!("decode repo record: {e}")))?; + let host = KnotHost::parse(value.knot.as_ref()) + .map_err(|e| XrpcError::InvalidRecord(format!("knot field: {e}")))?; + let name = pick_human_slug(rkey.as_ref(), value.name.as_deref()).ok_or_else(|| { + XrpcError::InvalidRecord("at-uri missing rkey and record missing name".to_string()) + })?; + let slug = RepoSlug::new(&did, &name) + .map_err(|e| XrpcError::InvalidRecord(format!("repo slug: {e}")))?; + Ok((host, slug)) +} + +fn pick_human_slug(rkey: Option<&Rkey>, name: Option<&str>) -> Option { + match rkey { + Some(r) if jacquard_common::types::tid::Tid::new(r.as_ref()).is_ok() => { + Some(name.unwrap_or(r.as_ref()).to_owned()) + } + Some(r) => Some(r.as_ref().to_owned()), + None => name.map(str::to_owned), + } +} + +fn filter_request_headers(client: &HeaderMap) -> HeaderMap { + FORWARDED_REQUEST_HEADERS + .iter() + .fold(HeaderMap::new(), |mut acc, name| { + if let Some(value) = client.get(*name) { + acc.insert((*name).clone(), value.clone()); + } + acc + }) +} + +fn upstream_to_axum(resp: ProxyResponse) -> Response { + let status = resp.status(); + let upstream_headers = resp.headers().clone(); + let body = Body::from_stream(resp.into_body_stream()); + let mut response = Response::builder() + .status(status) + .body(body) + .expect("response body construction must succeed"); + let response_headers = response.headers_mut(); + PASSTHROUGH_HEADERS.iter().for_each(|name| { + if let Some(value) = upstream_headers.get(*name) { + response_headers.insert((*name).clone(), value.clone()); + } + }); + response +} + +async fn dispatch_proxy( + state: AppState, + headers: HeaderMap, + nsid: Nsid, + host: KnotHost, + params: ProxyParams, +) -> Result { + let forward: Vec<(&str, &str)> = params + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect(); + let allowed = filter_request_headers(&headers); + let upstream = state + .knots + .forward(&host, &nsid, &forward, allowed) + .await + .map_err(map_proxy_error)?; + Ok(upstream_to_axum(upstream)) +} + +fn extract_param( + params: ProxyParams, + key: &str, +) -> Result, XrpcError> { + let (matching, rest): (ProxyParams, ProxyParams) = + params.into_iter().partition(|(k, _)| k == key); + match matching.as_slice() { + [] => Ok(None), + [_] => Ok(matching.into_iter().next().map(|(_, v)| (v, rest))), + _ => Err(XrpcError::InvalidParams(format!( + "{key} parameter must appear at most once, got {}", + matching.len(), + ))), + } +} + +async fn proxy_repo_handler( + state: AppState, + headers: HeaderMap, + params: ProxyParams, + nsid: Nsid, +) -> Result { + let (repo_raw, rest) = extract_param(params, REPO_PARAM)? + .ok_or_else(|| XrpcError::InvalidParams("missing repo".into()))?; + let repo_uri = parse_uri(&repo_raw)?; + let (host, slug) = resolve_knot_target(&state, repo_uri).await?; + let forward = rest + .into_iter() + .chain(std::iter::once(( + REPO_PARAM.to_owned(), + slug.as_str().to_owned(), + ))) + .collect(); + dispatch_proxy(state, headers, nsid, host, forward).await +} + +async fn proxy_knot_handler( + state: AppState, + headers: HeaderMap, + params: ProxyParams, + nsid: Nsid, +) -> Result { + let (knot_raw, forward) = extract_param(params, KNOT_HOST_PARAM)? + .ok_or_else(|| XrpcError::InvalidParams("missing knot".into()))?; + let host = + KnotHost::parse(&knot_raw).map_err(|e| XrpcError::InvalidParams(format!("knot: {e}")))?; + validate_client_supplied_knot(&state, &host)?; + dispatch_proxy(state, headers, nsid, host, forward).await +} diff --git a/bobbin/crates/xrpc/tests/aggregation.rs b/bobbin/crates/xrpc/tests/aggregation.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/xrpc/tests/aggregation.rs @@ -0,0 +1,2057 @@ +use std::sync::Arc; + +use axum::body::{Body, to_bytes}; +use bobbin_edge_index::{ + Coverage, CoverageWatch, EdgeStore, HydrantCursor, IssueStateKind, PageToken, PullStatusKind, + StateIndex, +}; +use bobbin_knot_proxy::{KnotHttpConfig, KnotProxy, KnotProxyConfig}; +use bobbin_record_lru::{CacheCapacity, LruRecordStore}; +use bobbin_resolver::RepoIdResolver; +use bobbin_runtime::{RuntimeHasher, SystemClock}; +use bobbin_search::{DEFAULT_WRITER_HEAP_BYTES, SearchIndex, SearchReader}; +use bobbin_slingshot_client::SlingshotClient; +use bobbin_types::edges::Edge; +use bobbin_types::ids::SubjectRef; +use bobbin_xrpc::{AppState, router}; +use futures::stream::{self, StreamExt}; +use http::{Request, StatusCode}; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::recordkey::Rkey; +use jacquard_common::types::string::AtUri; +use serde_json::{Value, json}; +use tower::ServiceExt; +use url::Url; +use url::form_urlencoded::byte_serialize; +use wiremock::matchers::{method, path, query_param}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const CID: &str = "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i"; + +fn at(s: &str) -> AtUri { + AtUri::new_owned(s).unwrap() +} + +fn did(s: &str) -> Did { + Did::new_owned(s).unwrap() +} + +fn rkey(s: &str) -> Rkey { + Rkey::new_owned(s).unwrap() +} + +fn nsid(s: &'static str) -> Nsid { + Nsid::new_static(s).unwrap() +} + +fn subj(s: &str) -> SubjectRef { + Did::::new_owned(s) + .map(SubjectRef::Did) + .unwrap_or_else(|_| SubjectRef::Uri(AtUri::new_owned(s).unwrap())) +} + +struct Harness { + server: MockServer, + edges: Arc, + coverage: Arc, + state: AppState, +} + +static EDGE_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + +fn next_sort_micros() -> u64 { + EDGE_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed) +} + +impl Harness { + async fn new() -> Self { + let server = MockServer::start().await; + let edges = Arc::new(EdgeStore::new(RuntimeHasher::default())); + let issue_states = Arc::new(StateIndex::new(RuntimeHasher::default())); + let pull_statuses = Arc::new(StateIndex::new(RuntimeHasher::default())); + let coverage = Arc::new(CoverageWatch::new()); + let state = AppState::new( + Arc::new(LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024))), + SlingshotClient::with_default_http(Url::parse(&server.uri()).unwrap()).unwrap(), + edges.clone(), + issue_states.clone(), + pull_statuses.clone(), + coverage.clone(), + Arc::new( + KnotProxy::new( + KnotProxyConfig::default(), + KnotHttpConfig::default(), + Arc::new(SystemClock::new()), + RuntimeHasher::default(), + ) + .unwrap(), + ), + Arc::new( + SearchIndex::new(DEFAULT_WRITER_HEAP_BYTES, Arc::new(SystemClock::new())).unwrap(), + ) as Arc, + Arc::new(RepoIdResolver::detached(RuntimeHasher::default())), + ); + Self { + server, + edges, + coverage, + state, + } + } + + fn add_edge( + &self, + kind: &Nsid, + subject: &AtUri, + source: &AtUri, + ) { + self.edges.add(Edge { + kind: kind.clone(), + subject: subj(subject.as_ref()), + source: source.clone(), + sort_micros: next_sort_micros(), + }); + } + + async fn mount( + &self, + did: &Did, + collection: &Nsid, + rkey: &Rkey, + value: Value, + ) { + let uri = format!( + "at://{}/{}/{}", + did.as_ref(), + collection.as_ref(), + rkey.as_ref() + ); + let body = json!({ "uri": uri, "cid": CID, "value": value }); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", did.as_ref())) + .and(query_param("collection", collection.as_ref())) + .and(query_param("rkey", rkey.as_ref())) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .mount(&self.server) + .await; + } + + fn promote_ready(&self, events: u64, cursor: u64) { + self.coverage.update(|_| Coverage::Ready { + events_processed: events, + last_cursor: HydrantCursor::new(cursor), + }); + } + + fn warming(&self, events: u64, cursor: u64) { + self.coverage.update(|_| Coverage::Warming { + events_processed: events, + last_cursor: HydrantCursor::new(cursor), + }); + } +} + +fn list_request(endpoint: &str, subject: &str, extras: &[(&str, &str)]) -> Request { + let mut qs = format!("subject={}", encode(subject)); + extras.iter().for_each(|(k, v)| { + qs.push('&'); + qs.push_str(k); + qs.push('='); + qs.push_str(&encode(v)); + }); + Request::builder() + .uri(format!("/xrpc/{endpoint}?{qs}")) + .body(Body::empty()) + .unwrap() +} + +fn encode(s: &str) -> String { + byte_serialize(s.as_bytes()).collect() +} + +async fn json_response(resp: axum::response::Response) -> (StatusCode, Value) { + let status = resp.status(); + let bytes = to_bytes(resp.into_body(), 1 << 20).await.unwrap(); + let parsed: Value = serde_json::from_slice(&bytes).expect("JSON body"); + (status, parsed) +} + +fn issue_body(repo_did: &Did, title: &str) -> Value { + json!({ + "$type": "sh.tangled.repo.issue", + "repo": repo_did.as_ref(), + "title": title, + "createdAt": "2026-05-01T00:00:00Z" + }) +} + +fn pull_body(repo_did: &Did, title: &str) -> Value { + json!({ + "$type": "sh.tangled.repo.pull", + "title": title, + "createdAt": "2026-05-01T00:00:00Z", + "rounds": [], + "target": { + "repo": repo_did.as_ref(), + "branch": "main" + } + }) +} + +fn star_body(subject_did: &Did) -> Value { + json!({ + "$type": "sh.tangled.feed.star", + "createdAt": "2026-05-01T00:00:00Z", + "subject": { + "$type": "sh.tangled.feed.star#repo", + "did": subject_did.as_ref() + } + }) +} + +fn follow_body(subject_did: &Did) -> Value { + json!({ + "$type": "sh.tangled.graph.follow", + "createdAt": "2026-05-01T00:00:00Z", + "subject": subject_did.as_ref() + }) +} + +#[tokio::test] +async fn list_issues_with_no_edges_returns_empty_items() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot(list_request( + "sh.tangled.repo.listIssues", + "at://did:plc:abalone", + &[], + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["items"], json!([])); + assert!(body["cursor"].is_null()); +} + +#[tokio::test] +async fn count_issues_with_no_edges_returns_zero() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot(list_request( + "sh.tangled.repo.countIssues", + "at://did:plc:abalone", + &[], + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["count"], json!(0)); + assert_eq!(body["distinctAuthors"], json!(0)); +} + +#[tokio::test] +async fn list_issues_hydrates_via_slingshot_when_edges_present() { + let h = Harness::new().await; + let repo = did("did:plc:abalone"); + let subject = at(&format!("at://{}", repo.as_ref())); + let owners = [ + ("did:plc:nel", "i1", "first"), + ("did:plc:olaren", "i2", "second"), + ]; + stream::iter(owners) + .for_each(|(d, r, title)| { + let h = &h; + let subject = subject.clone(); + let repo = repo.clone(); + async move { + let d_did = did(d); + let rk = rkey(r); + h.add_edge( + &nsid("sh.tangled.repo.issue"), + &subject, + &at(&format!( + "at://{}/sh.tangled.repo.issue/{}", + d_did.as_ref(), + rk.as_ref() + )), + ); + h.mount( + &d_did, + &nsid("sh.tangled.repo.issue"), + &rk, + issue_body(&repo, title), + ) + .await; + } + }) + .await; + + let app = router(h.state.clone()); + let resp = app + .oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[], + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().expect("items array"); + assert_eq!(items.len(), 2); + let titles: Vec<&str> = items + .iter() + .map(|v| v["value"]["title"].as_str().unwrap()) + .collect(); + assert!(titles.contains(&"first")); + assert!(titles.contains(&"second")); + assert_eq!(items[0]["cid"], CID); + assert!(items[0]["uri"].as_str().unwrap().starts_with("at://")); +} + +#[tokio::test] +async fn count_distinct_authors_dedupes_per_author() { + let h = Harness::new().await; + let subject = at("at://did:plc:abalone"); + h.add_edge( + &nsid("sh.tangled.feed.star"), + &subject, + &at("at://did:plc:nel/sh.tangled.feed.star/s1"), + ); + h.add_edge( + &nsid("sh.tangled.feed.star"), + &subject, + &at("at://did:plc:nel/sh.tangled.feed.star/s2"), + ); + h.add_edge( + &nsid("sh.tangled.feed.star"), + &subject, + &at("at://did:plc:olaren/sh.tangled.feed.star/s3"), + ); + + let app = router(h.state.clone()); + let resp = app + .oneshot(list_request( + "sh.tangled.feed.countStars", + subject.as_ref(), + &[], + )) + .await + .unwrap(); + let (_, body) = json_response(resp).await; + assert_eq!(body["count"], json!(3)); + assert_eq!(body["distinctAuthors"], json!(2)); +} + +#[tokio::test] +async fn list_items_stable_across_coverage_promotion() { + let h = Harness::new().await; + let subject = at("at://did:plc:abalone"); + let nel = did("did:plc:nel"); + h.add_edge( + &nsid("sh.tangled.feed.star"), + &subject, + &at(&format!("at://{}/sh.tangled.feed.star/s1", nel.as_ref())), + ); + h.mount( + &nel, + &nsid("sh.tangled.feed.star"), + &rkey("s1"), + star_body(&did("did:plc:abalone")), + ) + .await; + + let app = router(h.state.clone()); + h.warming(1, 5); + let (_, before) = json_response( + app.clone() + .oneshot(list_request( + "sh.tangled.feed.listStars", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(before["items"].as_array().unwrap().len(), 1); + + h.promote_ready(2, 9); + let (_, after) = json_response( + app.oneshot(list_request( + "sh.tangled.feed.listStars", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!( + after["items"].as_array().unwrap().len(), + before["items"].as_array().unwrap().len(), + ); + assert_eq!(after["items"], before["items"]); +} + +#[tokio::test] +async fn list_paginates_via_cursor() { + let h = Harness::new().await; + let subject = at("at://did:plc:abalone"); + let repo = did("did:plc:abalone"); + let owners = [ + ("did:plc:nel", "i1"), + ("did:plc:olaren", "i2"), + ("did:plc:teq", "i3"), + ("did:plc:lyna", "i4"), + ("did:plc:bailey", "i5"), + ]; + stream::iter(owners) + .for_each(|(d, r)| { + let h = &h; + let subject = subject.clone(); + let repo = repo.clone(); + async move { + let d_did = did(d); + let rk = rkey(r); + h.add_edge( + &nsid("sh.tangled.repo.issue"), + &subject, + &at(&format!( + "at://{}/sh.tangled.repo.issue/{}", + d_did.as_ref(), + rk.as_ref() + )), + ); + h.mount( + &d_did, + &nsid("sh.tangled.repo.issue"), + &rk, + issue_body(&repo, &format!("issue-{}", rk.as_ref())), + ) + .await; + } + }) + .await; + + let app = router(h.state.clone()); + let (_, page1) = json_response( + app.clone() + .oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[("limit", "2")], + )) + .await + .unwrap(), + ) + .await; + let page1_items = page1["items"].as_array().unwrap().clone(); + assert_eq!(page1_items.len(), 2); + let cursor = page1["cursor"] + .as_str() + .expect("first page must yield a cursor") + .to_owned(); + assert!( + PageToken::decode_token(&cursor).is_ok(), + "cursor must be a TID-shaped token" + ); + + let (_, page2) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[("limit", "10"), ("cursor", &cursor)], + )) + .await + .unwrap(), + ) + .await; + let page2_items = page2["items"].as_array().unwrap().clone(); + assert_eq!(page2_items.len(), 3); + assert!(page2["cursor"].is_null(), "tail page must not promise more"); + + let union: Vec<&str> = page1_items + .iter() + .chain(page2_items.iter()) + .map(|item| item["uri"].as_str().unwrap()) + .collect(); + assert_eq!(union.len(), owners.len(), "union covers every owner"); + let mut sorted = union.clone(); + sorted.sort(); + sorted.dedup(); + assert_eq!(sorted.len(), owners.len(), "no duplicates across pages"); +} + +#[tokio::test] +async fn pagination_unaffected_by_coverage_promotion() { + let h = Harness::new().await; + let subject = at("at://did:plc:abalone"); + let repo = did("did:plc:abalone"); + let owners = [("did:plc:nel", "i1"), ("did:plc:olaren", "i2")]; + stream::iter(owners) + .for_each(|(d, r)| { + let h = &h; + let subject = subject.clone(); + let repo = repo.clone(); + async move { + let d_did = did(d); + let rk = rkey(r); + h.add_edge( + &nsid("sh.tangled.repo.issue"), + &subject, + &at(&format!( + "at://{}/sh.tangled.repo.issue/{}", + d_did.as_ref(), + rk.as_ref() + )), + ); + h.mount( + &d_did, + &nsid("sh.tangled.repo.issue"), + &rk, + issue_body(&repo, &format!("issue-{}", rk.as_ref())), + ) + .await; + } + }) + .await; + + h.warming(1, 5); + let app = router(h.state.clone()); + let (_, page1) = json_response( + app.clone() + .oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[("limit", "1")], + )) + .await + .unwrap(), + ) + .await; + let cursor = page1["cursor"].as_str().unwrap().to_owned(); + + h.promote_ready(2, 9); + let (_, page2) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[("limit", "10"), ("cursor", &cursor)], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(page2["items"].as_array().unwrap().len(), 1); +} + +#[tokio::test] +async fn invalid_cursor_returns_400() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot(list_request( + "sh.tangled.repo.listIssues", + "at://did:plc:abalone", + &[("cursor", "not-a-number")], + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["error"], "InvalidRequest"); +} + +#[tokio::test] +async fn list_follows_subject_is_followee_did() { + let h = Harness::new().await; + let followee = did("did:plc:bailey"); + let subject = at(&format!("at://{}", followee.as_ref())); + h.add_edge( + &nsid("sh.tangled.graph.follow"), + &subject, + &at("at://did:plc:nel/sh.tangled.graph.follow/f1"), + ); + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.graph.follow"), + &rkey("f1"), + follow_body(&followee), + ) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.graph.listFollows", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["value"]["subject"], followee.as_ref()); +} + +#[tokio::test] +async fn upstream_failure_during_hydration_drops_only_that_item() { + let h = Harness::new().await; + let subject = at("at://did:plc:squid"); + let kind = nsid("sh.tangled.repo.issue"); + let repo = did("did:plc:squid"); + h.add_edge( + &kind, + &subject, + &at("at://did:plc:nel/sh.tangled.repo.issue/ok"), + ); + h.add_edge( + &kind, + &subject, + &at("at://did:plc:teq/sh.tangled.repo.issue/flaky"), + ); + h.mount( + &did("did:plc:nel"), + &kind, + &rkey("ok"), + issue_body(&repo, "kelp survey"), + ) + .await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", "did:plc:teq")) + .and(query_param("collection", "sh.tangled.repo.issue")) + .and(query_param("rkey", "flaky")) + .respond_with(ResponseTemplate::new(503)) + .mount(&h.server) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().expect("items array"); + assert_eq!(items.len(), 1, "flaky item dropped, healthy sibling kept"); + assert_eq!( + items[0]["uri"].as_str().unwrap(), + "at://did:plc:nel/sh.tangled.repo.issue/ok", + ); +} + +#[tokio::test] +async fn transient_failure_keeps_edge_so_count_stays_whole() { + let h = Harness::new().await; + let subject = at("at://did:plc:squid"); + let kind = nsid("sh.tangled.repo.issue"); + let repo = did("did:plc:squid"); + h.add_edge( + &kind, + &subject, + &at("at://did:plc:nel/sh.tangled.repo.issue/ok"), + ); + h.add_edge( + &kind, + &subject, + &at("at://did:plc:teq/sh.tangled.repo.issue/flaky"), + ); + h.mount( + &did("did:plc:nel"), + &kind, + &rkey("ok"), + issue_body(&repo, "kelp survey"), + ) + .await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", "did:plc:teq")) + .and(query_param("collection", "sh.tangled.repo.issue")) + .and(query_param("rkey", "flaky")) + .respond_with(ResponseTemplate::new(503)) + .mount(&h.server) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.clone() + .oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["items"].as_array().unwrap().len(), 1); + + let (cstatus, cbody) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.countIssues", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(cstatus, StatusCode::OK); + assert_eq!( + cbody["count"], + json!(2), + "a transient 503 must not evict the edge, count stays whole", + ); +} + +#[tokio::test] +async fn gone_item_is_evicted_so_count_converges_to_list() { + let h = Harness::new().await; + let subject = at("at://did:plc:squid"); + let kind = nsid("sh.tangled.repo.issue"); + let repo = did("did:plc:squid"); + h.add_edge( + &kind, + &subject, + &at("at://did:plc:nel/sh.tangled.repo.issue/ok"), + ); + h.add_edge( + &kind, + &subject, + &at("at://did:plc:teq/sh.tangled.repo.issue/gone"), + ); + h.mount( + &did("did:plc:nel"), + &kind, + &rkey("ok"), + issue_body(&repo, "kelp survey"), + ) + .await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", "did:plc:teq")) + .and(query_param("collection", "sh.tangled.repo.issue")) + .and(query_param("rkey", "gone")) + .respond_with(ResponseTemplate::new(404)) + .mount(&h.server) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.clone() + .oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["items"].as_array().unwrap().len(), 1, "gone item dropped from the page"); + + let (cstatus, cbody) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.countIssues", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(cstatus, StatusCode::OK); + assert_eq!( + cbody["count"], + json!(1), + "a definitive 404 must evict the dead edge so count matches the list", + ); +} + +#[tokio::test] +async fn handle_authority_subject_is_400() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let cases = [ + "sh.tangled.feed.listStars", + "sh.tangled.feed.countStars", + "sh.tangled.graph.listFollows", + "sh.tangled.graph.countFollows", + "sh.tangled.repo.listIssues", + "sh.tangled.repo.countIssues", + "sh.tangled.repo.listPulls", + "sh.tangled.repo.countPulls", + "sh.tangled.repo.issue.listComments", + "sh.tangled.repo.issue.countComments", + ]; + stream::iter(cases) + .for_each(|endpoint| { + let app = app.clone(); + async move { + let resp = app + .oneshot(list_request(endpoint, "at://oyster.cafe", &[])) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{endpoint}"); + assert_eq!(body["error"], "InvalidRequest", "{endpoint}"); + assert!( + body["message"] + .as_str() + .unwrap_or_default() + .contains("did, not a handle"), + "{endpoint}: {}", + body["message"] + ); + } + }) + .await; +} + +#[tokio::test] +async fn empty_subject_is_400() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot(list_request("sh.tangled.repo.listIssues", "", &[])) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["error"], "InvalidRequest"); +} + +#[tokio::test] +async fn limit_below_min_or_above_max_is_400() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let cases = [("0", "below"), ("1001", "above")]; + stream::iter(cases) + .for_each(|(limit, label)| { + let app = app.clone(); + async move { + let resp = app + .oneshot(list_request( + "sh.tangled.repo.listIssues", + "at://did:plc:abalone", + &[("limit", limit)], + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "limit {label}"); + assert_eq!(body["error"], "InvalidRequest", "limit {label}"); + } + }) + .await; +} + +#[tokio::test] +async fn count_after_remove_source_returns_zero() { + let h = Harness::new().await; + let subject = at("at://did:plc:abalone"); + let source = at("at://did:plc:nel/sh.tangled.feed.star/s1"); + h.add_edge(&nsid("sh.tangled.feed.star"), &subject, &source); + h.edges.remove_source(&source); + + let app = router(h.state.clone()); + let (_, body) = json_response( + app.oneshot(list_request( + "sh.tangled.feed.countStars", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(body["count"], json!(0)); + assert_eq!(body["distinctAuthors"], json!(0)); +} + +#[tokio::test] +async fn list_issue_comments_hydrates_end_to_end() { + let h = Harness::new().await; + let issue_uri = at("at://did:plc:abalone/sh.tangled.repo.issue/i1"); + let nel = did("did:plc:nel"); + let rk = rkey("c1"); + h.add_edge( + &nsid("sh.tangled.repo.issue.comment"), + &issue_uri, + &at(&format!( + "at://{}/sh.tangled.repo.issue.comment/{}", + nel.as_ref(), + rk.as_ref() + )), + ); + h.mount( + &nel, + &nsid("sh.tangled.repo.issue.comment"), + &rk, + json!({ + "$type": "sh.tangled.repo.issue.comment", + "issue": issue_uri.as_ref(), + "body": "thoughts", + "createdAt": "2026-05-01T00:00:00Z" + }), + ) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.issue.listComments", + issue_uri.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["value"]["body"], json!("thoughts")); + assert_eq!(items[0]["value"]["issue"], json!(issue_uri.as_ref())); +} + +#[tokio::test] +async fn list_item_cid_is_present() { + let h = Harness::new().await; + let subject = at("at://did:plc:abalone"); + let nel = did("did:plc:nel"); + h.add_edge( + &nsid("sh.tangled.feed.star"), + &subject, + &at(&format!("at://{}/sh.tangled.feed.star/s1", nel.as_ref())), + ); + h.mount( + &nel, + &nsid("sh.tangled.feed.star"), + &rkey("s1"), + star_body(&did("did:plc:abalone")), + ) + .await; + + let app = router(h.state.clone()); + let (_, body) = json_response( + app.oneshot(list_request( + "sh.tangled.feed.listStars", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + let item = &body["items"][0]; + assert!( + item.as_object().unwrap().contains_key("cid"), + "list items must mirror getRecord output shape and include cid" + ); + assert_eq!(item["cid"], json!(CID)); +} + +#[tokio::test] +async fn count_issue_comments_subjects_on_issue_uri() { + let h = Harness::new().await; + let issue_uri = at("at://did:plc:abalone/sh.tangled.repo.issue/i1"); + h.add_edge( + &nsid("sh.tangled.repo.issue.comment"), + &issue_uri, + &at("at://did:plc:nel/sh.tangled.repo.issue.comment/c1"), + ); + h.add_edge( + &nsid("sh.tangled.repo.issue.comment"), + &issue_uri, + &at("at://did:plc:olaren/sh.tangled.repo.issue.comment/c2"), + ); + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.issue.countComments", + issue_uri.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["count"], json!(2)); + assert_eq!(body["distinctAuthors"], json!(2)); +} + +#[tokio::test] +async fn list_item_404_dropped_not_404_for_subject() { + let h = Harness::new().await; + let subject = at("at://did:plc:squid"); + let kind = nsid("sh.tangled.repo.issue"); + let repo = did("did:plc:squid"); + h.add_edge( + &kind, + &subject, + &at("at://did:plc:nel/sh.tangled.repo.issue/live"), + ); + h.add_edge( + &kind, + &subject, + &at("at://did:plc:teq/sh.tangled.repo.issue/missing"), + ); + h.mount( + &did("did:plc:nel"), + &kind, + &rkey("live"), + issue_body(&repo, "kelp survives"), + ) + .await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", "did:plc:teq")) + .and(query_param("collection", "sh.tangled.repo.issue")) + .and(query_param("rkey", "missing")) + .respond_with(ResponseTemplate::new(404).set_body_json(json!({ + "error": "RecordNotFound", + "message": "could not find record" + }))) + .mount(&h.server) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "a stale-index 404 drops that item, it must not 404 or 502 the subject's list", + ); + let items = body["items"].as_array().expect("items array"); + assert_eq!(items.len(), 1, "stale 404 item dropped, live sibling kept"); + assert_eq!( + items[0]["uri"].as_str().unwrap(), + "at://did:plc:nel/sh.tangled.repo.issue/live", + ); +} + +#[tokio::test] +async fn list_item_with_wrong_type_tag_dropped() { + let h = Harness::new().await; + let subject = at("at://did:plc:squid"); + let kind = nsid("sh.tangled.feed.star"); + h.add_edge( + &kind, + &subject, + &at("at://did:plc:nel/sh.tangled.feed.star/good"), + ); + h.add_edge( + &kind, + &subject, + &at("at://did:plc:teq/sh.tangled.feed.star/wrong"), + ); + h.mount( + &did("did:plc:nel"), + &kind, + &rkey("good"), + star_body(&did("did:plc:squid")), + ) + .await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", "did:plc:teq")) + .and(query_param("collection", "sh.tangled.feed.star")) + .and(query_param("rkey", "wrong")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": "at://did:plc:teq/sh.tangled.feed.star/wrong", + "cid": CID, + "value": { + "$type": "sh.tangled.feed.reaction", + "createdAt": "2026-05-01T00:00:00Z", + "subject": "at://did:plc:squid" + } + }))) + .mount(&h.server) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.feed.listStars", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().expect("items array"); + assert_eq!(items.len(), 1, "wrong-type item dropped, valid star kept"); + assert_eq!( + items[0]["uri"].as_str().unwrap(), + "at://did:plc:nel/sh.tangled.feed.star/good", + ); +} + +#[tokio::test] +async fn list_item_with_mismatched_collection_dropped() { + let h = Harness::new().await; + let subject = at("at://did:plc:squid"); + let kind = nsid("sh.tangled.repo.issue"); + let repo = did("did:plc:squid"); + h.add_edge( + &kind, + &subject, + &at("at://did:plc:nel/sh.tangled.repo.issue/live"), + ); + h.add_edge( + &kind, + &subject, + &at("at://did:plc:teq/sh.tangled.feed.star/whelk"), + ); + h.mount( + &did("did:plc:nel"), + &kind, + &rkey("live"), + issue_body(&repo, "kelp survives"), + ) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "a mismatched-collection index edge must not 400 the subject's list", + ); + let items = body["items"].as_array().expect("items array"); + assert_eq!( + items.len(), + 1, + "mismatched-collection edge dropped, live sibling kept" + ); + assert_eq!( + items[0]["uri"].as_str().unwrap(), + "at://did:plc:nel/sh.tangled.repo.issue/live", + ); +} + +#[tokio::test] +async fn bare_did_endpoints_reject_at_uri_subject() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let cases = [ + "sh.tangled.graph.listFollows", + "sh.tangled.graph.countFollows", + ]; + stream::iter(cases) + .for_each(|endpoint| { + let app = app.clone(); + async move { + let resp = app + .oneshot(list_request( + endpoint, + "at://did:plc:abalone/sh.tangled.repo/r1", + &[], + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{endpoint}"); + assert_eq!(body["error"], "InvalidRequest", "{endpoint}"); + assert!( + body["message"] + .as_str() + .unwrap_or_default() + .contains("bare did"), + "{endpoint}: {}", + body["message"], + ); + } + }) + .await; +} + +#[tokio::test] +async fn repo_pointing_endpoints_reject_at_uri_subject() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let cases = [ + "sh.tangled.repo.listIssues", + "sh.tangled.repo.countIssues", + "sh.tangled.repo.listPulls", + "sh.tangled.repo.countPulls", + "sh.tangled.repo.listArtifacts", + "sh.tangled.repo.countArtifacts", + ]; + stream::iter(cases) + .for_each(|endpoint| { + let app = app.clone(); + async move { + let resp = app + .oneshot(list_request( + endpoint, + "at://did:plc:abalone/sh.tangled.repo/r1", + &[], + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "{endpoint} must reject rkey-form subjects since rkeys are unstable; clients must send the repoDID", + ); + assert!( + body["message"] + .as_str() + .unwrap_or_default() + .contains("bare did"), + "{endpoint}: {}", + body["message"], + ); + } + }) + .await; +} + +#[tokio::test] +async fn repo_pointing_endpoints_accept_bare_did() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let cases = [ + "sh.tangled.repo.listIssues", + "sh.tangled.repo.countIssues", + "sh.tangled.repo.listPulls", + "sh.tangled.repo.countPulls", + "sh.tangled.repo.listArtifacts", + "sh.tangled.repo.countArtifacts", + ]; + stream::iter(cases) + .for_each(|endpoint| { + let app = app.clone(); + async move { + let resp = app + .oneshot(list_request(endpoint, "did:plc:abalone", &[])) + .await + .unwrap(); + let (status, _body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK, "{endpoint} must accept bare did"); + } + }) + .await; +} + +#[tokio::test] +async fn issue_collection_endpoints_reject_bare_did_or_wrong_collection() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let endpoints = [ + "sh.tangled.repo.issue.listComments", + "sh.tangled.repo.issue.countComments", + ]; + let inputs = [ + "at://did:plc:abalone", + "at://did:plc:abalone/sh.tangled.repo/r1", + ]; + let cases = endpoints + .iter() + .copied() + .flat_map(|endpoint| inputs.iter().copied().map(move |input| (endpoint, input))); + stream::iter(cases) + .for_each(|(endpoint, input)| { + let app = app.clone(); + async move { + let resp = app + .oneshot(list_request(endpoint, input, &[])) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{endpoint} input={input}"); + assert!( + body["message"] + .as_str() + .unwrap_or_default() + .contains("sh.tangled.repo.issue/"), + "{endpoint} input={input}: {}", + body["message"], + ); + } + }) + .await; +} + +#[tokio::test] +async fn star_endpoints_reject_unrelated_collection() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let endpoints = ["sh.tangled.feed.listStars", "sh.tangled.feed.countStars"]; + stream::iter(endpoints) + .for_each(|endpoint| { + let app = app.clone(); + async move { + let resp = app + .oneshot(list_request( + endpoint, + "at://did:plc:abalone/sh.tangled.knot/k1", + &[], + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{endpoint}"); + let msg = body["message"].as_str().unwrap_or_default(); + assert!(msg.contains("sh.tangled.string"), "{endpoint}: {msg}",); + } + }) + .await; +} + +#[tokio::test] +async fn star_endpoints_reject_repo_uri_subject() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot(list_request( + "sh.tangled.feed.countStars", + "at://did:plc:abalone/sh.tangled.repo/r1", + &[], + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "rkey-form repo URI must be rejected; clients must send the repoDID directly", + ); + let msg = body["message"].as_str().unwrap_or_default(); + assert!(msg.contains("sh.tangled.string"), "{msg}"); +} + +#[tokio::test] +async fn star_endpoints_accept_string_subject_form() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot(list_request( + "sh.tangled.feed.countStars", + "at://did:plc:abalone/sh.tangled.string/k1", + &[], + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["count"], json!(0)); +} + +#[tokio::test] +async fn list_after_remove_source_returns_empty_items() { + let h = Harness::new().await; + let subject = at("at://did:plc:abalone"); + let source = at("at://did:plc:nel/sh.tangled.feed.star/s1"); + h.add_edge(&nsid("sh.tangled.feed.star"), &subject, &source); + h.edges.remove_source(&source); + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.feed.listStars", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["items"], json!([])); + assert!(body["cursor"].is_null()); +} + +#[tokio::test] +async fn list_pulls_hydrates_via_slingshot_when_edges_present() { + let h = Harness::new().await; + let target_did = did("did:plc:abalone"); + let subject = at(&format!("at://{}", target_did.as_ref())); + let source_did = did("did:plc:nel"); + let rk = rkey("p1"); + h.add_edge( + &nsid("sh.tangled.repo.pull"), + &subject, + &at(&format!( + "at://{}/sh.tangled.repo.pull/{}", + source_did.as_ref(), + rk.as_ref() + )), + ); + h.mount( + &source_did, + &nsid("sh.tangled.repo.pull"), + &rk, + json!({ + "$type": "sh.tangled.repo.pull", + "title": "ship it", + "createdAt": "2026-05-01T00:00:00Z", + "rounds": [], + "target": {"repo": target_did.as_ref(), "branch": "main"}, + }), + ) + .await; + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listPulls", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["value"]["title"], json!("ship it")); + assert_eq!( + items[0]["value"]["target"]["repo"], + json!(target_did.as_ref()) + ); +} + +#[tokio::test] +async fn count_pulls_returns_distinct_authors() { + let h = Harness::new().await; + let subject = at("at://did:plc:abalone"); + h.add_edge( + &nsid("sh.tangled.repo.pull"), + &subject, + &at("at://did:plc:nel/sh.tangled.repo.pull/p1"), + ); + h.add_edge( + &nsid("sh.tangled.repo.pull"), + &subject, + &at("at://did:plc:olaren/sh.tangled.repo.pull/p2"), + ); + h.add_edge( + &nsid("sh.tangled.repo.pull"), + &subject, + &at("at://did:plc:nel/sh.tangled.repo.pull/p3"), + ); + let app = router(h.state.clone()); + let (_, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.countPulls", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(body["count"], json!(3)); + assert_eq!(body["distinctAuthors"], json!(2)); +} + +#[tokio::test] +async fn extractor_to_xrpc_round_trip_for_star() { + let h = Harness::new().await; + let subject_did = did("did:plc:abalone"); + let source_did = did("did:plc:nel"); + let rk = rkey("s1"); + let source = at(&format!( + "at://{}/sh.tangled.feed.star/{}", + source_did.as_ref(), + rk.as_ref() + )); + let body = star_body(&subject_did); + let parsed = + bobbin_types::edges::Record::from_json_value(&nsid("sh.tangled.feed.star"), body.clone()) + .expect("parse star record"); + parsed + .extract_edges(&source) + .expect("extract") + .into_iter() + .for_each(|e| h.edges.add(e)); + h.mount(&source_did, &nsid("sh.tangled.feed.star"), &rk, body) + .await; + + let app = router(h.state.clone()); + let (status, json) = json_response( + app.oneshot(list_request( + "sh.tangled.feed.listStars", + &format!("at://{}", subject_did.as_ref()), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "extractor key must match handler subject, body was {json}", + ); + let items = json["items"].as_array().unwrap(); + assert_eq!(items.len(), 1, "expected exactly one star edge"); + assert_eq!( + items[0]["value"]["subject"]["did"], + json!(subject_did.as_ref()) + ); +} + +#[tokio::test] +async fn list_issues_includes_state_comment_count_and_state_updated_at() { + let h = Harness::new().await; + let repo = did("did:plc:limpet"); + let subject = at(&format!("at://{}", repo.as_ref())); + let issue_uri = at("at://did:plc:nel/sh.tangled.repo.issue/i1"); + h.add_edge(&nsid("sh.tangled.repo.issue"), &subject, &issue_uri); + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo.issue"), + &rkey("i1"), + issue_body(&repo, "hi"), + ) + .await; + h.add_edge( + &nsid("sh.tangled.repo.issue.comment"), + &issue_uri, + &at("at://did:plc:olaren/sh.tangled.repo.issue.comment/c1"), + ); + h.add_edge( + &nsid("sh.tangled.repo.issue.comment"), + &issue_uri, + &at("at://did:plc:teq/sh.tangled.repo.issue.comment/c2"), + ); + + h.state.issue_states.upsert( + at("at://did:plc:nel/sh.tangled.repo.issue.state/s1"), + issue_uri.clone(), + 1_777_593_600_000_000, + IssueStateKind::Open, + ); + h.state.issue_states.upsert( + at("at://did:plc:nel/sh.tangled.repo.issue.state/s2"), + issue_uri.clone(), + 1_777_593_700_000_000, + IssueStateKind::Closed, + ); + + let app = router(h.state.clone()); + let resp = app + .oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[], + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + let item = &body["items"][0]; + assert_eq!(item["state"], json!("closed")); + assert_eq!(item["commentCount"], json!(2)); + let updated = item["stateUpdatedAt"] + .as_str() + .expect("stateUpdatedAt must serialize as RFC3339 string"); + assert!( + updated.starts_with("2026-"), + "expected 2026 timestamp, got {updated}" + ); +} + +#[tokio::test] +async fn list_issues_defaults_to_open_when_no_state_record() { + let h = Harness::new().await; + let repo = did("did:plc:limpet"); + let subject = at(&format!("at://{}", repo.as_ref())); + let issue_uri = at("at://did:plc:nel/sh.tangled.repo.issue/i1"); + h.add_edge(&nsid("sh.tangled.repo.issue"), &subject, &issue_uri); + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo.issue"), + &rkey("i1"), + issue_body(&repo, "no state yet"), + ) + .await; + + let app = router(h.state.clone()); + let (_status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + let item = &body["items"][0]; + assert_eq!( + item["state"], + json!("open"), + "absent state record defaults to open" + ); + assert!( + item.get("stateUpdatedAt").is_none(), + "stateUpdatedAt must be absent without a state record", + ); + assert_eq!(item["commentCount"], json!(0)); +} + +#[tokio::test] +async fn list_issues_author_filter_restricts_to_matching_did() { + let h = Harness::new().await; + let repo = did("did:plc:limpet"); + let subject = at(&format!("at://{}", repo.as_ref())); + let owners = [ + ("did:plc:nel", "n1"), + ("did:plc:nel", "n2"), + ("did:plc:olaren", "o1"), + ("did:plc:olaren", "o2"), + ]; + stream::iter(owners) + .for_each(|(d, r)| { + let h = &h; + let subject = subject.clone(); + let repo = repo.clone(); + async move { + let d_did = did(d); + let rk = rkey(r); + h.add_edge( + &nsid("sh.tangled.repo.issue"), + &subject, + &at(&format!( + "at://{}/sh.tangled.repo.issue/{}", + d_did.as_ref(), + rk.as_ref() + )), + ); + h.mount( + &d_did, + &nsid("sh.tangled.repo.issue"), + &rk, + issue_body(&repo, &format!("issue-{}", rk.as_ref())), + ) + .await; + } + }) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[("author", "did:plc:nel")], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().expect("items array"); + assert_eq!(items.len(), 2, "two issues authored by nel"); + let all_nel = items + .iter() + .all(|i| i["uri"].as_str().unwrap().starts_with("at://did:plc:nel/")); + assert!(all_nel, "every returned uri must be authored by nel"); +} + +#[tokio::test] +async fn list_issues_invalid_author_returns_400() { + let h = Harness::new().await; + let subject = "at://did:plc:limpet".to_owned(); + let app = router(h.state.clone()); + let resp = app + .oneshot(list_request( + "sh.tangled.repo.listIssues", + &subject, + &[("author", "not-a-did")], + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn list_pulls_includes_merged_state_and_comment_count() { + let h = Harness::new().await; + let repo = did("did:plc:limpet"); + let subject = at(&format!("at://{}", repo.as_ref())); + let pull_uri = at("at://did:plc:nel/sh.tangled.repo.pull/p1"); + h.add_edge(&nsid("sh.tangled.repo.pull"), &subject, &pull_uri); + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo.pull"), + &rkey("p1"), + pull_body(&repo, "fix bug"), + ) + .await; + h.add_edge( + &nsid("sh.tangled.repo.pull.comment"), + &pull_uri, + &at("at://did:plc:teq/sh.tangled.repo.pull.comment/c1"), + ); + h.state.pull_statuses.upsert( + at("at://did:plc:nel/sh.tangled.repo.pull.status/s1"), + pull_uri.clone(), + 1_777_593_600_000_000, + PullStatusKind::Open, + ); + h.state.pull_statuses.upsert( + at("at://did:plc:nel/sh.tangled.repo.pull.status/s2"), + pull_uri.clone(), + 1_777_593_800_000_000, + PullStatusKind::Merged, + ); + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listPulls", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let item = &body["items"][0]; + assert_eq!(item["state"], json!("merged")); + assert_eq!(item["commentCount"], json!(1)); +} + +#[tokio::test] +async fn list_issues_state_filter_open_includes_records_without_state() { + let h = Harness::new().await; + let repo = did("did:plc:limpet"); + let subject = at(&format!("at://{}", repo.as_ref())); + let issue_uri = at("at://did:plc:nel/sh.tangled.repo.issue/i1"); + h.add_edge(&nsid("sh.tangled.repo.issue"), &subject, &issue_uri); + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo.issue"), + &rkey("i1"), + issue_body(&repo, "fresh"), + ) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[("state", "open")], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().expect("items array"); + assert_eq!( + items.len(), + 1, + "absent state record still matches state=open" + ); +} + +#[tokio::test] +async fn list_issues_state_filter_ignores_third_party_state_source() { + let h = Harness::new().await; + let repo = did("did:plc:limpet"); + let subject = at(&format!("at://{}", repo.as_ref())); + let issue_uri = at("at://did:plc:nel/sh.tangled.repo.issue/i1"); + h.add_edge(&nsid("sh.tangled.repo.issue"), &subject, &issue_uri); + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo.issue"), + &rkey("i1"), + issue_body(&repo, "open issue"), + ) + .await; + h.state.issue_states.upsert( + at("at://did:plc:nautilus/sh.tangled.repo.issue.state/spoof"), + issue_uri.clone(), + 1_777_593_800_000_000, + IssueStateKind::Closed, + ); + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[("state", "open")], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().expect("items array"); + assert_eq!( + items.len(), + 1, + "third-party Closed record must not flip filter result for state=open", + ); + assert_eq!(items[0]["state"], json!("open")); + assert!( + items[0].get("stateUpdatedAt").is_none(), + "third-party state source must not surface stateUpdatedAt", + ); +} + +#[tokio::test] +async fn list_pulls_status_filter_ignores_third_party_status_source() { + let h = Harness::new().await; + let repo = did("did:plc:limpet"); + let subject = at(&format!("at://{}", repo.as_ref())); + let pull_uri = at("at://did:plc:nel/sh.tangled.repo.pull/p1"); + h.add_edge(&nsid("sh.tangled.repo.pull"), &subject, &pull_uri); + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo.pull"), + &rkey("p1"), + pull_body(&repo, "wip"), + ) + .await; + h.state.pull_statuses.upsert( + at("at://did:plc:nautilus/sh.tangled.repo.pull.status/spoof"), + pull_uri.clone(), + 1_777_593_800_000_000, + PullStatusKind::Merged, + ); + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listPulls", + subject.as_ref(), + &[("status", "merged")], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().expect("items array"); + assert_eq!( + items.len(), + 0, + "third-party Merged record must not satisfy status=merged" + ); +} + +#[tokio::test] +async fn list_issues_state_filter_accepts_repo_owner_state_source() { + let h = Harness::new().await; + let repo_owner = did("did:plc:limpet"); + let subject = at(&format!("at://{}", repo_owner.as_ref())); + let issue_uri = at("at://did:plc:nel/sh.tangled.repo.issue/i1"); + h.add_edge(&nsid("sh.tangled.repo.issue"), &subject, &issue_uri); + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo.issue"), + &rkey("i1"), + issue_body(&repo_owner, "owner closed"), + ) + .await; + h.state.issue_states.upsert( + at("at://did:plc:limpet/sh.tangled.repo.issue.state/legit"), + issue_uri.clone(), + 1_777_593_800_000_000, + IssueStateKind::Closed, + ); + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[("state", "closed")], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().expect("items array"); + assert_eq!( + items.len(), + 1, + "repo-owner state record must satisfy state=closed" + ); + assert_eq!(items[0]["state"], json!("closed")); +} + +#[tokio::test] +async fn list_issues_order_asc_returns_oldest_first() { + let h = Harness::new().await; + let repo = did("did:plc:limpet"); + let subject = at(&format!("at://{}", repo.as_ref())); + let rkeys = ["a", "b", "c"]; + stream::iter(rkeys) + .for_each(|r| { + let h = &h; + let subject = subject.clone(); + let repo = repo.clone(); + async move { + let rk = rkey(r); + let issue_uri = at(&format!( + "at://did:plc:nel/sh.tangled.repo.issue/{}", + rk.as_ref() + )); + h.add_edge(&nsid("sh.tangled.repo.issue"), &subject, &issue_uri); + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo.issue"), + &rk, + issue_body(&repo, &format!("issue-{}", rk.as_ref())), + ) + .await; + } + }) + .await; + + let app = router(h.state.clone()); + let (_, asc) = json_response( + app.clone() + .oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[("order", "asc")], + )) + .await + .unwrap(), + ) + .await; + let (_, desc) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listIssues", + subject.as_ref(), + &[("order", "desc")], + )) + .await + .unwrap(), + ) + .await; + let asc_uris: Vec<_> = asc["items"] + .as_array() + .unwrap() + .iter() + .map(|i| i["uri"].as_str().unwrap().to_owned()) + .collect(); + let desc_uris: Vec<_> = desc["items"] + .as_array() + .unwrap() + .iter() + .map(|i| i["uri"].as_str().unwrap().to_owned()) + .collect(); + let mut reversed = asc_uris.clone(); + reversed.reverse(); + assert_eq!(asc_uris.len(), 3); + assert_eq!(desc_uris, reversed, "desc must be exact reverse of asc"); +} + +#[tokio::test] +async fn list_issues_by_state_filter_narrows_results() { + let h = Harness::new().await; + let author = did("did:plc:nel"); + let repo = did("did:plc:limpet"); + let open_uri = at("at://did:plc:nel/sh.tangled.repo.issue/open1"); + let closed_uri = at("at://did:plc:nel/sh.tangled.repo.issue/closed1"); + let author_subject = at(&format!("at://{}", author.as_ref())); + h.edges.add(Edge { + kind: nsid("sh.tangled.repo.issue.by"), + subject: SubjectRef::Did(author.clone()), + source: open_uri.clone(), + sort_micros: next_sort_micros(), + }); + h.edges.add(Edge { + kind: nsid("sh.tangled.repo.issue.by"), + subject: SubjectRef::Did(author.clone()), + source: closed_uri.clone(), + sort_micros: next_sort_micros(), + }); + h.mount( + &author, + &nsid("sh.tangled.repo.issue"), + &rkey("open1"), + issue_body(&repo, "still open"), + ) + .await; + h.mount( + &author, + &nsid("sh.tangled.repo.issue"), + &rkey("closed1"), + issue_body(&repo, "shut"), + ) + .await; + h.state.issue_states.upsert( + at("at://did:plc:nel/sh.tangled.repo.issue.state/s1"), + closed_uri.clone(), + 1_777_593_800_000_000, + IssueStateKind::Closed, + ); + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listIssuesBy", + author_subject.as_ref(), + &[("state", "closed")], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().expect("items array"); + assert_eq!( + items.len(), + 1, + "only the closed issue survives state=closed" + ); + assert_eq!(items[0]["uri"], json!(closed_uri.as_ref())); +} diff --git a/bobbin/crates/xrpc/tests/bulk.rs b/bobbin/crates/xrpc/tests/bulk.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/xrpc/tests/bulk.rs @@ -0,0 +1,486 @@ +use std::sync::Arc; + +use axum::body::{Body, to_bytes}; +use bobbin_edge_index::{CoverageWatch, EdgeStore, StateIndex}; +use bobbin_knot_proxy::{KnotHttpConfig, KnotProxy, KnotProxyConfig}; +use bobbin_record_lru::{CacheCapacity, LruRecordStore}; +use bobbin_resolver::RepoIdResolver; +use bobbin_runtime::{RuntimeHasher, SystemClock}; +use bobbin_search::{DEFAULT_WRITER_HEAP_BYTES, SearchIndex, SearchReader}; +use bobbin_slingshot_client::SlingshotClient; +use bobbin_xrpc::{AppState, router}; +use http::{Request, StatusCode}; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::handle::Handle; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::recordkey::Rkey; +use serde_json::{Value, json}; +use tower::ServiceExt; +use url::Url; +use url::form_urlencoded::byte_serialize; +use wiremock::matchers::{method, path, query_param}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const CID: &str = "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i"; + +fn did(s: &str) -> Did { + Did::new_owned(s).unwrap() +} + +fn rkey(s: &str) -> Rkey { + Rkey::new_owned(s).unwrap() +} + +fn nsid(s: &'static str) -> Nsid { + Nsid::new_static(s).unwrap() +} + +fn handle(s: &str) -> Handle { + Handle::new_owned(s).unwrap() +} + +struct Harness { + server: MockServer, + state: AppState, +} + +impl Harness { + async fn new() -> Self { + let server = MockServer::start().await; + let coverage = Arc::new(CoverageWatch::new()); + let state = AppState::new( + Arc::new(LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024))), + SlingshotClient::with_default_http(Url::parse(&server.uri()).unwrap()).unwrap(), + Arc::new(EdgeStore::new(RuntimeHasher::default())), + Arc::new(StateIndex::new(RuntimeHasher::default())), + Arc::new(StateIndex::new(RuntimeHasher::default())), + coverage, + Arc::new( + KnotProxy::new( + KnotProxyConfig::default(), + KnotHttpConfig::default(), + Arc::new(SystemClock::new()), + RuntimeHasher::default(), + ) + .unwrap(), + ), + Arc::new( + SearchIndex::new(DEFAULT_WRITER_HEAP_BYTES, Arc::new(SystemClock::new())).unwrap(), + ) as Arc, + Arc::new(RepoIdResolver::detached(RuntimeHasher::default())), + ); + Self { server, state } + } + + async fn mount( + &self, + did: &Did, + collection: &Nsid, + rkey: &Rkey, + value: Value, + ) { + let uri = format!( + "at://{}/{}/{}", + did.as_ref(), + collection.as_ref(), + rkey.as_ref() + ); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", did.as_ref())) + .and(query_param("collection", collection.as_ref())) + .and(query_param("rkey", rkey.as_ref())) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": uri, + "cid": CID, + "value": value, + }))) + .mount(&self.server) + .await; + } + + async fn mount_404( + &self, + did: &Did, + collection: &Nsid, + rkey: &Rkey, + ) { + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", did.as_ref())) + .and(query_param("collection", collection.as_ref())) + .and(query_param("rkey", rkey.as_ref())) + .respond_with( + ResponseTemplate::new(404) + .set_body_json(json!({"error": "RecordNotFound", "message": "missing"})), + ) + .mount(&self.server) + .await; + } +} + +fn enc(s: &str) -> String { + byte_serialize(s.as_bytes()).collect() +} + +fn bulk_request(endpoint: &str, key: &str, values: &[&str]) -> Request { + let qs = values + .iter() + .map(|v| format!("{key}={}", enc(v))) + .collect::>() + .join("&"); + Request::builder() + .uri(format!("/xrpc/{endpoint}?{qs}")) + .body(Body::empty()) + .unwrap() +} + +async fn json_response(resp: axum::response::Response) -> (StatusCode, Value) { + let status = resp.status(); + let bytes = to_bytes(resp.into_body(), 1 << 20).await.unwrap(); + let parsed: Value = serde_json::from_slice(&bytes).expect("JSON body"); + (status, parsed) +} + +fn issue_body(repo_did: &Did, title: &str) -> Value { + json!({ + "$type": "sh.tangled.repo.issue", + "repo": repo_did.as_ref(), + "title": title, + "createdAt": "2026-05-01T00:00:00Z" + }) +} + +fn pull_body(target_repo: &Did, title: &str) -> Value { + json!({ + "$type": "sh.tangled.repo.pull", + "title": title, + "createdAt": "2026-05-01T00:00:00Z", + "rounds": [], + "target": { + "branch": "main", + "repo": target_repo.as_ref() + } + }) +} + +fn repo_body(name: &str) -> Value { + json!({ + "$type": "sh.tangled.repo", + "name": name, + "knot": "oyster.cafe", + "createdAt": "2026-05-01T00:00:00Z" + }) +} + +fn profile_body(handle: &Handle) -> Value { + json!({ + "$type": "sh.tangled.actor.profile", + "bluesky": false, + "preferredHandle": handle.as_ref() + }) +} + +#[tokio::test] +async fn get_repos_returns_all_resolved_records() { + let h = Harness::new().await; + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo"), + &rkey("abalone"), + repo_body("abalone"), + ) + .await; + h.mount( + &did("did:plc:teq"), + &nsid("sh.tangled.repo"), + &rkey("limpet"), + repo_body("limpet"), + ) + .await; + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(bulk_request( + "sh.tangled.repo.getRepos", + "repos", + &[ + "at://did:plc:nel/sh.tangled.repo/abalone", + "at://did:plc:teq/sh.tangled.repo/limpet", + ], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 2); + let names: Vec<&str> = items + .iter() + .map(|v| v["value"]["name"].as_str().unwrap()) + .collect(); + assert!(names.contains(&"abalone")); + assert!(names.contains(&"limpet")); +} + +#[tokio::test] +async fn get_profiles_returns_all_resolved_profiles() { + let h = Harness::new().await; + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.actor.profile"), + &rkey("self"), + profile_body(&handle("witchcraft.systems")), + ) + .await; + h.mount( + &did("did:plc:teq"), + &nsid("sh.tangled.actor.profile"), + &rkey("self"), + profile_body(&handle("olaren.dev")), + ) + .await; + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(bulk_request( + "sh.tangled.actor.getProfiles", + "actors", + &[ + "at://did:plc:nel/sh.tangled.actor.profile/self", + "at://did:plc:teq/sh.tangled.actor.profile/self", + ], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 2); +} + +#[tokio::test] +async fn get_issues_returns_all_resolved_issues() { + let h = Harness::new().await; + let repo = did("did:plc:abalone"); + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo.issue"), + &rkey("i1"), + issue_body(&repo, "first"), + ) + .await; + h.mount( + &did("did:plc:olaren"), + &nsid("sh.tangled.repo.issue"), + &rkey("i2"), + issue_body(&repo, "second"), + ) + .await; + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(bulk_request( + "sh.tangled.repo.getIssues", + "issues", + &[ + "at://did:plc:nel/sh.tangled.repo.issue/i1", + "at://did:plc:olaren/sh.tangled.repo.issue/i2", + ], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 2); +} + +#[tokio::test] +async fn get_pulls_returns_all_resolved_pulls() { + let h = Harness::new().await; + let target = did("did:plc:abalone"); + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo.pull"), + &rkey("p1"), + pull_body(&target, "patch one"), + ) + .await; + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(bulk_request( + "sh.tangled.repo.getPulls", + "pulls", + &["at://did:plc:nel/sh.tangled.repo.pull/p1"], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["value"]["title"], json!("patch one")); +} + +#[tokio::test] +async fn missing_records_are_dropped_silently() { + let h = Harness::new().await; + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo"), + &rkey("abalone"), + repo_body("abalone"), + ) + .await; + h.mount_404( + &did("did:plc:teq"), + &nsid("sh.tangled.repo"), + &rkey("ghost"), + ) + .await; + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(bulk_request( + "sh.tangled.repo.getRepos", + "repos", + &[ + "at://did:plc:nel/sh.tangled.repo/abalone", + "at://did:plc:teq/sh.tangled.repo/ghost", + ], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!( + items.len(), + 1, + "missing records must be dropped not fail the bulk call" + ); + assert_eq!(items[0]["value"]["name"], json!("abalone")); +} + +#[tokio::test] +async fn transient_failure_drops_only_that_record() { + let h = Harness::new().await; + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo"), + &rkey("conch"), + repo_body("conch"), + ) + .await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", "did:plc:teq")) + .and(query_param("collection", "sh.tangled.repo")) + .and(query_param("rkey", "flaky")) + .respond_with(ResponseTemplate::new(503)) + .mount(&h.server) + .await; + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(bulk_request( + "sh.tangled.repo.getRepos", + "repos", + &[ + "at://did:plc:nel/sh.tangled.repo/conch", + "at://did:plc:teq/sh.tangled.repo/flaky", + ], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!( + items.len(), + 1, + "transient upstream failure drops that record, it must not fail the bulk call" + ); + assert_eq!(items[0]["value"]["name"], json!("conch")); +} + +#[tokio::test] +async fn wrong_collection_uri_fails_bulk_request() { + let h = Harness::new().await; + h.mount( + &did("did:plc:nel"), + &nsid("sh.tangled.repo"), + &rkey("conch"), + repo_body("conch"), + ) + .await; + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(bulk_request( + "sh.tangled.repo.getRepos", + "repos", + &[ + "at://did:plc:nel/sh.tangled.repo/conch", + "at://did:plc:teq/sh.tangled.repo.issue/whelk", + ], + )) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "a wrong-collection uri must fail the whole bulk request" + ); + assert_eq!(body["error"], "InvalidRequest"); +} + +#[tokio::test] +async fn empty_uri_list_is_rejected() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot( + Request::builder() + .uri("/xrpc/sh.tangled.repo.getRepos") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn over_limit_uri_list_is_rejected() { + let h = Harness::new().await; + let uris: Vec = (0..51) + .map(|i| format!("at://did:plc:nel/sh.tangled.repo/r{i}")) + .collect(); + let refs: Vec<&str> = uris.iter().map(|s| s.as_str()).collect(); + let app = router(h.state.clone()); + let resp = app + .oneshot(bulk_request("sh.tangled.repo.getRepos", "repos", &refs)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn malformed_uri_in_list_returns_400() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot(bulk_request( + "sh.tangled.repo.getRepos", + "repos", + &["not-a-uri"], + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} diff --git a/bobbin/crates/xrpc/tests/cold_start.rs b/bobbin/crates/xrpc/tests/cold_start.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/xrpc/tests/cold_start.rs @@ -0,0 +1,639 @@ +use std::sync::Arc; + +use axum::body::{Body, to_bytes}; +use bobbin_edge_index::{CoverageWatch, EdgeStore, StateIndex}; +use bobbin_knot_proxy::{KnotHttpConfig, KnotProxy, KnotProxyConfig}; +use bobbin_record_lru::{CacheCapacity, LruRecordStore}; +use bobbin_resolver::RepoIdResolver; +use bobbin_runtime::{RuntimeHasher, SystemClock}; +use bobbin_search::{DEFAULT_WRITER_HEAP_BYTES, SearchIndex, SearchReader}; +use bobbin_slingshot_client::SlingshotClient; +use bobbin_xrpc::{AppState, router}; +use futures::stream::{self, StreamExt}; +use http::{Request, StatusCode}; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::recordkey::Rkey; +use serde_json::{Value, json}; +use tower::ServiceExt; +use url::Url; +use url::form_urlencoded::byte_serialize; +use wiremock::matchers::{method, path, query_param}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const CID: &str = "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i"; + +fn did(s: &str) -> Did { + Did::new_owned(s).unwrap() +} + +fn rkey(s: &str) -> Rkey { + Rkey::new_owned(s).unwrap() +} + +fn nsid(s: &'static str) -> Nsid { + Nsid::new_static(s).unwrap() +} + +async fn fresh_app(server_uri: &Url) -> AppState { + AppState::new( + Arc::new(LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024))), + SlingshotClient::with_default_http(server_uri.clone()).unwrap(), + Arc::new(EdgeStore::new(RuntimeHasher::default())), + Arc::new(StateIndex::new(RuntimeHasher::default())), + Arc::new(StateIndex::new(RuntimeHasher::default())), + Arc::new(CoverageWatch::new()), + Arc::new( + KnotProxy::new( + KnotProxyConfig::default(), + KnotHttpConfig::default(), + Arc::new(SystemClock::new()), + RuntimeHasher::default(), + ) + .unwrap(), + ), + Arc::new(SearchIndex::new(DEFAULT_WRITER_HEAP_BYTES, Arc::new(SystemClock::new())).unwrap()) + as Arc, + Arc::new(RepoIdResolver::detached(RuntimeHasher::default())), + ) +} + +async fn mount_record( + server: &MockServer, + did: &Did, + collection: &Nsid, + rkey: &Rkey, + value: Value, +) { + let uri = format!( + "at://{}/{}/{}", + did.as_ref(), + collection.as_ref(), + rkey.as_ref() + ); + let body = json!({ "uri": uri, "cid": CID, "value": value }); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", did.as_ref())) + .and(query_param("collection", collection.as_ref())) + .and(query_param("rkey", rkey.as_ref())) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .mount(server) + .await; +} + +fn xrpc_request(endpoint: &str, param: &str, value: &str) -> Request { + let encoded: String = byte_serialize(value.as_bytes()).collect(); + Request::builder() + .uri(format!("/xrpc/{endpoint}?{param}={encoded}")) + .body(Body::empty()) + .unwrap() +} + +async fn json_response(resp: axum::response::Response) -> (StatusCode, Value) { + let status = resp.status(); + let bytes = to_bytes(resp.into_body(), 1 << 20).await.unwrap(); + let parsed: Value = serde_json::from_slice(&bytes).expect("response is JSON"); + (status, parsed) +} + +#[tokio::test] +async fn cold_start_serves_all_four_point_lookups() { + let server = MockServer::start().await; + let clam = did("did:plc:clam"); + + mount_record( + &server, + &clam, + &nsid("sh.tangled.repo"), + &rkey("r1"), + json!({ + "$type": "sh.tangled.repo", + "name": "clam", + "knot": "oyster.cafe", + "createdAt": "2026-05-01T00:00:00Z" + }), + ) + .await; + + mount_record( + &server, + &clam, + &nsid("sh.tangled.actor.profile"), + &rkey("self"), + json!({ + "$type": "sh.tangled.actor.profile", + "bluesky": false, + "description": "clam shell" + }), + ) + .await; + + mount_record( + &server, + &clam, + &nsid("sh.tangled.repo.issue"), + &rkey("i1"), + json!({ + "$type": "sh.tangled.repo.issue", + "repo": "did:plc:limpet", + "title": "broken", + "createdAt": "2026-05-01T00:00:00Z" + }), + ) + .await; + + mount_record( + &server, + &clam, + &nsid("sh.tangled.repo.pull"), + &rkey("p1"), + json!({ + "$type": "sh.tangled.repo.pull", + "title": "ship", + "createdAt": "2026-05-01T00:00:00Z", + "rounds": [], + "target": {"repo": "did:plc:limpet", "branch": "main"} + }), + ) + .await; + + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + + let cases = [ + ( + "sh.tangled.repo.getRepo", + "repo", + format!("at://{}/sh.tangled.repo/r1", clam.as_ref()), + "knot", + json!("oyster.cafe"), + ), + ( + "sh.tangled.actor.getProfile", + "actor", + format!("at://{}/sh.tangled.actor.profile/self", clam.as_ref()), + "description", + json!("clam shell"), + ), + ( + "sh.tangled.repo.getIssue", + "issue", + format!("at://{}/sh.tangled.repo.issue/i1", clam.as_ref()), + "title", + json!("broken"), + ), + ( + "sh.tangled.repo.getPull", + "pull", + format!("at://{}/sh.tangled.repo.pull/p1", clam.as_ref()), + "title", + json!("ship"), + ), + ]; + + stream::iter(cases) + .for_each(|(endpoint, param, at_uri, field, expected)| { + let app = app.clone(); + async move { + let resp = app + .oneshot(xrpc_request(endpoint, param, &at_uri)) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK, "{endpoint} status"); + assert_eq!(body["uri"], at_uri, "{endpoint} uri"); + assert_eq!(body["cid"], CID, "{endpoint} cid"); + assert_eq!( + body["value"][field], expected, + "{endpoint} body field {field}" + ); + } + }) + .await; +} + +#[tokio::test] +async fn second_call_is_served_from_lru() { + let server = MockServer::start().await; + let uni = did("did:plc:uni"); + let mock = Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", uni.as_ref())) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": format!("at://{}/sh.tangled.repo/r1", uni.as_ref()), + "cid": CID, + "value": { + "$type": "sh.tangled.repo", + "name": "uni", + "knot": "witchcraft.systems", + "createdAt": "2026-05-01T00:00:00Z" + } + }))) + .expect(1) + .mount_as_scoped(&server) + .await; + + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let req_uri = format!("at://{}/sh.tangled.repo/r1", uni.as_ref()); + + stream::iter(0..3) + .for_each(|_| { + let app = app.clone(); + let req_uri = req_uri.clone(); + async move { + let resp = app + .oneshot(xrpc_request("sh.tangled.repo.getRepo", "repo", &req_uri)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + } + }) + .await; + + drop(mock); +} + +#[tokio::test] +async fn collection_mismatch_is_400() { + let server = MockServer::start().await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let resp = app + .oneshot(xrpc_request( + "sh.tangled.repo.getRepo", + "repo", + "at://did:plc:clam/sh.tangled.actor.profile/self", + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn handle_authority_is_400() { + let server = MockServer::start().await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let resp = app + .oneshot(xrpc_request( + "sh.tangled.repo.getRepo", + "repo", + "at://witchcraft.systems/sh.tangled.repo/r1", + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn slingshot_404_propagates_as_404() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let resp = app + .oneshot(xrpc_request( + "sh.tangled.repo.getRepo", + "repo", + "at://did:plc:clam/sh.tangled.repo/missing", + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn wrong_record_type_is_502() { + let server = MockServer::start().await; + mount_record( + &server, + &did("did:plc:clam"), + &nsid("sh.tangled.repo"), + &rkey("r1"), + json!({ + "$type": "sh.tangled.knot", + "knot": "oyster.cafe", + "createdAt": "2026-05-01T00:00:00Z" + }), + ) + .await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let resp = app + .oneshot(xrpc_request( + "sh.tangled.repo.getRepo", + "repo", + "at://did:plc:clam/sh.tangled.repo/r1", + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert_eq!(body["error"], "InvalidRecord"); +} + +#[tokio::test] +async fn wrong_type_does_not_poison_cache() { + let server = MockServer::start().await; + let mock = Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", "did:plc:clam")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": "at://did:plc:clam/sh.tangled.repo/r1", + "cid": CID, + "value": { + "$type": "sh.tangled.knot", + "knot": "oyster.cafe", + "createdAt": "2026-05-01T00:00:00Z" + } + }))) + .expect(2) + .mount_as_scoped(&server) + .await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let req = || { + xrpc_request( + "sh.tangled.repo.getRepo", + "repo", + "at://did:plc:clam/sh.tangled.repo/r1", + ) + }; + let first = app.clone().oneshot(req()).await.unwrap(); + assert_eq!(first.status(), StatusCode::BAD_GATEWAY); + let second = app.clone().oneshot(req()).await.unwrap(); + assert_eq!(second.status(), StatusCode::BAD_GATEWAY); + drop(mock); +} + +#[tokio::test] +async fn profile_with_empty_preferred_handle_is_tolerated() { + let server = MockServer::start().await; + let nel = did("did:plc:nel"); + mount_record( + &server, + &nel, + &nsid("sh.tangled.actor.profile"), + &rkey("self"), + json!({ + "$type": "sh.tangled.actor.profile", + "bluesky": true, + "preferredHandle": "", + "description": "empty handle, valid profile" + }), + ) + .await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let at_uri = format!("at://{}/sh.tangled.actor.profile/self", nel.as_ref()); + let resp = app + .oneshot(xrpc_request( + "sh.tangled.actor.getProfile", + "actor", + &at_uri, + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK, "status: {body}"); + assert_eq!(body["uri"], at_uri); + assert_eq!(body["value"]["description"], "empty handle, valid profile"); + assert!(body["value"]["preferredHandle"].is_null()); +} + +#[tokio::test] +async fn missing_uri_param_returns_json_envelope() { + let server = MockServer::start().await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let req = Request::builder() + .uri("/xrpc/sh.tangled.repo.getRepo") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["error"], "InvalidRequest"); + assert!(body["message"].is_string()); +} + +#[tokio::test] +async fn malformed_at_uri_returns_400_envelope() { + let server = MockServer::start().await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let resp = app + .oneshot(xrpc_request( + "sh.tangled.repo.getRepo", + "repo", + "definitely-not-an-at-uri", + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["error"], "InvalidRequest"); +} + +#[tokio::test] +async fn upstream_uri_mismatch_routes_to_invalid_record() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", "did:plc:clam")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": "at://did:plc:limpet/sh.tangled.repo/elsewhere", + "cid": CID, + "value": { + "$type": "sh.tangled.repo", + "knot": "oyster.cafe", + "createdAt": "2026-05-01T00:00:00Z" + } + }))) + .mount(&server) + .await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let resp = app + .oneshot(xrpc_request( + "sh.tangled.repo.getRepo", + "repo", + "at://did:plc:clam/sh.tangled.repo/r1", + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert_eq!(body["error"], "InvalidRecord"); +} + +#[tokio::test] +async fn upstream_garbage_cid_routes_to_invalid_record() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": "at://did:plc:clam/sh.tangled.repo/r1", + "cid": "not-a-real-cid", + "value": { + "$type": "sh.tangled.repo", + "knot": "oyster.cafe", + "createdAt": "2026-05-01T00:00:00Z" + } + }))) + .mount(&server) + .await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let resp = app + .oneshot(xrpc_request( + "sh.tangled.repo.getRepo", + "repo", + "at://did:plc:clam/sh.tangled.repo/r1", + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert_eq!(body["error"], "InvalidRecord"); +} + +#[tokio::test] +async fn oversize_upstream_body_routes_to_upstream_failed() { + let server = MockServer::start().await; + let payload = vec![b'x'; 8 * 1024 * 1024]; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/json") + .set_body_bytes(payload), + ) + .mount(&server) + .await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let resp = app + .oneshot(xrpc_request( + "sh.tangled.repo.getRepo", + "repo", + "at://did:plc:clam/sh.tangled.repo/r1", + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert_eq!(body["error"], "UpstreamFailed"); +} + +#[tokio::test] +async fn upstream_503_routes_to_upstream_failed() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let resp = app + .oneshot(xrpc_request( + "sh.tangled.repo.getRepo", + "repo", + "at://did:plc:clam/sh.tangled.repo/r1", + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert_eq!(body["error"], "UpstreamFailed"); +} + +#[tokio::test] +async fn get_repo_by_repo_did_returns_observed_record() { + let server = MockServer::start().await; + let owner_did = did("did:plc:scallop"); + let rk = rkey("r1"); + let repo_did = did("did:plc:limpet"); + mount_record( + &server, + &owner_did, + &nsid("sh.tangled.repo"), + &rk, + json!({ + "$type": "sh.tangled.repo", + "name": "scallop", + "knot": "oyster.cafe", + "createdAt": "2026-05-01T00:00:00Z", + "repoDid": repo_did.as_ref(), + }), + ) + .await; + + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + state + .resolver + .observe(owner_did.clone(), rk.clone(), Some(repo_did.clone())) + .await; + + let app = router(state); + let resp = app + .oneshot(xrpc_request( + "sh.tangled.repo.getRepoByRepoDid", + "repoDid", + repo_did.as_ref(), + )) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + body["uri"], + format!( + "at://{}/sh.tangled.repo/{}", + owner_did.as_ref(), + rk.as_ref() + ) + ); + assert_eq!(body["value"]["name"], "scallop"); + assert_eq!(body["value"]["repoDid"], repo_did.as_ref()); +} + +#[tokio::test] +async fn get_repo_by_repo_did_404_when_unobserved() { + let server = MockServer::start().await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let resp = app + .oneshot(xrpc_request( + "sh.tangled.repo.getRepoByRepoDid", + "repoDid", + "did:plc:whelk", + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn get_repo_by_repo_did_400_on_invalid_did() { + let server = MockServer::start().await; + let state = fresh_app(&Url::parse(&server.uri()).unwrap()).await; + let app = router(state); + let resp = app + .oneshot(xrpc_request( + "sh.tangled.repo.getRepoByRepoDid", + "repoDid", + "not-a-did", + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} diff --git a/bobbin/crates/xrpc/tests/coverage.rs b/bobbin/crates/xrpc/tests/coverage.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/xrpc/tests/coverage.rs @@ -0,0 +1,126 @@ +use std::sync::Arc; + +use axum::body::{Body, to_bytes}; +use bobbin_edge_index::{Coverage, CoverageWatch, EdgeStore, HydrantCursor, StateIndex}; +use bobbin_knot_proxy::{KnotHttpConfig, KnotProxy, KnotProxyConfig}; +use bobbin_record_lru::{CacheCapacity, LruRecordStore}; +use bobbin_resolver::RepoIdResolver; +use bobbin_runtime::{RuntimeHasher, SystemClock}; +use bobbin_search::{DEFAULT_WRITER_HEAP_BYTES, SearchIndex, SearchReader}; +use bobbin_slingshot_client::SlingshotClient; +use bobbin_xrpc::{AppState, router}; +use http::{Request, StatusCode}; +use serde_json::{Value, json}; +use tower::ServiceExt; +use url::Url; +use wiremock::MockServer; + +struct Harness { + coverage: Arc, + state: AppState, +} + +impl Harness { + async fn new() -> Self { + let server = MockServer::start().await; + let coverage = Arc::new(CoverageWatch::new()); + let state = AppState::new( + Arc::new(LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024))), + SlingshotClient::with_default_http(Url::parse(&server.uri()).unwrap()).unwrap(), + Arc::new(EdgeStore::new(RuntimeHasher::default())), + Arc::new(StateIndex::new(RuntimeHasher::default())), + Arc::new(StateIndex::new(RuntimeHasher::default())), + coverage.clone(), + Arc::new( + KnotProxy::new( + KnotProxyConfig::default(), + KnotHttpConfig::default(), + Arc::new(SystemClock::new()), + RuntimeHasher::default(), + ) + .unwrap(), + ), + Arc::new( + SearchIndex::new(DEFAULT_WRITER_HEAP_BYTES, Arc::new(SystemClock::new())).unwrap(), + ) as Arc, + Arc::new(RepoIdResolver::detached(RuntimeHasher::default())), + ); + Self { coverage, state } + } +} + +fn coverage_request() -> Request { + Request::builder() + .uri("/xrpc/sh.tangled.bobbin.getCoverage") + .body(Body::empty()) + .unwrap() +} + +async fn json_response(resp: axum::response::Response) -> (StatusCode, Value) { + let status = resp.status(); + let bytes = to_bytes(resp.into_body(), 1 << 20).await.unwrap(); + let parsed: Value = serde_json::from_slice(&bytes).expect("JSON body"); + (status, parsed) +} + +#[tokio::test] +async fn defaults_to_warming_at_zero() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let (status, body) = json_response(app.oneshot(coverage_request()).await.unwrap()).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["ready"], json!(false)); + assert_eq!(body["eventsProcessed"], json!(0)); + assert_eq!(body["lastCursor"], json!(0)); +} + +#[tokio::test] +async fn reflects_warming_state() { + let h = Harness::new().await; + h.coverage.update(|_| Coverage::Warming { + events_processed: 7, + last_cursor: HydrantCursor::new(21), + }); + let app = router(h.state.clone()); + let (status, body) = json_response(app.oneshot(coverage_request()).await.unwrap()).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["ready"], json!(false)); + assert_eq!(body["eventsProcessed"], json!(7)); + assert_eq!(body["lastCursor"], json!(21)); +} + +#[tokio::test] +async fn reflects_ready_state() { + let h = Harness::new().await; + h.coverage.update(|_| Coverage::Ready { + events_processed: 99, + last_cursor: HydrantCursor::new(5000), + }); + let app = router(h.state.clone()); + let (status, body) = json_response(app.oneshot(coverage_request()).await.unwrap()).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["ready"], json!(true)); + assert_eq!(body["eventsProcessed"], json!(99)); + assert_eq!(body["lastCursor"], json!(5000)); +} + +#[tokio::test] +async fn promotion_flips_ready_field() { + let h = Harness::new().await; + let app = router(h.state.clone()); + + h.coverage.update(|_| Coverage::Warming { + events_processed: 1, + last_cursor: HydrantCursor::new(5), + }); + let (_, before) = json_response(app.clone().oneshot(coverage_request()).await.unwrap()).await; + assert_eq!(before["ready"], json!(false)); + + h.coverage.update(|_| Coverage::Ready { + events_processed: 2, + last_cursor: HydrantCursor::new(9), + }); + let (_, after) = json_response(app.oneshot(coverage_request()).await.unwrap()).await; + assert_eq!(after["ready"], json!(true)); + assert_eq!(after["lastCursor"], json!(9)); +} diff --git a/bobbin/crates/xrpc/tests/extended.rs b/bobbin/crates/xrpc/tests/extended.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/xrpc/tests/extended.rs @@ -0,0 +1,873 @@ +use std::sync::Arc; + +use axum::body::{Body, to_bytes}; +use bobbin_edge_index::{CoverageWatch, EdgeStore, StateIndex}; +use bobbin_knot_proxy::{KnotHttpConfig, KnotProxy, KnotProxyConfig}; +use bobbin_record_lru::{CacheCapacity, LruRecordStore}; +use bobbin_resolver::RepoIdResolver; +use bobbin_runtime::{RuntimeHasher, SystemClock}; +use bobbin_search::{DEFAULT_WRITER_HEAP_BYTES, SearchIndex, SearchReader}; +use bobbin_slingshot_client::SlingshotClient; +use bobbin_types::edges::Edge; +use bobbin_types::ids::SubjectRef; +use bobbin_xrpc::{AppState, router}; +use http::{Request, StatusCode}; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::recordkey::Rkey; +use jacquard_common::types::string::AtUri; +use serde_json::{Value, json}; +use tower::ServiceExt; +use url::Url; +use url::form_urlencoded::byte_serialize; +use wiremock::matchers::{method, path, query_param}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const CID: &str = "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i"; +const TAG_BYTES: &str = "AAAAAAAAAAAAAAAAAAAAAAAAAAA="; +const ARTIFACT_LINK: &str = "bafkreigh2akiscaildc7gnvtklbsfhdgwz72eolmpckbqr5ej26byp3uli"; + +fn at(s: &str) -> AtUri { + AtUri::new_owned(s).unwrap() +} + +fn did(s: &str) -> Did { + Did::new_owned(s).unwrap() +} + +fn rkey(s: &str) -> Rkey { + Rkey::new_owned(s).unwrap() +} + +fn nsid(s: &'static str) -> Nsid { + Nsid::new_static(s).unwrap() +} + +fn subj(s: &str) -> SubjectRef { + Did::::new_owned(s) + .map(SubjectRef::Did) + .unwrap_or_else(|_| SubjectRef::Uri(AtUri::new_owned(s).unwrap())) +} + +struct Harness { + server: MockServer, + edges: Arc, + state: AppState, +} + +impl Harness { + async fn new() -> Self { + let server = MockServer::start().await; + let edges = Arc::new(EdgeStore::new(RuntimeHasher::default())); + let coverage = Arc::new(CoverageWatch::new()); + let state = AppState::new( + Arc::new(LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024))), + SlingshotClient::with_default_http(Url::parse(&server.uri()).unwrap()).unwrap(), + edges.clone(), + Arc::new(StateIndex::new(RuntimeHasher::default())), + Arc::new(StateIndex::new(RuntimeHasher::default())), + coverage, + Arc::new( + KnotProxy::new( + KnotProxyConfig::default(), + KnotHttpConfig::default(), + Arc::new(SystemClock::new()), + RuntimeHasher::default(), + ) + .unwrap(), + ), + Arc::new( + SearchIndex::new(DEFAULT_WRITER_HEAP_BYTES, Arc::new(SystemClock::new())).unwrap(), + ) as Arc, + Arc::new(RepoIdResolver::detached(RuntimeHasher::default())), + ); + Self { + server, + edges, + state, + } + } + + fn add_edge( + &self, + kind: &Nsid, + subject: &AtUri, + source: &AtUri, + ) { + static EDGE_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + self.edges.add(Edge { + kind: kind.clone(), + subject: subj(subject.as_ref()), + source: source.clone(), + sort_micros: EDGE_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed), + }); + } + + async fn mount( + &self, + did: &Did, + collection: &Nsid, + rkey: &Rkey, + value: Value, + ) { + let uri = format!( + "at://{}/{}/{}", + did.as_ref(), + collection.as_ref(), + rkey.as_ref() + ); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", did.as_ref())) + .and(query_param("collection", collection.as_ref())) + .and(query_param("rkey", rkey.as_ref())) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": uri, + "cid": CID, + "value": value, + }))) + .mount(&self.server) + .await; + } +} + +fn list_request(endpoint: &str, subject: &str, extras: &[(&str, &str)]) -> Request { + let mut qs = format!("subject={}", encode(subject)); + extras.iter().for_each(|(k, v)| { + qs.push('&'); + qs.push_str(k); + qs.push('='); + qs.push_str(&encode(v)); + }); + Request::builder() + .uri(format!("/xrpc/{endpoint}?{qs}")) + .body(Body::empty()) + .unwrap() +} + +fn encode(s: &str) -> String { + byte_serialize(s.as_bytes()).collect() +} + +async fn json_response(resp: axum::response::Response) -> (StatusCode, Value) { + let status = resp.status(); + let bytes = to_bytes(resp.into_body(), 1 << 20).await.unwrap(); + let parsed: Value = serde_json::from_slice(&bytes).expect("JSON body"); + (status, parsed) +} + +fn label_definition_body(name: &str) -> Value { + json!({ + "$type": "sh.tangled.label.definition", + "createdAt": "2026-05-01T00:00:00Z", + "name": name, + "scope": ["sh.tangled.repo.issue"], + "valueType": {"type": "boolean", "format": "any"} + }) +} + +fn label_op_body(subject: &AtUri, def_uri: &AtUri, value: &str) -> Value { + json!({ + "$type": "sh.tangled.label.op", + "performedAt": "2026-05-01T00:00:00Z", + "subject": subject.as_ref(), + "add": [{"key": def_uri.as_ref(), "value": value}], + "delete": [] + }) +} + +fn pipeline_body(repo_did: &Did) -> Value { + json!({ + "$type": "sh.tangled.pipeline", + "workflows": [], + "triggerMetadata": { + "kind": "manual", + "repo": { + "did": "did:plc:teq", + "repoDid": repo_did.as_ref(), + "knot": "nel.pet", + "defaultBranch": "main" + } + } + }) +} + +fn pipeline_body_owner_only(owner_did: &Did) -> Value { + json!({ + "$type": "sh.tangled.pipeline", + "workflows": [], + "triggerMetadata": { + "kind": "manual", + "repo": { + "did": owner_did.as_ref(), + "knot": "nel.pet", + "defaultBranch": "main" + } + } + }) +} + +fn pipeline_status_body(pipeline_uri: &AtUri) -> Value { + json!({ + "$type": "sh.tangled.pipeline.status", + "createdAt": "2026-05-01T00:00:00Z", + "pipeline": pipeline_uri.as_ref(), + "workflow": pipeline_uri.as_ref(), + "status": "success" + }) +} + +fn artifact_body(repo_did: &Did, name: &str) -> Value { + json!({ + "$type": "sh.tangled.repo.artifact", + "createdAt": "2026-05-01T00:00:00Z", + "name": name, + "repoDid": repo_did.as_ref(), + "tag": {"$bytes": TAG_BYTES}, + "artifact": { + "$type": "blob", + "ref": {"$link": ARTIFACT_LINK}, + "mimeType": "application/octet-stream", + "size": 12 + } + }) +} + +fn knot_member_body(subject_did: &Did) -> Value { + json!({ + "$type": "sh.tangled.knot.member", + "createdAt": "2026-05-01T00:00:00Z", + "subject": subject_did.as_ref(), + "domain": "oyster.cafe" + }) +} + +fn spindle_member_body(subject_did: &Did) -> Value { + json!({ + "$type": "sh.tangled.spindle.member", + "createdAt": "2026-05-01T00:00:00Z", + "subject": subject_did.as_ref(), + "instance": "spin.nel.pet" + }) +} + +fn string_body(filename: &str, contents: &str) -> Value { + json!({ + "$type": "sh.tangled.string", + "createdAt": "2026-05-01T00:00:00Z", + "filename": filename, + "description": "test fixture", + "contents": contents + }) +} + +#[tokio::test] +async fn list_label_definitions_keys_on_owner_did() { + let h = Harness::new().await; + let owner = did("did:plc:abalone"); + let rk = rkey("bug"); + let source = at(&format!( + "at://{}/sh.tangled.label.definition/{}", + owner.as_ref(), + rk.as_ref() + )); + h.add_edge( + &nsid("sh.tangled.label.definition"), + &at(&format!("at://{}", owner.as_ref())), + &source, + ); + h.mount( + &owner, + &nsid("sh.tangled.label.definition"), + &rk, + label_definition_body("bug"), + ) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.label.listDefinitions", + &format!("at://{}", owner.as_ref()), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["value"]["name"], json!("bug")); + assert_eq!( + items[0]["value"]["scope"][0], + json!("sh.tangled.repo.issue") + ); +} + +#[tokio::test] +async fn count_label_definitions_dedupes_per_author() { + let h = Harness::new().await; + let owner = did("did:plc:abalone"); + let subject = at(&format!("at://{}", owner.as_ref())); + h.add_edge( + &nsid("sh.tangled.label.definition"), + &subject, + &at(&format!( + "at://{}/sh.tangled.label.definition/bug", + owner.as_ref() + )), + ); + h.add_edge( + &nsid("sh.tangled.label.definition"), + &subject, + &at(&format!( + "at://{}/sh.tangled.label.definition/wontfix", + owner.as_ref() + )), + ); + + let app = router(h.state.clone()); + let (_, body) = json_response( + app.oneshot(list_request( + "sh.tangled.label.countDefinitions", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(body["count"], json!(2)); + assert_eq!(body["distinctAuthors"], json!(1)); +} + +#[tokio::test] +async fn list_label_ops_accepts_issue_subject() { + let h = Harness::new().await; + let issue_uri = at("at://did:plc:abalone/sh.tangled.repo.issue/i1"); + let author = did("did:plc:nel"); + let rk = rkey("op1"); + let def_uri = at("at://did:plc:abalone/sh.tangled.label.definition/bug"); + h.add_edge( + &nsid("sh.tangled.label.op"), + &issue_uri, + &at(&format!( + "at://{}/sh.tangled.label.op/{}", + author.as_ref(), + rk.as_ref() + )), + ); + h.mount( + &author, + &nsid("sh.tangled.label.op"), + &rk, + label_op_body(&issue_uri, &def_uri, "true"), + ) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.label.listOps", + issue_uri.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["value"]["subject"], json!(issue_uri.as_ref())); + assert_eq!(items[0]["value"]["add"][0]["key"], json!(def_uri.as_ref())); +} + +#[tokio::test] +async fn list_label_ops_pull_subject_round_trip() { + let h = Harness::new().await; + let pull_uri = at("at://did:plc:abalone/sh.tangled.repo.pull/p1"); + let author = did("did:plc:bailey"); + let rk = rkey("op1"); + let def_uri = at("at://did:plc:abalone/sh.tangled.label.definition/wontfix"); + let source = at(&format!( + "at://{}/sh.tangled.label.op/{}", + author.as_ref(), + rk.as_ref() + )); + let body = label_op_body(&pull_uri, &def_uri, "true"); + let parsed = + bobbin_types::edges::Record::from_json_value(&nsid("sh.tangled.label.op"), body.clone()) + .expect("parse label.op record"); + parsed + .extract_edges(&source) + .expect("extract") + .into_iter() + .for_each(|e| h.edges.add(e)); + h.mount(&author, &nsid("sh.tangled.label.op"), &rk, body) + .await; + + let app = router(h.state.clone()); + let (status, json) = json_response( + app.oneshot(list_request( + "sh.tangled.label.listOps", + pull_uri.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = json["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["value"]["subject"], json!(pull_uri.as_ref())); + assert_eq!(items[0]["value"]["add"][0]["key"], json!(def_uri.as_ref())); +} + +#[tokio::test] +async fn list_label_ops_rejects_bare_did_subject() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.label.listOps", + "at://did:plc:abalone", + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + let msg = body["message"].as_str().unwrap_or_default(); + assert!( + msg.contains("sh.tangled.repo.issue") && msg.contains("sh.tangled.repo.pull"), + "message must list allowed collections, got {msg}" + ); +} + +#[tokio::test] +async fn list_label_ops_rejects_unrelated_collection() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let (status, _) = json_response( + app.oneshot(list_request( + "sh.tangled.label.listOps", + "at://did:plc:abalone/sh.tangled.repo/r1", + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn list_pipelines_keys_on_repo_did() { + let h = Harness::new().await; + let repo_did = did("did:plc:abalone"); + let subject = at(&format!("at://{}", repo_did.as_ref())); + let spindle_did = did("did:plc:lyna"); + let rk = rkey("pl1"); + h.add_edge( + &nsid("sh.tangled.pipeline"), + &subject, + &at(&format!( + "at://{}/sh.tangled.pipeline/{}", + spindle_did.as_ref(), + rk.as_ref() + )), + ); + h.mount( + &spindle_did, + &nsid("sh.tangled.pipeline"), + &rk, + pipeline_body(&repo_did), + ) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.pipeline.listPipelines", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!( + items[0]["value"]["triggerMetadata"]["repo"]["repoDid"], + json!(repo_did.as_ref()) + ); +} + +#[tokio::test] +async fn count_pipelines_returns_zero_when_no_edges() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let (_, body) = json_response( + app.oneshot(list_request( + "sh.tangled.pipeline.countPipelines", + "at://did:plc:abalone", + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(body["count"], json!(0)); +} + +#[tokio::test] +async fn list_pipeline_statuses_keys_on_pipeline_uri() { + let h = Harness::new().await; + let pipeline_uri = at("at://did:plc:lyna/sh.tangled.pipeline/pl1"); + let author = did("did:plc:bailey"); + let rk = rkey("s1"); + h.add_edge( + &nsid("sh.tangled.pipeline.status"), + &pipeline_uri, + &at(&format!( + "at://{}/sh.tangled.pipeline.status/{}", + author.as_ref(), + rk.as_ref() + )), + ); + h.mount( + &author, + &nsid("sh.tangled.pipeline.status"), + &rk, + pipeline_status_body(&pipeline_uri), + ) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.pipeline.listStatuses", + pipeline_uri.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["value"]["pipeline"], json!(pipeline_uri.as_ref())); + assert_eq!(items[0]["value"]["status"], json!("success")); +} + +#[tokio::test] +async fn pipeline_status_endpoint_rejects_bare_did_subject() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.pipeline.listStatuses", + "at://did:plc:lyna", + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!( + body["message"] + .as_str() + .unwrap_or_default() + .contains("sh.tangled.pipeline/"), + "{body}" + ); +} + +#[tokio::test] +async fn list_artifacts_keys_on_repo_did() { + let h = Harness::new().await; + let repo_did = did("did:plc:abalone"); + let subject = at(&format!("at://{}", repo_did.as_ref())); + let owner = did("did:plc:nel"); + let rk = rkey("a1"); + h.add_edge( + &nsid("sh.tangled.repo.artifact"), + &subject, + &at(&format!( + "at://{}/sh.tangled.repo.artifact/{}", + owner.as_ref(), + rk.as_ref() + )), + ); + h.mount( + &owner, + &nsid("sh.tangled.repo.artifact"), + &rk, + artifact_body(&repo_did, "out.bin"), + ) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.repo.listArtifacts", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["value"]["name"], json!("out.bin")); + assert_eq!(items[0]["value"]["repoDid"], json!(repo_did.as_ref())); +} + +#[tokio::test] +async fn list_knot_members_keys_on_subject_did() { + let h = Harness::new().await; + let subject_did = did("did:plc:nel"); + let subject = at(&format!("at://{}", subject_did.as_ref())); + let admin = did("did:plc:teq"); + let rk = rkey("m1"); + h.add_edge( + &nsid("sh.tangled.knot.member"), + &subject, + &at(&format!( + "at://{}/sh.tangled.knot.member/{}", + admin.as_ref(), + rk.as_ref() + )), + ); + h.mount( + &admin, + &nsid("sh.tangled.knot.member"), + &rk, + knot_member_body(&subject_did), + ) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.knot.listMembers", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["value"]["subject"], json!(subject_did.as_ref())); + assert_eq!(items[0]["value"]["domain"], json!("oyster.cafe")); +} + +#[tokio::test] +async fn list_spindle_members_keys_on_subject_did() { + let h = Harness::new().await; + let subject_did = did("did:plc:olaren"); + let subject = at(&format!("at://{}", subject_did.as_ref())); + let admin = did("did:plc:teq"); + let rk = rkey("m1"); + h.add_edge( + &nsid("sh.tangled.spindle.member"), + &subject, + &at(&format!( + "at://{}/sh.tangled.spindle.member/{}", + admin.as_ref(), + rk.as_ref() + )), + ); + h.mount( + &admin, + &nsid("sh.tangled.spindle.member"), + &rk, + spindle_member_body(&subject_did), + ) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.spindle.listMembers", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["value"]["subject"], json!(subject_did.as_ref())); + assert_eq!(items[0]["value"]["instance"], json!("spin.nel.pet")); +} + +#[tokio::test] +async fn list_strings_keys_on_owner_did() { + let h = Harness::new().await; + let owner = did("did:plc:abalone"); + let subject = at(&format!("at://{}", owner.as_ref())); + let rk = rkey("k1"); + h.add_edge( + &nsid("sh.tangled.string"), + &subject, + &at(&format!( + "at://{}/sh.tangled.string/{}", + owner.as_ref(), + rk.as_ref() + )), + ); + h.mount( + &owner, + &nsid("sh.tangled.string"), + &rk, + string_body("snippet.rs", "fn main() {}"), + ) + .await; + + let app = router(h.state.clone()); + let (status, body) = json_response( + app.oneshot(list_request( + "sh.tangled.string.listStrings", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["value"]["filename"], json!("snippet.rs")); + assert_eq!(items[0]["value"]["contents"], json!("fn main() {}")); +} + +#[tokio::test] +async fn count_strings_dedupes_per_owner() { + let h = Harness::new().await; + let owner = did("did:plc:abalone"); + let subject = at(&format!("at://{}", owner.as_ref())); + ["k1", "k2", "k3"].iter().for_each(|r| { + h.add_edge( + &nsid("sh.tangled.string"), + &subject, + &at(&format!("at://{}/sh.tangled.string/{}", owner.as_ref(), r)), + ); + }); + + let app = router(h.state.clone()); + let (_, body) = json_response( + app.oneshot(list_request( + "sh.tangled.string.countStrings", + subject.as_ref(), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(body["count"], json!(3)); + assert_eq!(body["distinctAuthors"], json!(1)); +} + +#[tokio::test] +async fn extractor_to_xrpc_round_trip_for_pipeline() { + let h = Harness::new().await; + let repo_did = did("did:plc:abalone"); + let spindle_did = did("did:plc:lyna"); + let rk = rkey("pl1"); + let source = at(&format!( + "at://{}/sh.tangled.pipeline/{}", + spindle_did.as_ref(), + rk.as_ref() + )); + let body = pipeline_body(&repo_did); + let parsed = + bobbin_types::edges::Record::from_json_value(&nsid("sh.tangled.pipeline"), body.clone()) + .expect("parse pipeline record"); + parsed + .extract_edges(&source) + .expect("extract") + .into_iter() + .for_each(|e| h.edges.add(e)); + h.mount(&spindle_did, &nsid("sh.tangled.pipeline"), &rk, body) + .await; + + let app = router(h.state.clone()); + let (status, json) = json_response( + app.oneshot(list_request( + "sh.tangled.pipeline.listPipelines", + &format!("at://{}", repo_did.as_ref()), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "extractor key must match handler subject, body was {json}" + ); + let items = json["items"].as_array().unwrap(); + assert_eq!(items.len(), 1, "expected exactly one pipeline edge"); +} + +#[tokio::test] +async fn list_pipelines_drops_records_without_resolvable_repo_did() { + let h = Harness::new().await; + let owner_did = did("did:plc:nel"); + let spindle_did = did("did:plc:lyna"); + let rk = rkey("pl1"); + let source = at(&format!( + "at://{}/sh.tangled.pipeline/{}", + spindle_did.as_ref(), + rk.as_ref() + )); + let body = pipeline_body_owner_only(&owner_did); + let parsed = + bobbin_types::edges::Record::from_json_value(&nsid("sh.tangled.pipeline"), body.clone()) + .expect("parse pipeline record"); + parsed + .extract_edges(&source) + .expect("extract") + .into_iter() + .for_each(|e| h.edges.add(e)); + h.mount(&spindle_did, &nsid("sh.tangled.pipeline"), &rk, body) + .await; + + let app = router(h.state.clone()); + let (status, json) = json_response( + app.oneshot(list_request( + "sh.tangled.pipeline.listPipelines", + &format!("at://{}", owner_did.as_ref()), + &[], + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK, "body was {json}"); + let items = json["items"].as_array().unwrap(); + assert!( + items.is_empty(), + "pipeline without resolvable repoDid must be dropped, got {json}" + ); +} diff --git a/bobbin/crates/xrpc/tests/knot_proxy.rs b/bobbin/crates/xrpc/tests/knot_proxy.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/xrpc/tests/knot_proxy.rs @@ -0,0 +1,982 @@ +use std::sync::Arc; +use std::time::Duration; + +use axum::body::{Body, to_bytes}; +use bobbin_edge_index::{CoverageWatch, EdgeStore, StateIndex}; +use bobbin_knot_proxy::{FailureThreshold, KnotHttpConfig, KnotProxy, KnotProxyConfig}; +use bobbin_record_lru::{CacheCapacity, LruRecordStore}; +use bobbin_resolver::RepoIdResolver; +use bobbin_runtime::{RuntimeHasher, SystemClock}; +use bobbin_search::{DEFAULT_WRITER_HEAP_BYTES, SearchIndex, SearchReader}; +use bobbin_slingshot_client::SlingshotClient; +use bobbin_xrpc::{AppState, router}; +use http::{Request, StatusCode}; +use jacquard_common::DefaultStr; +use jacquard_common::types::did::Did; +use jacquard_common::types::recordkey::Rkey; +use serde_json::{Value, json}; +use tower::ServiceExt; +use url::Url; +use url::form_urlencoded::byte_serialize; +use wiremock::matchers::{header_exists, method, path, query_param}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const CID: &str = "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i"; + +fn did(s: &str) -> Did { + Did::new_owned(s).unwrap() +} + +fn rkey(s: &str) -> Rkey { + Rkey::new_owned(s).unwrap() +} + +fn test_config() -> KnotProxyConfig { + KnotProxyConfig { + failure_threshold: FailureThreshold::new(2).unwrap(), + cooldown: Duration::from_millis(80), + allow_private_hosts: true, + require_https: false, + } +} + +fn test_http_config() -> KnotHttpConfig { + KnotHttpConfig { + connect_timeout: Duration::from_millis(500), + read_timeout: Duration::from_secs(2), + } +} + +struct Harness { + slingshot: MockServer, + knot: MockServer, + state: AppState, +} + +impl Harness { + async fn new() -> Self { + Self::with_config(test_config()).await + } + + async fn with_config(config: KnotProxyConfig) -> Self { + let slingshot_server = MockServer::start().await; + let knot_server = MockServer::start().await; + let state = AppState::new( + Arc::new(LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024))), + SlingshotClient::with_default_http(Url::parse(&slingshot_server.uri()).unwrap()) + .unwrap(), + Arc::new(EdgeStore::new(RuntimeHasher::default())), + Arc::new(StateIndex::new(RuntimeHasher::default())), + Arc::new(StateIndex::new(RuntimeHasher::default())), + Arc::new(CoverageWatch::new()), + Arc::new( + KnotProxy::new( + config, + test_http_config(), + Arc::new(SystemClock::new()), + RuntimeHasher::default(), + ) + .unwrap(), + ), + Arc::new( + SearchIndex::new(DEFAULT_WRITER_HEAP_BYTES, Arc::new(SystemClock::new())).unwrap(), + ) as Arc, + Arc::new(RepoIdResolver::detached(RuntimeHasher::default())), + ); + Self { + slingshot: slingshot_server, + knot: knot_server, + state, + } + } + + async fn mount_repo_record(&self, did: &Did, rkey: &Rkey, name: &str) { + self.mount_repo_record_inner(did, rkey, Some(name)).await; + } + + async fn mount_repo_record_rkey_as_name(&self, did: &Did, rkey: &Rkey) { + self.mount_repo_record_inner(did, rkey, None).await; + } + + async fn mount_repo_record_inner( + &self, + did: &Did, + rkey: &Rkey, + name: Option<&str>, + ) { + let knot_value = self.knot.uri(); + let mut record = json!({ + "$type": "sh.tangled.repo", + "createdAt": "2026-05-01T00:00:00Z", + "knot": knot_value, + }); + if let Some(n) = name { + record["name"] = json!(n); + } + let uri = format!("at://{}/sh.tangled.repo/{}", did.as_ref(), rkey.as_ref()); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", did.as_ref())) + .and(query_param("collection", "sh.tangled.repo")) + .and(query_param("rkey", rkey.as_ref())) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": uri, + "cid": CID, + "value": record, + }))) + .mount(&self.slingshot) + .await; + } + + async fn call(&self, path_and_query: &str) -> http::Response { + self.call_with_headers(path_and_query, &[]).await + } + + async fn call_with_headers( + &self, + path_and_query: &str, + client_headers: &[(&str, &str)], + ) -> http::Response { + let builder = client_headers + .iter() + .fold(Request::builder().uri(path_and_query), |b, (k, v)| { + b.header(*k, *v) + }); + router(self.state.clone()) + .oneshot(builder.body(Body::empty()).unwrap()) + .await + .expect("router infallible") + } +} + +fn enc(s: &str) -> String { + byte_serialize(s.as_bytes()).collect() +} + +async fn body_string(resp: http::Response) -> String { + let body = to_bytes(resp.into_body(), 64 * 1024).await.unwrap(); + String::from_utf8(body.to_vec()).expect("response body is utf-8") +} + +async fn body_value(resp: http::Response) -> Value { + let s = body_string(resp).await; + serde_json::from_str(&s).unwrap_or_else(|e| panic!("body not json: {e}: {s}")) +} + +#[tokio::test] +async fn proxies_repo_blob_with_did_slash_name_repo_param() { + let h = Harness::new().await; + let tid = "3jzfcijpj2z2a"; + h.mount_repo_record(&did("did:plc:abalone"), &rkey(tid), "barnacle") + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .and(query_param("repo", "did:plc:abalone/barnacle")) + .and(query_param("ref", "main")) + .and(query_param("path", "README.md")) + .respond_with( + ResponseTemplate::new(200) + .set_body_raw(r#"{"path":"README.md","content":"hi"}"#, "application/json"), + ) + .mount(&h.knot) + .await; + + let target = format!( + "/xrpc/sh.tangled.repo.blob?repo={}&ref=main&path=README.md", + enc(&format!("at://did:plc:abalone/sh.tangled.repo/{tid}")), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers().get("content-type").unwrap(), + "application/json", + ); + let v = body_value(resp).await; + assert_eq!(v["path"], "README.md"); + assert_eq!(v["content"], "hi"); +} + +#[tokio::test] +async fn modern_rkey_as_name_uses_rkey_even_when_name_field_set() { + let h = Harness::new().await; + h.mount_repo_record(&did("did:plc:abalone"), &rkey("core"), "Tangled Core") + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.getDefaultBranch")) + .and(query_param("repo", "did:plc:abalone/core")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "hash": "abc", + "name": "main", + "when": "2026-05-01T00:00:00Z", + }))) + .mount(&h.knot) + .await; + + let target = format!( + "/xrpc/sh.tangled.repo.getDefaultBranch?repo={}", + enc("at://did:plc:abalone/sh.tangled.repo/core"), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::OK); + let v = body_value(resp).await; + assert_eq!(v["name"], "main"); +} + +#[tokio::test] +async fn modern_rkey_as_name_works_when_name_field_null() { + let h = Harness::new().await; + h.mount_repo_record_rkey_as_name(&did("did:plc:abalone"), &rkey("core")) + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.getDefaultBranch")) + .and(query_param("repo", "did:plc:abalone/core")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "hash": "abc", + "name": "main", + "when": "2026-05-01T00:00:00Z", + }))) + .mount(&h.knot) + .await; + + let target = format!( + "/xrpc/sh.tangled.repo.getDefaultBranch?repo={}", + enc("at://did:plc:abalone/sh.tangled.repo/core"), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::OK); +} + +#[tokio::test] +async fn legacy_tid_rkey_falls_back_to_name_field() { + let h = Harness::new().await; + let tid_rkey = "3jzfcijpj2z2a"; + h.mount_repo_record(&did("did:plc:abalone"), &rkey(tid_rkey), "dotfiles") + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.getDefaultBranch")) + .and(query_param("repo", "did:plc:abalone/dotfiles")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "hash": "abc", + "name": "main", + "when": "2026-05-01T00:00:00Z", + }))) + .mount(&h.knot) + .await; + + let target = format!( + "/xrpc/sh.tangled.repo.getDefaultBranch?repo={}", + enc(&format!("at://did:plc:abalone/sh.tangled.repo/{tid_rkey}")), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::OK); +} + +#[tokio::test] +async fn tid_rkey_without_name_falls_back_to_tid() { + let h = Harness::new().await; + let tid_rkey = "3jzfcijpj2z2a"; + h.mount_repo_record_rkey_as_name(&did("did:plc:abalone"), &rkey(tid_rkey)) + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.getDefaultBranch")) + .and(query_param("repo", format!("did:plc:abalone/{tid_rkey}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "hash": "abc", + "name": "main", + "when": "2026-05-01T00:00:00Z", + }))) + .mount(&h.knot) + .await; + let target = format!( + "/xrpc/sh.tangled.repo.getDefaultBranch?repo={}", + enc(&format!("at://did:plc:abalone/sh.tangled.repo/{tid_rkey}")), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::OK); +} + +#[tokio::test] +async fn streams_binary_archive_through_proxy() { + let h = Harness::new().await; + let tid = "3jzfcijpj2z2b"; + h.mount_repo_record(&did("did:plc:limpet"), &rkey(tid), "kelp") + .await; + let payload: Vec = (0u8..=255).collect(); + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.archive")) + .and(query_param("repo", "did:plc:limpet/kelp")) + .and(query_param("ref", "v1")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/gzip") + .set_body_bytes(payload.clone()), + ) + .mount(&h.knot) + .await; + + let target = format!( + "/xrpc/sh.tangled.repo.archive?repo={}&ref=v1", + enc(&format!("at://did:plc:limpet/sh.tangled.repo/{tid}")), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers().get("content-type").unwrap(), + "application/gzip", + ); + let body = to_bytes(resp.into_body(), 4 * 1024).await.unwrap(); + assert_eq!(body.as_ref(), payload.as_slice()); +} + +#[tokio::test] +async fn missing_repo_param_returns_400() { + let h = Harness::new().await; + let resp = h.call("/xrpc/sh.tangled.repo.blob?ref=main").await; + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let v = body_value(resp).await; + assert_eq!(v["error"], "InvalidRequest"); +} + +#[tokio::test] +async fn unknown_repo_propagates_404_from_slingshot() { + let h = Harness::new().await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .respond_with(ResponseTemplate::new(404).set_body_string("not found")) + .mount(&h.slingshot) + .await; + let target = format!( + "/xrpc/sh.tangled.repo.blob?repo={}&ref=main", + enc("at://did:plc:abalone/sh.tangled.repo/missing"), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + let v = body_value(resp).await; + assert_eq!(v["error"], "RecordNotFound"); +} + +#[tokio::test] +async fn knot_5xx_routes_to_upstream_failed() { + let h = Harness::new().await; + h.mount_repo_record(&did("did:plc:abalone"), &rkey("r1"), "barnacle") + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with(ResponseTemplate::new(503)) + .mount(&h.knot) + .await; + let target = format!( + "/xrpc/sh.tangled.repo.blob?repo={}&ref=main&path=x", + enc("at://did:plc:abalone/sh.tangled.repo/r1"), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::BAD_GATEWAY); + let v = body_value(resp).await; + assert_eq!(v["error"], "UpstreamFailed"); +} + +#[tokio::test] +async fn knot_4xx_passes_through_unchanged() { + let h = Harness::new().await; + h.mount_repo_record(&did("did:plc:abalone"), &rkey("r1"), "barnacle") + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with(ResponseTemplate::new(404).set_body_raw( + r#"{"error":"FileNotFound","message":"nope"}"#, + "application/json", + )) + .mount(&h.knot) + .await; + let target = format!( + "/xrpc/sh.tangled.repo.blob?repo={}&ref=main&path=missing", + enc("at://did:plc:abalone/sh.tangled.repo/r1"), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + let v = body_value(resp).await; + assert_eq!(v["error"], "FileNotFound"); +} + +#[tokio::test] +async fn breaker_opens_after_threshold_then_short_circuits() { + let h = Harness::new().await; + h.mount_repo_record(&did("did:plc:abalone"), &rkey("r1"), "barnacle") + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with(ResponseTemplate::new(503)) + .mount(&h.knot) + .await; + let target = format!( + "/xrpc/sh.tangled.repo.blob?repo={}&ref=main&path=x", + enc("at://did:plc:abalone/sh.tangled.repo/r1"), + ); + let r1 = h.call(&target).await; + assert_eq!(r1.status(), StatusCode::BAD_GATEWAY); + let _ = body_string(r1).await; + let r2 = h.call(&target).await; + assert_eq!(r2.status(), StatusCode::BAD_GATEWAY); + let _ = body_string(r2).await; + let r3 = h.call(&target).await; + assert_eq!(r3.status(), StatusCode::BAD_GATEWAY); + let v = body_value(r3).await; + assert!( + v["message"] + .as_str() + .unwrap_or_default() + .contains("circuit breaker open"), + "third call must be short-circuited by breaker, got {v}", + ); +} + +#[tokio::test] +async fn proxy_owner_uses_knot_query_param() { + let h = Harness::new().await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.owner")) + .respond_with( + ResponseTemplate::new(200) + .set_body_raw(r#"{"owner":"did:plc:nautilus"}"#, "application/json"), + ) + .mount(&h.knot) + .await; + let target = format!("/xrpc/sh.tangled.owner?knot={}", enc(&h.knot.uri())); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::OK); + let v = body_value(resp).await; + assert_eq!(v["owner"], "did:plc:nautilus"); +} + +#[tokio::test] +async fn proxy_knot_version_uses_knot_query_param() { + let h = Harness::new().await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.knot.version")) + .respond_with( + ResponseTemplate::new(200).set_body_raw(r#"{"version":"0.42"}"#, "application/json"), + ) + .mount(&h.knot) + .await; + let target = format!("/xrpc/sh.tangled.knot.version?knot={}", enc(&h.knot.uri())); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::OK); + let v = body_value(resp).await; + assert_eq!(v["version"], "0.42"); +} + +#[tokio::test] +async fn proxy_knot_list_keys_forwards_pagination_params() { + let h = Harness::new().await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.knot.listKeys")) + .and(query_param("limit", "5")) + .and(query_param("cursor", "abc")) + .respond_with(ResponseTemplate::new(200).set_body_raw(r#"{"keys":[]}"#, "application/json")) + .mount(&h.knot) + .await; + let target = format!( + "/xrpc/sh.tangled.knot.listKeys?knot={}&limit=5&cursor=abc", + enc(&h.knot.uri()), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::OK); +} + +#[tokio::test] +async fn missing_knot_param_on_knot_route_returns_400() { + let h = Harness::new().await; + let resp = h.call("/xrpc/sh.tangled.knot.version").await; + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let v = body_value(resp).await; + assert_eq!(v["error"], "InvalidRequest"); +} + +#[tokio::test] +async fn second_proxy_call_skips_slingshot_via_lru() { + let h = Harness::new().await; + let tid = "3jzfcijpj2z2c"; + h.mount_repo_record(&did("did:plc:abalone"), &rkey(tid), "barnacle") + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.tree")) + .and(query_param("repo", "did:plc:abalone/barnacle")) + .and(query_param("ref", "main")) + .respond_with( + ResponseTemplate::new(200) + .set_body_raw(r#"{"ref":"main","files":[]}"#, "application/json"), + ) + .mount(&h.knot) + .await; + let target = format!( + "/xrpc/sh.tangled.repo.tree?repo={}&ref=main", + enc(&format!("at://did:plc:abalone/sh.tangled.repo/{tid}")), + ); + let r1 = h.call(&target).await; + assert_eq!(r1.status(), StatusCode::OK); + let _ = body_string(r1).await; + let r2 = h.call(&target).await; + assert_eq!(r2.status(), StatusCode::OK); + let _ = body_string(r2).await; + let received = h.slingshot.received_requests().await.unwrap(); + let getrecord = received + .iter() + .filter(|r| r.url.path() == "/xrpc/com.atproto.repo.getRecord") + .count(); + assert_eq!( + getrecord, 1, + "slingshot must be hit exactly once because the LRU serves the second proxy call", + ); +} + +#[tokio::test] +async fn does_not_inject_auth_or_atproto_proxy_headers() { + let h = Harness::new().await; + h.mount_repo_record(&did("did:plc:abalone"), &rkey("r1"), "barnacle") + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .and(header_exists("user-agent")) + .respond_with( + ResponseTemplate::new(200).set_body_raw(r#"{"path":"x"}"#, "application/json"), + ) + .mount(&h.knot) + .await; + let target = format!( + "/xrpc/sh.tangled.repo.blob?repo={}&ref=main&path=x", + enc("at://did:plc:abalone/sh.tangled.repo/r1"), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::OK); + let received = h.knot.received_requests().await.unwrap(); + let knot_call = received + .iter() + .find(|r| r.url.path() == "/xrpc/sh.tangled.repo.blob") + .expect("knot received the proxied call"); + assert!( + knot_call.headers.get("authorization").is_none(), + "bobbin must not inject auth, anonymous read by design", + ); + assert!( + knot_call.headers.get("atproto-proxy").is_none(), + "bobbin is not an atproto-proxy chain", + ); + assert!( + knot_call.headers.get("atproto-accept-labelers").is_none(), + "bobbin does not negotiate labelers with knots", + ); +} + +#[tokio::test] +async fn forwards_range_and_conditional_request_headers() { + let h = Harness::new().await; + let tid = "3jzfcijpj2z2d"; + h.mount_repo_record(&did("did:plc:limpet"), &rkey(tid), "kelp") + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.archive")) + .and(query_param("repo", "did:plc:limpet/kelp")) + .respond_with( + ResponseTemplate::new(206) + .insert_header("content-type", "application/octet-stream") + .insert_header("content-range", "bytes 0-99/2048") + .insert_header("accept-ranges", "bytes") + .insert_header("etag", "\"v1\"") + .set_body_bytes(vec![0u8; 100]), + ) + .mount(&h.knot) + .await; + + let target = format!( + "/xrpc/sh.tangled.repo.archive?repo={}&ref=v1", + enc(&format!("at://did:plc:limpet/sh.tangled.repo/{tid}")), + ); + let resp = h + .call_with_headers( + &target, + &[ + ("range", "bytes=0-99"), + ("if-none-match", "\"old\""), + ("if-modified-since", "Wed, 01 May 2026 00:00:00 GMT"), + ], + ) + .await; + assert_eq!(resp.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!( + resp.headers().get("content-range").unwrap(), + "bytes 0-99/2048" + ); + assert_eq!(resp.headers().get("accept-ranges").unwrap(), "bytes"); + assert_eq!(resp.headers().get("etag").unwrap(), "\"v1\""); + + let received = h.knot.received_requests().await.unwrap(); + let knot_call = received + .iter() + .find(|r| r.url.path() == "/xrpc/sh.tangled.repo.archive") + .expect("knot received the proxied call"); + assert_eq!(knot_call.headers.get("range").unwrap(), "bytes=0-99"); + assert_eq!(knot_call.headers.get("if-none-match").unwrap(), "\"old\""); + assert_eq!( + knot_call.headers.get("if-modified-since").unwrap(), + "Wed, 01 May 2026 00:00:00 GMT", + ); +} + +#[tokio::test] +async fn drops_disallowed_client_headers() { + let h = Harness::new().await; + h.mount_repo_record(&did("did:plc:abalone"), &rkey("r1"), "barnacle") + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with( + ResponseTemplate::new(200).set_body_raw(r#"{"path":"x"}"#, "application/json"), + ) + .mount(&h.knot) + .await; + let target = format!( + "/xrpc/sh.tangled.repo.blob?repo={}&path=x", + enc("at://did:plc:abalone/sh.tangled.repo/r1"), + ); + let resp = h + .call_with_headers( + &target, + &[ + ("authorization", "Bearer secret"), + ("cookie", "sid=evil"), + ("x-custom", "should-not-pass"), + ], + ) + .await; + assert_eq!(resp.status(), StatusCode::OK); + let received = h.knot.received_requests().await.unwrap(); + let knot_call = received + .iter() + .find(|r| r.url.path() == "/xrpc/sh.tangled.repo.blob") + .expect("knot received the proxied call"); + assert!(knot_call.headers.get("authorization").is_none()); + assert!(knot_call.headers.get("cookie").is_none()); + assert!(knot_call.headers.get("x-custom").is_none()); +} + +#[tokio::test] +async fn rejects_client_supplied_loopback_under_strict_config() { + let strict = KnotProxyConfig { + allow_private_hosts: false, + ..test_config() + }; + let h = Harness::with_config(strict).await; + let resp = h + .call(&format!( + "/xrpc/sh.tangled.knot.version?knot={}", + enc("http://127.0.0.1:9"), + )) + .await; + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let v = body_value(resp).await; + assert_eq!(v["error"], "InvalidRequest"); + let msg = v["message"].as_str().unwrap_or_default().to_owned(); + assert!( + msg.contains("loopback") || msg.contains("blocked"), + "message should explain block reason, got {msg}", + ); +} + +#[tokio::test] +async fn rejects_client_supplied_link_local_metadata_endpoint() { + let strict = KnotProxyConfig { + allow_private_hosts: false, + ..test_config() + }; + let h = Harness::with_config(strict).await; + let resp = h + .call(&format!( + "/xrpc/sh.tangled.knot.version?knot={}", + enc("http://169.254.169.254"), + )) + .await; + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let v = body_value(resp).await; + assert_eq!(v["error"], "InvalidRequest"); +} + +#[tokio::test] +async fn record_with_private_knot_returns_invalid_record() { + let strict = KnotProxyConfig { + allow_private_hosts: false, + ..test_config() + }; + let h = Harness::with_config(strict).await; + let owner = did("did:plc:abalone"); + let rk = rkey("r1"); + let record = json!({ + "$type": "sh.tangled.repo", + "createdAt": "2026-05-01T00:00:00Z", + "knot": "http://10.0.0.5:3000", + "name": "barnacle", + }); + let uri = format!("at://{}/sh.tangled.repo/{}", owner.as_ref(), rk.as_ref()); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", owner.as_ref())) + .and(query_param("collection", "sh.tangled.repo")) + .and(query_param("rkey", rk.as_ref())) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": uri, + "cid": CID, + "value": record, + }))) + .mount(&h.slingshot) + .await; + let resp = h + .call(&format!( + "/xrpc/sh.tangled.repo.blob?repo={}&path=x", + enc(&uri), + )) + .await; + assert_eq!(resp.status(), StatusCode::BAD_GATEWAY); + let v = body_value(resp).await; + assert_eq!(v["error"], "InvalidRecord"); +} + +#[tokio::test] +async fn strips_basic_auth_from_credentialed_knot_url() { + let h = Harness::new().await; + let parsed = Url::parse(&h.knot.uri()).unwrap(); + let knot_with_creds = format!( + "{}://attacker:secret@{}:{}/", + parsed.scheme(), + parsed.host_str().unwrap(), + parsed.port().unwrap(), + ); + let owner = did("did:plc:abalone"); + let rk = rkey("r1"); + let record = json!({ + "$type": "sh.tangled.repo", + "createdAt": "2026-05-01T00:00:00Z", + "knot": knot_with_creds, + "name": "barnacle", + }); + let uri = format!("at://{}/sh.tangled.repo/{}", owner.as_ref(), rk.as_ref()); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", owner.as_ref())) + .and(query_param("collection", "sh.tangled.repo")) + .and(query_param("rkey", rk.as_ref())) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": uri, + "cid": CID, + "value": record, + }))) + .mount(&h.slingshot) + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with( + ResponseTemplate::new(200).set_body_raw(r#"{"path":"x"}"#, "application/json"), + ) + .mount(&h.knot) + .await; + let target = format!("/xrpc/sh.tangled.repo.blob?repo={}&path=x", enc(&uri)); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::OK); + let received = h.knot.received_requests().await.unwrap(); + let knot_call = received + .iter() + .find(|r| r.url.path() == "/xrpc/sh.tangled.repo.blob") + .expect("knot received the proxied call"); + assert!( + knot_call.headers.get("authorization").is_none(), + "userinfo in knot field must not become an Authorization header", + ); +} + +#[tokio::test] +async fn knot_redirect_surfaces_as_upstream_failed() { + let h = Harness::new().await; + let secondary = MockServer::start().await; + h.mount_repo_record(&did("did:plc:abalone"), &rkey("r1"), "barnacle") + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.blob")) + .respond_with( + ResponseTemplate::new(302) + .insert_header("location", &format!("{}/secret", secondary.uri())), + ) + .mount(&h.knot) + .await; + Mock::given(method("GET")) + .and(path("/secret")) + .respond_with(ResponseTemplate::new(200).set_body_string("leaked")) + .mount(&secondary) + .await; + let resp = h + .call(&format!( + "/xrpc/sh.tangled.repo.blob?repo={}&path=x", + enc("at://did:plc:abalone/sh.tangled.repo/r1"), + )) + .await; + assert_eq!(resp.status(), StatusCode::BAD_GATEWAY); + let v = body_value(resp).await; + assert_eq!(v["error"], "UpstreamFailed"); + let received = secondary.received_requests().await.unwrap(); + assert!(received.is_empty(), "redirect target must not be dialled"); +} + +#[tokio::test] +async fn forwards_repeated_query_params() { + let h = Harness::new().await; + h.mount_repo_record(&did("did:plc:limpet"), &rkey("r4"), "kelp") + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.tags")) + .respond_with(ResponseTemplate::new(200).set_body_raw(r#"{"tags":[]}"#, "application/json")) + .mount(&h.knot) + .await; + let target = format!( + "/xrpc/sh.tangled.repo.tags?repo={}&filter=alpha&filter=beta", + enc("at://did:plc:limpet/sh.tangled.repo/r4"), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::OK); + let received = h.knot.received_requests().await.unwrap(); + let knot_call = received + .iter() + .find(|r| r.url.path() == "/xrpc/sh.tangled.repo.tags") + .expect("knot received the proxied call"); + let filters: Vec = knot_call + .url + .query_pairs() + .filter(|(k, _)| k == "filter") + .map(|(_, v)| v.into_owned()) + .collect(); + assert_eq!(filters, vec!["alpha".to_owned(), "beta".to_owned()]); +} + +#[tokio::test] +async fn duplicate_repo_param_rejected_as_invalid_request() { + let h = Harness::new().await; + let target = format!( + "/xrpc/sh.tangled.repo.blob?repo={}&repo={}", + enc("at://did:plc:abalone/sh.tangled.repo/r1"), + enc("at://did:plc:limpet/sh.tangled.repo/r2"), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let v = body_value(resp).await; + assert_eq!(v["error"], "InvalidRequest"); + assert!( + v["message"] + .as_str() + .unwrap_or_default() + .contains("repo parameter must appear at most once"), + "got {v}", + ); +} + +#[tokio::test] +async fn duplicate_knot_param_rejected_as_invalid_request() { + let h = Harness::new().await; + let target = format!( + "/xrpc/sh.tangled.knot.version?knot={}&knot={}", + enc("https://oyster.cafe"), + enc("https://nel.pet"), + ); + let resp = h.call(&target).await; + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let v = body_value(resp).await; + assert_eq!(v["error"], "InvalidRequest"); +} + +#[tokio::test] +async fn rejects_client_supplied_plaintext_when_https_required() { + let strict = KnotProxyConfig { + require_https: true, + ..test_config() + }; + let h = Harness::with_config(strict).await; + let resp = h + .call(&format!( + "/xrpc/sh.tangled.knot.version?knot={}", + enc("http://oyster.cafe"), + )) + .await; + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let v = body_value(resp).await; + assert_eq!(v["error"], "InvalidRequest"); + assert!( + v["message"] + .as_str() + .unwrap_or_default() + .contains("must be https"), + "got {v}", + ); +} + +#[tokio::test] +async fn record_with_plaintext_knot_returns_invalid_record_when_https_required() { + let strict = KnotProxyConfig { + require_https: true, + allow_private_hosts: true, + ..test_config() + }; + let h = Harness::with_config(strict).await; + let owner = did("did:plc:abalone"); + let rk = rkey("r1"); + let record = json!({ + "$type": "sh.tangled.repo", + "createdAt": "2026-05-01T00:00:00Z", + "knot": "http://oyster.cafe", + "name": "barnacle", + }); + let uri = format!("at://{}/sh.tangled.repo/{}", owner.as_ref(), rk.as_ref()); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", owner.as_ref())) + .and(query_param("collection", "sh.tangled.repo")) + .and(query_param("rkey", rk.as_ref())) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": uri, + "cid": CID, + "value": record, + }))) + .mount(&h.slingshot) + .await; + let resp = h + .call(&format!( + "/xrpc/sh.tangled.repo.blob?repo={}&path=x", + enc(&uri), + )) + .await; + assert_eq!(resp.status(), StatusCode::BAD_GATEWAY); + let v = body_value(resp).await; + assert_eq!(v["error"], "InvalidRecord"); + assert!( + v["message"] + .as_str() + .unwrap_or_default() + .contains("requires https"), + "got {v}", + ); +} + +#[tokio::test] +async fn knot_not_modified_passes_through() { + let h = Harness::new().await; + h.mount_repo_record(&did("did:plc:limpet"), &rkey("r5"), "kelp") + .await; + Mock::given(method("GET")) + .and(path("/xrpc/sh.tangled.repo.archive")) + .respond_with(ResponseTemplate::new(304).insert_header("etag", "\"v1\"")) + .mount(&h.knot) + .await; + let target = format!( + "/xrpc/sh.tangled.repo.archive?repo={}&ref=v1", + enc("at://did:plc:limpet/sh.tangled.repo/r5"), + ); + let resp = h + .call_with_headers(&target, &[("if-none-match", "\"v1\"")]) + .await; + assert_eq!(resp.status(), StatusCode::NOT_MODIFIED); + assert_eq!(resp.headers().get("etag").unwrap(), "\"v1\""); +} diff --git a/bobbin/crates/xrpc/tests/search.rs b/bobbin/crates/xrpc/tests/search.rs new file mode 100644 --- /dev/null +++ b/bobbin/crates/xrpc/tests/search.rs @@ -0,0 +1,832 @@ +use std::sync::Arc; + +use axum::body::{Body, to_bytes}; +use bobbin_edge_index::{Coverage, CoverageWatch, EdgeStore, HydrantCursor, StateIndex}; +use bobbin_knot_proxy::{KnotHttpConfig, KnotProxy, KnotProxyConfig}; +use bobbin_record_lru::{CacheCapacity, LruRecordStore}; +use bobbin_resolver::RepoIdResolver; +use bobbin_runtime::{RuntimeHasher, SystemClock}; +use bobbin_search::{DEFAULT_WRITER_HEAP_BYTES, SearchIndex, SearchReader}; +use bobbin_slingshot_client::SlingshotClient; +use bobbin_types::search::{SearchDoc, SearchSink}; +use bobbin_xrpc::{AppState, router}; +use http::{Request, StatusCode}; +use jacquard_common::DefaultStr; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::recordkey::Rkey; +use jacquard_common::types::string::{AtUri, Did}; +use serde_json::{Value, json}; +use tower::ServiceExt; +use url::Url; +use url::form_urlencoded::byte_serialize; +use wiremock::matchers::{method, path, query_param}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const CID: &str = "bafyreieqygohnz2zqyvtvktbjpvhutphobcmbsnt4q5lc36ri7vpcmoz4i"; + +fn at(s: &str) -> AtUri { + AtUri::new_owned(s).unwrap() +} + +fn did(s: &str) -> Did { + Did::new_owned(s).unwrap() +} + +fn rkey(s: &str) -> Rkey { + Rkey::new_owned(s).unwrap() +} + +fn nsid(s: &'static str) -> Nsid { + Nsid::new_static(s).unwrap() +} + +fn enc(s: &str) -> String { + byte_serialize(s.as_bytes()).collect() +} + +struct Harness { + server: MockServer, + coverage: Arc, + search: Arc, + state: AppState, +} + +impl Harness { + async fn new() -> Self { + let server = MockServer::start().await; + let coverage = Arc::new(CoverageWatch::new()); + let search = Arc::new( + SearchIndex::new(DEFAULT_WRITER_HEAP_BYTES, Arc::new(SystemClock::new())).unwrap(), + ); + let state = AppState::new( + Arc::new(LruRecordStore::new(CacheCapacity::from_bytes(64 * 1024))), + SlingshotClient::with_default_http(Url::parse(&server.uri()).unwrap()).unwrap(), + Arc::new(EdgeStore::new(RuntimeHasher::default())), + Arc::new(StateIndex::new(RuntimeHasher::default())), + Arc::new(StateIndex::new(RuntimeHasher::default())), + coverage.clone(), + Arc::new( + KnotProxy::new( + KnotProxyConfig::default(), + KnotHttpConfig::default(), + Arc::new(SystemClock::new()), + RuntimeHasher::default(), + ) + .unwrap(), + ), + search.clone() as Arc, + Arc::new(RepoIdResolver::detached(RuntimeHasher::default())), + ); + Self { + server, + coverage, + search, + state, + } + } + + async fn index_issue( + &self, + did: &Did, + rkey: &Rkey, + title: &str, + body: &str, + ) { + self.index_issue_at(did, rkey, title, body, None, None) + .await; + } + + async fn index_issue_at( + &self, + did: &Did, + rkey: &Rkey, + title: &str, + body: &str, + created_at: Option, + repo: Option<&Did>, + ) { + let uri = format!( + "at://{}/sh.tangled.repo.issue/{}", + did.as_ref(), + rkey.as_ref() + ); + self.search + .upsert(SearchDoc { + uri: at(&uri), + nsid: nsid("sh.tangled.repo.issue"), + title: title.to_owned(), + body: body.to_owned(), + author: Some(did.clone()), + created_at, + repo: repo.cloned(), + }) + .await; + self.search.flush().await; + self.mount_issue(did, rkey, title, body).await; + } + + async fn index_string( + &self, + did: &Did, + rkey: &Rkey, + filename: &str, + contents: &str, + ) { + let uri = format!("at://{}/sh.tangled.string/{}", did.as_ref(), rkey.as_ref()); + self.search + .upsert(SearchDoc { + uri: at(&uri), + nsid: nsid("sh.tangled.string"), + title: filename.to_owned(), + body: contents.to_owned(), + author: None, + created_at: None, + repo: None, + }) + .await; + self.search.flush().await; + self.mount_string(did, rkey, filename, contents).await; + } + + async fn mount_issue( + &self, + did: &Did, + rkey: &Rkey, + title: &str, + body: &str, + ) { + let uri = format!( + "at://{}/sh.tangled.repo.issue/{}", + did.as_ref(), + rkey.as_ref() + ); + let value = json!({ + "$type": "sh.tangled.repo.issue", + "repo": "did:plc:abalone", + "title": title, + "body": body, + "createdAt": "2026-05-01T00:00:00Z" + }); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", did.as_ref())) + .and(query_param("collection", "sh.tangled.repo.issue")) + .and(query_param("rkey", rkey.as_ref())) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": uri, + "cid": CID, + "value": value, + }))) + .mount(&self.server) + .await; + } + + async fn mount_issue_with_raw_value( + &self, + did: &Did, + rkey: &Rkey, + raw_value: &str, + ) { + let uri = format!( + "at://{}/sh.tangled.repo.issue/{}", + did.as_ref(), + rkey.as_ref() + ); + let body = format!(r#"{{"uri":"{uri}","cid":"{CID}","value":{raw_value}}}"#,); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", did.as_ref())) + .and(query_param("collection", "sh.tangled.repo.issue")) + .and(query_param("rkey", rkey.as_ref())) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/json") + .set_body_string(body), + ) + .mount(&self.server) + .await; + } + + async fn mount_string( + &self, + did: &Did, + rkey: &Rkey, + filename: &str, + contents: &str, + ) { + let uri = format!("at://{}/sh.tangled.string/{}", did.as_ref(), rkey.as_ref()); + let value = json!({ + "$type": "sh.tangled.string", + "filename": filename, + "description": "field notes", + "contents": contents, + "createdAt": "2026-05-01T00:00:00Z" + }); + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", did.as_ref())) + .and(query_param("collection", "sh.tangled.string")) + .and(query_param("rkey", rkey.as_ref())) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": uri, + "cid": CID, + "value": value, + }))) + .mount(&self.server) + .await; + } + + fn warming(&self, events: u64, cursor: u64) { + self.coverage.update(|_| Coverage::Warming { + events_processed: events, + last_cursor: HydrantCursor::new(cursor), + }); + } + + fn promote_ready(&self, events: u64, cursor: u64) { + self.coverage.update(|_| Coverage::Ready { + events_processed: events, + last_cursor: HydrantCursor::new(cursor), + }); + } +} + +fn search_request(extras: &[(&str, &str)]) -> Request { + let qs = extras + .iter() + .map(|(k, v)| format!("{k}={}", enc(v))) + .collect::>() + .join("&"); + Request::builder() + .uri(format!("/xrpc/sh.tangled.search.query?{qs}")) + .body(Body::empty()) + .unwrap() +} + +async fn json_response(resp: axum::response::Response) -> (StatusCode, Value) { + let status = resp.status(); + let bytes = to_bytes(resp.into_body(), 1 << 20).await.unwrap(); + let parsed: Value = serde_json::from_slice(&bytes).expect("JSON body"); + (status, parsed) +} + +#[tokio::test] +async fn empty_query_returns_no_hits() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot(search_request(&[("q", "barnacle")])) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["hits"], json!([])); + assert!(body["cursor"].is_null()); +} + +#[tokio::test] +async fn indexed_issue_hydrates_typed_value() { + let h = Harness::new().await; + h.index_issue( + &did("did:plc:nel"), + &rkey("abcabcabcabcz"), + "barnacle pagination overflow", + "scrolling resets when the cursor wraps", + ) + .await; + let app = router(h.state.clone()); + let resp = app + .oneshot(search_request(&[("q", "barnacle")])) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + let hits = body["hits"].as_array().expect("hits array"); + assert_eq!(hits.len(), 1); + assert_eq!( + hits[0]["uri"].as_str().unwrap(), + "at://did:plc:nel/sh.tangled.repo.issue/abcabcabcabcz", + ); + assert_eq!(hits[0]["nsid"], json!("sh.tangled.repo.issue")); + assert_eq!(hits[0]["cid"], CID); + assert_eq!( + hits[0]["value"]["$type"], + json!("sh.tangled.repo.issue"), + "hit value is typed via SearchableRecord serialization", + ); + assert_eq!( + hits[0]["value"]["title"], + json!("barnacle pagination overflow"), + ); + assert_eq!(hits[0]["value"]["repo"], json!("did:plc:abalone")); + assert!(hits[0]["score"].as_f64().unwrap() > 0.0); +} + +#[tokio::test] +async fn nsid_filter_narrows_to_single_collection() { + let h = Harness::new().await; + h.index_issue( + &did("did:plc:nel"), + &rkey("i1"), + "anemone tide", + "high water", + ) + .await; + h.index_string( + &did("did:plc:teq"), + &rkey("k1"), + "anemone.md", + "anemone notes", + ) + .await; + let app = router(h.state.clone()); + let resp = app + .clone() + .oneshot(search_request(&[ + ("q", "anemone"), + ("nsid", "sh.tangled.string"), + ])) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + let hits = body["hits"].as_array().unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0]["nsid"], json!("sh.tangled.string")); + assert_eq!(hits[0]["value"]["$type"], json!("sh.tangled.string")); + assert_eq!(hits[0]["value"]["filename"], json!("anemone.md")); +} + +#[tokio::test] +async fn hit_set_stable_across_coverage_promotion() { + let h = Harness::new().await; + h.index_issue( + &did("did:plc:nel"), + &rkey("i1"), + "limpet survey", + "tidal pools", + ) + .await; + let app = router(h.state.clone()); + h.warming(3, 5); + let (_, before) = json_response( + app.clone() + .oneshot(search_request(&[("q", "limpet")])) + .await + .unwrap(), + ) + .await; + let before_hits = before["hits"].as_array().unwrap().clone(); + assert_eq!(before_hits.len(), 1); + + h.promote_ready(9, 11); + let (_, after) = json_response( + app.oneshot(search_request(&[("q", "limpet")])) + .await + .unwrap(), + ) + .await; + let after_hits = after["hits"].as_array().unwrap(); + assert_eq!(after_hits.len(), before_hits.len()); + assert_eq!(after_hits[0]["uri"], before_hits[0]["uri"]); +} + +#[tokio::test] +async fn pagination_round_trips_via_cursor() { + let h = Harness::new().await; + let names = ["nel", "olaren", "teq", "lyna", "bailey"]; + for (i, owner) in names.iter().enumerate() { + h.index_issue( + &did(&format!("did:plc:{owner}")), + &rkey(&format!("r{i}")), + "anemone tides", + "shell", + ) + .await; + } + let app = router(h.state.clone()); + let (_, page1) = json_response( + app.clone() + .oneshot(search_request(&[("q", "anemone"), ("limit", "2")])) + .await + .unwrap(), + ) + .await; + assert_eq!(page1["hits"].as_array().unwrap().len(), 2); + let cursor = page1["cursor"].as_str().expect("more pages").to_owned(); + + let (_, page2) = json_response( + app.clone() + .oneshot(search_request(&[ + ("q", "anemone"), + ("limit", "2"), + ("cursor", &cursor), + ])) + .await + .unwrap(), + ) + .await; + assert_eq!(page2["hits"].as_array().unwrap().len(), 2); + let cursor2 = page2["cursor"].as_str().expect("more pages").to_owned(); + + let (_, page3) = json_response( + app.oneshot(search_request(&[ + ("q", "anemone"), + ("limit", "2"), + ("cursor", &cursor2), + ])) + .await + .unwrap(), + ) + .await; + assert_eq!(page3["hits"].as_array().unwrap().len(), 1); + assert!(page3["cursor"].is_null()); +} + +#[tokio::test] +async fn empty_q_returns_400() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app.oneshot(search_request(&[("q", " ")])).await.unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["error"], json!("InvalidRequest")); +} + +#[tokio::test] +async fn invalid_cursor_returns_400() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot(search_request(&[("q", "anything"), ("cursor", "not-hex")])) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["error"], json!("InvalidRequest")); +} + +#[tokio::test] +async fn invalid_nsid_returns_400() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot(search_request(&[ + ("q", "anything"), + ("nsid", "not a real nsid"), + ])) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["error"], json!("InvalidRequest")); +} + +#[tokio::test] +async fn tombstoned_hit_silently_dropped_from_results() { + let h = Harness::new().await; + h.search + .upsert(SearchDoc { + uri: at("at://did:plc:nel/sh.tangled.repo.issue/i1"), + nsid: nsid("sh.tangled.repo.issue"), + title: "kelp".to_owned(), + body: "ocean".to_owned(), + author: None, + created_at: None, + repo: None, + }) + .await; + h.search.flush().await; + h.index_issue( + &did("did:plc:teq"), + &rkey("i2"), + "kelp survives", + "still here", + ) + .await; + let app = router(h.state.clone()); + let resp = app.oneshot(search_request(&[("q", "kelp")])).await.unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + let hits = body["hits"].as_array().expect("hits array"); + assert_eq!(hits.len(), 1, "tombstoned hit dropped, sibling kept"); + assert_eq!( + hits[0]["uri"].as_str().unwrap(), + "at://did:plc:teq/sh.tangled.repo.issue/i2", + ); +} + +#[tokio::test] +async fn upstream_5xx_during_hydration_drops_only_that_hit() { + let h = Harness::new().await; + h.index_issue( + &did("did:plc:nel"), + &rkey("i1"), + "kelp survey", + "still here", + ) + .await; + h.search + .upsert(SearchDoc { + uri: at("at://did:plc:teq/sh.tangled.repo.issue/i2"), + nsid: nsid("sh.tangled.repo.issue"), + title: "kelp drift".to_owned(), + body: "upstream is flaky".to_owned(), + author: Some(did("did:plc:teq")), + created_at: None, + repo: None, + }) + .await; + h.search.flush().await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", "did:plc:teq")) + .and(query_param("collection", "sh.tangled.repo.issue")) + .and(query_param("rkey", "i2")) + .respond_with(ResponseTemplate::new(503)) + .mount(&h.server) + .await; + let app = router(h.state.clone()); + let resp = app.oneshot(search_request(&[("q", "kelp")])).await.unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + let hits = body["hits"].as_array().expect("hits array"); + assert_eq!(hits.len(), 1, "flaky hit dropped, healthy sibling kept"); + assert_eq!( + hits[0]["uri"].as_str().unwrap(), + "at://did:plc:nel/sh.tangled.repo.issue/i1", + ); +} + +#[tokio::test] +async fn second_query_short_circuits_via_lru_without_re_querying_slingshot() { + let h = Harness::new().await; + h.search + .upsert(SearchDoc { + uri: at("at://did:plc:nel/sh.tangled.repo.issue/i1"), + nsid: nsid("sh.tangled.repo.issue"), + title: "kelp".to_owned(), + body: "ocean".to_owned(), + author: None, + created_at: None, + repo: None, + }) + .await; + h.search.flush().await; + Mock::given(method("GET")) + .and(path("/xrpc/com.atproto.repo.getRecord")) + .and(query_param("repo", "did:plc:nel")) + .and(query_param("collection", "sh.tangled.repo.issue")) + .and(query_param("rkey", "i1")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uri": "at://did:plc:nel/sh.tangled.repo.issue/i1", + "cid": CID, + "value": { + "$type": "sh.tangled.repo.issue", + "repo": "did:plc:abalone", + "title": "kelp", + "body": "ocean", + "createdAt": "2026-05-01T00:00:00Z" + } + }))) + .expect(1) + .mount(&h.server) + .await; + let app = router(h.state.clone()); + let first = app + .clone() + .oneshot(search_request(&[("q", "kelp")])) + .await + .unwrap(); + let _ = json_response(first).await; + let second = app.oneshot(search_request(&[("q", "kelp")])).await.unwrap(); + let _ = json_response(second).await; +} + +#[tokio::test] +async fn author_filter_narrows_to_matching_did() { + let h = Harness::new().await; + h.index_issue(&did("did:plc:nel"), &rkey("i1"), "kelp tide", "") + .await; + h.index_issue(&did("did:plc:teq"), &rkey("i2"), "kelp wave", "") + .await; + let app = router(h.state.clone()); + let resp = app + .oneshot(search_request(&[("q", "kelp"), ("author", "did:plc:nel")])) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + let hits = body["hits"].as_array().unwrap(); + assert_eq!(hits.len(), 1); + assert!( + hits[0]["uri"] + .as_str() + .unwrap() + .starts_with("at://did:plc:nel/") + ); +} + +#[tokio::test] +async fn since_until_window_filters_by_created_at() { + let h = Harness::new().await; + let early = 1_700_000_000; + let mid = 1_750_000_000; + let late = 1_800_000_000; + h.index_issue_at( + &did("did:plc:nel"), + &rkey("i1"), + "kelp early", + "", + Some(early), + None, + ) + .await; + h.index_issue_at( + &did("did:plc:nel"), + &rkey("i2"), + "kelp mid", + "", + Some(mid), + None, + ) + .await; + h.index_issue_at( + &did("did:plc:nel"), + &rkey("i3"), + "kelp late", + "", + Some(late), + None, + ) + .await; + let app = router(h.state.clone()); + let resp = app + .oneshot(search_request(&[ + ("q", "kelp"), + ("since", "2025-01-01T00:00:00Z"), + ("until", "2027-01-01T00:00:00Z"), + ])) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + let hits = body["hits"].as_array().unwrap(); + assert_eq!(hits.len(), 1, "only the mid record falls in [2025, 2027)"); + assert_eq!( + hits[0]["uri"].as_str().unwrap(), + "at://did:plc:nel/sh.tangled.repo.issue/i2" + ); +} + +#[tokio::test] +async fn repo_filter_scopes_to_owning_repo() { + let h = Harness::new().await; + let abalone = did("did:plc:abalone"); + let limpet = did("did:plc:limpet"); + h.index_issue_at( + &did("did:plc:nel"), + &rkey("i1"), + "kelp one", + "", + None, + Some(&abalone), + ) + .await; + h.index_issue_at( + &did("did:plc:teq"), + &rkey("i2"), + "kelp two", + "", + None, + Some(&limpet), + ) + .await; + let app = router(h.state.clone()); + let resp = app + .oneshot(search_request(&[("q", "kelp"), ("repo", abalone.as_ref())])) + .await + .unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + let hits = body["hits"].as_array().unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!( + hits[0]["uri"].as_str().unwrap(), + "at://did:plc:nel/sh.tangled.repo.issue/i1" + ); +} + +#[tokio::test] +async fn invalid_author_did_returns_400() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot(search_request(&[("q", "kelp"), ("author", "not-a-did")])) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn invalid_since_returns_400() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot(search_request(&[("q", "kelp"), ("since", "yesterday")])) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn since_after_until_returns_400() { + let h = Harness::new().await; + let app = router(h.state.clone()); + let resp = app + .oneshot(search_request(&[ + ("q", "kelp"), + ("since", "2027-01-01T00:00:00Z"), + ("until", "2025-01-01T00:00:00Z"), + ])) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn search_recovers_hit_with_duplicate_dollar_type() { + let h = Harness::new().await; + let owner = did("did:plc:nel"); + let key = rkey("dupezzzzzzzzz"); + h.search + .upsert(SearchDoc { + uri: at(&format!( + "at://{}/sh.tangled.repo.issue/{}", + owner.as_ref(), + key.as_ref() + )), + nsid: nsid("sh.tangled.repo.issue"), + title: "meow regression".to_owned(), + body: "dup type field at end of object".to_owned(), + author: Some(owner.clone()), + created_at: None, + repo: None, + }) + .await; + h.search.flush().await; + let raw = r#"{"$type":"sh.tangled.repo.issue","repo":"did:plc:scallop","title":"meow regression","createdAt":"2026-05-01T00:00:00Z","$type":"sh.tangled.repo.issue"}"#; + h.mount_issue_with_raw_value(&owner, &key, raw).await; + let app = router(h.state.clone()); + let resp = app.oneshot(search_request(&[("q", "meow")])).await.unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + let hits = body["hits"].as_array().expect("hits array"); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0]["value"]["title"], json!("meow regression")); + assert_eq!(hits[0]["value"]["repo"], json!("did:plc:scallop")); +} + +#[tokio::test] +async fn search_drops_undecodable_hit_and_returns_others() { + let h = Harness::new().await; + let good_owner = did("did:plc:nel"); + let good_key = rkey("goodzzzzzzzzz"); + let bad_owner = did("did:plc:teq"); + let bad_key = rkey("badzzzzzzzzzz"); + h.index_issue(&good_owner, &good_key, "kelp survey", "good body") + .await; + h.search + .upsert(SearchDoc { + uri: at(&format!( + "at://{}/sh.tangled.repo.issue/{}", + bad_owner.as_ref(), + bad_key.as_ref() + )), + nsid: nsid("sh.tangled.repo.issue"), + title: "kelp drift".to_owned(), + body: "missing required fields".to_owned(), + author: Some(bad_owner.clone()), + created_at: None, + repo: None, + }) + .await; + h.search.flush().await; + let unrecoverable = r#"{"$type":"sh.tangled.repo.issue"}"#; + h.mount_issue_with_raw_value(&bad_owner, &bad_key, unrecoverable) + .await; + let app = router(h.state.clone()); + let resp = app.oneshot(search_request(&[("q", "kelp")])).await.unwrap(); + let (status, body) = json_response(resp).await; + assert_eq!(status, StatusCode::OK); + let hits = body["hits"].as_array().expect("hits array"); + assert_eq!(hits.len(), 1); + assert_eq!( + hits[0]["uri"].as_str().unwrap(), + format!( + "at://{}/sh.tangled.repo.issue/{}", + good_owner.as_ref(), + good_key.as_ref() + ), + ); +} diff --git a/bobbin/default.nix b/bobbin/default.nix new file mode 100644 --- /dev/null +++ b/bobbin/default.nix @@ -0,0 +1,36 @@ +{ + lib, + rustPlatform, + tangled, + ... +}: + +rustPlatform.buildRustPackage { + pname = "bobbin"; + version = "main"; + + src = ./.; + + cargoHash = "sha256-apY29WbquD3Eh2yvyRqPPTjm60JZMkty4Vwi1ERZ2+Q="; + + cargoBuildFlags = [ + "--bin" + "bobbin" + "--package" + "bobbin" + ]; + + preBuild = '' + export BOBBIN_LEXICONS_DIR=${tangled}/lexicons + ''; + + doCheck = false; + + meta = { + description = "tangled appview"; + homepage = "https://tangled.org/oyster.cafe/bobbin"; + license = lib.licenses.mit; + mainProgram = "bobbin"; + }; +} + diff --git a/bobbin/example.toml b/bobbin/example.toml new file mode 100644 --- /dev/null +++ b/bobbin/example.toml @@ -0,0 +1,163 @@ +[server] +# Addresses the XRPC server listens on. When using as an env var, comma-separated. +# +# Can also be specified via environment variable `BOBBIN_BIND`. +# +# Default value: ["127.0.0.1:8090", "[::1]:8090"] +#binds = ["127.0.0.1:8090", "[::1]:8090"] + +# The amount of time in seconds to allow in-flight requests to drain after sigterm +# before forcing the listener closed. +# +# Can also be specified via environment variable `BOBBIN_SHUTDOWN_GRACE_SECS`. +# +# Default value: 30 +#shutdown_grace_secs = 30 + +# Address for the `/debug/mem` and `/debug/heap` introspection endpoints, +# for example `127.0.0.1:8091`. Empty disables them so the debug surface is +# never reachable on the public listener. Bind to loopback only. +# +# Can also be specified via environment variable `BOBBIN_DEBUG_BIND`. +# +# Default value: "" +#debug_bind = "" + +[hydrant] +# Base URL of the hydrant instance - the cursor-replayable /stream lives +# under this. Use `ws://` or `wss://` - `http://` and `https://` +# are rewritten to the corresponding ws scheme at connection time! +# +# Can also be specified via environment variable `BOBBIN_HYDRANT_URL`. +# +# Default value: "http://127.0.0.1:13010" +#url = "http://127.0.0.1:13010" + +# The cursor to request on first connect. Reconnects ignore this and resume +# strictly after the last cursor seen internally. +# +# Can also be specified via environment variable `BOBBIN_START_CURSOR`. +# +# Default value: 0 +#start_cursor = 0 + +[ingest] +# Concurrent in-flight resolves during ingest. One slot per slingshot rtt. +# Default 16 sits at the measured throughput +# knee for cold replay against a healthy hydrant. The committer stays +# serial so that like, cursor and Spur allocation order are preserved. +# +# Can also be specified via environment variable `BOBBIN_INGEST_PARALLELISM`. +# +# Default value: 16 +#parallelism = 16 + +[backpressure] +# Estimated peak transient anonymous bytes a single hydrating request holds: +# roughly the page of decoded records plus its serialized copy. Combined with +# the detected cgroup memory limit to size the in-flight cap on heavy +# endpoints. Has no effect when no cgroup memory limit is present, the heavy +# path stays unbounded exactly as before. +# +# Can also be specified via environment variable `BOBBIN_BACKPRESSURE_PER_REQUEST_ANON_BYTES`. +# +# Default value: 2097152 +#per_request_anon_bytes = 2097152 + +# How often the adaptive watcher samples memory.current/memory.max and +# adjusts the in-flight cap. Ignored when no cgroup memory limit is present. +# +# Can also be specified via environment variable `BOBBIN_BACKPRESSURE_ADJUST_INTERVAL_MS`. +# +# Default value: 500 +#adjust_interval_ms = 500 + +# memory.current/memory.max ratio below which the watcher additively raises +# the in-flight cap back toward its ceiling. Sits below the memory.high +# throttle band so the kernel, not this loop, handles mild pressure. +# +# Can also be specified via environment variable `BOBBIN_BACKPRESSURE_RELIEVE_BELOW_RATIO`. +# +# Default value: 0.85 +#relieve_below_ratio = 0.85 + +# memory.current/memory.max ratio above which the watcher multiplicatively +# cuts the in-flight cap and purges jemalloc arenas. Set above the memory.high +# watermark so this loop acts as the anti-OOM backstop, not a duplicate throttle. +# +# Can also be specified via environment variable `BOBBIN_BACKPRESSURE_TIGHTEN_ABOVE_RATIO`. +# +# Default value: 0.92 +#tighten_above_ratio = 0.92 + +# Bytes held back for the edge index, state indexes, and other derived +# state that grows after startup. Folded into the heavy-request reserve so +# the static in-flight cap leaves the index room to fill. Has no effect when +# no cgroup memory limit is present. +# +# Can also be specified via environment variable `BOBBIN_BACKPRESSURE_RESERVED_INDEX_BYTES`. +# +# Default value: 67108864 +#reserved_index_bytes = 67108864 + +[slingshot] +# Base URL of a slingshot instance. Used for record bodies and identity. +# +# Can also be specified via environment variable `BOBBIN_SLINGSHOT_URL`. +# +# Default value: "http://127.0.0.1:13011" +#url = "http://127.0.0.1:13011" + +[record_cache] +# Bound of bytes on the in-process record LRU. Records evict on a weighted +# LRU policy keyed on URI plus payload length. +# +# Can also be specified via environment variable `BOBBIN_RECORD_LRU_BYTES`. +# +# Default value: 67108864 +#lru_bytes = 67108864 + +[search] +# The heap size in bytes for the in-mem tantivy writer. Larger values +# trade RAM for fewer segment merges - the index itself lives in +# `RamDirectory` and is rebuilt from hydrant replay on every restart. +# +# Can also be specified via environment variable `BOBBIN_SEARCH_HEAP_BYTES`. +# +# Default value: 50000000 +#heap_bytes = 50000000 + +[knot] +# Whether to allow the knot proxy to dial private/loopback addresses. Off in +# production - on for local testing against a knotserver on localhost. +# +# Can also be specified via environment variable `BOBBIN_KNOT_ALLOW_PRIVATE`. +# +# Default value: false +#allow_private = false + +# Require https on knot hosts. Disable only when proxying to a local +# knot for development. +# +# Can also be specified via environment variable `BOBBIN_KNOT_REQUIRE_HTTPS`. +# +# Default value: true +#require_https = true + +[log] +# Log emitter format. `text` produces human-readable output for local +# development. `json` emits one structured object per line for log +# shippers in production. +# +# Can also be specified via environment variable `BOBBIN_LOG_FORMAT`. +# +# Default value: "text" +#format = "text" + +# `tracing-subscriber` env-filter directive. Defaults to `info` across +# every span - override with `BOBBIN_LOG=bobbin_xrpc=debug,info` etc. +# +# Can also be specified via environment variable `BOBBIN_LOG`. +# +# Default value: "info" +#filter = "info" -- tangled.sh