From 01b872239dd259199fbe4ae473654e2098bccae4 Mon Sep 17 00:00:00 2001 From: dawn <90008@klbr.net> Date: Mon, 10 Aug 2026 22:32:39 +0000 Subject: [PATCH] [tests] enforce representation boundary coverage --- Cargo.lock | 4 +++- Cargo.toml | 3 ++- .beads/interactions.jsonl | 1 + src/config.rs | 2 +- src/filter.rs | 12 ++++++++++++ src/ops.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/patch.rs | 34 +++++++++++++++++++++++++++++++++- src/pds_meta.rs | 37 +++++++++++++++++++++++++++++++++++++ src/resolver.rs | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/types.rs | 158 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/api_db.nu | 32 ++++++++++++++++++++++++++++++++ tests/api_xrpc.nu | 147 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/backlinks.nu | 333 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- tests/common.nu | 41 +++++++++++++++++++++++++---------------- tests/mock_pds.nu | 16 +++++++++++++--- tests/representation_audit.rs | 994 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/representation_inventory.tsv | 806 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/stream_live_backfill.nu | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++--------------------------- tests/stream_relay_jetstream_subscribe.nu | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------- src/api/debug.rs | 450 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----- src/api/ingestion.rs | 38 ++++++++++++++++++++++++++++++++++++++ src/api/jetstream.rs | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------------------------------------------- src/api/mod.rs | 26 ++++++++++++++++++++++++++ src/api/redactions.rs | 17 +++++++++-------- src/api/repos.rs | 1 - src/backlinks/api.rs | 24 +++++++++++++++++++++--- src/backlinks/mod.rs | 10 ++++++---- src/backlinks/store.rs | 35 +++++++++++++++++++++++++++++++++++ src/bin/backfill_strategy_bench.rs | 29 +++++++++++++++++++++++++++++ src/config/env.rs | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------- src/config/types.rs | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------- src/crawler/relay_first.rs | 14 +++++++++++--- src/db/indexer.rs | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/db/mod.rs | 32 ++++++++++++++++++++++++++++++++ src/db/schema.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/db/storage_mode.rs | 17 +++++++++++++++++ src/db/types.rs | 260 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------- src/ingest/stream.rs | 41 +++++++++++++++++++++++++++++++++++++++-- src/ingest/validation.rs | 18 ++++++++++++++++++ src/types/v2.rs | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/types/v4.rs | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/types/v7.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ src/util/mod.rs | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/helpers/pds_status.nu | 4 ---- src/api/xrpc/com_atproto_describe_repo.rs | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---- src/api/xrpc/mod.rs | 2 +- src/api/xrpc/request_crawl.rs | 42 ++++++++++++++++++++++++++++++++++++++++++ src/control/repos/indexer.rs | 11 ++++++++++- src/control/repos/mod.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/control/stream/indexer.rs | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------- src/crawler/list_repos/checker.rs | 28 ++++++++++++++++++++++++++++ src/db/keys/indexer.rs | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- src/db/keys/mod.rs | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/db/migration/mod.rs | 172 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/db/migration/v2.rs | 42 ++++++++++++++++++++++++++++++++++++++++++ src/db/migration/v4.rs | 31 +++++++++++++++++++++++++++++++ src/db/migration/v7.rs | 7 ++++++- src/ingest/firehose_stats/relay.rs | 27 +++++++++++++++++++++++++++ src/ingest/firehose_stats/source.rs | 47 +++++++++++++++++++++++++++++++++++++++++++++++ src/ingest/indexer/shard.rs | 47 +++++++++++++++++++++++++++++++++++++++++++++++ src/ingest/stream/codec.rs | 135 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/ingest/stream/types.rs | 204 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------ src/types/event/jetstream.rs | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- src/types/event/stream.rs | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/db/migration/v10/events.rs | 1 + src/ingest/relay/sink/relay.rs | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 66 file(s) changed, 5600 insertion(s)(+), 467 deletion(s)(-) diff --git a/Cargo.lock b/Cargo.lock --- a/Cargo.lock +++ b/Cargo.lock @@ -1530,13 +1530,14 @@ "scc", "serde", "serde_bytes", + "serde_html_form", "serde_ipld_dagcbor", "serde_json", - "serde_urlencoded", "sha2", "smol_str", "snmalloc-rs", "socket2", + "syn", "tempfile", "thiserror 2.0.18", "tikv-jemallocator", @@ -3217,6 +3218,7 @@ "form_urlencoded", "indexmap", "itoa", + "ryu", "serde_core", ] diff --git a/Cargo.toml b/Cargo.toml --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ fjall = { git = "https://github.com/90-008/fjall.git", rev = "85a8ad9b3d327c39263cc547fa9944e7535e802f", features = ["lz4", "zstd"], version = "3.1.2" } lsm-tree = { git = "https://github.com/90-008/lsm-tree.git", features = ["lz4", "zstd"] } serde_ipld_dagcbor = "0.6" -serde_urlencoded = "0.7" +serde_html_form = "0.3" fluent-uri = "0.4.1" zstd = { version = "0.13", optional = true } @@ -79,6 +79,7 @@ [dev-dependencies] tempfile = "3.26.0" +syn = { version = "2", features = ["full", "visit"] } [[bin]] name = "replay_bench" diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl --- a/.beads/interactions.jsonl +++ b/.beads/interactions.jsonl @@ -86,3 +86,4 @@ {"id":"int-31e5dd29f03926cd1b773b3379c0ddb3","kind":"field_change","created_at":"2026-08-05T13:28:53.901038Z","actor":"dawn","issue_id":"hydrant-adq","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented and fully validated on the final working tree."}} {"id":"int-d47b701c216df2d75a8351144aa4e648","kind":"field_change","created_at":"2026-08-05T13:28:54.108148Z","actor":"dawn","issue_id":"hydrant-0gg","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented and fully validated on the final working tree."}} {"id":"int-a476263d50fdc6414edb9ac747ba69b6","kind":"field_change","created_at":"2026-08-05T13:28:54.314066Z","actor":"dawn","issue_id":"hydrant-4hr","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented and fully validated on the final working tree."}} +{"id":"int-ccc4d70168c95b110d3b7e29fcf40451","kind":"field_change","created_at":"2026-08-10T22:32:35.356361Z","actor":"dawn","issue_id":"hydrant-dom","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}} diff --git a/src/config.rs b/src/config.rs --- a/src/config.rs +++ b/src/config.rs @@ -475,7 +475,7 @@ } #[test] - #[cfg(feature = "backlinks")] + #[cfg(all(feature = "backlinks", feature = "indexer_stream"))] fn ephemeral_rejected_when_backlinks_feature_enabled() { let mut config = Config::default(); config.ephemeral = true; diff --git a/src/filter.rs b/src/filter.rs --- a/src/filter.rs +++ b/src/filter.rs @@ -78,4 +78,16 @@ assert!(!filter.matches_collection("sh.tangledfoo.repo")); assert!(!filter.matches_collection("sh.tangled")); } + #[test] + fn filter_mode_persisted_golden_fixtures() { + for (mode, msgpack, json) in [ + (FilterMode::Filter, "a666696c746572", "\"filter\""), + (FilterMode::Full, "a466756c6c", "\"full\""), + ] { + let encoded = rmp_serde::to_vec(&mode).unwrap(); + assert_eq!(hex::encode(&encoded), msgpack); + assert_eq!(rmp_serde::from_slice::(&encoded).unwrap(), mode); + assert_eq!(serde_json::to_string(&mode).unwrap(), json); + } + } } diff --git a/src/ops.rs b/src/ops.rs --- a/src/ops.rs +++ b/src/ops.rs @@ -280,3 +280,59 @@ Ok((collection, rkey)) } + +#[cfg(all(test, feature = "indexer_stream"))] +mod tests { + use super::*; + use crate::config::Config; + use jacquard_common::{CowStr, types::string::Handle}; + + fn ephemeral(event: BroadcastEvent) -> Box> { + let BroadcastEvent::Ephemeral(event) = event else { + panic!("expected ephemeral event"); + }; + event + } + + #[test] + fn ephemeral_identity_and_account_json_contracts() { + let tmp = tempfile::tempdir().unwrap(); + let db = Db::open(&Config { + database_path: tmp.path().to_path_buf(), + ..Default::default() + }) + .unwrap(); + let did = Did::new("did:web:example.com").unwrap(); + + let identity = ephemeral(make_identity_event( + &db, + IdentityEvt { + did: did.clone(), + handle: Some(Handle::new("alice.test").unwrap()), + }, + )); + let identity_json = serde_json::to_value(identity.as_ref()).unwrap(); + assert_eq!(identity_json["type"], "identity"); + assert_eq!(identity_json["identity"]["did"], "did:web:example.com"); + assert_eq!(identity_json["identity"]["handle"], "alice.test"); + assert!(identity_json.get("record").is_none()); + assert!(identity_json.get("account").is_none()); + + let account = ephemeral(make_account_event( + &db, + AccountEvt { + did, + active: false, + status: Some(CowStr::Borrowed("stinkpot")), + }, + )); + let account_json = serde_json::to_value(account.as_ref()).unwrap(); + assert_eq!(account_json["id"].as_u64(), Some(identity.id + 1)); + assert_eq!(account_json["type"], "account"); + assert_eq!(account_json["account"]["did"], "did:web:example.com"); + assert_eq!(account_json["account"]["active"], false); + assert_eq!(account_json["account"]["status"], "stinkpot"); + assert!(account_json.get("record").is_none()); + assert!(account_json.get("identity").is_none()); + } +} diff --git a/src/patch.rs b/src/patch.rs --- a/src/patch.rs +++ b/src/patch.rs @@ -1,11 +1,43 @@ use serde::{Deserialize, Serialize}; /// apply a bool patch or set replacement for a single set update. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(untagged)] pub(crate) enum SetUpdate { /// replace the entire set with this list Set(Vec), /// patch: true = add, false = remove Patch(std::collections::HashMap), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn set_update_json_markers_are_structurally_disjoint() { + for input in [r#"[]"#, r#"["a","b"]"#] { + let update: SetUpdate = serde_json::from_str(input).unwrap(); + assert!(matches!(update, SetUpdate::Set(_)), "{input}"); + } + for input in [r#"{}"#, r#"{"a":true,"b":false}"#] { + let update: SetUpdate = serde_json::from_str(input).unwrap(); + assert!(matches!(update, SetUpdate::Patch(_)), "{input}"); + } + for input in [r#""a""#, "null", "true", "1"] { + assert!(serde_json::from_str::(input).is_err(), "{input}"); + } + } + + #[test] + fn set_update_json_roundtrips_both_variants() { + for update in [ + SetUpdate::Set(vec!["a".to_owned(), "b".to_owned()]), + SetUpdate::Patch([("a".to_owned(), true), ("b".to_owned(), false)].into()), + ] { + let encoded = serde_json::to_vec(&update).unwrap(); + let decoded: SetUpdate = serde_json::from_slice(&encoded).unwrap(); + assert_eq!(decoded, update); + } + } } diff --git a/src/pds_meta.rs b/src/pds_meta.rs --- a/src/pds_meta.rs +++ b/src/pds_meta.rs @@ -159,3 +159,40 @@ tier } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn host_status_persisted_golden_fixtures() { + for (status, msgpack, json) in [ + (HostStatus::Active, "a6416374697665", "\"Active\""), + (HostStatus::Idle, "a449646c65", "\"Idle\""), + (HostStatus::Offline, "a74f66666c696e65", "\"Offline\""), + ( + HostStatus::Throttled, + "a95468726f74746c6564", + "\"Throttled\"", + ), + (HostStatus::Banned, "a642616e6e6564", "\"Banned\""), + ] { + let encoded = rmp_serde::to_vec(&status).unwrap(); + assert_eq!(hex::encode(&encoded), msgpack); + assert_eq!( + rmp_serde::from_slice::(&encoded).unwrap(), + status + ); + assert_eq!(serde_json::to_string(&status).unwrap(), json); + let external: jacquard_api::com_atproto::sync::HostStatus<'static> = status.into(); + let expected_external = match status { + HostStatus::Active => jacquard_api::com_atproto::sync::HostStatus::Active, + HostStatus::Idle => jacquard_api::com_atproto::sync::HostStatus::Idle, + HostStatus::Offline => jacquard_api::com_atproto::sync::HostStatus::Offline, + HostStatus::Throttled => jacquard_api::com_atproto::sync::HostStatus::Throttled, + HostStatus::Banned => jacquard_api::com_atproto::sync::HostStatus::Banned, + }; + assert_eq!(external, expected_external); + } + } +} diff --git a/src/resolver.rs b/src/resolver.rs --- a/src/resolver.rs +++ b/src/resolver.rs @@ -260,3 +260,57 @@ #[derive(Debug, Diagnostic, Error)] #[error("no atproto signing key in DID doc for {0}")] pub struct NoSigningKeyError(Did<'static>); + +#[cfg(test)] +mod tests { + use super::*; + use axum::{Router, http::header, response::IntoResponse}; + + async fn spawn_doc_server(body: String) -> Url { + let app = Router::new().fallback(move || { + let body = body.clone(); + async move { ([(header::CONTENT_TYPE, "application/json")], body).into_response() } + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + Url::parse(&format!("http://{addr}/")).unwrap() + } + + #[tokio::test] + async fn raw_did_doc_json_contract_and_size_limit() { + let did = Did::new_static("did:plc:ewvi7nxzyoun6zhxrhs64oiz").unwrap(); + let body = serde_json::json!({ + "@context": ["https://www.w3.org/ns/did/v1"], + "id": did.as_str(), + "alsoKnownAs": ["at://alice.test"], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.example" + }], + "verificationMethod": [] + }) + .to_string(); + let resolver = Resolver::new(vec![spawn_doc_server(body).await], 10); + + let (data, handle) = resolver.resolve_raw_doc(&did).await.unwrap(); + let json = serde_json::to_value(data).unwrap(); + assert_eq!(json["id"], did.as_str()); + assert_eq!(json["service"][0]["serviceEndpoint"], "https://pds.example"); + assert_eq!(handle.unwrap().as_str(), "alice.test"); + + let oversized = Resolver::new( + vec![spawn_doc_server("x".repeat(MAX_DID_DOC_BYTES + 1)).await], + 10, + ); + let error = oversized.resolve_raw_doc(&did).await.unwrap_err(); + assert!( + error + .to_string() + .contains("DID doc response exceeds 16384 byte limit") + ); + } +} diff --git a/src/types.rs b/src/types.rs --- a/src/types.rs +++ b/src/types.rs @@ -326,4 +326,162 @@ Ok(()) } + + #[test] + fn commit_and_repo_status_conversion_contracts() { + let did = Did::new("did:plc:ewvi7nxzyoun6zhxrhs64oiz").unwrap(); + let rev = jacquard_common::types::tid::Tid::new("3jzfcijpj2z2a").unwrap(); + let data = jacquard_repo::mst::util::compute_cid(b"root").unwrap(); + let prev = jacquard_repo::mst::util::compute_cid(b"prev").unwrap(); + let atp = AtpCommit { + did: did.clone(), + version: 3, + data, + rev: rev.clone(), + prev: Some(prev), + sig: bytes::Bytes::from_static(b"\x01\x02\x03"), + }; + + let stored = Commit::from(atp); + assert_eq!(stored.version, 3); + assert_eq!(stored.data, data); + assert_eq!(stored.prev, Some(prev)); + assert_eq!(stored.rev.to_tid(), rev); + assert_eq!(stored.sig.as_ref(), b"\x01\x02\x03"); + + let restored = stored.into_atp_commit(did.clone()).unwrap(); + assert_eq!(restored.did, did); + assert_eq!(restored.version, 3); + assert_eq!(restored.data, data); + assert_eq!(restored.prev, Some(prev)); + assert_eq!(restored.rev, rev); + assert_eq!(restored.sig.as_ref(), b"\x01\x02\x03"); + + let mut sentinel = Commit::from(restored); + sentinel.version = -1; + assert!(sentinel.into_atp_commit(did).is_none()); + + for (status, display) in [ + (RepoStatus::Synced, "synced"), + (RepoStatus::Error("stinkpot".into()), "error(stinkpot)"), + (RepoStatus::Deactivated, "deactivated"), + (RepoStatus::Takendown, "takendown"), + (RepoStatus::Suspended, "suspended"), + (RepoStatus::Deleted, "deleted"), + (RepoStatus::Desynchronized, "desynchronized"), + (RepoStatus::Throttled, "throttled"), + ] { + assert_eq!(status.to_string(), display); + } + } + #[cfg(feature = "indexer")] + #[test] + fn current_persisted_state_golden_fixtures() { + let states = [ + ResyncState::Error { + kind: ResyncErrorKind::Ratelimited, + retry_count: 7, + next_retry: -42, + }, + ResyncState::Error { + kind: ResyncErrorKind::Transport, + retry_count: 8, + next_retry: 0, + }, + ResyncState::Error { + kind: ResyncErrorKind::Generic, + retry_count: 9, + next_retry: 42, + }, + ResyncState::Gone { + status: RepoStatus::Deleted, + }, + ]; + let encoded = states + .iter() + .map(|state| rmp_serde::to_vec(state).unwrap()) + .collect::>(); + assert_eq!( + encoded.iter().map(hex::encode).collect::>(), + [ + "81a54572726f7293ab526174656c696d6974656407d0d6", + "81a54572726f7293a95472616e73706f72740800", + "81a54572726f7293a747656e65726963092a", + "81a4476f6e6591a744656c65746564", + ] + ); + assert!(matches!( + rmp_serde::from_slice::(&encoded[0]).unwrap(), + ResyncState::Error { + kind: ResyncErrorKind::Ratelimited, + retry_count: 7, + next_retry: -42, + } + )); + assert!(matches!( + rmp_serde::from_slice::(&encoded[1]).unwrap(), + ResyncState::Error { + kind: ResyncErrorKind::Transport, + retry_count: 8, + next_retry: 0, + } + )); + assert!(matches!( + rmp_serde::from_slice::(&encoded[2]).unwrap(), + ResyncState::Error { + kind: ResyncErrorKind::Generic, + retry_count: 9, + next_retry: 42, + } + )); + assert!(matches!( + rmp_serde::from_slice::(&encoded[3]).unwrap(), + ResyncState::Gone { + status: RepoStatus::Deleted, + } + )); + + let metadata = RepoMetadata { + tracked: true, + index_id: 7, + }; + let encoded = rmp_serde::to_vec(&metadata).unwrap(); + assert_eq!(hex::encode(&encoded), "92c307"); + let decoded: RepoMetadata = rmp_serde::from_slice(&encoded).unwrap(); + assert!(decoded.tracked); + assert_eq!(decoded.index_id, 7); + } + + #[cfg(feature = "indexer")] + #[test] + fn operator_redaction_json_contracts() { + for (target, expected) in [ + (DeleteBodyTarget::Head, "\"head\""), + (DeleteBodyTarget::History, "\"history\""), + (DeleteBodyTarget::All, "\"all\""), + ] { + let encoded = serde_json::to_string(&target).unwrap(); + assert_eq!(encoded, expected); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + target + ); + } + + let report = DeleteBodiesReport { + heads_redacted: 1, + history_redacted: 7, + event_bodies_redacted: 3, + body_bytes_redacted: 11, + buffered_commits_deleted: 2, + buffered_commit_bytes_deleted: 13, + }; + let encoded = serde_json::to_value(&report).unwrap(); + assert_eq!(encoded["heads_redacted"], 1); + assert_eq!(encoded["history_redacted"], 7); + assert_eq!(encoded["event_bodies_redacted"], 3); + assert_eq!(encoded["body_bytes_redacted"], 11); + assert_eq!(encoded["buffered_commits_deleted"], 2); + assert_eq!(encoded["buffered_commit_bytes_deleted"], 13); + } } diff --git a/tests/api_db.nu b/tests/api_db.nu new file mode 100644 --- /dev/null +++ b/tests/api_db.nu @@ -0,0 +1,32 @@ +#!/usr/bin/env nu +use common.nu * + +def main [] { + let port = resolve-test-port 3022 + let url = $"http://localhost:($port)" + let db_path = (mktemp -d -t hydrant_api_db.XXXXXX) + let binary = build-hydrant + let instance = (with-env { + HYDRANT_CRAWLER_URLS: "" + HYDRANT_RELAY_HOSTS: "" + } { + start-hydrant $binary $db_path $port + }) + + if not (wait-for-api $url) { + fail "hydrant did not start" $instance.pid + } + + http post -f -e -H [Content-Length 0] $"($url)/db/compact" "" + | assert-status 200 "POST /db/compact" $instance.pid + let train = ( + http post -f -e -H [Content-Length 0] $"($url)/db/train" "" + ) + $train | assert-status 500 "POST /db/train without samples" $instance.pid + if not ($train.body | str contains "no samples found") { + fail "POST /db/train did not explain that training requires samples" $instance.pid + } + + try { kill $instance.pid } + print "database maintenance endpoints passed!" +} diff --git a/tests/api_xrpc.nu b/tests/api_xrpc.nu new file mode 100644 --- /dev/null +++ b/tests/api_xrpc.nu @@ -0,0 +1,147 @@ +#!/usr/bin/env nu +use common.nu * + +const COLLECTION = "app.bsky.feed.post" +const RKEY = "stinkpot" + + +def assert-equal [actual: any, expected: any, label: string] { + if $actual != $expected { + fail $"($label): expected ($expected), got ($actual)" + } +} + + +def assert-record [record: record, seed: record, label: string] { + assert-equal $record.uri $"at://($seed.did)/($seed.collection)/($seed.rkey)" $"($label) uri" + assert-equal $record.cid $seed.cid $"($label) cid" + assert-equal $record.value.'$type' $seed.collection $"($label) body type" + assert-equal $record.value.text "eight-byte rkey fixture" $"($label) body text" +} + + +def test-record-endpoints [url: string, debug_url: string, seed: record] { + let record = (http get $"($url)/xrpc/com.atproto.repo.getRecord?repo=($seed.did)&collection=($seed.collection)&rkey=($seed.rkey)") + assert-record $record $seed "getRecord" + + let listed = (http get $"($url)/xrpc/com.atproto.repo.listRecords?repo=($seed.did)&collection=($seed.collection)") + assert-equal ($listed.records | length) 1 "listRecords count" + assert-record ($listed.records | first) $seed "listRecords" + + let at_uri = $"at://($seed.did)/($seed.collection)/($seed.rkey)" + for path in [ + "blue.microcosm.repo.getRecordByUri" + "com.bad-example.repo.getUriRecord" + ] { + let by_uri = (http get $"($url)/xrpc/($path)?at_uri=($at_uri)&cid=($seed.cid)") + assert-record $by_uri $seed $path + } + + let count = (http get $"($url)/xrpc/systems.gaze.hydrant.countRecords?identifier=($seed.did)&collection=($seed.collection)") + assert-equal $count.count 1 "countRecords" + + let debug_count = (http get $"($debug_url)/debug/count?did=($seed.did)&collection=($seed.collection)") + assert-equal $debug_count.count 1 "debug count" +} + + +def test-repo-endpoints [url: string, seed: record] { + let latest = (http get $"($url)/xrpc/com.atproto.sync.getLatestCommit?did=($seed.did)") + assert-equal $latest.cid $seed.commit_cid "getLatestCommit cid" + assert-equal $latest.rev $seed.rev "getLatestCommit rev" + + let status = (http get $"($url)/xrpc/com.atproto.sync.getRepoStatus?did=($seed.did)") + assert-equal $status.did $seed.did "getRepoStatus did" + assert-equal $status.active true "getRepoStatus active" + assert-equal $status.rev $seed.rev "getRepoStatus rev" + if ('status' in $status) { + fail "synced getRepoStatus must omit status" + } + + let repos = (http get $"($url)/xrpc/com.atproto.sync.listRepos?limit=1") + assert-equal ($repos.repos | length) 1 "listRepos count" + let listed = ($repos.repos | first) + assert-equal $listed.did $seed.did "listRepos did" + assert-equal $listed.head $seed.commit_cid "listRepos head" + assert-equal $listed.rev $seed.rev "listRepos rev" + assert-equal $listed.active true "listRepos active" + + let described = (http get $"($url)/xrpc/systems.gaze.hydrant.describeRepo?identifier=($seed.did)") + assert-equal $described.did $seed.did "describeRepo did" + assert-equal $described.handle "handle.invalid" "describeRepo invalid handle" + assert-equal $described.pds $"https://($seed.hostname)/" "describeRepo pds" + if not ($described.signing_key | str starts-with "z") { + fail "describeRepo signing key is not rendered as multibase" + } + assert-equal ($described.collections | sort) [$seed.collection] "describeRepo collections" + + for path in [ + "blue.microcosm.identity.resolveMiniDoc" + "com.bad-example.identity.resolveMiniDoc" + ] { + let doc = (http get $"($url)/xrpc/($path)?identifier=($seed.did)") + assert-equal $doc.did $seed.did $"($path) did" + assert-equal $doc.handle "handle.invalid" $"($path) handle" + assert-equal $doc.pds $"https://($seed.hostname)/" $"($path) pds" + assert-equal $doc.signing_key $described.signing_key $"($path) signing key" + } + + let car = (http get $"($url)/xrpc/com.atproto.sync.getRepo?did=($seed.did)") + if (($car | bytes length) < 100) { + fail "getRepo returned an implausibly small CAR" + } +} + + +def test-host-endpoints [url: string, seed: record] { + let host = (http get $"($url)/xrpc/com.atproto.sync.getHostStatus?hostname=($seed.hostname)") + assert-equal $host.hostname $seed.hostname "getHostStatus hostname" + assert-equal $host.seq 42 "getHostStatus seq" + assert-equal $host.accountCount 1 "getHostStatus account count" + assert-equal $host.status "active" "getHostStatus status" + + let hosts = (http get $"($url)/xrpc/com.atproto.sync.listHosts?limit=1") + assert-equal ($hosts.hosts | length) 1 "listHosts count" + let listed = ($hosts.hosts | first) + assert-equal $listed.hostname $seed.hostname "listHosts hostname" + assert-equal $listed.seq 42 "listHosts seq" + assert-equal $listed.accountCount 1 "listHosts account count" + assert-equal $listed.status "active" "listHosts status" +} + + +def main [] { + let port = resolve-test-port 3024 + let url = $"http://localhost:($port)" + let debug_url = $"http://localhost:($port + 1)" + let db_path = (mktemp -d -t hydrant_api_xrpc.XXXXXX) + let binary = build-hydrant + let instance = (with-env { + HYDRANT_MODE: "filter" + HYDRANT_ENABLE_CRAWLER: "false" + HYDRANT_ENABLE_FIREHOSE: "false" + HYDRANT_ENABLE_BACKFILL: "false" + } { + start-hydrant $binary $db_path $port + }) + + if not (wait-for-api $url) { + fail "hydrant did not start" $instance.pid + } + + try { + let seed = (http post -H [Content-Length 0] $"($debug_url)/debug/seed_xrpc_repo" "") + assert-equal $seed.collection $COLLECTION "seed collection" + assert-equal $seed.rkey $RKEY "seed rkey" + + test-record-endpoints $url $debug_url $seed + test-repo-endpoints $url $seed + test-host-endpoints $url $seed + print "deterministic XRPC representation boundaries passed" + } catch { |err| + try { kill --force $instance.pid } + fail $"XRPC representation harness failed: ($err.msg)" + } + + kill --force $instance.pid +} diff --git a/tests/backlinks.nu b/tests/backlinks.nu --- a/tests/backlinks.nu +++ b/tests/backlinks.nu @@ -1,263 +1,138 @@ #!/usr/bin/env nu -# tests the blue.microcosm.links backlinks API endpoints. -# -# backfills a single repo and verifies: -# 1. getBacklinks returns a result for a subject URI extracted from an indexed like record -# 2. getBacklinksCount is >= number of getBacklinks results for the same subject -# 3. cursor pagination returns each result exactly once with no duplicates -# 4. source filter restricts results to the specified collection -# 5. reverse=true returns the same set as forward, with inverted order verified when -# a subject with 2+ backlinks is found -# -# usage: nu tests/backlinks.nu +# tests the blue.microcosm.links backlinks API with deterministic local fixtures use common.nu * -# paginate through all backlinks for subject+source using limit=2, return all entries -def get-all-backlinks [url: string, subject: string, source: string] { +def get-all-backlinks [url: string, subject: string, source: string, reverse: bool] { mut all = [] - mut cursor = null + mut cursor = "" loop { - let params = if $cursor != null { - $"subject=($subject | url encode)&source=($source)&limit=2&cursor=($cursor)" + let reverse_param = if $reverse { "&reverse=true" } else { "" } + let params = if ($cursor | is-empty) { + $"subject=($subject | url encode)&source=($source | url encode)&limit=2($reverse_param)" } else { - $"subject=($subject | url encode)&source=($source)&limit=2" + $"subject=($subject | url encode)&source=($source | url encode)&limit=2&cursor=($cursor | url encode)($reverse_param)" } - let resp = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?($params)") - $all = ($all | append $resp.backlinks) - let next = ($resp | get --optional cursor) - if ($next | is-empty) { break } + let response = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?($params)") + $all = ($all | append $response.backlinks) + let next = ($response | get --optional cursor) + if ($next | is-empty) { + break + } $cursor = $next } $all } -# scan up to `limit` reply posts looking for a thread root that has 2+ backlinks -# from app.bsky.feed.post. returns a record {subject, count} or null. -def find-multi-backlink-subject [url: string, did: string, limit: int] { - let posts_resp = (try { - http get $"($url)/xrpc/com.atproto.repo.listRecords?repo=($did)&collection=app.bsky.feed.post&limit=($limit)" - } catch { - return null - }) - - for post in $posts_resp.records { - let reply = ($post.value | get --optional reply) - if ($reply | is-empty) { continue } - let root_uri = $reply.root.uri - let count_resp = (try { - http get $"($url)/xrpc/blue.microcosm.links.getBacklinksCount?subject=($root_uri | url encode)" - } catch { - continue - }) - if $count_resp.count >= 2 { - return {subject: $root_uri, count: $count_resp.count} - } - } - - null -} - -# verify getBacklinks and getBacklinksCount return sensible results for sampled likes. -# returns an error string on failure, or empty string on success. -def check-basic [url: string, likes: list] { - print "checking basic backlinks and count..." - - for like in $likes { - let subject = $like.value.subject.uri - print $" subject: ($subject)" - - let bl_resp = (try { - http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject | url encode)&source=app.bsky.feed.like" - } catch { - return $"getBacklinks request failed for ($subject)" - }) - - if ($bl_resp.backlinks | is-empty) { - return $"expected at least 1 backlink for ($subject)" - } - let bl_count = ($bl_resp.backlinks | length) - print $" getBacklinks: ($bl_count) results" - - let ct_resp = (try { - http get $"($url)/xrpc/blue.microcosm.links.getBacklinksCount?subject=($subject | url encode)&source=app.bsky.feed.like" - } catch { - return $"getBacklinksCount request failed for ($subject)" - }) - - # count may exceed backlinks results because stale index entries (records deleted - # after indexing) are counted but skipped during result collection - if $ct_resp.count < ($bl_resp.backlinks | length) { - return $"count ($ct_resp.count) < backlinks result count ($bl_resp.backlinks | length) for ($subject)" - } - print $" getBacklinksCount: ($ct_resp.count)" - } - - "" -} - -# verify that cursor pagination returns every result exactly once with no duplicates. -# returns an error string on failure, or empty string on success. -def check-pagination [url: string, likes: list] { - print "checking cursor pagination..." - let subject = ($likes | first).value.subject.uri - print $" subject: ($subject)" - - # fetch all in a single shot with a large limit - let full = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject | url encode)&source=app.bsky.feed.like&limit=100") - let full_uris = ($full.backlinks | each { |b| $b.uri } | sort) - - # paginate through the same subject with limit=2 - let paged = (get-all-backlinks $url $subject "app.bsky.feed.like") - let paged_uris = ($paged | each { |b| $b.uri } | sort) - - print $" full fetch: ($full_uris | length), paginated: ($paged_uris | length)" - - if $full_uris != $paged_uris { - return "paginated results differ from single-page results" - } - - let unique_count = ($paged_uris | uniq | length) - if $unique_count != ($paged_uris | length) { - return "duplicate URIs found in paginated results" - } - - print " pagination OK" - "" -} - -# verify source filter and that forward/reverse return the same set. -# returns an error string on failure, or empty string on success. -def check-source-filter [url: string, likes: list] { - print "checking source filter..." - let subject = ($likes | first).value.subject.uri - print $" subject: ($subject)" - - let fwd = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject | url encode)&source=app.bsky.feed.like&limit=50") - let rev = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject | url encode)&source=app.bsky.feed.like&limit=50&reverse=true") - - # source filter: all returned URIs must belong to app.bsky.feed.like - let bad = ($fwd.backlinks | where { |b| not ($b.uri | str contains "/app.bsky.feed.like/") }) - if not ($bad | is-empty) { - return $"source filter returned non-like records: ($bad)" - } - let fwd_count = ($fwd.backlinks | length) - print $" source filter OK: ($fwd_count) likes" - - # reverse must return the same set - let fwd_sorted = ($fwd.backlinks | each { |b| $b.uri } | sort) - let rev_sorted = ($rev.backlinks | each { |b| $b.uri } | sort) - if $fwd_sorted != $rev_sorted { - return "forward and reverse scans returned different sets" - } - - print " reverse set equality OK" - "" -} - -# verify that reverse=true actually inverts the order using a subject with 2+ backlinks. -# returns an error string on failure, or empty string on success. -def check-reverse-ordering [url: string, subject: string, expected_count: int] { - print $"checking reverse ordering... subject has ($expected_count) backlinks..." - print $" subject: ($subject)" - - let fwd = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject | url encode)&limit=50") - let rev = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject | url encode)&limit=50&reverse=true") - - let fwd_uris = ($fwd.backlinks | each { |b| $b.uri }) - let rev_uris = ($rev.backlinks | each { |b| $b.uri }) - - if ($fwd_uris | length) < 2 { - return $"expected >= 2 forward results for ordering test, got ($fwd_uris | length)" - } - - if $fwd_uris != ($rev_uris | reverse) { - return "reverse order is not the inverse of forward order" - } - - print $" order inversion verified with ($fwd_uris | length) results" - "" -} - def main [] { - let did = ($env | get --optional BACKLINKS_TEST_DID | default "did:plc:oc6vwdlmk2kqyida5i74d3p5") let port = resolve-test-port 3020 - let url = $"http://localhost:($port)" + let debug_port = resolve-test-debug-port ($port + 1) + let url = $"http://127.0.0.1:($port)" + let debug_url = $"http://127.0.0.1:($debug_port)" let db_path = (mktemp -d -t hydrant_backlinks.XXXXXX) print $"database path: ($db_path)" - let binary = (build-hydrant-features "backlinks") - let instance = (with-env { HYDRANT_ENABLE_CRAWLER: "false", HYDRANT_ENABLE_FIREHOSE: "false" } { + let instance = (with-env { + HYDRANT_RELAY_HOSTS: "", + HYDRANT_CRAWLER_URLS: "", + HYDRANT_ENABLE_CRAWLER: "false", + HYDRANT_ENABLE_FIREHOSE: "false", + HYDRANT_ENABLE_BACKFILL: "false", + } { start-hydrant $binary $db_path $port }) if not (wait-for-api $url) { - print "ERROR: hydrant failed to start" - try { kill -9 $instance.pid } - rm -rf $db_path - exit 1 + fail "hydrant failed to start" $instance.pid } - print $"adding ($did) to tracking..." - http put -t application/json $"($url)/repos" [{ did: $did }] + let seeded = ( + http post -f -e -H [Content-Length 0] + $"($debug_url)/debug/seed_backlinks?like_count=5&post_count=2" "" + ) + if $seeded.status != 200 { + fail $"backlink fixture seed returned ($seeded.status)" $instance.pid + } + let fixture = $seeded.body + let subject = $fixture.subject + let did = $fixture.did + let subject_param = ($subject | url encode) - if not (wait-for-backfill $url) { - print "ERROR: backfill timed out or failed" - try { kill -9 $instance.pid } - rm -rf $db_path - exit 1 + print "checking unfiltered results and count..." + let all = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject_param)&limit=100") + let all_count = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinksCount?subject=($subject_param)") + let expected_total = ($fixture.like_count + $fixture.post_count) + if ($all.backlinks | length) != $expected_total or $all_count.count != $expected_total { + fail $"expected ($expected_total) backlinks, got results=($all.backlinks | length) count=($all_count.count)" $instance.pid } - # fetch a small set of like records to use as test subjects - print "fetching likes from indexed repo..." - let likes_resp = (try { - http get $"($url)/xrpc/com.atproto.repo.listRecords?repo=($did)&collection=app.bsky.feed.like&limit=5" - } catch { - print "ERROR: could not fetch like records" - try { kill -9 $instance.pid } - rm -rf $db_path - exit 1 - }) - - let likes = $likes_resp.records - if ($likes | is-empty) { - print "SKIP: no like records found for test DID. cannot verify backlinks" - try { kill -9 $instance.pid } - rm -rf $db_path - exit 0 + print "checking collection and path filters..." + let like_source = "app.bsky.feed.like" + let like_source_param = ($like_source | url encode) + let likes = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject_param)&source=($like_source_param)&limit=100") + let like_count = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinksCount?subject=($subject_param)&source=($like_source_param)") + if ($likes.backlinks | length) != $fixture.like_count or $like_count.count != $fixture.like_count { + fail "like source filter returned the wrong cardinality" $instance.pid } - print $"found ($likes | length) likes for testing" - - # find a thread root with 2+ backlinks for ordering verification - print "scanning posts for a thread root with 2+ backlinks..." - let multi = (find-multi-backlink-subject $url $did 100) - if ($multi | is-empty) { - print " note: no thread root with 2+ backlinks found, ordering verification will be skipped" - } else { - print $" found subject with ($multi.count) backlinks: ($multi.subject)" + if ($likes.backlinks | any { |entry| not ($entry.uri | str contains "/app.bsky.feed.like/") }) { + fail "like source filter returned another collection" $instance.pid + } + if not ($likes.backlinks | any { |entry| $entry.uri | str ends-with "/stinkpot" }) { + fail "eight-byte rkey was not preserved through backlink projection" $instance.pid + } + if ($likes.backlinks | any { |entry| ($entry.cid | is-empty) }) { + fail "backlink result omitted a record CID" $instance.pid } - let basic_err = (check-basic $url $likes) - let paging_err = (check-pagination $url $likes) - let srcfilt_err = (check-source-filter $url $likes) - let ordering_err = if not ($multi | is-empty) { - check-reverse-ordering $url $multi.subject $multi.count - } else { - print "skipping reverse ordering check (no multi-backlink subject found)" - "" + let path_source = ("app.bsky.feed.like:subject.uri" | url encode) + let path_filtered = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject_param)&source=($path_source)&limit=100") + if ($path_filtered.backlinks | length) != $fixture.like_count { + fail "collection:path filter returned the wrong cardinality" $instance.pid } - let failed = ([$basic_err, $paging_err, $srcfilt_err, $ordering_err] | where { |e| not ($e | is-empty) }) - - print "stopping hydrant..." - try { kill -9 $instance.pid } - rm -rf $db_path - - if ($failed | is-empty) { - print "all backlinks tests PASSED" - exit 0 - } else { - for err in $failed { print $"FAILED: ($err)" } - exit 1 + let post_source = ("app.bsky.feed.post" | url encode) + let posts = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject_param)&source=($post_source)&limit=100") + if ($posts.backlinks | length) != $fixture.post_count { + fail "post source filter returned the wrong cardinality" $instance.pid } + + print "checking cursor pagination..." + let paged = (get-all-backlinks $url $subject $like_source false) + let full_uris = ($likes.backlinks | get uri) + let paged_uris = ($paged | get uri) + if $paged_uris != $full_uris or ($paged_uris | uniq | length) != $fixture.like_count { + fail "cursor pagination skipped or duplicated backlinks" $instance.pid + } + + print "checking reverse and DID filters..." + let reverse = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject_param)&source=($like_source_param)&limit=100&reverse=true") + if ($reverse.backlinks | get uri) != ($full_uris | reverse) { + fail "reverse scan did not invert result order" $instance.pid + } + let reverse_paged = (get-all-backlinks $url $subject $like_source true) + if ($reverse_paged | get uri) != ($full_uris | reverse) { + fail "reverse cursor pagination skipped or duplicated backlinks" $instance.pid + } + + let matching_did = ($did | url encode) + let matching = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject_param)&did=($matching_did)&limit=100") + if ($matching.backlinks | length) != $expected_total { + fail "matching DID filter dropped backlinks" $instance.pid + } + let other_did = ("did:plc:ewvi7nxzyoun6zhxrhs64oiz" | url encode) + let excluded = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject_param)&did=($other_did)&limit=100") + if not ($excluded.backlinks | is-empty) { + fail "non-matching DID filter returned backlinks" $instance.pid + } + let repeated = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject_param)&did=($other_did)&did=($matching_did)&limit=100") + if ($repeated.backlinks | length) != $expected_total { + fail "repeated DID filters were not decoded as an array" $instance.pid + } + let repeated_count = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinksCount?subject=($subject_param)&did=($other_did)&did=($matching_did)") + if $repeated_count.count != $expected_total { + fail "repeated DID count filters were not decoded as an array" $instance.pid + } + + try { kill $instance.pid } + print "all backlinks tests PASSED" } diff --git a/tests/common.nu b/tests/common.nu --- a/tests/common.nu +++ b/tests/common.nu @@ -113,15 +113,19 @@ | where { |r| $r.reason? == "compiler-artifact" and $r.executable? != null and ($r.target?.name? == "hydrant") }) if ($items | is-empty) { - for candidate in ["target/aarch64-apple-darwin/debug/hydrant" "target/debug/hydrant"] { - if ($candidate | path exists) { - return $candidate - } - } - error make {msg: "could not determine hydrant executable path"} - } else { - $items | last | get executable + error make {msg: "cargo succeeded without reporting a hydrant executable"} } + + $items | last | get executable +} + +def require-build-success [out: record] { + if $out.exit_code != 0 { + let diagnostics = ([$out.stdout $out.stderr] | str join "\n" | str trim) + error make {msg: $"cargo build failed:\n($diagnostics)"} + } + + parse-hydrant-executable $out.stdout } # build the hydrant binary @@ -130,8 +134,8 @@ return $env.HYDRANT_BINARY } print "building hydrant..." - let out = (^cargo build --message-format json err> /dev/null | complete) - parse-hydrant-executable $out.stdout + let out = (^cargo build --message-format json | complete) + require-build-success $out } # build the hydrant binary with extra cargo features (space-separated string) @@ -140,8 +144,8 @@ return $env.HYDRANT_BINARY } print $"building hydrant with features: ($features)..." - let out = (^cargo build --features $features --message-format json err> /dev/null | complete) - parse-hydrant-executable $out.stdout + let out = (^cargo build --features $features --message-format json | complete) + require-build-success $out } # build the hydrant binary for relay only (no default features) @@ -150,8 +154,8 @@ return $env.HYDRANT_BINARY } print "building hydrant for relay only..." - let out = (^cargo build --no-default-features --features relay --message-format json err> /dev/null | complete) - parse-hydrant-executable $out.stdout + let out = (^cargo build --no-default-features --features relay --message-format json | complete) + require-build-success $out } # build the hydrant binary for relay with jetstream support (no default features) @@ -160,12 +164,17 @@ return $env.HYDRANT_BINARY } print "building hydrant for relay with jetstream..." - let out = (^cargo build --no-default-features --features relay,jetstream --message-format json err> /dev/null | complete) - parse-hydrant-executable $out.stdout + let out = (^cargo build --no-default-features --features relay,jetstream --message-format json | complete) + require-build-success $out } # start hydrant in the background export def start-hydrant [binary: string, db_path: string, port: int] { + let occupied = (^bash -c $"echo > /dev/tcp/127.0.0.1/($port)" | complete) + if $occupied.exit_code == 0 { + error make {msg: $"cannot start test hydrant: port ($port) is already in use"} + } + let log_file = $"($db_path)/hydrant.log" print $"starting hydrant - logs at ($log_file)..." diff --git a/tests/mock_pds.nu b/tests/mock_pds.nu --- a/tests/mock_pds.nu +++ b/tests/mock_pds.nu @@ -1,9 +1,19 @@ +def port-open [port: int] { + (^bash -c $"echo > /dev/tcp/127.0.0.1/($port)" | complete).exit_code == 0 +} + export def start-mock-pds [port: int] { - # kill any stale process from a previous failed run holding this port - try { bash -c $"fuser -k ($port)/tcp" } catch {} - sleep 100ms + if (port-open $port) { + error make {msg: $"cannot start mock PDS: port ($port) is already in use"} + } let log_file = (mktemp) let pid = (bash -c $"websocat --no-async-stdio -s ($port) >($log_file) 2>&1 & echo $!" | str trim | into int) + sleep 100ms + if not (port-open $port) { + try { kill $pid } + let log = (try { open --raw $log_file | into string | str trim } catch { "" }) + error make {msg: $"mock PDS failed to listen on port ($port): ($log)"} + } { pid: $pid, log: $log_file } } diff --git a/tests/representation_audit.rs b/tests/representation_audit.rs new file mode 100644 --- /dev/null +++ b/tests/representation_audit.rs @@ -0,0 +1,994 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +use syn::visit::{self, Visit}; +use syn::{ + Attribute, Expr, ExprCall, ExprMethodCall, GenericArgument, ImplItemFn, ItemConst, ItemEnum, + ItemFn, ItemImpl, ItemMod, ItemStatic, ItemStruct, ItemUnion, PathArguments, Type, Visibility, +}; +const INVENTORY_PATH: &str = "tests/representation_inventory.tsv"; + +#[derive(Default)] +struct Surface { + boundaries: BTreeSet, + tests: BTreeSet, +} + +struct Scanner<'a> { + relative_path: &'a str, + modules: Vec, + impl_name: Option, + function: Option, + test_only: bool, + surface: &'a mut Surface, +} + +impl Scanner<'_> { + fn location(&self, symbol: &str) -> String { + let mut parts = vec![self.relative_path.to_owned()]; + parts.extend(self.modules.iter().cloned()); + if let Some(name) = &self.impl_name { + parts.push(name.clone()); + } + parts.push(symbol.to_owned()); + parts.join("::") + } + + fn owner(&self) -> String { + self.function + .as_deref() + .map(|function| self.location(function)) + .unwrap_or_else(|| self.location("")) + } + + fn record(&mut self, kind: &str, symbol: &str) { + self.surface + .boundaries + .insert(format!("{kind}:{}", self.location(symbol))); + } + + fn record_call(&mut self, kind: &str, callee: &str) { + self.surface + .boundaries + .insert(format!("{kind}:{}::{callee}", self.owner())); + } + + fn enter_function(&mut self, name: String, block: &syn::Block) { + let previous = self.function.replace(name); + self.visit_block(block); + self.function = previous; + } + + fn record_serde_attributes(&mut self, owner: &str, attrs: &[Attribute]) { + for attr in attrs.iter().filter(|attr| attr.path().is_ident("serde")) { + let value = match &attr.meta { + syn::Meta::List(list) => list.tokens.to_string(), + syn::Meta::NameValue(_) => "".to_owned(), + syn::Meta::Path(_) => String::new(), + }; + let value = value.split_whitespace().collect::(); + self.record("serde-attribute", &format!("{owner}::serde({value})")); + } + } + + fn record_function_categories(&mut self, name: &str, visibility: &Visibility) { + let is_key_codec_module = self.relative_path.starts_with("src/db/keys/") + || self.relative_path == "src/db/keys.rs"; + let is_embedded_key_codec = matches!( + self.relative_path, + "src/backlinks/store.rs" | "src/db/filter.rs" | "src/db/pds_meta.rs" + ) && looks_like_key_codec(name) + || self.relative_path.starts_with("src/db/migration/") && looks_like_key_codec(name); + if is_key_codec_module || is_embedded_key_codec { + self.record("db-key-codec", name); + } + if looks_like_serde_helper(name) { + self.record("serde-helper", name); + } + if self.relative_path.starts_with("src/db/migration/") { + self.record("migration-transform", name); + } + if self.relative_path == "src/config/env.rs" + || ((self.relative_path.starts_with("src/config/") + || self.relative_path == "src/config.rs") + && (name == "parse" || name.starts_with("parse_"))) + { + self.record("config-parser", name); + } + if self.relative_path == "src/ingest/stream/codec.rs" { + self.record("firehose-codec", name); + } + if self.relative_path == "src/car.rs" { + self.record("car-codec", name); + } + if (self.relative_path.starts_with("src/api/") || self.relative_path == "src/api.rs") + && !matches!(visibility, Visibility::Inherited) + { + self.record("api-boundary", name); + } + if self.relative_path.starts_with("src/control/stream/") + && !matches!(visibility, Visibility::Inherited) + { + self.record("stream-projection", name); + } + } +} + +impl<'ast> Visit<'ast> for Scanner<'_> { + fn visit_item_mod(&mut self, node: &'ast ItemMod) { + let previous_test_only = self.test_only; + self.test_only |= is_test_only(&node.attrs); + self.modules.push(node.ident.to_string()); + if let Some((_, items)) = &node.content { + for item in items { + self.visit_item(item); + } + } + self.modules.pop(); + self.test_only = previous_test_only; + } + + fn visit_item_struct(&mut self, node: &'ast ItemStruct) { + if !self.test_only { + let name = node.ident.to_string(); + if has_serde_derive(&node.attrs) { + self.record("serde-type", &name); + } + self.record_serde_attributes(&name, &node.attrs); + for (index, field) in node.fields.iter().enumerate() { + let field_name = field + .ident + .as_ref() + .map(ToString::to_string) + .unwrap_or_else(|| index.to_string()); + self.record_serde_attributes(&format!("{name}::{field_name}"), &field.attrs); + } + } + visit::visit_item_struct(self, node); + } + + fn visit_item_enum(&mut self, node: &'ast ItemEnum) { + if !self.test_only { + let name = node.ident.to_string(); + if has_serde_derive(&node.attrs) { + self.record("serde-type", &name); + } + self.record_serde_attributes(&name, &node.attrs); + for variant in &node.variants { + let variant_name = format!("{name}::{}", variant.ident); + self.record_serde_attributes(&variant_name, &variant.attrs); + for (index, field) in variant.fields.iter().enumerate() { + let field_name = field + .ident + .as_ref() + .map(ToString::to_string) + .unwrap_or_else(|| index.to_string()); + self.record_serde_attributes( + &format!("{variant_name}::{field_name}"), + &field.attrs, + ); + } + } + } + visit::visit_item_enum(self, node); + } + + fn visit_item_union(&mut self, node: &'ast ItemUnion) { + if !self.test_only { + let name = node.ident.to_string(); + if has_serde_derive(&node.attrs) { + self.record("serde-type", &name); + } + self.record_serde_attributes(&name, &node.attrs); + for field in &node.fields.named { + self.record_serde_attributes( + &format!("{name}::{}", field.ident.as_ref().unwrap()), + &field.attrs, + ); + } + } + visit::visit_item_union(self, node); + } + + fn visit_item_const(&mut self, node: &'ast ItemConst) { + if !self.test_only && records_storage_constant(self.relative_path, &node.ident.to_string()) + { + self.record("db-key-constant", &node.ident.to_string()); + } + visit::visit_item_const(self, node); + } + + fn visit_item_static(&mut self, node: &'ast ItemStatic) { + if !self.test_only && records_storage_constant(self.relative_path, &node.ident.to_string()) + { + self.record("db-key-constant", &node.ident.to_string()); + } + visit::visit_item_static(self, node); + } + + fn visit_item_impl(&mut self, node: &'ast ItemImpl) { + if self.test_only || is_test_only(&node.attrs) { + return; + } + + let self_name = type_name(&node.self_ty); + let conversion_owner = type_signature(&node.self_ty); + if let Some((_, path, _)) = &node.trait_ + && let Some(trait_name) = path + .segments + .last() + .map(|segment| segment.ident.to_string()) + { + match trait_name.as_str() { + "Serialize" | "Deserialize" => { + self.record("manual-serde", &format!("{self_name}::{trait_name}")) + } + "FromStr" => self.record("text-parser", &format!("{self_name}::{trait_name}")), + "From" | "TryFrom" | "IntoStatic" | "Display" | "AsRef" + if records_representation_conversions(self.relative_path) => + { + self.record( + "representation-conversion", + &format!("{conversion_owner}::{}", path_signature(path)), + ) + } + "Schema" if self.relative_path == "src/db/schema.rs" => { + self.record("db-keyspace", &self_name) + } + _ => {} + } + } + + let previous = self.impl_name.replace(self_name); + for item in &node.items { + self.visit_impl_item(item); + } + self.impl_name = previous; + } + + fn visit_item_fn(&mut self, node: &'ast ItemFn) { + let name = node.sig.ident.to_string(); + if self.test_only || is_test_only(&node.attrs) { + if is_test(&node.attrs) { + self.surface + .tests + .insert(format!("rust:{}::{name}", self.relative_path)); + } + return; + } + + self.record_function_categories(&name, &node.vis); + self.enter_function(name, &node.block); + } + + fn visit_impl_item_fn(&mut self, node: &'ast ImplItemFn) { + let name = node.sig.ident.to_string(); + if self.test_only || is_test_only(&node.attrs) { + if is_test(&node.attrs) { + self.surface + .tests + .insert(format!("rust:{}::{name}", self.relative_path)); + } + return; + } + + self.record_function_categories(&name, &node.vis); + self.enter_function(name, &node.block); + } + + fn visit_expr_call(&mut self, node: &'ast ExprCall) { + if !self.test_only + && let Expr::Path(path) = node.func.as_ref() + { + let callee = path + .path + .segments + .iter() + .map(|segment| segment.ident.to_string()) + .collect::>() + .join("::"); + if is_codec_call(&callee) { + self.record_call("codec-call", &callee); + } + } + visit::visit_expr_call(self, node); + } + + fn visit_expr_method_call(&mut self, node: &'ast ExprMethodCall) { + if !self.test_only { + let method = node.method.to_string(); + if matches!( + method.as_str(), + "serialize" | "deserialize" | "to_be_bytes" | "to_le_bytes" + ) || (self.relative_path == "src/config/env.rs" && method == "parse") + { + self.record_call("codec-call", &format!(".{method}")); + } + } + visit::visit_expr_method_call(self, node); + } +} + +fn records_representation_conversions(relative_path: &str) -> bool { + matches!( + relative_path, + "src/config.rs" + | "src/config/types.rs" + | "src/control/pds.rs" + | "src/db/types.rs" + | "src/ingest/stream/types.rs" + | "src/pds_meta.rs" + | "src/types.rs" + ) || relative_path.starts_with("src/types/event/") +} + +fn records_storage_constant(relative_path: &str, name: &str) -> bool { + let storage_source = relative_path.starts_with("src/db/") || relative_path == "src/db.rs"; + let embedded_key_source = matches!( + relative_path, + "src/backlinks/store.rs" | "src/filter.rs" | "src/pds_meta.rs" + ); + (storage_source || embedded_key_source) + && (name == "SEP" + || name.starts_with("TAG_") + || name.contains("_KEY") + || name.contains("_PREFIX") + || name.ends_with("_SEP")) +} + +fn looks_like_key_codec(name: &str) -> bool { + name.ends_with("_key") + || name.contains("_key_") + || name.ends_with("_prefix") + || name.contains("_prefix_") +} + +fn looks_like_serde_helper(name: &str) -> bool { + name != "serialize" + && name != "deserialize" + && (name.starts_with("serialize_") + || name.starts_with("deserialize_") + || name.starts_with("ser_") + || name.starts_with("deser_") + || name.contains("_serialize_") + || name.contains("_deserialize_") + || name.contains("_ser_") + || name.contains("_deser_") + || name.ends_with("_ser") + || name.ends_with("_deser")) +} + +fn is_codec_call(callee: &str) -> bool { + const ROOTS: &[&str] = &[ + "rmp_serde::", + "serde_json::", + "serde_ipld_dagcbor::", + "serde_html_form::", + "iroh_car::", + "data_encoding::", + "multibase::", + "hex::", + ]; + ROOTS.iter().any(|root| callee.contains(root)) + || callee.contains("CarReader") + || callee.contains("CarWriter") + || callee.ends_with("::serialize") + || callee.ends_with("::deserialize") + || callee.ends_with("::from_be_bytes") + || callee.ends_with("::from_le_bytes") +} + +fn path_signature(path: &syn::Path) -> String { + path.segments + .iter() + .map(|segment| { + let arguments = match &segment.arguments { + PathArguments::None => String::new(), + PathArguments::AngleBracketed(arguments) => { + let types = arguments + .args + .iter() + .filter_map(|argument| match argument { + GenericArgument::Type(ty) => Some(type_signature(ty)), + GenericArgument::AssocType(assoc) => { + Some(format!("{}={}", assoc.ident, type_signature(&assoc.ty))) + } + _ => None, + }) + .collect::>(); + format!("<{}>", types.join(",")) + } + PathArguments::Parenthesized(_) => "(..)".to_owned(), + }; + format!("{}{arguments}", segment.ident) + }) + .collect::>() + .join("::") +} + +fn type_signature(ty: &Type) -> String { + match ty { + Type::Path(path) => path_signature(&path.path), + Type::Reference(reference) => format!("&{}", type_signature(&reference.elem)), + Type::Slice(slice) => format!("[{}]", type_signature(&slice.elem)), + Type::Array(array) => format!("[{};_]", type_signature(&array.elem)), + Type::Tuple(tuple) => format!( + "({})", + tuple + .elems + .iter() + .map(type_signature) + .collect::>() + .join(",") + ), + _ => "".to_owned(), + } +} + +fn type_name(ty: &Type) -> String { + match ty { + Type::Path(path) => path + .path + .segments + .last() + .map(|segment| segment.ident.to_string()) + .unwrap_or_else(|| "".to_owned()), + Type::Reference(reference) => type_name(&reference.elem), + _ => "".to_owned(), + } +} + +fn has_serde_derive(attrs: &[Attribute]) -> bool { + attrs.iter().any(|attr| { + attr.path().is_ident("derive") + && match &attr.meta { + syn::Meta::List(list) => { + let derives = list.tokens.to_string(); + derives + .split(|ch: char| !ch.is_alphanumeric() && ch != '_') + .any(|derive| matches!(derive, "Serialize" | "Deserialize")) + } + _ => false, + } + }) +} + +fn is_test(attrs: &[Attribute]) -> bool { + attrs.iter().any(|attr| { + attr.path() + .segments + .last() + .is_some_and(|segment| segment.ident == "test") + }) +} + +fn is_test_only(attrs: &[Attribute]) -> bool { + attrs.iter().any(|attr| { + is_test(std::slice::from_ref(attr)) + || (attr.path().is_ident("cfg") + && matches!(&attr.meta, syn::Meta::List(list) if list.tokens.to_string().split(|ch: char| !ch.is_alphanumeric() && ch != '_').any(|word| word == "test"))) + }) +} + +fn rust_files(root: &Path) -> Vec { + fn collect(path: &Path, files: &mut Vec) { + let mut entries = fs::read_dir(path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())) + .collect::, _>>() + .unwrap_or_else(|error| panic!("failed to enumerate {}: {error}", path.display())); + entries.sort_by_key(|entry| entry.path()); + for entry in entries { + let path = entry.path(); + if path.is_dir() { + collect(&path, files); + } else if path.extension().is_some_and(|extension| extension == "rs") { + files.push(path); + } + } + } + + let mut files = Vec::new(); + collect(root, &mut files); + files +} + +fn discover(manifest: &Path) -> Surface { + let mut surface = Surface::default(); + for path in rust_files(&manifest.join("src")) { + let relative = path + .strip_prefix(manifest) + .expect("source path must be below manifest") + .to_string_lossy() + .replace('\\', "/"); + let source = fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())); + let syntax = syn::parse_file(&source) + .unwrap_or_else(|error| panic!("failed to parse {}: {error}", path.display())); + let mut scanner = Scanner { + relative_path: &relative, + modules: Vec::new(), + impl_name: None, + function: None, + test_only: false, + surface: &mut surface, + }; + scanner.visit_file(&syntax); + } + surface +} + +fn load_inventory(manifest: &Path) -> BTreeMap { + let path = manifest.join(INVENTORY_PATH); + let input = fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())); + let mut inventory = BTreeMap::new(); + for (index, line) in input.lines().enumerate() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let (boundary, coverage) = line + .split_once('\t') + .unwrap_or_else(|| panic!("{}:{} must contain one tab", path.display(), index + 1)); + assert!( + inventory + .insert(boundary.to_owned(), coverage.to_owned()) + .is_none(), + "{}:{} duplicates boundary {boundary}", + path.display(), + index + 1 + ); + } + inventory +} + +fn proposed_coverage(boundary: &str, surface: &Surface) -> String { + let location = boundary + .split_once(':') + .map(|(_, location)| location) + .expect("boundary has a kind"); + let source = location + .split_once("::") + .map(|(source, _)| source) + .expect("boundary has a source"); + + let exact = if source == "src/api/mod.rs" && boundary.contains("MultiQuery") { + Some("rust:src/api/jetstream.rs::jetstream_query_accepts_repeated_values") + } else if source == "src/api/debug.rs" + && (boundary.contains("seed_xrpc") || boundary.contains("DebugSeedXrpc")) + { + Some(if boundary.contains("rmp_serde::to_vec") { + "nu:tests/stream_live_backfill.nu" + } else { + "nu:tests/api_xrpc.nu" + }) + } else if source == "src/api/xrpc/com_atproto_describe_repo.rs" { + Some( + "rust:src/api/xrpc/com_atproto_describe_repo.rs::describe_repo_json_projection_contract", + ) + } else if source == "src/api/xrpc/request_crawl.rs" { + Some("rust:src/api/xrpc/request_crawl.rs::banned_request_crawl_json_contract") + } else if source == "src/api/xrpc/subscribe_repos.rs" { + Some("nu:tests/relay_subscribe_repos_ping.nu") + } else if source.starts_with("src/api/xrpc/") { + Some("nu:tests/api_xrpc.nu") + } else if source == "src/api/db.rs" { + Some("nu:tests/api_db.nu") + } else if source == "src/api/mod.rs" { + Some("nu:tests/debug_endpoints.nu") + } else if source == "src/api/stats.rs" { + Some("nu:tests/debug_endpoints.nu") + } else if source == "src/api/ws.rs" { + Some("nu:tests/stream_ping.nu") + } else if source == "src/config/env.rs" { + Some("rust:src/config/env.rs::env_text_parser_contracts") + } else if source == "src/control/indexer.rs" { + Some("nu:tests/debug_endpoints.nu") + } else if source == "src/jetstream.rs" { + Some("nu:tests/stream_relay_jetstream_subscribe.nu") + } else if source == "src/resolver.rs" { + Some("rust:src/resolver.rs::raw_did_doc_json_contract_and_size_limit") + } else if source == "src/db/types.rs" { + Some( + if boundary.starts_with("representation-conversion:") + || boundary.starts_with("db-key-constant:") + { + "rust:src/db/types.rs::db_representation_conversions_are_lossless" + } else if boundary.contains("TrimmedDid") { + "rust:src/control/stream/indexer.rs::legacy_string_trimmed_did_replays_after_reopen" + } else if boundary.contains("DbRkey") { + "rust:src/db/types.rs::dbrkey_string_markers_preserve_every_utf8_length" + } else if boundary.contains("DbTid") { + "rust:src/db/types.rs::dbrkey_msgpack_binary_marker_preserves_tid" + } else { + "rust:src/db/types.rs::db_action_and_did_key_representation_contracts" + }, + ) + } else if source == "src/patch.rs" { + Some("rust:src/patch.rs::set_update_json_markers_are_structurally_disjoint") + } else if source == "src/config/types.rs" { + Some(if boundary.contains("CrawlerMode") { + "rust:src/config/types.rs::crawler_mode_string_and_msgpack_contracts" + } else if boundary.contains("RateTier") + || boundary.contains("CrawlerSource") + || boundary.contains("FirehoseSource") + { + "rust:src/config/types.rs::source_and_rate_tier_parser_contracts" + } else { + "rust:src/config/types.rs::text_config_parser_contracts" + }) + } else if source == "src/config.rs" && boundary.starts_with("representation-conversion:") { + Some("rust:src/config.rs::display_reports_history_ttl") + } else if source == "src/filter.rs" { + Some("rust:src/filter.rs::filter_mode_persisted_golden_fixtures") + } else if source == "src/pds_meta.rs" { + Some("rust:src/pds_meta.rs::host_status_persisted_golden_fixtures") + } else if source == "src/crawler/list_repos/checker.rs" { + Some("rust:src/crawler/list_repos/checker.rs::retry_state_persisted_golden_fixture") + } else if source == "src/crawler/relay_first.rs" { + Some("rust:src/crawler/relay_first.rs::pass_cursor_round_trips") + } else if source == "src/db/keys/indexer.rs" { + Some("rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts") + } else if source == "src/db/keys/mod.rs" || source == "src/db/keys/v1.rs" { + Some("rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts") + } else if source == "src/db/storage_mode.rs" { + Some("rust:src/db/storage_mode.rs::storage_mode_marker_persisted_golden_fixture") + } else if source == "src/db/schema.rs" && boundary.starts_with("db-keyspace:") { + Some("rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema") + } else if source == "src/db/mod.rs" && boundary.contains("FirehoseSourceMeta") { + Some("rust:src/db/mod.rs::firehose_source_meta_golden_fixture") + } else if source == "src/db/mod.rs" { + Some("rust:src/types.rs::current_persisted_state_golden_fixtures") + } else if source == "src/db/migration/mod.rs" && boundary.starts_with("db-key-") { + Some( + if boundary.contains("LEGACY_BLOCKS_KEYSPACE") + || boundary.contains("LEGACY_RECORDS_KEYSPACE") + { + "rust:src/db/migration/v10.rs::v10_rewrites_records_and_retires_legacy_storage" + } else if boundary.contains("pass_cursor_key") + || boundary.contains("MIGRATION_CURSOR_PREFIX") + { + "rust:src/db/migration/mod.rs::cursor_keys_are_distinct_per_version_and_pass" + } else { + "rust:src/db/migration/mod.rs::applied_migration_keys_track_per_migration_status" + }, + ) + } else if source == "src/db/migration/v10.rs" { + Some(if boundary.contains("rewrite_filter_key") { + "rust:src/db/migration/v10.rs::v10_finishes_v9_filter_keys_skipped_by_a_reduced_feature_build" + } else { + "rust:src/db/migration/v10.rs::v10_rewrites_records_and_retires_legacy_storage" + }) + } else if matches!( + source, + "src/db/migration/v1.rs" + | "src/db/migration/v3.rs" + | "src/db/migration/v5.rs" + | "src/db/migration/v6.rs" + ) { + Some( + "rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries", + ) + } else if source == "src/db/migration/v2.rs" { + Some(if boundary.starts_with("serde-") { + "rust:src/db/migration/v2.rs::v1_repo_state_input_golden_fixture" + } else { + "rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries" + }) + } else if source == "src/db/migration/v4.rs" { + Some(if boundary.starts_with("serde-") { + "rust:src/db/migration/v4.rs::v2_repo_state_input_golden_fixture" + } else { + "rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries" + }) + } else if source == "src/types/v2.rs" { + Some("rust:src/types/v2.rs::v2_persisted_schema_golden_fixture") + } else if source == "src/types/v4.rs" { + Some("rust:src/types/v4.rs::v4_persisted_schema_golden_fixtures") + } else if source == "src/types/v7.rs" { + Some("rust:src/types/v7.rs::v7_persisted_schema_golden_fixture") + } else if source == "src/types.rs" { + Some(if boundary.contains("DeleteBod") { + "rust:src/types.rs::operator_redaction_json_contracts" + } else if boundary.contains("RepoState") && boundary.contains("IntoStatic") { + "rust:src/types.rs::into_static_preserves_per_event_clocks" + } else if boundary.starts_with("representation-conversion:") { + "rust:src/types.rs::commit_and_repo_status_conversion_contracts" + } else { + "rust:src/types.rs::current_persisted_state_golden_fixtures" + }) + } else if source == "src/ingest/indexer/shard.rs" { + Some( + "rust:src/ingest/indexer/shard.rs::resync_buffer_commit_msgpack_round_trip_preserves_path", + ) + } else if source == "src/ingest/relay/sink/relay.rs" { + Some( + "rust:src/ingest/relay/sink/relay.rs::relay_ephemeral_projection_preserves_record_representation", + ) + } else if source == "src/ingest/validation.rs" { + Some( + "rust:src/ingest/validation.rs::record_dag_cbor_projection_accepts_data_and_rejects_invalid_bytes", + ) + } else if source == "src/ops/backlink_ops.rs" { + Some("nu:tests/backlinks.nu") + } else if source == "src/ops/record_events.rs" { + Some("nu:tests/stream_cursor_replay.nu") + } else if source == "src/ingest/stream/types.rs" { + Some(if boundary.contains("::Datetime") { + "rust:src/ingest/stream/types.rs::firehose_datetime_accepts_zoned_and_legacy_unzoned_values" + } else if boundary.contains("RepoOpAction") + || boundary.contains("AccountStatus") + || boundary.contains("InfoName") + { + "rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts" + } else if boundary.contains("::Sync") { + "rust:src/ingest/stream/codec.rs::firehose_frame_mock_decodes_sync_binary_blocks" + } else if boundary.contains("::Commit") || boundary.contains("::RepoOp") { + "rust:src/ingest/stream.rs::test_decode_commit_empty_since" + } else if boundary.contains("::Account") { + "rust:src/ingest/stream.rs::test_decode_account" + } else if boundary.contains("::Identity") { + "rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity" + } else if boundary.contains("::Info") { + "rust:src/ingest/stream/codec.rs::firehose_frame_maps_outdated_cursor_info" + } else { + "rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts" + }) + } else if source == "src/ingest/stream/codec.rs" { + Some("rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity") + } else if source == "src/ingest/stream.rs" { + Some("rust:src/ingest/stream.rs::test_decode_commit_empty_since") + } else if source == "src/control/repos/indexer.rs" { + Some(if boundary.contains("generate_car") { + "rust:src/control/repos/indexer.rs::generate_car_reports_a_deleted_head_body" + } else if boundary.contains("get_record") || boundary.contains("list_records") { + "rust:src/control/repos/indexer.rs::get_and_list_hide_redacted_heads" + } else { + "rust:src/control/repos/indexer.rs::drop_repo_excludes_untracks_and_redacts_all_body_storage" + }) + } else if source == "src/api/jetstream.rs" { + Some( + if boundary.contains("Subscriber") || boundary.contains("handle_options_message") { + "rust:src/api/jetstream.rs::jetstream_options_message_json_contract" + } else { + "rust:src/api/jetstream.rs::jetstream_query_accepts_repeated_values" + }, + ) + } else if source == "src/api/ingestion.rs" { + Some("rust:src/api/ingestion.rs::ingestion_json_contracts_follow_compiled_mode") + } else if source == "src/types/event/jetstream.rs" { + Some(if boundary.contains("::Relay") { + "rust:src/types/event/jetstream.rs::relay_jetstream_event_golden_fixtures" + } else if boundary.contains("::Commit") { + "rust:src/types/event/jetstream.rs::indexer_jetstream_event_golden_fixture" + } else { + "rust:src/types/event/jetstream.rs::common_jetstream_event_golden_fixtures" + }) + } else if source == "src/bin/backfill_strategy_bench.rs" { + Some("rust:src/bin/backfill_strategy_bench.rs::benchmark_api_json_fixtures_decode") + } else if source == "src/backlinks/store.rs" { + Some( + if boundary.contains("forward_key") || boundary.contains("forward_did_prefix") { + "rust:src/backlinks/store.rs::forward_did_prefix_matches_key" + } else if boundary.contains("reverse_scan_prefix") { + "rust:src/backlinks/store.rs::reverse_key_scan_prefix_with_collection_and_path" + } else if boundary.contains("reverse_key") + || boundary.contains("source_from_reverse_key") + { + "rust:src/backlinks/store.rs::source_from_reverse_key_roundtrip" + } else { + "rust:src/backlinks/store.rs::forward_value_persisted_golden_fixture" + }, + ) + } else if source == "src/util/mod.rs" { + Some(if boundary.contains("ser_status_code") { + "rust:src/util/mod.rs::http_status_serde_contract" + } else { + "rust:src/util/mod.rs::xrpc_output_string_serializers_are_stable" + }) + } else if source == "src/control/stream/indexer.rs" { + Some( + "rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim", + ) + } else if source == "src/control/repos/mod.rs" { + Some(if boundary.contains("collections") { + "rust:src/control/repos/mod.rs::collections_decodes_big_endian_counts" + } else { + "rust:src/control/repos/mod.rs::repo_info_json_contract" + }) + } else if source == "src/db/indexer.rs" && boundary.contains("load_persisted_crawler_sources") { + Some("rust:src/db/indexer.rs::persisted_crawler_sources_decode_mode_and_url") + } else if source == "src/db/indexer.rs" && boundary.contains("record_count") { + Some("rust:src/db/indexer.rs::record_count_codec_updates_and_rejects_invalid_bytes") + } else if source == "src/ingest/firehose_stats/source.rs" { + Some("rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract") + } else if source == "src/ingest/firehose_stats/relay.rs" { + Some("rust:src/ingest/firehose_stats/relay.rs::relay_worker_stats_snapshot_json_contract") + } else if source == "src/control/pds.rs" && boundary.starts_with("representation-conversion:") { + Some("nu:tests/api_pds_tiers.nu") + } else if source == "src/db/filter.rs" && boundary.starts_with("db-key-") { + Some(if boundary.contains("exclude_key") { + "rust:src/db/filter.rs::test_exclude_key_trimmed" + } else { + "rust:src/db/filter.rs::test_filter_keys" + }) + } else if source == "src/db/pds_meta.rs" && boundary.starts_with("db-key-") { + Some(if boundary.contains("tier") { + "nu:tests/api_pds_tiers.nu" + } else { + "nu:tests/pds_host_status_transitions.nu" + }) + } else if source == "src/types/event/stream.rs" { + Some(if boundary.contains("StoredData") { + "rust:src/types/event/stream.rs::stored_data_persisted_golden_fixtures" + } else if boundary.contains("StoredEvent") { + "rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim" + } else { + "rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant" + }) + } else { + None + }; + if let Some(coverage) = exact { + return coverage.to_owned(); + } + + if let Some(test) = surface + .tests + .iter() + .find(|test| test.starts_with(&format!("rust:{source}::"))) + { + return test.clone(); + } + + let nu = if source.starts_with("src/api/crawler.rs") { + "tests/api_crawler_sources.nu" + } else if source.starts_with("src/api/firehose.rs") { + "tests/api_firehose_sources.nu" + } else if source.starts_with("src/api/pds.rs") && boundary.to_ascii_lowercase().contains("ban") + { + "tests/api_pds_banned.nu" + } else if source.starts_with("src/api/pds.rs") { + "tests/api_pds_tiers.nu" + } else if source.starts_with("src/api/repos.rs") || source.starts_with("src/control/repos/") { + "tests/api_repos.nu" + } else if source.starts_with("src/api/debug.rs") || source.starts_with("src/db/schema.rs") { + "tests/debug_endpoints.nu" + } else if source.starts_with("src/api/filter.rs") + || source == "src/filter.rs" + || source == "src/control/filter.rs" + { + "tests/signal_filter.nu" + } else if source.starts_with("src/api/stream.rs") + || source.starts_with("src/control/stream/") + || source.starts_with("src/types/event/") + { + "tests/stream_cursor_replay.nu" + } else if source.starts_with("src/api/xrpc/subscribe_repos.rs") { + "tests/relay_subscribe_repos_ping.nu" + } else if source.starts_with("src/api/xrpc/") + || source.starts_with("src/backfill/") + || source.starts_with("src/ops") + { + "tests/stream_live_backfill.nu" + } else if source.starts_with("src/backlinks/") { + "tests/backlinks.nu" + } else if source.starts_with("src/crawler/") { + "tests/crawler_full_network.nu" + } else if source.starts_with("src/ingest/") { + "tests/stream_live_backfill.nu" + } else if source.starts_with("src/control/firehose.rs") + || source.starts_with("src/ingest/firehose_stats/") + { + "tests/api_firehose_sources.nu" + } else if source.starts_with("src/control/crawler.rs") { + "tests/api_crawler_sources.nu" + } else if source.starts_with("src/control/pds.rs") || source == "src/pds_meta.rs" { + "tests/pds_host_status_transitions.nu" + } else if source.starts_with("src/control/stats.rs") { + "tests/debug_endpoints.nu" + } else if source.starts_with("src/db/") { + "tests/debug_endpoints.nu" + } else { + "tests/representation_unassigned" + }; + format!("nu:{nu}") +} + +fn inventory_errors(manifest: &Path) -> Vec { + let surface = discover(manifest); + let inventory = load_inventory(manifest); + let inventoried = inventory.keys().cloned().collect::>(); + let mut errors = Vec::new(); + + errors.extend( + surface + .boundaries + .difference(&inventoried) + .map(|boundary| format!("missing: {boundary}\t")), + ); + errors.extend( + inventoried + .difference(&surface.boundaries) + .map(|boundary| format!("stale: {boundary}")), + ); + errors.extend(inventory.into_iter().filter_map(|(boundary, coverage)| { + let valid = match coverage.strip_prefix("rust:") { + Some(_) => surface.tests.contains(&coverage), + None => coverage + .strip_prefix("nu:") + .or_else(|| coverage.strip_prefix("fixture:")) + .is_some_and(|path| manifest.join(path).is_file()), + }; + (!valid).then(|| format!("missing coverage target: {boundary}\t{coverage}")) + })); + errors +} + +#[test] +fn representation_audit_rejects_uncovered_and_stale_boundaries() { + let tmp = tempfile::tempdir().unwrap(); + fs::create_dir_all(tmp.path().join("src")).unwrap(); + fs::create_dir_all(tmp.path().join("tests")).unwrap(); + fs::write( + tmp.path().join("src/lib.rs"), + r#" +pub fn persist(value: &u64) -> Vec { + rmp_serde::to_vec(value).unwrap() +} + +#[cfg(test)] +mod tests { + #[test] + fn round_trip() {} +} +"#, + ) + .unwrap(); + fs::write( + tmp.path().join(INVENTORY_PATH), + "# machine-checked representation boundary\tbehavioral coverage\n", + ) + .unwrap(); + + let errors = inventory_errors(tmp.path()); + assert_eq!( + errors, + ["missing: codec-call:src/lib.rs::persist::rmp_serde::to_vec\t"] + ); + + fs::write( + tmp.path().join(INVENTORY_PATH), + concat!( + "codec-call:src/lib.rs::persist::rmp_serde::to_vec", + "\trust:src/lib.rs::round_trip\n", + "serde-type:src/lib.rs::Removed", + "\trust:src/lib.rs::round_trip\n", + ), + ) + .unwrap(); + let errors = inventory_errors(tmp.path()); + assert_eq!(errors, ["stale: serde-type:src/lib.rs::Removed"]); +} + +#[test] +#[ignore] +fn print_proposed_representation_inventory() { + let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); + let surface = discover(manifest); + println!("# machine-checked representation boundary\tbehavioral coverage"); + for boundary in &surface.boundaries { + println!("{boundary}\t{}", proposed_coverage(boundary, &surface)); + } +} + +#[test] +fn representation_audit_inventory_is_complete() { + let errors = inventory_errors(Path::new(env!("CARGO_MANIFEST_DIR"))); + assert!( + errors.is_empty(), + "representation inventory is incomplete:\n{}", + errors.join("\n") + ); +} diff --git a/tests/representation_inventory.tsv b/tests/representation_inventory.tsv new file mode 100644 --- /dev/null +++ b/tests/representation_inventory.tsv @@ -0,0 +1,806 @@ +# machine-checked representation boundary behavioral coverage +api-boundary:src/api/crawler.rs::add_source nu:tests/api_crawler_sources.nu +api-boundary:src/api/crawler.rs::list_sources nu:tests/api_crawler_sources.nu +api-boundary:src/api/crawler.rs::remove_source nu:tests/api_crawler_sources.nu +api-boundary:src/api/crawler.rs::reset_cursor nu:tests/api_crawler_sources.nu +api-boundary:src/api/crawler.rs::router nu:tests/api_crawler_sources.nu +api-boundary:src/api/db.rs::handle_compact nu:tests/api_db.nu +api-boundary:src/api/db.rs::handle_train_dict nu:tests/api_db.nu +api-boundary:src/api/db.rs::router nu:tests/api_db.nu +api-boundary:src/api/debug.rs::handle_debug_compact nu:tests/debug_endpoints.nu +api-boundary:src/api/debug.rs::handle_debug_count nu:tests/debug_endpoints.nu +api-boundary:src/api/debug.rs::handle_debug_ephemeral_ttl_tick nu:tests/debug_endpoints.nu +api-boundary:src/api/debug.rs::handle_debug_get nu:tests/debug_endpoints.nu +api-boundary:src/api/debug.rs::handle_debug_iter nu:tests/debug_endpoints.nu +api-boundary:src/api/debug.rs::handle_debug_seed_backlinks nu:tests/debug_endpoints.nu +api-boundary:src/api/debug.rs::handle_debug_seed_events nu:tests/debug_endpoints.nu +api-boundary:src/api/debug.rs::handle_debug_seed_relay_jetstream_commits nu:tests/debug_endpoints.nu +api-boundary:src/api/debug.rs::handle_debug_seed_watermark nu:tests/debug_endpoints.nu +api-boundary:src/api/debug.rs::handle_debug_seed_xrpc_repo nu:tests/api_xrpc.nu +api-boundary:src/api/debug.rs::router nu:tests/debug_endpoints.nu +api-boundary:src/api/filter.rs::handle_get_filter nu:tests/signal_filter.nu +api-boundary:src/api/filter.rs::handle_patch_filter nu:tests/signal_filter.nu +api-boundary:src/api/filter.rs::router nu:tests/signal_filter.nu +api-boundary:src/api/firehose.rs::add_source nu:tests/api_firehose_sources.nu +api-boundary:src/api/firehose.rs::get_diagnostics nu:tests/api_firehose_sources.nu +api-boundary:src/api/firehose.rs::get_source nu:tests/api_firehose_sources.nu +api-boundary:src/api/firehose.rs::list_sources nu:tests/api_firehose_sources.nu +api-boundary:src/api/firehose.rs::remove_source nu:tests/api_firehose_sources.nu +api-boundary:src/api/firehose.rs::reset_cursor nu:tests/api_firehose_sources.nu +api-boundary:src/api/firehose.rs::router nu:tests/api_firehose_sources.nu +api-boundary:src/api/ingestion.rs::get_ingestion rust:src/api/ingestion.rs::ingestion_json_contracts_follow_compiled_mode +api-boundary:src/api/ingestion.rs::patch_ingestion rust:src/api/ingestion.rs::ingestion_json_contracts_follow_compiled_mode +api-boundary:src/api/ingestion.rs::router rust:src/api/ingestion.rs::ingestion_json_contracts_follow_compiled_mode +api-boundary:src/api/jetstream.rs::handle_subscribe rust:src/api/jetstream.rs::jetstream_query_accepts_repeated_values +api-boundary:src/api/mod.rs::serve nu:tests/debug_endpoints.nu +api-boundary:src/api/mod.rs::serve_debug nu:tests/debug_endpoints.nu +api-boundary:src/api/pds.rs::ban nu:tests/api_pds_banned.nu +api-boundary:src/api/pds.rs::list_banned nu:tests/api_pds_banned.nu +api-boundary:src/api/pds.rs::list_rate_tiers nu:tests/api_pds_tiers.nu +api-boundary:src/api/pds.rs::list_tiers nu:tests/api_pds_tiers.nu +api-boundary:src/api/pds.rs::remove_tier nu:tests/api_pds_tiers.nu +api-boundary:src/api/pds.rs::router nu:tests/api_pds_tiers.nu +api-boundary:src/api/pds.rs::set_tier nu:tests/api_pds_tiers.nu +api-boundary:src/api/pds.rs::unban nu:tests/api_pds_banned.nu +api-boundary:src/api/redactions.rs::handle_post_redaction rust:src/api/redactions.rs::test_parse_redaction_target_invalid +api-boundary:src/api/redactions.rs::parse_redaction_target rust:src/api/redactions.rs::test_parse_redaction_target_invalid +api-boundary:src/api/redactions.rs::router rust:src/api/redactions.rs::test_parse_redaction_target_invalid +api-boundary:src/api/repos.rs::handle_delete_repos nu:tests/api_repos.nu +api-boundary:src/api/repos.rs::handle_delete_single_repo nu:tests/api_repos.nu +api-boundary:src/api/repos.rs::handle_get_repo nu:tests/api_repos.nu +api-boundary:src/api/repos.rs::handle_get_repos nu:tests/api_repos.nu +api-boundary:src/api/repos.rs::handle_post_resync nu:tests/api_repos.nu +api-boundary:src/api/repos.rs::handle_put_repos nu:tests/api_repos.nu +api-boundary:src/api/repos.rs::router nu:tests/api_repos.nu +api-boundary:src/api/stats.rs::get_stats nu:tests/debug_endpoints.nu +api-boundary:src/api/stream.rs::handle_stream nu:tests/stream_cursor_replay.nu +api-boundary:src/api/stream.rs::router nu:tests/stream_cursor_replay.nu +api-boundary:src/api/ws.rs::control_frame_only_limits nu:tests/stream_ping.nu +api-boundary:src/api/ws.rs::run_socket nu:tests/stream_ping.nu +api-boundary:src/api/xrpc/com_atproto_describe_repo.rs::handle rust:src/api/xrpc/com_atproto_describe_repo.rs::describe_repo_json_projection_contract +api-boundary:src/api/xrpc/count_records.rs::handle nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/describe_repo.rs::handle nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/get_host_status.rs::handle nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/get_latest_commit.rs::handle nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/get_record.rs::handle nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/get_record_by_uri.rs::handle_bad_example nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/get_record_by_uri.rs::handle_blue_microcosm nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/get_repo.rs::handle nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/get_repo_status.rs::handle nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/get_repo_status.rs::repo_status_to_api nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/list_hosts.rs::handle nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/list_records.rs::handle nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/list_repos.rs::handle nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/mod.rs::router nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/request_crawl.rs::handle rust:src/api/xrpc/request_crawl.rs::banned_request_crawl_json_contract +api-boundary:src/api/xrpc/resolve_mini_doc.rs::handle_bad_example nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/resolve_mini_doc.rs::handle_blue_microcosm nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/resolve_mini_doc.rs::resolve_mini_doc nu:tests/api_xrpc.nu +api-boundary:src/api/xrpc/subscribe_repos.rs::handle nu:tests/relay_subscribe_repos_ping.nu +car-codec:src/car.rs::car_header_bounds rust:src/car.rs::parses_rooted_car_with_zero_copy_payloads +car-codec:src/car.rs::parse_car rust:src/car.rs::parses_rooted_car_with_zero_copy_payloads +car-codec:src/car.rs::parse_car_blocks rust:src/car.rs::parses_rooted_car_with_zero_copy_payloads +car-codec:src/car.rs::parse_car_blocks_from rust:src/car.rs::parses_rooted_car_with_zero_copy_payloads +car-codec:src/car.rs::read_uvarint rust:src/car.rs::parses_rooted_car_with_zero_copy_payloads +car-codec:src/car.rs::validate_block_cids rust:src/car.rs::parses_rooted_car_with_zero_copy_payloads +codec-call:src/api/debug.rs::handle_debug_seed_backlinks::serde_ipld_dagcbor::from_slice nu:tests/debug_endpoints.nu +codec-call:src/api/debug.rs::handle_debug_seed_backlinks::serde_ipld_dagcbor::to_vec nu:tests/debug_endpoints.nu +codec-call:src/api/debug.rs::handle_debug_seed_watermark::.to_be_bytes nu:tests/debug_endpoints.nu +codec-call:src/api/debug.rs::handle_debug_seed_xrpc_repo::.to_be_bytes nu:tests/api_xrpc.nu +codec-call:src/api/debug.rs::handle_debug_seed_xrpc_repo::rmp_serde::to_vec nu:tests/stream_live_backfill.nu +codec-call:src/api/debug.rs::handle_debug_seed_xrpc_repo::serde_ipld_dagcbor::to_vec nu:tests/api_xrpc.nu +codec-call:src/api/jetstream.rs::handle_options_message::serde_json::from_slice rust:src/api/jetstream.rs::jetstream_options_message_json_contract +codec-call:src/api/jetstream.rs::handle_options_message::serde_json::from_value rust:src/api/jetstream.rs::jetstream_options_message_json_contract +codec-call:src/api/mod.rs::MultiQuery::from_request_parts::serde_html_form::from_str rust:src/api/jetstream.rs::jetstream_query_accepts_repeated_values +codec-call:src/api/repos.rs::did_list_response::serde_json::to_string nu:tests/api_repos.nu +codec-call:src/api/repos.rs::handle_get_repos::serde_json::to_string nu:tests/api_repos.nu +codec-call:src/api/repos.rs::parse_body::serde_json::from_str nu:tests/api_repos.nu +codec-call:src/api/stream.rs::handle_socket::serde_json::to_string nu:tests/stream_cursor_replay.nu +codec-call:src/api/xrpc/mod.rs::ExtractXrpc::from_request::serde_html_form::from_str nu:tests/api_xrpc.nu +codec-call:src/api/xrpc/mod.rs::ExtractXrpc::from_request::serde_json::from_slice nu:tests/api_xrpc.nu +codec-call:src/backfill/manager.rs::queue_gone_backfills::rmp_serde::from_slice rust:src/backfill/manager.rs::eligible_repo_moves_from_resync_to_pending +codec-call:src/backfill/manager.rs::queue_resync_if::rmp_serde::from_slice rust:src/backfill/manager.rs::eligible_repo_moves_from_resync_to_pending +codec-call:src/backfill/manager.rs::retry_worker::rmp_serde::from_slice rust:src/backfill/manager.rs::eligible_repo_moves_from_resync_to_pending +codec-call:src/backfill/sparse.rs::fetch_block_chunk::serde_json::from_slice rust:src/backfill/sparse.rs::fetches_independent_get_blocks_chunks_concurrently +codec-call:src/backfill/worker/process.rs::fetch_full_repo_car::serde_json::from_slice rust:src/backfill/worker/process.rs::full_get_repo_maps_inactive_status +codec-call:src/backfill/worker/process.rs::process_did::rmp_serde::from_slice rust:src/backfill/worker/process.rs::full_get_repo_maps_inactive_status +codec-call:src/backfill/worker/process.rs::process_did::rmp_serde::to_vec rust:src/backfill/worker/process.rs::full_get_repo_maps_inactive_status +codec-call:src/backfill/worker/task.rs::did_task::rmp_serde::from_slice rust:src/backfill/worker/task.rs::excluded_orphan_pending_entry_is_removed_before_work +codec-call:src/backfill/worker/task.rs::did_task::rmp_serde::to_vec rust:src/backfill/worker/task.rs::excluded_orphan_pending_entry_is_removed_before_work +codec-call:src/backlinks/store.rs::delete_record::rmp_serde::from_slice rust:src/backlinks/store.rs::forward_value_persisted_golden_fixture +codec-call:src/backlinks/store.rs::delete_repo::rmp_serde::from_slice rust:src/backlinks/store.rs::forward_value_persisted_golden_fixture +codec-call:src/backlinks/store.rs::index_record::rmp_serde::from_slice rust:src/backlinks/store.rs::forward_value_persisted_golden_fixture +codec-call:src/backlinks/store.rs::index_record::rmp_serde::to_vec rust:src/backlinks/store.rs::forward_value_persisted_golden_fixture +codec-call:src/car.rs::parse_car::iroh_car::CarHeader::decode rust:src/car.rs::parses_rooted_car_with_zero_copy_payloads +codec-call:src/config/env.rs::Config::from_env::.parse rust:src/config/env.rs::env_text_parser_contracts +codec-call:src/config/env.rs::parse_new_host_limit_value::.parse rust:src/config/env.rs::env_text_parser_contracts +codec-call:src/config/types.rs::CrawlerMode::deserialize::String::deserialize rust:src/config/types.rs::crawler_mode_string_and_msgpack_contracts +codec-call:src/control/crawler.rs::CrawlerHandle::add_source::rmp_serde::to_vec rust:src/control/crawler.rs::reset_cursor_clears_relay_first_progress_for_only_that_source +codec-call:src/control/firehose.rs::FirehoseHandle::add_seeded_sources::rmp_serde::to_vec nu:tests/api_firehose_sources.nu +codec-call:src/control/firehose.rs::FirehoseHandle::add_source::rmp_serde::to_vec nu:tests/api_firehose_sources.nu +codec-call:src/control/hosts.rs::Hydrant::get_host_status::i64::from_be_bytes rust:src/control/hosts.rs::list_hosts_includes_seeded_hosts_without_cursors +codec-call:src/control/indexer.rs::Hydrant::seed_events_for_bench::rmp_serde::to_vec nu:tests/debug_endpoints.nu +codec-call:src/control/indexer.rs::Hydrant::seed_events_for_bench::serde_ipld_dagcbor::to_vec nu:tests/debug_endpoints.nu +codec-call:src/control/repos/indexer.rs::RepoHandle::generate_car::CarWriter::new rust:src/control/repos/indexer.rs::generate_car_reports_a_deleted_head_body +codec-call:src/control/repos/indexer.rs::RepoHandle::get_record::serde_ipld_dagcbor::from_slice rust:src/control/repos/indexer.rs::get_and_list_hide_redacted_heads +codec-call:src/control/repos/indexer.rs::RepoHandle::list_records::serde_ipld_dagcbor::from_slice rust:src/control/repos/indexer.rs::get_and_list_hide_redacted_heads +codec-call:src/control/repos/indexer.rs::ReposControl::iter_pending::.to_be_bytes rust:src/control/repos/indexer.rs::drop_repo_excludes_untracks_and_redacts_all_body_storage +codec-call:src/control/repos/indexer.rs::ReposControl::iter_pending::u64::from_be_bytes rust:src/control/repos/indexer.rs::drop_repo_excludes_untracks_and_redacts_all_body_storage +codec-call:src/control/repos/mod.rs::RepoHandle::collections::u64::from_be_bytes rust:src/control/repos/mod.rs::collections_decodes_big_endian_counts +codec-call:src/control/seed.rs::seed_one::serde_json::from_slice rust:src/control/seed.rs::apply_seed_snapshot_persists_statuses_and_preserves_absent_cursor +codec-call:src/control/stream/indexer.rs::read_event_replay_chunk::rmp_serde::from_slice rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim +codec-call:src/control/stream/indexer.rs::stored_to_event::serde_ipld_dagcbor::from_slice rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim +codec-call:src/control/stream/indexer.rs::stored_to_event::serde_json::value::to_raw_value rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim +codec-call:src/control/stream/jetstream.rs::jetstream_event_to_bytes::rmp_serde::from_slice nu:tests/stream_cursor_replay.nu +codec-call:src/control/stream/jetstream.rs::jetstream_event_to_bytes::serde_ipld_dagcbor::from_slice nu:tests/stream_cursor_replay.nu +codec-call:src/control/stream/jetstream.rs::jetstream_event_to_bytes::serde_json::to_vec nu:tests/stream_cursor_replay.nu +codec-call:src/control/stream/jetstream.rs::jetstream_event_to_bytes::serde_json::value::to_raw_value nu:tests/stream_cursor_replay.nu +codec-call:src/control/stream/jetstream.rs::read_jetstream_replay_chunk::rmp_serde::from_slice nu:tests/stream_cursor_replay.nu +codec-call:src/crawler/by_collection.rs::ByCollectionProducer::crawl::serde_json::from_slice nu:tests/crawler_full_network.nu +codec-call:src/crawler/list_repos/checker.rs::SignalChecker::check_repo_signals::serde_json::from_slice rust:src/crawler/list_repos/checker.rs::retry_state_persisted_golden_fixture +codec-call:src/crawler/list_repos/checker.rs::SignalChecker::check_signals_batch::rmp_serde::to_vec rust:src/crawler/list_repos/checker.rs::retry_state_persisted_golden_fixture +codec-call:src/crawler/list_repos/producer.rs::ListReposProducer::crawl::serde_json::from_slice rust:src/crawler/list_repos/producer.rs::inactive_unknown_repo_is_not_backfilled +codec-call:src/crawler/list_repos/producer.rs::ListReposProducer::get_cursor::rmp_serde::from_slice rust:src/crawler/list_repos/producer.rs::inactive_unknown_repo_is_not_backfilled +codec-call:src/crawler/list_repos/producer.rs::pass_cursor_updates::.to_be_bytes rust:src/crawler/list_repos/producer.rs::inactive_unknown_repo_is_not_backfilled +codec-call:src/crawler/list_repos/producer.rs::pass_cursor_updates::rmp_serde::to_vec rust:src/crawler/list_repos/producer.rs::inactive_unknown_repo_is_not_backfilled +codec-call:src/crawler/list_repos/retry.rs::RetryProducer::process_queue::rmp_serde::from_slice nu:tests/crawler_full_network.nu +codec-call:src/crawler/relay_first.rs::DeepDiscoveryProducer::crawl_pass::serde_json::from_slice rust:src/crawler/relay_first.rs::pass_cursor_round_trips +codec-call:src/crawler/relay_first.rs::DeepDiscoveryProducer::load_cursor::rmp_serde::from_slice rust:src/crawler/relay_first.rs::pass_cursor_round_trips +codec-call:src/crawler/relay_first.rs::DeepDiscoveryProducer::parse_deep_page::serde_json::from_slice rust:src/crawler/relay_first.rs::pass_cursor_round_trips +codec-call:src/crawler/relay_first.rs::serialize_cursor::rmp_serde::to_vec rust:src/crawler/relay_first.rs::pass_cursor_round_trips +codec-call:src/crawler/worker.rs::reconcile_listing::rmp_serde::to_vec rust:src/crawler/worker.rs::changed_listing_emits_one_account_event +codec-call:src/db/compaction.rs::tid_micros::u64::from_be_bytes rust:src/db/compaction.rs::huge_ttl_cannot_wrap_the_cutoff_into_the_future +codec-call:src/db/counts.rs::Db::stage_count_deltas::.to_be_bytes rust:src/db/counts.rs::checkpoint_skips_inflight_deltas +codec-call:src/db/counts.rs::set_count_delta_watermark::.to_be_bytes rust:src/db/counts.rs::checkpoint_skips_inflight_deltas +codec-call:src/db/counts.rs::set_ks_count::.to_be_bytes rust:src/db/counts.rs::checkpoint_skips_inflight_deltas +codec-call:src/db/ephemeral.rs::ttl_tick_inner::.to_be_bytes rust:src/db/ephemeral.rs::relay_leveled_compaction_does_not_reclaim_prefix_tombstones +codec-call:src/db/filter.rs::apply_patch::rmp_serde::to_vec rust:src/db/filter.rs::test_apply_and_load +codec-call:src/db/filter.rs::load::rmp_serde::from_slice rust:src/db/filter.rs::test_apply_and_load +codec-call:src/db/indexer.rs::get_record_count::u64::from_be_bytes rust:src/db/indexer.rs::record_count_codec_updates_and_rejects_invalid_bytes +codec-call:src/db/indexer.rs::load_persisted_crawler_sources::rmp_serde::from_slice rust:src/db/indexer.rs::persisted_crawler_sources_decode_mode_and_url +codec-call:src/db/indexer.rs::set_record_count::.to_be_bytes rust:src/db/indexer.rs::record_count_codec_updates_and_rejects_invalid_bytes +codec-call:src/db/indexer.rs::update_record_count::.to_be_bytes rust:src/db/indexer.rs::record_count_codec_updates_and_rejects_invalid_bytes +codec-call:src/db/indexer.rs::update_record_count::u64::from_be_bytes rust:src/db/indexer.rs::record_count_codec_updates_and_rejects_invalid_bytes +codec-call:src/db/keys/indexer.rs::event_watermark_key::.to_be_bytes rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +codec-call:src/db/keys/indexer.rs::pending_key::.to_be_bytes rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +codec-call:src/db/keys/mod.rs::count_delta_key::.to_be_bytes rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +codec-call:src/db/keys/mod.rs::count_delta_start_key::.to_be_bytes rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +codec-call:src/db/keys/mod.rs::event_key::.to_be_bytes rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +codec-call:src/db/keys/mod.rs::jetstream_event_key::.to_be_bytes rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +codec-call:src/db/keys/mod.rs::parse_count_delta_key::u64::from_be_bytes rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +codec-call:src/db/keys/mod.rs::parse_jetstream_event_key::u64::from_be_bytes rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +codec-call:src/db/keys/mod.rs::relay_event_key::.to_be_bytes rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +codec-call:src/db/keys/mod.rs::relay_event_watermark_key::.to_be_bytes rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +codec-call:src/db/keyspaces.rs::RelayDb::init::u64::from_be_bytes nu:tests/debug_endpoints.nu +codec-call:src/db/keyspaces.rs::StreamDb::event_head::u64::from_be_bytes nu:tests/debug_endpoints.nu +codec-call:src/db/keyspaces.rs::StreamDb::init::u64::from_be_bytes nu:tests/debug_endpoints.nu +codec-call:src/db/lifecycle_counts.rs::gauge_from_resync::rmp_serde::from_slice rust:src/db/lifecycle_counts.rs::duplicate_pending_completion_is_idempotent +codec-call:src/db/migration/mod.rs::pass_cursor_key::.to_be_bytes rust:src/db/migration/mod.rs::a_chunk_that_fails_partway_commits_nothing +codec-call:src/db/migration/mod.rs::run::.to_be_bytes rust:src/db/migration/mod.rs::a_chunk_that_fails_partway_commits_nothing +codec-call:src/db/migration/v10/events.rs::decode_event::rmp_serde::from_slice rust:src/db/migration/v10/events.rs::cid_mismatch_aborts_without_clearing_blocks +codec-call:src/db/migration/v2.rs::repo_state_root_commit::rmp_serde::from_slice rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +codec-call:src/db/migration/v2.rs::repo_state_root_commit::rmp_serde::to_vec rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +codec-call:src/db/migration/v3.rs::firehose_source_is_pds::rmp_serde::to_vec rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +codec-call:src/db/migration/v4.rs::repo_state_active::rmp_serde::from_slice rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +codec-call:src/db/migration/v4.rs::repo_state_active::rmp_serde::to_vec rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +codec-call:src/db/migration/v6.rs::rebuild_pds_account_counts::rmp_serde::from_slice rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +codec-call:src/db/migration/v7.rs::repo_state_event_clocks::rmp_serde::from_slice rust:src/db/migration/v7.rs::repo_state_event_clock_migration_rewrites_v6_repo_states +codec-call:src/db/migration/v7.rs::repo_state_event_clocks::rmp_serde::to_vec rust:src/db/migration/v7.rs::repo_state_event_clock_migration_rewrites_v6_repo_states +codec-call:src/db/mod.rs::deser_repo_meta::rmp_serde::from_slice rust:src/types.rs::current_persisted_state_golden_fixtures +codec-call:src/db/mod.rs::deser_repo_state::rmp_serde::from_slice rust:src/types.rs::current_persisted_state_golden_fixtures +codec-call:src/db/mod.rs::get_firehose_cursor::i64::from_be_bytes rust:src/types.rs::current_persisted_state_golden_fixtures +codec-call:src/db/mod.rs::load_pds_daily_adds::u64::from_be_bytes rust:src/types.rs::current_persisted_state_golden_fixtures +codec-call:src/db/mod.rs::load_persisted_firehose_sources::rmp_serde::from_slice rust:src/types.rs::current_persisted_state_golden_fixtures +codec-call:src/db/mod.rs::save_pds_daily_adds::.to_be_bytes rust:src/types.rs::current_persisted_state_golden_fixtures +codec-call:src/db/mod.rs::ser_repo_meta::rmp_serde::to_vec rust:src/types.rs::current_persisted_state_golden_fixtures +codec-call:src/db/mod.rs::ser_repo_state::rmp_serde::to_vec rust:src/types.rs::current_persisted_state_golden_fixtures +codec-call:src/db/mod.rs::set_firehose_cursor::.to_be_bytes rust:src/types.rs::current_persisted_state_golden_fixtures +codec-call:src/db/pds_meta.rs::v5::load_statuses::rmp_serde::from_slice nu:tests/debug_endpoints.nu +codec-call:src/db/pds_meta.rs::v5::set_status::rmp_serde::to_vec nu:tests/debug_endpoints.nu +codec-call:src/db/schema.rs::::hex::encode rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::EventBodies::render_value::hex::encode rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::EventBodies::render_value::serde_ipld_dagcbor::from_slice rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::Events::render_value::hex::encode rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::Events::render_value::rmp_serde::from_slice rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::Events::render_value::serde_json::to_value rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::Heads::render_value::hex::encode rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::Heads::render_value::serde_ipld_dagcbor::from_slice rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::History::render_value::hex::encode rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::History::render_value::serde_ipld_dagcbor::from_slice rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::JetstreamEvents::render_value::hex::encode rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::JetstreamEvents::render_value::rmp_serde::from_slice rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::JetstreamEvents::render_value::serde_json::to_value rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::Repos::render_value::hex::encode rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::Repos::render_value::rmp_serde::from_slice rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::Repos::render_value::serde_json::to_value rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::msgpack_or_hex::hex::encode rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::msgpack_or_hex::rmp_serde::from_slice rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::msgpack_or_hex::serde_json::to_value rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::render_counter::hex::encode rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::render_counter::u64::from_be_bytes rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::u64_be_key_parse::.to_be_bytes rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/schema.rs::u64_be_key_render::u64::from_be_bytes rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +codec-call:src/db/storage_mode.rs::adopt::rmp_serde::to_vec_named rust:src/db/storage_mode.rs::storage_mode_marker_persisted_golden_fixture +codec-call:src/db/storage_mode.rs::infer_existing_ephemeral::rmp_serde::from_slice rust:src/db/storage_mode.rs::storage_mode_marker_persisted_golden_fixture +codec-call:src/db/storage_mode.rs::preflight::rmp_serde::from_slice rust:src/db/storage_mode.rs::storage_mode_marker_persisted_golden_fixture +codec-call:src/db/types.rs::DbTid::add_micros::.to_be_bytes rust:src/db/types.rs::dbrkey_msgpack_binary_marker_preserves_tid +codec-call:src/db/types.rs::DbTid::add_micros::u64::from_be_bytes rust:src/db/types.rs::dbrkey_msgpack_binary_marker_preserves_tid +codec-call:src/db/types.rs::DbTid::from::.to_be_bytes rust:src/db/types.rs::dbrkey_msgpack_binary_marker_preserves_tid +codec-call:src/db/types.rs::DbTid::to_smolstr::u64::from_be_bytes rust:src/db/types.rs::dbrkey_msgpack_binary_marker_preserves_tid +codec-call:src/db/types.rs::DidKey::encode::multibase::encode rust:src/db/types.rs::db_action_and_did_key_representation_contracts +codec-call:src/db/types.rs::DidKey::from_did_key::multibase::decode rust:src/db/types.rs::db_action_and_did_key_representation_contracts +codec-call:src/ingest/indexer/shard.rs::FirehoseWorker::drain_resync_buffer::rmp_serde::from_slice rust:src/ingest/indexer/shard.rs::resync_buffer_commit_msgpack_round_trip_preserves_path +codec-call:src/ingest/relay/sink/relay.rs::build_relay_commit_ephemeral::serde_ipld_dagcbor::from_slice rust:src/ingest/relay/sink/relay.rs::relay_ephemeral_projection_preserves_record_representation +codec-call:src/ingest/relay/sink/relay.rs::build_relay_commit_ephemeral::serde_json::value::to_raw_value rust:src/ingest/relay/sink/relay.rs::relay_ephemeral_projection_preserves_record_representation +codec-call:src/ingest/stream/codec.rs::decode_frame::Deserialize::deserialize rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +codec-call:src/ingest/stream/codec.rs::decode_frame::ErrorFrame::deserialize rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +codec-call:src/ingest/stream/codec.rs::decode_frame::EventHeader::deserialize rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +codec-call:src/ingest/stream/codec.rs::decode_frame::serde_ipld_dagcbor::de::Deserializer::from_slice rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +codec-call:src/ingest/stream/codec.rs::encode_error_frame::serde_ipld_dagcbor::to_vec rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +codec-call:src/ingest/stream/codec.rs::encode_frame::serde_ipld_dagcbor::to_vec rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +codec-call:src/ingest/validation.rs::validate_commit::serde_ipld_dagcbor::from_slice rust:src/ingest/validation.rs::record_dag_cbor_projection_accepts_data_and_rejects_invalid_bytes +codec-call:src/jetstream.rs::build_ephemeral_from_stored::serde_ipld_dagcbor::from_slice nu:tests/stream_relay_jetstream_subscribe.nu +codec-call:src/jetstream.rs::build_ephemeral_from_stored::serde_json::value::to_raw_value nu:tests/stream_relay_jetstream_subscribe.nu +codec-call:src/jetstream.rs::stage_event::rmp_serde::to_vec nu:tests/stream_relay_jetstream_subscribe.nu +codec-call:src/jetstream.rs::stage_event::serde_json::to_vec nu:tests/stream_relay_jetstream_subscribe.nu +codec-call:src/ops.rs::persist_to_resync_buffer::rmp_serde::to_vec_named rust:src/ops.rs::ephemeral_identity_and_account_json_contracts +codec-call:src/ops.rs::transition_repo::rmp_serde::to_vec rust:src/ops.rs::ephemeral_identity_and_account_json_contracts +codec-call:src/ops/backlink_ops.rs::index_record::serde_ipld_dagcbor::from_slice nu:tests/backlinks.nu +codec-call:src/ops/record_events.rs::enabled::RecordEmitter::emit::rmp_serde::to_vec nu:tests/stream_cursor_replay.nu +codec-call:src/resolver.rs::Resolver::resolve_raw_doc::serde_json::from_slice rust:src/resolver.rs::raw_did_doc_json_contract_and_size_limit +codec-call:src/sparse_mst.rs::SparseScanner::scan_node::serde_ipld_dagcbor::from_slice rust:src/sparse_mst.rs::ancestor_bounds_prune_out_of_range_boundary_subtrees +codec-call:src/sparse_mst.rs::mst_node_layer::serde_ipld_dagcbor::from_slice rust:src/sparse_mst.rs::ancestor_bounds_prune_out_of_range_boundary_subtrees +codec-call:src/util/mod.rs::deser_status_code::Option::deserialize rust:src/util/mod.rs::http_status_serde_contract +config-parser:src/config/env.rs::Config::from_env rust:src/config/env.rs::env_text_parser_contracts +config-parser:src/config/env.rs::dotenv_assignments rust:src/config/env.rs::env_text_parser_contracts +config-parser:src/config/env.rs::load_dotenv rust:src/config/env.rs::env_text_parser_contracts +config-parser:src/config/env.rs::parse_new_host_limit rust:src/config/env.rs::env_text_parser_contracts +config-parser:src/config/env.rs::parse_new_host_limit_value rust:src/config/env.rs::env_text_parser_contracts +config-parser:src/config/types.rs::CrawlerSource::parse rust:src/config/types.rs::source_and_rate_tier_parser_contracts +config-parser:src/config/types.rs::FirehoseSource::parse rust:src/config/types.rs::source_and_rate_tier_parser_contracts +config-parser:src/config/types.rs::RateTier::parse rust:src/config/types.rs::source_and_rate_tier_parser_contracts +db-key-codec:src/backlinks/store.rs::forward_did_prefix rust:src/backlinks/store.rs::forward_did_prefix_matches_key +db-key-codec:src/backlinks/store.rs::forward_key rust:src/backlinks/store.rs::forward_did_prefix_matches_key +db-key-codec:src/backlinks/store.rs::reverse_key rust:src/backlinks/store.rs::source_from_reverse_key_roundtrip +db-key-codec:src/backlinks/store.rs::reverse_scan_prefix rust:src/backlinks/store.rs::reverse_key_scan_prefix_with_collection_and_path +db-key-codec:src/backlinks/store.rs::source_from_reverse_key rust:src/backlinks/store.rs::source_from_reverse_key_roundtrip +db-key-codec:src/db/filter.rs::collection_key rust:src/db/filter.rs::test_filter_keys +db-key-codec:src/db/filter.rs::exclude_key rust:src/db/filter.rs::test_exclude_key_trimmed +db-key-codec:src/db/filter.rs::signal_key rust:src/db/filter.rs::test_filter_keys +db-key-codec:src/db/keys/indexer.rs::block_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::by_collection_cursor_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::by_collection_cursor_prefix rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::count_collection_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::crawler_cursor_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::crawler_retry_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::crawler_retry_parse_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::crawler_source_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::event_body_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::event_watermark_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::history_hit_for rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::history_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::history_range_after rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::history_record_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::parse_rkey_text rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::pending_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::record_cid_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::record_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::record_key_trimmed rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::record_prefix_collection rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::record_prefix_did rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::record_prefix_trimmed rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::redaction_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::relay_first_cursor_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::relay_first_cursor_prefix rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::relay_first_handled_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::relay_first_hosts_cursor_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::relay_first_pass_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::relay_first_pds_cursor_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::resync_buffer_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::resync_buffer_prefix rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::split_record_cid_key rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::split_record_suffix rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/indexer.rs::trimmed_did_len rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::count_delta_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::count_delta_start_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::count_gauge_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::count_keyspace_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::did_collection_prefix rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::event_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::firehose_source_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::jetstream_event_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::parse_count_delta_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::parse_jetstream_event_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::pds_account_count_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::relay_event_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::relay_event_watermark_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::repo_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/mod.rs::repo_metadata_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/v1.rs::firehose_cursor_key rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/keys/v1.rs::firehose_cursor_key_from_url rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-codec:src/db/migration/mod.rs::migration_applied_key rust:src/db/migration/mod.rs::applied_migration_keys_track_per_migration_status +db-key-codec:src/db/migration/mod.rs::pass_cursor_key rust:src/db/migration/mod.rs::cursor_keys_are_distinct_per_version_and_pass +db-key-codec:src/db/migration/v10.rs::is_current_key rust:src/db/migration/v10.rs::v10_rewrites_records_and_retires_legacy_storage +db-key-codec:src/db/migration/v10.rs::parse_legacy_key rust:src/db/migration/v10.rs::v10_rewrites_records_and_retires_legacy_storage +db-key-codec:src/db/migration/v10.rs::rewrite_filter_key rust:src/db/migration/v10.rs::v10_finishes_v9_filter_keys_skipped_by_a_reduced_feature_build +db-key-codec:src/db/migration/v8.rs::count_gauge_key_from_repo_key rust:src/db/migration/v8.rs::lifecycle_count_migration_rebuilds_skewed_counts_and_drops_old_deltas +db-key-codec:src/db/migration/v8.rs::repo_metadata_key_from_repo_key rust:src/db/migration/v8.rs::lifecycle_count_migration_rebuilds_skewed_counts_and_drops_old_deltas +db-key-codec:src/db/pds_meta.rs::v5::pds_status_key nu:tests/pds_host_status_transitions.nu +db-key-codec:src/db/pds_meta.rs::v5::pds_tier_key nu:tests/api_pds_tiers.nu +db-key-constant:src/db/filter.rs::COLLECTION_PREFIX rust:src/db/filter.rs::test_filter_keys +db-key-constant:src/db/filter.rs::EXCLUDE_PREFIX rust:src/db/filter.rs::test_filter_keys +db-key-constant:src/db/filter.rs::MODE_KEY rust:src/db/filter.rs::test_filter_keys +db-key-constant:src/db/filter.rs::SIGNAL_PREFIX rust:src/db/filter.rs::test_filter_keys +db-key-constant:src/db/keys/indexer.rs::BY_COLLECTION_CURSOR_PREFIX rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-constant:src/db/keys/indexer.rs::CRAWLER_CURSOR_PREFIX rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-constant:src/db/keys/indexer.rs::CRAWLER_RETRY_PREFIX rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-constant:src/db/keys/indexer.rs::CRAWLER_SOURCE_PREFIX rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-constant:src/db/keys/indexer.rs::EVENT_WATERMARK_PREFIX rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-constant:src/db/keys/indexer.rs::REC_PATH_SEP rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-constant:src/db/keys/indexer.rs::REC_SEP rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-constant:src/db/keys/indexer.rs::RELAY_FIRST_CURSOR_PREFIX rust:src/db/keys/indexer.rs::indexer_key_codec_inventory_contracts +db-key-constant:src/db/keys/mod.rs::COUNT_COLLECTION_PREFIX rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-constant:src/db/keys/mod.rs::COUNT_DELTA_PREFIX rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-constant:src/db/keys/mod.rs::COUNT_DELTA_WATERMARK_KEY rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-constant:src/db/keys/mod.rs::COUNT_GAUGE_PREFIX rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-constant:src/db/keys/mod.rs::COUNT_KS_PREFIX rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-constant:src/db/keys/mod.rs::FIREHOSE_CURSOR_PREFIX rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-constant:src/db/keys/mod.rs::FIREHOSE_SOURCE_PREFIX rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-constant:src/db/keys/mod.rs::PDS_DAILY_ADDS_KEY rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-constant:src/db/keys/mod.rs::RELAY_EVENT_WATERMARK_PREFIX rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-constant:src/db/keys/mod.rs::REPO_METADATA_PREFIX rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-constant:src/db/keys/mod.rs::SEP rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-constant:src/db/keys/mod.rs::VERSIONING_KEY rust:src/db/keys/mod.rs::shared_key_codec_inventory_contracts +db-key-constant:src/db/migration/mod.rs::LEGACY_BLOCKS_KEYSPACE rust:src/db/migration/v10.rs::v10_rewrites_records_and_retires_legacy_storage +db-key-constant:src/db/migration/mod.rs::LEGACY_RECORDS_KEYSPACE rust:src/db/migration/v10.rs::v10_rewrites_records_and_retires_legacy_storage +db-key-constant:src/db/migration/mod.rs::MIGRATION_APPLIED_PREFIX rust:src/db/migration/mod.rs::applied_migration_keys_track_per_migration_status +db-key-constant:src/db/migration/mod.rs::MIGRATION_CURSOR_PREFIX rust:src/db/migration/mod.rs::cursor_keys_are_distinct_per_version_and_pass +db-key-constant:src/db/migration/v5.rs::v4::PDS_BANNED_PREFIX rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +db-key-constant:src/db/migration/v5.rs::v4::PDS_TIER_PREFIX rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +db-key-constant:src/db/storage_mode.rs::STORAGE_MODE_KEY rust:src/db/storage_mode.rs::storage_mode_marker_persisted_golden_fixture +db-key-constant:src/db/types.rs::TAG_PLC rust:src/db/types.rs::db_representation_conversions_are_lossless +db-key-constant:src/db/types.rs::TAG_WEB rust:src/db/types.rs::db_representation_conversions_are_lossless +db-keyspace:src/db/schema.rs::Backlinks rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::Counts rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::Crawler rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::Cursors rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::EventBodies rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::Events rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::Filter rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::Heads rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::History rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::JetstreamEvents rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::Pending rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::Redactions rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::RelayEvents rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::RepoMetadata rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::Repos rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::Resync rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +db-keyspace:src/db/schema.rs::ResyncBuffer rust:src/db/schema.rs::keyspace_names_match_the_compiled_storage_schema +firehose-codec:src/ingest/stream/codec.rs::decode_frame rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +firehose-codec:src/ingest/stream/codec.rs::encode_error_frame rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +firehose-codec:src/ingest/stream/codec.rs::encode_frame rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +manual-serde:src/config/types.rs::CrawlerMode::Deserialize rust:src/config/types.rs::crawler_mode_string_and_msgpack_contracts +manual-serde:src/config/types.rs::CrawlerMode::Serialize rust:src/config/types.rs::crawler_mode_string_and_msgpack_contracts +manual-serde:src/db/types.rs::DbRkey::Deserialize rust:src/db/types.rs::dbrkey_string_markers_preserve_every_utf8_length +manual-serde:src/db/types.rs::TrimmedDid::Deserialize rust:src/control/stream/indexer.rs::legacy_string_trimmed_did_replays_after_reopen +manual-serde:src/db/types.rs::TrimmedDid::Serialize rust:src/control/stream/indexer.rs::legacy_string_trimmed_did_replays_after_reopen +manual-serde:src/ingest/stream/types.rs::AccountStatus::Deserialize rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +manual-serde:src/ingest/stream/types.rs::AccountStatus::Serialize rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +manual-serde:src/ingest/stream/types.rs::InfoName::Deserialize rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +manual-serde:src/ingest/stream/types.rs::InfoName::Serialize rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +manual-serde:src/ingest/stream/types.rs::RepoOpAction::Deserialize rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +manual-serde:src/ingest/stream/types.rs::RepoOpAction::Serialize rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +migration-transform:src/db/migration/mod.rs::is_migration_applied rust:src/db/migration/mod.rs::a_chunk_that_fails_partway_commits_nothing +migration-transform:src/db/migration/mod.rs::legacy_block rust:src/db/migration/mod.rs::a_chunk_that_fails_partway_commits_nothing +migration-transform:src/db/migration/mod.rs::legacy_blocks rust:src/db/migration/mod.rs::a_chunk_that_fails_partway_commits_nothing +migration-transform:src/db/migration/mod.rs::legacy_records rust:src/db/migration/mod.rs::a_chunk_that_fails_partway_commits_nothing +migration-transform:src/db/migration/mod.rs::migration_applied_key rust:src/db/migration/mod.rs::a_chunk_that_fails_partway_commits_nothing +migration-transform:src/db/migration/mod.rs::pass_cursor_key rust:src/db/migration/mod.rs::a_chunk_that_fails_partway_commits_nothing +migration-transform:src/db/migration/mod.rs::read_version rust:src/db/migration/mod.rs::a_chunk_that_fails_partway_commits_nothing +migration-transform:src/db/migration/mod.rs::run rust:src/db/migration/mod.rs::a_chunk_that_fails_partway_commits_nothing +migration-transform:src/db/migration/mod.rs::run_chunk rust:src/db/migration/mod.rs::a_chunk_that_fails_partway_commits_nothing +migration-transform:src/db/migration/mod.rs::run_pass rust:src/db/migration/mod.rs::a_chunk_that_fails_partway_commits_nothing +migration-transform:src/db/migration/v1.rs::stable_firehose_cursors rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +migration-transform:src/db/migration/v10.rs::is_current_key rust:src/db/migration/v10.rs::v10_rewrites_records_and_retires_legacy_storage +migration-transform:src/db/migration/v10.rs::parse_legacy_key rust:src/db/migration/v10.rs::v10_rewrites_records_and_retires_legacy_storage +migration-transform:src/db/migration/v10.rs::retire_legacy_stores rust:src/db/migration/v10.rs::v10_rewrites_records_and_retires_legacy_storage +migration-transform:src/db/migration/v10.rs::rewrite_filter_key rust:src/db/migration/v10.rs::v10_finishes_v9_filter_keys_skipped_by_a_reduced_feature_build +migration-transform:src/db/migration/v10.rs::rewrite_record rust:src/db/migration/v10.rs::v10_rewrites_records_and_retires_legacy_storage +migration-transform:src/db/migration/v10/events.rs::body_matches rust:src/db/migration/v10/events.rs::cid_mismatch_aborts_without_clearing_blocks +migration-transform:src/db/migration/v10/events.rs::decode_event rust:src/db/migration/v10/events.rs::cid_mismatch_aborts_without_clearing_blocks +migration-transform:src/db/migration/v10/events.rs::materialize_pointer_event rust:src/db/migration/v10/events.rs::cid_mismatch_aborts_without_clearing_blocks +migration-transform:src/db/migration/v10/events.rs::retire_blocks rust:src/db/migration/v10/events.rs::cid_mismatch_aborts_without_clearing_blocks +migration-transform:src/db/migration/v10/events.rs::stage_archive_body rust:src/db/migration/v10/events.rs::cid_mismatch_aborts_without_clearing_blocks +migration-transform:src/db/migration/v2.rs::repo_state_root_commit rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +migration-transform:src/db/migration/v3.rs::firehose_source_is_pds rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +migration-transform:src/db/migration/v3.rs::is_known_relay rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +migration-transform:src/db/migration/v4.rs::repo_state_active rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +migration-transform:src/db/migration/v5.rs::pds_meta_layout rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +migration-transform:src/db/migration/v6.rs::rebuild_pds_account_counts rust:src/db/migration/mod.rs::v0_database_migrates_all_early_representation_boundaries +migration-transform:src/db/migration/v7.rs::repo_state_event_clocks rust:src/db/migration/v7.rs::repo_state_event_clock_migration_rewrites_v6_repo_states +migration-transform:src/db/migration/v8.rs::count_gauge_key_from_repo_key rust:src/db/migration/v8.rs::lifecycle_count_migration_rebuilds_skewed_counts_and_drops_old_deltas +migration-transform:src/db/migration/v8.rs::is_lifecycle_count rust:src/db/migration/v8.rs::lifecycle_count_migration_rebuilds_skewed_counts_and_drops_old_deltas +migration-transform:src/db/migration/v8.rs::legacy_migration_applied rust:src/db/migration/v8.rs::lifecycle_count_migration_rebuilds_skewed_counts_and_drops_old_deltas +migration-transform:src/db/migration/v8.rs::primary_lifecycle_gauge rust:src/db/migration/v8.rs::lifecycle_count_migration_rebuilds_skewed_counts_and_drops_old_deltas +migration-transform:src/db/migration/v8.rs::rebuild_lifecycle_counts rust:src/db/migration/v8.rs::lifecycle_count_migration_rebuilds_skewed_counts_and_drops_old_deltas +migration-transform:src/db/migration/v8.rs::repo_metadata_key_from_repo_key rust:src/db/migration/v8.rs::lifecycle_count_migration_rebuilds_skewed_counts_and_drops_old_deltas +migration-transform:src/db/migration/v9.rs::migrate_v9 rust:src/db/migration/v9.rs::test_migration_v9 +representation-conversion:src/config.rs::Config::fmt::Display rust:src/config.rs::display_reports_history_ttl +representation-conversion:src/config/types.rs::BackfillStrategy::fmt::Display rust:src/config/types.rs::text_config_parser_contracts +representation-conversion:src/config/types.rs::Compression::fmt::Display rust:src/config/types.rs::text_config_parser_contracts +representation-conversion:src/config/types.rs::CrawlerMode::fmt::Display rust:src/config/types.rs::crawler_mode_string_and_msgpack_contracts +representation-conversion:src/config/types.rs::SignatureVerification::fmt::Display rust:src/config/types.rs::text_config_parser_contracts +representation-conversion:src/control/pds.rs::PdsTierDefinition::From nu:tests/api_pds_tiers.nu +representation-conversion:src/db/types.rs::DbAction::Display rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::DbAction::TryFrom<&str> rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::DbRkey::Display rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::DbRkey::From<&str> rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::DbRkey::From rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::DbRkey::From rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::DbTid::Display rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::DbTid::From<&Tid> rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::DidKey<>::From> rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::DidKey<>::std::fmt::Display rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::Tid::From rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::TrimmedDid<>::Display rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::TrimmedDid<>::From<&Did<>> rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::TrimmedDid<>::TryFrom<&[u8]> rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::UserKey::From<&TrimmedDid<>> rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/db/types.rs::UserKey::From> rust:src/db/types.rs::db_representation_conversions_are_lossless +representation-conversion:src/ingest/stream/types.rs::AccountStatus<>::AsRef rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::AccountStatus<>::From<&str> rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::AccountStatus<>::From rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::AccountStatus<>::core::fmt::Display rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::AccountStatus<>::jacquard_common::IntoStatic rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::Datetime::TryFrom rust:src/ingest/stream/types.rs::firehose_datetime_accepts_zoned_and_legacy_unzoned_values +representation-conversion:src/ingest/stream/types.rs::Datetime::jacquard_common::IntoStatic rust:src/ingest/stream/types.rs::firehose_datetime_accepts_zoned_and_legacy_unzoned_values +representation-conversion:src/ingest/stream/types.rs::InfoName<>::AsRef rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::InfoName<>::From<&str> rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::InfoName<>::From rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::InfoName<>::core::fmt::Display rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::InfoName<>::jacquard_common::IntoStatic rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::RepoOpAction<>::AsRef rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::RepoOpAction<>::From<&str> rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::RepoOpAction<>::From rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::RepoOpAction<>::core::fmt::Display rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/ingest/stream/types.rs::RepoOpAction<>::jacquard_common::IntoStatic rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +representation-conversion:src/pds_meta.rs::jacquard_api::com_atproto::sync::HostStatus<>::From rust:src/pds_meta.rs::host_status_persisted_golden_fixtures +representation-conversion:src/types.rs::Commit::From> rust:src/types.rs::commit_and_repo_status_conversion_contracts +representation-conversion:src/types.rs::RepoState<>::IntoStatic rust:src/types.rs::into_static_preserves_per_event_clocks +representation-conversion:src/types.rs::RepoStatus::Display rust:src/types.rs::commit_and_repo_status_conversion_contracts +representation-conversion:src/types/event/stream.rs::EventType::AsRef rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/api/firehose.rs::AddSourceRequest::is_pds::serde(default) nu:tests/api_firehose_sources.nu +serde-attribute:src/api/ingestion.rs::IngestionPatch::backfill::serde(default) rust:src/api/ingestion.rs::ingestion_json_contracts_follow_compiled_mode +serde-attribute:src/api/ingestion.rs::IngestionPatch::crawler::serde(default) rust:src/api/ingestion.rs::ingestion_json_contracts_follow_compiled_mode +serde-attribute:src/api/ingestion.rs::IngestionPatch::firehose::serde(default) rust:src/api/ingestion.rs::ingestion_json_contracts_follow_compiled_mode +serde-attribute:src/api/jetstream.rs::JetstreamQuery::compress::serde(default) rust:src/api/jetstream.rs::jetstream_query_accepts_repeated_values +serde-attribute:src/api/jetstream.rs::JetstreamQuery::max_message_size_bytes::serde(default,rename="maxMessageSizeBytes",alias="maxSize") rust:src/api/jetstream.rs::jetstream_query_accepts_repeated_values +serde-attribute:src/api/jetstream.rs::JetstreamQuery::require_hello::serde(default,rename="requireHello") rust:src/api/jetstream.rs::jetstream_query_accepts_repeated_values +serde-attribute:src/api/jetstream.rs::JetstreamQuery::wanted_collections::serde(default,rename="wantedCollections") rust:src/api/jetstream.rs::jetstream_query_accepts_repeated_values +serde-attribute:src/api/jetstream.rs::JetstreamQuery::wanted_dids::serde(default,rename="wantedDids") rust:src/api/jetstream.rs::jetstream_query_accepts_repeated_values +serde-attribute:src/api/jetstream.rs::JetstreamQuery::wanted_event_types::serde(default,rename="wantedEventTypes") rust:src/api/jetstream.rs::jetstream_query_accepts_repeated_values +serde-attribute:src/api/jetstream.rs::SubscriberOptionsUpdatePayload::max_message_size_bytes::serde(default,rename="maxMessageSizeBytes",alias="maxSize") rust:src/api/jetstream.rs::jetstream_options_message_json_contract +serde-attribute:src/api/jetstream.rs::SubscriberOptionsUpdatePayload::wanted_collections::serde(default,rename="wantedCollections") rust:src/api/jetstream.rs::jetstream_options_message_json_contract +serde-attribute:src/api/jetstream.rs::SubscriberOptionsUpdatePayload::wanted_dids::serde(default,rename="wantedDids") rust:src/api/jetstream.rs::jetstream_options_message_json_contract +serde-attribute:src/api/jetstream.rs::SubscriberOptionsUpdatePayload::wanted_event_types::serde(default,rename="wantedEventTypes") rust:src/api/jetstream.rs::jetstream_options_message_json_contract +serde-attribute:src/api/jetstream.rs::SubscriberSourcedMessage::kind::serde(rename="type") rust:src/api/jetstream.rs::jetstream_options_message_json_contract +serde-attribute:src/api/redactions.rs::RedactionQueryParams::exclude::serde(default) rust:src/api/redactions.rs::test_parse_redaction_target_invalid +serde-attribute:src/api/redactions.rs::RedactionQueryParams::target::serde(default) rust:src/api/redactions.rs::test_parse_redaction_target_invalid +serde-attribute:src/api/xrpc/count_records.rs::CountRecordsRequestData::identifier::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/describe_repo.rs::DescribeRepoOutput::collections::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/describe_repo.rs::DescribeRepoOutput::did::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/describe_repo.rs::DescribeRepoOutput::handle::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/describe_repo.rs::DescribeRepoOutput::signing_key::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/describe_repo.rs::DescribeRepoOutput::signing_key::serde(serialize_with="crate::util::did_key_serialize_str") nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/describe_repo.rs::DescribeRepoRequestData::identifier::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/get_record_by_uri.rs::BadExampleGetUriRecordRequestData::at_uri::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/get_record_by_uri.rs::BadExampleGetUriRecordRequestData::cid::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/get_record_by_uri.rs::BadExampleGetUriRecordRequestData::cid::serde(default) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/get_record_by_uri.rs::GetRecordByUriRequestData::at_uri::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/get_record_by_uri.rs::GetRecordByUriRequestData::cid::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/get_record_by_uri.rs::GetRecordByUriRequestData::cid::serde(default) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/resolve_mini_doc.rs::BlueMicrocosmResolveMiniDocRequestData::identifier::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/resolve_mini_doc.rs::ResolveMiniDocOutput::did::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/resolve_mini_doc.rs::ResolveMiniDocOutput::handle::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/resolve_mini_doc.rs::ResolveMiniDocOutput::signing_key::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/resolve_mini_doc.rs::ResolveMiniDocOutput::signing_key::serde(serialize_with="crate::util::did_key_serialize_str") nu:tests/api_xrpc.nu +serde-attribute:src/api/xrpc/resolve_mini_doc.rs::ResolveMiniDocRequestData::identifier::serde(borrow) nu:tests/api_xrpc.nu +serde-attribute:src/backlinks/api.rs::GetBacklinksCountParams::did::serde(default) rust:src/backlinks/api.rs::backlinks_query_accepts_single_and_repeated_dids +serde-attribute:src/backlinks/api.rs::GetBacklinksOutput::cursor::serde(skip_serializing_if="Option::is_none") rust:src/backlinks/api.rs::backlinks_query_accepts_single_and_repeated_dids +serde-attribute:src/backlinks/api.rs::GetBacklinksParams::did::serde(default) rust:src/backlinks/api.rs::backlinks_query_accepts_single_and_repeated_dids +serde-attribute:src/bin/backfill_strategy_bench.rs::DidService::kind::serde(rename="type") rust:src/bin/backfill_strategy_bench.rs::benchmark_api_json_fixtures_decode +serde-attribute:src/bin/backfill_strategy_bench.rs::DidService::service_endpoint::serde(rename="serviceEndpoint") rust:src/bin/backfill_strategy_bench.rs::benchmark_api_json_fixtures_decode +serde-attribute:src/control/firehose.rs::FirehoseSourceInfo::host_status::serde(skip_serializing_if="Option::is_none") nu:tests/api_firehose_sources.nu +serde-attribute:src/control/firehose.rs::FirehoseSourceInfo::last_failure::serde(skip_serializing_if="Option::is_none") nu:tests/api_firehose_sources.nu +serde-attribute:src/control/firehose.rs::FirehoseSourceInfo::pds::serde(skip_serializing_if="Option::is_none") nu:tests/api_firehose_sources.nu +serde-attribute:src/control/firehose.rs::FirehoseSourceInfo::retry_in_secs::serde(skip_serializing_if="Option::is_none") nu:tests/api_firehose_sources.nu +serde-attribute:src/control/firehose.rs::FirehoseSourceInfo::stats::serde(skip_serializing_if="Option::is_none") nu:tests/api_firehose_sources.nu +serde-attribute:src/control/firehose.rs::FirehoseSourceInfo::throttled_until::serde(skip_serializing_if="Option::is_none") nu:tests/api_firehose_sources.nu +serde-attribute:src/control/repos/mod.rs::RepoInfo::data::serde(serialize_with="crate::util::opt_cid_serialize_str") rust:src/control/repos/mod.rs::repo_info_json_contract +serde-attribute:src/control/repos/mod.rs::RepoInfo::data::serde(skip_serializing_if="Option::is_none") rust:src/control/repos/mod.rs::repo_info_json_contract +serde-attribute:src/control/repos/mod.rs::RepoInfo::handle::serde(skip_serializing_if="Option::is_none") rust:src/control/repos/mod.rs::repo_info_json_contract +serde-attribute:src/control/repos/mod.rs::RepoInfo::last_message_at::serde(skip_serializing_if="Option::is_none") rust:src/control/repos/mod.rs::repo_info_json_contract +serde-attribute:src/control/repos/mod.rs::RepoInfo::last_updated_at::serde(skip_serializing_if="Option::is_none") rust:src/control/repos/mod.rs::repo_info_json_contract +serde-attribute:src/control/repos/mod.rs::RepoInfo::pds::serde(skip_serializing_if="Option::is_none") rust:src/control/repos/mod.rs::repo_info_json_contract +serde-attribute:src/control/repos/mod.rs::RepoInfo::rev::serde(skip_serializing_if="Option::is_none") rust:src/control/repos/mod.rs::repo_info_json_contract +serde-attribute:src/control/repos/mod.rs::RepoInfo::signing_key::serde(serialize_with="crate::util::opt_did_key_serialize_str") rust:src/control/repos/mod.rs::repo_info_json_contract +serde-attribute:src/control/repos/mod.rs::RepoInfo::signing_key::serde(skip_serializing_if="Option::is_none") rust:src/control/repos/mod.rs::repo_info_json_contract +serde-attribute:src/control/repos/mod.rs::RepoInfo::status::serde(serialize_with="crate::util::repo_status_serialize_str") rust:src/control/repos/mod.rs::repo_info_json_contract +serde-attribute:src/control/stream/jetstream.rs::JetstreamAccount::status::serde(skip_serializing_if="Option::is_none") nu:tests/stream_cursor_replay.nu +serde-attribute:src/control/stream/jetstream.rs::JetstreamCommit::cid::serde(skip_serializing_if="Option::is_none") nu:tests/stream_cursor_replay.nu +serde-attribute:src/control/stream/jetstream.rs::JetstreamCommit::record::serde(skip_serializing_if="Option::is_none") nu:tests/stream_cursor_replay.nu +serde-attribute:src/control/stream/jetstream.rs::JetstreamEvent::payload::serde(flatten) nu:tests/stream_cursor_replay.nu +serde-attribute:src/control/stream/jetstream.rs::JetstreamIdentity::handle::serde(skip_serializing_if="Option::is_none") nu:tests/stream_cursor_replay.nu +serde-attribute:src/control/stream/jetstream.rs::JetstreamPayload::Account::account::serde(borrow) nu:tests/stream_cursor_replay.nu +serde-attribute:src/control/stream/jetstream.rs::JetstreamPayload::Identity::identity::serde(borrow) nu:tests/stream_cursor_replay.nu +serde-attribute:src/control/stream/jetstream.rs::JetstreamPayload::serde(tag="kind",rename_all="lowercase") nu:tests/stream_cursor_replay.nu +serde-attribute:src/crawler/list_repos/checker.rs::RetryState::status::serde(deserialize_with="crate::util::deser_status_code") rust:src/crawler/list_repos/checker.rs::retry_state_persisted_golden_fixture +serde-attribute:src/crawler/list_repos/checker.rs::RetryState::status::serde(serialize_with="crate::util::ser_status_code") rust:src/crawler/list_repos/checker.rs::retry_state_persisted_golden_fixture +serde-attribute:src/db/migration/v2.rs::OldRepoState::handle::serde(borrow) rust:src/db/migration/v2.rs::v1_repo_state_input_golden_fixture +serde-attribute:src/db/migration/v2.rs::OldRepoState::last_message_time::serde(default) rust:src/db/migration/v2.rs::v1_repo_state_input_golden_fixture +serde-attribute:src/db/migration/v2.rs::OldRepoState::pds::serde(borrow) rust:src/db/migration/v2.rs::v1_repo_state_input_golden_fixture +serde-attribute:src/db/migration/v2.rs::OldRepoState::serde(bound(deserialize="'i:'de")) rust:src/db/migration/v2.rs::v1_repo_state_input_golden_fixture +serde-attribute:src/db/migration/v2.rs::OldRepoState::signing_key::serde(borrow) rust:src/db/migration/v2.rs::v1_repo_state_input_golden_fixture +serde-attribute:src/db/migration/v4.rs::OldRepoState::handle::serde(borrow) rust:src/db/migration/v4.rs::v2_repo_state_input_golden_fixture +serde-attribute:src/db/migration/v4.rs::OldRepoState::pds::serde(borrow) rust:src/db/migration/v4.rs::v2_repo_state_input_golden_fixture +serde-attribute:src/db/migration/v4.rs::OldRepoState::serde(bound(deserialize="'i:'de")) rust:src/db/migration/v4.rs::v2_repo_state_input_golden_fixture +serde-attribute:src/db/migration/v4.rs::OldRepoState::signing_key::serde(borrow) rust:src/db/migration/v4.rs::v2_repo_state_input_golden_fixture +serde-attribute:src/db/mod.rs::FirehoseSourceMeta::is_pds::serde(default) rust:src/db/mod.rs::firehose_source_meta_golden_fixture +serde-attribute:src/db/types.rs::DbRkey::serde(untagged) rust:src/db/types.rs::dbrkey_string_markers_preserve_every_utf8_length +serde-attribute:src/db/types.rs::DbTid::0::serde(with="serde_bytes") rust:src/db/types.rs::dbrkey_msgpack_binary_marker_preserves_tid +serde-attribute:src/db/types.rs::DidKey::0::serde(borrow) rust:src/db/types.rs::db_action_and_did_key_representation_contracts +serde-attribute:src/db/types.rs::DidKey::0::serde(with="serde_bytes") rust:src/db/types.rs::db_action_and_did_key_representation_contracts +serde-attribute:src/filter.rs::FilterMode::serde(rename_all="snake_case") rust:src/filter.rs::filter_mode_persisted_golden_fixtures +serde-attribute:src/ingest/firehose_stats/source.rs::FirehoseStatsSnapshot::last_connect_attempt_at::serde(skip_serializing_if="Option::is_none") rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract +serde-attribute:src/ingest/firehose_stats/source.rs::FirehoseStatsSnapshot::last_connected_at::serde(skip_serializing_if="Option::is_none") rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract +serde-attribute:src/ingest/firehose_stats/source.rs::FirehoseStatsSnapshot::last_decoded_at::serde(skip_serializing_if="Option::is_none") rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract +serde-attribute:src/ingest/firehose_stats/source.rs::FirehoseStatsSnapshot::last_error_at::serde(skip_serializing_if="Option::is_none") rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract +serde-attribute:src/ingest/firehose_stats/source.rs::FirehoseStatsSnapshot::last_error_kind::serde(skip_serializing_if="Option::is_none") rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract +serde-attribute:src/ingest/firehose_stats/source.rs::FirehoseStatsSnapshot::last_forwarded_at::serde(skip_serializing_if="Option::is_none") rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract +serde-attribute:src/ingest/firehose_stats/source.rs::FirehoseStatsSnapshot::last_frame_at::serde(skip_serializing_if="Option::is_none") rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract +serde-attribute:src/ingest/firehose_stats/source.rs::FirehoseStatsSnapshot::last_seq::serde(skip_serializing_if="Option::is_none") rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract +serde-attribute:src/ingest/firehose_stats/source.rs::FirehoseStatsSnapshot::last_start_cursor::serde(skip_serializing_if="Option::is_none") rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract +serde-attribute:src/ingest/firehose_stats/source.rs::FirehoseStatsSnapshot::max_seq::serde(skip_serializing_if="Option::is_none") rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract +serde-attribute:src/ingest/stream/codec.rs::EncodeErrorFrame::message::serde(skip_serializing_if="Option::is_none") rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +serde-attribute:src/ingest/stream/types.rs::Account::did::serde(borrow) rust:src/ingest/stream.rs::test_decode_account +serde-attribute:src/ingest/stream/types.rs::Account::serde(rename_all="camelCase") rust:src/ingest/stream.rs::test_decode_account +serde-attribute:src/ingest/stream/types.rs::Commit::blobs::serde(borrow) rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::Commit::blocks::serde(with="jacquard_common::serde_bytes_helper") rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::Commit::commit::serde(borrow) rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::Commit::ops::serde(borrow) rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::Commit::prev_data::serde(borrow) rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::Commit::prev_data::serde(default) rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::Commit::prev_data::serde(skip_serializing_if="Option::is_none") rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::Commit::repo::serde(borrow) rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::Commit::serde(rename_all="camelCase") rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::Commit::since::serde(deserialize_with="deserialize_tid_or_empty") rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::Commit::too_big::serde(default) rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::Datetime::serde(try_from="String") rust:src/ingest/stream/types.rs::firehose_datetime_accepts_zoned_and_legacy_unzoned_values +serde-attribute:src/ingest/stream/types.rs::Identity::did::serde(borrow) rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +serde-attribute:src/ingest/stream/types.rs::Identity::handle::serde(borrow) rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +serde-attribute:src/ingest/stream/types.rs::Identity::handle::serde(default) rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +serde-attribute:src/ingest/stream/types.rs::Identity::handle::serde(skip_serializing_if="Option::is_none") rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +serde-attribute:src/ingest/stream/types.rs::Identity::serde(rename_all="camelCase") rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +serde-attribute:src/ingest/stream/types.rs::Info::message::serde(borrow) rust:src/ingest/stream/codec.rs::firehose_frame_maps_outdated_cursor_info +serde-attribute:src/ingest/stream/types.rs::Info::message::serde(skip_serializing_if="Option::is_none") rust:src/ingest/stream/codec.rs::firehose_frame_maps_outdated_cursor_info +serde-attribute:src/ingest/stream/types.rs::Info::name::serde(borrow) rust:src/ingest/stream/codec.rs::firehose_frame_maps_outdated_cursor_info +serde-attribute:src/ingest/stream/types.rs::Info::serde(rename_all="camelCase") rust:src/ingest/stream/codec.rs::firehose_frame_maps_outdated_cursor_info +serde-attribute:src/ingest/stream/types.rs::RepoOp::action::serde(borrow) rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::RepoOp::cid::serde(borrow) rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::RepoOp::path::serde(borrow) rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::RepoOp::prev::serde(borrow) rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::RepoOp::prev::serde(default) rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::RepoOp::prev::serde(skip_serializing_if="std::option::Option::is_none") rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::RepoOp::serde(rename_all="camelCase") rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-attribute:src/ingest/stream/types.rs::Sync::blocks::serde(with="jacquard_common::serde_bytes_helper") rust:src/ingest/stream/codec.rs::firehose_frame_mock_decodes_sync_binary_blocks +serde-attribute:src/ingest/stream/types.rs::Sync::did::serde(borrow) rust:src/ingest/stream/codec.rs::firehose_frame_mock_decodes_sync_binary_blocks +serde-attribute:src/ingest/stream/types.rs::Sync::rev::serde(borrow) rust:src/ingest/stream/codec.rs::firehose_frame_mock_decodes_sync_binary_blocks +serde-attribute:src/ingest/stream/types.rs::Sync::serde(rename_all="camelCase") rust:src/ingest/stream/codec.rs::firehose_frame_mock_decodes_sync_binary_blocks +serde-attribute:src/patch.rs::SetUpdate::serde(untagged) rust:src/patch.rs::set_update_json_markers_are_structurally_disjoint +serde-attribute:src/types.rs::DeleteBodyTarget::serde(rename_all="snake_case") rust:src/types.rs::operator_redaction_json_contracts +serde-attribute:src/types/event/jetstream.rs::StoredJetstreamEvent::Account::did::serde(borrow) rust:src/types/event/jetstream.rs::common_jetstream_event_golden_fixtures +serde-attribute:src/types/event/jetstream.rs::StoredJetstreamEvent::Account::status::serde(borrow) rust:src/types/event/jetstream.rs::common_jetstream_event_golden_fixtures +serde-attribute:src/types/event/jetstream.rs::StoredJetstreamEvent::Commit::collection::serde(borrow) rust:src/types/event/jetstream.rs::indexer_jetstream_event_golden_fixture +serde-attribute:src/types/event/jetstream.rs::StoredJetstreamEvent::Commit::did::serde(borrow) rust:src/types/event/jetstream.rs::indexer_jetstream_event_golden_fixture +serde-attribute:src/types/event/jetstream.rs::StoredJetstreamEvent::Commit::live::serde(default="default_true") rust:src/types/event/jetstream.rs::indexer_jetstream_event_golden_fixture +serde-attribute:src/types/event/jetstream.rs::StoredJetstreamEvent::Identity::did::serde(borrow) rust:src/types/event/jetstream.rs::common_jetstream_event_golden_fixtures +serde-attribute:src/types/event/jetstream.rs::StoredJetstreamEvent::Identity::handle::serde(borrow) rust:src/types/event/jetstream.rs::common_jetstream_event_golden_fixtures +serde-attribute:src/types/event/jetstream.rs::StoredJetstreamEvent::RelayAccount::did::serde(borrow) rust:src/types/event/jetstream.rs::relay_jetstream_event_golden_fixtures +serde-attribute:src/types/event/jetstream.rs::StoredJetstreamEvent::RelayCommit::collection::serde(borrow) rust:src/types/event/jetstream.rs::relay_jetstream_event_golden_fixtures +serde-attribute:src/types/event/jetstream.rs::StoredJetstreamEvent::RelayCommit::did::serde(borrow) rust:src/types/event/jetstream.rs::relay_jetstream_event_golden_fixtures +serde-attribute:src/types/event/jetstream.rs::StoredJetstreamEvent::RelayIdentity::did::serde(borrow) rust:src/types/event/jetstream.rs::relay_jetstream_event_golden_fixtures +serde-attribute:src/types/event/jetstream.rs::StoredJetstreamEvent::serde(bound(deserialize="'i:'de")) rust:src/types/event/jetstream.rs::common_jetstream_event_golden_fixtures +serde-attribute:src/types/event/stream.rs::AccountEvt::did::serde(borrow) rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::AccountEvt::status::serde(skip_serializing_if="Option::is_none") rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::IdentityEvt::did::serde(borrow) rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::IdentityEvt::handle::serde(skip_serializing_if="Option::is_none") rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::MarshallableEvt::account::serde(borrow) rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::MarshallableEvt::account::serde(skip_serializing_if="Option::is_none") rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::MarshallableEvt::identity::serde(borrow) rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::MarshallableEvt::identity::serde(skip_serializing_if="Option::is_none") rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::MarshallableEvt::kind::serde(rename="type") rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::MarshallableEvt::kind::serde(serialize_with="event_type_ser_str") rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::MarshallableEvt::record::serde(borrow) rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::MarshallableEvt::record::serde(skip_serializing_if="Option::is_none") rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::RecordEvt::cid::serde(serialize_with="crate::util::opt_cid_serialize_str") rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::RecordEvt::cid::serde(skip_serializing_if="Option::is_none") rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::RecordEvt::did::serde(borrow) rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::RecordEvt::record::serde(skip_serializing_if="Option::is_none") rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-attribute:src/types/event/stream.rs::StoredData::Block::serde(with="jacquard_common::serde_bytes_helper") rust:src/types/event/stream.rs::stored_data_persisted_golden_fixtures +serde-attribute:src/types/event/stream.rs::StoredEvent::collection::serde(borrow) rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim +serde-attribute:src/types/event/stream.rs::StoredEvent::data::serde(default) rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim +serde-attribute:src/types/event/stream.rs::StoredEvent::data::serde(skip_serializing_if="StoredData::is_nothing") rust:src/types/event/stream.rs::stored_data_persisted_golden_fixtures +serde-attribute:src/types/event/stream.rs::StoredEvent::did::serde(borrow) rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim +serde-attribute:src/types/event/stream.rs::StoredEvent::live::serde(default) rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim +serde-attribute:src/types/event/stream.rs::StoredEvent::serde(bound(deserialize="'i:'de")) rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim +serde-attribute:src/types/v2.rs::Commit::sig::serde(with="jacquard_common::serde_bytes_helper") rust:src/types/v2.rs::v2_persisted_schema_golden_fixture +serde-attribute:src/types/v2.rs::RepoState::handle::serde(borrow) rust:src/types/v2.rs::v2_persisted_schema_golden_fixture +serde-attribute:src/types/v2.rs::RepoState::pds::serde(borrow) rust:src/types/v2.rs::v2_persisted_schema_golden_fixture +serde-attribute:src/types/v2.rs::RepoState::serde(bound(deserialize="'i:'de")) rust:src/types/v2.rs::v2_persisted_schema_golden_fixture +serde-attribute:src/types/v2.rs::RepoState::signing_key::serde(borrow) rust:src/types/v2.rs::v2_persisted_schema_golden_fixture +serde-attribute:src/types/v4.rs::RepoState::handle::serde(borrow) rust:src/types/v4.rs::v4_persisted_schema_golden_fixtures +serde-attribute:src/types/v4.rs::RepoState::pds::serde(borrow) rust:src/types/v4.rs::v4_persisted_schema_golden_fixtures +serde-attribute:src/types/v4.rs::RepoState::serde(bound(deserialize="'i:'de")) rust:src/types/v4.rs::v4_persisted_schema_golden_fixtures +serde-attribute:src/types/v4.rs::RepoState::signing_key::serde(borrow) rust:src/types/v4.rs::v4_persisted_schema_golden_fixtures +serde-attribute:src/types/v7.rs::RepoState::handle::serde(borrow) rust:src/types/v7.rs::v7_persisted_schema_golden_fixture +serde-attribute:src/types/v7.rs::RepoState::pds::serde(borrow) rust:src/types/v7.rs::v7_persisted_schema_golden_fixture +serde-attribute:src/types/v7.rs::RepoState::serde(bound(deserialize="'i:'de")) rust:src/types/v7.rs::v7_persisted_schema_golden_fixture +serde-attribute:src/types/v7.rs::RepoState::signing_key::serde(borrow) rust:src/types/v7.rs::v7_persisted_schema_golden_fixture +serde-helper:src/crawler/relay_first.rs::serialize_cursor rust:src/crawler/relay_first.rs::pass_cursor_round_trips +serde-helper:src/db/mod.rs::deser_repo_meta rust:src/types.rs::current_persisted_state_golden_fixtures +serde-helper:src/db/mod.rs::deser_repo_state rust:src/types.rs::current_persisted_state_golden_fixtures +serde-helper:src/db/mod.rs::ser_repo_meta rust:src/types.rs::current_persisted_state_golden_fixtures +serde-helper:src/db/mod.rs::ser_repo_state rust:src/types.rs::current_persisted_state_golden_fixtures +serde-helper:src/ingest/stream/types.rs::deserialize_tid_or_empty rust:src/ingest/stream/types.rs::firehose_scalar_wire_contracts +serde-helper:src/types/event/stream.rs::event_type_ser_str rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-helper:src/util/mod.rs::deser_status_code rust:src/util/mod.rs::http_status_serde_contract +serde-helper:src/util/mod.rs::did_key_serialize_str rust:src/util/mod.rs::xrpc_output_string_serializers_are_stable +serde-helper:src/util/mod.rs::opt_cid_serialize_str rust:src/util/mod.rs::xrpc_output_string_serializers_are_stable +serde-helper:src/util/mod.rs::opt_did_key_serialize_str rust:src/util/mod.rs::xrpc_output_string_serializers_are_stable +serde-helper:src/util/mod.rs::repo_status_serialize_str rust:src/util/mod.rs::xrpc_output_string_serializers_are_stable +serde-helper:src/util/mod.rs::ser_status_code rust:src/util/mod.rs::http_status_serde_contract +serde-type:src/api/crawler.rs::AddSourceRequest nu:tests/api_crawler_sources.nu +serde-type:src/api/crawler.rs::RemoveSourceRequest nu:tests/api_crawler_sources.nu +serde-type:src/api/crawler.rs::ResetCursorBody nu:tests/api_crawler_sources.nu +serde-type:src/api/debug.rs::DebugCompactRequest nu:tests/debug_endpoints.nu +serde-type:src/api/debug.rs::DebugCountRequest nu:tests/debug_endpoints.nu +serde-type:src/api/debug.rs::DebugCountResponse nu:tests/debug_endpoints.nu +serde-type:src/api/debug.rs::DebugGetRequest nu:tests/debug_endpoints.nu +serde-type:src/api/debug.rs::DebugGetResponse nu:tests/debug_endpoints.nu +serde-type:src/api/debug.rs::DebugIterRequest nu:tests/debug_endpoints.nu +serde-type:src/api/debug.rs::DebugIterResponse nu:tests/debug_endpoints.nu +serde-type:src/api/debug.rs::DebugSeedBacklinksRequest nu:tests/debug_endpoints.nu +serde-type:src/api/debug.rs::DebugSeedBacklinksResponse nu:tests/debug_endpoints.nu +serde-type:src/api/debug.rs::DebugSeedEventsRequest nu:tests/debug_endpoints.nu +serde-type:src/api/debug.rs::DebugSeedRelayJetstreamCommitsRequest nu:tests/debug_endpoints.nu +serde-type:src/api/debug.rs::DebugSeedWatermarkRequest nu:tests/debug_endpoints.nu +serde-type:src/api/debug.rs::DebugSeedXrpcRepoResponse nu:tests/api_xrpc.nu +serde-type:src/api/filter.rs::PatchFilterBody nu:tests/signal_filter.nu +serde-type:src/api/firehose.rs::AddSourceRequest nu:tests/api_firehose_sources.nu +serde-type:src/api/firehose.rs::GetSourceQuery nu:tests/api_firehose_sources.nu +serde-type:src/api/firehose.rs::ListSourcesQuery nu:tests/api_firehose_sources.nu +serde-type:src/api/firehose.rs::RemoveSourceRequest nu:tests/api_firehose_sources.nu +serde-type:src/api/firehose.rs::ResetCursorBody nu:tests/api_firehose_sources.nu +serde-type:src/api/ingestion.rs::IngestionPatch rust:src/api/ingestion.rs::ingestion_json_contracts_follow_compiled_mode +serde-type:src/api/ingestion.rs::IngestionStatus rust:src/api/ingestion.rs::ingestion_json_contracts_follow_compiled_mode +serde-type:src/api/jetstream.rs::JetstreamQuery rust:src/api/jetstream.rs::jetstream_query_accepts_repeated_values +serde-type:src/api/jetstream.rs::SubscriberOptionsUpdatePayload rust:src/api/jetstream.rs::jetstream_options_message_json_contract +serde-type:src/api/jetstream.rs::SubscriberSourcedMessage rust:src/api/jetstream.rs::jetstream_options_message_json_contract +serde-type:src/api/pds.rs::BanBody nu:tests/api_pds_banned.nu +serde-type:src/api/pds.rs::RemoveTierQuery nu:tests/api_pds_tiers.nu +serde-type:src/api/pds.rs::SetTierBody nu:tests/api_pds_tiers.nu +serde-type:src/api/pds.rs::TiersResponse nu:tests/api_pds_tiers.nu +serde-type:src/api/redactions.rs::RedactionQueryParams rust:src/api/redactions.rs::test_parse_redaction_target_invalid +serde-type:src/api/repos.rs::GetReposParams nu:tests/api_repos.nu +serde-type:src/api/repos.rs::RepoRequest nu:tests/api_repos.nu +serde-type:src/api/stream.rs::StreamQuery nu:tests/stream_cursor_replay.nu +serde-type:src/api/xrpc/count_records.rs::CountRecordsOutput nu:tests/api_xrpc.nu +serde-type:src/api/xrpc/count_records.rs::CountRecordsRequestData nu:tests/api_xrpc.nu +serde-type:src/api/xrpc/describe_repo.rs::DescribeRepoOutput nu:tests/api_xrpc.nu +serde-type:src/api/xrpc/describe_repo.rs::DescribeRepoRequestData nu:tests/api_xrpc.nu +serde-type:src/api/xrpc/get_record_by_uri.rs::BadExampleGetUriRecordRequestData nu:tests/api_xrpc.nu +serde-type:src/api/xrpc/get_record_by_uri.rs::GetRecordByUriRequestData nu:tests/api_xrpc.nu +serde-type:src/api/xrpc/resolve_mini_doc.rs::BlueMicrocosmResolveMiniDocRequestData nu:tests/api_xrpc.nu +serde-type:src/api/xrpc/resolve_mini_doc.rs::ResolveMiniDocOutput nu:tests/api_xrpc.nu +serde-type:src/api/xrpc/resolve_mini_doc.rs::ResolveMiniDocRequestData nu:tests/api_xrpc.nu +serde-type:src/api/xrpc/subscribe_repos.rs::SubscribeReposQuery nu:tests/relay_subscribe_repos_ping.nu +serde-type:src/backlinks/api.rs::Backlink rust:src/backlinks/api.rs::backlinks_query_accepts_single_and_repeated_dids +serde-type:src/backlinks/api.rs::GetBacklinksCountOutput rust:src/backlinks/api.rs::backlinks_query_accepts_single_and_repeated_dids +serde-type:src/backlinks/api.rs::GetBacklinksCountParams rust:src/backlinks/api.rs::backlinks_query_accepts_single_and_repeated_dids +serde-type:src/backlinks/api.rs::GetBacklinksOutput rust:src/backlinks/api.rs::backlinks_query_accepts_single_and_repeated_dids +serde-type:src/backlinks/api.rs::GetBacklinksParams rust:src/backlinks/api.rs::backlinks_query_accepts_single_and_repeated_dids +serde-type:src/bin/backfill_strategy_bench.rs::DidDoc rust:src/bin/backfill_strategy_bench.rs::benchmark_api_json_fixtures_decode +serde-type:src/bin/backfill_strategy_bench.rs::DidService rust:src/bin/backfill_strategy_bench.rs::benchmark_api_json_fixtures_decode +serde-type:src/bin/backfill_strategy_bench.rs::ListReposByCollectionOutput rust:src/bin/backfill_strategy_bench.rs::benchmark_api_json_fixtures_decode +serde-type:src/bin/backfill_strategy_bench.rs::RepoHit rust:src/bin/backfill_strategy_bench.rs::benchmark_api_json_fixtures_decode +serde-type:src/control/crawler.rs::CrawlerSourceInfo rust:src/control/crawler.rs::reset_cursor_clears_relay_first_progress_for_only_that_source +serde-type:src/control/filter.rs::FilterSnapshot nu:tests/signal_filter.nu +serde-type:src/control/firehose.rs::FirehoseDiagnosticsInfo nu:tests/api_firehose_sources.nu +serde-type:src/control/firehose.rs::FirehoseFailureInfo nu:tests/api_firehose_sources.nu +serde-type:src/control/firehose.rs::FirehosePdsInfo nu:tests/api_firehose_sources.nu +serde-type:src/control/firehose.rs::FirehoseSourceInfo nu:tests/api_firehose_sources.nu +serde-type:src/control/pds.rs::PdsTierAssignment nu:tests/pds_host_status_transitions.nu +serde-type:src/control/pds.rs::PdsTierDefinition nu:tests/pds_host_status_transitions.nu +serde-type:src/control/repos/mod.rs::RepoInfo rust:src/control/repos/mod.rs::repo_info_json_contract +serde-type:src/control/stats.rs::StatsResponse nu:tests/debug_endpoints.nu +serde-type:src/control/stream/jetstream.rs::JetstreamAccount nu:tests/stream_cursor_replay.nu +serde-type:src/control/stream/jetstream.rs::JetstreamCommit nu:tests/stream_cursor_replay.nu +serde-type:src/control/stream/jetstream.rs::JetstreamEvent nu:tests/stream_cursor_replay.nu +serde-type:src/control/stream/jetstream.rs::JetstreamIdentity nu:tests/stream_cursor_replay.nu +serde-type:src/control/stream/jetstream.rs::JetstreamPayload nu:tests/stream_cursor_replay.nu +serde-type:src/crawler/list_repos/checker.rs::RetryState rust:src/crawler/list_repos/checker.rs::retry_state_persisted_golden_fixture +serde-type:src/crawler/relay_first.rs::PassCursor rust:src/crawler/relay_first.rs::pass_cursor_round_trips +serde-type:src/db/migration/v2.rs::OldRepoState rust:src/db/migration/v2.rs::v1_repo_state_input_golden_fixture +serde-type:src/db/migration/v4.rs::OldRepoState rust:src/db/migration/v4.rs::v2_repo_state_input_golden_fixture +serde-type:src/db/mod.rs::FirehoseSourceMeta rust:src/db/mod.rs::firehose_source_meta_golden_fixture +serde-type:src/db/storage_mode.rs::StorageModeMarker rust:src/db/storage_mode.rs::storage_mode_marker_persisted_golden_fixture +serde-type:src/db/types.rs::DbAction rust:src/db/types.rs::db_action_and_did_key_representation_contracts +serde-type:src/db/types.rs::DbRkey rust:src/db/types.rs::dbrkey_string_markers_preserve_every_utf8_length +serde-type:src/db/types.rs::DbTid rust:src/db/types.rs::dbrkey_msgpack_binary_marker_preserves_tid +serde-type:src/db/types.rs::DidKey rust:src/db/types.rs::db_action_and_did_key_representation_contracts +serde-type:src/filter.rs::FilterConfig rust:src/filter.rs::filter_mode_persisted_golden_fixtures +serde-type:src/filter.rs::FilterMode rust:src/filter.rs::filter_mode_persisted_golden_fixtures +serde-type:src/ingest/firehose_stats/relay.rs::RelayShardStatsSnapshot rust:src/ingest/firehose_stats/relay.rs::relay_worker_stats_snapshot_json_contract +serde-type:src/ingest/firehose_stats/relay.rs::RelayWorkerStatsSnapshot rust:src/ingest/firehose_stats/relay.rs::relay_worker_stats_snapshot_json_contract +serde-type:src/ingest/firehose_stats/source.rs::FirehoseMessageStats rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract +serde-type:src/ingest/firehose_stats/source.rs::FirehoseStatsSnapshot rust:src/ingest/firehose_stats/source.rs::firehose_stats_snapshot_json_contract +serde-type:src/ingest/stream/codec.rs::EncodeErrorFrame rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +serde-type:src/ingest/stream/codec.rs::EncodeErrorHeader rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +serde-type:src/ingest/stream/codec.rs::EncodeHeader rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +serde-type:src/ingest/stream/codec.rs::ErrorFrame rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +serde-type:src/ingest/stream/codec.rs::EventHeader rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +serde-type:src/ingest/stream/types.rs::Account rust:src/ingest/stream.rs::test_decode_account +serde-type:src/ingest/stream/types.rs::Commit rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-type:src/ingest/stream/types.rs::Datetime rust:src/ingest/stream/types.rs::firehose_datetime_accepts_zoned_and_legacy_unzoned_values +serde-type:src/ingest/stream/types.rs::Identity rust:src/ingest/stream/codec.rs::firehose_frame_fixture_decodes_identity +serde-type:src/ingest/stream/types.rs::Info rust:src/ingest/stream/codec.rs::firehose_frame_maps_outdated_cursor_info +serde-type:src/ingest/stream/types.rs::RepoOp rust:src/ingest/stream.rs::test_decode_commit_empty_since +serde-type:src/ingest/stream/types.rs::Sync rust:src/ingest/stream/codec.rs::firehose_frame_mock_decodes_sync_binary_blocks +serde-type:src/patch.rs::SetUpdate rust:src/patch.rs::set_update_json_markers_are_structurally_disjoint +serde-type:src/pds_meta.rs::HostStatus rust:src/pds_meta.rs::host_status_persisted_golden_fixtures +serde-type:src/types.rs::DeleteBodiesReport rust:src/types.rs::operator_redaction_json_contracts +serde-type:src/types.rs::DeleteBodyTarget rust:src/types.rs::operator_redaction_json_contracts +serde-type:src/types.rs::ResyncErrorKind rust:src/types.rs::current_persisted_state_golden_fixtures +serde-type:src/types.rs::ResyncState rust:src/types.rs::current_persisted_state_golden_fixtures +serde-type:src/types/event/jetstream.rs::StoredJetstreamEvent rust:src/types/event/jetstream.rs::common_jetstream_event_golden_fixtures +serde-type:src/types/event/stream.rs::AccountEvt rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-type:src/types/event/stream.rs::EventType rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-type:src/types/event/stream.rs::IdentityEvt rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-type:src/types/event/stream.rs::MarshallableEvt rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-type:src/types/event/stream.rs::RecordEvt rust:src/types/event/stream.rs::stream_event_json_envelopes_cover_every_variant +serde-type:src/types/event/stream.rs::StoredData rust:src/types/event/stream.rs::stored_data_persisted_golden_fixtures +serde-type:src/types/event/stream.rs::StoredEvent rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim +serde-type:src/types/v2.rs::Commit rust:src/types/v2.rs::v2_persisted_schema_golden_fixture +serde-type:src/types/v2.rs::RepoState rust:src/types/v2.rs::v2_persisted_schema_golden_fixture +serde-type:src/types/v2.rs::RepoStatus rust:src/types/v2.rs::v2_persisted_schema_golden_fixture +serde-type:src/types/v4.rs::RepoMetadata rust:src/types/v4.rs::v4_persisted_schema_golden_fixtures +serde-type:src/types/v4.rs::RepoState rust:src/types/v4.rs::v4_persisted_schema_golden_fixtures +serde-type:src/types/v4.rs::RepoStatus rust:src/types/v4.rs::v4_persisted_schema_golden_fixtures +serde-type:src/types/v7.rs::RepoState rust:src/types/v7.rs::v7_persisted_schema_golden_fixture +stream-projection:src/control/stream/engine.rs::clear_replay_blocked rust:src/control/stream/engine.rs::ordered_stream_closes_when_output_queue_is_full +stream-projection:src/control/stream/engine.rs::next_expected_seq rust:src/control/stream/engine.rs::ordered_stream_closes_when_output_queue_is_full +stream-projection:src/control/stream/engine.rs::note_replay_blocked rust:src/control/stream/engine.rs::ordered_stream_closes_when_output_queue_is_full +stream-projection:src/control/stream/engine.rs::replay_chunk_size_for rust:src/control/stream/engine.rs::ordered_stream_closes_when_output_queue_is_full +stream-projection:src/control/stream/engine.rs::run_ordered_stream rust:src/control/stream/engine.rs::ordered_stream_closes_when_output_queue_is_full +stream-projection:src/control/stream/engine.rs::send_stream_error rust:src/control/stream/engine.rs::ordered_stream_closes_when_output_queue_is_full +stream-projection:src/control/stream/engine.rs::stream_seq_after rust:src/control/stream/engine.rs::ordered_stream_closes_when_output_queue_is_full +stream-projection:src/control/stream/indexer.rs::event_stream_thread rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim +stream-projection:src/control/stream/indexer.rs::resolve_event_body rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim +stream-projection:src/control/stream/indexer.rs::stored_to_event rust:src/control/stream/indexer.rs::persisted_eight_byte_string_rkey_replays_verbatim +stream-projection:src/control/stream/jetstream.rs::jetstream_stream_thread nu:tests/stream_cursor_replay.nu +stream-projection:src/control/stream/relay.rs::relay_stream_thread nu:tests/stream_cursor_replay.nu +stream-projection:src/control/stream/types.rs::PendingLiveEvents::new nu:tests/stream_cursor_replay.nu +stream-projection:src/control/stream/types.rs::PendingLiveEvents::next_sequence nu:tests/stream_cursor_replay.nu +stream-projection:src/control/stream/types.rs::PendingLiveEvents::pop_front nu:tests/stream_cursor_replay.nu +stream-projection:src/control/stream/types.rs::PendingLiveEvents::push nu:tests/stream_cursor_replay.nu +stream-projection:src/control/stream/types.rs::PendingLiveEvents::push_front nu:tests/stream_cursor_replay.nu +stream-projection:src/control/stream/types.rs::PendingLiveEvents::take_persisted_after nu:tests/stream_cursor_replay.nu +stream-projection:src/control/stream/types.rs::StreamOptions::from_config nu:tests/stream_cursor_replay.nu +stream-projection:src/control/stream/types.rs::StreamTooSlow::lagged nu:tests/stream_cursor_replay.nu +stream-projection:src/control/stream/types.rs::StreamTooSlow::pending_limit nu:tests/stream_cursor_replay.nu +stream-projection:src/control/stream/types.rs::StreamTooSlow::send_timeout nu:tests/stream_cursor_replay.nu +text-parser:src/config/types.rs::BackfillStrategy::FromStr rust:src/config/types.rs::text_config_parser_contracts +text-parser:src/config/types.rs::Compression::FromStr rust:src/config/types.rs::text_config_parser_contracts +text-parser:src/config/types.rs::CrawlerMode::FromStr rust:src/config/types.rs::crawler_mode_string_and_msgpack_contracts +text-parser:src/config/types.rs::SignatureVerification::FromStr rust:src/config/types.rs::text_config_parser_contracts diff --git a/tests/stream_live_backfill.nu b/tests/stream_live_backfill.nu --- a/tests/stream_live_backfill.nu +++ b/tests/stream_live_backfill.nu @@ -2,40 +2,64 @@ use common.nu * def main [] { - let did = "did:web:guestbook.gaze.systems" + let tracked_did = "did:plc:qfpnj4og54vl56wngdriaxug" let port = resolve-test-port 3002 + let debug_port = resolve-test-debug-port ($port + 1) let url = $"http://localhost:($port)" + let debug_url = $"http://localhost:($debug_port)" let ws_url = $"ws://127.0.0.1:($port)/stream" let db_path = (mktemp -d -t hydrant_stream_live_backfill.XXXXXX) - print $"testing live /stream events during backfill for ($did)..." + print "testing live /stream events while backfill work is pending..." print $"database path: ($db_path)" let binary = build-hydrant - let instance = start-hydrant $binary $db_path $port + let instance = (with-env { + HYDRANT_RELAY_HOSTS: "", + HYDRANT_CRAWLER_URLS: "", + HYDRANT_ENABLE_FIREHOSE: "false", + HYDRANT_ENABLE_CRAWLER: "false", + HYDRANT_ENABLE_BACKFILL: "false", + HYDRANT_PLC_URL: "http://127.0.0.1:1" + } { + start-hydrant $binary $db_path $port + }) if not (wait-for-api $url) { fail "api failed to start" $instance.pid } let live_output = $"($db_path)/stream_live.txt" - print $"starting stream listener -> ($live_output)" - let stream_pid = (bash -c $"websocat --no-async-stdio -n '($ws_url)' > '($live_output)' 2>&1 & echo $!" | str trim | into int) - print $"stream listener pid: ($stream_pid)" - sleep 2sec + let stream_pid = ( + bash -c $"websocat --no-async-stdio -n '($ws_url)' > '($live_output)' 2>&1 & echo $!" + | str trim + | into int + ) + sleep 500ms - print $"adding repo ($did) to tracking..." - http put -t application/json $"($url)/repos" [{ did: $did }] - - if not (wait-for-backfill $url) { - try { kill $stream_pid } - fail "backfill failed or timed out" $instance.pid + http put -t application/json $"($url)/repos" [{ did: $tracked_did }] + let pending = (http get $"($url)/stats").counts.pending | into int + if $pending != 1 { + fail $"expected one queued backfill, got ($pending)" $stream_pid $instance.pid } - for i in 1..50 { + let seed = ( + http post -H [Content-Length 0] $"($debug_url)/debug/seed_xrpc_repo" "" + ) + + mut live_events = [] + for _ in 1..50 { if ($live_output | path exists) { - let live_content = (open $live_output | str trim) - if not ($live_content | is-empty) { + $live_events = ( + open --raw $live_output + | into string + | str trim + | lines + | each { |line| try { $line | from json } catch { null } } + | compact + | where { |event| ($event | describe | str starts-with "record") } + ) + if ($live_events | is-not-empty) { break } } @@ -43,21 +67,20 @@ } try { kill $stream_pid } - sleep 1sec + sleep 100ms - if not ($live_output | path exists) { - fail "stream output file was not created" $instance.pid + if ($live_events | is-empty) { + fail "no live record event received while backfill was pending" $instance.pid } - let live_content = (open $live_output | str trim) - if ($live_content | is-empty) { - fail "no live events received during backfill" $instance.pid + let event = $live_events | first + if $event.id != $seed.event_id { + fail $"expected event ($seed.event_id), got ($event.id)" $instance.pid } - - let live_messages = ($live_content | lines) - let first = ($live_messages | first | from json) - print $"received ($live_messages | length) live events; first event id=($first.id), type=($first.type)" + if $event.type != "record" or $event.record.rkey != "stinkpot" { + fail $"unexpected live event: ($event | to json --raw)" $instance.pid + } try { kill $instance.pid } - print "live /stream during backfill passed!" + print "live /stream event during pending backfill passed!" } diff --git a/tests/stream_relay_jetstream_subscribe.nu b/tests/stream_relay_jetstream_subscribe.nu --- a/tests/stream_relay_jetstream_subscribe.nu +++ b/tests/stream_relay_jetstream_subscribe.nu @@ -1,22 +1,11 @@ #!/usr/bin/env nu -# Tests the Jetstream-compatible /subscribe WebSocket endpoint. -# -# Starts hydrant with the jetstream feature, connects to wss://bsky.network -# as the upstream firehose with full-network mode, and verifies that events -# arrive and are correctly formatted. Crawler and backfill are disabled so -# only the single firehose connection is used (no PDS connections). -# -# Does not require credentials — relies on the public bsky.network firehose. +# tests the Jetstream-compatible /subscribe WebSocket endpoint +# local debug fixtures exercise the real live, replay, and filter paths +# without credentials or an external network service use common.nu * -def collect-ws-json [ws_url: string, output_file: string, duration: duration] { - let pid_file = $"($output_file).pid" - bash -c $"websocat --no-async-stdio -n '($ws_url)' > '($output_file)' 2>&1 & echo $! > '($pid_file)'" - sleep $duration - let pid = (open $pid_file | str trim | into int) - try { kill $pid } - sleep 200ms +def read-ws-json [output_file: string] { if ($output_file | path exists) { open --raw $output_file | into string @@ -28,6 +17,42 @@ } else { [] } +} + +def collect-ws-json [ws_url: string, output_file: string, duration: duration] { + let pid_file = $"($output_file).pid" + bash -c $"websocat --no-async-stdio -n '($ws_url)' > '($output_file)' 2>&1 & echo $! > '($pid_file)'" + sleep $duration + let pid = (open $pid_file | str trim | into int) + try { kill $pid } + sleep 200ms + read-ws-json $output_file +} + +def collect-seeded-ws-json [ + ws_url: string, + debug_url: string, + output_file: string, + count: int, + hydrant_pid: int, +] { + let pid_file = $"($output_file).pid" + bash -c $"websocat --no-async-stdio -n '($ws_url)' > '($output_file)' 2>&1 & echo $! > '($pid_file)'" + sleep 1sec + let response = ( + http post -f -e -H [Content-Length 0] + $"($debug_url)/debug/seed_relay_jetstream_commits?count=($count)" "" + ) + if $response.status != 200 { + let pid = (open $pid_file | str trim | into int) + try { kill $pid } + fail $"relay Jetstream fixture seed returned ($response.status)" $hydrant_pid + } + sleep 1sec + let pid = (open $pid_file | str trim | into int) + try { kill $pid } + sleep 200ms + read-ws-json $output_file } def assert-no-error-events [events: list, label: string, ...pids: int] { @@ -61,6 +86,7 @@ let port = resolve-test-port 3019 let url = $"http://localhost:($port)" let ws_base = $"ws://127.0.0.1:($port)" + let debug_url = $"http://127.0.0.1:(resolve-test-debug-port ($port + 1))" let db_path = (mktemp -d -t hydrant_stream_relay_jetstream_subscribe.XXXXXX) print "testing Jetstream /subscribe..." @@ -68,7 +94,9 @@ let binary = build-hydrant-relay-jetstream let instance = (with-env { - HYDRANT_RELAY_HOSTS: "wss://bsky.network", + HYDRANT_RELAY_HOSTS: "", + HYDRANT_CRAWLER_URLS: "", + HYDRANT_ENABLE_FIREHOSE: "false", HYDRANT_FULL_NETWORK: "true", HYDRANT_SEED_HOSTS: "" } { @@ -84,9 +112,10 @@ # capture cursor_us before the live window so replay is guaranteed to cover it let cursor_us = ((date now | format date "%s" | into int) * 1_000_000) let live_file = $"($db_path)/live.txt" - # bsky.network has high throughput; 15 seconds gives a large event sample - # and a reasonable chance of capturing identity/account events alongside commits - let events = (collect-ws-json $"($ws_base)/subscribe" $live_file 15sec) + # subscribe first, then stage enough deterministic events to exercise live delivery + let events = ( + collect-seeded-ws-json $"($ws_base)/subscribe" $debug_url $live_file 24 $instance.pid + ) print $"received ($events | length) events" assert-no-error-events $events "live subscription" $instance.pid @@ -108,7 +137,7 @@ } print "commit event structure is correct" - # validate identity events if any arrived (lazy RelayIdentity inflation path) + # validate the deterministic identity fixture (lazy RelayIdentity inflation path) let identity_events = ($events | where { |e| ($e | get -o kind | default "") == "identity" }) print $"identity events: ($identity_events | length)" for evt in $identity_events { @@ -121,11 +150,15 @@ } } } - if ($identity_events | length) > 0 { - print "identity event structure is correct \(lazy RelayIdentity path\)" + if ($identity_events | length) != 1 { + fail $"expected one identity fixture event, got ($identity_events | length)" $instance.pid } + if ($identity_events | first | get -o identity.handle | default "") != "alice.test" { + fail "identity fixture handle was not projected" $instance.pid + } + print "identity event structure is correct \(lazy RelayIdentity path\)" - # validate account events if any arrived (lazy RelayAccount inflation path) + # validate the deterministic account fixture (lazy RelayAccount inflation path) let account_events = ($events | where { |e| ($e | get -o kind | default "") == "account" }) print $"account events: ($account_events | length)" for evt in $account_events { @@ -138,9 +171,14 @@ } } } - if ($account_events | length) > 0 { - print "account event structure is correct \(lazy RelayAccount path\)" + if ($account_events | length) != 1 { + fail $"expected one account fixture event, got ($account_events | length)" $instance.pid } + let account = ($account_events | first | get account) + if $account.active != false or ($account | get -o status | default "") != "deactivated" { + fail "account fixture state was not projected" $instance.pid + } + print "account event structure is correct \(lazy RelayAccount path\)" # --- scenario: cursor replay --- print "--- scenario: cursor replay ---" @@ -148,7 +186,7 @@ print $"replaying from cursor_us: ($cursor_us)" let replay_file = $"($db_path)/replay.txt" - let replay_events = (collect-ws-json $"($ws_base)/subscribe?cursor=($cursor_us)" $replay_file 8sec) + let replay_events = (collect-ws-json $"($ws_base)/subscribe?cursor=($cursor_us)" $replay_file 2sec) print $"cursor replay: ($replay_events | length) events" assert-no-error-events $replay_events "cursor replay" $instance.pid @@ -157,6 +195,12 @@ fail $"cursor replay: expected >= 1 commit event, got ($replay_commits | length)" $instance.pid } print $"cursor replay: ($replay_commits | length) commit events" + if (($replay_events | where { |e| ($e | get -o kind | default "") == "identity" }) | length) != 1 { + fail "cursor replay did not inflate the identity fixture" $instance.pid + } + if (($replay_events | where { |e| ($e | get -o kind | default "") == "account" }) | length) != 1 { + fail "cursor replay did not inflate the account fixture" $instance.pid + } # verify monotonic time_us ordering let replay_timed = ($replay_events | where { |e| ($e | get -o time_us | default 0) > 0 }) @@ -178,7 +222,7 @@ let collection = "app.bsky.feed.post" let col_file = $"($db_path)/col_filter.txt" let col_url = $"($ws_base)/subscribe?cursor=($cursor_us)&wantedCollections=($collection)" - let col_events = (collect-ws-json $col_url $col_file 8sec) + let col_events = (collect-ws-json $col_url $col_file 2sec) let col_commits = ($col_events | where { |e| ($e | get -o kind | default "") == "commit" }) print $"wantedCollections=($collection): ($col_commits | length) commit events" if ($col_commits | length) < 1 { @@ -201,7 +245,7 @@ print $"filtering to DID: ($sample_did)" let did_file = $"($db_path)/did_filter.txt" let did_url = $"($ws_base)/subscribe?cursor=($cursor_us)&wantedDids=($sample_did)" - let did_events = (collect-ws-json $did_url $did_file 8sec) + let did_events = (collect-ws-json $did_url $did_file 2sec) print $"wantedDids=($sample_did): ($did_events | length) events" let did_jetstream_events = ($did_events | where { |e| ($e | get -o did | is-not-empty) }) if ($did_jetstream_events | any { |e| ($e | get -o did | default "") != $sample_did }) { diff --git a/src/api/debug.rs b/src/api/debug.rs --- a/src/api/debug.rs +++ b/src/api/debug.rs @@ -2,17 +2,33 @@ #[cfg(feature = "indexer")] use crate::db::keys; use crate::db::registry; +#[cfg(feature = "indexer_stream")] +use crate::db::types::{DbAction, DbRkey, DbTid, DidKey, TrimmedDid}; +#[cfg(feature = "indexer_stream")] +use crate::types::{ + BroadcastEvent, Commit, RepoMetadata, RepoState, RepoStatus, StoredData, StoredEvent, +}; use axum::routing::{get, post}; use axum::{ Json, extract::{Query, State}, http::StatusCode, }; +#[cfg(feature = "indexer_stream")] +use bytes::Bytes; +#[cfg(feature = "indexer_stream")] +use jacquard_common::CowStr; #[cfg(feature = "indexer")] use jacquard_common::types::ident::AtIdentifier; +#[cfg(feature = "indexer_stream")] +use jacquard_common::types::{did::Did, tid::Tid}; +#[cfg(feature = "indexer_stream")] +use jacquard_repo::{MemoryBlockStore, Mst}; use miette::IntoDiagnostic; use serde::{Deserialize, Serialize}; use serde_json::Value; +#[cfg(feature = "indexer_stream")] +use std::borrow::Cow; use std::sync::Arc; #[cfg(feature = "indexer")] @@ -26,6 +42,19 @@ #[derive(Serialize)] pub struct DebugCountResponse { pub count: usize, +} + +#[cfg(feature = "indexer_stream")] +#[derive(Serialize)] +pub struct DebugSeedXrpcRepoResponse { + pub did: String, + pub collection: &'static str, + pub rkey: &'static str, + pub cid: String, + pub commit_cid: String, + pub rev: String, + pub event_id: u64, + pub hostname: &'static str, } pub fn router() -> axum::Router> { @@ -43,8 +72,20 @@ .route("/debug/seed_watermark", post(handle_debug_seed_watermark)) .route("/debug/seed_events", post(handle_debug_seed_events)); + #[cfg(all(feature = "relay", feature = "jetstream"))] + let r = r.route( + "/debug/seed_relay_jetstream_commits", + post(handle_debug_seed_relay_jetstream_commits), + ); + + #[cfg(all(feature = "indexer_stream", feature = "backlinks"))] + let r = r.route("/debug/seed_backlinks", post(handle_debug_seed_backlinks)); + #[cfg(feature = "indexer")] let r = r.route("/debug/count", get(handle_debug_count)); + + #[cfg(feature = "indexer_stream")] + let r = r.route("/debug/seed_xrpc_repo", post(handle_debug_seed_xrpc_repo)); r } @@ -65,22 +106,148 @@ let count = state .db .run(move |db| { - let ks = db - .keyspace_by_name("records") - .expect("records keyspace exists in indexer mode"); - let start_key = prefix.clone(); // rkeys cannot contain 0xff, so every key in the collection sorts // below `prefix ff` let mut end_key = prefix.clone(); end_key.push(0xff); - Ok::<_, miette::Report>(ks.range(start_key..end_key).count()) + Ok::<_, miette::Report>(db.indexer.record_range(start_key..end_key).count()) }) .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; Ok(Json(DebugCountResponse { count })) +} + +#[cfg(feature = "indexer_stream")] +pub async fn handle_debug_seed_xrpc_repo( + State(state): State>, +) -> Result, StatusCode> { + const DID: &str = "did:plc:ewvi7nxzyoun6zhxrhs64oiz"; + const COLLECTION: &str = "app.bsky.feed.post"; + const RKEY: &str = "stinkpot"; + const REV: &str = "3jzfcijpj2z2a"; + const HOSTNAME: &str = "pds.example"; + + let did = Did::new(DID).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let rev = Tid::new(REV).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let body = Bytes::from( + serde_ipld_dagcbor::to_vec(&serde_json::json!({ + "$type": COLLECTION, + "text": "eight-byte rkey fixture", + })) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, + ); + let cid = jacquard_repo::mst::util::compute_cid(&body) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + + let store = Arc::new(MemoryBlockStore::new()); + let mut mst = Mst::new(store); + mst.add_mut(&format!("{COLLECTION}/{RKEY}"), cid) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + mst.persist() + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let root = mst + .get_pointer() + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + + let mut repo_state = RepoState::backfilling(); + repo_state.status = RepoStatus::Synced; + repo_state.root = Some(Commit { + data: root, + prev: None, + rev: DbTid::from(&rev), + sig: Bytes::from_static(b"fixture-signature"), + version: 3, + }); + repo_state.pds = Some(CowStr::Owned(format!("https://{HOSTNAME}").into())); + repo_state.signing_key = Some(DidKey(Cow::Owned( + [0xed, 0x01] + .into_iter() + .chain(std::iter::repeat_n(7, 32)) + .collect(), + ))); + + let commit_cid = repo_state + .root + .clone() + .and_then(|commit| commit.into_atp_commit(did.clone())) + .ok_or(StatusCode::INTERNAL_SERVER_ERROR)? + .to_cid() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let record_key = keys::record_key(&did, COLLECTION, &crate::db::types::DbRkey::new(RKEY)); + let repo_key = keys::repo_key(&did); + let metadata_key = keys::repo_metadata_key(&did); + let collection_count_key = keys::count_collection_key(&did, COLLECTION); + let host_count_key = keys::pds_account_count_key(HOSTNAME); + let host_cursor_key = keys::firehose_cursor_key(HOSTNAME); + let stored_event = StoredEvent { + live: true, + did: TrimmedDid::from(&did).into_static(), + rev: DbTid::from(&rev), + collection: CowStr::Borrowed(COLLECTION), + rkey: DbRkey::new(RKEY), + action: DbAction::Create, + data: StoredData::Ptr(cid), + }; + + let event_id = state + .db + .run(move |db| { + let mut batch = db.inner.batch(); + db.indexer + .stage_record(&mut batch, record_key, body.as_ref()); + batch.insert(&db.repos, repo_key, crate::db::ser_repo_state(&repo_state)?); + batch.insert( + &db.repo_metadata, + metadata_key, + crate::db::ser_repo_meta(&RepoMetadata::backfilling(1))?, + ); + batch.insert(&db.counts, collection_count_key, 1_u64.to_be_bytes()); + batch.insert(&db.counts, host_count_key, 1_u64.to_be_bytes()); + batch.insert(&db.cursors, host_cursor_key, 42_i64.to_be_bytes()); + let event_id = db + .stream + .next_event_id + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + db.stream.stage_event( + &mut batch, + keys::event_key(event_id), + rmp_serde::to_vec(&stored_event).into_diagnostic()?, + ); + batch.commit().into_diagnostic()?; + Ok::<_, miette::Report>(event_id) + }) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let _ = state + .db + .stream + .event_tx + .send(BroadcastEvent::Persisted(event_id)); + let host_count_key = keys::pds_account_count_key(HOSTNAME); + let current_host_count = state.db.get_count_sync(&host_count_key); + state + .db + .update_count(&host_count_key, 1_i64 - current_host_count as i64); + crate::pds_meta::PdsMeta::update_host(&state.pds_meta, HOSTNAME, |host| { + host.status = crate::pds_meta::HostStatus::Active; + }); + + Ok(Json(DebugSeedXrpcRepoResponse { + did: DID.to_string(), + collection: COLLECTION, + rkey: RKEY, + cid: cid.to_string(), + commit_cid: commit_cid.to_string(), + rev: REV.to_string(), + event_id, + hostname: HOSTNAME, + })) } #[derive(Deserialize)] @@ -356,4 +523,277 @@ .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; Ok(StatusCode::OK) +} + +#[cfg(all(feature = "relay", feature = "jetstream"))] +#[derive(Deserialize)] +pub struct DebugSeedRelayJetstreamCommitsRequest { + pub count: u64, +} + +#[cfg(all(feature = "relay", feature = "jetstream"))] +pub async fn handle_debug_seed_relay_jetstream_commits( + State(state): State>, + Query(req): Query, +) -> Result { + let count = usize::try_from(req.count) + .ok() + .filter(|count| *count <= 10_000) + .ok_or(StatusCode::BAD_REQUEST)?; + state + .db + .run(move |db| { + use crate::db::types::TrimmedDid; + use crate::ingest::stream::{ + Account, AccountStatus, Commit, Datetime, Identity, RepoOp, RepoOpAction, + SubscribeReposMessage, decode_frame, encode_frame, + }; + use crate::types::StoredJetstreamEvent; + use jacquard_common::CowStr; + use jacquard_common::types::cid::CidLink; + use jacquard_common::types::string::{Did, Handle, Tid}; + use std::sync::atomic::Ordering; + + let did = Did::new("did:plc:ewvi7nxzyoun6zhxrhs64oiz").into_diagnostic()?; + let commit_cid = CidLink::from( + jacquard_repo::mst::util::compute_cid(b"relay-jetstream-fixture") + .into_diagnostic()?, + ); + let rev: Tid = "3kzbif5moe22m".parse().into_diagnostic()?; + let time = Datetime::try_from("2026-08-10T12:34:56Z".to_owned()).into_diagnostic()?; + let trimmed_did = TrimmedDid::from(&did).into_static(); + let mut batch = db.inner.batch(); + let mut broadcasts = Vec::with_capacity(count + 2); + + for index in 0..count { + let relay_seq = db.relay.next_seq.fetch_add(1, Ordering::SeqCst); + let rkey = format!("fixture{index}"); + let commit = Commit { + blobs: Vec::new(), + blocks: bytes::Bytes::new(), + commit: commit_cid.clone(), + ops: vec![RepoOp { + action: RepoOpAction::Delete, + cid: None, + path: CowStr::Owned(format!("app.bsky.feed.post/{rkey}").into()), + prev: None, + }], + prev_data: None, + rebase: false, + repo: did.clone(), + rev: rev.clone(), + seq: relay_seq as i64, + since: None, + time: time.clone(), + too_big: false, + }; + let frame = encode_frame("#commit", &commit)?; + let SubscribeReposMessage::Commit(decoded) = + decode_frame(frame.as_ref()).map_err(|err| miette::miette!("{err}"))? + else { + return Err(miette::miette!("relay fixture did not decode as a commit")); + }; + if decoded.ops.len() != 1 { + return Err(miette::miette!("relay fixture lost its operation")); + } + batch.insert( + &db.relay.events, + crate::db::keys::relay_event_key(relay_seq), + frame.as_ref(), + ); + broadcasts.push(crate::jetstream::stage_event( + &mut batch, + db, + StoredJetstreamEvent::RelayCommit { + did: trimmed_did.clone(), + collection: CowStr::Borrowed("app.bsky.feed.post"), + relay_seq, + op_index: 0, + }, + Some(crate::jetstream::JetstreamEphemeral { + did: did.as_str().to_owned(), + rev: rev.as_str().to_owned(), + operation: "delete".to_owned(), + collection: "app.bsky.feed.post".to_owned(), + rkey, + record: None, + cid: None, + live: true, + }), + )?); + } + let relay_seq = db.relay.next_seq.fetch_add(1, Ordering::SeqCst); + let account = Account { + active: false, + did: did.clone(), + seq: relay_seq as i64, + status: Some(AccountStatus::Deactivated), + time: time.clone(), + }; + let frame = encode_frame("#account", &account)?; + let SubscribeReposMessage::Account(decoded) = + decode_frame(frame.as_ref()).map_err(|err| miette::miette!("{err}"))? + else { + return Err(miette::miette!( + "relay fixture did not decode as an account" + )); + }; + if decoded.active || decoded.status != Some(AccountStatus::Deactivated) { + return Err(miette::miette!("relay account fixture lost its state")); + } + batch.insert( + &db.relay.events, + crate::db::keys::relay_event_key(relay_seq), + frame.as_ref(), + ); + broadcasts.push(crate::jetstream::stage_event( + &mut batch, + db, + StoredJetstreamEvent::RelayAccount { + did: trimmed_did.clone(), + relay_seq, + }, + None, + )?); + + let relay_seq = db.relay.next_seq.fetch_add(1, Ordering::SeqCst); + let identity = Identity { + did: did.clone(), + handle: Some(Handle::new("alice.test").into_diagnostic()?), + seq: relay_seq as i64, + time: time.clone(), + }; + let frame = encode_frame("#identity", &identity)?; + let SubscribeReposMessage::Identity(decoded) = + decode_frame(frame.as_ref()).map_err(|err| miette::miette!("{err}"))? + else { + return Err(miette::miette!( + "relay fixture did not decode as an identity" + )); + }; + if decoded.handle.as_ref().map(Handle::as_str) != Some("alice.test") { + return Err(miette::miette!("relay identity fixture lost its handle")); + } + batch.insert( + &db.relay.events, + crate::db::keys::relay_event_key(relay_seq), + frame.as_ref(), + ); + broadcasts.push(crate::jetstream::stage_event( + &mut batch, + db, + StoredJetstreamEvent::RelayIdentity { + did: trimmed_did, + relay_seq, + }, + None, + )?); + + batch.commit().into_diagnostic()?; + for broadcast in broadcasts { + let _ = db.jetstream.tx.send(broadcast); + } + Ok::<_, miette::Report>(()) + }) + .await + .map_err(|err| { + tracing::error!(%err, "failed to seed relay Jetstream fixture"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + Ok(StatusCode::OK) +} + +#[cfg(all(feature = "indexer_stream", feature = "backlinks"))] +#[derive(Deserialize)] +pub struct DebugSeedBacklinksRequest { + pub like_count: u64, + pub post_count: u64, +} + +#[cfg(all(feature = "indexer_stream", feature = "backlinks"))] +#[derive(Serialize)] +pub struct DebugSeedBacklinksResponse { + pub did: String, + pub subject: String, + pub like_count: u64, + pub post_count: u64, +} + +#[cfg(all(feature = "indexer_stream", feature = "backlinks"))] +pub async fn handle_debug_seed_backlinks( + State(state): State>, + Query(req): Query, +) -> Result, StatusCode> { + req.like_count + .checked_add(req.post_count) + .filter(|count| *count <= 10_000) + .ok_or(StatusCode::BAD_REQUEST)?; + state + .db + .run(move |db| { + use crate::db::types::DbRkey; + use jacquard_common::types::string::Did; + + const DID: &str = "did:plc:wqstj3k5tslmm246baaf3tpa"; + const SUBJECT: &str = "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/root"; + const SUBJECT_CID: &str = "bafyreicldnax5mfobeba2pnv7fgsdfz2563gc72gyi5cnhyuqfnk3jz7aa"; + let did = Did::new(DID).into_diagnostic()?; + let mut batch = db.inner.batch(); + + for (collection, count) in [ + ("app.bsky.feed.like", req.like_count), + ("app.bsky.feed.post", req.post_count), + ] { + for index in 0..count { + let rkey = if collection == "app.bsky.feed.like" && index == 0 { + "stinkpot".to_owned() + } else { + let prefix = if collection == "app.bsky.feed.like" { + "like" + } else { + "post" + }; + format!("{prefix}{index:04}") + }; + let body = serde_ipld_dagcbor::to_vec(&serde_json::json!({ + "$type": collection, + "subject": { + "uri": SUBJECT, + "cid": SUBJECT_CID, + }, + "createdAt": "2026-08-10T12:34:56Z", + })) + .into_diagnostic()?; + let value = serde_ipld_dagcbor::from_slice::(&body) + .into_diagnostic()?; + let record_key = + crate::db::keys::record_key(&did, collection, &DbRkey::new(&rkey)); + db.indexer + .stage_record(&mut batch, record_key, body.as_slice()); + crate::backlinks::store::index_record( + &mut batch, + &db.backlinks, + did.as_str(), + collection, + &rkey, + &value, + )?; + } + } + + batch.commit().into_diagnostic()?; + Ok::<_, miette::Report>(DebugSeedBacklinksResponse { + did: DID.to_owned(), + subject: SUBJECT.to_owned(), + like_count: req.like_count, + post_count: req.post_count, + }) + }) + .await + .map(Json) + .map_err(|err| { + tracing::error!(%err, "failed to seed backlinks fixture"); + StatusCode::INTERNAL_SERVER_ERROR + }) } diff --git a/src/api/ingestion.rs b/src/api/ingestion.rs --- a/src/api/ingestion.rs +++ b/src/api/ingestion.rs @@ -73,3 +73,41 @@ } StatusCode::OK } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ingestion_json_contracts_follow_compiled_mode() { + let patch: IngestionPatch = serde_json::from_str(r#"{"firehose":false}"#).unwrap(); + assert_eq!(patch.firehose, Some(false)); + #[cfg(feature = "indexer")] + { + assert_eq!(patch.crawler, None); + assert_eq!(patch.backfill, None); + } + + let status = IngestionStatus { + #[cfg(feature = "indexer")] + crawler: true, + firehose: false, + #[cfg(feature = "indexer")] + backfill: true, + }; + let encoded = serde_json::to_value(status).unwrap(); + assert_eq!(encoded["firehose"], false); + #[cfg(feature = "indexer")] + { + assert_eq!(encoded["crawler"], true); + assert_eq!(encoded["backfill"], true); + } + #[cfg(not(feature = "indexer"))] + { + assert!(encoded.get("crawler").is_none()); + assert!(encoded.get("backfill").is_none()); + } + + assert!(serde_json::from_str::(r#"{"firehose":"false"}"#).is_err()); + } +} diff --git a/src/api/jetstream.rs b/src/api/jetstream.rs --- a/src/api/jetstream.rs +++ b/src/api/jetstream.rs @@ -1,68 +1,26 @@ use std::time::Duration; -use axum::extract::{Query, State}; +use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::{IntoResponse, Response}; use axum_tws::{Message, WebSocket, WebSocketUpgrade}; use futures::{SinkExt, StreamExt}; -use serde::{Deserialize, Deserializer}; +use serde::Deserialize; use tokio::time::{MissedTickBehavior, interval, timeout}; use tracing::{debug, warn}; +use crate::api::MultiQuery; use crate::control::{Hydrant, JetstreamFilter, JetstreamSubscriberOptions}; const PING_INTERVAL: Duration = Duration::from_secs(30); const CLOSE_TIMEOUT: Duration = Duration::from_secs(1); const MAX_SUBSCRIBER_MESSAGE_BYTES: usize = 10_000_000; -// serde_urlencoded (used by axum's query extractor) cannot deserialize a single -// query param value into Vec — it expects repeated keys for sequences. -// this visitor accepts both a bare string (single value) and a proper sequence. -fn deserialize_string_or_seq<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - struct StringOrSeq; - - impl<'de> serde::de::Visitor<'de> for StringOrSeq { - type Value = Vec; - - fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "a string or sequence of strings") - } - - fn visit_str(self, v: &str) -> Result, E> { - Ok(vec![v.to_owned()]) - } - - fn visit_seq>( - self, - mut seq: A, - ) -> Result, A::Error> { - let mut out = Vec::new(); - while let Some(v) = seq.next_element()? { - out.push(v); - } - Ok(out) - } - } - - deserializer.deserialize_any(StringOrSeq) -} - #[derive(Deserialize)] pub struct JetstreamQuery { - #[serde( - default, - rename = "wantedCollections", - deserialize_with = "deserialize_string_or_seq" - )] + #[serde(default, rename = "wantedCollections")] wanted_collections: Vec, - #[serde( - default, - rename = "wantedDids", - deserialize_with = "deserialize_string_or_seq" - )] + #[serde(default, rename = "wantedDids")] wanted_dids: Vec, #[serde(default, rename = "maxMessageSizeBytes", alias = "maxSize")] max_message_size_bytes: Option, @@ -71,17 +29,13 @@ compress: bool, #[serde(default, rename = "requireHello")] require_hello: bool, - #[serde( - default, - rename = "wantedEventTypes", - deserialize_with = "deserialize_string_or_seq" - )] + #[serde(default, rename = "wantedEventTypes")] wanted_event_types: Vec, } pub async fn handle_subscribe( State(hydrant): State, - Query(query): Query, + MultiQuery(query): MultiQuery, headers: HeaderMap, ws: WebSocketUpgrade, ) -> Response { @@ -370,5 +324,58 @@ .unwrap() .unwrap(); assert!(res_compressed.is_binary()); + } + #[test] + fn jetstream_query_accepts_repeated_values() { + let query: JetstreamQuery = serde_html_form::from_str( + "wantedCollections=app.bsky.feed.post&wantedCollections=app.bsky.feed.like\ + &wantedDids=did%3Aplc%3Atest&wantedEventTypes=live", + ) + .unwrap(); + assert_eq!( + query.wanted_collections, + ["app.bsky.feed.post", "app.bsky.feed.like"] + ); + assert_eq!(query.wanted_dids, ["did:plc:test"]); + assert_eq!(query.wanted_event_types, ["live"]); + + let defaults: JetstreamQuery = serde_html_form::from_str("").unwrap(); + assert!(defaults.wanted_collections.is_empty()); + assert!(defaults.wanted_dids.is_empty()); + assert!(defaults.wanted_event_types.is_empty()); + } + + #[test] + fn jetstream_options_message_json_contract() { + let options = + JetstreamFilter::new(JetstreamSubscriberOptions::parse(&[], &[], 0, &[]).unwrap()); + let message = Message::text( + serde_json::json!({ + "type": "options_update", + "payload": { + "wantedCollections": ["app.bsky.feed.post"], + "wantedDids": ["did:web:example.com"], + "maxSize": 4096, + "wantedEventTypes": ["live"], + }, + }) + .to_string(), + ); + assert_eq!(handle_options_message(message, &options), Ok(true)); + assert_eq!(options.max_message_size_bytes(), 4096); + + let ignored = Message::text(r#"{"type":"stinkpot","payload":{}}"#); + assert_eq!(handle_options_message(ignored, &options), Ok(false)); + assert!(handle_options_message(Message::text("not json"), &options).is_err()); + } + + #[test] + fn jetstream_message_size_conversion_rejects_invalid_ranges() { + assert_eq!(parse_max_message_size(None), 0); + assert_eq!(parse_max_message_size(Some(-1)), 0); + assert_eq!(parse_max_message_size(Some(0)), 0); + assert_eq!(parse_max_message_size(Some(1)), 1); + assert_eq!(parse_max_message_size(Some(u32::MAX.into())), u32::MAX); + assert_eq!(parse_max_message_size(Some(i64::from(u32::MAX) + 1)), 0); } } diff --git a/src/api/mod.rs b/src/api/mod.rs --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -7,6 +7,32 @@ const LISTEN_BACKLOG: i32 = 1024; +#[cfg(any(feature = "jetstream", feature = "backlinks"))] +pub struct MultiQuery(pub T); + +#[cfg(any(feature = "jetstream", feature = "backlinks"))] +impl axum::extract::FromRequestParts for MultiQuery +where + T: serde::de::DeserializeOwned, + S: Send + Sync, +{ + type Rejection = (axum::http::StatusCode, String); + + async fn from_request_parts( + parts: &mut axum::http::request::Parts, + _state: &S, + ) -> Result { + serde_html_form::from_str(parts.uri.query().unwrap_or_default()) + .map(Self) + .map_err(|error| { + ( + axum::http::StatusCode::BAD_REQUEST, + format!("failed to deserialize query string: {error}"), + ) + }) + } +} + #[cfg(feature = "indexer")] mod crawler; mod db; diff --git a/src/api/redactions.rs b/src/api/redactions.rs --- a/src/api/redactions.rs +++ b/src/api/redactions.rs @@ -3,20 +3,20 @@ use crate::control::Hydrant; #[cfg(feature = "indexer")] +use crate::types::{DeleteBodiesReport, DeleteBodyTarget}; +#[cfg(feature = "indexer")] use axum::{ + Json, extract::{Path, Query, State}, http::StatusCode, routing::post, - Json, }; -#[cfg(feature = "indexer")] -use jacquard_common::types::{did::Did, nsid::Nsid, string::Rkey}; #[cfg(feature = "indexer")] use jacquard_common::IntoStatic; #[cfg(feature = "indexer")] -use serde::Deserialize; +use jacquard_common::types::{did::Did, nsid::Nsid, string::Rkey}; #[cfg(feature = "indexer")] -use crate::types::{DeleteBodiesReport, DeleteBodyTarget}; +use serde::Deserialize; #[cfg(feature = "indexer")] #[derive(Deserialize, Debug, Default)] @@ -28,7 +28,7 @@ } #[cfg(feature = "indexer")] - #[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] pub enum RedactionTarget { Repo(Did<'static>), Record { @@ -39,7 +39,7 @@ } #[cfg(feature = "indexer")] - pub fn parse_redaction_target(raw: &str) -> Result { +pub fn parse_redaction_target(raw: &str) -> Result { let s = raw.strip_prefix("at://").unwrap_or(raw); let segments: Vec<&str> = s.split('/').filter(|seg| !seg.is_empty()).collect(); match segments.as_slice() { @@ -146,7 +146,8 @@ expected ); assert_eq!( - parse_redaction_target("at://did:plc:testrepo/app.bsky.feed.post/3kzbif5moe22m").unwrap(), + parse_redaction_target("at://did:plc:testrepo/app.bsky.feed.post/3kzbif5moe22m") + .unwrap(), expected ); } diff --git a/src/api/repos.rs b/src/api/repos.rs --- a/src/api/repos.rs +++ b/src/api/repos.rs @@ -36,7 +36,6 @@ pub did: String, } - #[derive(Deserialize)] pub struct GetReposParams { pub limit: Option, diff --git a/src/backlinks/api.rs b/src/backlinks/api.rs --- a/src/backlinks/api.rs +++ b/src/backlinks/api.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use smol_str::SmolStr; -use crate::control::Hydrant; +use crate::{api::MultiQuery, control::Hydrant}; pub fn router() -> Router { Router::new() @@ -56,7 +56,7 @@ pub async fn handle_get_backlinks( State(hydrant): State, - axum::extract::Query(params): axum::extract::Query, + MultiQuery(params): MultiQuery, ) -> Result, StatusCode> { let limit = params.limit.unwrap_or(50).min(100) as usize; let reverse = params.reverse.unwrap_or(false); @@ -106,7 +106,7 @@ pub async fn handle_get_backlinks_count( State(hydrant): State, - axum::extract::Query(params): axum::extract::Query, + MultiQuery(params): MultiQuery, ) -> Result, StatusCode> { let mut count_q = hydrant.backlinks.count(params.subject); if !params.did.is_empty() { @@ -122,4 +122,22 @@ .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; Ok(Json(GetBacklinksCountOutput { count })) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn backlinks_query_accepts_single_and_repeated_dids() { + let fetch: GetBacklinksParams = serde_html_form::from_str( + "subject=at%3A%2F%2Ftarget&did=did%3Aplc%3Aone&did=did%3Aweb%3Atwo", + ) + .unwrap(); + assert_eq!(fetch.did, ["did:plc:one", "did:web:two"]); + + let count: GetBacklinksCountParams = + serde_html_form::from_str("subject=at%3A%2F%2Ftarget&did=did%3Aplc%3Aone").unwrap(); + assert_eq!(count.did, ["did:plc:one"]); + } } diff --git a/src/backlinks/mod.rs b/src/backlinks/mod.rs --- a/src/backlinks/mod.rs +++ b/src/backlinks/mod.rs @@ -156,14 +156,11 @@ let mut backlinks = Vec::with_capacity(self.limit); let mut next_cursor = None; + let mut last_returned_key = None; for item in iter { let key = item.key().into_diagnostic()?; if !key.starts_with(scan_prefix.as_slice()) { - break; - } - if backlinks.len() >= self.limit { - next_cursor = Some(key.to_vec()); break; } @@ -180,11 +177,16 @@ // record deleted after backlink was written continue; }; + if backlinks.len() >= self.limit { + next_cursor = last_returned_key; + break; + } let at_uri = format!("at://{did}/{col}/{rkey}"); backlinks.push(BacklinkEntry { uri: SmolStr::new(at_uri), cid: SmolStr::new(cid), }); + last_returned_key = Some(key.to_vec()); } Ok(BacklinksPage { diff --git a/src/backlinks/store.rs b/src/backlinks/store.rs --- a/src/backlinks/store.rs +++ b/src/backlinks/store.rs @@ -99,6 +99,12 @@ let did = std::str::from_utf8(parts.next()?).ok()?; let path = std::str::from_utf8(parts.next()?).ok()?; let col = std::str::from_utf8(parts.next()?).ok()?; + let target = parts.next()?; + let prefix = parts.next()?; + if prefix != b"r" || target.is_empty() || parts.next().is_some() { + return None; + } + std::str::from_utf8(target).ok()?; Some((col, path, did, rkey)) } @@ -311,5 +317,34 @@ let prefix = forward_did_prefix(&did); let other_key = forward_key("did:plc:xyz999", "app.bsky.feed.post", "tid123"); assert!(!other_key.starts_with(&prefix)); + } + #[test] + fn reverse_key_parser_rejects_malformed_keys() { + for key in [ + b"f|target|collection|path|did|rkey".as_slice(), + b"r||collection|path|did|rkey", + b"r|target|collection|path|did", + b"x|r|target|collection|path|did|rkey", + b"r|\xff|collection|path|did|rkey", + ] { + assert!(source_from_reverse_key(key).is_none()); + } + } + + #[test] + fn forward_value_persisted_golden_fixture() { + let links = vec![ + (".subject.uri".to_owned(), TARGET.to_owned()), + (".subject.cid".to_owned(), "bafy-stinkpot".to_owned()), + ]; + let encoded = rmp_serde::to_vec(&links).unwrap(); + assert_eq!( + hex::encode(&encoded), + "9292ac2e7375626a6563742e757269d92761743a2f2f6469643a706c633a6162632f6170702e62736b792e666565642e706f73742f74696492ac2e7375626a6563742e636964ad626166792d7374696e6b706f74" + ); + assert_eq!( + rmp_serde::from_slice::>(&encoded).unwrap(), + links + ); } } diff --git a/src/bin/backfill_strategy_bench.rs b/src/bin/backfill_strategy_bench.rs --- a/src/bin/backfill_strategy_bench.rs +++ b/src/bin/backfill_strategy_bench.rs @@ -483,3 +483,32 @@ .expect("validated url") .to_owned() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn benchmark_api_json_fixtures_decode() { + let list: ListReposByCollectionOutput = serde_json::from_value(serde_json::json!({ + "repos": [{"did": "did:web:example.com"}], + })) + .unwrap(); + assert_eq!(list.repos.len(), 1); + assert_eq!(list.repos[0].did, "did:web:example.com"); + + let doc: DidDoc = serde_json::from_value(serde_json::json!({ + "service": [{ + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.example", + }], + })) + .unwrap(); + assert_eq!(doc.service.len(), 1); + assert_eq!(doc.service[0].kind, "AtprotoPersonalDataServer"); + assert_eq!( + doc.service[0].service_endpoint.as_str(), + "https://pds.example/" + ); + } +} diff --git a/src/config/env.rs b/src/config/env.rs --- a/src/config/env.rs +++ b/src/config/env.rs @@ -31,25 +31,28 @@ use crate::__cfg as cfg; /// loads `.env` from the current directory, setting any variables not already in the environment. -fn load_dotenv() { - let Ok(contents) = std::fs::read_to_string(".env") else { - return; - }; - for line in contents.lines() { +fn dotenv_assignments(contents: &str) -> impl Iterator { + contents.lines().filter_map(|line| { let line = line.trim(); if line.is_empty() || line.starts_with('#') { - continue; + return None; } - let Some((key, val)) = line.split_once('=') else { - continue; - }; - let key = key.trim(); + let (key, val) = line.split_once('=')?; let val = val.trim(); let val = val .strip_prefix('"') .and_then(|v| v.strip_suffix('"')) .or_else(|| val.strip_prefix('\'').and_then(|v| v.strip_suffix('\''))) .unwrap_or(val); + Some((key.trim(), val)) + }) +} + +fn load_dotenv() { + let Ok(contents) = std::fs::read_to_string(".env") else { + return; + }; + for (key, val) in dotenv_assignments(&contents) { if std::env::var(key).is_err() { // SAFETY: single-threaded at startup; no other threads are reading env yet. unsafe { std::env::set_var(key, val) }; @@ -58,10 +61,17 @@ } fn parse_new_host_limit(default: Option) -> Option { - match std::env::var("HYDRANT_NEW_HOST_LIMIT") { - Ok(s) if s.trim().eq_ignore_ascii_case("none") => None, - Ok(s) => s.trim().parse().ok().or(default), - Err(_) => default, + parse_new_host_limit_value( + std::env::var("HYDRANT_NEW_HOST_LIMIT").ok().as_deref(), + default, + ) +} + +fn parse_new_host_limit_value(value: Option<&str>, default: Option) -> Option { + match value.map(str::trim) { + Some(value) if value.eq_ignore_ascii_case("none") => None, + Some(value) => value.parse().ok().or(default), + None => default, } } @@ -409,5 +419,41 @@ }; config.validate()?; Ok(config) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn env_text_parser_contracts() { + let parsed = dotenv_assignments( + r#" + # ignored + BARE = value + DOUBLE="two words" + SINGLE='three words' + INVALID + "#, + ) + .collect::>(); + assert_eq!( + parsed, + [ + ("BARE", "value"), + ("DOUBLE", "two words"), + ("SINGLE", "three words"), + ] + ); + + assert_eq!(parse_new_host_limit_value(None, Some(10)), Some(10)); + assert_eq!(parse_new_host_limit_value(Some(" 42 "), Some(10)), Some(42)); + assert_eq!(parse_new_host_limit_value(Some("NoNe"), Some(10)), None); + assert_eq!( + parse_new_host_limit_value(Some("invalid"), Some(10)), + Some(10) + ); + assert_eq!(parse_new_host_limit_value(Some("invalid"), None), None); } } diff --git a/src/config/types.rs b/src/config/types.rs --- a/src/config/types.rs +++ b/src/config/types.rs @@ -273,19 +273,84 @@ use super::*; #[test] - fn relay_first_crawler_mode_round_trips() { - assert_eq!( - "relay_first".parse::().unwrap(), - CrawlerMode::RelayFirst - ); - assert_eq!( - "relay-first".parse::().unwrap(), - CrawlerMode::RelayFirst - ); - assert_eq!(CrawlerMode::RelayFirst.to_string(), "relay_first"); - assert_eq!( - serde_json::from_str::("\"relay_first\"").unwrap(), - CrawlerMode::RelayFirst - ); + fn crawler_mode_string_and_msgpack_contracts() { + for (mode, canonical, alias) in [ + (CrawlerMode::ListRepos, "list_repos", "list-repos"), + (CrawlerMode::RelayFirst, "relay_first", "relay-first"), + (CrawlerMode::ByCollection, "by_collection", "by-collection"), + ] { + assert_eq!(canonical.parse::().unwrap(), mode); + assert_eq!(alias.parse::().unwrap(), mode); + assert_eq!(mode.to_string(), canonical); + + let json = serde_json::to_string(&mode).unwrap(); + assert_eq!(json, format!("\"{canonical}\"")); + assert_eq!(serde_json::from_str::(&json).unwrap(), mode); + + let msgpack = rmp_serde::to_vec(&mode).unwrap(); + assert_eq!( + rmp_serde::from_slice::(&msgpack).unwrap(), + mode + ); + } + assert!("unknown".parse::().is_err()); + assert!(serde_json::from_str::("7").is_err()); + } + + #[test] + fn text_config_parser_contracts() { + for (raw, expected) in [ + ("lz4", Compression::Lz4), + ("zstd", Compression::Zstd), + ("none", Compression::None), + ] { + assert_eq!(raw.parse::().unwrap(), expected); + assert_eq!(expected.to_string(), raw); + } + assert!("gzip".parse::().is_err()); + + for (raw, expected) in [ + ("full", SignatureVerification::Full), + ("backfill-only", SignatureVerification::BackfillOnly), + ("none", SignatureVerification::None), + ] { + assert_eq!(raw.parse::().unwrap(), expected); + assert_eq!(expected.to_string(), raw); + } + assert!("sometimes".parse::().is_err()); + + for (raw, expected) in [ + ("full", BackfillStrategy::Full), + ("sparse-filter", BackfillStrategy::SparseFilter), + ("auto", BackfillStrategy::Auto), + ] { + assert_eq!(raw.parse::().unwrap(), expected); + assert_eq!(expected.to_string(), raw); + } + assert!("sparse".parse::().is_err()); + } + + #[test] + fn source_and_rate_tier_parser_contracts() { + let tier = RateTier::parse("10/0.5/100/1000/7").unwrap(); + assert_eq!(tier.per_second_base, 10); + assert_eq!(tier.per_second_account_mul, 0.5); + assert_eq!(tier.per_hour, 100); + assert_eq!(tier.per_day, 1000); + assert_eq!(tier.account_limit, Some(7)); + assert!(RateTier::parse("10/0.5/100").is_none()); + assert!(RateTier::parse("bad/0.5/100/1000").is_none()); + + let crawler = + CrawlerSource::parse("relay-first::https://relay.example", CrawlerMode::ListRepos) + .unwrap(); + assert_eq!(crawler.mode, CrawlerMode::RelayFirst); + assert_eq!(crawler.url.as_str(), "https://relay.example/"); + assert!(CrawlerSource::parse("bad::not a url", CrawlerMode::ListRepos).is_none()); + + let firehose = FirehoseSource::parse("pds::wss://pds.example").unwrap(); + assert!(firehose.is_pds); + assert_eq!(firehose.url.as_str(), "wss://pds.example/"); + assert!(FirehoseSource::parse("not a url").is_none()); } } diff --git a/src/crawler/relay_first.rs b/src/crawler/relay_first.rs --- a/src/crawler/relay_first.rs +++ b/src/crawler/relay_first.rs @@ -605,12 +605,20 @@ #[test] fn pass_cursor_round_trips() -> Result<()> { - let cursor = PassCursor::active(7, Some("next".into())); + let cursor = PassCursor::active(7, Some("stinkpot".into())); + let encoded = serialize_cursor(&cursor)?; + assert_eq!(hex::encode(&encoded), "9307a87374696e6b706f74c2"); assert_eq!( - rmp_serde::from_slice::(&serialize_cursor(&cursor)?).into_diagnostic()?, + rmp_serde::from_slice::(&encoded).into_diagnostic()?, cursor ); - assert!(PassCursor::complete(7).complete); + let complete = PassCursor::complete(7); + let encoded = serialize_cursor(&complete)?; + assert_eq!(hex::encode(&encoded), "9307c0c3"); + assert_eq!( + rmp_serde::from_slice::(&encoded).into_diagnostic()?, + complete + ); Ok(()) } diff --git a/src/db/indexer.rs b/src/db/indexer.rs --- a/src/db/indexer.rs +++ b/src/db/indexer.rs @@ -973,4 +973,73 @@ Ok(()) } + + #[test] + fn record_count_codec_updates_and_rejects_invalid_bytes() -> Result<()> { + let (_tmp, db) = test_db()?; + let did = did(DID); + let key = keys::count_collection_key(&did, COLLECTION); + + let mut batch = db.inner.batch(); + set_record_count(&mut batch, &db, &did, COLLECTION, 7); + batch.commit().into_diagnostic()?; + assert_eq!( + db.counts.get(&key).into_diagnostic()?.unwrap().as_ref(), + 7_u64.to_be_bytes() + ); + assert_eq!(get_record_count(&db, &did, COLLECTION)?, 7); + + let mut batch = db.inner.batch(); + update_record_count(&mut batch, &db, &did, COLLECTION, -9)?; + batch.commit().into_diagnostic()?; + assert_eq!(get_record_count(&db, &did, COLLECTION)?, 0); + + db.counts.insert(&key, [0_u8; 7]).into_diagnostic()?; + assert!(get_record_count(&db, &did, COLLECTION).is_err()); + let mut batch = db.inner.batch(); + assert!(update_record_count(&mut batch, &db, &did, COLLECTION, 1).is_err()); + Ok(()) + } + + #[test] + fn persisted_crawler_sources_decode_mode_and_url() -> Result<()> { + let (_tmp, db) = test_db()?; + let fixtures = [ + ( + Url::parse("https://relay.example/x").into_diagnostic()?, + crate::config::CrawlerMode::RelayFirst, + ), + ( + Url::parse("https://collection.example/").into_diagnostic()?, + crate::config::CrawlerMode::ByCollection, + ), + ]; + for (url, mode) in &fixtures { + db.crawler + .insert( + keys::crawler_source_key(url.as_str()), + rmp_serde::to_vec(mode).into_diagnostic()?, + ) + .into_diagnostic()?; + } + + let mut decoded = load_persisted_crawler_sources(&db)?; + decoded.sort_by(|a, b| a.url.as_str().cmp(b.url.as_str())); + let mut expected = fixtures + .into_iter() + .map(|(url, mode)| crate::config::CrawlerSource { url, mode }) + .collect::>(); + expected.sort_by(|a, b| a.url.as_str().cmp(b.url.as_str())); + assert_eq!(decoded.len(), expected.len()); + for (decoded, expected) in decoded.iter().zip(expected) { + assert_eq!(decoded.url, expected.url); + assert_eq!(decoded.mode, expected.mode); + } + + db.crawler + .insert(keys::crawler_source_key("not a url"), [0xc0]) + .into_diagnostic()?; + assert!(load_persisted_crawler_sources(&db).is_err()); + Ok(()) + } } diff --git a/src/db/mod.rs b/src/db/mod.rs --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -308,4 +308,36 @@ Ok(()) } + #[test] + fn firehose_source_meta_golden_fixture() { + let encoded = rmp_serde::to_vec(&FirehoseSourceMeta { is_pds: true }).unwrap(); + assert_eq!(hex::encode(&encoded), "91c3"); + let decoded: FirehoseSourceMeta = rmp_serde::from_slice(&encoded).unwrap(); + assert!(decoded.is_pds); + + let legacy_default: FirehoseSourceMeta = rmp_serde::from_slice(&[0x90]).unwrap(); + assert!(!legacy_default.is_pds); + } + + #[tokio::test] + async fn firehose_cursor_big_endian_roundtrip() -> Result<()> { + let tmp = tempfile::tempdir().into_diagnostic()?; + let cfg = Config { + database_path: tmp.path().to_path_buf(), + ..Config::default() + }; + let db = Db::open(&cfg)?; + let relay = Url::parse("https://relay.example").into_diagnostic()?; + let cursor = i64::from_be_bytes(*b"stinkpot"); + + set_firehose_cursor(&db, &relay, cursor)?; + let raw = db + .cursors + .get(keys::firehose_cursor_key_from_url(&relay)) + .into_diagnostic()? + .unwrap(); + assert_eq!(raw.as_ref(), b"stinkpot"); + assert_eq!(get_firehose_cursor(&db, &relay).await?, Some(cursor)); + Ok(()) + } } diff --git a/src/db/schema.rs b/src/db/schema.rs --- a/src/db/schema.rs +++ b/src/db/schema.rs @@ -756,3 +756,59 @@ .data_block_restart_interval_policy(RestartIntervalPolicy::new([16, 32])) } } + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + #[test] + fn keyspace_names_match_the_compiled_storage_schema() { + let tmp = tempfile::tempdir().unwrap(); + let db = crate::db::Db::open(&crate::config::Config { + database_path: tmp.path().to_path_buf(), + ..Default::default() + }) + .unwrap(); + + let expected = BTreeSet::from([ + "repos", + "repo_metadata", + "cursors", + "counts", + "filter", + "crawler", + ]); + #[cfg(any( + feature = "backlinks", + feature = "indexer", + feature = "indexer_stream", + feature = "jetstream", + feature = "relay" + ))] + let mut expected = expected; + #[cfg(feature = "backlinks")] + expected.insert("backlinks"); + #[cfg(feature = "indexer")] + expected.extend([ + "heads", + "history", + "redactions", + "pending", + "resync", + "resync_buffer", + ]); + #[cfg(feature = "indexer_stream")] + expected.extend(["events", "event_bodies"]); + #[cfg(feature = "jetstream")] + expected.insert("jetstream_events"); + #[cfg(feature = "relay")] + expected.insert("relay_events"); + + let actual = db + .all_keyspaces() + .into_iter() + .map(|(name, _)| name) + .collect::>(); + assert_eq!(actual, expected); + } +} diff --git a/src/db/storage_mode.rs b/src/db/storage_mode.rs --- a/src/db/storage_mode.rs +++ b/src/db/storage_mode.rs @@ -228,6 +228,23 @@ } #[test] + fn storage_mode_marker_persisted_golden_fixture() { + let marker = StorageModeMarker { + links_only: false, + ephemeral: true, + }; + let encoded = rmp_serde::to_vec_named(&marker).unwrap(); + assert_eq!( + hex::encode(&encoded), + "82aa6c696e6b735f6f6e6c79c2a9657068656d6572616cc3" + ); + assert_eq!( + rmp_serde::from_slice::(&encoded).unwrap(), + marker + ); + } + + #[test] #[cfg(all(feature = "indexer_stream", not(feature = "backlinks")))] fn ephemeral_mode_is_immutable() { let tmp = tempfile::tempdir().unwrap(); diff --git a/src/db/types.rs b/src/db/types.rs --- a/src/db/types.rs +++ b/src/db/types.rs @@ -38,6 +38,20 @@ TrimmedDid::Other(s) => TrimmedDid::Other(s.into_static()), } } + fn from_trimmed_str(value: &'s str) -> Self { + let value = value.strip_prefix("did:").unwrap_or(value); + if let Some(rest) = value.strip_prefix("plc:") + && rest.len() == 24 + && let Ok(bytes) = BASE32_NOPAD.decode(rest.to_ascii_uppercase().as_bytes()) + && let Ok(bytes) = bytes.try_into() + { + return TrimmedDid::Plc(bytes); + } + if let Some(rest) = value.strip_prefix("web:") { + return TrimmedDid::Web(CowStr::Borrowed(rest)); + } + TrimmedDid::Other(CowStr::Borrowed(value)) + } pub fn to_did(&self) -> Did<'static> { match self { @@ -87,20 +101,7 @@ impl<'a> From<&'a Did<'a>> for TrimmedDid<'a> { fn from(did: &'a Did<'a>) -> Self { - let s = did.as_str(); - if let Some(rest) = s.strip_prefix("did:plc:") { - if rest.len() == 24 { - // decode - if let Ok(bytes) = BASE32_NOPAD.decode(rest.to_ascii_uppercase().as_bytes()) - && bytes.len() == 15 - { - return TrimmedDid::Plc(bytes.try_into().unwrap()); - } - } - } else if let Some(rest) = s.strip_prefix("did:web:") { - return TrimmedDid::Web(CowStr::Borrowed(rest)); - } - TrimmedDid::Other(CowStr::Borrowed(s.trim_start_matches("did:"))) + Self::from_trimmed_str(did.as_str()) } } @@ -163,28 +164,77 @@ impl<'de> Deserialize<'de> for TrimmedDid<'de> { fn deserialize>(deserializer: D) -> Result { struct DidVisitor; + + fn static_from_bytes(value: &[u8]) -> Result, E> + where + E: serde::de::Error, + { + TrimmedDid::try_from(value) + .map(TrimmedDid::into_static) + .map_err(E::custom) + } + impl<'de> serde::de::Visitor<'de> for DidVisitor { type Value = TrimmedDid<'de>; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("bytes (tagged) or string (legacy)") } - fn visit_bytes(self, v: &[u8]) -> Result + fn visit_str(self, value: &str) -> Result where E: serde::de::Error, { - TrimmedDid::try_from(v) - .map(|td| td.into_static()) - .map_err(E::custom) + Ok(TrimmedDid::from_trimmed_str(value).into_static()) } - fn visit_borrowed_bytes(self, v: &'de [u8]) -> Result + fn visit_borrowed_str(self, value: &'de str) -> Result where E: serde::de::Error, { - TrimmedDid::try_from(v).map_err(E::custom) + Ok(TrimmedDid::from_trimmed_str(value)) + } + + fn visit_string(self, value: String) -> Result + where + E: serde::de::Error, + { + Ok(TrimmedDid::from_trimmed_str(&value).into_static()) + } + + fn visit_bytes(self, value: &[u8]) -> Result + where + E: serde::de::Error, + { + static_from_bytes(value) + } + + fn visit_borrowed_bytes(self, value: &'de [u8]) -> Result + where + E: serde::de::Error, + { + TrimmedDid::try_from(value).map_err(E::custom) + } + + fn visit_byte_buf(self, value: Vec) -> Result + where + E: serde::de::Error, + { + static_from_bytes(&value) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut bytes = Vec::with_capacity(seq.size_hint().unwrap_or(0).min(2048)); + while let Some(byte) = seq.next_element()? { + bytes.push(byte); + } + static_from_bytes(&bytes) } } + deserializer.deserialize_any(DidVisitor) } } @@ -488,6 +538,67 @@ } #[test] + fn db_representation_conversions_are_lossless() { + for did in [ + "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + "did:web:example.com", + "did:example:opaque", + ] { + let did = Did::new(did).unwrap(); + let trimmed = TrimmedDid::from(&did); + let mut bytes = Vec::new(); + trimmed.write_to_vec(&mut bytes); + + assert_eq!(TrimmedDid::try_from(bytes.as_slice()).unwrap(), trimmed); + assert_eq!(trimmed.to_did(), did); + assert_eq!(trimmed.to_string(), did.as_str().trim_start_matches("did:")); + assert_eq!( + UserKey::from(trimmed.clone()), + UserKey::from(&trimmed), + "owned and borrowed key conversions diverged for {did}" + ); + assert_eq!(trimmed.clone().into_static(), trimmed); + } + + for malformed in [ + &[][..], + &[TAG_PLC][..], + &[TAG_PLC, 0][..], + &[TAG_WEB, 0xff][..], + &[0xff][..], + ] { + assert!(TrimmedDid::try_from(malformed).is_err()); + } + + let tid = Tid::new("3jzfcijpj2z2a").unwrap(); + let db_tid = DbTid::from(&tid); + assert_eq!(Tid::from(db_tid), tid); + assert_eq!(db_tid.to_string(), tid.as_str()); + + for value in ["self", "stinkpot", "3jzfcijpj2z2a"] { + let from_str = DbRkey::from(value); + assert_eq!(DbRkey::from(value.to_owned()), from_str); + assert_eq!(DbRkey::from(SmolStr::from(value)), from_str); + assert_eq!(from_str.to_string(), value); + } + + use jacquard_common::types::crypto::KeyCodec; + for (codec, prefix) in [ + (KeyCodec::Ed25519, &[0xed, 0x01][..]), + (KeyCodec::Secp256k1, &[0xe7, 0x01][..]), + (KeyCodec::P256, &[0x80, 0x24][..]), + (KeyCodec::Unknown(0x42), &[0x42][..]), + ] { + let key = DidKey::from(PublicKey { + codec, + bytes: Cow::Borrowed(&[1, 2, 3]), + }); + assert_eq!(key.0.strip_suffix(&[1, 2, 3]), Some(prefix)); + assert_eq!(DidKey::from_did_key(&key.to_string()).unwrap(), key); + } + } + + #[test] fn test_dbrkey() { let tid_str = "3jzfcijpj2z2a"; let rkey = DbRkey::new(tid_str); @@ -535,5 +646,114 @@ let decoded: DbRkey = serde_json::from_slice(&encoded).unwrap(); assert_eq!(decoded, expected); + } + #[test] + fn trimmed_did_accepts_legacy_msgpack_strings() { + for did in [ + "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + "did:web:example.com", + "did:example:opaque", + ] { + let parsed = Did::new(did).unwrap(); + let expected = TrimmedDid::from(&parsed); + for legacy in [did, did.strip_prefix("did:").unwrap()] { + let encoded = rmp_serde::to_vec(legacy).unwrap(); + + let decoded: TrimmedDid<'_> = rmp_serde::from_slice(&encoded).unwrap(); + assert_eq!(decoded, expected); + assert_eq!(decoded.to_did().as_str(), did); + } + } + } + + #[test] + fn trimmed_did_binary_and_json_sequence_roundtrip() { + for (did, golden) in [ + ( + "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + "c4100125aa8fb6f9c3a8df64f789e5ee3919", + ), + ("did:web:example.com", "c40c026578616d706c652e636f6d"), + ("did:example:opaque", "c40e6578616d706c653a6f7061717565"), + ] { + let parsed = Did::new(did).unwrap(); + let expected = TrimmedDid::from(&parsed); + + let msgpack = rmp_serde::to_vec(&expected).unwrap(); + assert_eq!(msgpack.first(), Some(&0xc4)); + assert_eq!(hex::encode(&msgpack), golden); + let from_msgpack: TrimmedDid<'_> = rmp_serde::from_slice(&msgpack).unwrap(); + assert_eq!(from_msgpack, expected); + + let json = serde_json::to_vec(&expected).unwrap(); + assert_eq!(json.first(), Some(&b'[')); + let from_json: TrimmedDid<'_> = serde_json::from_slice(&json).unwrap(); + assert_eq!(from_json, expected); + } + } + + #[test] + fn dbrkey_string_markers_preserve_every_utf8_length() { + let mut values = (0..=24) + .map(|length| "x".repeat(length)) + .collect::>(); + values.extend(["stinkpot".to_owned(), "éééé".to_owned()]); + + for value in values { + let expected = DbRkey::Str(value.clone().into()); + let encoded = rmp_serde::to_vec(&expected).unwrap(); + let decoded: DbRkey = rmp_serde::from_slice(&encoded).unwrap(); + assert_eq!(decoded, expected, "failed for {value:?}"); + } + } + + #[test] + fn dbrkey_binary_marker_requires_exactly_eight_bytes() { + for length in [0, 1, 7, 9, 16] { + let encoded = + rmp_serde::to_vec(&serde_bytes::ByteBuf::from(vec![b'x'; length])).unwrap(); + assert!(rmp_serde::from_slice::(&encoded).is_err()); + } + + let encoded = rmp_serde::to_vec(&serde_bytes::ByteBuf::from(b"stinkpot".to_vec())).unwrap(); + let decoded: DbRkey = rmp_serde::from_slice(&encoded).unwrap(); + assert!(matches!(decoded, DbRkey::Tid(_))); + } + + #[test] + fn dbrkey_json_sequence_requires_exactly_eight_bytes() { + for length in [0, 1, 7, 9, 16] { + let encoded = serde_json::to_vec(&vec![0_u8; length]).unwrap(); + assert!(serde_json::from_slice::(&encoded).is_err()); + } + + let decoded: DbRkey = serde_json::from_slice(b"[0,1,2,3,4,5,6,7]").unwrap(); + assert_eq!( + decoded, + DbRkey::Tid(DbTid::new_from_bytes([0, 1, 2, 3, 4, 5, 6, 7])) + ); + } + + #[test] + fn db_action_and_did_key_representation_contracts() { + for (action, golden) in [ + (DbAction::Create, "a6437265617465"), + (DbAction::Update, "a6557064617465"), + (DbAction::Delete, "a644656c657465"), + ] { + let msgpack = rmp_serde::to_vec(&action).unwrap(); + assert_eq!(hex::encode(&msgpack), golden); + assert_eq!(rmp_serde::from_slice::(&msgpack).unwrap(), action); + assert_eq!(DbAction::try_from(action.as_str()).unwrap(), action); + } + + let expected = DidKey(Cow::Borrowed(&[0xed, 0x01, 0x02, 0x03])); + let msgpack = rmp_serde::to_vec(&expected).unwrap(); + assert_eq!(hex::encode(&msgpack), "c404ed010203"); + let decoded: DidKey<'_> = rmp_serde::from_slice(&msgpack).unwrap(); + assert_eq!(decoded, expected); + + let display = expected.to_string(); + assert_eq!(DidKey::from_did_key(&display).unwrap(), expected); } } diff --git a/src/ingest/stream.rs b/src/ingest/stream.rs --- a/src/ingest/stream.rs +++ b/src/ingest/stream.rs @@ -186,8 +186,17 @@ fn test_decode_account() { const FRAME: &[u8] = b"omF0aCNhY2NvdW50Ym9wAaRjZGlkeCBkaWQ6cGxjOjNuNDNncWY2YTZua3J3MzU1cnNjNnJ0ZGNzZXEbAAAAAlP5Zt5kdGltZXgbMjAyNi0wMi0yNFQxNDowNToyMC41MjE1NDY5ZmFjdGl2ZfU="; let bytes = data_encoding::BASE64.decode(FRAME).unwrap(); - let msg = decode_frame(&bytes).unwrap(); - assert!(matches!(msg, SubscribeReposMessage::Account(_))); + let SubscribeReposMessage::Account(account) = decode_frame(&bytes).unwrap() else { + panic!("expected account"); + }; + assert_eq!(account.did.as_str(), "did:plc:3n43gqf6a6nkrw355rsc6rtd"); + assert!(account.active); + assert!(account.status.is_none()); + assert_eq!(account.seq, 9_998_788_318); + assert_eq!( + account.time.0.to_rfc3339(), + "2026-02-24T14:05:20.521546900+00:00" + ); } /// regression: some relays send `since: ""` (empty string) instead of null/absent for the initial commit. @@ -202,6 +211,34 @@ panic!("expected Commit"); }; assert!(c.since.is_none(), "since should be None for empty string"); + let stored = rmp_serde::to_vec_named(c.as_ref()).unwrap(); + use sha2::Digest as _; + assert_eq!( + (stored.len(), hex::encode(sha2::Sha256::digest(&stored)),), + ( + 523, + "cb3bb18e50d66f71c22b468f69df1b553922bb1073c2f7841fd3cae00317e260".to_owned(), + ) + ); + let replayed: super::types::Commit<'_> = rmp_serde::from_slice(&stored).unwrap(); + assert!(replayed.since.is_none()); + assert_eq!(replayed.repo, c.repo); + assert_eq!(replayed.rev, c.rev); + assert_eq!(replayed.blobs, c.blobs); + assert_eq!(replayed.blocks, c.blocks); + assert_eq!(replayed.commit, c.commit); + assert_eq!(replayed.ops.len(), c.ops.len()); + for (actual, expected) in replayed.ops.iter().zip(&c.ops) { + assert_eq!(actual.action, expected.action); + assert_eq!(actual.cid, expected.cid); + assert_eq!(actual.path, expected.path); + assert_eq!(actual.prev, expected.prev); + } + assert_eq!(replayed.prev_data, c.prev_data); + assert_eq!(replayed.rebase, c.rebase); + assert_eq!(replayed.seq, c.seq); + assert_eq!(replayed.time.0, c.time.0); + assert_eq!(replayed.too_big, c.too_big); } #[cfg(feature = "relay")] diff --git a/src/ingest/validation.rs b/src/ingest/validation.rs --- a/src/ingest/validation.rs +++ b/src/ingest/validation.rs @@ -445,3 +445,21 @@ Ok(()) }) } + +#[cfg(test)] +mod tests { + #[test] + fn record_dag_cbor_projection_accepts_data_and_rejects_invalid_bytes() { + let encoded = serde_ipld_dagcbor::to_vec(&serde_json::json!({ + "$type": "app.bsky.feed.post", + "text": "stinkpot", + })) + .unwrap(); + let decoded = serde_ipld_dagcbor::from_slice::(&encoded).unwrap(); + assert!(matches!(decoded, jacquard_common::Data::Object(_))); + assert!( + serde_ipld_dagcbor::from_slice::(&[0xff]).is_err(), + "break marker is not a standalone record value" + ); + } +} diff --git a/src/types/v2.rs b/src/types/v2.rs --- a/src/types/v2.rs +++ b/src/types/v2.rs @@ -42,3 +42,79 @@ #[serde(borrow)] pub handle: Option>, } + +#[cfg(test)] +mod tests { + use super::*; + use jacquard_common::types::string::Tid; + + #[test] + fn v2_persisted_schema_golden_fixture() { + for (status, golden) in [ + (RepoStatus::Backfilling, "ab4261636b66696c6c696e67"), + (RepoStatus::Synced, "a653796e636564"), + ( + RepoStatus::Error("stinkpot".into()), + "81a54572726f72a87374696e6b706f74", + ), + (RepoStatus::Deactivated, "ab4465616374697661746564"), + (RepoStatus::Takendown, "a954616b656e646f776e"), + (RepoStatus::Suspended, "a953757370656e646564"), + ] { + let encoded = rmp_serde::to_vec(&status).unwrap(); + assert_eq!(hex::encode(&encoded), golden); + assert_eq!( + rmp_serde::from_slice::(&encoded).unwrap(), + status + ); + } + + let state = RepoState { + status: RepoStatus::Error("stinkpot".into()), + root: Some(Commit { + version: 3, + rev: DbTid::from(&Tid::new("3jzfcijpj2z2a").unwrap()), + data: jacquard_repo::mst::util::compute_cid(b"root").unwrap(), + prev: None, + sig: Bytes::from_static(b"\x01\x02\x03"), + }), + last_message_time: Some(-1), + last_updated_at: 42, + tracked: true, + index_id: 7, + signing_key: Some(DidKey(std::borrow::Cow::Borrowed(&[0xed, 1, 2, 3]))), + pds: Some("https://pds.example".into()), + handle: Some(Handle::new("alice.test").unwrap()), + }; + let encoded = rmp_serde::to_vec(&state).unwrap(); + assert_eq!( + hex::encode(&encoded), + "9981a54572726f72a87374696e6b706f749503c40817fd6873eaf07c06c424017112204813494d137e1631bba301d5acab6e7bb7aa74ce1185d456565ef51d737677b2c0c403010203ff2ac307c404ed010203b368747470733a2f2f7064732e6578616d706c65aa616c6963652e74657374" + ); + + let decoded: RepoState<'_> = rmp_serde::from_slice(&encoded).unwrap(); + assert!(matches!(&decoded.status, RepoStatus::Error(value) if value == "stinkpot")); + let root = decoded.root.unwrap(); + assert_eq!(root.version, 3); + assert_eq!(root.rev, DbTid::from(&Tid::new("3jzfcijpj2z2a").unwrap())); + assert_eq!( + root.data, + jacquard_repo::mst::util::compute_cid(b"root").unwrap() + ); + assert!(root.prev.is_none()); + assert_eq!(root.sig.as_ref(), b"\x01\x02\x03"); + assert_eq!(decoded.last_message_time, Some(-1)); + assert_eq!(decoded.last_updated_at, 42); + assert!(decoded.tracked); + assert_eq!(decoded.index_id, 7); + assert_eq!( + decoded.signing_key.as_ref().map(|key| key.0.as_ref()), + Some([0xed, 1, 2, 3].as_slice()) + ); + assert_eq!(decoded.pds.as_deref(), Some("https://pds.example")); + assert_eq!( + decoded.handle.as_ref().map(Handle::as_str), + Some("alice.test") + ); + } +} diff --git a/src/types/v4.rs b/src/types/v4.rs --- a/src/types/v4.rs +++ b/src/types/v4.rs @@ -38,3 +38,73 @@ pub tracked: bool, pub index_id: u64, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn v4_persisted_schema_golden_fixtures() { + for (status, golden) in [ + (RepoStatus::Synced, "a653796e636564"), + ( + RepoStatus::Error("stinkpot".into()), + "81a54572726f72a87374696e6b706f74", + ), + (RepoStatus::Deactivated, "ab4465616374697661746564"), + (RepoStatus::Takendown, "a954616b656e646f776e"), + (RepoStatus::Suspended, "a953757370656e646564"), + (RepoStatus::Deleted, "a744656c65746564"), + (RepoStatus::Desynchronized, "ae446573796e6368726f6e697a6564"), + (RepoStatus::Throttled, "a95468726f74746c6564"), + ] { + let encoded = rmp_serde::to_vec(&status).unwrap(); + assert_eq!(hex::encode(&encoded), golden); + assert_eq!( + rmp_serde::from_slice::(&encoded).unwrap(), + status + ); + } + + let state = RepoState { + active: false, + status: RepoStatus::Throttled, + root: None, + last_message_time: Some(7), + last_updated_at: 42, + signing_key: Some(DidKey(std::borrow::Cow::Borrowed(&[0xed, 1, 2, 3]))), + pds: Some("https://pds.example".into()), + handle: Some(Handle::new("alice.test").unwrap()), + }; + let encoded = rmp_serde::to_vec(&state).unwrap(); + assert_eq!( + hex::encode(&encoded), + "98c2a95468726f74746c6564c0072ac404ed010203b368747470733a2f2f7064732e6578616d706c65aa616c6963652e74657374" + ); + let decoded: RepoState<'_> = rmp_serde::from_slice(&encoded).unwrap(); + assert!(!decoded.active); + assert_eq!(decoded.status, RepoStatus::Throttled); + assert_eq!(decoded.last_message_time, Some(7)); + assert_eq!(decoded.last_updated_at, 42); + assert!(decoded.root.is_none()); + assert_eq!( + decoded.signing_key.as_ref().map(|key| key.0.as_ref()), + Some([0xed, 1, 2, 3].as_slice()) + ); + assert_eq!(decoded.pds.as_deref(), Some("https://pds.example")); + assert_eq!( + decoded.handle.as_ref().map(Handle::as_str), + Some("alice.test") + ); + + let metadata = RepoMetadata { + tracked: true, + index_id: 7, + }; + let encoded = rmp_serde::to_vec(&metadata).unwrap(); + assert_eq!(hex::encode(&encoded), "92c307"); + let decoded: RepoMetadata = rmp_serde::from_slice(&encoded).unwrap(); + assert!(decoded.tracked); + assert_eq!(decoded.index_id, 7); + } +} diff --git a/src/types/v7.rs b/src/types/v7.rs --- a/src/types/v7.rs +++ b/src/types/v7.rs @@ -21,3 +21,47 @@ #[serde(borrow)] pub handle: Option>, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn v7_persisted_schema_golden_fixture() { + let state = RepoState { + active: true, + status: RepoStatus::Synced, + root: None, + last_message_time: Some(7), + last_identity_time: Some(8), + last_account_time: Some(9), + last_updated_at: 42, + signing_key: Some(DidKey(std::borrow::Cow::Borrowed(&[0xed, 1, 2, 3]))), + pds: Some("https://pds.example".into()), + handle: Some(Handle::new("alice.test").unwrap()), + }; + let encoded = rmp_serde::to_vec(&state).unwrap(); + assert_eq!( + hex::encode(&encoded), + "9ac3a653796e636564c00708092ac404ed010203b368747470733a2f2f7064732e6578616d706c65aa616c6963652e74657374" + ); + + let decoded: RepoState<'_> = rmp_serde::from_slice(&encoded).unwrap(); + assert!(decoded.active); + assert_eq!(decoded.status, RepoStatus::Synced); + assert_eq!(decoded.last_message_time, Some(7)); + assert_eq!(decoded.last_identity_time, Some(8)); + assert_eq!(decoded.last_account_time, Some(9)); + assert_eq!(decoded.last_updated_at, 42); + assert!(decoded.root.is_none()); + assert_eq!( + decoded.signing_key.as_ref().map(|key| key.0.as_ref()), + Some([0xed, 1, 2, 3].as_slice()) + ); + assert_eq!(decoded.pds.as_deref(), Some("https://pds.example")); + assert_eq!( + decoded.handle.as_ref().map(Handle::as_str), + Some("alice.test") + ); + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -312,3 +312,60 @@ } Ok(buf.freeze()) } + +#[cfg(test)] +mod tests { + use super::*; + use serde::Serialize; + + #[derive(Deserialize, Serialize)] + struct StatusFixture { + #[serde( + serialize_with = "ser_status_code", + deserialize_with = "deser_status_code" + )] + status: Option, + } + + #[derive(Serialize)] + struct OutputFixture<'a> { + #[serde(serialize_with = "opt_cid_serialize_str")] + cid: &'a Option, + #[serde(serialize_with = "did_key_serialize_str")] + did_key: &'a DidKey<'a>, + #[serde(serialize_with = "opt_did_key_serialize_str")] + optional_did_key: &'a Option>, + #[serde(serialize_with = "repo_status_serialize_str")] + status: &'a RepoStatus, + } + + #[test] + fn http_status_serde_contract() { + for status in [None, Some(StatusCode::OK), Some(CONNECTION_TIMEOUT)] { + let encoded = serde_json::to_vec(&StatusFixture { status }).unwrap(); + let decoded: StatusFixture = serde_json::from_slice(&encoded).unwrap(); + assert_eq!(decoded.status, status); + } + assert!(serde_json::from_str::(r#"{"status":99}"#).is_err()); + assert!(serde_json::from_str::(r#"{"status":1000}"#).is_err()); + } + + #[test] + fn xrpc_output_string_serializers_are_stable() { + let cid = Some(jacquard_repo::mst::util::compute_cid(b"body").unwrap()); + let did_key = DidKey(std::borrow::Cow::Borrowed(&[0xed, 1, 2, 3])); + let optional_did_key = Some(did_key.clone()); + let encoded = serde_json::to_value(OutputFixture { + cid: &cid, + did_key: &did_key, + optional_did_key: &optional_did_key, + status: &RepoStatus::Throttled, + }) + .unwrap(); + + assert_eq!(encoded["cid"], cid.unwrap().to_string()); + assert_eq!(encoded["did_key"], did_key.encode()); + assert_eq!(encoded["optional_did_key"], did_key.encode()); + assert_eq!(encoded["status"], "throttled"); + } +} diff --git a/tests/helpers/pds_status.nu b/tests/helpers/pds_status.nu --- a/tests/helpers/pds_status.nu +++ b/tests/helpers/pds_status.nu @@ -18,8 +18,6 @@ fail "hydrant did not start" $instance.pid } - try { bash -c $"fuser -k ($mock_port)/tcp" } catch {} - sleep 100ms print "adding offline mock pds via firehose sources..." http post -t application/json $"($url)/firehose/sources" { @@ -154,8 +152,6 @@ fail "hydrant instance did not start" $instance.pid } - try { bash -c $"fuser -k ($mock_port)/tcp" } catch {} - sleep 100ms http post -t application/json $"($url)/firehose/sources" { url: $"ws://($mock_host):($mock_port)/", diff --git a/src/api/xrpc/com_atproto_describe_repo.rs b/src/api/xrpc/com_atproto_describe_repo.rs --- a/src/api/xrpc/com_atproto_describe_repo.rs +++ b/src/api/xrpc/com_atproto_describe_repo.rs @@ -7,6 +7,23 @@ use super::*; +fn describe_repo_output( + did: jacquard_common::types::did::Did<'static>, + handle: jacquard_common::types::string::Handle<'static>, + handle_is_correct: bool, + did_doc: jacquard_common::types::value::Data<'static>, + collections: Vec>, +) -> DescribeRepoOutput<'static> { + DescribeRepoOutput { + did, + handle, + handle_is_correct, + did_doc, + collections, + extra_data: Default::default(), + } +} + pub async fn handle( State(hydrant): State, ExtractXrpc(req): ExtractXrpc, @@ -40,12 +57,55 @@ repo.collections().map_err(|e| internal_error(nsid, e)), )?; - Ok(Json(DescribeRepoOutput { + Ok(Json(describe_repo_output( did, handle, handle_is_correct, did_doc, - collections: collections.into_keys().collect(), - extra_data: Default::default(), - })) + collections.into_keys().collect(), + ))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn describe_repo_json_projection_contract() { + let output = describe_repo_output( + jacquard_common::types::did::Did::new("did:plc:ewvi7nxzyoun6zhxrhs64oiz").unwrap(), + jacquard_common::types::string::Handle::new("alice.test").unwrap(), + true, + serde_json::from_str( + r##"{ + "id": "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.example" + }] + }"##, + ) + .unwrap(), + vec![jacquard_common::types::nsid::Nsid::new("app.bsky.feed.post").unwrap()], + ); + + assert_eq!( + serde_json::to_value(output).unwrap(), + serde_json::json!({ + "collections": ["app.bsky.feed.post"], + "did": "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + "didDoc": { + "id": "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.example", + }], + }, + "handle": "alice.test", + "handleIsCorrect": true, + }) + ); + } } diff --git a/src/api/xrpc/mod.rs b/src/api/xrpc/mod.rs --- a/src/api/xrpc/mod.rs +++ b/src/api/xrpc/mod.rs @@ -156,7 +156,7 @@ XrpcMethod::Query => { let query = req.uri().query().unwrap_or(""); let res: E::Request<'_> = - serde_urlencoded::from_str(query).map_err(|e| bad_request(nsid, e))?; + serde_html_form::from_str(query).map_err(|e| bad_request(nsid, e))?; Ok(ExtractXrpc(res.into_static())) } XrpcMethod::Procedure(_) => { diff --git a/src/api/xrpc/request_crawl.rs b/src/api/xrpc/request_crawl.rs --- a/src/api/xrpc/request_crawl.rs +++ b/src/api/xrpc/request_crawl.rs @@ -1,3 +1,5 @@ +#[cfg(test)] +use jacquard_api::com_atproto::sync::request_crawl::RequestCrawl; use jacquard_api::com_atproto::sync::request_crawl::{ RequestCrawlError, RequestCrawlRequest, RequestCrawlResponse, }; @@ -58,4 +60,44 @@ .map_err(|e| internal_error(nsid, e))?; Ok(StatusCode::OK) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::response::IntoResponse; + + #[tokio::test] + async fn banned_request_crawl_json_contract() -> miette::Result<()> { + let tmp = tempfile::tempdir().map_err(|e| miette::miette!(e))?; + let hydrant = Hydrant::new(crate::config::Config { + database_path: tmp.path().to_path_buf(), + ..Default::default() + }) + .await?; + hydrant.pds.ban("blocked.example").await?; + + let error = handle( + State(hydrant), + ExtractXrpc::(RequestCrawl { + hostname: "blocked.example".into(), + extra_data: Default::default(), + }), + ) + .await + .expect_err("banned host must be rejected"); + let response = error.into_response(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = axum::body::to_bytes(response.into_body(), 1024) + .await + .map_err(|e| miette::miette!(e))?; + assert_eq!( + serde_json::from_slice::(&body).map_err(|e| miette::miette!(e))?, + serde_json::json!({ + "error": "HostBanned", + "message": "host is banned", + }) + ); + Ok(()) + } } diff --git a/src/control/repos/indexer.rs b/src/control/repos/indexer.rs --- a/src/control/repos/indexer.rs +++ b/src/control/repos/indexer.rs @@ -724,6 +724,13 @@ batch.commit().into_diagnostic()?; let handle = ReposControl(state).get(&did); + let visible_record = handle + .get_record("app.bsky.feed.post", "a") + .await? + .expect("visible record"); + let visible_json = serde_json::to_value(&visible_record.value).into_diagnostic()?; + assert_eq!(visible_json["$type"], "app.bsky.feed.post"); + assert_eq!(visible_json["text"], "visible"); assert!( handle .get_record("app.bsky.feed.post", "b") @@ -820,7 +827,9 @@ batch.commit().into_diagnostic()?; let repos = ReposControl(state.clone()); - let report = repos.drop_repo(&did, crate::types::DeleteBodyTarget::All, true).await?; + let report = repos + .drop_repo(&did, crate::types::DeleteBodyTarget::All, true) + .await?; assert_eq!(report.heads_redacted, 1); assert_eq!(report.history_redacted, 1); assert_eq!(report.buffered_commits_deleted, 1); diff --git a/src/control/repos/mod.rs b/src/control/repos/mod.rs --- a/src/control/repos/mod.rs +++ b/src/control/repos/mod.rs @@ -206,6 +206,62 @@ ); Ok(()) } + + #[test] + fn repo_info_json_contract() { + let cid = jacquard_repo::mst::util::compute_cid(b"root").unwrap(); + let signing_key = DidKey(std::borrow::Cow::Borrowed(&[0xed, 1, 2, 3])); + let signing_key_encoded = signing_key.encode(); + let info = RepoInfo { + did: Did::new_static("did:web:example.com").unwrap(), + status: RepoStatus::Throttled, + tracked: false, + rev: Some(Tid::new("3jzfcijpj2z2a").unwrap()), + data: Some(cid), + handle: Some(Handle::new("alice.test").unwrap()), + pds: Some(Url::parse("https://pds.example").unwrap()), + signing_key: Some(signing_key), + last_updated_at: DateTime::from_timestamp_secs(1_783_785_600), + last_message_at: DateTime::from_timestamp_millis(1_783_785_600_123), + }; + + let encoded = serde_json::to_value(info).unwrap(); + assert_eq!(encoded["did"], "did:web:example.com"); + assert_eq!(encoded["status"], "throttled"); + assert_eq!(encoded["tracked"], false); + assert_eq!(encoded["rev"], "3jzfcijpj2z2a"); + assert_eq!(encoded["data"], cid.to_string()); + assert_eq!(encoded["handle"], "alice.test"); + assert_eq!(encoded["pds"], "https://pds.example/"); + assert_eq!(encoded["signing_key"], signing_key_encoded); + assert_eq!(encoded["last_updated_at"], "2026-07-11T16:00:00Z"); + assert_eq!(encoded["last_message_at"], "2026-07-11T16:00:00.123Z"); + } + + #[cfg(feature = "indexer")] + #[tokio::test] + async fn collections_decodes_big_endian_counts() { + let tmp = tempfile::tempdir().unwrap(); + let state = Arc::new( + AppState::new(&crate::config::Config { + database_path: tmp.path().to_path_buf(), + ..Default::default() + }) + .unwrap(), + ); + let did = Did::new_static("did:web:example.com").unwrap(); + state + .db + .counts + .insert( + keys::count_collection_key(&did, "app.bsky.feed.post"), + 42_u64.to_be_bytes(), + ) + .unwrap(); + + let counts = ReposControl(state).get(&did).collections().await.unwrap(); + assert_eq!(counts[&Nsid::new_static("app.bsky.feed.post").unwrap()], 42); + } } pub struct ListedRecord { diff --git a/src/control/stream/indexer.rs b/src/control/stream/indexer.rs --- a/src/control/stream/indexer.rs +++ b/src/control/stream/indexer.rs @@ -431,22 +431,104 @@ #[test] fn persisted_eight_byte_string_rkey_replays_verbatim() { - let (_tmp, state) = test_state(); - let did = jacquard_common::types::string::Did::new(DID).unwrap(); - let stored = StoredEvent { - live: false, - did: TrimmedDid::from(&did), - rev: rev("3kzbif5moe22m"), - collection: CowStr::Borrowed(COL), - rkey: DbRkey::Str("stinkpot".into()), - action: crate::db::types::DbAction::Create, - data: StoredData::Block(bytes::Bytes::from_static(b"\xa0")), + let tmp = tempfile::tempdir().unwrap(); + let config = Config { + database_path: tmp.path().to_path_buf(), + ..Default::default() }; - let serialized = rmp_serde::to_vec(&stored).unwrap(); - let deserialized: StoredEvent = rmp_serde::from_slice(&serialized).unwrap(); + let event_id = 153_752; - let event = stored_to_event(&state, 153_752, deserialized, None).unwrap(); - assert_eq!(event.record.unwrap().rkey.as_ref(), "stinkpot"); + { + let state = AppState::new(&config).unwrap(); + let did = jacquard_common::types::string::Did::new(DID).unwrap(); + let stored = StoredEvent { + live: false, + did: TrimmedDid::from(&did), + rev: rev("3kzbif5moe22m"), + collection: CowStr::Borrowed(COL), + rkey: DbRkey::Str("stinkpot".into()), + action: crate::db::types::DbAction::Create, + data: StoredData::Block(bytes::Bytes::from_static(b"\xa0")), + }; + let serialized = rmp_serde::to_vec(&stored).unwrap(); + assert_eq!( + hex::encode(&serialized), + "97c2c4100125aa8fb6f9c3a8df64f789e5ee3919c408187cee58e5450012b26170702e62736b792e666565642e706f7374a87374696e6b706f74a643726561746581a5426c6f636bc401a0" + ); + let mut batch = state.db.inner.batch(); + state + .db + .stream + .stage_event(&mut batch, keys::event_key(event_id), serialized); + batch.commit().unwrap(); + state.db.persist().unwrap(); + } + + let state = AppState::new(&config).unwrap(); + let replay = read_event_replay_chunk(&state, None, event_id, 8); + assert_eq!(replay.last_seen_seq, Some(event_id)); + assert_eq!(replay.events.len(), 1); + + let json = serde_json::to_value(&replay.events[0]).unwrap(); + assert_eq!(json["id"], event_id); + assert_eq!(json["type"], "record"); + assert_eq!(json["record"]["did"], DID); + assert_eq!(json["record"]["collection"], COL); + assert_eq!(json["record"]["rkey"], "stinkpot"); + assert_eq!(json["record"]["record"], serde_json::json!({})); + } + + #[test] + fn legacy_string_trimmed_did_replays_after_reopen() { + #[derive(serde::Serialize)] + struct LegacyStoredEvent<'a> { + live: bool, + did: &'a str, + rev: DbTid, + collection: &'a str, + rkey: &'a str, + action: crate::db::types::DbAction, + data: StoredData, + } + + let tmp = tempfile::tempdir().unwrap(); + let config = Config { + database_path: tmp.path().to_path_buf(), + ..Default::default() + }; + let event_id = 153_753; + { + let state = AppState::new(&config).unwrap(); + let stored = LegacyStoredEvent { + live: false, + did: DID.strip_prefix("did:").unwrap(), + rev: rev("3kzbif5moe22m"), + collection: COL, + rkey: "stinkpot", + action: crate::db::types::DbAction::Create, + data: StoredData::Block(bytes::Bytes::from_static(b"\xa0")), + }; + let serialized = rmp_serde::to_vec(&stored).unwrap(); + assert_eq!( + hex::encode(&serialized), + "97c2bc706c633a65777669376e787a796f756e367a687872687336346f697ac408187cee58e5450012b26170702e62736b792e666565642e706f7374a87374696e6b706f74a643726561746581a5426c6f636bc401a0" + ); + let mut batch = state.db.inner.batch(); + state + .db + .stream + .stage_event(&mut batch, keys::event_key(event_id), serialized); + batch.commit().unwrap(); + state.db.persist().unwrap(); + } + + let state = AppState::new(&config).unwrap(); + let replay = read_event_replay_chunk(&state, None, event_id, 1); + assert_eq!(replay.last_seen_seq, Some(event_id)); + assert_eq!(replay.events.len(), 1); + let json = serde_json::to_value(&replay.events[0]).unwrap(); + assert_eq!(json["record"]["did"], DID); + assert_eq!(json["record"]["rkey"], "stinkpot"); } #[test] diff --git a/src/crawler/list_repos/checker.rs b/src/crawler/list_repos/checker.rs --- a/src/crawler/list_repos/checker.rs +++ b/src/crawler/list_repos/checker.rs @@ -360,3 +360,31 @@ .collect()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retry_state_persisted_golden_fixture() { + let state = RetryState { + after: DateTime::parse_from_rfc3339("2026-08-10T12:34:56Z") + .unwrap() + .with_timezone(&Utc), + duration: TimeDelta::seconds(90), + attempts: 3, + status: Some(StatusCode::TOO_MANY_REQUESTS), + }; + let encoded = rmp_serde::to_vec(&state).unwrap(); + assert_eq!( + hex::encode(&encoded), + "94b4323032362d30382d31305431323a33343a35365a925a0003cd01ad" + ); + + let decoded: RetryState = rmp_serde::from_slice(&encoded).unwrap(); + assert_eq!(decoded.after, state.after); + assert_eq!(decoded.duration, state.duration); + assert_eq!(decoded.attempts, 3); + assert_eq!(decoded.status, Some(StatusCode::TOO_MANY_REQUESTS)); + } +} diff --git a/src/db/keys/indexer.rs b/src/db/keys/indexer.rs --- a/src/db/keys/indexer.rs +++ b/src/db/keys/indexer.rs @@ -245,7 +245,10 @@ } pub fn crawler_retry_parse_key(key: &[u8]) -> miette::Result> { - TrimmedDid::try_from(&key[CRAWLER_RETRY_PREFIX.len()..]) + let did = key + .strip_prefix(CRAWLER_RETRY_PREFIX) + .ok_or_else(|| miette::miette!("invalid crawler retry key"))?; + TrimmedDid::try_from(did) } pub const CRAWLER_CURSOR_PREFIX: &[u8] = b"crawler_cursor|"; @@ -446,5 +449,101 @@ let early = history_key(&rec, &DbTid::from(&Tid::new("3kzbif5moe22m").unwrap())); let late = history_key(&rec, &DbTid::from(&Tid::new("3kzbif5mof33m").unwrap())); assert!(early < late); + } + #[test] + fn indexer_key_codec_inventory_contracts() { + let did = did(); + let trimmed = TrimmedDid::from(&did); + let rkey = DbRkey::Str("stinkpot".into()); + let rev = DbTid::from(&Tid::new("3kzbif5moe22m").unwrap()); + + assert_eq!(pending_key(42), 42_u64.to_be_bytes()); + #[cfg(feature = "indexer_stream")] + assert_eq!( + event_watermark_key(42), + [EVENT_WATERMARK_PREFIX, 42_u64.to_be_bytes().as_slice()].concat() + ); + + let record_prefix = record_prefix_trimmed(&trimmed); + assert_eq!(trimmed_did_len(&record_prefix), Some(trimmed.len())); + assert_eq!(record_prefix, record_prefix_did(&did)); + let collection_prefix = record_prefix_collection(&did, "app.bsky.feed.post"); + let record = record_key_trimmed(&trimmed, "app.bsky.feed.post", &rkey); + assert_eq!(record, record_key(&did, "app.bsky.feed.post", &rkey)); + assert!(record.starts_with(&collection_prefix)); + assert_eq!(parse_rkey_text(b"stinkpot").unwrap(), rkey); + assert!(parse_rkey_text(&[0xff]).is_err()); + assert!(split_record_suffix(b"missing-separator").is_err()); + + let history = history_key(&record, &rev); + assert_eq!(history_record_key(&history), Some(record.as_slice())); + assert_eq!(history_record_key(b"short"), None); + + let cid = jacquard_repo::mst::util::compute_cid(b"body").unwrap(); + let redaction = redaction_key(&record, &cid); + assert_eq!( + split_record_cid_key(&redaction), + Some((record.as_slice(), cid)) + ); + assert!(split_record_cid_key(&record).is_none()); + #[cfg(feature = "indexer_stream")] + assert_eq!(event_body_key(&record, &cid), redaction); + + let count_prefix = super::super::did_collection_prefix(&did); + assert_eq!( + count_collection_key(&did, "app.bsky.feed.post"), + [count_prefix.as_slice(), b"app.bsky.feed.post"].concat() + ); + let resync_prefix = resync_buffer_prefix(&did); + assert_eq!( + resync_buffer_key(&did, rev), + [resync_prefix.as_slice(), rev.as_bytes()].concat() + ); + + let retry = crawler_retry_key(&did); + assert_eq!(crawler_retry_parse_key(&retry).unwrap(), trimmed); + assert!(crawler_retry_parse_key(b"bad").is_err()); + assert!(crawler_retry_parse_key(b"nope|did:web:example.com").is_err()); + + assert_eq!( + crawler_cursor_key("https://relay.example"), + b"crawler_cursor|https://relay.example" + ); + assert_eq!( + relay_first_cursor_prefix("https://relay.example"), + b"relay_first_cursor|https://relay.example|" + ); + assert_eq!( + relay_first_pass_key("https://relay.example"), + b"relay_first_cursor|https://relay.example|pass" + ); + assert_eq!( + relay_first_handled_key("https://relay.example"), + b"relay_first_cursor|https://relay.example|handled" + ); + assert_eq!( + relay_first_hosts_cursor_key("https://relay.example"), + b"relay_first_cursor|https://relay.example|hosts" + ); + assert_eq!( + relay_first_pds_cursor_key("https://relay.example", "pds.example"), + b"relay_first_cursor|https://relay.example|pds|pds.example" + ); + assert_eq!( + by_collection_cursor_prefix("https://relay.example"), + b"by_collection_cursor|https://relay.example|" + ); + assert_eq!( + by_collection_cursor_key("https://relay.example", "app.bsky.feed.post"), + b"by_collection_cursor|https://relay.example|app.bsky.feed.post" + ); + assert_eq!( + crawler_source_key("https://relay.example"), + b"src|https://relay.example" + ); + assert_eq!( + block_key("app.bsky.feed.post", &[1, 2, 3]), + b"app.bsky.feed.post|\x01\x02\x03" + ); } } diff --git a/src/db/keys/mod.rs b/src/db/keys/mod.rs --- a/src/db/keys/mod.rs +++ b/src/db/keys/mod.rs @@ -177,3 +177,81 @@ /// value layout: [day: u64 BE][count: u64 BE] = 16 bytes. #[cfg(feature = "relay")] pub const PDS_DAILY_ADDS_KEY: &[u8] = b"pds_daily_adds"; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shared_key_codec_inventory_contracts() { + let did = Did::new("did:plc:ewvi7nxzyoun6zhxrhs64oiz").unwrap(); + let repo = repo_key(&did); + assert_eq!( + repo_metadata_key(&did), + [REPO_METADATA_PREFIX, repo.as_slice()].concat() + ); + + assert_eq!(count_keyspace_key("heads"), b"k|heads"); + let delta = count_delta_key(42, "heads"); + assert_eq!(parse_count_delta_key(&delta).unwrap(), (42, "heads")); + assert_eq!( + count_delta_start_key(42), + &delta[..COUNT_DELTA_PREFIX.len() + 8] + ); + assert!(parse_count_delta_key(b"bad").is_err()); + let mut bad_separator = delta.clone(); + bad_separator[COUNT_DELTA_PREFIX.len() + 8] = b'/'; + assert!(parse_count_delta_key(&bad_separator).is_err()); + let mut bad_utf8 = count_delta_start_key(42); + bad_utf8.extend_from_slice(&[SEP, 0xff]); + assert!(parse_count_delta_key(&bad_utf8).is_err()); + + #[cfg(feature = "indexer")] + assert_eq!( + count_gauge_key(&did), + [COUNT_GAUGE_PREFIX, repo.as_slice()].concat() + ); + assert_eq!( + did_collection_prefix(&did), + [COUNT_COLLECTION_PREFIX, repo.as_slice(), &[SEP]].concat() + ); + assert_eq!( + firehose_source_key("https://relay.example"), + b"firehose|https://relay.example" + ); + assert_eq!(pds_account_count_key("pds.example"), "p|pds.example"); + + assert_eq!( + firehose_cursor_key("relay.example"), + b"firehose_cursor|relay.example" + ); + let url = url::Url::parse("https://relay.example:8443/xrpc").unwrap(); + assert_eq!( + firehose_cursor_key_from_url(&url), + b"firehose_cursor|relay.example:8443" + ); + + #[cfg(any(feature = "indexer_stream", feature = "relay"))] + assert_eq!(event_key(42), 42_u64.to_be_bytes()); + + #[cfg(feature = "relay")] + { + assert_eq!(relay_event_key(42), 42_u64.to_be_bytes()); + assert_eq!( + relay_event_watermark_key(42), + [ + RELAY_EVENT_WATERMARK_PREFIX, + 42_u64.to_be_bytes().as_slice() + ] + .concat() + ); + } + + #[cfg(feature = "jetstream")] + { + let key = jetstream_event_key(42, 7); + assert_eq!(parse_jetstream_event_key(&key).unwrap(), (42, 7)); + assert!(parse_jetstream_event_key(&key[..15]).is_err()); + } + } +} diff --git a/src/db/migration/mod.rs b/src/db/migration/mod.rs --- a/src/db/migration/mod.rs +++ b/src/db/migration/mod.rs @@ -617,6 +617,178 @@ (0..count as u64).map(|i| (i, i)).collect() } + #[cfg(feature = "indexer")] + #[test] + fn v0_database_migrates_all_early_representation_boundaries() -> Result<()> { + use crate::config::Config; + use crate::db::{ + keys, + types::{DbTid, DidKey}, + }; + use crate::types::v2::RepoStatus as V2RepoStatus; + use jacquard_common::CowStr; + use jacquard_common::types::{ + string::{Did, Handle}, + tid::Tid, + }; + + let tmp = tempfile::tempdir().into_diagnostic()?; + let cfg = Config { + database_path: tmp.path().to_path_buf(), + ..Default::default() + }; + let did = Did::new("did:web:migration.test").into_diagnostic()?; + let repo_key = keys::repo_key(&did); + let old_cursor_url = url::Url::parse("wss://relay.example:7443/x").into_diagnostic()?; + let old_cursor_key = [ + keys::FIREHOSE_CURSOR_PREFIX, + old_cursor_url.as_str().as_bytes(), + ] + .concat(); + let new_cursor_key = keys::v1::firehose_cursor_key_from_url(&old_cursor_url); + let relay_source = "wss://bsky.network"; + let pds_source = "wss://pds.example"; + + { + let db = Db::open(&cfg)?; + let old_state = v2::OldRepoState { + status: V2RepoStatus::Error("stinkpot".into()), + rev: Some(DbTid::from(&Tid::new("3jzfcijpj2z2a").unwrap())), + data: Some(jacquard_repo::mst::util::compute_cid(b"root")?), + last_message_time: Some(-1), + last_updated_at: 42, + tracked: true, + index_id: 7, + signing_key: Some(DidKey(std::borrow::Cow::Borrowed(&[0xed, 1, 2, 3]))), + pds: Some(CowStr::Borrowed("https://pds.example")), + handle: Some(Handle::new("alice.test").into_diagnostic()?), + }; + let mut batch = db.inner.batch(); + batch.insert( + &db.repos, + &repo_key, + rmp_serde::to_vec(&old_state).into_diagnostic()?, + ); + batch.insert(&db.cursors, &old_cursor_key, 42i64.to_be_bytes()); + batch.insert(&db.crawler, keys::firehose_source_key(relay_source), []); + batch.insert(&db.crawler, keys::firehose_source_key(pds_source), []); + batch.insert(&db.filter, b"pt|pds.example", b"trusted"); + batch.insert(&db.filter, b"pb|banned.example", []); + batch.insert( + &db.counts, + keys::count_keyspace_key("p|stale.example"), + 99u64.to_be_bytes(), + ); + batch.commit().into_diagnostic()?; + rewind_version_for_test(&db, 0)?; + db.persist()?; + } + + let db = Db::open(&cfg)?; + assert!(!db.cursors.contains_key(&old_cursor_key).into_diagnostic()?); + let cursor = db + .cursors + .get(&new_cursor_key) + .into_diagnostic()? + .expect("v1 cursor"); + assert_eq!( + i64::from_be_bytes(cursor.as_ref().try_into().into_diagnostic()?), + 42 + ); + + for (source, is_pds) in [(relay_source, false), (pds_source, true)] { + let encoded = db + .crawler + .get(keys::firehose_source_key(source)) + .into_diagnostic()? + .expect("v3 source metadata"); + let metadata: crate::db::FirehoseSourceMeta = + rmp_serde::from_slice(&encoded).into_diagnostic()?; + assert_eq!(metadata.is_pds, is_pds); + } + + let state_bytes = db + .repos + .get(&repo_key) + .into_diagnostic()? + .expect("migrated repo state"); + let state = crate::db::deser_repo_state(&state_bytes)?; + assert!(state.active); + assert!(matches!( + &state.status, + crate::types::RepoStatus::Error(error) if error == "stinkpot" + )); + let root = state.root.as_ref().expect("migrated commit"); + assert_eq!(root.version, -1); + assert_eq!(root.rev, DbTid::from(&Tid::new("3jzfcijpj2z2a").unwrap())); + assert_eq!(root.data, jacquard_repo::mst::util::compute_cid(b"root")?); + assert!(root.prev.is_none()); + assert!(root.sig.is_empty()); + assert_eq!(state.last_message_time, Some(-1)); + assert_eq!(state.last_identity_time, None); + assert_eq!(state.last_account_time, None); + assert_eq!(state.last_updated_at, 42); + assert_eq!( + state.signing_key.as_ref().map(|key| key.0.as_ref()), + Some([0xed, 1, 2, 3].as_slice()) + ); + assert_eq!(state.pds.as_deref(), Some("https://pds.example")); + assert_eq!( + state.handle.as_ref().map(Handle::as_str), + Some("alice.test") + ); + let metadata = crate::db::deser_repo_meta( + &db.repo_metadata + .get(keys::repo_metadata_key(&did)) + .into_diagnostic()? + .expect("v4 repo metadata"), + )?; + assert!(metadata.tracked); + assert_eq!(metadata.index_id, 7); + + assert_eq!( + db.filter + .get(crate::db::pds_meta::pds_tier_key("pds.example")) + .into_diagnostic()? + .as_deref(), + Some(b"trusted".as_slice()) + ); + let banned = db + .filter + .get(crate::db::pds_meta::pds_status_key("banned.example")) + .into_diagnostic()? + .expect("v5 banned status"); + assert_eq!( + rmp_serde::from_slice::(&banned).into_diagnostic()?, + crate::pds_meta::HostStatus::Banned + ); + assert!( + !db.counts + .contains_key(keys::count_keyspace_key("p|stale.example")) + .into_diagnostic()? + ); + let account_count = db + .counts + .get(keys::count_keyspace_key("p|pds.example")) + .into_diagnostic()? + .expect("v6 PDS account count"); + assert_eq!( + u64::from_be_bytes(account_count.as_ref().try_into().into_diagnostic()?), + 1 + ); + + let version = db + .counts + .get(VERSIONING_KEY) + .into_diagnostic()? + .expect("schema version"); + assert_eq!( + u64::from_be_bytes(version.as_ref().try_into().into_diagnostic()?), + LATEST_VERSION + ); + Ok(()) + } + #[test] fn chunked_pass_rewrites_every_entry_across_many_chunks() -> Result<()> { let (_tmp, db) = open_db()?; diff --git a/src/db/migration/v2.rs b/src/db/migration/v2.rs --- a/src/db/migration/v2.rs +++ b/src/db/migration/v2.rs @@ -68,3 +68,45 @@ Ok(true) } + +#[cfg(test)] +mod tests { + use super::*; + use jacquard_common::types::tid::Tid; + + #[test] + fn v1_repo_state_input_golden_fixture() { + let state = OldRepoState { + status: RepoStatus::Error("stinkpot".into()), + rev: Some(DbTid::from(&Tid::new("3jzfcijpj2z2a").unwrap())), + data: Some(jacquard_repo::mst::util::compute_cid(b"root").unwrap()), + last_message_time: Some(-1), + last_updated_at: 42, + tracked: true, + index_id: 7, + signing_key: Some(DidKey(std::borrow::Cow::Borrowed(&[0xed, 1, 2, 3]))), + pds: Some(CowStr::Borrowed("https://pds.example")), + handle: Some(Handle::new("alice.test").unwrap()), + }; + let encoded = rmp_serde::to_vec(&state).unwrap(); + assert_eq!( + hex::encode(&encoded), + "9a81a54572726f72a87374696e6b706f74c40817fd6873eaf07c06c424017112204813494d137e1631bba301d5acab6e7bb7aa74ce1185d456565ef51d737677b2ff2ac307c404ed010203b368747470733a2f2f7064732e6578616d706c65aa616c6963652e74657374" + ); + + let decoded: OldRepoState<'_> = rmp_serde::from_slice(&encoded).unwrap(); + assert!(matches!(&decoded.status, RepoStatus::Error(value) if value == "stinkpot")); + assert_eq!(decoded.rev, state.rev); + assert_eq!(decoded.data, state.data); + assert_eq!(decoded.last_message_time, Some(-1)); + assert_eq!(decoded.last_updated_at, 42); + assert!(decoded.tracked); + assert_eq!(decoded.index_id, 7); + assert_eq!(decoded.signing_key, state.signing_key); + assert_eq!(decoded.pds.as_deref(), Some("https://pds.example")); + assert_eq!( + decoded.handle.as_ref().map(Handle::as_str), + Some("alice.test") + ); + } +} diff --git a/src/db/migration/v4.rs b/src/db/migration/v4.rs --- a/src/db/migration/v4.rs +++ b/src/db/migration/v4.rs @@ -87,3 +87,34 @@ Ok(true) } + +#[cfg(test)] +mod tests { + use super::*; + use jacquard_common::types::string::Handle; + + #[test] + fn v2_repo_state_input_golden_fixture() { + let encoded = hex::decode( + "9981a54572726f72a87374696e6b706f749503c40817fd6873eaf07c06c424017112204813494d137e1631bba301d5acab6e7bb7aa74ce1185d456565ef51d737677b2c0c403010203ff2ac307c404ed010203b368747470733a2f2f7064732e6578616d706c65aa616c6963652e74657374", + ) + .unwrap(); + let decoded: OldRepoState<'_> = rmp_serde::from_slice(&encoded).unwrap(); + + assert!(matches!( + &decoded.status, + crate::types::v2::RepoStatus::Error(value) if value == "stinkpot" + )); + assert_eq!(decoded.root.unwrap().sig.as_ref(), b"\x01\x02\x03"); + assert_eq!(decoded.last_message_time, Some(-1)); + assert_eq!(decoded.last_updated_at, 42); + assert!(decoded.tracked); + assert_eq!(decoded.index_id, 7); + assert_eq!(decoded.signing_key.unwrap().0.as_ref(), &[0xed, 1, 2, 3]); + assert_eq!(decoded.pds.as_deref(), Some("https://pds.example")); + assert_eq!( + decoded.handle.as_ref().map(Handle::as_str), + Some("alice.test") + ); + } +} diff --git a/src/db/migration/v7.rs b/src/db/migration/v7.rs --- a/src/db/migration/v7.rs +++ b/src/db/migration/v7.rs @@ -101,9 +101,14 @@ .get(keys::VERSIONING_KEY) .into_diagnostic()? .expect("db version should be set"); + let expected_version = if cfg!(feature = "indexer") { + crate::db::migration::LATEST_VERSION + } else { + 7 + }; assert_eq!( u64::from_be_bytes(version_bytes.as_ref().try_into().into_diagnostic()?), - crate::db::migration::LATEST_VERSION + expected_version ); Ok(()) diff --git a/src/ingest/firehose_stats/relay.rs b/src/ingest/firehose_stats/relay.rs --- a/src/ingest/firehose_stats/relay.rs +++ b/src/ingest/firehose_stats/relay.rs @@ -448,3 +448,30 @@ pub last_seq: Option, pub max_seq: Option, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn relay_worker_stats_snapshot_json_contract() { + let worker = RelayWorkerStats::default(); + assert_eq!( + serde_json::to_value(worker.snapshot()).unwrap(), + serde_json::json!({"shards": []}) + ); + + let shard = worker.shard(7); + shard.record_received(42); + shard.record_info(); + shard.record_message_kind(RelayMessageKind::Commit); + let encoded = serde_json::to_value(worker.snapshot()).unwrap(); + let snapshot = &encoded["shards"][0]; + assert_eq!(snapshot["id"], 7); + assert_eq!(snapshot["received_messages"], 1); + assert_eq!(snapshot["info_messages"], 1); + assert_eq!(snapshot["commit_messages"], 1); + assert_eq!(snapshot["last_seq"], 42); + assert_eq!(snapshot["max_seq"], 42); + } +} diff --git a/src/ingest/firehose_stats/source.rs b/src/ingest/firehose_stats/source.rs --- a/src/ingest/firehose_stats/source.rs +++ b/src/ingest/firehose_stats/source.rs @@ -273,3 +273,50 @@ pub account: u64, pub info: u64, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn firehose_stats_snapshot_json_contract() { + let stats = FirehoseSourceStats::default(); + let empty = serde_json::to_value(stats.snapshot()).unwrap(); + assert_eq!(empty["connection_attempts"], 0); + assert_eq!( + empty["message_kinds"], + serde_json::json!({ + "commit": 0, + "sync": 0, + "identity": 0, + "account": 0, + "info": 0, + }) + ); + for absent in [ + "last_connect_attempt_at", + "last_connected_at", + "last_frame_at", + "last_decoded_at", + "last_forwarded_at", + "last_error_at", + "last_start_cursor", + "last_seq", + "max_seq", + "last_error_kind", + ] { + assert!(empty.get(absent).is_none(), "{absent} was not omitted"); + } + + stats.record_connect_attempt(Some(42)); + stats.record_decoded("commit", Some(73)); + stats.record_connect_error("transport"); + let populated = serde_json::to_value(stats.snapshot()).unwrap(); + assert_eq!(populated["connection_attempts"], 1); + assert_eq!(populated["message_kinds"]["commit"], 1); + assert_eq!(populated["last_start_cursor"], 42); + assert_eq!(populated["last_seq"], 73); + assert_eq!(populated["max_seq"], 73); + assert_eq!(populated["last_error_kind"], "transport"); + } +} diff --git a/src/ingest/indexer/shard.rs b/src/ingest/indexer/shard.rs --- a/src/ingest/indexer/shard.rs +++ b/src/ingest/indexer/shard.rs @@ -613,3 +613,50 @@ Ok(repo_state) } } + +#[cfg(test)] +mod tests { + use jacquard_common::{ + CowStr, + types::{ + cid::CidLink, + string::{Did, Tid}, + }, + }; + + use super::*; + use crate::ingest::stream::{Datetime, RepoOp, RepoOpAction}; + + #[test] + fn resync_buffer_commit_msgpack_round_trip_preserves_path() { + let cid = CidLink::from(jacquard_repo::mst::util::compute_cid(b"commit").unwrap()); + let commit = Commit { + blobs: Vec::new(), + blocks: bytes::Bytes::new(), + commit: cid, + ops: vec![RepoOp { + action: RepoOpAction::Delete, + cid: None, + path: CowStr::Borrowed("app.bsky.feed.post/stinkpot"), + prev: None, + }], + prev_data: None, + rebase: false, + repo: Did::new("did:plc:ewvi7nxzyoun6zhxrhs64oiz").unwrap(), + rev: Tid::new("3kzbif5moe22m").unwrap(), + seq: 42, + since: None, + time: Datetime::try_from("2026-08-10T12:34:56Z".to_owned()).unwrap(), + too_big: false, + }; + + let encoded = rmp_serde::to_vec_named(&commit).unwrap(); + let decoded: Commit = rmp_serde::from_slice(&encoded).unwrap(); + assert_eq!(decoded.repo, commit.repo); + assert_eq!(decoded.rev, commit.rev); + assert_eq!(decoded.seq, commit.seq); + assert_eq!(decoded.ops.len(), 1); + assert_eq!(decoded.ops[0].path.as_ref(), "app.bsky.feed.post/stinkpot"); + assert_eq!(decoded.ops[0].action, RepoOpAction::Delete); + } +} diff --git a/src/ingest/stream/codec.rs b/src/ingest/stream/codec.rs --- a/src/ingest/stream/codec.rs +++ b/src/ingest/stream/codec.rs @@ -93,3 +93,138 @@ ); Ok(bytes::Bytes::from(buf)) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ingest::stream::{Datetime, Identity, Sync}; + use bytes::Bytes; + use jacquard_common::{CowStr, types::string::Did}; + + fn frame(header: EventHeader, body: &T) -> Vec { + let mut bytes = serde_ipld_dagcbor::to_vec(&header).unwrap(); + bytes.extend_from_slice(&serde_ipld_dagcbor::to_vec(body).unwrap()); + bytes + } + + #[test] + fn firehose_frame_fixture_decodes_identity() { + let did = Did::new("did:plc:ewvi7nxzyoun6zhxrhs64oiz").unwrap(); + let body = Identity { + did, + handle: None, + seq: 42, + time: serde_json::from_str::(r#""2026-08-10T12:34:56Z""#).unwrap(), + }; + let bytes = frame( + EventHeader { + op: 1, + t: Some("#identity".to_owned()), + }, + &body, + ); + assert_eq!( + hex::encode(&bytes), + "a2617469236964656e74697479626f7001a36364696478206469643a706c633a65777669376e787a796f756e367a687872687336346f697a63736571182a6474696d6574323032362d30382d31305431323a33343a35365a" + ); + + let decoded = decode_frame(&bytes).unwrap(); + let SubscribeReposMessage::Identity(identity) = decoded else { + panic!("expected identity"); + }; + assert_eq!(identity.did.as_str(), "did:plc:ewvi7nxzyoun6zhxrhs64oiz"); + assert_eq!(identity.seq, 42); + assert_eq!(identity.time.0.to_rfc3339(), "2026-08-10T12:34:56+00:00"); + assert!(identity.handle.is_none()); + } + + #[test] + fn firehose_frame_mock_decodes_sync_binary_blocks() { + let bytes = frame( + EventHeader { + op: 1, + t: Some("#sync".to_owned()), + }, + &Sync { + blocks: Bytes::from_static(b"stinkpot"), + did: Did::new("did:web:example.com").unwrap(), + rev: CowStr::Borrowed("3jzfcijpj2z2a"), + seq: 42, + time: serde_json::from_str::(r#""2026-08-10T12:34:56Z""#).unwrap(), + }, + ); + assert_eq!( + hex::encode(&bytes), + "a26174652373796e63626f7001a563646964736469643a7765623a6578616d706c652e636f6d637265766d336a7a6663696a706a327a326163736571182a6474696d6574323032362d30382d31305431323a33343a35365a66626c6f636b73487374696e6b706f74" + ); + + let SubscribeReposMessage::Sync(sync) = decode_frame(&bytes).unwrap() else { + panic!("expected sync"); + }; + assert_eq!(sync.blocks.as_ref(), b"stinkpot"); + assert_eq!(sync.did.as_str(), "did:web:example.com"); + assert_eq!(sync.rev.as_ref(), "3jzfcijpj2z2a"); + assert_eq!(sync.seq, 42); + assert_eq!(sync.time.0.to_rfc3339(), "2026-08-10T12:34:56+00:00"); + } + + #[test] + fn firehose_frame_rejects_header_and_relay_errors() { + let unknown_op = serde_ipld_dagcbor::to_vec(&EventHeader { + op: 7, + t: Some("#identity".to_owned()), + }) + .unwrap(); + assert!(matches!( + decode_frame(&unknown_op), + Err(FirehoseError::UnknownOp(7)) + )); + + let missing_type = serde_ipld_dagcbor::to_vec(&EventHeader { op: 1, t: None }).unwrap(); + assert!(matches!( + decode_frame(&missing_type), + Err(FirehoseError::MissingType) + )); + + let unknown_type = serde_ipld_dagcbor::to_vec(&EventHeader { + op: 1, + t: Some("#unknown".to_owned()), + }) + .unwrap(); + assert!(matches!( + decode_frame(&unknown_type), + Err(FirehoseError::UnknownType(value)) if value == "#unknown" + )); + + let relay_error = frame( + EventHeader { op: -1, t: None }, + &serde_json::json!({ + "error": "ConsumerTooSlow", + "message": "retry later", + }), + ); + assert!(matches!( + decode_frame(&relay_error), + Err(FirehoseError::RelayError { error, message }) + if error == "ConsumerTooSlow" && message.as_deref() == Some("retry later") + )); + } + + #[test] + fn firehose_frame_maps_outdated_cursor_info() { + let bytes = frame( + EventHeader { + op: 1, + t: Some("#info".to_owned()), + }, + &Info { + message: Some("cursor is in the future".into()), + name: InfoName::OutdatedCursor, + }, + ); + assert!(matches!( + decode_frame(&bytes), + Err(FirehoseError::FutureCursor) + )); + } +} diff --git a/src/ingest/stream/types.rs b/src/ingest/stream/types.rs --- a/src/ingest/stream/types.rs +++ b/src/ingest/stream/types.rs @@ -107,8 +107,32 @@ where D: serde::Deserializer<'de>, { - let s = <&'de str>::deserialize(deserializer)?; - Ok(Self::from(s)) + struct ActionVisitor; + + impl<'de> serde::de::Visitor<'de> for ActionVisitor { + type Value = RepoOpAction<'de>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a repository operation action") + } + + fn visit_str(self, value: &str) -> Result { + Ok(RepoOpAction::from(value.to_owned())) + } + + fn visit_borrowed_str( + self, + value: &'de str, + ) -> Result { + Ok(RepoOpAction::from(value)) + } + + fn visit_string(self, value: String) -> Result { + Ok(RepoOpAction::from(value)) + } + } + + deserializer.deserialize_str(ActionVisitor) } } @@ -285,8 +309,32 @@ where D: serde::Deserializer<'de>, { - let s = <&'de str>::deserialize(deserializer)?; - Ok(Self::from(s)) + struct StatusVisitor; + + impl<'de> serde::de::Visitor<'de> for StatusVisitor { + type Value = AccountStatus<'de>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("an account status") + } + + fn visit_str(self, value: &str) -> Result { + Ok(AccountStatus::from(value.to_owned())) + } + + fn visit_borrowed_str( + self, + value: &'de str, + ) -> Result { + Ok(AccountStatus::from(value)) + } + + fn visit_string(self, value: String) -> Result { + Ok(AccountStatus::from(value)) + } + } + + deserializer.deserialize_str(StatusVisitor) } } @@ -397,8 +445,32 @@ where D: serde::Deserializer<'de>, { - let s = <&'de str>::deserialize(deserializer)?; - Ok(Self::from(s)) + struct NameVisitor; + + impl<'de> serde::de::Visitor<'de> for NameVisitor { + type Value = InfoName<'de>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a subscribeRepos info name") + } + + fn visit_str(self, value: &str) -> Result { + Ok(InfoName::from(value.to_owned())) + } + + fn visit_borrowed_str( + self, + value: &'de str, + ) -> Result { + Ok(InfoName::from(value)) + } + + fn visit_string(self, value: String) -> Result { + Ok(InfoName::from(value)) + } + } + + deserializer.deserialize_str(NameVisitor) } } @@ -446,5 +518,125 @@ SubscribeReposMessage::Sync(s) => &s.did, _ => return None, }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Deserialize)] + struct Since { + #[serde(deserialize_with = "deserialize_tid_or_empty")] + since: Option, + } + + #[test] + fn firehose_scalar_wire_contracts() { + for value in ["create", "update", "delete", "stinkpot"] { + let expected = RepoOpAction::from(value); + let encoded = serde_json::to_vec(&expected).unwrap(); + let decoded: RepoOpAction<'_> = serde_json::from_slice(&encoded).unwrap(); + assert_eq!(decoded, expected); + assert_eq!(RepoOpAction::from(value.to_owned()), expected); + assert_eq!(expected.as_ref(), value); + assert_eq!(expected.to_string(), value); + assert_eq!( + jacquard_common::IntoStatic::into_static(expected.clone()).as_ref(), + value + ); + } + + for value in [ + "takendown", + "suspended", + "deleted", + "deactivated", + "desynchronized", + "throttled", + "stinkpot", + ] { + let expected = AccountStatus::from(value); + let encoded = serde_json::to_vec(&expected).unwrap(); + let decoded: AccountStatus<'_> = serde_json::from_slice(&encoded).unwrap(); + assert_eq!(decoded, expected); + assert_eq!(AccountStatus::from(value.to_owned()), expected); + assert_eq!(expected.as_ref(), value); + assert_eq!(expected.to_string(), value); + assert_eq!( + jacquard_common::IntoStatic::into_static(expected.clone()).as_ref(), + value + ); + } + + for value in ["OutdatedCursor", "stinkpot"] { + let expected = InfoName::from(value); + let encoded = serde_json::to_vec(&expected).unwrap(); + let decoded: InfoName<'_> = serde_json::from_slice(&encoded).unwrap(); + assert_eq!(decoded, expected); + assert_eq!(InfoName::from(value.to_owned()), expected); + assert_eq!(expected.as_ref(), value); + assert_eq!(expected.to_string(), value); + assert_eq!( + jacquard_common::IntoStatic::into_static(expected.clone()).as_ref(), + value + ); + } + + let action: RepoOpAction<'_> = + RepoOpAction::deserialize(serde::de::value::StringDeserializer::< + serde::de::value::Error, + >::new("stinkpot".to_owned())) + .unwrap(); + assert_eq!(action, RepoOpAction::Other("stinkpot".into())); + let status: AccountStatus<'_> = + AccountStatus::deserialize(serde::de::value::StringDeserializer::< + serde::de::value::Error, + >::new("stinkpot".to_owned())) + .unwrap(); + assert_eq!(status, AccountStatus::Other("stinkpot".into())); + let name: InfoName<'_> = InfoName::deserialize(serde::de::value::StringDeserializer::< + serde::de::value::Error, + >::new("stinkpot".to_owned())) + .unwrap(); + assert_eq!(name, InfoName::Other("stinkpot".into())); + } + + #[test] + fn firehose_datetime_accepts_zoned_and_legacy_unzoned_values() { + let zoned: Datetime = serde_json::from_str(r#""2026-08-10T12:34:56.123+02:00""#).unwrap(); + assert_eq!(zoned.0.to_rfc3339(), "2026-08-10T12:34:56.123+02:00"); + + let unzoned: Datetime = serde_json::from_str(r#""2026-08-10T12:34:56.123""#).unwrap(); + assert_eq!(unzoned.0.to_rfc3339(), "2026-08-10T12:34:56.123+00:00"); + assert_eq!( + jacquard_common::IntoStatic::into_static(zoned.clone()), + zoned + ); + + assert!(serde_json::from_str::(r#""not-a-datetime""#).is_err()); + } + + #[test] + fn firehose_since_accepts_null_empty_and_tid_only() { + assert!( + serde_json::from_str::(r#"{"since":null}"#) + .unwrap() + .since + .is_none() + ); + assert!( + serde_json::from_str::(r#"{"since":""}"#) + .unwrap() + .since + .is_none() + ); + + let tid = serde_json::from_str::(r#"{"since":"3jzfcijpj2z2a"}"#) + .unwrap() + .since + .unwrap(); + assert_eq!(tid.as_str(), "3jzfcijpj2z2a"); + assert!(serde_json::from_str::(r#"{"since":"bad"}"#).is_err()); } } diff --git a/src/types/event/jetstream.rs b/src/types/event/jetstream.rs --- a/src/types/event/jetstream.rs +++ b/src/types/event/jetstream.rs @@ -11,7 +11,7 @@ true } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(bound(deserialize = "'i: 'de"))] pub(crate) enum StoredJetstreamEvent<'i> { #[cfg(feature = "indexer_stream")] @@ -167,6 +167,99 @@ seq, time, }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use jacquard_common::types::string::Did; + + fn did() -> TrimmedDid<'static> { + let did = Did::new("did:plc:ewvi7nxzyoun6zhxrhs64oiz").unwrap(); + TrimmedDid::from(&did).into_static() + } + fn time() -> Datetime { + serde_json::from_str(r#""2026-08-10T12:34:56Z""#).unwrap() + } + + fn assert_fixture(event: &StoredJetstreamEvent<'_>, golden: &str) { + let encoded = rmp_serde::to_vec(event).unwrap(); + assert_eq!(hex::encode(&encoded), golden); + let decoded: StoredJetstreamEvent<'_> = rmp_serde::from_slice(&encoded).unwrap(); + assert_eq!(&decoded, event); + } + + #[test] + fn common_jetstream_event_golden_fixtures() { + assert_fixture( + &StoredJetstreamEvent::Account { + did: did(), + active: false, + status: Some("stinkpot".into()), + seq: 42, + time: time(), + }, + "81a74163636f756e7495c4100125aa8fb6f9c3a8df64f789e5ee3919c2a87374696e6b706f742ab4323032362d30382d31305431323a33343a35365a", + ); + assert_fixture( + &StoredJetstreamEvent::Identity { + did: did(), + handle: Some(Handle::new("alice.test").unwrap()), + seq: 42, + time: time(), + }, + "81a84964656e7469747994c4100125aa8fb6f9c3a8df64f789e5ee3919aa616c6963652e746573742ab4323032362d30382d31305431323a33343a35365a", + ); + } + + #[cfg(feature = "indexer_stream")] + #[test] + fn indexer_jetstream_event_golden_fixture() { + let event = StoredJetstreamEvent::Commit { + did: did(), + collection: "app.bsky.feed.post".into(), + event_id: 42, + live: false, + }; + assert_fixture( + &event, + "81a6436f6d6d697494c4100125aa8fb6f9c3a8df64f789e5ee3919b26170702e62736b792e666565642e706f73742ac2", + ); + assert!(!event.is_live()); + assert_eq!(event.collection(), Some("app.bsky.feed.post")); + } + + #[cfg(feature = "relay")] + #[test] + fn relay_jetstream_event_golden_fixtures() { + for (event, golden) in [ + ( + StoredJetstreamEvent::RelayCommit { + did: did(), + collection: "app.bsky.feed.post".into(), + relay_seq: 42, + op_index: 7, + }, + "81ab52656c6179436f6d6d697494c4100125aa8fb6f9c3a8df64f789e5ee3919b26170702e62736b792e666565642e706f73742a07", + ), + ( + StoredJetstreamEvent::RelayAccount { + did: did(), + relay_seq: 42, + }, + "81ac52656c61794163636f756e7492c4100125aa8fb6f9c3a8df64f789e5ee39192a", + ), + ( + StoredJetstreamEvent::RelayIdentity { + did: did(), + relay_seq: 42, + }, + "81ad52656c61794964656e7469747992c4100125aa8fb6f9c3a8df64f789e5ee39192a", + ), + ] { + assert_fixture(&event, golden); } } } diff --git a/src/types/event/stream.rs b/src/types/event/stream.rs --- a/src/types/event/stream.rs +++ b/src/types/event/stream.rs @@ -135,3 +135,122 @@ pub stored: StoredEvent<'static>, pub inline_block: Option, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stream_event_json_envelopes_cover_every_variant() { + let did = Did::new("did:web:example.com").unwrap(); + let cid = jacquard_repo::mst::util::compute_cid(b"body").unwrap(); + let record = MarshallableEvt { + id: 1, + kind: EventType::Record, + record: Some(RecordEvt { + live: false, + did: did.clone(), + rev: Tid::new("3jzfcijpj2z2a").unwrap(), + collection: Nsid::new("app.bsky.feed.post").unwrap(), + rkey: Rkey::new("stinkpot").unwrap(), + action: CowStr::Borrowed("create"), + record: None, + cid: Some(cid), + }), + identity: None, + account: None, + }; + assert_eq!( + serde_json::to_value(&record).unwrap(), + serde_json::json!({ + "id": 1, + "type": "record", + "record": { + "live": false, + "did": "did:web:example.com", + "rev": "3jzfcijpj2z2a", + "collection": "app.bsky.feed.post", + "rkey": "stinkpot", + "action": "create", + "cid": cid.to_string(), + }, + }) + ); + + let identity = MarshallableEvt { + id: 2, + kind: EventType::Identity, + record: None, + identity: Some(IdentityEvt { + did: did.clone(), + handle: None, + }), + account: None, + }; + assert_eq!( + serde_json::to_value(&identity).unwrap(), + serde_json::json!({ + "id": 2, + "type": "identity", + "identity": {"did": "did:web:example.com"}, + }) + ); + + let account = MarshallableEvt { + id: 3, + kind: EventType::Account, + record: None, + identity: None, + account: Some(AccountEvt { + did, + active: true, + status: None, + }), + }; + assert_eq!( + serde_json::to_value(&account).unwrap(), + serde_json::json!({ + "id": 3, + "type": "account", + "account": { + "did": "did:web:example.com", + "active": true, + }, + }) + ); + } + + #[test] + fn stored_data_persisted_golden_fixtures() { + let cid = jacquard_repo::mst::util::compute_cid(b"body").unwrap(); + let variants = [ + StoredData::Nothing, + StoredData::Ptr(cid), + StoredData::Block(Bytes::from_static(b"stinkpot")), + ]; + let encoded = variants + .iter() + .map(|value| rmp_serde::to_vec(value).unwrap()) + .collect::>(); + assert_eq!( + encoded.iter().map(hex::encode).collect::>(), + [ + "a74e6f7468696e67", + "81a3507472c42401711220230d8358dc8e8890b4c58deeb62912ee2f20357ae92a5cc861b98e68fe31acb5", + "81a5426c6f636bc4087374696e6b706f74", + ] + ); + assert!(matches!( + rmp_serde::from_slice::(&encoded[0]).unwrap(), + StoredData::Nothing + )); + assert!(matches!( + rmp_serde::from_slice::(&encoded[1]).unwrap(), + StoredData::Ptr(decoded) if decoded == cid + )); + assert!(matches!( + rmp_serde::from_slice::(&encoded[2]).unwrap(), + StoredData::Block(decoded) if decoded.as_ref() == b"stinkpot" + )); + } +} diff --git a/src/db/migration/v10/events.rs b/src/db/migration/v10/events.rs --- a/src/db/migration/v10/events.rs +++ b/src/db/migration/v10/events.rs @@ -299,6 +299,7 @@ Ok(()) } + #[cfg(not(feature = "backlinks"))] #[test] fn inline_ephemeral_body_stays_owned_by_its_event() -> Result<()> { let tmp = tempfile::tempdir().into_diagnostic()?; diff --git a/src/ingest/relay/sink/relay.rs b/src/ingest/relay/sink/relay.rs --- a/src/ingest/relay/sink/relay.rs +++ b/src/ingest/relay/sink/relay.rs @@ -295,3 +295,72 @@ live: true, }) } + +#[cfg(all(test, feature = "jetstream"))] +mod tests { + use std::collections::BTreeMap; + + use jacquard_common::{ + CowStr, + types::{ + cid::CidLink, + string::{Did, Tid}, + }, + }; + + use super::*; + use crate::ingest::stream::{Datetime, RepoOp, RepoOpAction}; + + #[test] + fn relay_ephemeral_projection_preserves_record_representation() { + let body = serde_ipld_dagcbor::to_vec(&serde_json::json!({ + "$type": "app.bsky.feed.post", + "text": "stinkpot", + })) + .unwrap(); + let cid = jacquard_repo::mst::util::compute_cid(&body).unwrap(); + let cid_link = CidLink::from(cid); + let did = Did::new("did:plc:ewvi7nxzyoun6zhxrhs64oiz").unwrap(); + let rev: Tid = "3kzbif5moe22m".parse().unwrap(); + let commit = Commit { + blobs: Vec::new(), + blocks: bytes::Bytes::new(), + commit: cid_link.clone(), + ops: vec![RepoOp { + action: RepoOpAction::Create, + cid: Some(cid_link), + path: CowStr::Borrowed("app.bsky.feed.post/stinkpot"), + prev: None, + }], + prev_data: None, + rebase: false, + repo: did, + rev: rev.clone(), + seq: 42, + since: None, + time: Datetime::try_from("2026-08-10T12:34:56Z".to_owned()).unwrap(), + too_big: false, + }; + let car = jacquard_repo::car::reader::ParsedCar { + root: cid, + blocks: BTreeMap::from([(cid, body.into())]), + }; + + let event = + build_relay_commit_ephemeral(&commit, 0, &CowStr::Borrowed("app.bsky.feed.post"), &car) + .unwrap(); + assert_eq!(event.did, commit.repo.as_str()); + assert_eq!(event.rev, rev.as_str()); + assert_eq!(event.operation, "create"); + assert_eq!(event.collection, "app.bsky.feed.post"); + assert_eq!(event.rkey, "stinkpot"); + assert_eq!(event.cid.as_deref(), Some(cid.to_string().as_str())); + assert_eq!( + serde_json::from_str::(event.record.unwrap().get()).unwrap(), + serde_json::json!({ + "$type": "app.bsky.feed.post", + "text": "stinkpot", + }) + ); + } +} -- tangled.sh