From 7c3293dfba99b7b181330a412ca699c47a240a55 Mon Sep 17 00:00:00 2001 From: dawn <90008@klbr.net> Date: Sat, 11 Jul 2026 15:21:49 +0300 Subject: [PATCH] [build] add feature matrix check and fix broken combos tests/feature_matrix.nu cargo-checks all 13 supported feature combinations and requires them error- and warning-free. fixes: - bare indexer / featureless: control/stream engine+types re-export gated (inner #![cfg] removed in favor of declaration-site cfg), Config::default ephemeral fields now cfg(not(relay)), is_pending fallback covers featureless, types/event gated to streaming modes - firehose-diagnostics: FirehoseSourceStats::snapshot visibility - relay: dead imports removed, indexer-only imports gated correctly, RepoState::synced gated any(test, relay) - unused re-exports and test-only items (LATEST_VERSION) tightened --- .beads/interactions.jsonl | 3 ++ AGENTS.md | 1 + src/api/xrpc/mod.rs | 1 + src/backfill/sparse.rs | 5 +- src/backfill/worker/process.rs | 4 +- src/config.rs | 4 +- src/control/hydrant.rs | 1 + src/control/repos/mod.rs | 2 +- src/control/stream.rs | 4 ++ src/control/stream/engine.rs | 2 - src/db/migration/mod.rs | 1 + src/db/migration/v8.rs | 1 + src/db/mod.rs | 1 + src/db/open.rs | 1 + src/ingest/firehose_stats.rs | 7 ++- src/ingest/firehose_stats/source.rs | 2 +- src/ingest/relay/context.rs | 1 + src/ingest/relay/handlers.rs | 5 +- src/state.rs | 1 + src/types.rs | 3 ++ src/types/event.rs | 22 +++++++- tests/feature_matrix.nu | 84 +++++++++++++++++++++++++++++ 22 files changed, 140 insertions(+), 16 deletions(-) create mode 100755 tests/feature_matrix.nu diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl index 4240166..6cab8d1 100644 --- a/.beads/interactions.jsonl +++ b/.beads/interactions.jsonl @@ -32,3 +32,6 @@ {"id":"int-c5142d1c6b1b4ef2d1e46dd4fdce9525","kind":"field_change","created_at":"2026-07-09T19:26:22.639873Z","actor":"dawn","issue_id":"hydrant-oth","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"switched reqwest to native root certificates and added a darwin-compatible devshell; checks and tests pass"}} {"id":"int-832ee20415313678de9d18490fa5ebb8","kind":"field_change","created_at":"2026-07-11T09:40:50.765918Z","actor":"dawn","issue_id":"hydrant-4hk","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented and verified proxy-pool scaling plus sparse probe reuse, one-request auto fast path, parallel getBlocks chunks, zero-copy CAR parsing, and benchmark request accounting."}} {"id":"int-40c64e6034cfb8a0dbcd83d93671f0e1","kind":"field_change","created_at":"2026-07-11T09:40:50.962019Z","actor":"dawn","issue_id":"hydrant-8u5","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented and verified proxy-pool scaling plus sparse probe reuse, one-request auto fast path, parallel getBlocks chunks, zero-copy CAR parsing, and benchmark request accounting."}} +{"id":"int-632f66bb8a0ffe9b50a6f4e98fff17ad","kind":"field_change","created_at":"2026-07-11T11:56:17.599247Z","actor":"dawn","issue_id":"hydrant-76x","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"shared car::validate_block_cids applied to sparse proof + getBlocks; full/ingest paths deduplicated; tests cover mismatch/match/disabled"}} +{"id":"int-2d8452d5606c2f560f0f91ac7bfa74e8","kind":"field_change","created_at":"2026-07-11T12:01:39.629304Z","actor":"dawn","issue_id":"hydrant-37h","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"SparseScanner carries NodeBounds through pending/visited scans; boundary subtrees pruned (test shows one fewer round), fetches deduped by CID, conflicting-bounds reuse rescanned deterministically; 69/69 lib tests"}} +{"id":"int-172481868fa58aa6281c7aa0c152276b","kind":"field_change","created_at":"2026-07-11T12:12:58.883838Z","actor":"dawn","issue_id":"hydrant-9ka.3","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"superseded: keep compile-time feature gates, don't compile all modes in; epic goal is composability/cfg consolidation (hydrant-9ka.2), not runtime mode selection"}} diff --git a/AGENTS.md b/AGENTS.md index b399b1c..8942ae1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,6 +126,7 @@ Hydrant uses multiple `fjall` keyspaces: ### Testing - `nu tests/run_all.nu` - Runs all tests in parallel with automatically assigned free ports. Pass `--skip-creds` to skip tests requiring `.env` credentials or external account fixtures, or `--only=[...]` to run a subset. +- `nu tests/feature_matrix.nu` - Checks every supported cargo feature combination compiles error- and warning-free (`--only [...]` for a subset). Run this after touching any `#[cfg(feature = ...)]` gate or adding/removing items in shared modules. The supported matrix lives in the script: `indexer` and `relay` are mutually exclusive modes; `indexer_stream`/`backlinks` require `indexer`; `jetstream` requires `indexer_stream` or `relay`; the bare no-features core must also build. - `nu tests/api_crawler_sources.nu` - Tests `/crawler/sources` CRUD plus dynamic/configured source restart behavior. - `nu tests/api_firehose_sources.nu` - Tests `/firehose/sources` CRUD behavior. - `nu tests/api_pds_tiers.nu` - Tests PDS tier APIs, tier persistence, and custom rate tiers. diff --git a/src/api/xrpc/mod.rs b/src/api/xrpc/mod.rs index 382f874..0c98b06 100644 --- a/src/api/xrpc/mod.rs +++ b/src/api/xrpc/mod.rs @@ -219,6 +219,7 @@ fn payload_too_large( } } +#[cfg(any(feature = "indexer", feature = "relay"))] fn rate_limited( nsid: &'static str, method: &'static str, diff --git a/src/backfill/sparse.rs b/src/backfill/sparse.rs index bad7ddd..99ceecb 100644 --- a/src/backfill/sparse.rs +++ b/src/backfill/sparse.rs @@ -1,7 +1,9 @@ use crate::backfill::client::ThrottledHttpClient; use crate::backfill::error::BackfillError; use crate::config::{BackfillStrategy, RateTier}; -use crate::db::types::{DbAction, DbRkey, TrimmedDid}; +use crate::db::types::{DbAction, DbRkey}; +#[cfg(feature = "indexer_stream")] +use crate::db::types::TrimmedDid; use crate::db::{self, CountDeltas, keys, ser_repo_state}; use crate::filter::{FilterConfig, FilterMode}; use crate::ops; @@ -25,6 +27,7 @@ use reqwest::StatusCode; use smol_str::{SmolStr, ToSmolStr}; use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; +#[cfg(feature = "indexer_stream")] use std::sync::atomic::Ordering; use tracing::{debug, trace, warn}; diff --git a/src/backfill/worker/process.rs b/src/backfill/worker/process.rs index a3dfd86..f18c48f 100644 --- a/src/backfill/worker/process.rs +++ b/src/backfill/worker/process.rs @@ -19,7 +19,9 @@ use crate::backfill::client::ThrottledHttpClient; use crate::backfill::error::BackfillError; use crate::backfill::sparse::{SparseBackfillResult, process_did_sparse}; use crate::config::BackfillStrategy; -use crate::db::types::{DbAction, DbRkey, TrimmedDid}; +use crate::db::types::{DbAction, DbRkey}; +#[cfg(feature = "indexer_stream")] +use crate::db::types::TrimmedDid; use crate::db::{self, CountDeltas, Db, keys, ser_repo_state}; use crate::filter::FilterMode; use crate::ops; diff --git a/src/config.rs b/src/config.rs index 4d3affa..98a4eba 100644 --- a/src/config.rs +++ b/src/config.rs @@ -260,11 +260,11 @@ impl Default for Config { const BASE_MEMTABLE_MB: u64 = 32; Self { database_path: PathBuf::from("./hydrant.db"), - #[cfg(feature = "indexer")] + #[cfg(not(feature = "relay"))] ephemeral: false, #[cfg(feature = "relay")] ephemeral: true, - #[cfg(feature = "indexer")] + #[cfg(not(feature = "relay"))] ephemeral_ttl: Duration::from_secs(3600), // 1 hour #[cfg(feature = "relay")] ephemeral_ttl: Duration::from_secs(3600 * 24 * 3), // 3 days diff --git a/src/control/hydrant.rs b/src/control/hydrant.rs index 9430f58..2206b04 100644 --- a/src/control/hydrant.rs +++ b/src/control/hydrant.rs @@ -61,6 +61,7 @@ impl Hydrant { &self.state.resolver } + #[cfg(any(feature = "indexer_stream", feature = "relay", feature = "jetstream"))] pub(crate) fn stream_send_timeout(&self) -> std::time::Duration { self.config.stream_send_timeout } diff --git a/src/control/repos/mod.rs b/src/control/repos/mod.rs index 72c1ff1..71c3760 100644 --- a/src/control/repos/mod.rs +++ b/src/control/repos/mod.rs @@ -368,7 +368,7 @@ impl<'i> RepoHandle<'i> { .map_err(|e| MiniDocError::Other(miette::miette!(e)))? .map_err(MiniDocError::Other)? }; - #[cfg(feature = "relay")] + #[cfg(not(feature = "indexer"))] let is_pending = false; if is_pending { diff --git a/src/control/stream.rs b/src/control/stream.rs index 51c56ff..e6cdc66 100644 --- a/src/control/stream.rs +++ b/src/control/stream.rs @@ -1,4 +1,6 @@ +#[cfg(any(feature = "indexer_stream", feature = "relay", feature = "jetstream"))] pub(crate) mod engine; +#[cfg(any(feature = "indexer_stream", feature = "relay", feature = "jetstream"))] pub(crate) mod types; #[cfg(feature = "indexer_stream")] @@ -20,5 +22,7 @@ pub(crate) use jetstream::{ JetstreamAccount, JetstreamCommit, JetstreamEvent, JetstreamIdentity, JetstreamPayload, }; +#[cfg(any(feature = "indexer_stream", feature = "relay", feature = "jetstream"))] pub(crate) use engine::*; +#[cfg(any(feature = "indexer_stream", feature = "relay", feature = "jetstream"))] pub(crate) use types::*; diff --git a/src/control/stream/engine.rs b/src/control/stream/engine.rs index 994844b..46e67e6 100644 --- a/src/control/stream/engine.rs +++ b/src/control/stream/engine.rs @@ -1,5 +1,3 @@ -#![cfg(any(feature = "indexer_stream", feature = "relay", feature = "jetstream"))] - use std::fmt; use std::num::NonZeroUsize; use std::time::{Duration, Instant}; diff --git a/src/db/migration/mod.rs b/src/db/migration/mod.rs index ca02e15..e5477b8 100644 --- a/src/db/migration/mod.rs +++ b/src/db/migration/mod.rs @@ -43,6 +43,7 @@ const MIGRATIONS: &[(&str, MigrationFn)] = &[ ("migrate_excludes_and_pds_keys", v9::migrate_v9), ]; +#[cfg(test)] pub(crate) const LATEST_VERSION: u64 = MIGRATIONS.len() as u64; fn read_version(db: &Db) -> Result { diff --git a/src/db/migration/v8.rs b/src/db/migration/v8.rs index c04599e..0d2e537 100644 --- a/src/db/migration/v8.rs +++ b/src/db/migration/v8.rs @@ -123,6 +123,7 @@ fn is_lifecycle_count(name: &str) -> bool { #[cfg(test)] mod tests { + #[cfg(feature = "indexer")] use super::*; #[cfg(feature = "indexer")] diff --git a/src/db/mod.rs b/src/db/mod.rs index f677ab9..2ee4ddb 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -130,6 +130,7 @@ impl Db { .into_diagnostic()? }; + #[cfg_attr(not(any(feature = "indexer", feature = "indexer_stream", feature = "jetstream", feature = "relay", feature = "backlinks")), allow(unused_mut))] let mut tasks = vec![ compact(self.repos.clone()), compact(self.cursors.clone()), diff --git a/src/db/open.rs b/src/db/open.rs index af6947d..de850ab 100644 --- a/src/db/open.rs +++ b/src/db/open.rs @@ -11,6 +11,7 @@ use std::collections::BTreeSet; use std::sync::atomic::AtomicI64; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; +#[cfg(any(feature = "indexer_stream", feature = "relay", feature = "jetstream"))] use tokio::sync::broadcast; use crate::config::{Compression, Config}; diff --git a/src/ingest/firehose_stats.rs b/src/ingest/firehose_stats.rs index d7cc208..497bb7d 100644 --- a/src/ingest/firehose_stats.rs +++ b/src/ingest/firehose_stats.rs @@ -2,11 +2,10 @@ mod relay; mod source; pub use relay::{ - HostAuthorityStatsOutcome, RelayMessageKind, RelayShardStats, RelayShardStatsSnapshot, - RelayShardTimings, RelayWorkerStats, RelayWorkerStatsSnapshot, RepoStateLoadOutcome, - ValidationStatsOutcome, + HostAuthorityStatsOutcome, RelayMessageKind, RelayShardStats, RelayShardTimings, + RelayWorkerStats, RelayWorkerStatsSnapshot, RepoStateLoadOutcome, ValidationStatsOutcome, }; -pub use source::{FirehoseMessageStats, FirehoseSourceStats, FirehoseStats, FirehoseStatsSnapshot}; +pub use source::{FirehoseSourceStats, FirehoseStats, FirehoseStatsSnapshot}; use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; use std::time::Duration; diff --git a/src/ingest/firehose_stats/source.rs b/src/ingest/firehose_stats/source.rs index 09993a2..c72190a 100644 --- a/src/ingest/firehose_stats/source.rs +++ b/src/ingest/firehose_stats/source.rs @@ -175,7 +175,7 @@ impl FirehoseSourceStats { *self.last_error_kind.lock() = Some(kind); } - fn snapshot(&self) -> FirehoseStatsSnapshot { + pub(super) fn snapshot(&self) -> FirehoseStatsSnapshot { FirehoseStatsSnapshot { connection_attempts: self.load_u64(&self.connection_attempts), successful_connections: self.load_u64(&self.successful_connections), diff --git a/src/ingest/relay/context.rs b/src/ingest/relay/context.rs index 2dcfe8f..4dcae3c 100644 --- a/src/ingest/relay/context.rs +++ b/src/ingest/relay/context.rs @@ -25,6 +25,7 @@ use super::{ WRONG_HOST_AUTHORITY_RECHECK_INTERVAL, WorkerMessage, map_repo_status_probe, }; use crate::ingest::stream::AccountStatus; +#[cfg(feature = "indexer")] use crate::ingest::stream::SubscribeReposMessage; use crate::ingest::validation::{ CommitValidationError, SyncValidationError, ValidatedCommit, ValidatedSync, ValidationContext, diff --git a/src/ingest/relay/handlers.rs b/src/ingest/relay/handlers.rs index 45199bd..b4303f2 100644 --- a/src/ingest/relay/handlers.rs +++ b/src/ingest/relay/handlers.rs @@ -15,13 +15,12 @@ use crate::db::types::TrimmedDid; use crate::ingest::stream::encode_frame; use crate::ingest::stream::{Account, AccountStatus, Commit, Identity, Sync}; use crate::ingest::validation::ValidatedCommit; -#[cfg(feature = "relay")] -use crate::types::RelayBroadcast; +#[cfg(feature = "indexer")] +use jacquard_common::IntoStatic; #[cfg(all(feature = "relay", feature = "jetstream"))] use crate::types::StoredJetstreamEvent; #[cfg(all(feature = "relay", feature = "jetstream"))] use jacquard_common::CowStr; -use jacquard_common::IntoStatic; use super::{RelayWorker, WorkerContext}; diff --git a/src/state.rs b/src/state.rs index 5e6c7da..760e014 100644 --- a/src/state.rs +++ b/src/state.rs @@ -222,6 +222,7 @@ impl AppState { .is_some_and(|l| count >= l) } + #[cfg(feature = "indexer")] pub(crate) fn resolve_pds_tier(&self, host: &str) -> crate::config::RateTier { let meta = self.pds_meta.load(); let override_name = meta.hosts.get(host).and_then(|h| h.tier.as_ref()); diff --git a/src/types.rs b/src/types.rs index 9b3b928..ccacb76 100644 --- a/src/types.rs +++ b/src/types.rs @@ -9,11 +9,13 @@ use smol_str::ToSmolStr; use crate::db::types::DbTid; use crate::resolver::MiniDoc; +#[cfg(any(feature = "indexer_stream", feature = "relay", feature = "jetstream"))] pub(crate) mod event; pub(crate) mod v2; pub(crate) mod v4; pub(crate) mod v7; +#[cfg(any(feature = "indexer_stream", feature = "relay", feature = "jetstream"))] pub(crate) use event::*; pub(crate) use v7::*; @@ -136,6 +138,7 @@ impl<'i> RepoState<'i> { } } + #[cfg(any(test, feature = "relay"))] pub fn synced() -> Self { Self { status: RepoStatus::Synced, diff --git a/src/types/event.rs b/src/types/event.rs index f95898e..769705c 100644 --- a/src/types/event.rs +++ b/src/types/event.rs @@ -1,13 +1,26 @@ +#[cfg(feature = "indexer_stream")] use bytes::Bytes; #[cfg(feature = "jetstream")] use jacquard_common::IntoStatic; +#[cfg(feature = "indexer_stream")] use jacquard_common::types::cid::IpldCid; +#[cfg(feature = "indexer_stream")] use jacquard_common::types::nsid::Nsid; +#[cfg(feature = "indexer_stream")] use jacquard_common::types::string::{Did, Rkey}; +#[cfg(feature = "indexer_stream")] use jacquard_common::types::tid::Tid; +#[cfg(any(feature = "indexer_stream", feature = "jetstream"))] use jacquard_common::{CowStr, types::string::Handle}; -use serde::{Deserialize, Serialize, Serializer}; +#[cfg(any(feature = "indexer_stream", feature = "jetstream"))] +use serde::Serialize; +#[cfg(feature = "indexer_stream")] +use serde::Serializer; +#[cfg(any(feature = "indexer_stream", feature = "jetstream"))] +use serde::Deserialize; +#[cfg(feature = "indexer_stream")] use serde_json::value::RawValue; +#[cfg(any(feature = "indexer_stream", feature = "jetstream"))] use std::fmt::Debug; #[cfg(any(feature = "indexer_stream", feature = "jetstream"))] @@ -17,6 +30,7 @@ use crate::db::types::{DbAction, DbRkey, DbTid}; #[cfg(feature = "jetstream")] use crate::ingest::stream::Datetime; +#[cfg(feature = "indexer_stream")] #[derive(Debug, Serialize, Clone)] pub enum EventType { Record, @@ -24,6 +38,7 @@ pub enum EventType { Account, } +#[cfg(feature = "indexer_stream")] impl AsRef for EventType { fn as_ref(&self) -> &str { match self { @@ -34,10 +49,12 @@ impl AsRef for EventType { } } +#[cfg(feature = "indexer_stream")] fn event_type_ser_str(v: &EventType, s: S) -> Result { s.serialize_str(v.as_ref()) } +#[cfg(feature = "indexer_stream")] #[derive(Debug, Serialize, Clone)] pub struct MarshallableEvt<'i> { pub id: u64, @@ -55,6 +72,7 @@ pub struct MarshallableEvt<'i> { pub account: Option>, } +#[cfg(feature = "indexer_stream")] #[derive(Debug, Serialize, Clone)] pub struct RecordEvt<'i> { pub live: bool, @@ -71,6 +89,7 @@ pub struct RecordEvt<'i> { pub cid: Option, } +#[cfg(feature = "indexer_stream")] #[derive(Debug, Serialize, Clone)] pub struct IdentityEvt<'i> { #[serde(borrow)] @@ -79,6 +98,7 @@ pub struct IdentityEvt<'i> { pub handle: Option>, } +#[cfg(feature = "indexer_stream")] #[derive(Debug, Serialize, Clone)] pub struct AccountEvt<'i> { #[serde(borrow)] diff --git a/tests/feature_matrix.nu b/tests/feature_matrix.nu new file mode 100755 index 0000000..1b51911 --- /dev/null +++ b/tests/feature_matrix.nu @@ -0,0 +1,84 @@ +#!/usr/bin/env nu +# check every supported cargo feature combination compiles without errors or warnings. +# +# usage: +# nu tests/feature_matrix.nu +# nu tests/feature_matrix.nu --only [relay indexer] + +# supported combinations. indexer and relay are mutually exclusive modes +# (enforced by compile_error! in src/lib.rs); everything else composes on +# top of exactly one mode (or none, for the bare shared core). +const combos = [ + { name: "none", features: "" } + { name: "indexer", features: "indexer" } + { name: "default", features: "indexer,indexer_stream" } + { name: "indexer-jetstream", features: "indexer,indexer_stream,jetstream" } + { name: "indexer-backlinks", features: "indexer,indexer_stream,backlinks" } + { name: "indexer-backlinks-nostream", features: "indexer,backlinks" } + { name: "indexer-user-keyspace", features: "indexer,indexer_stream,user-keyspace" } + { name: "indexer-diagnostics", features: "indexer,indexer_stream,firehose-diagnostics" } + { name: "indexer-persist-sync-all", features: "indexer,indexer_stream,__persist_sync_all" } + { name: "relay", features: "relay" } + { name: "relay-jetstream", features: "relay,jetstream" } + { name: "relay-diagnostics", features: "relay,firehose-diagnostics" } + { name: "relay-user-keyspace", features: "relay,user-keyspace" } +] + +def check-combo [combo: record] { + mut args = [check --all-targets --no-default-features] + if ($combo.features | is-not-empty) { + $args = $args | append [--features $combo.features] + } + + let result = (cargo ...$args | complete) + let diagnostics = $result.stderr + | lines + | where {|line| + ($line starts-with "error") or ( + ($line starts-with "warning:") + and not ($line | str contains "generated") + and not ($line | str contains "build failed") + ) + } + + { + name: $combo.name + features: $combo.features + success: ($result.exit_code == 0 and ($diagnostics | is-empty)) + diagnostics: $diagnostics + stderr: $result.stderr + } +} + +def main [--only: list = []] { + let selected = if ($only | is-empty) { + $combos + } else { + $combos | where {|c| $c.name in $only } + } + if ($selected | is-empty) { + print $"no combos match ($only); known: ($combos | get name | str join ', ')" + exit 1 + } + + mut failures = [] + for combo in $selected { + print $"checking ($combo.name) [($combo.features)]..." + let result = (check-combo $combo) + if not $result.success { + print $" FAILED:" + for line in $result.diagnostics { + print $" ($line)" + } + $failures = $failures | append $result + } + } + + print "" + if ($failures | is-empty) { + print $"feature matrix OK: ($selected | length) combos error- and warning-free" + } else { + print $"feature matrix FAILED: ($failures | get name | str join ', ')" + exit 1 + } +} -- 2.51.2