diff --git a/.gitignore b/.gitignore index 303a859..0315cbf 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ node_modules/ /src/env.rs # Local databases (SQLite projection + Hydrant fjall store). /data/ +/hydrant.db/ \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 31ec519..27470c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,6 +94,7 @@ tokio = { version = "1", features = ["rt", "macros"] } [build-dependencies] dotenvy = "0.15" +serde_json = "1.0" [profile.web-release] inherits = "release" diff --git a/build.rs b/build.rs index bf94749..ffddcac 100644 --- a/build.rs +++ b/build.rs @@ -8,9 +8,11 @@ //! here: the server binary loads `.env` directly via `dotenvy` and reads it into //! a typed `ServerConfig` struct (see `src/indexing/config.rs`). +use serde_json::Value; use std::env; -use std::fs::File; +use std::fs::{self, File}; use std::io::Write; +use std::path::{Path, PathBuf}; fn main() { println!("cargo:rerun-if-changed=.env"); @@ -21,7 +23,7 @@ fn main() { f.write_all( b"// This file is automatically generated by build.rs.\n\ // Frontend (WASM) compile-time configuration.\n\ - // Do not edit; regenerate by rebuilding.\n\n", + // Do not edit; regenerate by rebuilding.\n", ) .expect("write env.rs header"); @@ -39,4 +41,54 @@ fn main() { f.write_all(line.as_bytes()).expect("write env.rs const"); } } + generate_indexed_collections(); +} +fn generate_indexed_collections() { + println!("cargo:rerun-if-changed=lexicons"); + let lexicon_root = Path::new("lexicons"); + let mut json_paths = Vec::new(); + collect_json_paths(lexicon_root, &mut json_paths); + json_paths.sort(); + let mut collections = Vec::new(); + for path in json_paths { + println!("cargo:rerun-if-changed={}", path.display()); + let contents = fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("read authored lexicon {}: {e}", path.display())); + let lexicon: Value = serde_json::from_str(&contents) + .unwrap_or_else(|e| panic!("parse authored lexicon JSON {}: {e}", path.display())); + if lexicon.pointer("/defs/main/type").and_then(Value::as_str) != Some("record") { + continue; + } + let id = lexicon + .get("id") + .and_then(Value::as_str) + .unwrap_or_else(|| panic!("record lexicon {} has no string id", path.display())); + if id.starts_with("space.polymodel.") { + collections.push(id.to_owned()); + } + } + collections.sort(); + let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR set by Cargo")); + let out_path = out_dir.join("polymodel_record_collections.rs"); + let mut out = File::create(&out_path) + .unwrap_or_else(|e| panic!("create generated collections {}: {e}", out_path.display())); + writeln!(out, "// Generated by build.rs from authored lexicons where defs.main.type == \"record\".\n// Do not edit.\npub const INDEXED_COLLECTIONS: &[&str] = &[").expect("write generated collections header"); + for collection in collections { + writeln!(out, " {collection:?},").expect("write generated collection"); + } + writeln!(out, "];").expect("write generated collections footer"); +} +fn collect_json_paths(dir: &Path, paths: &mut Vec) { + let entries = + fs::read_dir(dir).unwrap_or_else(|e| panic!("read lexicon dir {}: {e}", dir.display())); + for entry in entries { + let entry = + entry.unwrap_or_else(|e| panic!("read lexicon dir entry in {}: {e}", dir.display())); + let path = entry.path(); + if path.is_dir() { + collect_json_paths(&path, paths); + } else if path.extension().and_then(|ext| ext.to_str()) == Some("json") { + paths.push(path); + } + } } diff --git a/src/indexing/setup.rs b/src/indexing/setup.rs index 1051367..f48ebdb 100644 --- a/src/indexing/setup.rs +++ b/src/indexing/setup.rs @@ -1,16 +1,18 @@ //! Hydrant ATProto indexer setup and configuration. //! -//! Initializes Hydrant from environment and applies a signal-mode filter so only -//! repos that publish `space.polymodel.*` records are indexed. The returned -//! handle is cheaply cloneable; the firehose/backfill driver is started later via -//! [`Hydrant::run`] in [`super::start_indexing`]. +//! Initializes Hydrant from environment and applies exact Polymodel record +//! collection filters. Hydrant's by-collection crawler sends signal entries +//! directly to `listReposByCollection`, so wildcard NSIDs must not be used here. +//! The returned handle is cheaply cloneable; the firehose/backfill driver is started later via [`Hydrant::run`] in [`super::start_indexing`]. use hydrant::FilterMode; use hydrant::control::Hydrant; - -/// NSID patterns to index: the whole Polymodel namespace, used for both -/// signal-mode discovery and the collection allowlist. -const COLLECTIONS: [&str; 1] = ["space.polymodel.*"]; +mod generated_collections { + include!(concat!(env!("OUT_DIR"), "/polymodel_record_collections.rs")); +} +/// Exact authored Polymodel repo record collections used for Hydrant discovery +/// and storage. Keep this exact because Hydrant sends signal entries to listReposByCollection. +const INDEXED_COLLECTIONS: &[&str] = generated_collections::INDEXED_COLLECTIONS; /// Initialize Hydrant from environment and configure signal-mode filtering. /// @@ -21,18 +23,73 @@ pub async fn init_hydrant() -> anyhow::Result { .await .map_err(|e| anyhow::anyhow!("hydrant init failed: {e}"))?; - // Signal mode: only index repos that emit space.polymodel.* records, and - // only store those collections. Code takes precedence over HYDRANT_FILTER_* - // env vars, but the env vars are still documented in .env.example. + // Hydrant signal mode discovers repos publishing authored Polymodel records. + // In Hydrant's default ByCollection crawler, signals are also the exact + // values sent to listReposByCollection, so do not use wildcard patterns. hydrant .filter .set_mode(FilterMode::Filter) - .set_signals(COLLECTIONS) - .set_collections(COLLECTIONS) + .set_signals(INDEXED_COLLECTIONS.iter().copied()) + .set_collections(INDEXED_COLLECTIONS.iter().copied()) .apply() .await .map_err(|e| anyhow::anyhow!("hydrant filter apply failed: {e}"))?; - tracing::info!("hydrant initialized with signal filter for space.polymodel.*"); + tracing::info!( + indexed_collections = INDEXED_COLLECTIONS.len(), + "hydrant initialized with exact authored record collection allowlist" + ); Ok(hydrant) } +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn hydrant_filters_use_exact_record_collections() { + assert_eq!( + INDEXED_COLLECTIONS, + generated_collections::INDEXED_COLLECTIONS + ); + for collection in INDEXED_COLLECTIONS { + assert!( + !collection.contains('*'), + "indexed collection must be exact, got {collection}" + ); + assert!( + !collection.ends_with(".*"), + "indexed collection must not be a wildcard, got {collection}" + ); + } + let expected_records = [ + "space.polymodel.actor.profile", + "space.polymodel.graph.like", + "space.polymodel.graph.list", + "space.polymodel.graph.listitem", + "space.polymodel.graph.save", + "space.polymodel.graph.tag", + "space.polymodel.library.model", + "space.polymodel.library.part", + "space.polymodel.library.thing", + ]; + for collection in expected_records { + assert!( + generated_collections::INDEXED_COLLECTIONS.contains(&collection), + "missing generated record collection {collection}" + ); + } + let excluded_non_records = [ + "space.polymodel.library.getFeed", + "space.polymodel.library.getThing", + "space.polymodel.library.getAuthorThings", + "space.polymodel.library.searchThings", + "space.polymodel.graph.getList", + "space.polymodel.library.defs", + ]; + for collection in excluded_non_records { + assert!( + !generated_collections::INDEXED_COLLECTIONS.contains(&collection), + "non-record lexicon must not be indexed as a collection: {collection}" + ); + } + } +}