diff --git a/Cargo.lock b/Cargo.lock index 1fc7357..a7e96a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1290,7 +1290,6 @@ dependencies = [ "chrono", "clap", "glob", - "hickory-resolver", "miette", "mlf-codegen", "mlf-codegen-go", @@ -1298,12 +1297,14 @@ dependencies = [ "mlf-codegen-typescript", "mlf-diagnostics", "mlf-lang", + "mlf-lexicon-fetcher", "mlf-validation", "reqwest", "serde", "serde_json", "sha2", "thiserror 2.0.17", + "tokio", "toml", ] @@ -1358,8 +1359,10 @@ dependencies = [ "mlf-codegen", "mlf-diagnostics", "mlf-lang", + "mlf-lexicon-fetcher", "serde", "serde_json", + "tokio", "toml", ] @@ -1375,6 +1378,19 @@ dependencies = [ "toml", ] +[[package]] +name = "mlf-lexicon-fetcher" +version = "0.1.0" +dependencies = [ + "async-trait", + "hickory-resolver", + "reqwest", + "serde", + "serde_json", + "thiserror 2.0.17", + "tokio", +] + [[package]] name = "mlf-lsp" version = "0.1.0" @@ -1801,9 +1817,7 @@ dependencies = [ "base64", "bytes", "encoding_rs", - "futures-channel", "futures-core", - "futures-util", "h2", "http", "http-body", diff --git a/Cargo.toml b/Cargo.toml index bbedee1..f20a7f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "mlf-cli", "mlf-codegen", "mlf-diagnostics", + "mlf-lexicon-fetcher", "mlf-lang", "mlf-lsp", "mlf-validation", "mlf-wasm", diff --git a/justfile b/justfile index 56f49eb..f541055 100644 --- a/justfile +++ b/justfile @@ -5,7 +5,7 @@ default: test # Run all tests (excluding problematic packages) -test: test-lang test-codegen test-diagnostics test-validation +test: test-lang test-codegen test-diagnostics test-lexicon-fetcher test-validation # Run only language tests (mlf-lang crate) test-lang: @@ -22,6 +22,11 @@ test-diagnostics: @echo "\nRunning diagnostics integration tests..." cargo test -p mlf-integration-tests --test diagnostics_integration -- --nocapture +# Run lexicon fetcher tests +test-lexicon-fetcher: + @echo "\nRunning lexicon fetcher tests..." + cargo test -p mlf-lexicon-fetcher -- --nocapture + # Run validation tests test-validation: @echo "\nRunning validation tests..." @@ -78,12 +83,13 @@ clean: # Show test statistics test-stats: @echo "Test Statistics:" - @echo " Lang tests: 21 tests in mlf-lang/tests/lang/" - @echo " Codegen tests: 10 tests in tests/codegen/lexicon/" - @echo " Diagnostics tests: 1 test in tests/diagnostics/" - @echo " Validation tests: 12 tests in mlf-validation" + @echo " Lang tests: 21 tests in mlf-lang/tests/lang/" + @echo " Codegen tests: 10 tests in tests/codegen/lexicon/" + @echo " Diagnostics tests: 1 test in tests/diagnostics/" + @echo " Lexicon fetcher tests: 33 tests in mlf-lexicon-fetcher/" + @echo " Validation tests: 12 tests in mlf-validation" @echo "" - @echo "Total integration tests: 44" + @echo "Total integration tests: 77" # List all test directories test-list: diff --git a/mlf-cli/Cargo.toml b/mlf-cli/Cargo.toml index 6219b41..3c6ecbc 100644 --- a/mlf-cli/Cargo.toml +++ b/mlf-cli/Cargo.toml @@ -13,6 +13,7 @@ mlf-lang = { path = "../mlf-lang" } mlf-validation = { path = "../mlf-validation" } mlf-codegen = { path = "../mlf-codegen" } mlf-diagnostics = { path = "../mlf-diagnostics" } +mlf-lexicon-fetcher = { path = "../mlf-lexicon-fetcher" } clap = { version = "4.5.48", features = ["derive"] } miette = { version = "7", features = ["fancy"] } thiserror = "2" @@ -20,9 +21,9 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" glob = "0.3" toml = "0.8" -reqwest = { version = "0.12", features = ["blocking", "json"] } +tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +reqwest = { version = "0.12", features = ["json"] } chrono = { version = "0.4", features = ["serde"] } -hickory-resolver = "0.24" sha2 = "0.10" # Optional code generator plugins diff --git a/mlf-cli/src/fetch.rs b/mlf-cli/src/fetch.rs index ce0ffb7..4091c96 100644 --- a/mlf-cli/src/fetch.rs +++ b/mlf-cli/src/fetch.rs @@ -1,8 +1,6 @@ use crate::config::{find_project_root, get_mlf_cache_dir, init_mlf_cache, ConfigError, MlfConfig, LockFile}; -use hickory_resolver::config::*; -use hickory_resolver::Resolver; +use mlf_lexicon_fetcher::{optimize_fetch_patterns, ProductionLexiconFetcher}; use miette::Diagnostic; -use serde::Deserialize; use sha2::{Digest, Sha256}; use std::collections::HashSet; use thiserror::Error; @@ -17,14 +15,6 @@ pub enum FetchError { #[diagnostic(code(mlf::fetch::init_failed))] InitFailed(#[source] std::io::Error), - #[error("DNS lookup failed: {0}")] - #[diagnostic(code(mlf::fetch::dns_error))] - DnsError(String), - - #[error("Failed to parse DID from TXT record: {0}")] - #[diagnostic(code(mlf::fetch::did_parse_error))] - DidParseError(String), - #[error("Failed to fetch lexicon from ATProto repo: {0}")] #[diagnostic(code(mlf::fetch::http_error))] HttpError(String), @@ -47,14 +37,9 @@ pub enum FetchError { } -#[derive(Debug, Deserialize)] -struct AtProtoRecord { - uri: String, - value: serde_json::Value, -} /// Main entry point for fetch command -pub fn run_fetch(nsid: Option, save: bool, update: bool, locked: bool) -> Result<(), FetchError> { +pub async fn run_fetch(nsid: Option, save: bool, update: bool, locked: bool) -> Result<(), FetchError> { // Validate flags if update && locked { return Err(FetchError::HttpError( @@ -76,7 +61,7 @@ pub fn run_fetch(nsid: Option, save: bool, update: bool, locked: bool) - let config_path = project_root.join("mlf.toml"); let config = MlfConfig::load(&config_path).map_err(FetchError::NoProjectRoot)?; - fetch_lexicon_with_lock(&namespace, &project_root, &mut lockfile)?; + fetch_lexicon_with_lock(&namespace, &project_root, &mut lockfile).await?; // Handle transitive dependencies if enabled if config.dependencies.allow_transitive_deps { @@ -85,7 +70,7 @@ pub fn run_fetch(nsid: Option, save: bool, update: bool, locked: bool) - &project_root, &mut lockfile, config.dependencies.optimize_transitive_fetches - )?; + ).await?; } // Save lockfile @@ -101,7 +86,7 @@ pub fn run_fetch(nsid: Option, save: bool, update: bool, locked: bool) - } None => { // Fetch all dependencies from mlf.toml - fetch_all_dependencies(&project_root, update, locked) + fetch_all_dependencies(&project_root, update, locked).await } } } @@ -132,7 +117,7 @@ fn ensure_project_root(current_dir: &std::path::Path) -> Result Result<(), FetchError> { +async fn fetch_all_dependencies(project_root: &std::path::Path, update: bool, locked: bool) -> Result<(), FetchError> { // Load mlf.toml let config_path = project_root.join("mlf.toml"); let config = MlfConfig::load(&config_path).map_err(FetchError::NoProjectRoot)?; @@ -160,7 +145,7 @@ fn fetch_all_dependencies(project_root: &std::path::Path, update: bool, locked: // In locked mode, we use the lockfile and verify nothing needs updating // For now, we'll just use the lockfile - verification can be enhanced later println!("Using locked dependencies from mlf-lock.toml"); - return fetch_from_lockfile(project_root, &existing_lockfile); + return fetch_from_lockfile(project_root, &existing_lockfile).await; } // Determine fetch mode @@ -192,7 +177,7 @@ fn fetch_all_dependencies(project_root: &std::path::Path, update: bool, locked: // Fetch initial dependencies for dep in &config.dependencies.dependencies { println!("\nFetching: {}", dep); - match fetch_lexicon_with_lock(dep, project_root, &mut lockfile) { + match fetch_lexicon_with_lock(dep, project_root, &mut lockfile).await { Ok(()) => { success_count += 1; fetched_nsids.insert(dep.clone()); @@ -205,7 +190,7 @@ fn fetch_all_dependencies(project_root: &std::path::Path, update: bool, locked: // If transitive dependencies are enabled, fetch them if allow_transitive { - fetch_transitive_dependencies(&project_root, &mut lockfile, config.dependencies.optimize_transitive_fetches)?; + fetch_transitive_dependencies(&project_root, &mut lockfile, config.dependencies.optimize_transitive_fetches).await?; } // Save the lockfile @@ -232,7 +217,7 @@ fn fetch_all_dependencies(project_root: &std::path::Path, update: bool, locked: } /// Fetch transitive dependencies by iteratively resolving unresolved references -fn fetch_transitive_dependencies( +async fn fetch_transitive_dependencies( project_root: &std::path::Path, lockfile: &mut LockFile, optimize_fetches: bool @@ -287,7 +272,7 @@ fn fetch_transitive_dependencies( println!("\nFetching transitive dependency: {}", pattern); fetched_nsids.insert(pattern.clone()); - match fetch_lexicon_with_lock(&pattern, project_root, lockfile) { + match fetch_lexicon_with_lock(&pattern, project_root, lockfile).await { Ok(()) => {} Err(e) => { eprintln!(" Warning: Failed to fetch {}: {}", pattern, e); @@ -320,7 +305,7 @@ fn fetch_transitive_dependencies( println!(" Fetching: {}", nsid); fetched_nsids.insert(nsid.clone()); - match fetch_lexicon_with_lock(&nsid, project_root, lockfile) { + match fetch_lexicon_with_lock(&nsid, project_root, lockfile).await { Ok(()) => {} Err(e) => { eprintln!(" Warning: Failed to fetch {}: {}", nsid, e); @@ -339,7 +324,7 @@ fn fetch_transitive_dependencies( println!("\nFetching transitive dependency: {}", nsid); fetched_nsids.insert(nsid.clone()); - match fetch_lexicon_with_lock(nsid, project_root, lockfile) { + match fetch_lexicon_with_lock(nsid, project_root, lockfile).await { Ok(()) => {} Err(e) => { // Don't fail the entire fetch for transitive deps @@ -355,7 +340,7 @@ fn fetch_transitive_dependencies( /// Fetch dependencies using the lockfile /// This refetches each lexicon from its recorded DID and verifies the checksum -fn fetch_from_lockfile(project_root: &std::path::Path, lockfile: &LockFile) -> Result<(), FetchError> { +async fn fetch_from_lockfile(project_root: &std::path::Path, lockfile: &LockFile) -> Result<(), FetchError> { if lockfile.lexicons.is_empty() { println!("Lockfile is empty"); return Ok(()); @@ -371,7 +356,7 @@ fn fetch_from_lockfile(project_root: &std::path::Path, lockfile: &LockFile) -> R println!("\nRefetching: {}", nsid); // Fetch the lexicon using the DID from lockfile - match fetch_specific_lexicon(nsid, &locked.did, &locked.checksum, project_root) { + match fetch_specific_lexicon(nsid, &locked.did, &locked.checksum, project_root).await { Ok(()) => { success_count += 1; } @@ -401,7 +386,7 @@ fn fetch_from_lockfile(project_root: &std::path::Path, lockfile: &LockFile) -> R } /// Fetch a specific lexicon by NSID from a known DID, verifying checksum -fn fetch_specific_lexicon( +async fn fetch_specific_lexicon( nsid: &str, did: &str, expected_checksum: &str, @@ -411,62 +396,74 @@ fn fetch_specific_lexicon( init_mlf_cache(project_root).map_err(FetchError::InitFailed)?; let mlf_dir = get_mlf_cache_dir(project_root); - // Fetch records from the DID - let records = fetch_lexicon_records(did)?; + // Create fetcher and fetch from known DID (bypassing DNS) + let fetcher = ProductionLexiconFetcher::production() + .await + .map_err(|e| FetchError::HttpError(format!("Failed to create fetcher: {}", e)))?; - // Find the specific NSID - for record in records { - let record_nsid = extract_nsid_from_record(&record)?; + let result = fetcher + .fetch_from_did_with_metadata(did, nsid) + .await + .map_err(|e| FetchError::HttpError(format!("Failed to fetch from DID: {}", e)))?; - if record_nsid == nsid { - // Found it! Process and verify checksum - let json_str = serde_json::to_string_pretty(&record.value)?; - let hash = calculate_hash(&json_str); + if result.lexicons.is_empty() { + return Err(FetchError::HttpError(format!( + "Lexicon {} not found in repo {}", + nsid, did + ))); + } - if hash != expected_checksum { - return Err(FetchError::HttpError(format!( - "Checksum mismatch for {}: expected {}, got {}", - nsid, expected_checksum, hash - ))); - } + // We should only get one lexicon for an exact NSID match + let fetched = &result.lexicons[0]; - // Save JSON - let mut json_path = mlf_dir.join("lexicons/json"); - for segment in nsid.split('.') { - json_path.push(segment); - } - json_path.set_extension("json"); + if fetched.nsid != nsid { + return Err(FetchError::HttpError(format!( + "Expected lexicon {}, but got {}", + nsid, fetched.nsid + ))); + } - if let Some(parent) = json_path.parent() { - std::fs::create_dir_all(parent)?; - } - std::fs::write(&json_path, &json_str)?; - println!(" → Saved JSON (checksum verified)"); + // Verify checksum + let json_str = serde_json::to_string_pretty(&fetched.lexicon)?; + let hash = calculate_hash(&json_str); - // Convert to MLF - let mlf_content = crate::generate::mlf::generate_mlf_from_json(&record.value) - .map_err(|e| FetchError::ConversionError(format!("{:?}", e)))?; + if hash != expected_checksum { + return Err(FetchError::HttpError(format!( + "Checksum mismatch for {}: expected {}, got {}", + nsid, expected_checksum, hash + ))); + } - let mut mlf_path = mlf_dir.join("lexicons/mlf"); - for segment in nsid.split('.') { - mlf_path.push(segment); - } - mlf_path.set_extension("mlf"); + // Save JSON + let mut json_path = mlf_dir.join("lexicons/json"); + for segment in nsid.split('.') { + json_path.push(segment); + } + json_path.set_extension("json"); - if let Some(parent) = mlf_path.parent() { - std::fs::create_dir_all(parent)?; - } - std::fs::write(&mlf_path, mlf_content)?; - println!(" → Converted to MLF"); + if let Some(parent) = json_path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&json_path, &json_str)?; + println!(" → Saved JSON (checksum verified)"); - return Ok(()); - } + // Convert to MLF + let mlf_content = crate::generate::mlf::generate_mlf_from_json(&fetched.lexicon) + .map_err(|e| FetchError::ConversionError(format!("{:?}", e)))?; + + let mut mlf_path = mlf_dir.join("lexicons/mlf"); + for segment in nsid.split('.') { + mlf_path.push(segment); } + mlf_path.set_extension("mlf"); - Err(FetchError::HttpError(format!( - "Lexicon {} not found in repo {}", - nsid, did - ))) + if let Some(parent) = mlf_path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&mlf_path, mlf_content)?; + println!(" → Converted to MLF"); + + Ok(()) } fn save_dependency(project_root: &std::path::Path, nsid: &str) -> Result<(), FetchError> { @@ -485,149 +482,94 @@ fn save_dependency(project_root: &std::path::Path, nsid: &str) -> Result<(), Fet Ok(()) } -pub fn fetch_lexicon(nsid: &str, project_root: &std::path::Path) -> Result<(), FetchError> { - let mut lockfile = LockFile::new(); - fetch_lexicon_with_lock(nsid, project_root, &mut lockfile) -} - -fn fetch_lexicon_with_lock(nsid: &str, project_root: &std::path::Path, lockfile: &mut LockFile) -> Result<(), FetchError> { +async fn fetch_lexicon_with_lock(nsid: &str, project_root: &std::path::Path, lockfile: &mut LockFile) -> Result<(), FetchError> { // Initialize .mlf directory init_mlf_cache(project_root).map_err(FetchError::InitFailed)?; - let mlf_dir = get_mlf_cache_dir(&project_root); - // Validate NSID format: must be specific (3+ segments) or use wildcard + // Validate NSID format validate_nsid_format(nsid)?; - // Check if it's a wildcard pattern - let is_wildcard = nsid.ends_with(".*"); - let nsid_pattern = if is_wildcard { - nsid.strip_suffix(".*").unwrap() - } else { - nsid - }; - - // Extract authority and name segments from NSID - // For "app.bsky.actor.profile", authority is "app.bsky", name is "actor.profile" - // For DNS lookup, we need "_lexicon.actor.bsky.app" - let (authority, name_segments) = extract_authority_and_name(nsid_pattern)?; println!("Fetching lexicons for pattern: {}", nsid); - // Step 1: DNS TXT lookup - let did = resolve_lexicon_did(&authority, &name_segments)?; - println!(" → Resolved DID: {}", did); + // Create the lexicon fetcher (encapsulates all DNS and HTTP logic) + let fetcher = ProductionLexiconFetcher::production() + .await + .map_err(|e| FetchError::HttpError(format!("Failed to create fetcher: {}", e)))?; - // Step 2: Query ATProto repo for lexicon schemas - let records = fetch_lexicon_records(&did)?; - println!(" → Found {} lexicon record(s)", records.len()); + // Fetch lexicons with metadata + let result = fetcher + .fetch_with_metadata(nsid) + .await + .map_err(|e| FetchError::HttpError(format!("Failed to fetch: {}", e)))?; - if records.is_empty() { + if result.lexicons.is_empty() { return Err(FetchError::HttpError(format!( - "No lexicon records found for {}", + "No lexicons matched pattern: {}", nsid ))); } - let mut processed_count = 0; - - // Step 3: Process each record - for record in records { - // Extract NSID from record URI or value - let record_nsid = extract_nsid_from_record(&record)?; - - // Match against pattern - let matches = if is_wildcard { - // Wildcard: match all records starting with the pattern - // For "app.bsky.actor.*", nsid_pattern is "app.bsky.actor" - // Should match "app.bsky.actor.defs", "app.bsky.actor.profile", etc. - let starts_with_pattern = record_nsid.starts_with(nsid_pattern); - let has_more_segments = record_nsid.len() > nsid_pattern.len(); - let is_direct_child = if starts_with_pattern && has_more_segments { - // Check if the next character after the pattern is a dot - record_nsid.chars().nth(nsid_pattern.len()) == Some('.') - } else { - false - }; - - starts_with_pattern && has_more_segments && is_direct_child - } else { - // Specific: exact match only - record_nsid == nsid - }; - - if !matches { - continue; - } + println!(" → Found {} lexicon record(s)", result.lexicons.len()); - println!(" Processing: {}", record_nsid); - processed_count += 1; + // Process each fetched lexicon + for fetched in &result.lexicons { + println!(" Processing: {}", fetched.nsid); - // Save JSON file with directory structure - // e.g., "place.stream.key" -> "place/stream/key.json" - let json_str = serde_json::to_string_pretty(&record.value)?; + // Save JSON file + let json_str = serde_json::to_string_pretty(&fetched.lexicon)?; let mut json_path = mlf_dir.join("lexicons/json"); - for segment in record_nsid.split('.') { + for segment in fetched.nsid.split('.') { json_path.push(segment); } json_path.set_extension("json"); - // Create parent directories if let Some(parent) = json_path.parent() { std::fs::create_dir_all(parent)?; } - std::fs::write(&json_path, &json_str)?; println!(" → Saved JSON to {}", json_path.display()); // Convert to MLF - let mlf_content = crate::generate::mlf::generate_mlf_from_json(&record.value) + let mlf_content = crate::generate::mlf::generate_mlf_from_json(&fetched.lexicon) .map_err(|e| FetchError::ConversionError(format!("{:?}", e)))?; - // Save MLF file with directory structure - // e.g., "place.stream.key" -> "place/stream/key.mlf" + // Save MLF file let mut mlf_path = mlf_dir.join("lexicons/mlf"); - for segment in record_nsid.split('.') { + for segment in fetched.nsid.split('.') { mlf_path.push(segment); } mlf_path.set_extension("mlf"); - // Create parent directories if let Some(parent) = mlf_path.parent() { std::fs::create_dir_all(parent)?; } - std::fs::write(&mlf_path, mlf_content)?; println!(" → Converted to MLF at {}", mlf_path.display()); - // Calculate hash of JSON content + // Calculate hash and extract dependencies for lockfile let hash = calculate_hash(&json_str); + let dependencies = extract_dependencies_from_json(&fetched.lexicon); - // Extract dependencies from JSON - let dependencies = extract_dependencies_from_json(&record.value); - - // Update lockfile - lockfile.add_lexicon(record_nsid.clone(), did.clone(), hash.clone(), dependencies); - } - - if processed_count == 0 { - return Err(FetchError::HttpError(format!( - "No lexicons matched pattern: {}", - nsid - ))); + // Update lockfile with DID from fetcher metadata + lockfile.add_lexicon(fetched.nsid.clone(), fetched.did.clone(), hash, dependencies); } - println!("✓ Successfully fetched {} lexicon(s) for {}", processed_count, nsid); + println!("✓ Successfully fetched {} lexicon(s) for {}", result.lexicons.len(), nsid); Ok(()) } fn validate_nsid_format(nsid: &str) -> Result<(), FetchError> { - // Remove wildcard suffix for validation - let nsid_base = nsid.strip_suffix(".*").unwrap_or(nsid); + // Remove wildcard suffix for validation (both .* and ._) + let nsid_base = nsid + .strip_suffix(".*") + .or_else(|| nsid.strip_suffix("._")) + .unwrap_or(nsid); let parts: Vec<&str> = nsid_base.split('.').collect(); // NSID must have at least 2 segments (authority) - // e.g., "place.stream", "place.stream.key", or "place.stream.*" + // e.g., "place.stream", "place.stream.key", "place.stream.*", or "place.stream._" if parts.len() < 2 { return Err(FetchError::InvalidNsid(format!( "NSID must have at least 2 segments (e.g., 'place.stream' or 'com.atproto.repo.strongRef'): {}", @@ -638,213 +580,6 @@ fn validate_nsid_format(nsid: &str) -> Result<(), FetchError> { Ok(()) } -fn extract_authority_and_name(nsid_pattern: &str) -> Result<(String, String), FetchError> { - // NSID format: authority.name(.name)* - // For "place.stream.key", authority is "place.stream" (first 2), name is "key" - // For "app.bsky.actor.profile", authority is "app.bsky" (first 2), name is "actor.profile" - let parts: Vec<&str> = nsid_pattern.split('.').collect(); - - if parts.len() < 2 { - return Err(FetchError::InvalidNsid(format!( - "NSID must have at least 2 segments: {}", - nsid_pattern - ))); - } - - // Authority is first 2 segments (reversed domain) - let authority = format!("{}.{}", parts[0], parts[1]); - - // Name segments are everything after the authority - let name_segments = if parts.len() > 2 { - parts[2..].join(".") - } else { - String::new() - }; - - Ok((authority, name_segments)) -} - -fn resolve_lexicon_did(authority: &str, name_segments: &str) -> Result { - // Reverse the authority for DNS lookup and prepend name segments - // For "app.bsky" + "actor": "_lexicon.actor.bsky.app" - // For "place.stream" + "key": "_lexicon.key.stream.place" - let auth_parts: Vec<&str> = authority.split('.').collect(); - let reversed_auth: Vec<&str> = auth_parts.iter().rev().copied().collect(); - - let dns_name = if name_segments.is_empty() { - // No name segments, just use reversed authority - // For "place.stream": "_lexicon.stream.place" - format!("_lexicon.{}", reversed_auth.join(".")) - } else { - // Prepend name segments before reversed authority - // For "app.bsky" + "actor": "_lexicon.actor.bsky.app" - format!("_lexicon.{}.{}", name_segments, reversed_auth.join(".")) - }; - - println!(" Looking up DNS TXT record: {}", dns_name); - - // Create DNS resolver - let resolver = Resolver::new(ResolverConfig::default(), ResolverOpts::default()) - .map_err(|e| FetchError::DnsError(format!("Failed to create DNS resolver: {}", e)))?; - - // Lookup TXT records - let response = resolver - .txt_lookup(&dns_name) - .map_err(|e| FetchError::DnsError(format!("DNS TXT lookup failed for {}: {}", dns_name, e)))?; - - // Parse TXT records to find DID - for txt_record in response.iter() { - for txt_data in txt_record.txt_data() { - let text = String::from_utf8_lossy(txt_data); - // Look for "did=did:plc:..." or "did=did:web:..." - if let Some(did_value) = text.strip_prefix("did=") { - return Ok(did_value.trim().to_string()); - } - } - } - - Err(FetchError::DidParseError(format!( - "No DID found in TXT record for {}", - dns_name - ))) -} - -fn fetch_lexicon_records(did: &str) -> Result, FetchError> { - // Query the ATProto repo for records in com.atproto.lexicon.schema collection - // We need to use the repo.listRecords XRPC endpoint - // Note: This API is paginated, so we need to fetch all pages - - // First, resolve the DID to a PDS endpoint - let pds_url = resolve_did_to_pds(did)?; - - println!(" → Using PDS: {}", pds_url); - - let mut all_records = Vec::new(); - let mut cursor: Option = None; - let mut page_num = 1; - - loop { - // Build URL with optional cursor - let url = if let Some(ref c) = cursor { - format!( - "{}/xrpc/com.atproto.repo.listRecords?repo={}&collection=com.atproto.lexicon.schema&cursor={}", - pds_url, did, c - ) - } else { - format!( - "{}/xrpc/com.atproto.repo.listRecords?repo={}&collection=com.atproto.lexicon.schema", - pds_url, did - ) - }; - - println!(" Fetching lexicon records (page {})...", page_num); - - let response = reqwest::blocking::get(&url) - .map_err(|e| FetchError::HttpError(format!("Failed to fetch records: {}", e)))?; - - if !response.status().is_success() { - return Err(FetchError::HttpError(format!( - "HTTP {} when fetching records", - response.status() - ))); - } - - let mut list_response: serde_json::Value = response - .json() - .map_err(|e| FetchError::HttpError(format!("Failed to parse response: {}", e)))?; - - // Extract records - if let Some(records_array) = list_response.get_mut("records") { - if let Some(records) = records_array.as_array_mut() { - for record_value in records.drain(..) { - let record: AtProtoRecord = serde_json::from_value(record_value) - .map_err(|e| FetchError::HttpError(format!("Failed to parse record: {}", e)))?; - all_records.push(record); - } - } - } - - // Check for cursor to continue pagination - cursor = list_response.get("cursor") - .and_then(|c| c.as_str()) - .map(|s| s.to_string()); - - if cursor.is_none() { - break; - } - - page_num += 1; - } - - Ok(all_records) -} - -fn resolve_did_to_pds(did: &str) -> Result { - // For did:web:, extract the domain - if let Some(domain) = did.strip_prefix("did:web:") { - return Ok(format!("https://{}", domain)); - } - - // For did:plc:, we need to query the PLC directory - if did.starts_with("did:plc:") { - // Query plc.directory to resolve DID document - let url = format!("https://plc.directory/{}", did); - - let response = reqwest::blocking::get(&url) - .map_err(|e| FetchError::HttpError(format!("Failed to resolve DID: {}", e)))?; - - if !response.status().is_success() { - return Err(FetchError::HttpError(format!( - "Failed to resolve DID {}: HTTP {}", - did, - response.status() - ))); - } - - let did_doc: serde_json::Value = response - .json() - .map_err(|e| FetchError::HttpError(format!("Failed to parse DID document: {}", e)))?; - - // Extract PDS endpoint from service array - if let Some(services) = did_doc.get("service").and_then(|v| v.as_array()) { - for service in services { - if service.get("type").and_then(|v| v.as_str()) == Some("AtprotoPersonalDataServer") { - if let Some(endpoint) = service.get("serviceEndpoint").and_then(|v| v.as_str()) { - return Ok(endpoint.trim_end_matches('/').to_string()); - } - } - } - } - - return Err(FetchError::HttpError(format!( - "No PDS endpoint found in DID document for {}", - did - ))); - } - - Err(FetchError::HttpError(format!( - "Unsupported DID method: {}", - did - ))) -} - -fn extract_nsid_from_record(record: &AtProtoRecord) -> Result { - // The record value should have an "id" field with the NSID - if let Some(id) = record.value.get("id").and_then(|v| v.as_str()) { - return Ok(id.to_string()); - } - - // Fallback: try to extract from URI - // URI format: at://did:plc:xxx/com.atproto.lexicon.schema/nsid - if let Some(rkey) = record.uri.split('/').last() { - return Ok(rkey.to_string()); - } - - Err(FetchError::HttpError(format!( - "Could not extract NSID from record: {}", - record.uri - ))) -} /// Calculate SHA-256 hash of content fn calculate_hash(content: &str) -> String { @@ -994,76 +729,3 @@ fn extract_namespace_pattern(type_ref: &str) -> String { } } -/// Optimize a set of NSIDs by collapsing them into the minimal set of fetch patterns -/// For example: ["app.bsky.actor.foo", "app.bsky.actor.bar"] -> ["app.bsky.actor.*"] -/// This function tries multiple grouping strategies to find the most efficient pattern -fn optimize_fetch_patterns(nsids: &HashSet) -> Vec { - use std::collections::BTreeMap; - - if nsids.is_empty() { - return Vec::new(); - } - - // Strategy 1: Try grouping by authority (first 2 segments) - // e.g., ["app.bsky.actor.foo", "app.bsky.feed.bar"] -> ["app.bsky.*"] - let mut authority_groups: BTreeMap> = BTreeMap::new(); - - for nsid in nsids { - let parts: Vec<&str> = nsid.split('.').collect(); - if parts.len() >= 2 { - let authority = format!("{}.{}", parts[0], parts[1]); - authority_groups.entry(authority).or_insert_with(Vec::new).push(nsid.clone()); - } - } - - // Strategy 2: Try grouping by namespace prefix (all but last segment) - // e.g., ["app.bsky.actor.foo", "app.bsky.actor.bar"] -> ["app.bsky.actor.*"] - let mut prefix_groups: BTreeMap> = BTreeMap::new(); - - for nsid in nsids { - let parts: Vec<&str> = nsid.split('.').collect(); - if parts.len() >= 3 { - let prefix = parts[..parts.len() - 1].join("."); - prefix_groups.entry(prefix).or_insert_with(Vec::new).push(nsid.clone()); - } - } - - let mut result = Vec::new(); - let mut handled_nsids = HashSet::new(); - - // First pass: Apply namespace-level grouping (more specific) - for (prefix, group) in &prefix_groups { - if group.len() >= 2 && !handled_nsids.contains(&group[0]) { - result.push(format!("{}.*", prefix)); - for nsid in group { - handled_nsids.insert(nsid.clone()); - } - } - } - - // Second pass: For remaining NSIDs, consider authority-level grouping - // Only use authority wildcard if we have 3+ different namespaces under same authority - for (authority, group) in &authority_groups { - let unhandled: Vec<&String> = group.iter() - .filter(|nsid| !handled_nsids.contains(*nsid)) - .collect(); - - if unhandled.len() >= 3 { - result.push(format!("{}.*", authority)); - for nsid in &unhandled { - handled_nsids.insert((*nsid).clone()); - } - } - } - - // Third pass: Add remaining individual NSIDs - for nsid in nsids { - if !handled_nsids.contains(nsid) { - result.push(nsid.clone()); - } - } - - // Sort for consistent output - result.sort(); - result -} diff --git a/mlf-cli/src/main.rs b/mlf-cli/src/main.rs index 3ea284a..4575387 100644 --- a/mlf-cli/src/main.rs +++ b/mlf-cli/src/main.rs @@ -112,7 +112,8 @@ enum GenerateCommands { }, } -fn main() { +#[tokio::main] +async fn main() { let cli = Cli::parse(); let result: Result<(), miette::Report> = match cli.command { @@ -141,7 +142,7 @@ fn main() { } }, Commands::Fetch { nsid, save, update, locked } => { - fetch::run_fetch(nsid, save, update, locked).into_diagnostic() + fetch::run_fetch(nsid, save, update, locked).await.into_diagnostic() } }; diff --git a/mlf-lang/tests/integration_test.rs b/mlf-lang/tests/integration_test.rs index 51efba9..b5c66c4 100644 --- a/mlf-lang/tests/integration_test.rs +++ b/mlf-lang/tests/integration_test.rs @@ -11,8 +11,10 @@ struct ExpectedResult { #[serde(default)] errors: Vec, #[serde(default)] + #[allow(dead_code)] warnings: Vec, #[serde(flatten)] + #[allow(dead_code)] extra: HashMap, } @@ -21,8 +23,10 @@ struct ExpectedError { #[serde(rename = "type")] error_type: String, #[serde(default)] + #[allow(dead_code)] name: Option, #[serde(default)] + #[allow(dead_code)] message: Option, } diff --git a/mlf-lexicon-fetcher/Cargo.toml b/mlf-lexicon-fetcher/Cargo.toml new file mode 100644 index 0000000..daf0bb7 --- /dev/null +++ b/mlf-lexicon-fetcher/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "mlf-lexicon-fetcher" +version = "0.1.0" +edition = "2024" +license = "MIT" + +[dependencies] +hickory-resolver = "0.24" +thiserror = "2.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +reqwest = { version = "0.12", features = ["json"] } +async-trait = "0.1" +tokio = { version = "1", features = ["rt"] } + +[dev-dependencies] +tokio = { version = "1", features = ["full"] } diff --git a/mlf-lexicon-fetcher/examples/usage.rs b/mlf-lexicon-fetcher/examples/usage.rs new file mode 100644 index 0000000..75dddce --- /dev/null +++ b/mlf-lexicon-fetcher/examples/usage.rs @@ -0,0 +1,72 @@ +// Example usage of mlf-lexicon-fetcher + +use mlf_lexicon_fetcher::{LexiconFetcher, MockDnsResolver, MockHttpClient}; +use serde_json::json; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Example 1: Fetch a single lexicon + println!("=== Example 1: Fetch Single Lexicon ==="); + + let mut dns_resolver = MockDnsResolver::new(); + dns_resolver.add_record("place.stream", "chat.profile", "did:plc:test123".to_string()); + + let mut http_client = MockHttpClient::new(); + http_client.add_lexicon( + "place.stream.chat.profile".to_string(), + json!({ + "lexicon": 1, + "id": "place.stream.chat.profile", + "defs": { + "main": { + "type": "record", + "description": "A user profile for chat" + } + } + }), + ); + + let fetcher = LexiconFetcher::new(dns_resolver, http_client); + + match fetcher.fetch("place.stream.chat.profile").await { + Ok(lexicon) => { + println!("Successfully fetched lexicon:"); + println!("{}", serde_json::to_string_pretty(&lexicon)?); + } + Err(e) => eprintln!("Error: {}", e), + } + + // Example 2: Fetch multiple lexicons with a pattern + println!("\n=== Example 2: Fetch Multiple Lexicons with Pattern ==="); + + let mut dns_resolver2 = MockDnsResolver::new(); + dns_resolver2.add_record("app.bsky", "feed", "did:plc:bsky123".to_string()); + + let mut http_client2 = MockHttpClient::new(); + http_client2.add_lexicon( + "app.bsky.feed.post".to_string(), + json!({"lexicon": 1, "id": "app.bsky.feed.post"}), + ); + http_client2.add_lexicon( + "app.bsky.feed.like".to_string(), + json!({"lexicon": 1, "id": "app.bsky.feed.like"}), + ); + http_client2.add_lexicon( + "app.bsky.feed.repost".to_string(), + json!({"lexicon": 1, "id": "app.bsky.feed.repost"}), + ); + + let fetcher2 = LexiconFetcher::new(dns_resolver2, http_client2); + + match fetcher2.fetch_pattern("app.bsky.feed.*").await { + Ok(lexicons) => { + println!("Successfully fetched {} lexicons:", lexicons.len()); + for (nsid, _lexicon) in lexicons { + println!(" - {}", nsid); + } + } + Err(e) => eprintln!("Error: {}", e), + } + + Ok(()) +} diff --git a/mlf-lexicon-fetcher/src/lib.rs b/mlf-lexicon-fetcher/src/lib.rs new file mode 100644 index 0000000..89ebc72 --- /dev/null +++ b/mlf-lexicon-fetcher/src/lib.rs @@ -0,0 +1,880 @@ +// MLF Lexicon Fetcher +// Resolves ATProto lexicon NSIDs to DIDs via DNS TXT records +// and fetches lexicon JSON via HTTP + +use async_trait::async_trait; +use hickory_resolver::config::{ResolverConfig, ResolverOpts}; +use hickory_resolver::TokioAsyncResolver; +use serde::Deserialize; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use thiserror::Error; + +#[derive(Debug, Deserialize)] +struct AtProtoRecord { + uri: String, + value: serde_json::Value, +} + +#[derive(Error, Debug)] +pub enum LexiconFetcherError { + #[error("Failed to create DNS resolver: {0}")] + ResolverCreationFailed(String), + + #[error("DNS lookup failed for {domain}: {error}")] + LookupFailed { domain: String, error: String }, + + #[error("No DID found in TXT record for {0}")] + NoDid(String), + + #[error("Invalid NSID format: {0}")] + InvalidNsid(String), + + #[error("HTTP request failed: {0}")] + HttpRequestFailed(String), + + #[error("Failed to parse JSON response: {0}")] + JsonParseFailed(String), + + #[error("Lexicon not found: {0}")] + LexiconNotFound(String), + + #[error("Invalid URL: {0}")] + InvalidUrl(String), +} + +pub type Result = std::result::Result; + +/// Trait for DNS resolution - allows mocking in tests +#[async_trait] +pub trait DnsResolver: Send + Sync { + /// Resolve an NSID to a DID via DNS TXT lookup + async fn resolve_lexicon_did(&self, authority: &str, name_segments: &str) -> Result; +} + +/// Real DNS resolver using hickory_resolver's async resolver +pub struct RealDnsResolver { + resolver: TokioAsyncResolver, +} + +impl RealDnsResolver { + pub async fn new() -> Result { + let resolver = TokioAsyncResolver::tokio(ResolverConfig::default(), ResolverOpts::default()); + Ok(Self { resolver }) + } + + pub async fn with_config(config: ResolverConfig, opts: ResolverOpts) -> Result { + let resolver = TokioAsyncResolver::tokio(config, opts); + Ok(Self { resolver }) + } +} + +#[async_trait] +impl DnsResolver for RealDnsResolver { + async fn resolve_lexicon_did(&self, authority: &str, name_segments: &str) -> Result { + let dns_name = construct_dns_name(authority, name_segments); + + // Lookup TXT records (async) + let response = self + .resolver + .txt_lookup(&dns_name) + .await + .map_err(|e| LexiconFetcherError::LookupFailed { + domain: dns_name.clone(), + error: e.to_string(), + })?; + + // Parse TXT records to find DID + for txt_record in response.iter() { + for txt_data in txt_record.txt_data() { + let text = String::from_utf8_lossy(txt_data); + // Look for "did=did:plc:..." or "did=did:web:..." + if let Some(did_value) = text.strip_prefix("did=") { + return Ok(did_value.trim().to_string()); + } + } + } + + Err(LexiconFetcherError::NoDid(dns_name)) + } +} + +/// Mock DNS resolver for testing +#[derive(Clone)] +pub struct MockDnsResolver { + records: Arc>>, +} + +impl MockDnsResolver { + pub fn new() -> Self { + Self { + records: Arc::new(Mutex::new(HashMap::new())), + } + } + + /// Add a mock DNS record (maps NSID authority+name to DID) + pub fn add_record(&mut self, authority: &str, name_segments: &str, did: String) { + let dns_name = construct_dns_name(authority, name_segments); + self.records.lock().unwrap().insert(dns_name, did); + } + + /// Add a mock record using full NSID + pub fn add_record_from_nsid(&mut self, nsid: &str, did: String) -> Result<()> { + let (authority, name_segments) = parse_nsid(nsid)?; + self.add_record(&authority, &name_segments, did); + Ok(()) + } +} + +impl Default for MockDnsResolver { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl DnsResolver for MockDnsResolver { + async fn resolve_lexicon_did(&self, authority: &str, name_segments: &str) -> Result { + let dns_name = construct_dns_name(authority, name_segments); + self.records + .lock() + .unwrap() + .get(&dns_name) + .cloned() + .ok_or_else(|| LexiconFetcherError::LookupFailed { + domain: dns_name.clone(), + error: "No mock record found".to_string(), + }) + } +} + +/// Construct DNS name from authority and name segments +/// For "app.bsky" + "actor": "_lexicon.actor.bsky.app" +/// For "place.stream" + "key": "_lexicon.key.stream.place" +pub fn construct_dns_name(authority: &str, name_segments: &str) -> String { + let auth_parts: Vec<&str> = authority.split('.').collect(); + let reversed_auth: Vec<&str> = auth_parts.iter().rev().copied().collect(); + + if name_segments.is_empty() { + // No name segments, just use reversed authority + // For "place.stream": "_lexicon.stream.place" + format!("_lexicon.{}", reversed_auth.join(".")) + } else { + // Prepend name segments before reversed authority + // For "app.bsky" + "actor": "_lexicon.actor.bsky.app" + format!("_lexicon.{}.{}", name_segments, reversed_auth.join(".")) + } +} + +/// Parse NSID into authority and name segments +/// For "place.stream.key", returns ("place.stream", "key") +/// For "app.bsky.actor.profile", returns ("app.bsky", "actor.profile") +pub fn parse_nsid(nsid: &str) -> Result<(String, String)> { + // NSID format: authority.name(.name)* + // Authority is first 2 segments (reversed domain) + let parts: Vec<&str> = nsid.split('.').collect(); + + if parts.len() < 2 { + return Err(LexiconFetcherError::InvalidNsid(format!( + "NSID must have at least 2 segments: {}", + nsid + ))); + } + + // Authority is first 2 segments + let authority = format!("{}.{}", parts[0], parts[1]); + + // Name segments are everything after the authority + let name_segments = if parts.len() > 2 { + parts[2..].join(".") + } else { + String::new() + }; + + Ok((authority, name_segments)) +} + +/// Trait for HTTP client - allows mocking in tests +#[async_trait] +pub trait HttpClient: Send + Sync { + /// Fetch a single lexicon by NSID from a DID's server + async fn fetch_lexicon(&self, did: &str, nsid: &str) -> Result; + + /// Fetch all lexicons matching a pattern (e.g., "place.stream.*") + async fn fetch_lexicons_pattern( + &self, + did: &str, + pattern: &str, + ) -> Result>; +} + +/// Real HTTP client using reqwest +pub struct RealHttpClient { + client: reqwest::Client, +} + +impl RealHttpClient { + pub fn new() -> Self { + Self { + client: reqwest::Client::new(), + } + } + + pub fn with_client(client: reqwest::Client) -> Self { + Self { client } + } +} + +impl Default for RealHttpClient { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl HttpClient for RealHttpClient { + async fn fetch_lexicon(&self, did: &str, nsid: &str) -> Result { + // Fetch all records from the DID's repo + let records = self.fetch_records_from_did(did).await?; + + // Find the specific NSID + for record in records { + let record_nsid = extract_nsid_from_record(&record)?; + if record_nsid == nsid { + return Ok(record.value); + } + } + + Err(LexiconFetcherError::LexiconNotFound(format!( + "Lexicon {} not found in repo {}", + nsid, did + ))) + } + + async fn fetch_lexicons_pattern( + &self, + did: &str, + pattern: &str, + ) -> Result> { + // Fetch all records from the DID's repo + let records = self.fetch_records_from_did(did).await?; + + let mut results = Vec::new(); + + // Handle exact match (no wildcard) + if !pattern.contains('*') && !pattern.contains('_') { + for record in records { + let record_nsid = extract_nsid_from_record(&record)?; + if record_nsid == pattern { + results.push((record_nsid, record.value)); + } + } + return Ok(results); + } + + // Handle wildcard patterns + if pattern.ends_with(".*") { + // "*" matches EVERYTHING + // For "place.stream.*", match all: place.stream.chat, place.stream.chat.profile, etc. + let base = pattern.strip_suffix(".*").unwrap(); + let prefix_with_dot = format!("{}.", base); + + for record in records { + let record_nsid = extract_nsid_from_record(&record)?; + if record_nsid.starts_with(&prefix_with_dot) { + results.push((record_nsid, record.value)); + } + } + } else if pattern.ends_with("._") { + // "_" matches only DIRECT CHILDREN + // For "place.stream._", match place.stream.chat but NOT place.stream.chat.profile + let base = pattern.strip_suffix("._").unwrap(); + let prefix_with_dot = format!("{}.", base); + + for record in records { + let record_nsid = extract_nsid_from_record(&record)?; + + if let Some(suffix) = record_nsid.strip_prefix(&prefix_with_dot) { + // Check if it's a direct child (no more dots in the suffix) + if !suffix.contains('.') && !suffix.is_empty() { + results.push((record_nsid, record.value)); + } + } + } + } + + Ok(results) + } +} + +impl RealHttpClient { + /// Fetch all lexicon records from a DID's ATProto repository + async fn fetch_records_from_did(&self, did: &str) -> Result> { + // Resolve DID to PDS URL + let pds_url = self.resolve_did_to_pds(did).await?; + + let mut all_records = Vec::new(); + let mut cursor: Option = None; + + // Paginate through all records + loop { + let url = if let Some(ref c) = cursor { + format!( + "{}/xrpc/com.atproto.repo.listRecords?repo={}&collection=com.atproto.lexicon.schema&cursor={}", + pds_url, did, c + ) + } else { + format!( + "{}/xrpc/com.atproto.repo.listRecords?repo={}&collection=com.atproto.lexicon.schema", + pds_url, did + ) + }; + + let response = self + .client + .get(&url) + .send() + .await + .map_err(|e| LexiconFetcherError::HttpRequestFailed(e.to_string()))?; + + if !response.status().is_success() { + return Err(LexiconFetcherError::HttpRequestFailed(format!( + "HTTP {} when fetching records from {}", + response.status(), + did + ))); + } + + let mut list_response: serde_json::Value = response + .json() + .await + .map_err(|e| LexiconFetcherError::JsonParseFailed(e.to_string()))?; + + // Extract records + if let Some(records_array) = list_response.get_mut("records") { + if let Some(records) = records_array.as_array_mut() { + for record_value in records.drain(..) { + let record: AtProtoRecord = serde_json::from_value(record_value) + .map_err(|e| LexiconFetcherError::JsonParseFailed(format!("Failed to parse record: {}", e)))?; + all_records.push(record); + } + } + } + + // Check for pagination cursor + cursor = list_response + .get("cursor") + .and_then(|c| c.as_str()) + .map(|s| s.to_string()); + + if cursor.is_none() { + break; + } + } + + Ok(all_records) + } + + /// Resolve a DID to its PDS URL + async fn resolve_did_to_pds(&self, did: &str) -> Result { + // For did:web:, extract the domain + if let Some(domain) = did.strip_prefix("did:web:") { + return Ok(format!("https://{}", domain)); + } + + // For did:plc:, query the PLC directory + if did.starts_with("did:plc:") { + let url = format!("https://plc.directory/{}", did); + + let response = self + .client + .get(&url) + .send() + .await + .map_err(|e| LexiconFetcherError::HttpRequestFailed(format!("Failed to resolve DID: {}", e)))?; + + if !response.status().is_success() { + return Err(LexiconFetcherError::HttpRequestFailed(format!( + "Failed to resolve DID {}: HTTP {}", + did, + response.status() + ))); + } + + let did_doc: serde_json::Value = response + .json() + .await + .map_err(|e| LexiconFetcherError::JsonParseFailed(format!("Failed to parse DID document: {}", e)))?; + + // Extract PDS endpoint from service array + if let Some(services) = did_doc.get("service").and_then(|v| v.as_array()) { + for service in services { + if service.get("type").and_then(|v| v.as_str()) == Some("AtprotoPersonalDataServer") { + if let Some(endpoint) = service.get("serviceEndpoint").and_then(|v| v.as_str()) { + return Ok(endpoint.trim_end_matches('/').to_string()); + } + } + } + } + + return Err(LexiconFetcherError::HttpRequestFailed(format!( + "No PDS endpoint found in DID document for {}", + did + ))); + } + + Err(LexiconFetcherError::InvalidUrl(format!( + "Unsupported DID format: {}", + did + ))) + } +} + +/// Mock HTTP client for testing +#[derive(Clone)] +pub struct MockHttpClient { + lexicons: Arc>>, +} + +impl MockHttpClient { + pub fn new() -> Self { + Self { + lexicons: Arc::new(Mutex::new(HashMap::new())), + } + } + + /// Add a mock lexicon response for a specific NSID + pub fn add_lexicon(&mut self, nsid: String, lexicon: serde_json::Value) { + self.lexicons.lock().unwrap().insert(nsid, lexicon); + } +} + +impl Default for MockHttpClient { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl HttpClient for MockHttpClient { + async fn fetch_lexicon(&self, _did: &str, nsid: &str) -> Result { + self.lexicons + .lock() + .unwrap() + .get(nsid) + .cloned() + .ok_or_else(|| LexiconFetcherError::LexiconNotFound(nsid.to_string())) + } + + async fn fetch_lexicons_pattern( + &self, + _did: &str, + pattern: &str, + ) -> Result> { + let lexicons = self.lexicons.lock().unwrap(); + let mut results = Vec::new(); + + // Handle exact match (no wildcard) + if !pattern.contains('*') && !pattern.contains('_') { + if let Some(lexicon) = lexicons.get(pattern) { + results.push((pattern.to_string(), lexicon.clone())); + } + return Ok(results); + } + + // Handle wildcard patterns + if pattern.ends_with(".*") { + // "*" matches EVERYTHING + // For "place.stream.*", match all: place.stream.chat, place.stream.chat.profile, etc. + let base = pattern.strip_suffix(".*").unwrap(); + let prefix_with_dot = format!("{}.", base); + + for (nsid, lexicon) in lexicons.iter() { + if nsid.starts_with(&prefix_with_dot) { + results.push((nsid.clone(), lexicon.clone())); + } + } + } else if pattern.ends_with("._") { + // "_" matches only DIRECT CHILDREN + // For "place.stream._", match place.stream.chat but NOT place.stream.chat.profile + let base = pattern.strip_suffix("._").unwrap(); + let prefix_with_dot = format!("{}.", base); + + for (nsid, lexicon) in lexicons.iter() { + if let Some(suffix) = nsid.strip_prefix(&prefix_with_dot) { + // Check if it's a direct child (no more dots in the suffix) + if !suffix.contains('.') && !suffix.is_empty() { + results.push((nsid.clone(), lexicon.clone())); + } + } + } + } + + Ok(results) + } +} + +/// Main lexicon fetcher that combines DNS resolution and HTTP fetching +pub struct LexiconFetcher { + dns_resolver: D, + http_client: H, +} + +impl LexiconFetcher { + pub fn new(dns_resolver: D, http_client: H) -> Self { + Self { + dns_resolver, + http_client, + } + } + + /// Fetch a single lexicon by NSID + /// Example: "place.stream.chat.profile" -> single lexicon JSON + pub async fn fetch(&self, nsid: &str) -> Result { + // Check if this is a wildcard pattern + if nsid.contains('*') || nsid.contains('_') { + return Err(LexiconFetcherError::InvalidNsid(format!( + "Use fetch_pattern() for wildcard patterns (* or _): {}", + nsid + ))); + } + + // Parse NSID into authority and name segments + let (authority, name_segments) = parse_nsid(nsid)?; + + // Resolve DID via DNS (async) + let did = self.dns_resolver.resolve_lexicon_did(&authority, &name_segments).await?; + + // Fetch lexicon via HTTP + self.http_client.fetch_lexicon(&did, nsid).await + } + + /// Fetch all lexicons matching a pattern + /// Examples: + /// - "place.stream.*" -> matches everything (place.stream.chat, place.stream.chat.profile, etc.) + /// - "place.stream._" -> matches direct children only (place.stream.chat, place.stream.key, but not place.stream.chat.profile) + pub async fn fetch_pattern(&self, pattern: &str) -> Result> { + // Parse pattern to extract authority + let (authority, name_pattern) = parse_nsid(pattern)?; + + // For DNS lookup, remove wildcard suffix (both .* and ._) + // For "place.stream.*" or "place.stream._", name_pattern is "*" or "_", so we use empty string + // For "place.stream.chat.*", name_pattern is "chat.*", so we use "chat" + let dns_name_segments = if name_pattern == "*" || name_pattern == "_" { + "" + } else if let Some(pos) = name_pattern.rfind(".*") { + &name_pattern[..pos] + } else if let Some(pos) = name_pattern.rfind("._") { + &name_pattern[..pos] + } else { + &name_pattern + }; + + // Resolve DID via DNS (async) + let did = self.dns_resolver.resolve_lexicon_did(&authority, dns_name_segments).await?; + + // Fetch lexicons matching pattern via HTTP + self.http_client.fetch_lexicons_pattern(&did, pattern).await + } +} + +/// Metadata about a fetched lexicon +#[derive(Debug, Clone)] +pub struct FetchedLexicon { + pub nsid: String, + pub lexicon: serde_json::Value, + pub did: String, +} + +/// Result of fetching one or more lexicons +#[derive(Debug)] +pub struct FetchResult { + pub lexicons: Vec, +} + +/// Convenience type for production use with real DNS and HTTP +pub type ProductionLexiconFetcher = LexiconFetcher; + +impl ProductionLexiconFetcher { + /// Create a new production fetcher with default configuration + pub async fn production() -> Result { + Ok(Self::new(RealDnsResolver::new().await?, RealHttpClient::new())) + } +} + +impl LexiconFetcher { + /// Fetch one or more lexicons and return metadata for lockfile tracking + /// Handles both exact NSIDs and patterns (* or _) + pub async fn fetch_with_metadata(&self, nsid: &str) -> Result { + // Parse pattern to extract authority and name segments + let (authority, name_segments) = parse_nsid(nsid)?; + + // For DNS lookup, remove wildcard suffix (both .* and ._) + let dns_name_segments = if name_segments == "*" || name_segments == "_" { + "" + } else if let Some(pos) = name_segments.rfind(".*") { + &name_segments[..pos] + } else if let Some(pos) = name_segments.rfind("._") { + &name_segments[..pos] + } else { + &name_segments + }; + + // Resolve DID via DNS (async) + let did = self.dns_resolver.resolve_lexicon_did(&authority, dns_name_segments).await?; + + // Fetch lexicons + let lexicons = if nsid.contains('*') || nsid.contains('_') { + self.http_client.fetch_lexicons_pattern(&did, nsid).await? + } else { + let lexicon = self.http_client.fetch_lexicon(&did, nsid).await?; + vec![(nsid.to_string(), lexicon)] + }; + + // Package results with metadata + let fetched_lexicons = lexicons + .into_iter() + .map(|(nsid, lexicon)| FetchedLexicon { + nsid, + lexicon, + did: did.clone(), + }) + .collect(); + + Ok(FetchResult { + lexicons: fetched_lexicons, + }) + } + + /// Fetch multiple NSIDs in sequence (no optimization) + pub async fn fetch_many(&self, nsids: &[String]) -> Result> { + let mut results = Vec::new(); + for nsid in nsids { + results.push(self.fetch_with_metadata(nsid).await?); + } + Ok(results) + } + + /// Fetch multiple NSIDs with optimization to reduce network requests + /// Groups similar NSIDs into wildcard patterns when beneficial + /// Example: ["app.bsky.actor.foo", "app.bsky.actor.bar"] -> fetches "app.bsky.actor.*" + pub async fn fetch_many_optimized(&self, nsids: &[String]) -> Result> { + use std::collections::HashSet; + + if nsids.is_empty() { + return Ok(Vec::new()); + } + + // Convert to HashSet for optimization + let nsids_set: HashSet = nsids.iter().cloned().collect(); + + // Optimize into minimal set of patterns + let optimized_patterns = optimize_fetch_patterns(&nsids_set); + + // Fetch each optimized pattern + let mut results = Vec::new(); + for pattern in optimized_patterns { + results.push(self.fetch_with_metadata(&pattern).await?); + } + + Ok(results) + } + + /// Fetch lexicon(s) from a known DID, bypassing DNS resolution + /// Useful when fetching from lockfile where DID is already known + /// Handles both exact NSIDs and patterns (* or _) + pub async fn fetch_from_did_with_metadata(&self, did: &str, nsid: &str) -> Result { + // Fetch lexicons directly from the DID + let lexicons = if nsid.contains('*') || nsid.contains('_') { + self.http_client.fetch_lexicons_pattern(did, nsid).await? + } else { + let lexicon = self.http_client.fetch_lexicon(did, nsid).await?; + vec![(nsid.to_string(), lexicon)] + }; + + // Package results with metadata + let fetched_lexicons = lexicons + .into_iter() + .map(|(nsid, lexicon)| FetchedLexicon { + nsid, + lexicon, + did: did.to_string(), + }) + .collect(); + + Ok(FetchResult { + lexicons: fetched_lexicons, + }) + } +} + +/// Convenience type for testing with mocks +pub type MockLexiconFetcher = LexiconFetcher; + +impl MockLexiconFetcher { + /// Create a new mock fetcher for testing + pub fn mock() -> Self { + Self::new(MockDnsResolver::new(), MockHttpClient::new()) + } +} + +/// Extract NSID from an ATProto record +fn extract_nsid_from_record(record: &AtProtoRecord) -> Result { + // The record value should have an "id" field with the NSID + if let Some(id) = record.value.get("id").and_then(|v| v.as_str()) { + return Ok(id.to_string()); + } + + // Fallback: try to extract from URI + // URI format: at://did:plc:xxx/com.atproto.lexicon.schema/nsid + if let Some(rkey) = record.uri.split('/').last() { + return Ok(rkey.to_string()); + } + + Err(LexiconFetcherError::HttpRequestFailed(format!( + "Could not extract NSID from record: {}", + record.uri + ))) +} + +/// Optimize a set of NSIDs by collapsing them into the minimal set of fetch patterns +/// For example: ["app.bsky.actor.foo", "app.bsky.actor.bar"] -> ["app.bsky.actor.*"] +/// This function tries multiple grouping strategies to find the most efficient pattern +pub fn optimize_fetch_patterns(nsids: &std::collections::HashSet) -> Vec { + use std::collections::{BTreeMap, HashSet}; + + if nsids.is_empty() { + return Vec::new(); + } + + // Strategy 1: Try grouping by authority (first 2 segments) + // e.g., ["app.bsky.actor.foo", "app.bsky.feed.bar"] -> ["app.bsky.*"] + let mut authority_groups: BTreeMap> = BTreeMap::new(); + + for nsid in nsids { + let parts: Vec<&str> = nsid.split('.').collect(); + if parts.len() >= 2 { + let authority = format!("{}.{}", parts[0], parts[1]); + authority_groups.entry(authority).or_insert_with(Vec::new).push(nsid.clone()); + } + } + + // Strategy 2: Try grouping by namespace prefix (all but last segment) + // e.g., ["app.bsky.actor.foo", "app.bsky.actor.bar"] -> ["app.bsky.actor.*"] + let mut prefix_groups: BTreeMap> = BTreeMap::new(); + + for nsid in nsids { + let parts: Vec<&str> = nsid.split('.').collect(); + if parts.len() >= 3 { + let prefix = parts[..parts.len() - 1].join("."); + prefix_groups.entry(prefix).or_insert_with(Vec::new).push(nsid.clone()); + } + } + + let mut result = Vec::new(); + let mut handled_nsids = HashSet::new(); + + // First pass: Apply namespace-level grouping (more specific) + for (prefix, group) in &prefix_groups { + if group.len() >= 2 && !handled_nsids.contains(&group[0]) { + result.push(format!("{}.*", prefix)); + for nsid in group { + handled_nsids.insert(nsid.clone()); + } + } + } + + // Second pass: For remaining NSIDs, consider authority-level grouping + // Only use authority wildcard if we have 3+ different namespaces under same authority + for (authority, group) in &authority_groups { + let unhandled: Vec<&String> = group.iter() + .filter(|nsid| !handled_nsids.contains(*nsid)) + .collect(); + + if unhandled.len() >= 3 { + result.push(format!("{}.*", authority)); + for nsid in &unhandled { + handled_nsids.insert((*nsid).clone()); + } + } + } + + // Third pass: Add remaining individual NSIDs + for nsid in nsids { + if !handled_nsids.contains(nsid) { + result.push(nsid.clone()); + } + } + + // Sort for consistent output + result.sort(); + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_construct_dns_name() { + assert_eq!( + construct_dns_name("place.stream", "key"), + "_lexicon.key.stream.place" + ); + assert_eq!( + construct_dns_name("app.bsky", "actor"), + "_lexicon.actor.bsky.app" + ); + assert_eq!( + construct_dns_name("app.bsky", "actor.profile"), + "_lexicon.actor.profile.bsky.app" + ); + assert_eq!( + construct_dns_name("place.stream", ""), + "_lexicon.stream.place" + ); + } + + #[test] + fn test_parse_nsid() { + let (auth, name) = parse_nsid("place.stream.key").unwrap(); + assert_eq!(auth, "place.stream"); + assert_eq!(name, "key"); + + let (auth, name) = parse_nsid("app.bsky.actor.profile").unwrap(); + assert_eq!(auth, "app.bsky"); + assert_eq!(name, "actor.profile"); + + let (auth, name) = parse_nsid("place.stream").unwrap(); + assert_eq!(auth, "place.stream"); + assert_eq!(name, ""); + + assert!(parse_nsid("invalid").is_err()); + } + + #[tokio::test] + async fn test_mock_dns_resolver() { + let mut resolver = MockDnsResolver::new(); + resolver.add_record("place.stream", "key", "did:plc:test123".to_string()); + + let did = resolver.resolve_lexicon_did("place.stream", "key").await.unwrap(); + assert_eq!(did, "did:plc:test123"); + + let result = resolver.resolve_lexicon_did("place.stream", "notfound").await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_mock_dns_resolver_from_nsid() { + let mut resolver = MockDnsResolver::new(); + resolver + .add_record_from_nsid("app.bsky.actor.profile", "did:plc:bsky123".to_string()) + .unwrap(); + + let did = resolver + .resolve_lexicon_did("app.bsky", "actor.profile") + .await + .unwrap(); + assert_eq!(did, "did:plc:bsky123"); + } +} diff --git a/mlf-lexicon-fetcher/tests/dns_scenarios.rs b/mlf-lexicon-fetcher/tests/dns_scenarios.rs new file mode 100644 index 0000000..0166a98 --- /dev/null +++ b/mlf-lexicon-fetcher/tests/dns_scenarios.rs @@ -0,0 +1,286 @@ +// Comprehensive DNS resolver tests covering various scenarios + +use mlf_lexicon_fetcher::{DnsResolver, MockDnsResolver}; + +#[tokio::test] +async fn test_successful_lookup() { + let mut resolver = MockDnsResolver::new(); + resolver.add_record("place.stream", "key", "did:plc:abc123def456".to_string()); + + let result = resolver.resolve_lexicon_did("place.stream", "key").await; + assert!(result.is_ok()); + assert_eq!(result.unwrap(), "did:plc:abc123def456"); +} + +#[tokio::test] +async fn test_dns_record_not_found() { + let resolver = MockDnsResolver::new(); + let result = resolver.resolve_lexicon_did("nonexistent.domain", "test").await; + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("No mock record found")); +} + +#[tokio::test] +async fn test_multiple_authority_types() { + let mut resolver = MockDnsResolver::new(); + + // app.bsky authority + resolver.add_record("app.bsky", "actor", "did:plc:bsky001".to_string()); + + // place.stream authority + resolver.add_record("place.stream", "chat", "did:plc:stream001".to_string()); + + // com.atproto authority + resolver.add_record("com.atproto", "repo", "did:plc:atproto001".to_string()); + + assert_eq!( + resolver.resolve_lexicon_did("app.bsky", "actor").await.unwrap(), + "did:plc:bsky001" + ); + assert_eq!( + resolver.resolve_lexicon_did("place.stream", "chat").await.unwrap(), + "did:plc:stream001" + ); + assert_eq!( + resolver.resolve_lexicon_did("com.atproto", "repo").await.unwrap(), + "did:plc:atproto001" + ); +} + +#[tokio::test] +async fn test_nested_name_segments() { + let mut resolver = MockDnsResolver::new(); + + // Single segment + resolver.add_record("app.bsky", "actor", "did:plc:single".to_string()); + + // Two segments + resolver.add_record("app.bsky", "actor.profile", "did:plc:double".to_string()); + + // Three segments + resolver.add_record("app.bsky", "actor.profile.detailed", "did:plc:triple".to_string()); + + assert_eq!( + resolver.resolve_lexicon_did("app.bsky", "actor").await.unwrap(), + "did:plc:single" + ); + assert_eq!( + resolver.resolve_lexicon_did("app.bsky", "actor.profile").await.unwrap(), + "did:plc:double" + ); + assert_eq!( + resolver.resolve_lexicon_did("app.bsky", "actor.profile.detailed").await.unwrap(), + "did:plc:triple" + ); +} + +#[tokio::test] +async fn test_empty_name_segments() { + let mut resolver = MockDnsResolver::new(); + + // Authority only (no name segments) + resolver.add_record("place.stream", "", "did:plc:root".to_string()); + + assert_eq!( + resolver.resolve_lexicon_did("place.stream", "").await.unwrap(), + "did:plc:root" + ); +} + +#[tokio::test] +async fn test_did_web_format() { + let mut resolver = MockDnsResolver::new(); + + resolver.add_record("example.com", "api", "did:web:example.com".to_string()); + + assert_eq!( + resolver.resolve_lexicon_did("example.com", "api").await.unwrap(), + "did:web:example.com" + ); +} + +#[tokio::test] +async fn test_did_plc_format() { + let mut resolver = MockDnsResolver::new(); + + // Real-world style PLC DIDs + resolver.add_record( + "app.bsky", + "feed", + "did:plc:z72i7hdynmk6r22z27h6tvur".to_string() + ); + + let did = resolver.resolve_lexicon_did("app.bsky", "feed").await.unwrap(); + assert!(did.starts_with("did:plc:")); + assert_eq!(did.len(), 32); // "did:plc:" (8) + 24 chars +} + +#[tokio::test] +async fn test_add_record_from_nsid() { + let mut resolver = MockDnsResolver::new(); + + // Add using full NSID + resolver.add_record_from_nsid("place.stream.key", "did:plc:test".to_string()).unwrap(); + resolver.add_record_from_nsid("app.bsky.actor.profile", "did:plc:bsky".to_string()).unwrap(); + + assert_eq!( + resolver.resolve_lexicon_did("place.stream", "key").await.unwrap(), + "did:plc:test" + ); + assert_eq!( + resolver.resolve_lexicon_did("app.bsky", "actor.profile").await.unwrap(), + "did:plc:bsky" + ); +} + +#[tokio::test] +async fn test_invalid_nsid_format() { + let mut resolver = MockDnsResolver::new(); + + // NSID with only one segment + let result = resolver.add_record_from_nsid("invalid", "did:plc:test".to_string()); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("at least 2 segments")); +} + +#[tokio::test] +async fn test_case_sensitivity() { + let mut resolver = MockDnsResolver::new(); + + // ATProto NSIDs are case-sensitive + resolver.add_record("App.Bsky", "Actor", "did:plc:uppercase".to_string()); + resolver.add_record("app.bsky", "actor", "did:plc:lowercase".to_string()); + + // These should be treated as different domains + assert_eq!( + resolver.resolve_lexicon_did("App.Bsky", "Actor").await.unwrap(), + "did:plc:uppercase" + ); + assert_eq!( + resolver.resolve_lexicon_did("app.bsky", "actor").await.unwrap(), + "did:plc:lowercase" + ); +} + +#[tokio::test] +async fn test_concurrent_lookups() { + use std::sync::Arc; + + let mut resolver = MockDnsResolver::new(); + resolver.add_record("app.bsky", "feed", "did:plc:concurrent".to_string()); + + let resolver = Arc::new(resolver); + let mut handles = vec![]; + + // Spawn 10 tasks doing concurrent lookups + for _ in 0..10 { + let resolver_clone = Arc::clone(&resolver); + let handle = tokio::spawn(async move { + resolver_clone.resolve_lexicon_did("app.bsky", "feed").await.unwrap() + }); + handles.push(handle); + } + + // All should succeed + for handle in handles { + assert_eq!(handle.await.unwrap(), "did:plc:concurrent"); + } +} + +#[tokio::test] +async fn test_wildcard_namespace_scenarios() { + let mut resolver = MockDnsResolver::new(); + + // Simulate multiple lexicons under same namespace + resolver.add_record("app.bsky", "actor.defs", "did:plc:bsky".to_string()); + resolver.add_record("app.bsky", "actor.profile", "did:plc:bsky".to_string()); + resolver.add_record("app.bsky", "feed.post", "did:plc:bsky".to_string()); + resolver.add_record("app.bsky", "feed.like", "did:plc:bsky".to_string()); + + // All should resolve to the same DID + assert_eq!( + resolver.resolve_lexicon_did("app.bsky", "actor.defs").await.unwrap(), + "did:plc:bsky" + ); + assert_eq!( + resolver.resolve_lexicon_did("app.bsky", "feed.post").await.unwrap(), + "did:plc:bsky" + ); +} + +#[tokio::test] +async fn test_dns_name_construction() { + use mlf_lexicon_fetcher::construct_dns_name; + + // Test various NSID patterns + assert_eq!( + construct_dns_name("place.stream", "key"), + "_lexicon.key.stream.place" + ); + assert_eq!( + construct_dns_name("app.bsky", "actor"), + "_lexicon.actor.bsky.app" + ); + assert_eq!( + construct_dns_name("com.atproto", "repo.strongRef"), + "_lexicon.repo.strongRef.atproto.com" + ); +} + +#[tokio::test] +async fn test_nsid_parsing() { + use mlf_lexicon_fetcher::parse_nsid; + + // Test various NSID formats + let (auth, name) = parse_nsid("place.stream.key").unwrap(); + assert_eq!(auth, "place.stream"); + assert_eq!(name, "key"); + + let (auth, name) = parse_nsid("app.bsky.actor.profile").unwrap(); + assert_eq!(auth, "app.bsky"); + assert_eq!(name, "actor.profile"); + + let (auth, name) = parse_nsid("com.atproto.repo.strongRef").unwrap(); + assert_eq!(auth, "com.atproto"); + assert_eq!(name, "repo.strongRef"); +} + +#[tokio::test] +async fn test_edge_case_empty_did() { + let mut resolver = MockDnsResolver::new(); + + // Empty DID should work (though not valid in practice) + resolver.add_record("test.com", "api", "".to_string()); + + assert_eq!( + resolver.resolve_lexicon_did("test.com", "api").await.unwrap(), + "" + ); +} + +#[tokio::test] +async fn test_very_long_nsid() { + let mut resolver = MockDnsResolver::new(); + + // Very long name segments + let long_name = "very.long.deeply.nested.namespace.path.to.definition"; + resolver.add_record("app.bsky", long_name, "did:plc:deep".to_string()); + + assert_eq!( + resolver.resolve_lexicon_did("app.bsky", long_name).await.unwrap(), + "did:plc:deep" + ); +} + +#[tokio::test] +async fn test_special_characters_in_nsid() { + let mut resolver = MockDnsResolver::new(); + + // Hyphens and numbers are valid in domain names + resolver.add_record("app-test.bsky-123", "actor", "did:plc:special".to_string()); + + assert_eq!( + resolver.resolve_lexicon_did("app-test.bsky-123", "actor").await.unwrap(), + "did:plc:special" + ); +} diff --git a/mlf-lexicon-fetcher/tests/lexicon_fetching.rs b/mlf-lexicon-fetcher/tests/lexicon_fetching.rs new file mode 100644 index 0000000..82110c6 --- /dev/null +++ b/mlf-lexicon-fetcher/tests/lexicon_fetching.rs @@ -0,0 +1,360 @@ +// Integration tests for full lexicon fetching flow (DNS + HTTP) + +use mlf_lexicon_fetcher::{ + LexiconFetcher, MockDnsResolver, MockHttpClient, LexiconFetcherError, +}; +use serde_json::json; + +#[tokio::test] +async fn test_fetch_single_lexicon() { + // Setup mock DNS resolver + let mut dns_resolver = MockDnsResolver::new(); + dns_resolver.add_record("place.stream", "chat.profile", "did:plc:test123".to_string()); + + // Setup mock HTTP client + let mut http_client = MockHttpClient::new(); + let lexicon_json = json!({ + "lexicon": 1, + "id": "place.stream.chat.profile", + "defs": { + "main": { + "type": "record", + "description": "A chat profile record" + } + } + }); + http_client.add_lexicon("place.stream.chat.profile".to_string(), lexicon_json.clone()); + + // Create fetcher + let fetcher = LexiconFetcher::new(dns_resolver, http_client); + + // Fetch single lexicon + let result = fetcher.fetch("place.stream.chat.profile").await; + assert!(result.is_ok()); + let fetched = result.unwrap(); + assert_eq!(fetched.get("id").unwrap().as_str().unwrap(), "place.stream.chat.profile"); +} + +#[tokio::test] +async fn test_fetch_pattern_multiple_lexicons() { + // Setup mock DNS resolver + let mut dns_resolver = MockDnsResolver::new(); + dns_resolver.add_record("place.stream", "chat", "did:plc:test123".to_string()); + + // Setup mock HTTP client with multiple lexicons + let mut http_client = MockHttpClient::new(); + + let profile_lexicon = json!({ + "lexicon": 1, + "id": "place.stream.chat.profile", + "defs": { "main": { "type": "record" } } + }); + let message_lexicon = json!({ + "lexicon": 1, + "id": "place.stream.chat.message", + "defs": { "main": { "type": "record" } } + }); + let room_lexicon = json!({ + "lexicon": 1, + "id": "place.stream.chat.room", + "defs": { "main": { "type": "record" } } + }); + + http_client.add_lexicon("place.stream.chat.profile".to_string(), profile_lexicon); + http_client.add_lexicon("place.stream.chat.message".to_string(), message_lexicon); + http_client.add_lexicon("place.stream.chat.room".to_string(), room_lexicon); + + // Create fetcher + let fetcher = LexiconFetcher::new(dns_resolver, http_client); + + // Fetch pattern + let result = fetcher.fetch_pattern("place.stream.chat.*").await; + assert!(result.is_ok()); + let lexicons = result.unwrap(); + + // Should get 3 lexicons + assert_eq!(lexicons.len(), 3); + + // Check all NSIDs are present + let nsids: Vec<&str> = lexicons.iter().map(|(nsid, _)| nsid.as_str()).collect(); + assert!(nsids.contains(&"place.stream.chat.profile")); + assert!(nsids.contains(&"place.stream.chat.message")); + assert!(nsids.contains(&"place.stream.chat.room")); +} + +#[tokio::test] +async fn test_fetch_lexicon_not_found() { + // Setup mock DNS resolver + let mut dns_resolver = MockDnsResolver::new(); + dns_resolver.add_record("place.stream", "chat.profile", "did:plc:test123".to_string()); + + // Setup mock HTTP client (but don't add the lexicon) + let http_client = MockHttpClient::new(); + + // Create fetcher + let fetcher = LexiconFetcher::new(dns_resolver, http_client); + + // Try to fetch non-existent lexicon + let result = fetcher.fetch("place.stream.chat.profile").await; + assert!(result.is_err()); + + match result { + Err(LexiconFetcherError::LexiconNotFound(nsid)) => { + assert_eq!(nsid, "place.stream.chat.profile"); + } + _ => panic!("Expected LexiconNotFound error"), + } +} + +#[tokio::test] +async fn test_fetch_dns_lookup_failed() { + // Setup mock DNS resolver (but don't add the record) + let dns_resolver = MockDnsResolver::new(); + + // Setup mock HTTP client + let http_client = MockHttpClient::new(); + + // Create fetcher + let fetcher = LexiconFetcher::new(dns_resolver, http_client); + + // Try to fetch with no DNS record + let result = fetcher.fetch("place.stream.chat.profile").await; + assert!(result.is_err()); + + match result { + Err(LexiconFetcherError::LookupFailed { domain, .. }) => { + assert_eq!(domain, "_lexicon.chat.profile.stream.place"); + } + _ => panic!("Expected LookupFailed error"), + } +} + +#[tokio::test] +async fn test_fetch_with_wildcard_returns_error() { + let dns_resolver = MockDnsResolver::new(); + let http_client = MockHttpClient::new(); + let fetcher = LexiconFetcher::new(dns_resolver, http_client); + + // Try to use wildcard with fetch() instead of fetch_pattern() + let result = fetcher.fetch("place.stream.*").await; + assert!(result.is_err()); + + match result { + Err(LexiconFetcherError::InvalidNsid(msg)) => { + assert!(msg.contains("fetch_pattern()")); + } + _ => panic!("Expected InvalidNsid error"), + } +} + +#[tokio::test] +async fn test_fetch_pattern_empty_results() { + // Setup mock DNS resolver + let mut dns_resolver = MockDnsResolver::new(); + dns_resolver.add_record("place.stream", "chat", "did:plc:test123".to_string()); + + // Setup mock HTTP client with no matching lexicons + let mut http_client = MockHttpClient::new(); + http_client.add_lexicon("place.stream.other.thing".to_string(), json!({})); + + // Create fetcher + let fetcher = LexiconFetcher::new(dns_resolver, http_client); + + // Fetch pattern that matches nothing + let result = fetcher.fetch_pattern("place.stream.chat.*").await; + assert!(result.is_ok()); + let lexicons = result.unwrap(); + assert_eq!(lexicons.len(), 0); +} + +#[tokio::test] +async fn test_multiple_authorities() { + // Setup mock DNS resolver with multiple authorities + let mut dns_resolver = MockDnsResolver::new(); + dns_resolver.add_record("place.stream", "chat.profile", "did:plc:stream123".to_string()); + dns_resolver.add_record("app.bsky", "actor.profile", "did:plc:bsky456".to_string()); + + // Setup mock HTTP client + let mut http_client = MockHttpClient::new(); + http_client.add_lexicon( + "place.stream.chat.profile".to_string(), + json!({"id": "place.stream.chat.profile"}), + ); + http_client.add_lexicon( + "app.bsky.actor.profile".to_string(), + json!({"id": "app.bsky.actor.profile"}), + ); + + // Create fetcher + let fetcher = LexiconFetcher::new(dns_resolver, http_client); + + // Fetch from both authorities + let result1 = fetcher.fetch("place.stream.chat.profile").await; + assert!(result1.is_ok()); + + let result2 = fetcher.fetch("app.bsky.actor.profile").await; + assert!(result2.is_ok()); +} + +#[tokio::test] +async fn test_nested_name_segments() { + // Setup mock DNS resolver + let mut dns_resolver = MockDnsResolver::new(); + dns_resolver.add_record( + "place.stream", + "chat.message.attachments.image", + "did:plc:test123".to_string(), + ); + + // Setup mock HTTP client + let mut http_client = MockHttpClient::new(); + http_client.add_lexicon( + "place.stream.chat.message.attachments.image".to_string(), + json!({"id": "place.stream.chat.message.attachments.image"}), + ); + + // Create fetcher + let fetcher = LexiconFetcher::new(dns_resolver, http_client); + + // Fetch deeply nested lexicon + let result = fetcher.fetch("place.stream.chat.message.attachments.image").await; + assert!(result.is_ok()); +} + +#[tokio::test] +async fn test_concurrent_fetches() { + use std::sync::Arc; + use tokio::task; + + // Setup mock DNS resolver + let mut dns_resolver = MockDnsResolver::new(); + dns_resolver.add_record("place.stream", "chat.profile", "did:plc:test123".to_string()); + + // Setup mock HTTP client + let mut http_client = MockHttpClient::new(); + http_client.add_lexicon( + "place.stream.chat.profile".to_string(), + json!({"id": "place.stream.chat.profile"}), + ); + + // Create fetcher and wrap in Arc + let fetcher = Arc::new(LexiconFetcher::new(dns_resolver, http_client)); + + // Spawn multiple concurrent fetch tasks + let mut handles = vec![]; + for _ in 0..10 { + let fetcher_clone = Arc::clone(&fetcher); + let handle = task::spawn(async move { + fetcher_clone.fetch("place.stream.chat.profile").await + }); + handles.push(handle); + } + + // All should succeed + for handle in handles { + let result = handle.await.unwrap(); + assert!(result.is_ok()); + } +} + +#[tokio::test] +async fn test_pattern_prefix_matching() { + // Setup mock DNS resolver + let mut dns_resolver = MockDnsResolver::new(); + dns_resolver.add_record("app.bsky", "feed", "did:plc:test123".to_string()); + + // Setup mock HTTP client + let mut http_client = MockHttpClient::new(); + http_client.add_lexicon("app.bsky.feed.post".to_string(), json!({"id": "app.bsky.feed.post"})); + http_client.add_lexicon("app.bsky.feed.like".to_string(), json!({"id": "app.bsky.feed.like"})); + http_client.add_lexicon("app.bsky.feed.repost".to_string(), json!({"id": "app.bsky.feed.repost"})); + http_client.add_lexicon("app.bsky.actor.profile".to_string(), json!({"id": "app.bsky.actor.profile"})); + + // Create fetcher + let fetcher = LexiconFetcher::new(dns_resolver, http_client); + + // Fetch only feed.* lexicons + let result = fetcher.fetch_pattern("app.bsky.feed.*").await; + assert!(result.is_ok()); + let lexicons = result.unwrap(); + + // Should only get feed.* lexicons (not actor.profile) + assert_eq!(lexicons.len(), 3); + for (nsid, _) in lexicons { + assert!(nsid.starts_with("app.bsky.feed.")); + } +} + +#[tokio::test] +async fn test_underscore_wildcard_direct_children() { + // Setup mock DNS resolver + let mut dns_resolver = MockDnsResolver::new(); + dns_resolver.add_record("place.stream", "", "did:plc:test123".to_string()); + + // Setup mock HTTP client with nested lexicons + let mut http_client = MockHttpClient::new(); + + // Direct children of place.stream + http_client.add_lexicon("place.stream.chat".to_string(), json!({"id": "place.stream.chat"})); + http_client.add_lexicon("place.stream.key".to_string(), json!({"id": "place.stream.key"})); + http_client.add_lexicon("place.stream.livestream".to_string(), json!({"id": "place.stream.livestream"})); + + // Nested children (should NOT match with _) + http_client.add_lexicon("place.stream.chat.profile".to_string(), json!({"id": "place.stream.chat.profile"})); + http_client.add_lexicon("place.stream.chat.message".to_string(), json!({"id": "place.stream.chat.message"})); + http_client.add_lexicon("place.stream.key.defs".to_string(), json!({"id": "place.stream.key.defs"})); + + // Create fetcher + let fetcher = LexiconFetcher::new(dns_resolver, http_client); + + // Test underscore wildcard (direct children only) + let result = fetcher.fetch_pattern("place.stream._").await; + assert!(result.is_ok()); + let lexicons = result.unwrap(); + + // Should only get 3 direct children + assert_eq!(lexicons.len(), 3); + + let nsids: Vec<&str> = lexicons.iter().map(|(nsid, _)| nsid.as_str()).collect(); + assert!(nsids.contains(&"place.stream.chat")); + assert!(nsids.contains(&"place.stream.key")); + assert!(nsids.contains(&"place.stream.livestream")); + + // Should NOT contain nested children + assert!(!nsids.contains(&"place.stream.chat.profile")); + assert!(!nsids.contains(&"place.stream.chat.message")); + assert!(!nsids.contains(&"place.stream.key.defs")); +} + +#[tokio::test] +async fn test_star_vs_underscore_wildcard() { + // Setup mock DNS resolver + let mut dns_resolver = MockDnsResolver::new(); + dns_resolver.add_record("place.stream", "", "did:plc:test123".to_string()); + + // Setup mock HTTP client with nested lexicons + let mut http_client = MockHttpClient::new(); + http_client.add_lexicon("place.stream.chat".to_string(), json!({"id": "place.stream.chat"})); + http_client.add_lexicon("place.stream.chat.profile".to_string(), json!({"id": "place.stream.chat.profile"})); + http_client.add_lexicon("place.stream.key".to_string(), json!({"id": "place.stream.key"})); + + // Create fetcher + let fetcher = LexiconFetcher::new(dns_resolver, http_client); + + // Test star wildcard (all descendants) + let star_result = fetcher.fetch_pattern("place.stream.*").await; + assert!(star_result.is_ok()); + let star_lexicons = star_result.unwrap(); + assert_eq!(star_lexicons.len(), 3); // All 3 lexicons + + // Test underscore wildcard (direct children only) + let underscore_result = fetcher.fetch_pattern("place.stream._").await; + assert!(underscore_result.is_ok()); + let underscore_lexicons = underscore_result.unwrap(); + assert_eq!(underscore_lexicons.len(), 2); // Only direct children (chat, key) + + let nsids: Vec<&str> = underscore_lexicons.iter().map(|(nsid, _)| nsid.as_str()).collect(); + assert!(nsids.contains(&"place.stream.chat")); + assert!(nsids.contains(&"place.stream.key")); + assert!(!nsids.contains(&"place.stream.chat.profile")); +} diff --git a/tests/Cargo.toml b/tests/Cargo.toml index bd5af74..91a5cea 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -12,9 +12,11 @@ path = "lib.rs" mlf-lang = { path = "../mlf-lang" } mlf-codegen = { path = "../mlf-codegen" } mlf-diagnostics = { path = "../mlf-diagnostics" } +mlf-lexicon-fetcher = { path = "../mlf-lexicon-fetcher" } serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } toml = "0.8" +tokio = { version = "1", features = ["full"] } [dev-dependencies] # Any additional test dependencies @@ -26,3 +28,7 @@ path = "codegen_integration.rs" [[test]] name = "diagnostics_integration" path = "diagnostics_integration.rs" + +[[test]] +name = "lexicon_fetcher_integration" +path = "lexicon_fetcher_integration.rs" diff --git a/tests/codegen_integration.rs b/tests/codegen_integration.rs index 5cb8d00..6f2a8d0 100644 --- a/tests/codegen_integration.rs +++ b/tests/codegen_integration.rs @@ -27,7 +27,7 @@ fn codegen_lexicon_tests() { }) .collect(); - let (passed, failed) = test_utils::run_and_report_tests:: Result<(), String>>(tests, "Codegen"); + let (_passed, failed) = test_utils::run_and_report_tests:: Result<(), String>>(tests, "Codegen"); if !failed.is_empty() { panic!( diff --git a/tests/diagnostics_integration.rs b/tests/diagnostics_integration.rs index 2721f81..54f1767 100644 --- a/tests/diagnostics_integration.rs +++ b/tests/diagnostics_integration.rs @@ -5,7 +5,6 @@ use mlf_diagnostics::{get_error_module_namespace_str, ValidationDiagnostic}; use mlf_integration_tests::test_utils; use mlf_lang::{parser::parse_lexicon, Workspace}; use serde::Deserialize; -use serde_json::Value; use std::fs; use std::path::Path; @@ -17,15 +16,19 @@ struct ExpectedDiagnostic { #[derive(Debug, Deserialize)] struct ExpectedError { + #[allow(dead_code)] code: String, message: String, #[serde(default)] + #[allow(dead_code)] span: Option, } #[derive(Debug, Deserialize)] struct ExpectedSpan { + #[allow(dead_code)] start: usize, + #[allow(dead_code)] end: usize, } @@ -47,7 +50,7 @@ fn diagnostics_tests() { }) .collect(); - let (passed, failed) = test_utils::run_and_report_tests:: Result<(), String>>(tests, "Diagnostics"); + let (_passed, failed) = test_utils::run_and_report_tests:: Result<(), String>>(tests, "Diagnostics"); if !failed.is_empty() { panic!( @@ -96,7 +99,7 @@ fn run_diagnostics_test(test_dir: &str) -> Result<(), String> { }; // 5. Create diagnostic - let diagnostic = ValidationDiagnostic::new( + let _diagnostic = ValidationDiagnostic::new( "input.mlf".to_string(), input.clone(), namespace.clone(), @@ -135,7 +138,7 @@ fn run_diagnostics_test(test_dir: &str) -> Result<(), String> { let actual_error = errors_in_module[i]; // Check error code - let actual_code = mlf_diagnostics::get_error_module_namespace_str(actual_error); + let _actual_code = mlf_diagnostics::get_error_module_namespace_str(actual_error); // Format the error message let actual_message = format!("{:?}", actual_error); diff --git a/tests/lexicon_fetcher/dns/basic_resolution/expected.json b/tests/lexicon_fetcher/dns/basic_resolution/expected.json new file mode 100644 index 0000000..684d048 --- /dev/null +++ b/tests/lexicon_fetcher/dns/basic_resolution/expected.json @@ -0,0 +1,4 @@ +{ + "status": "success", + "did": "did:plc:test123" +} diff --git a/tests/lexicon_fetcher/dns/basic_resolution/test.toml b/tests/lexicon_fetcher/dns/basic_resolution/test.toml new file mode 100644 index 0000000..c3a0943 --- /dev/null +++ b/tests/lexicon_fetcher/dns/basic_resolution/test.toml @@ -0,0 +1,4 @@ +[test] +description = "Basic DNS resolution for place.stream.chat.profile" +nsid = "place.stream.chat.profile" +did = "did:plc:test123" diff --git a/tests/lexicon_fetcher/dns/missing_record/expected.json b/tests/lexicon_fetcher/dns/missing_record/expected.json new file mode 100644 index 0000000..56aff91 --- /dev/null +++ b/tests/lexicon_fetcher/dns/missing_record/expected.json @@ -0,0 +1,4 @@ +{ + "status": "error", + "error": "lookup_failed" +} diff --git a/tests/lexicon_fetcher/dns/missing_record/test.toml b/tests/lexicon_fetcher/dns/missing_record/test.toml new file mode 100644 index 0000000..fb15d7f --- /dev/null +++ b/tests/lexicon_fetcher/dns/missing_record/test.toml @@ -0,0 +1,4 @@ +[test] +description = "DNS lookup for non-existent record" +nsid = "nonexistent.domain.test" +should_fail = true diff --git a/tests/lexicon_fetcher/dns/nested_segments/expected.json b/tests/lexicon_fetcher/dns/nested_segments/expected.json new file mode 100644 index 0000000..ec3b7f5 --- /dev/null +++ b/tests/lexicon_fetcher/dns/nested_segments/expected.json @@ -0,0 +1,4 @@ +{ + "status": "success", + "did": "did:plc:nested789" +} diff --git a/tests/lexicon_fetcher/dns/nested_segments/test.toml b/tests/lexicon_fetcher/dns/nested_segments/test.toml new file mode 100644 index 0000000..be5436a --- /dev/null +++ b/tests/lexicon_fetcher/dns/nested_segments/test.toml @@ -0,0 +1,4 @@ +[test] +description = "DNS resolution for deeply nested NSID app.bsky.actor.profile.detailed" +nsid = "app.bsky.actor.profile.detailed" +did = "did:plc:nested789" diff --git a/tests/lexicon_fetcher/dns/wildcard_pattern/expected.json b/tests/lexicon_fetcher/dns/wildcard_pattern/expected.json new file mode 100644 index 0000000..d96d407 --- /dev/null +++ b/tests/lexicon_fetcher/dns/wildcard_pattern/expected.json @@ -0,0 +1,4 @@ +{ + "status": "success", + "did": "did:plc:test456" +} diff --git a/tests/lexicon_fetcher/dns/wildcard_pattern/test.toml b/tests/lexicon_fetcher/dns/wildcard_pattern/test.toml new file mode 100644 index 0000000..974de08 --- /dev/null +++ b/tests/lexicon_fetcher/dns/wildcard_pattern/test.toml @@ -0,0 +1,4 @@ +[test] +description = "DNS resolution for wildcard pattern place.stream.chat.*" +nsid = "place.stream.chat.*" +did = "did:plc:test456" diff --git a/tests/lexicon_fetcher_integration.rs b/tests/lexicon_fetcher_integration.rs new file mode 100644 index 0000000..3bf693a --- /dev/null +++ b/tests/lexicon_fetcher_integration.rs @@ -0,0 +1,220 @@ +// Workspace-level integration tests for lexicon fetcher +// Tests mlf-lexicon-fetcher working with mocks + +use mlf_lexicon_fetcher::{DnsResolver, LexiconFetcher, MockDnsResolver, MockHttpClient}; +use serde::{Deserialize, Serialize}; +use std::fs; +use std::path::Path; + +#[derive(Debug, Deserialize)] +struct TestConfig { + test: TestMetadata, +} + +#[derive(Debug, Deserialize)] +struct TestMetadata { + description: String, + nsid: String, + #[serde(default)] + did: Option, + #[serde(default)] + should_fail: bool, +} + +#[derive(Debug, Deserialize, Serialize, PartialEq)] +struct ExpectedResult { + status: String, + #[serde(skip_serializing_if = "Option::is_none")] + did: Option, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +#[tokio::test] +async fn lexicon_fetcher_dns_tests() { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let test_base = format!("{}/lexicon_fetcher/dns", manifest_dir); + let test_dirs = discover_test_dirs(&test_base); + + let mut tests = Vec::new(); + + for test_dir in test_dirs { + let test_name = format!( + "lexicon_fetcher/dns/{}", + Path::new(&test_dir).file_name().unwrap().to_str().unwrap() + ); + let result = run_dns_test(&test_dir).await; + tests.push((test_name, result)); + } + + let (passed, failed) = run_and_report_tests(tests, "Lexicon Fetcher DNS"); + + if !failed.is_empty() { + panic!( + "\nFailed tests:\n{}", + failed + .iter() + .map(|(name, err)| format!(" - {}: {}", name, err)) + .collect::>() + .join("\n") + ); + } + + assert!(passed > 0, "No tests were run"); +} + +async fn run_dns_test(test_dir: &str) -> Result<(), String> { + // Load test configuration + let config_path = format!("{}/test.toml", test_dir); + let config_str = fs::read_to_string(&config_path) + .map_err(|e| format!("Failed to read test.toml: {}", e))?; + let config: TestConfig = toml::from_str(&config_str) + .map_err(|e| format!("Failed to parse test.toml: {}", e))?; + + // Load expected output + let expected_path = format!("{}/expected.json", test_dir); + let expected_str = fs::read_to_string(&expected_path) + .map_err(|e| format!("Failed to read expected.json: {}", e))?; + let expected: ExpectedResult = serde_json::from_str(&expected_str) + .map_err(|e| format!("Failed to parse expected.json: {}", e))?; + + // Setup mocks + let mut dns_resolver = MockDnsResolver::new(); + let http_client = MockHttpClient::new(); + + // Add DNS record if expected to succeed + if expected.status == "success" { + let (authority, name_segments) = parse_nsid(&config.test.nsid)?; + let did = config.test.did.clone().ok_or_else(|| "Missing DID in test config".to_string())?; + + // For wildcard patterns, remove the .* suffix + let dns_name = if name_segments.ends_with(".*") { + name_segments.strip_suffix(".*").unwrap() + } else { + &name_segments + }; + + dns_resolver.add_record(&authority, dns_name, did.clone()); + } + + // Create fetcher + let fetcher = LexiconFetcher::new(dns_resolver, http_client); + + // Test DNS resolution by attempting to extract the DID + // We can't directly test DNS resolution without HTTP, so we test the full flow + // but expect it to fail at HTTP stage (which is fine for DNS testing) + let (authority, name_segments) = parse_nsid(&config.test.nsid)?; + + // For wildcard patterns, strip the .* suffix + let dns_name = if name_segments.ends_with(".*") { + name_segments.strip_suffix(".*").unwrap().to_string() + } else { + name_segments + }; + + // Create a test resolver directly to check DNS + let mut test_dns = MockDnsResolver::new(); + if expected.status == "success" { + let did = config.test.did.clone().ok_or_else(|| "Missing DID in test config".to_string())?; + test_dns.add_record(&authority, &dns_name, did.clone()); + + // Verify DNS resolution + match test_dns.resolve_lexicon_did(&authority, &dns_name).await { + Ok(resolved_did) => { + if resolved_did != did { + return Err(format!("DID mismatch: expected {}, got {}", did, resolved_did)); + } + // Check against expected + if let Some(expected_did) = &expected.did { + if &resolved_did != expected_did { + return Err(format!("DID mismatch with expected: expected {}, got {}", expected_did, resolved_did)); + } + } + } + Err(e) => return Err(format!("DNS resolution failed: {:?}", e)), + } + } else { + // Expected to fail + match test_dns.resolve_lexicon_did(&authority, &dns_name).await { + Ok(_) => return Err("Expected DNS lookup to fail, but it succeeded".to_string()), + Err(_) => { + // Success - it failed as expected + } + } + } + + Ok(()) +} + +fn parse_nsid(nsid: &str) -> Result<(String, String), String> { + // Remove wildcard if present for parsing + let nsid_base = nsid.strip_suffix(".*").unwrap_or(nsid); + + let parts: Vec<&str> = nsid_base.split('.').collect(); + if parts.len() < 2 { + return Err(format!("Invalid NSID: {}", nsid)); + } + + let authority = format!("{}.{}", parts[0], parts[1]); + let name_segments = if parts.len() > 2 { + let mut segments = parts[2..].join("."); + // Re-add wildcard if original had it + if nsid.ends_with(".*") { + segments.push_str(".*"); + } + segments + } else if nsid.ends_with(".*") { + ".*".to_string() + } else { + String::new() + }; + + Ok((authority, name_segments)) +} + +fn discover_test_dirs(base: &str) -> Vec { + let base_path = Path::new(base); + if !base_path.exists() { + return vec![]; + } + + let mut dirs: Vec = fs::read_dir(base_path) + .unwrap() + .filter_map(|entry| { + let entry = entry.ok()?; + let path = entry.path(); + if path.is_dir() { + Some(path.to_str()?.to_string()) + } else { + None + } + }) + .collect(); + + dirs.sort(); + dirs +} + +fn run_and_report_tests( + tests: Vec<(String, Result<(), String>)>, + test_type: &str, +) -> (usize, Vec<(String, String)>) { + let mut passed = 0; + let mut failed = Vec::new(); + + for (test_name, result) in tests { + match result { + Ok(()) => { + println!("✓ {}", test_name); + passed += 1; + } + Err(err) => { + println!("✗ {}: {}", test_name, err); + failed.push((test_name, err)); + } + } + } + + println!("\n{} Results: {} passed, {} failed", test_type, passed, failed.len()); + (passed, failed) +}