diff --git a/mlf-lexicon-fetcher/README.md b/mlf-lexicon-fetcher/README.md new file mode 100644 index 0000000..8832e9a --- /dev/null +++ b/mlf-lexicon-fetcher/README.md @@ -0,0 +1,137 @@ +# mlf-lexicon-fetcher + +ATProto Lexicon fetcher with DNS resolution and HTTP client. + +Resolves lexicon NSIDs to DIDs via DNS TXT records and fetches lexicon JSON from ATProto repositories. + +## Features + +- **DNS Resolution**: Resolves NSIDs to DIDs using DNS TXT records (RFC spec) +- **HTTP Fetching**: Fetches lexicons from ATProto repositories via XRPC +- **Pattern Matching**: Supports wildcard patterns (`.*` and `._`) +- **Network Optimization**: Groups similar NSIDs to reduce HTTP requests +- **Lockfile Support**: Fetches from known DIDs, bypassing DNS resolution +- **Testable**: Mock DNS and HTTP clients for testing + +## Usage + +### Basic Fetching + +```rust +use mlf_lexicon_fetcher::ProductionLexiconFetcher; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Create a production fetcher (real DNS + HTTP) + let fetcher = ProductionLexiconFetcher::production().await?; + + // Fetch a single lexicon with metadata + let result = fetcher.fetch_with_metadata("app.bsky.feed.post").await?; + + for lexicon in result.lexicons { + println!("NSID: {}", lexicon.nsid); + println!("DID: {}", lexicon.did); + println!("Lexicon: {}", serde_json::to_string_pretty(&lexicon.lexicon)?); + } + + Ok(()) +} +``` + +### Pattern Matching + +```rust +// Fetch all lexicons under a namespace +let result = fetcher.fetch_with_metadata("app.bsky.feed.*").await?; +println!("Fetched {} lexicons", result.lexicons.len()); + +// Fetch direct children only +let result = fetcher.fetch_with_metadata("app.bsky._").await?; +``` + +### Optimized Batch Fetching + +```rust +use mlf_lexicon_fetcher::ProductionLexiconFetcher; + +// Fetch many NSIDs with automatic optimization +let nsids = vec![ + "app.bsky.actor.profile".to_string(), + "app.bsky.actor.defs".to_string(), + "app.bsky.feed.post".to_string(), + "app.bsky.feed.like".to_string(), +]; + +// Automatically groups into patterns like "app.bsky.actor.*" and "app.bsky.feed.*" +let results = fetcher.fetch_many_optimized(&nsids).await?; +``` + +### Lockfile Mode (Skip DNS) + +```rust +// When you already have the DID (e.g., from a lockfile) +let result = fetcher.fetch_from_did_with_metadata( + "did:plc:abc123", + "app.bsky.feed.post" +).await?; +``` + +### Advanced: Pattern Optimization + +```rust +use std::collections::HashSet; +use mlf_lexicon_fetcher::optimize_fetch_patterns; + +let nsids: HashSet = vec![ + "app.bsky.actor.foo".to_string(), + "app.bsky.actor.bar".to_string(), + "app.bsky.feed.post".to_string(), +].into_iter().collect(); + +// Returns: ["app.bsky.actor.*", "app.bsky.feed.post"] +let optimized = optimize_fetch_patterns(&nsids); +``` + +## API + +### Main Types + +- `ProductionLexiconFetcher` - Ready-to-use fetcher with real DNS and HTTP +- `LexiconFetcher` - Generic fetcher (for custom DNS/HTTP implementations) +- `FetchResult` - Contains fetched lexicons with metadata (NSID, DID, JSON) + +### Key Methods + +- `fetch_with_metadata(nsid)` - Fetch single/pattern, returns metadata +- `fetch_many(nsids)` - Fetch multiple NSIDs sequentially +- `fetch_many_optimized(nsids)` - Fetch multiple NSIDs with pattern optimization +- `fetch_from_did_with_metadata(did, nsid)` - Skip DNS, fetch from known DID + +### Utilities + +- `optimize_fetch_patterns(nsids)` - Group NSIDs into minimal patterns +- `parse_nsid(nsid)` - Parse NSID into authority and name segments +- `construct_dns_name(authority, name)` - Build DNS TXT record name + +## Testing + +Use mock implementations for testing: + +```rust +use mlf_lexicon_fetcher::{LexiconFetcher, MockDnsResolver, MockHttpClient}; + +let mut dns = MockDnsResolver::new(); +dns.add_record("app.bsky", "feed", "did:plc:test".to_string()); + +let mut http = MockHttpClient::new(); +http.add_lexicon( + "app.bsky.feed.post".to_string(), + serde_json::json!({"lexicon": 1, "id": "app.bsky.feed.post"}) +); + +let fetcher = LexiconFetcher::new(dns, http); +``` + +## License + +MIT diff --git a/mlf-lexicon-fetcher/examples/usage.rs b/mlf-lexicon-fetcher/examples/usage.rs index 75dddce..e86b9bd 100644 --- a/mlf-lexicon-fetcher/examples/usage.rs +++ b/mlf-lexicon-fetcher/examples/usage.rs @@ -5,8 +5,8 @@ use serde_json::json; #[tokio::main] async fn main() -> Result<(), Box> { - // Example 1: Fetch a single lexicon - println!("=== Example 1: Fetch Single Lexicon ==="); + // Example 1: Fetch with metadata (NEW!) + println!("=== Example 1: Fetch with Metadata ==="); let mut dns_resolver = MockDnsResolver::new(); dns_resolver.add_record("place.stream", "chat.profile", "did:plc:test123".to_string()); @@ -28,16 +28,21 @@ async fn main() -> Result<(), Box> { let fetcher = LexiconFetcher::new(dns_resolver, http_client); - match fetcher.fetch("place.stream.chat.profile").await { - Ok(lexicon) => { - println!("Successfully fetched lexicon:"); - println!("{}", serde_json::to_string_pretty(&lexicon)?); + // New API returns metadata (DID, NSID, lexicon) + match fetcher.fetch_with_metadata("place.stream.chat.profile").await { + Ok(result) => { + println!("Successfully fetched {} lexicon(s):", result.lexicons.len()); + for fetched in result.lexicons { + println!(" NSID: {}", fetched.nsid); + println!(" DID: {}", fetched.did); + println!(" Lexicon: {}", serde_json::to_string_pretty(&fetched.lexicon)?); + } } Err(e) => eprintln!("Error: {}", e), } - // Example 2: Fetch multiple lexicons with a pattern - println!("\n=== Example 2: Fetch Multiple Lexicons with Pattern ==="); + // Example 2: Fetch multiple with pattern + println!("\n=== Example 2: Fetch Multiple with Pattern ==="); let mut dns_resolver2 = MockDnsResolver::new(); dns_resolver2.add_record("app.bsky", "feed", "did:plc:bsky123".to_string()); @@ -58,11 +63,68 @@ async fn main() -> Result<(), Box> { let fetcher2 = LexiconFetcher::new(dns_resolver2, http_client2); - match fetcher2.fetch_pattern("app.bsky.feed.*").await { - Ok(lexicons) => { - println!("Successfully fetched {} lexicons:", lexicons.len()); - for (nsid, _lexicon) in lexicons { - println!(" - {}", nsid); + match fetcher2.fetch_with_metadata("app.bsky.feed.*").await { + Ok(result) => { + println!("Successfully fetched {} lexicons:", result.lexicons.len()); + for fetched in result.lexicons { + println!(" - {} (from {})", fetched.nsid, fetched.did); + } + } + Err(e) => eprintln!("Error: {}", e), + } + + // Example 3: Optimized batch fetching (NEW!) + println!("\n=== Example 3: Optimized Batch Fetching ==="); + + let mut dns_resolver3 = MockDnsResolver::new(); + dns_resolver3.add_record("app.bsky", "actor", "did:plc:bsky123".to_string()); + + let mut http_client3 = MockHttpClient::new(); + http_client3.add_lexicon( + "app.bsky.actor.profile".to_string(), + json!({"lexicon": 1, "id": "app.bsky.actor.profile"}), + ); + http_client3.add_lexicon( + "app.bsky.actor.defs".to_string(), + json!({"lexicon": 1, "id": "app.bsky.actor.defs"}), + ); + + let fetcher3 = LexiconFetcher::new(dns_resolver3, http_client3); + + let nsids = vec![ + "app.bsky.actor.profile".to_string(), + "app.bsky.actor.defs".to_string(), + ]; + + // Automatically optimizes to "app.bsky.actor.*" pattern + match fetcher3.fetch_many_optimized(&nsids).await { + Ok(results) => { + println!("Optimized fetch completed:"); + for result in results { + println!(" Batch of {} lexicons", result.lexicons.len()); + } + } + Err(e) => eprintln!("Error: {}", e), + } + + // Example 4: Fetch from known DID (NEW!) + println!("\n=== Example 4: Fetch from Known DID (Lockfile Mode) ==="); + + let dns_resolver4 = MockDnsResolver::new(); + let mut http_client4 = MockHttpClient::new(); + http_client4.add_lexicon( + "app.bsky.feed.post".to_string(), + json!({"lexicon": 1, "id": "app.bsky.feed.post"}), + ); + + let fetcher4 = LexiconFetcher::new(dns_resolver4, http_client4); + + // Skip DNS resolution when DID is already known (e.g., from lockfile) + match fetcher4.fetch_from_did_with_metadata("did:plc:bsky123", "app.bsky.feed.post").await { + Ok(result) => { + println!("Fetched from known DID:"); + for fetched in result.lexicons { + println!(" - {} (bypassed DNS)", fetched.nsid); } } Err(e) => eprintln!("Error: {}", e), diff --git a/website/content/docs/cli/07-fetch.md b/website/content/docs/cli/07-fetch.md index e2aef7f..03e5999 100644 --- a/website/content/docs/cli/07-fetch.md +++ b/website/content/docs/cli/07-fetch.md @@ -31,7 +31,8 @@ mlf fetch --save **Arguments:** - `[NSID]` - Optional NSID or pattern to fetch: - Specific lexicon: `com.example.forum.post` - - Wildcard pattern: `com.example.forum.*` + - All descendants: `com.example.forum.*` (matches `post`, `comment`, `comment.reply`, etc.) + - Direct children only: `com.example.forum._` (matches `post`, `comment`, but NOT `comment.reply`) - Real-world example: `app.bsky.feed.*` **Options:** @@ -183,13 +184,31 @@ mlf fetch com.example.forum.post This downloads only the `com.example.forum.post` lexicon. -### Fetch with Wildcard +### Fetch with Wildcards +**All descendants (`.*`):** ```bash mlf fetch com.example.forum.* ``` +Downloads all lexicons under the `com.example.forum` namespace (recursive). -This downloads all lexicons under the `com.example.forum` namespace. +**Example:** If the namespace contains: +- `com.example.forum.post` +- `com.example.forum.comment` +- `com.example.forum.comment.reply` + +The `.*` pattern fetches **all three**. + +**Direct children only (`._`):** +```bash +mlf fetch com.example.forum._ +``` +Downloads only direct children of `com.example.forum` (non-recursive). + +Using the same example namespace, the `._` pattern fetches: +- `com.example.forum.post` ✓ +- `com.example.forum.comment` ✓ +- `com.example.forum.comment.reply` ✗ (skipped, not a direct child) ### Fetch and Save