diff --git a/Cargo.lock b/Cargo.lock index a7e96a1..c39cc24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1362,6 +1362,7 @@ dependencies = [ "mlf-lexicon-fetcher", "serde", "serde_json", + "tempfile", "tokio", "toml", ] diff --git a/justfile b/justfile index f541055..179b133 100644 --- a/justfile +++ b/justfile @@ -42,10 +42,12 @@ test-workspace: @echo "\nRunning workspace resolution tests..." cargo test -p mlf-integration-tests --test workspace_integration -- --nocapture -# Run real-world lexicon tests (when implemented) +# Run real-world round-trip tests (network-dependent, ignored by default) test-real-world: - @echo "\nRunning real-world lexicon tests..." - cargo test -p mlf-integration-tests --test real_world_integration -- --nocapture + @echo "\n🌐 Running real-world round-trip tests (fetches from network)..." + @echo "This will download lexicons from: app.bsky.*, net.anisota.*, place.stream.*, pub.leaflet.*" + @echo "" + cargo test -p mlf-integration-tests --test real_world_roundtrip -- --ignored --nocapture # Run all workspace tests (excluding problematic packages) test-all: diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..67e2daa --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,2 @@ +# Real-world test artifacts (generated during test runs) +real_world/roundtrip/diffs/ diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 91a5cea..6348e11 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -17,6 +17,7 @@ serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } toml = "0.8" tokio = { version = "1", features = ["full"] } +tempfile = "3.8" [dev-dependencies] # Any additional test dependencies @@ -32,3 +33,7 @@ path = "diagnostics_integration.rs" [[test]] name = "lexicon_fetcher_integration" path = "lexicon_fetcher_integration.rs" + +[[test]] +name = "real_world_roundtrip" +path = "real_world/roundtrip.rs" diff --git a/tests/README.md b/tests/README.md index be94e09..f1a6afd 100644 --- a/tests/README.md +++ b/tests/README.md @@ -34,8 +34,9 @@ tests/ ## Current Status ### ✅ Implemented -- **mlf-lang/tests/lang/** - 17 tests for parsing and validation -- **tests/codegen/lexicon/** - 4 tests for lexicon generation +- **mlf-lang/tests/lang/** - 21 tests for parsing and validation +- **tests/codegen/lexicon/** - 10 tests for lexicon generation +- **tests/real_world_roundtrip** - Round-trip test (JSON → MLF → JSON) with real lexicons ### 🚧 Planned @@ -80,11 +81,14 @@ tests/ - **workspace/precedence** - Resolution order (local > home > std) - **workspace/sibling_files** - Multi-file modules -#### Real-World Tests -- **real_world/bsky** - Full app.bsky.* lexicons -- **real_world/place_stream** - Full place.stream.* lexicons -- **real_world/atproto** - Full com.atproto.* lexicons -- **real_world/bidirectional** - Roundtrip MLF ↔ JSON +#### Real-World Tests ✅ +- **real_world/** - Tests using real lexicons from production networks + - **roundtrip** - Round-trip test: JSON → MLF → JSON + - Fetches real lexicons (app.bsky.*, net.anisota.*, place.stream.*, pub.leaflet.*) + - Validates accurate conversion both ways + - Writes diff files to `real_world/roundtrip/diffs/` (gitignored) + - Run with: `just test-real-world` + - See `real_world/README.md` for details ## Running Tests @@ -100,11 +104,8 @@ just test-lang # Language tests (17 tests) just test-codegen # Codegen tests (4 tests) just test-validation # Validation tests (12 tests) -# Future test categories -just test-cli # CLI integration tests -just test-diagnostics # Error message tests -just test-workspace # Multi-file resolution tests -just test-real-world # Full lexicon suites +# Network-dependent tests (run explicitly) +just test-real-world # Round-trip test: fetch real lexicons, convert MLF→JSON, verify # Other useful commands just test-all # All workspace tests (includes unit tests) diff --git a/tests/real_world/README.md b/tests/real_world/README.md new file mode 100644 index 0000000..df3c15d --- /dev/null +++ b/tests/real_world/README.md @@ -0,0 +1,58 @@ +# Real-World Tests + +Tests that fetch and validate real lexicons from production networks. + +## Tests + +### `roundtrip.rs` - Round-Trip Test + +Validates that MLF can accurately convert lexicons: JSON → MLF → JSON + +**What it does:** +1. Fetches real lexicons from production networks: + - `app.bsky.actor.*`, `app.bsky.feed.*`, `app.bsky.graph.*` + - `net.anisota.*` + - `place.stream.*` + - `pub.leaflet.*` +2. Converts downloaded JSON to MLF (automatic during fetch) +3. Generates JSON back from MLF files +4. Compares original vs regenerated JSON + +**Running:** +```bash +# Using just +just test-real-world + +# Using cargo +cargo test -p mlf-integration-tests --test real_world_roundtrip -- --ignored --nocapture +``` + +**Network-dependent:** This test fetches from real networks, so it: +- Is marked `#[ignore]` by default +- Requires internet connectivity +- Takes 30-60 seconds to run + +**Diff files:** When differences are found, the test writes three files per lexicon to `roundtrip/diffs/`: +- `{nsid}.original.json` - The original fetched JSON +- `{nsid}.generated.json` - The regenerated JSON from MLF +- `{nsid}.diff` - Unified diff output (`diff -u`) + +Files are organized into subdirectories: +- `diffs/acceptable/` - Acceptable differences (field ordering, `$type` fields) +- `diffs/failure/` - Structural differences that indicate bugs + +These files are gitignored but persisted locally for review. + +## Adding New Tests + +To add more real-world test sources: + +1. Edit `TEST_SOURCES` in `roundtrip.rs` +2. Ensure the NSID has published DNS TXT records +3. Test with `mlf fetch ` first to verify + +## Notes + +- These tests validate the core MLF workflow end-to-end +- Failures indicate bugs in either JSON→MLF or MLF→JSON conversion +- Review diff files to diagnose what changed diff --git a/tests/real_world/roundtrip.rs b/tests/real_world/roundtrip.rs new file mode 100644 index 0000000..af8b54e --- /dev/null +++ b/tests/real_world/roundtrip.rs @@ -0,0 +1,505 @@ +// Real-world round-trip tests: JSON → MLF → JSON +// +// These tests fetch real lexicons from the network, convert them to MLF, +// then generate JSON back and verify the round-trip is accurate. +// +// Run with: cargo test --test real_world_roundtrip -- --ignored --nocapture + +use std::collections::HashSet; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use tempfile::TempDir; + +/// Real-world lexicon sources to test +/// These use specific namespaces that have DNS TXT records published +const TEST_SOURCES: &[&str] = &[ + // Bluesky - use specific namespaces since top-level doesn't have TXT record + "app.bsky.actor.*", + "app.bsky.feed.*", + "app.bsky.graph.*", + // Other networks + "net.anisota.*", + "place.stream.*", + "pub.leaflet.*", +]; + +#[test] +#[ignore] // Network-dependent test, run explicitly with --ignored +fn test_real_world_roundtrip() { + println!("\n🌐 Real-World Round-Trip Test"); + println!("=============================\n"); + + // Create temp directory for test workspace + let temp_dir = TempDir::new().expect("Failed to create temp directory"); + let workspace_path = temp_dir.path(); + + println!("📁 Test workspace: {}\n", workspace_path.display()); + + // Step 1: Initialize MLF project + println!("1️⃣ Initializing MLF project..."); + init_mlf_project(workspace_path).expect("Failed to initialize project"); + + // Step 2: Fetch real lexicons + println!("\n2️⃣ Fetching real lexicons from network..."); + for source in TEST_SOURCES { + println!(" Fetching: {}", source); + fetch_lexicons(workspace_path, source).expect(&format!("Failed to fetch {}", source)); + } + + // Step 3: Copy MLF files to standard lexicons directory + println!("\n3️⃣ Copying MLF files to standard lexicons directory..."); + let source_mlf_dir = workspace_path.join(".mlf/lexicons/mlf"); + let lexicons_dir = workspace_path.join("lexicons"); + copy_mlf_files(&source_mlf_dir, &lexicons_dir).expect("Failed to copy MLF files"); + + // Step 4: Generate JSON from MLF + println!("\n4️⃣ Generating JSON from MLF files..."); + let output_dir = workspace_path.join("generated-lexicons"); + generate_json_from_mlf(workspace_path, &output_dir).expect("Failed to generate JSON"); + + // Step 5: Compare original vs regenerated JSON + println!("\n5️⃣ Comparing original vs regenerated JSON..."); + let original_dir = workspace_path.join(".mlf/lexicons/json"); + + // Write diffs to tests/real_world/roundtrip/diffs/ (persisted, gitignored) + let diffs_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("real_world/roundtrip/diffs"); + + let stats = compare_json_files(&original_dir, &output_dir, &diffs_dir) + .expect("Failed to compare JSON files"); + + // Step 6: Report results + println!("\n📊 Round-Trip Test Results"); + println!("==========================="); + println!("Total lexicons tested: {}", stats.total); + println!("Perfect matches: {}", stats.perfect_matches); + println!("Acceptable differences: {}", stats.acceptable_diffs); + println!("Failures: {}", stats.failures); + + if !stats.failed_lexicons.is_empty() { + println!("\n❌ Failed lexicons:"); + for (nsid, reason) in &stats.failed_lexicons { + println!(" - {}: {}", nsid, reason); + } + } + + if stats.acceptable_diffs > 0 || stats.failures > 0 { + println!("\n📁 Diff files written to: {}", diffs_dir.display()); + println!(" Review these files to see what changed between original and regenerated JSON"); + } + + // Assert that we have no failures + assert_eq!( + stats.failures, 0, + "Round-trip test failed for {} lexicon(s). Check diff files in {}", + stats.failures, + diffs_dir.display() + ); + + println!("\n✅ All round-trip tests passed!"); +} + +/// Initialize an MLF project with mlf.toml +fn init_mlf_project(workspace_path: &Path) -> Result<(), String> { + // Create mlf.toml + let mlf_toml = r#" +[package] +name = "roundtrip-test" +version = "0.1.0" + +[dependencies] +dependencies = [] +allow_transitive_deps = true +optimize_transitive_fetches = false +"#; + + fs::write(workspace_path.join("mlf.toml"), mlf_toml) + .map_err(|e| format!("Failed to write mlf.toml: {}", e))?; + + Ok(()) +} + +/// Fetch lexicons using `mlf fetch` +fn fetch_lexicons(workspace_path: &Path, nsid_pattern: &str) -> Result<(), String> { + let output = Command::new("mlf") + .arg("fetch") + .arg(nsid_pattern) + .current_dir(workspace_path) + .output() + .map_err(|e| format!("Failed to execute mlf fetch: {}", e))?; + + if !output.status.success() { + return Err(format!( + "mlf fetch failed:\n{}", + String::from_utf8_lossy(&output.stderr) + )); + } + + Ok(()) +} + +/// Copy MLF files from .mlf/lexicons/mlf to lexicons/ +fn copy_mlf_files(source_dir: &Path, dest_dir: &Path) -> Result<(), String> { + if !source_dir.exists() { + return Err(format!("Source directory not found: {}", source_dir.display())); + } + + fn copy_recursive(src: &Path, dst: &Path) -> std::io::Result<()> { + fs::create_dir_all(dst)?; + for entry in fs::read_dir(src)? { + let entry = entry?; + let src_path = entry.path(); + let dst_path = dst.join(entry.file_name()); + + if src_path.is_dir() { + copy_recursive(&src_path, &dst_path)?; + } else { + fs::copy(&src_path, &dst_path)?; + } + } + Ok(()) + } + + copy_recursive(source_dir, dest_dir) + .map_err(|e| format!("Failed to copy MLF files: {}", e))?; + + let mlf_count = find_mlf_files(dest_dir)?.len(); + println!(" Copied {} MLF files", mlf_count); + + Ok(()) +} + +/// Generate JSON from MLF files using `mlf generate lexicon` +fn generate_json_from_mlf(workspace_dir: &Path, output_dir: &Path) -> Result<(), String> { + let lexicons_dir = workspace_dir.join("lexicons"); + + if !lexicons_dir.exists() { + return Err(format!("Lexicons directory not found: {}", lexicons_dir.display())); + } + + // Create output directory + fs::create_dir_all(output_dir) + .map_err(|e| format!("Failed to create output directory: {}", e))?; + + // Generate all JSON files at once by passing the lexicons directory + // This allows proper dependency resolution between MLF files + println!(" Generating JSON files..."); + let output = Command::new("mlf") + .arg("generate") + .arg("lexicon") + .arg("-i") + .arg("lexicons") + .arg("-o") + .arg(output_dir) + .current_dir(workspace_dir) + .output() + .map_err(|e| format!("Failed to execute mlf generate: {}", e))?; + + if !output.status.success() { + return Err(format!( + "mlf generate lexicon failed:\n{}", + String::from_utf8_lossy(&output.stderr) + )); + } + + println!(" Generated JSON successfully"); + + Ok(()) +} + +/// Find all .mlf files recursively +fn find_mlf_files(dir: &Path) -> Result, String> { + let mut mlf_files = Vec::new(); + + fn walk_dir(dir: &Path, files: &mut Vec) -> std::io::Result<()> { + if dir.is_dir() { + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + walk_dir(&path, files)?; + } else if path.extension().and_then(|s| s.to_str()) == Some("mlf") { + files.push(path); + } + } + } + Ok(()) + } + + walk_dir(dir, &mut mlf_files).map_err(|e| format!("Failed to walk directory: {}", e))?; + Ok(mlf_files) +} + +#[derive(Debug)] +struct ComparisonStats { + total: usize, + perfect_matches: usize, + acceptable_diffs: usize, + failures: usize, + failed_lexicons: Vec<(String, String)>, +} + +/// Compare original JSON with regenerated JSON +fn compare_json_files( + original_dir: &Path, + generated_dir: &Path, + diffs_dir: &Path, +) -> Result { + // Create diffs directory + fs::create_dir_all(diffs_dir) + .map_err(|e| format!("Failed to create diffs directory: {}", e))?; + let mut stats = ComparisonStats { + total: 0, + perfect_matches: 0, + acceptable_diffs: 0, + failures: 0, + failed_lexicons: Vec::new(), + }; + + // Find all JSON files in original directory + let original_files = find_json_files(original_dir)?; + stats.total = original_files.len(); + + println!(" Comparing {} lexicon files...", stats.total); + + for original_file in original_files { + let relative_path = original_file + .strip_prefix(original_dir) + .map_err(|e| format!("Failed to strip prefix: {}", e))?; + + let generated_file = generated_dir.join(relative_path); + + // Extract NSID from path for reporting + let nsid = relative_path + .with_extension("") + .to_str() + .unwrap() + .replace(std::path::MAIN_SEPARATOR, "."); + + if !generated_file.exists() { + stats.failures += 1; + stats.failed_lexicons + .push((nsid.clone(), "Generated file not found".to_string())); + eprintln!(" ✗ {}: Generated file not found", nsid); + continue; + } + + // Read and parse JSON files + let original_json = fs::read_to_string(&original_file) + .map_err(|e| format!("Failed to read original: {}", e))?; + let generated_json = fs::read_to_string(&generated_file) + .map_err(|e| format!("Failed to read generated: {}", e))?; + + let original: serde_json::Value = serde_json::from_str(&original_json) + .map_err(|e| format!("Failed to parse original JSON: {}", e))?; + let generated: serde_json::Value = serde_json::from_str(&generated_json) + .map_err(|e| format!("Failed to parse generated JSON: {}", e))?; + + // Compare with allowed differences + match compare_lexicon_json(&original, &generated) { + ComparisonResult::Perfect => { + stats.perfect_matches += 1; + println!(" ✓ {} (perfect match)", nsid); + } + ComparisonResult::AcceptableDifferences(diffs) => { + stats.acceptable_diffs += 1; + println!(" ✓ {} (acceptable diffs: {})", nsid, diffs.join(", ")); + + // Write diff file for acceptable differences + write_diff_file(diffs_dir, &nsid, &original_json, &generated_json, "acceptable") + .unwrap_or_else(|e| eprintln!("Warning: Failed to write diff: {}", e)); + } + ComparisonResult::Failure(reason) => { + stats.failures += 1; + stats.failed_lexicons.push((nsid.clone(), reason.clone())); + eprintln!(" ✗ {}: {}", nsid, reason); + + // Write diff file for failures + write_diff_file(diffs_dir, &nsid, &original_json, &generated_json, "failure") + .unwrap_or_else(|e| eprintln!("Warning: Failed to write diff: {}", e)); + } + } + } + + Ok(stats) +} + +/// Find all JSON files recursively +fn find_json_files(dir: &Path) -> Result, String> { + let mut json_files = Vec::new(); + + fn walk_dir(dir: &Path, files: &mut Vec) -> std::io::Result<()> { + if dir.is_dir() { + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + walk_dir(&path, files)?; + } else if path.extension().and_then(|s| s.to_str()) == Some("json") { + files.push(path); + } + } + } + Ok(()) + } + + walk_dir(dir, &mut json_files).map_err(|e| format!("Failed to walk directory: {}", e))?; + Ok(json_files) +} + +#[derive(Debug)] +enum ComparisonResult { + Perfect, + AcceptableDifferences(Vec), + Failure(String), +} + +/// Compare two lexicon JSON objects, allowing certain acceptable differences +fn compare_lexicon_json( + original: &serde_json::Value, + generated: &serde_json::Value, +) -> ComparisonResult { + let mut acceptable_diffs = Vec::new(); + + // Strip $type fields (these are often added/removed) + let original_stripped = strip_dollar_type(original); + let generated_stripped = strip_dollar_type(generated); + + // Check if they're identical after stripping $type + if original_stripped == generated_stripped { + return ComparisonResult::Perfect; + } + + // Allow $type differences + if has_only_dollar_type_diff(&original_stripped, &generated_stripped) { + acceptable_diffs.push("$type fields".to_string()); + } + + // Check for field ordering differences (same fields, different order) + if has_only_ordering_diff(&original_stripped, &generated_stripped) { + acceptable_diffs.push("field ordering".to_string()); + return ComparisonResult::AcceptableDifferences(acceptable_diffs); + } + + // If we have acceptable diffs, return them + if !acceptable_diffs.is_empty() { + return ComparisonResult::AcceptableDifferences(acceptable_diffs); + } + + // Otherwise, it's a failure + ComparisonResult::Failure(format!( + "Structural differences detected" + )) +} + +/// Recursively strip $type fields from JSON +fn strip_dollar_type(value: &serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(map) => { + let mut new_map = serde_json::Map::new(); + for (k, v) in map { + if k != "$type" { + new_map.insert(k.clone(), strip_dollar_type(v)); + } + } + serde_json::Value::Object(new_map) + } + serde_json::Value::Array(arr) => { + serde_json::Value::Array(arr.iter().map(strip_dollar_type).collect()) + } + _ => value.clone(), + } +} + +/// Check if the only difference is $type fields +fn has_only_dollar_type_diff(v1: &serde_json::Value, v2: &serde_json::Value) -> bool { + // After stripping $type, they should be equal + v1 == v2 +} + +/// Write diff files showing differences between original and generated JSON +fn write_diff_file( + diffs_dir: &Path, + nsid: &str, + original_json: &str, + generated_json: &str, + diff_type: &str, +) -> Result<(), String> { + // Create subdirectory based on diff type + let type_dir = diffs_dir.join(diff_type); + fs::create_dir_all(&type_dir) + .map_err(|e| format!("Failed to create diff type directory: {}", e))?; + + // Create base filename from NSID + let base_filename = nsid.replace('.', "_"); + + // Write original JSON + let original_path = type_dir.join(format!("{}.original.json", base_filename)); + fs::write(&original_path, original_json) + .map_err(|e| format!("Failed to write original JSON: {}", e))?; + + // Write generated JSON + let generated_path = type_dir.join(format!("{}.generated.json", base_filename)); + fs::write(&generated_path, generated_json) + .map_err(|e| format!("Failed to write generated JSON: {}", e))?; + + // Run diff command and save output + let diff_path = type_dir.join(format!("{}.diff", base_filename)); + let diff_output = Command::new("diff") + .arg("-u") + .arg(&original_path) + .arg(&generated_path) + .output() + .map_err(|e| format!("Failed to run diff command: {}", e))?; + + // diff returns exit code 1 when files differ, which is expected + // Only error if exit code is 2+ (indicates an error running diff) + if diff_output.status.code() == Some(2) { + return Err(format!("diff command error: {}", String::from_utf8_lossy(&diff_output.stderr))); + } + + // Write diff output + fs::write(&diff_path, &diff_output.stdout) + .map_err(|e| format!("Failed to write diff output: {}", e))?; + + Ok(()) +} + +/// Check if the only difference is field ordering in objects +fn has_only_ordering_diff(v1: &serde_json::Value, v2: &serde_json::Value) -> bool { + match (v1, v2) { + (serde_json::Value::Object(map1), serde_json::Value::Object(map2)) => { + // Check if they have the same keys + let keys1: HashSet<_> = map1.keys().collect(); + let keys2: HashSet<_> = map2.keys().collect(); + + if keys1 != keys2 { + return false; + } + + // Check if all values match (recursively) + for key in keys1 { + let val1 = &map1[key]; + let val2 = &map2[key]; + + if !has_only_ordering_diff(val1, val2) && val1 != val2 { + return false; + } + } + + true + } + (serde_json::Value::Array(arr1), serde_json::Value::Array(arr2)) => { + // Arrays must match exactly (order matters) + if arr1.len() != arr2.len() { + return false; + } + + arr1.iter() + .zip(arr2.iter()) + .all(|(v1, v2)| has_only_ordering_diff(v1, v2) || v1 == v2) + } + _ => v1 == v2, + } +} diff --git a/tests/real_world/roundtrip/.gitignore b/tests/real_world/roundtrip/.gitignore new file mode 100644 index 0000000..7b39412 --- /dev/null +++ b/tests/real_world/roundtrip/.gitignore @@ -0,0 +1,2 @@ +# Diff files generated by round-trip tests +diffs/