From 2e7ce70a5e79a96e416284ee4eee5b87097a0fce Mon Sep 17 00:00:00 2001 From: Matt Stavola Date: Tue, 14 Apr 2026 23:03:23 -0400 Subject: [PATCH] Switch to nextest + datatest-stable --- .config/nextest.toml | 21 +++ Cargo.lock | 92 ++++++++++++- justfile | 40 +++--- mlf-lang/Cargo.toml | 6 + mlf-lang/tests/integration_test.rs | 122 +++++------------ tests/Cargo.toml | 5 + tests/codegen_integration.rs | 89 ++++-------- tests/diagnostics_integration.rs | 110 +++++---------- tests/lexicon_fetcher_integration.rs | 195 +++++++-------------------- tests/lexicon_to_mlf_integration.rs | 57 ++------ tests/test_utils.rs | 98 +------------- 11 files changed, 294 insertions(+), 541 deletions(-) create mode 100644 .config/nextest.toml diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 0000000..715941a --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,21 @@ +# Default profile — used by `just test` and ad-hoc `cargo nextest run`. +[profile.default] +# Surface failing test output inline as it happens, then repeat the +# summary at the end. On success, stay quiet — use `just test-verbose` +# (which passes `--no-capture`) when the per-subcase ✓/✗ lines from +# fixture runners are wanted. +failure-output = "immediate-final" +success-output = "never" +# Don't stop after the first failure; a bug usually breaks several +# fixtures and we want to see them all in one run. +fail-fast = false + +# Network-gated real-world roundtrip. Absorb flaky DNS/HTTP with a +# couple of retries, warn loudly when a single attempt takes more than +# 30s (a dead resolver, usually), and hard-kill after 3 minutes to +# prevent wedged CI. Deterministic tests still fail fast via the +# default profile's settings. +[[profile.default.overrides]] +filter = "test(test_real_world_roundtrip)" +retries = 2 +slow-timeout = { period = "30s", terminate-after = 6 } diff --git a/Cargo.lock b/Cargo.lock index d62918c..2f277f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,7 +61,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ "anstyle", - "anstyle-parse", + "anstyle-parse 0.2.7", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse 1.0.0", "anstyle-query", "anstyle-wincon", "colorchoice", @@ -84,6 +99,15 @@ dependencies = [ "utf8parse", ] +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + [[package]] name = "anstyle-query" version = "1.1.4" @@ -168,6 +192,21 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -225,6 +264,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "camino" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48" + [[package]] name = "cc" version = "1.2.40" @@ -307,7 +352,7 @@ version = "4.5.48" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9" dependencies = [ - "anstream", + "anstream 0.6.21", "anstyle", "clap_lex", "strsim", @@ -397,6 +442,18 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +[[package]] +name = "datatest-stable" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a867d7322eb69cf3a68a5426387a25b45cb3b9c5ee41023ee6cea92e2afadd82" +dependencies = [ + "camino", + "fancy-regex", + "libtest-mimic", + "walkdir", +] + [[package]] name = "deranged" version = "0.5.4" @@ -485,6 +542,23 @@ dependencies = [ "windows-sys 0.61.1", ] +[[package]] +name = "escape8259" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" + +[[package]] +name = "fancy-regex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1141,6 +1215,18 @@ dependencies = [ "libc", ] +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream 1.0.0", + "anstyle", + "clap", + "escape8259", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -1356,6 +1442,7 @@ dependencies = [ name = "mlf-integration-tests" version = "0.1.0" dependencies = [ + "datatest-stable", "mlf-cli", "mlf-codegen", "mlf-diagnostics", @@ -1372,6 +1459,7 @@ dependencies = [ name = "mlf-lang" version = "0.1.0" dependencies = [ + "datatest-stable", "include_dir", "mlf-integration-tests", "nom 8.0.0", diff --git a/justfile b/justfile index 179b133..a9f5d62 100644 --- a/justfile +++ b/justfile @@ -1,62 +1,68 @@ # MLF Test Suite # Run with: just test +# +# Test runner: cargo-nextest. Install with `just install-tools`. # Default: run all tests default: test -# Run all tests (excluding problematic packages) -test: test-lang test-codegen test-diagnostics test-lexicon-fetcher test-validation +# Install developer tools (test runner, etc.) +install-tools: + cargo install cargo-nextest --locked + +# Run every test across the workspace (excluding packages we can't build +# locally). Nextest already parallelises across binaries, so a single +# invocation is both faster and more comprehensive than chaining the +# per-suite recipes below — those remain useful for targeted runs. +test: + @echo "Running all workspace tests..." + cargo nextest run --workspace --exclude tree-sitter-mlf --exclude mlf-wasm # Run only language tests (mlf-lang crate) test-lang: @echo "Running mlf-lang integration tests..." - cargo test -p mlf-lang --test integration_test -- --nocapture + cargo nextest run -p mlf-lang --test integration_test # Run codegen integration tests (multi-crate) test-codegen: @echo "\nRunning codegen integration tests..." - cargo test -p mlf-integration-tests --test codegen_integration -- --nocapture + cargo nextest run -p mlf-integration-tests --test codegen_integration # Run diagnostics integration tests (multi-crate) test-diagnostics: @echo "\nRunning diagnostics integration tests..." - cargo test -p mlf-integration-tests --test diagnostics_integration -- --nocapture + cargo nextest run -p mlf-integration-tests --test diagnostics_integration # Run lexicon fetcher tests test-lexicon-fetcher: @echo "\nRunning lexicon fetcher tests..." - cargo test -p mlf-lexicon-fetcher -- --nocapture + cargo nextest run -p mlf-lexicon-fetcher # Run validation tests test-validation: @echo "\nRunning validation tests..." - cargo test -p mlf-validation + cargo nextest run -p mlf-validation # Run CLI integration tests (when implemented) test-cli: @echo "\nRunning CLI integration tests..." - cargo test -p mlf-integration-tests --test cli_integration -- --nocapture + cargo nextest run -p mlf-integration-tests --test cli_integration # Run workspace resolution tests (when implemented) test-workspace: @echo "\nRunning workspace resolution tests..." - cargo test -p mlf-integration-tests --test workspace_integration -- --nocapture + cargo nextest run -p mlf-integration-tests --test workspace_integration # Run real-world round-trip tests (network-dependent, ignored by default) test-real-world: @echo "\n🌐 Running real-world round-trip tests (fetches from network)..." @echo "This will download lexicons from: app.bsky.*, net.anisota.*, place.stream.*, pub.leaflet.*" @echo "" - cargo test -p mlf-integration-tests --test real_world_roundtrip -- --ignored --nocapture - -# Run all workspace tests (excluding problematic packages) -test-all: - @echo "Running all workspace tests..." - cargo test --workspace --exclude tree-sitter-mlf --exclude mlf-wasm + cargo nextest run -p mlf-integration-tests --test real_world_roundtrip --run-ignored only -# Run tests with verbose output +# Run tests with verbose output (stdout streamed live, tests run serially) test-verbose: - cargo test --workspace --exclude tree-sitter-mlf --exclude mlf-wasm -- --nocapture + cargo nextest run --workspace --exclude tree-sitter-mlf --exclude mlf-wasm --no-capture # Quick check without running tests check: diff --git a/mlf-lang/Cargo.toml b/mlf-lang/Cargo.toml index 5d7f6c7..a0772f1 100644 --- a/mlf-lang/Cargo.toml +++ b/mlf-lang/Cargo.toml @@ -13,6 +13,12 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" toml = "0.8" mlf-integration-tests = { path = "../tests" } +datatest-stable = "0.3" + +[[test]] +name = "integration_test" +path = "tests/integration_test.rs" +harness = false [features] default = ["std"] diff --git a/mlf-lang/tests/integration_test.rs b/mlf-lang/tests/integration_test.rs index b5c66c4..92630a5 100644 --- a/mlf-lang/tests/integration_test.rs +++ b/mlf-lang/tests/integration_test.rs @@ -1,3 +1,7 @@ +// Language-level integration tests. +// Each leaf directory under `tests/lang///` contains a +// `test.mlf` (plus optional support files) and an `expected.json`. + use mlf_integration_tests::test_utils; use mlf_lang::{parser::parse_lexicon, Workspace}; use serde::Deserialize; @@ -30,36 +34,32 @@ struct ExpectedError { message: Option, } -fn run_lang_test(test_dir: &Path) -> Result<(), String> { - let test_name = test_dir.file_name().unwrap().to_str().unwrap(); +fn run_lang_test(test_mlf: &Path) -> datatest_stable::Result<()> { + let test_dir = test_mlf.parent().ok_or("test.mlf has no parent directory")?; + let test_name = test_dir + .file_name() + .and_then(|s| s.to_str()) + .ok_or("Could not derive test name")?; - // Load test configuration - let config = test_utils::load_test_config(test_dir, |name| format!("test.{}", name))?; + let config = test_utils::load_test_config(test_dir, |name| format!("test.{}", name)) + .map_err(|e| format!("{}", e))?; - // Load expected result let expected_path = test_dir.join("expected.json"); if !expected_path.exists() { - return Err(format!("No expected.json found for test {}", test_name)); + return Err(format!("No expected.json found for test {}", test_name).into()); } - let expected_json = fs::read_to_string(&expected_path) - .map_err(|e| format!("Failed to read expected.json: {}", e))?; - let expected: ExpectedResult = serde_json::from_str(&expected_json) - .map_err(|e| format!("Failed to parse expected.json: {}", e))?; + let expected_json = fs::read_to_string(&expected_path)?; + let expected: ExpectedResult = serde_json::from_str(&expected_json)?; - // Create workspace with std library - let mut ws = Workspace::with_std() - .map_err(|e| format!("Failed to create workspace: {:?}", e))?; + let mut ws = Workspace::with_std().map_err(|e| format!("Failed to create workspace: {:?}", e))?; - // Load modules as specified in test.toml - let mut module_files: Vec<(String, PathBuf)> = config.modules.iter() - .map(|(filename, namespace)| { - let path = test_dir.join(filename); - (namespace.clone(), path) - }) + // Support files load before test.mlf so cross-module refs resolve. + let mut module_files: Vec<(String, PathBuf)> = config + .modules + .iter() + .map(|(filename, namespace)| (namespace.clone(), test_dir.join(filename))) .collect(); - - // Sort to ensure deterministic order (test.mlf should come after supporting files) module_files.sort_by(|a, b| { let a_is_test = a.1.file_stem().unwrap() == "test"; let b_is_test = b.1.file_stem().unwrap() == "test"; @@ -72,99 +72,47 @@ fn run_lang_test(test_dir: &Path) -> Result<(), String> { for (namespace, mlf_file) in module_files { if !mlf_file.exists() { - return Err(format!("Module file not found: {}", mlf_file.display())); + return Err(format!("Module file not found: {}", mlf_file.display()).into()); } - - let content = fs::read_to_string(&mlf_file) - .map_err(|e| format!("Failed to read {}: {}", mlf_file.display(), e))?; - + let content = fs::read_to_string(&mlf_file)?; let lexicon = parse_lexicon(&content) .map_err(|e| format!("Failed to parse {}: {:?}", mlf_file.display(), e))?; - ws.add_module(namespace, lexicon) .map_err(|e| format!("Failed to add module {}: {:?}", mlf_file.display(), e))?; } - // Resolve the workspace let result = ws.resolve(); - // Check if the result matches expectations match (expected.status.as_str(), result) { - ("success", Ok(())) => { - // Success case - check no unexpected errors - Ok(()) - } - ("success", Err(errors)) => { - Err(format!( - "Test {} expected success but got errors: {:?}", - test_name, errors - )) - } + ("success", Ok(())) => Ok(()), + ("success", Err(errors)) => Err(format!( + "Test {} expected success but got errors: {:?}", + test_name, errors + ) + .into()), ("error", Ok(())) => { - Err(format!( - "Test {} expected errors but succeeded", - test_name - )) + Err(format!("Test {} expected errors but succeeded", test_name).into()) } ("error", Err(actual_errors)) => { - // Error case - verify we got the expected error types for expected_error in &expected.errors { let found = actual_errors.errors.iter().any(|err| { let err_type = format!("{:?}", err); err_type.contains(&expected_error.error_type) }); - if !found { return Err(format!( "Test {} expected error type '{}' but didn't find it in: {:?}", test_name, expected_error.error_type, actual_errors - )); + ) + .into()); } } Ok(()) } - _ => Err(format!("Unknown expected status: {}", expected.status)), + (status, _) => Err(format!("Unknown expected status: {}", status).into()), } } -fn discover_and_run_tests(base_dir: &str) -> Vec<(String, Result<(), String>)> { - test_utils::discover_categorized_tests(base_dir) - .into_iter() - .map(|(test_name, test_path)| { - let result = run_lang_test(&test_path); - (test_name, result) - }) - .collect() -} - -#[test] -fn lang_tests() { - let results = discover_and_run_tests("tests/lang"); - - let mut failed = Vec::new(); - let mut passed = 0; - - for (test_name, result) in results { - match result { - Ok(()) => { - println!("✓ {}", test_name); - passed += 1; - } - Err(err) => { - println!("✗ {}: {}", test_name, err); - failed.push((test_name, err)); - } - } - } - - println!("\nResults: {} passed, {} failed", passed, failed.len()); - - if !failed.is_empty() { - panic!("\nFailed tests:\n{}", - failed.iter() - .map(|(name, err)| format!(" - {}: {}", name, err)) - .collect::>() - .join("\n") - ); - } +datatest_stable::harness! { + { test = run_lang_test, root = "tests/lang", pattern = r"^.*/test\.mlf$" }, } diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 58d49ac..f12cb50 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -19,6 +19,7 @@ serde = { version = "1.0", features = ["derive"] } toml = "0.8" tokio = { version = "1", features = ["full"] } tempfile = "3.8" +datatest-stable = "0.3" [dev-dependencies] # Any additional test dependencies @@ -26,14 +27,17 @@ tempfile = "3.8" [[test]] name = "codegen_integration" path = "codegen_integration.rs" +harness = false [[test]] name = "diagnostics_integration" path = "diagnostics_integration.rs" +harness = false [[test]] name = "lexicon_fetcher_integration" path = "lexicon_fetcher_integration.rs" +harness = false [[test]] name = "real_world_roundtrip" @@ -42,3 +46,4 @@ path = "real_world/roundtrip.rs" [[test]] name = "lexicon_to_mlf_integration" path = "lexicon_to_mlf_integration.rs" +harness = false diff --git a/tests/codegen_integration.rs b/tests/codegen_integration.rs index 6f2a8d0..f251135 100644 --- a/tests/codegen_integration.rs +++ b/tests/codegen_integration.rs @@ -1,5 +1,6 @@ -// Workspace-level integration tests for code generation -// Tests mlf-lang + mlf-codegen working together +// Workspace-level integration tests for code generation. +// Each subdirectory of `codegen/lexicon/` is a test fixture containing +// `input.mlf`, `expected.json`, and optionally `test.toml`. use mlf_codegen::generate_lexicon; use mlf_integration_tests::test_utils; @@ -8,39 +9,6 @@ use serde_json::Value; use std::fs; use std::path::Path; -#[test] -fn codegen_lexicon_tests() { - // CARGO_MANIFEST_DIR points to tests/ directory - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - let test_base = format!("{}/codegen/lexicon", manifest_dir); - let test_dirs = test_utils::discover_test_dirs(&test_base); - - let tests: Vec<(String, Result<(), String>)> = test_dirs - .into_iter() - .map(|test_dir| { - let test_name = format!( - "codegen/lexicon/{}", - Path::new(&test_dir).file_name().unwrap().to_str().unwrap() - ); - let result = run_lexicon_test(&test_dir); - (test_name, result) - }) - .collect(); - - let (_passed, failed) = test_utils::run_and_report_tests:: Result<(), String>>(tests, "Codegen"); - - if !failed.is_empty() { - panic!( - "\nFailed tests:\n{}", - failed - .iter() - .map(|(name, err)| format!(" - {}: {}", name, err)) - .collect::>() - .join("\n") - ); - } -} - fn derive_namespace_fallback(test_name: &str) -> String { match test_name { "basic_record" => "app.bsky.feed.post".to_string(), @@ -48,56 +16,45 @@ fn derive_namespace_fallback(test_name: &str) -> String { } } -fn run_lexicon_test(test_dir: &str) -> Result<(), String> { - // 1. Load test configuration - let config = test_utils::load_test_config( - Path::new(test_dir), - |test_name| derive_namespace_fallback(test_name) - )?; +fn run_lexicon_test(input_path: &Path) -> datatest_stable::Result<()> { + let test_dir = input_path.parent().ok_or("input.mlf has no parent directory")?; - // 2. Get namespace from config - let namespace = config.test.namespace - .ok_or_else(|| "No namespace specified in test.toml".to_string())?; + let config = test_utils::load_test_config(test_dir, derive_namespace_fallback) + .map_err(|e| format!("{}", e))?; - // 3. Read input.mlf - let input_path = format!("{}/input.mlf", test_dir); - let input = fs::read_to_string(&input_path) - .map_err(|e| format!("Failed to read input.mlf: {}", e))?; + let namespace = config + .test + .namespace + .ok_or("No namespace specified in test.toml")?; - // 4. Parse with mlf-lang - let lexicon = parse_lexicon(&input) - .map_err(|e| format!("Failed to parse: {:?}", e))?; + let input = fs::read_to_string(input_path)?; + let lexicon = parse_lexicon(&input).map_err(|e| format!("Failed to parse: {:?}", e))?; - // 4. Create workspace and resolve - let mut ws = Workspace::with_std() - .map_err(|e| format!("Failed to create workspace: {:?}", e))?; + let mut ws = Workspace::with_std().map_err(|e| format!("Failed to create workspace: {:?}", e))?; ws.add_module(namespace.clone(), lexicon) .map_err(|e| format!("Failed to add module: {:?}", e))?; ws.resolve() .map_err(|e| format!("Failed to resolve: {:?}", e))?; - // 5. Get lexicon from workspace - let lexicon = ws.get_lexicon(&namespace) - .ok_or_else(|| "Module not found".to_string())?; + let lexicon = ws.get_lexicon(&namespace).ok_or("Module not found")?; - // 6. Generate lexicon JSON let output_json = generate_lexicon(&namespace, lexicon, &ws); - // 7. Read expected output - let expected_path = format!("{}/expected.json", test_dir); - let expected_str = fs::read_to_string(&expected_path) - .map_err(|e| format!("Failed to read expected.json: {}", e))?; - let expected_json: Value = serde_json::from_str(&expected_str) - .map_err(|e| format!("Failed to parse expected.json: {}", e))?; + let expected_str = fs::read_to_string(test_dir.join("expected.json"))?; + let expected_json: Value = serde_json::from_str(&expected_str)?; - // 8. Compare (normalize both to avoid whitespace issues) if output_json != expected_json { return Err(format!( "Output mismatch:\nExpected:\n{}\n\nGot:\n{}", serde_json::to_string_pretty(&expected_json).unwrap(), serde_json::to_string_pretty(&output_json).unwrap() - )); + ) + .into()); } Ok(()) } + +datatest_stable::harness! { + { test = run_lexicon_test, root = "codegen/lexicon", pattern = r"^.*/input\.mlf$" }, +} diff --git a/tests/diagnostics_integration.rs b/tests/diagnostics_integration.rs index 54f1767..6667d1a 100644 --- a/tests/diagnostics_integration.rs +++ b/tests/diagnostics_integration.rs @@ -1,5 +1,6 @@ -// Workspace-level integration tests for diagnostics -// Tests mlf-lang + mlf-diagnostics working together +// Workspace-level integration tests for diagnostics. +// Each subdirectory of `diagnostics/` is a fixture with `input.mlf`, +// `expected.json`, and optionally `test.toml`. use mlf_diagnostics::{get_error_module_namespace_str, ValidationDiagnostic}; use mlf_integration_tests::test_utils; @@ -32,73 +33,31 @@ struct ExpectedSpan { end: usize, } -#[test] -fn diagnostics_tests() { - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - let test_base = format!("{}/diagnostics", manifest_dir); - let test_dirs = test_utils::discover_test_dirs(&test_base); - - let tests: Vec<(String, Result<(), String>)> = test_dirs - .into_iter() - .map(|test_dir| { - let test_name = format!( - "diagnostics/{}", - Path::new(&test_dir).file_name().unwrap().to_str().unwrap() - ); - let result = run_diagnostics_test(&test_dir); - (test_name, result) - }) - .collect(); - - let (_passed, failed) = test_utils::run_and_report_tests:: Result<(), String>>(tests, "Diagnostics"); - - if !failed.is_empty() { - panic!( - "\nFailed tests:\n{}", - failed - .iter() - .map(|(name, err)| format!(" - {}: {}", name, err)) - .collect::>() - .join("\n") - ); - } -} - -fn derive_namespace_fallback(test_name: &str) -> String { - format!("test.{}", test_name) -} - -fn run_diagnostics_test(test_dir: &str) -> Result<(), String> { - // 1. Load test configuration - let config = test_utils::load_test_config( - Path::new(test_dir), - |test_name| derive_namespace_fallback(test_name) - )?; +fn run_diagnostics_test(input_path: &Path) -> datatest_stable::Result<()> { + let test_dir = input_path.parent().ok_or("input.mlf has no parent directory")?; - let namespace = config.test.namespace - .ok_or_else(|| "No namespace specified in test.toml".to_string())?; + let config = test_utils::load_test_config(test_dir, |name| format!("test.{}", name)) + .map_err(|e| format!("{}", e))?; - // 2. Read input.mlf - let input_path = format!("{}/input.mlf", test_dir); - let input = fs::read_to_string(&input_path) - .map_err(|e| format!("Failed to read input.mlf: {}", e))?; + let namespace = config + .test + .namespace + .ok_or("No namespace specified in test.toml")?; - // 3. Parse with mlf-lang - let lexicon = parse_lexicon(&input) - .map_err(|e| format!("Failed to parse: {:?}", e))?; + let input = fs::read_to_string(input_path)?; + let lexicon = parse_lexicon(&input).map_err(|e| format!("Failed to parse: {:?}", e))?; - // 4. Create workspace and resolve (expect errors) - let mut ws = Workspace::with_std() - .map_err(|e| format!("Failed to create workspace: {:?}", e))?; + let mut ws = Workspace::with_std().map_err(|e| format!("Failed to create workspace: {:?}", e))?; ws.add_module(namespace.clone(), lexicon) .map_err(|e| format!("Failed to add module: {:?}", e))?; let validation_errors = match ws.resolve() { - Ok(()) => return Err("Expected validation errors but got none".to_string()), + Ok(()) => return Err("Expected validation errors but got none".into()), Err(errors) => errors, }; - // 5. Create diagnostic + // Construct the diagnostic to ensure the wiring still works, even if we + // don't assert on its rendered form here. let _diagnostic = ValidationDiagnostic::new( "input.mlf".to_string(), input.clone(), @@ -106,53 +65,44 @@ fn run_diagnostics_test(test_dir: &str) -> Result<(), String> { validation_errors.clone(), ); - // 6. Read expected output - let expected_path = format!("{}/expected.json", test_dir); - let expected_str = fs::read_to_string(&expected_path) - .map_err(|e| format!("Failed to read expected.json: {}", e))?; - let expected: ExpectedDiagnostic = serde_json::from_str(&expected_str) - .map_err(|e| format!("Failed to parse expected.json: {}", e))?; + let expected_str = fs::read_to_string(test_dir.join("expected.json"))?; + let expected: ExpectedDiagnostic = serde_json::from_str(&expected_str)?; - // 7. Filter errors to this module let errors_in_module: Vec<_> = validation_errors .errors .iter() .filter(|e| get_error_module_namespace_str(e) == namespace) .collect(); - // 8. Verify error count if errors_in_module.len() != expected.error_count { return Err(format!( "Expected {} errors but got {}", expected.error_count, errors_in_module.len() - )); + ) + .into()); } - // 9. Verify each error for (i, expected_error) in expected.errors.iter().enumerate() { - if i >= errors_in_module.len() { - return Err(format!("Expected error #{} but only got {} errors", i + 1, errors_in_module.len())); - } - - let actual_error = errors_in_module[i]; - - // Check error code - let _actual_code = mlf_diagnostics::get_error_module_namespace_str(actual_error); + let actual_error = errors_in_module + .get(i) + .ok_or_else(|| format!("Expected error #{} but only got {} errors", i + 1, errors_in_module.len()))?; - // Format the error message let actual_message = format!("{:?}", actual_error); - - // Verify message contains expected text if !actual_message.contains(&expected_error.message) { return Err(format!( "Error #{}: Expected message to contain '{}' but got: {}", i + 1, expected_error.message, actual_message - )); + ) + .into()); } } Ok(()) } + +datatest_stable::harness! { + { test = run_diagnostics_test, root = "diagnostics", pattern = r"^.*/input\.mlf$" }, +} diff --git a/tests/lexicon_fetcher_integration.rs b/tests/lexicon_fetcher_integration.rs index 3bf693a..f6758cd 100644 --- a/tests/lexicon_fetcher_integration.rs +++ b/tests/lexicon_fetcher_integration.rs @@ -1,7 +1,8 @@ -// Workspace-level integration tests for lexicon fetcher -// Tests mlf-lexicon-fetcher working with mocks +// Workspace-level integration tests for the lexicon fetcher. +// Each subdirectory of `lexicon_fetcher/dns/` is a fixture with `test.toml` +// and `expected.json`. -use mlf_lexicon_fetcher::{DnsResolver, LexiconFetcher, MockDnsResolver, MockHttpClient}; +use mlf_lexicon_fetcher::{DnsResolver, MockDnsResolver}; use serde::{Deserialize, Serialize}; use std::fs; use std::path::Path; @@ -13,11 +14,13 @@ struct TestConfig { #[derive(Debug, Deserialize)] struct TestMetadata { + #[allow(dead_code)] description: String, nsid: String, #[serde(default)] did: Option, #[serde(default)] + #[allow(dead_code)] should_fail: bool, } @@ -30,124 +33,66 @@ struct ExpectedResult { error: Option, } -#[tokio::test] -async fn lexicon_fetcher_dns_tests() { - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - let test_base = format!("{}/lexicon_fetcher/dns", manifest_dir); - let test_dirs = discover_test_dirs(&test_base); - - let mut tests = Vec::new(); - - for test_dir in test_dirs { - let test_name = format!( - "lexicon_fetcher/dns/{}", - Path::new(&test_dir).file_name().unwrap().to_str().unwrap() - ); - let result = run_dns_test(&test_dir).await; - tests.push((test_name, result)); - } - - let (passed, failed) = run_and_report_tests(tests, "Lexicon Fetcher DNS"); - - if !failed.is_empty() { - panic!( - "\nFailed tests:\n{}", - failed - .iter() - .map(|(name, err)| format!(" - {}: {}", name, err)) - .collect::>() - .join("\n") - ); - } - - assert!(passed > 0, "No tests were run"); +fn run_dns_test(config_path: &Path) -> datatest_stable::Result<()> { + // The async body runs on a per-test Tokio runtime; datatest-stable is + // sync and spawning a fresh current-thread runtime keeps fixtures + // isolated from each other. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + runtime.block_on(run_dns_test_async(config_path)) } -async fn run_dns_test(test_dir: &str) -> Result<(), String> { - // Load test configuration - let config_path = format!("{}/test.toml", test_dir); - let config_str = fs::read_to_string(&config_path) - .map_err(|e| format!("Failed to read test.toml: {}", e))?; - let config: TestConfig = toml::from_str(&config_str) - .map_err(|e| format!("Failed to parse test.toml: {}", e))?; - - // Load expected output - let expected_path = format!("{}/expected.json", test_dir); - let expected_str = fs::read_to_string(&expected_path) - .map_err(|e| format!("Failed to read expected.json: {}", e))?; - let expected: ExpectedResult = serde_json::from_str(&expected_str) - .map_err(|e| format!("Failed to parse expected.json: {}", e))?; - - // Setup mocks - let mut dns_resolver = MockDnsResolver::new(); - let http_client = MockHttpClient::new(); - - // Add DNS record if expected to succeed - if expected.status == "success" { - let (authority, name_segments) = parse_nsid(&config.test.nsid)?; - let did = config.test.did.clone().ok_or_else(|| "Missing DID in test config".to_string())?; - - // For wildcard patterns, remove the .* suffix - let dns_name = if name_segments.ends_with(".*") { - name_segments.strip_suffix(".*").unwrap() - } else { - &name_segments - }; +async fn run_dns_test_async(config_path: &Path) -> datatest_stable::Result<()> { + let test_dir = config_path + .parent() + .ok_or("test.toml has no parent directory")?; - dns_resolver.add_record(&authority, dns_name, did.clone()); - } + let config_str = fs::read_to_string(config_path)?; + let config: TestConfig = toml::from_str(&config_str)?; - // Create fetcher - let fetcher = LexiconFetcher::new(dns_resolver, http_client); + let expected_str = fs::read_to_string(test_dir.join("expected.json"))?; + let expected: ExpectedResult = serde_json::from_str(&expected_str)?; - // Test DNS resolution by attempting to extract the DID - // We can't directly test DNS resolution without HTTP, so we test the full flow - // but expect it to fail at HTTP stage (which is fine for DNS testing) let (authority, name_segments) = parse_nsid(&config.test.nsid)?; + let dns_name = name_segments + .strip_suffix(".*") + .map(str::to_owned) + .unwrap_or(name_segments); - // For wildcard patterns, strip the .* suffix - let dns_name = if name_segments.ends_with(".*") { - name_segments.strip_suffix(".*").unwrap().to_string() - } else { - name_segments - }; - - // Create a test resolver directly to check DNS let mut test_dns = MockDnsResolver::new(); if expected.status == "success" { - let did = config.test.did.clone().ok_or_else(|| "Missing DID in test config".to_string())?; + let did = config + .test + .did + .clone() + .ok_or("Missing DID in test config")?; test_dns.add_record(&authority, &dns_name, did.clone()); - // Verify DNS resolution - match test_dns.resolve_lexicon_did(&authority, &dns_name).await { - Ok(resolved_did) => { - if resolved_did != did { - return Err(format!("DID mismatch: expected {}, got {}", did, resolved_did)); - } - // Check against expected - if let Some(expected_did) = &expected.did { - if &resolved_did != expected_did { - return Err(format!("DID mismatch with expected: expected {}, got {}", expected_did, resolved_did)); - } - } - } - Err(e) => return Err(format!("DNS resolution failed: {:?}", e)), + let resolved = test_dns + .resolve_lexicon_did(&authority, &dns_name) + .await + .map_err(|e| format!("DNS resolution failed: {:?}", e))?; + if resolved != did { + return Err(format!("DID mismatch: expected {}, got {}", did, resolved).into()); } - } else { - // Expected to fail - match test_dns.resolve_lexicon_did(&authority, &dns_name).await { - Ok(_) => return Err("Expected DNS lookup to fail, but it succeeded".to_string()), - Err(_) => { - // Success - it failed as expected + if let Some(expected_did) = &expected.did { + if &resolved != expected_did { + return Err(format!( + "DID mismatch with expected: expected {}, got {}", + expected_did, resolved + ) + .into()); } } + } else if test_dns.resolve_lexicon_did(&authority, &dns_name).await.is_ok() { + return Err("Expected DNS lookup to fail, but it succeeded".into()); } Ok(()) } fn parse_nsid(nsid: &str) -> Result<(String, String), String> { - // Remove wildcard if present for parsing let nsid_base = nsid.strip_suffix(".*").unwrap_or(nsid); let parts: Vec<&str> = nsid_base.split('.').collect(); @@ -158,7 +103,6 @@ fn parse_nsid(nsid: &str) -> Result<(String, String), String> { let authority = format!("{}.{}", parts[0], parts[1]); let name_segments = if parts.len() > 2 { let mut segments = parts[2..].join("."); - // Re-add wildcard if original had it if nsid.ends_with(".*") { segments.push_str(".*"); } @@ -172,49 +116,6 @@ fn parse_nsid(nsid: &str) -> Result<(String, String), String> { Ok((authority, name_segments)) } -fn discover_test_dirs(base: &str) -> Vec { - let base_path = Path::new(base); - if !base_path.exists() { - return vec![]; - } - - let mut dirs: Vec = fs::read_dir(base_path) - .unwrap() - .filter_map(|entry| { - let entry = entry.ok()?; - let path = entry.path(); - if path.is_dir() { - Some(path.to_str()?.to_string()) - } else { - None - } - }) - .collect(); - - dirs.sort(); - dirs -} - -fn run_and_report_tests( - tests: Vec<(String, Result<(), String>)>, - test_type: &str, -) -> (usize, Vec<(String, String)>) { - let mut passed = 0; - let mut failed = Vec::new(); - - for (test_name, result) in tests { - match result { - Ok(()) => { - println!("✓ {}", test_name); - passed += 1; - } - Err(err) => { - println!("✗ {}: {}", test_name, err); - failed.push((test_name, err)); - } - } - } - - println!("\n{} Results: {} passed, {} failed", test_type, passed, failed.len()); - (passed, failed) +datatest_stable::harness! { + { test = run_dns_test, root = "lexicon_fetcher/dns", pattern = r"^.*/test\.toml$" }, } diff --git a/tests/lexicon_to_mlf_integration.rs b/tests/lexicon_to_mlf_integration.rs index 7778769..180655a 100644 --- a/tests/lexicon_to_mlf_integration.rs +++ b/tests/lexicon_to_mlf_integration.rs @@ -5,64 +5,31 @@ // - `expected.mlf`: the expected MLF source produced by the converter use mlf_cli::generate::mlf::generate_mlf_from_json; -use mlf_integration_tests::test_utils; use serde_json::Value; use std::fs; use std::path::Path; -#[test] -fn lexicon_to_mlf_tests() { - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - let test_base = format!("{}/lexicon_to_mlf", manifest_dir); - let test_dirs = test_utils::discover_test_dirs(&test_base); +fn run_case(input_path: &Path) -> datatest_stable::Result<()> { + let test_dir = input_path.parent().ok_or("input.json has no parent directory")?; - let tests: Vec<(String, Result<(), String>)> = test_dirs - .into_iter() - .map(|test_dir| { - let test_name = format!( - "lexicon_to_mlf/{}", - Path::new(&test_dir).file_name().unwrap().to_str().unwrap() - ); - let result = run_case(&test_dir); - (test_name, result) - }) - .collect(); + let input = fs::read_to_string(input_path)?; + let json: Value = serde_json::from_str(&input)?; - let (_passed, failed) = - test_utils::run_and_report_tests:: Result<(), String>>(tests, "Lexicon→MLF"); + let output = generate_mlf_from_json(&json).map_err(|e| format!("{:?}", e))?; - if !failed.is_empty() { - panic!( - "\nFailed tests:\n{}", - failed - .iter() - .map(|(name, err)| format!(" - {}: {}", name, err)) - .collect::>() - .join("\n") - ); - } -} - -fn run_case(test_dir: &str) -> Result<(), String> { - let input_path = format!("{}/input.json", test_dir); - let input = fs::read_to_string(&input_path) - .map_err(|e| format!("Failed to read input.json: {}", e))?; - let json: Value = serde_json::from_str(&input) - .map_err(|e| format!("Failed to parse input.json: {}", e))?; - - let output = generate_mlf_from_json(&json) - .map_err(|e| format!("Failed to generate MLF: {}", e))?; - - let expected_path = format!("{}/expected.mlf", test_dir); - let expected = fs::read_to_string(&expected_path) - .map_err(|e| format!("Failed to read expected.mlf: {}", e))?; + let expected = fs::read_to_string(test_dir.join("expected.mlf"))?; if output != expected { return Err(format!( "Output mismatch:\n--- expected ---\n{}\n--- got ---\n{}", expected, output - )); + ) + .into()); } Ok(()) } + +datatest_stable::harness! { + { test = run_case, root = "lexicon_to_mlf", pattern = r"^.*/input\.json$" }, +} diff --git a/tests/test_utils.rs b/tests/test_utils.rs index 3838a2d..306898c 100644 --- a/tests/test_utils.rs +++ b/tests/test_utils.rs @@ -2,7 +2,7 @@ use serde::Deserialize; use std::collections::HashMap; use std::fs; -use std::path::{Path, PathBuf}; +use std::path::Path; #[derive(Debug, Deserialize)] pub struct TestConfig { @@ -57,99 +57,3 @@ pub fn load_test_config( toml::from_str(&config_str).map_err(|e| format!("Failed to parse test.toml: {}", e)) } -/// Discover all test directories in a base path. -/// Returns a sorted list of test directory paths. -pub fn discover_test_dirs(base: &str) -> Vec { - let base_path = Path::new(base); - if !base_path.exists() { - return vec![]; - } - - let mut dirs: Vec = fs::read_dir(base_path) - .unwrap() - .filter_map(|entry| { - let entry = entry.ok()?; - let path = entry.path(); - if path.is_dir() { - Some(path.to_str()?.to_string()) - } else { - None - } - }) - .collect(); - - dirs.sort(); - dirs -} - -/// Discover tests organized by category (e.g., lang/namespace_imports/*, lang/constraints/*) -pub fn discover_categorized_tests(base_dir: &str) -> Vec<(String, PathBuf)> { - let mut results = Vec::new(); - let base_path = Path::new(base_dir); - - if !base_path.exists() { - return results; - } - - // Walk through category directories - for category_entry in fs::read_dir(base_path).unwrap() { - let category_entry = category_entry.unwrap(); - let category_path = category_entry.path(); - - if !category_path.is_dir() { - continue; - } - - let category_name = category_path.file_name().unwrap().to_str().unwrap(); - - // Each category contains test directories - for test_entry in fs::read_dir(&category_path).unwrap() { - let test_entry = test_entry.unwrap(); - let test_path = test_entry.path(); - - if !test_path.is_dir() { - continue; - } - - let test_name = format!( - "{}/{}", - category_name, - test_path.file_name().unwrap().to_str().unwrap() - ); - - results.push((test_name, test_path)); - } - } - - results.sort_by(|a, b| a.0.cmp(&b.0)); - results -} - -/// Run tests and print results with ✓/✗ indicators. -/// Returns (passed_count, failed_tests). -pub fn run_and_report_tests( - tests: Vec<(String, Result<(), String>)>, - test_type: &str, -) -> (usize, Vec<(String, String)>) -where - F: Fn(&str) -> Result<(), String>, -{ - let mut passed = 0; - let mut failed = Vec::new(); - - for (test_name, result) in tests { - match result { - Ok(()) => { - println!("✓ {}", test_name); - passed += 1; - } - Err(err) => { - println!("✗ {}: {}", test_name, err); - failed.push((test_name, err)); - } - } - } - - println!("\n{} Results: {} passed, {} failed", test_type, passed, failed.len()); - (passed, failed) -} -- 2.51.2