From f6c9da99f54fbef8a1cdb2eb58728e452babb96a Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Sun, 23 Aug 2026 13:20:12 -0400 Subject: [PATCH] test(forces): a corpus of .mul files other people wrote Every check so far was against documents this repository chose: the 28 an install ships, the fixtures beside the code, the 2,000 the generator makes up. scripts/fetch-mul-corpus.sh fetches 271 files from five public repositories, pinned by commit and neither committed nor redistributed, and the new test reports how many read, how many come back the same document, how many are a fixed point and how many are handed back as they arrived. The conformance suite prints the same four figures for the files a MegaMek install ships, and now compares the units as well as the root - Mul::root keeps an empty entity where each unit stood, so comparing it alone never looked inside a machine. --- crates/helm-bv/tests/conformance.rs | 61 +++++++++- crates/helm-unitfile/tests/corpus.rs | 171 +++++++++++++++++++++++++++ scripts/fetch-mul-corpus.sh | 77 ++++++++++++ 3 files changed, 305 insertions(+), 4 deletions(-) create mode 100644 crates/helm-unitfile/tests/corpus.rs create mode 100755 scripts/fetch-mul-corpus.sh diff --git a/crates/helm-bv/tests/conformance.rs b/crates/helm-bv/tests/conformance.rs index 9c69893..27928b2 100644 --- a/crates/helm-bv/tests/conformance.rs +++ b/crates/helm-bv/tests/conformance.rs @@ -1302,16 +1302,29 @@ fn every_mul_megamek_ships_survives_a_round_trip() { let mut units = 0; let mut scored = 0; + let mut unreadable = 0; + let mut same_document = 0; + let mut fixed_point = 0; + let mut untouched = 0; let mut wrong: Vec = Vec::new(); for path in &files { let Ok(text) = std::fs::read_to_string(path) else { + unreadable += 1; continue; }; let Ok(once) = helm_unitfile::parse_mul(&text) else { + unreadable += 1; wrong.push(format!("{}: could not be read", path.display())); continue; }; let written = helm_unitfile::write_mul(&once); + // Nearly never, and counted so that the figure is read rather than + // assumed: this writer prints a document rather than editing text, so + // a file it did not write is reformatted the first time it is + // written. + if written == text { + untouched += 1; + } let twice = match helm_unitfile::parse_mul(&written) { Ok(twice) => twice, Err(e) => { @@ -1325,7 +1338,9 @@ fn every_mul_megamek_ships_survives_a_round_trip() { // Written again from what we wrote, which is where a store's promise // that an unedited save changes nothing either holds or does not. let again = helm_unitfile::write_mul(&twice); - if again != written { + if again == written { + fixed_point += 1; + } else { let differs = written .lines() .zip(again.lines()) @@ -1344,7 +1359,19 @@ fn every_mul_megamek_ships_survives_a_round_trip() { // carries ``, quirks, bomb loads and camouflage that helm has // no rules about, and those are exactly the things a rewrite drops // without anything noticing. - if once.root != twice.root { + // + // The units as well as the root: `Mul::root` keeps an empty entity + // where each unit stood, so comparing only the root compares the + // scaffolding and never looks inside a machine. + let sources = |mul: &helm_unitfile::Mul| { + mul.units + .iter() + .map(|u| u.source.clone()) + .collect::>() + }; + if once.root == twice.root && sources(&once) == sources(&twice) { + same_document += 1; + } else { let (a, b) = (&once.root, &twice.root); wrong.push(format!( "{}: the document changed on the way round ({} children became {})", @@ -1375,9 +1402,26 @@ fn every_mul_megamek_ships_survives_a_round_trip() { } } + // The same three figures the external corpus prints, so the two can be + // read side by side: what an install ships is one writer's output, and a + // player's file is not. + let read = files.len() - unreadable; + println!( + "{} files, {read} read ({:.1}%), {units} units, {scored} of them scored", + files.len(), + percent(read, files.len()) + ); + println!( + " {same_document} came back the same document ({:.1}% of those read)", + percent(same_document, read) + ); + println!( + " {fixed_point} written again to the same bytes ({:.1}% of those read)", + percent(fixed_point, read) + ); println!( - "{} files, {units} units, {scored} of them scored, all worth the same and written back byte for byte", - files.len() + " {untouched} were already in the form this writes ({:.1}% of those read)", + percent(untouched, read) ); for line in wrong.iter().take(10) { println!(" {line}"); @@ -1389,6 +1433,15 @@ fn every_mul_megamek_ships_survives_a_round_trip() { ); } +/// A count as a share of another, for a corpus figure that is read rather than +/// asserted. +fn percent(part: usize, whole: usize) -> f64 { + if whole == 0 { + return 0.0; + } + part as f64 * 100.0 / whole as f64 +} + fn collect_muls(dir: &std::path::Path, out: &mut Vec) { let Ok(entries) = std::fs::read_dir(dir) else { return; diff --git a/crates/helm-unitfile/tests/corpus.rs b/crates/helm-unitfile/tests/corpus.rs new file mode 100644 index 0000000..cc35e44 --- /dev/null +++ b/crates/helm-unitfile/tests/corpus.rs @@ -0,0 +1,171 @@ +//! Files other people wrote. +//! +//! Every other check on the `.mul` reader is against documents this repository +//! chose: the 28 a MegaMek install ships, the fixtures beside the code, and +//! the 2,000 the generator makes up. All three are one writer's idea of the +//! format, or ours. This one is forces people saved and put somewhere public, +//! over several years and out of several tools, which is the population a file +//! arriving from a player is actually drawn from. +//! +//! `scripts/fetch-mul-corpus.sh` puts the corpus somewhere and prints where. +//! Nothing here is committed and nothing is redistributed. +//! +//! It measures rather than only asserting. Three figures, and they are not the +//! same question: +//! +//! - how many files can be read at all; +//! - how many come back from a round trip the same document, element for +//! element and attribute for attribute; +//! - how many are a fixed point - written, read and written again to the same +//! bytes. +//! +//! And a fourth that is expected to be nearly nothing: how many files are +//! handed back byte for byte as they arrived. This writer prints a document +//! rather than editing text, so a file it did not write is reformatted the +//! first time it is written whatever the reader did with it. The number is +//! here because it is the one somebody will assume is high. + +use std::path::{Path, PathBuf}; + +/// Every `.mul` under the corpus directory. +fn collect(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + collect(&path, out); + } else if path + .extension() + .is_some_and(|e| e.eq_ignore_ascii_case("mul")) + { + out.push(path); + } + } + out.sort(); +} + +/// The document as read, entities included. +/// +/// `Mul::root` keeps an empty entity where each unit stood - the unit holds +/// the real one - so a check that compares only the root compares the +/// scaffolding and never looks inside a machine. +fn document(mul: &helm_unitfile::Mul) -> (&helm_unitfile::Kept, Vec<&helm_unitfile::Kept>) { + (&mul.root, mul.units.iter().map(|u| &u.source).collect()) +} + +fn percent(part: usize, whole: usize) -> f64 { + if whole == 0 { + return 0.0; + } + part as f64 * 100.0 / whole as f64 +} + +#[test] +#[ignore = "needs a corpus; set HELM_MUL_CORPUS (scripts/fetch-mul-corpus.sh)"] +fn files_other_people_wrote_survive_a_round_trip() { + let Ok(root) = std::env::var("HELM_MUL_CORPUS") else { + panic!("set HELM_MUL_CORPUS to a corpus directory; scripts/fetch-mul-corpus.sh makes one"); + }; + let mut files = Vec::new(); + collect(Path::new(&root), &mut files); + assert!(!files.is_empty(), "{root} holds no .mul files"); + + let (mut read, mut same_document, mut fixed_point, mut untouched) = (0, 0, 0, 0); + let mut unreadable: Vec = Vec::new(); + let mut lost: Vec = Vec::new(); + let mut unstable: Vec = Vec::new(); + + for path in &files { + let name = path.file_name().map_or_else( + || path.display().to_string(), + |n| n.to_string_lossy().into(), + ); + let Ok(text) = std::fs::read_to_string(path) else { + unreadable.push(format!("{name}: not UTF-8")); + continue; + }; + let once = match helm_unitfile::parse_mul(&text) { + Ok(once) => once, + Err(e) => { + unreadable.push(format!("{name}: {e}")); + continue; + } + }; + read += 1; + + let first = helm_unitfile::write_mul(&once); + if first == text { + untouched += 1; + } + let Ok(twice) = helm_unitfile::parse_mul(&first) else { + lost.push(format!("{name}: what we wrote is not readable")); + continue; + }; + if document(&once) == document(&twice) { + same_document += 1; + } else { + lost.push(format!( + "{name}: {} units became {}", + once.units.len(), + twice.units.len() + )); + } + + let second = helm_unitfile::write_mul(&twice); + if second == first { + fixed_point += 1; + } else { + let differs = first + .lines() + .zip(second.lines()) + .find(|(a, b)| a != b) + .map_or_else( + || String::from("the file changed length"), + |(a, b)| format!("`{}` became `{}`", a.trim(), b.trim()), + ); + unstable.push(format!("{name}: {differs}")); + } + } + + let total = files.len(); + println!("{total} files from {root}"); + println!(" {read} read ({:.1}%)", percent(read, total)); + println!( + " {same_document} came back the same document ({:.1}% of those read)", + percent(same_document, read) + ); + println!( + " {fixed_point} written again to the same bytes ({:.1}% of those read)", + percent(fixed_point, read) + ); + println!( + " {untouched} were already in the form this writes ({:.1}% of those read)", + percent(untouched, read) + ); + for (what, lines) in [ + ("could not be read", &unreadable), + ("changed on the way round", &lost), + ("were not a fixed point", &unstable), + ] { + if lines.is_empty() { + continue; + } + println!("\n{} {what}:", lines.len()); + for line in lines.iter().take(10) { + println!(" {line}"); + } + } + + // A file that cannot be read is a gap in the reader worth knowing about, + // but it is not this test's subject and some of a public corpus is + // genuinely broken. What may not happen is a file that reads and is then + // quietly changed by being written back. + assert!(lost.is_empty(), "{} documents changed", lost.len()); + assert!( + unstable.is_empty(), + "{} were not a fixed point", + unstable.len() + ); +} diff --git a/scripts/fetch-mul-corpus.sh b/scripts/fetch-mul-corpus.sh new file mode 100755 index 0000000..480f6f7 --- /dev/null +++ b/scripts/fetch-mul-corpus.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Fetch `.mul` files other people wrote, which is the only corpus that says +# whether this repository's reader survives contact with the format as it is +# actually used. +# +# ./scripts/fetch-mul-corpus.sh +# ./scripts/fetch-mul-corpus.sh /somewhere/else +# +# Prints the corpus path on stdout, so it composes: +# +# HELM_MUL_CORPUS="$(./scripts/fetch-mul-corpus.sh)" \ +# cargo test -p helm-unitfile --test corpus -- --ignored --nocapture +# +# A MegaMek install ships 28 `.mul` files and they are all MegaMek's own, +# written by one writer for one purpose. These are forces people saved: some +# from MegaMek, some from MekHQ, some from tools that are neither, over +# several years of the format. That is the population a file arriving from a +# player is drawn from. +# +# Nothing fetched here is committed and nothing is redistributed, the same +# posture this repository takes to a MegaMek install. The repositories below +# are public and pinned by commit, and their licences are their own - one is +# GPL-2.0, two say nothing at all - which is a reason to read them and not a +# reason to ship them. +# +# A repository that moves does not change what is tested: the pin is the +# corpus. Repointing one is a commit, so a figure that moves has something +# that says why. +set -euo pipefail + +# owner/repo commit +REPOS=( + "MegaMek/mm-data 5e630eab38ce3dbfb032343fa9504d94043c2d67" + "nx78/shackclient 557953c0b727059c1c8ca60d3fa8e724d00f9299" + "mothmind/clusterstrike-crew-game df96e911b382ebc52f279ebb858fd105ac492305" + "Eudicods/let-the-bots-fight ef4ee58a86df72c6af1b3b7f99d94dd6887fdc96" + "maldun/UltraMekCore 8d4d8484311e1552eeb929fcdef1e061be544943" +) + +out="${1:-$HOME/.cache/helm/mul-corpus}" +mkdir -p "$out" + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +for entry in "${REPOS[@]}"; do + read -r repo sha <<<"$entry" + slug="${repo//\//-}" + dest="$out/$slug" + # A pin that is already unpacked is the same bytes it was: nothing to do. + if [[ -f "$dest/.commit" ]] && [[ "$(cat "$dest/.commit")" == "$sha" ]]; then + echo "have $repo@${sha:0:8}" >&2 + continue + fi + + echo "fetching $repo@${sha:0:8}" >&2 + tarball="$work/$slug.tar.gz" + curl -fsSL "https://codeload.github.com/$repo/tar.gz/$sha" -o "$tarball" + + rm -rf "$dest" + mkdir -p "$dest" + # Only the `.mul` files, and flattened: a name that keeps its path is a name + # a failing test can be traced back to the file it came from. + unpacked="$work/$slug" + mkdir -p "$unpacked" + tar -xzf "$tarball" -C "$unpacked" --strip-components=1 --wildcards '*.mul' 2>/dev/null || true + found=0 + while IFS= read -r -d '' file; do + rel="${file#"$unpacked"/}" + cp "$file" "$dest/${rel//\//_}" + found=$((found + 1)) + done < <(find "$unpacked" -name '*.mul' -print0) + echo "$sha" >"$dest/.commit" + echo " $found files" >&2 +done + +echo "$out" -- 2.51.2