diff --git a/crates/helm-bv/tests/conformance.rs b/crates/helm-bv/tests/conformance.rs index 9c69893..27928b2 100644 --- a/crates/helm-bv/tests/conformance.rs +++ b/crates/helm-bv/tests/conformance.rs @@ -1302,16 +1302,29 @@ fn every_mul_megamek_ships_survives_a_round_trip() { let mut units = 0; let mut scored = 0; + let mut unreadable = 0; + let mut same_document = 0; + let mut fixed_point = 0; + let mut untouched = 0; let mut wrong: Vec = Vec::new(); for path in &files { let Ok(text) = std::fs::read_to_string(path) else { + unreadable += 1; continue; }; let Ok(once) = helm_unitfile::parse_mul(&text) else { + unreadable += 1; wrong.push(format!("{}: could not be read", path.display())); continue; }; let written = helm_unitfile::write_mul(&once); + // Nearly never, and counted so that the figure is read rather than + // assumed: this writer prints a document rather than editing text, so + // a file it did not write is reformatted the first time it is + // written. + if written == text { + untouched += 1; + } let twice = match helm_unitfile::parse_mul(&written) { Ok(twice) => twice, Err(e) => { @@ -1325,7 +1338,9 @@ fn every_mul_megamek_ships_survives_a_round_trip() { // Written again from what we wrote, which is where a store's promise // that an unedited save changes nothing either holds or does not. let again = helm_unitfile::write_mul(&twice); - if again != written { + if again == written { + fixed_point += 1; + } else { let differs = written .lines() .zip(again.lines()) @@ -1344,7 +1359,19 @@ fn every_mul_megamek_ships_survives_a_round_trip() { // carries ``, quirks, bomb loads and camouflage that helm has // no rules about, and those are exactly the things a rewrite drops // without anything noticing. - if once.root != twice.root { + // + // The units as well as the root: `Mul::root` keeps an empty entity + // where each unit stood, so comparing only the root compares the + // scaffolding and never looks inside a machine. + let sources = |mul: &helm_unitfile::Mul| { + mul.units + .iter() + .map(|u| u.source.clone()) + .collect::>() + }; + if once.root == twice.root && sources(&once) == sources(&twice) { + same_document += 1; + } else { let (a, b) = (&once.root, &twice.root); wrong.push(format!( "{}: the document changed on the way round ({} children became {})", @@ -1375,9 +1402,26 @@ fn every_mul_megamek_ships_survives_a_round_trip() { } } + // The same three figures the external corpus prints, so the two can be + // read side by side: what an install ships is one writer's output, and a + // player's file is not. + let read = files.len() - unreadable; + println!( + "{} files, {read} read ({:.1}%), {units} units, {scored} of them scored", + files.len(), + percent(read, files.len()) + ); + println!( + " {same_document} came back the same document ({:.1}% of those read)", + percent(same_document, read) + ); + println!( + " {fixed_point} written again to the same bytes ({:.1}% of those read)", + percent(fixed_point, read) + ); println!( - "{} files, {units} units, {scored} of them scored, all worth the same and written back byte for byte", - files.len() + " {untouched} were already in the form this writes ({:.1}% of those read)", + percent(untouched, read) ); for line in wrong.iter().take(10) { println!(" {line}"); @@ -1389,6 +1433,15 @@ fn every_mul_megamek_ships_survives_a_round_trip() { ); } +/// A count as a share of another, for a corpus figure that is read rather than +/// asserted. +fn percent(part: usize, whole: usize) -> f64 { + if whole == 0 { + return 0.0; + } + part as f64 * 100.0 / whole as f64 +} + fn collect_muls(dir: &std::path::Path, out: &mut Vec) { let Ok(entries) = std::fs::read_dir(dir) else { return; diff --git a/crates/helm-unitfile/tests/corpus.rs b/crates/helm-unitfile/tests/corpus.rs new file mode 100644 index 0000000..cc35e44 --- /dev/null +++ b/crates/helm-unitfile/tests/corpus.rs @@ -0,0 +1,171 @@ +//! Files other people wrote. +//! +//! Every other check on the `.mul` reader is against documents this repository +//! chose: the 28 a MegaMek install ships, the fixtures beside the code, and +//! the 2,000 the generator makes up. All three are one writer's idea of the +//! format, or ours. This one is forces people saved and put somewhere public, +//! over several years and out of several tools, which is the population a file +//! arriving from a player is actually drawn from. +//! +//! `scripts/fetch-mul-corpus.sh` puts the corpus somewhere and prints where. +//! Nothing here is committed and nothing is redistributed. +//! +//! It measures rather than only asserting. Three figures, and they are not the +//! same question: +//! +//! - how many files can be read at all; +//! - how many come back from a round trip the same document, element for +//! element and attribute for attribute; +//! - how many are a fixed point - written, read and written again to the same +//! bytes. +//! +//! And a fourth that is expected to be nearly nothing: how many files are +//! handed back byte for byte as they arrived. This writer prints a document +//! rather than editing text, so a file it did not write is reformatted the +//! first time it is written whatever the reader did with it. The number is +//! here because it is the one somebody will assume is high. + +use std::path::{Path, PathBuf}; + +/// Every `.mul` under the corpus directory. +fn collect(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + collect(&path, out); + } else if path + .extension() + .is_some_and(|e| e.eq_ignore_ascii_case("mul")) + { + out.push(path); + } + } + out.sort(); +} + +/// The document as read, entities included. +/// +/// `Mul::root` keeps an empty entity where each unit stood - the unit holds +/// the real one - so a check that compares only the root compares the +/// scaffolding and never looks inside a machine. +fn document(mul: &helm_unitfile::Mul) -> (&helm_unitfile::Kept, Vec<&helm_unitfile::Kept>) { + (&mul.root, mul.units.iter().map(|u| &u.source).collect()) +} + +fn percent(part: usize, whole: usize) -> f64 { + if whole == 0 { + return 0.0; + } + part as f64 * 100.0 / whole as f64 +} + +#[test] +#[ignore = "needs a corpus; set HELM_MUL_CORPUS (scripts/fetch-mul-corpus.sh)"] +fn files_other_people_wrote_survive_a_round_trip() { + let Ok(root) = std::env::var("HELM_MUL_CORPUS") else { + panic!("set HELM_MUL_CORPUS to a corpus directory; scripts/fetch-mul-corpus.sh makes one"); + }; + let mut files = Vec::new(); + collect(Path::new(&root), &mut files); + assert!(!files.is_empty(), "{root} holds no .mul files"); + + let (mut read, mut same_document, mut fixed_point, mut untouched) = (0, 0, 0, 0); + let mut unreadable: Vec = Vec::new(); + let mut lost: Vec = Vec::new(); + let mut unstable: Vec = Vec::new(); + + for path in &files { + let name = path.file_name().map_or_else( + || path.display().to_string(), + |n| n.to_string_lossy().into(), + ); + let Ok(text) = std::fs::read_to_string(path) else { + unreadable.push(format!("{name}: not UTF-8")); + continue; + }; + let once = match helm_unitfile::parse_mul(&text) { + Ok(once) => once, + Err(e) => { + unreadable.push(format!("{name}: {e}")); + continue; + } + }; + read += 1; + + let first = helm_unitfile::write_mul(&once); + if first == text { + untouched += 1; + } + let Ok(twice) = helm_unitfile::parse_mul(&first) else { + lost.push(format!("{name}: what we wrote is not readable")); + continue; + }; + if document(&once) == document(&twice) { + same_document += 1; + } else { + lost.push(format!( + "{name}: {} units became {}", + once.units.len(), + twice.units.len() + )); + } + + let second = helm_unitfile::write_mul(&twice); + if second == first { + fixed_point += 1; + } else { + let differs = first + .lines() + .zip(second.lines()) + .find(|(a, b)| a != b) + .map_or_else( + || String::from("the file changed length"), + |(a, b)| format!("`{}` became `{}`", a.trim(), b.trim()), + ); + unstable.push(format!("{name}: {differs}")); + } + } + + let total = files.len(); + println!("{total} files from {root}"); + println!(" {read} read ({:.1}%)", percent(read, total)); + println!( + " {same_document} came back the same document ({:.1}% of those read)", + percent(same_document, read) + ); + println!( + " {fixed_point} written again to the same bytes ({:.1}% of those read)", + percent(fixed_point, read) + ); + println!( + " {untouched} were already in the form this writes ({:.1}% of those read)", + percent(untouched, read) + ); + for (what, lines) in [ + ("could not be read", &unreadable), + ("changed on the way round", &lost), + ("were not a fixed point", &unstable), + ] { + if lines.is_empty() { + continue; + } + println!("\n{} {what}:", lines.len()); + for line in lines.iter().take(10) { + println!(" {line}"); + } + } + + // A file that cannot be read is a gap in the reader worth knowing about, + // but it is not this test's subject and some of a public corpus is + // genuinely broken. What may not happen is a file that reads and is then + // quietly changed by being written back. + assert!(lost.is_empty(), "{} documents changed", lost.len()); + assert!( + unstable.is_empty(), + "{} were not a fixed point", + unstable.len() + ); +} diff --git a/scripts/fetch-mul-corpus.sh b/scripts/fetch-mul-corpus.sh new file mode 100755 index 0000000..480f6f7 --- /dev/null +++ b/scripts/fetch-mul-corpus.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Fetch `.mul` files other people wrote, which is the only corpus that says +# whether this repository's reader survives contact with the format as it is +# actually used. +# +# ./scripts/fetch-mul-corpus.sh +# ./scripts/fetch-mul-corpus.sh /somewhere/else +# +# Prints the corpus path on stdout, so it composes: +# +# HELM_MUL_CORPUS="$(./scripts/fetch-mul-corpus.sh)" \ +# cargo test -p helm-unitfile --test corpus -- --ignored --nocapture +# +# A MegaMek install ships 28 `.mul` files and they are all MegaMek's own, +# written by one writer for one purpose. These are forces people saved: some +# from MegaMek, some from MekHQ, some from tools that are neither, over +# several years of the format. That is the population a file arriving from a +# player is drawn from. +# +# Nothing fetched here is committed and nothing is redistributed, the same +# posture this repository takes to a MegaMek install. The repositories below +# are public and pinned by commit, and their licences are their own - one is +# GPL-2.0, two say nothing at all - which is a reason to read them and not a +# reason to ship them. +# +# A repository that moves does not change what is tested: the pin is the +# corpus. Repointing one is a commit, so a figure that moves has something +# that says why. +set -euo pipefail + +# owner/repo commit +REPOS=( + "MegaMek/mm-data 5e630eab38ce3dbfb032343fa9504d94043c2d67" + "nx78/shackclient 557953c0b727059c1c8ca60d3fa8e724d00f9299" + "mothmind/clusterstrike-crew-game df96e911b382ebc52f279ebb858fd105ac492305" + "Eudicods/let-the-bots-fight ef4ee58a86df72c6af1b3b7f99d94dd6887fdc96" + "maldun/UltraMekCore 8d4d8484311e1552eeb929fcdef1e061be544943" +) + +out="${1:-$HOME/.cache/helm/mul-corpus}" +mkdir -p "$out" + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +for entry in "${REPOS[@]}"; do + read -r repo sha <<<"$entry" + slug="${repo//\//-}" + dest="$out/$slug" + # A pin that is already unpacked is the same bytes it was: nothing to do. + if [[ -f "$dest/.commit" ]] && [[ "$(cat "$dest/.commit")" == "$sha" ]]; then + echo "have $repo@${sha:0:8}" >&2 + continue + fi + + echo "fetching $repo@${sha:0:8}" >&2 + tarball="$work/$slug.tar.gz" + curl -fsSL "https://codeload.github.com/$repo/tar.gz/$sha" -o "$tarball" + + rm -rf "$dest" + mkdir -p "$dest" + # Only the `.mul` files, and flattened: a name that keeps its path is a name + # a failing test can be traced back to the file it came from. + unpacked="$work/$slug" + mkdir -p "$unpacked" + tar -xzf "$tarball" -C "$unpacked" --strip-components=1 --wildcards '*.mul' 2>/dev/null || true + found=0 + while IFS= read -r -d '' file; do + rel="${file#"$unpacked"/}" + cp "$file" "$dest/${rel//\//_}" + found=$((found + 1)) + done < <(find "$unpacked" -name '*.mul' -print0) + echo "$sha" >"$dest/.commit" + echo " $found files" >&2 +done + +echo "$out"