//! `helm bv-report`: how this repository's battle values compare with //! MegaMek's, and every way of asking that question. //! //! The whole library, one design's working beside MegaMek's, the designs that //! disagree grouped by what they have in common, what each rule fires on, and //! what a recompute costs. All of it prints; none of it writes a file. use helm_core::Catalogue; use crate::inputs::{install_and_dump, read_library, resolve_version}; use crate::options::parse_opts; /// Print how helm-bv's answers compare with MegaMek's, and exit non-zero /// while any of them disagree. /// /// The same report the conformance test pins, printed for a human. It is /// deliberately the same run - `Conformance::over` is one function - so that a /// number seen here and a number seen in CI cannot differ by anything except /// their inputs. pub(crate) fn bv_report(args: &[String]) -> Result<(), String> { let opts = parse_opts(args)?; let (megamek, bridge) = install_and_dump( &opts, "--bridge-dir is required: there is nothing to check against without it", )?; let version = resolve_version(&megamek, opts.version.clone()); let library = read_library(&megamek)?; let catalogue = helm_bridge::read_catalogue(&bridge.join("equipment.jsonl")).map_err(|e| e.to_string())?; let mut megamek_bv: std::collections::BTreeMap = helm_bridge::read_units(&bridge.join("units.jsonl")) .map_err(|e| e.to_string())? .into_iter() .filter_map(|s| { let mut b = helm_core::BvBreakdown::new(&s.name); b.battle_value = Some(s.battle_value?); Some((s.name, b)) }) .collect(); // The bv dumps carry the working of MegaMek's calculation and not only its // answer, so each rating can be checked on its own - one file per // calculator. A bridge dump made before they existed still checks the // total. // // They also carry a better total than the summary does: MegaMek's own // summary cache records the base figure for a platoon whose anti-Mek // skill moves it, and the calculator records what it plays with. See // `crates/helm-bv/UPSTREAM.md`. let mut found = false; for file in [ "bv.jsonl", "bv-vehicle.jsonl", "bv-ba.jsonl", "bv-infantry.jsonl", ] { let path = bridge.join(file); if !path.is_file() { continue; } found = true; for b in helm_bridge::read_bv(&path).map_err(|e| e.to_string())? { megamek_bv.insert(b.name.clone(), b); } } if !found { eprintln!( "no bv dump in {}: only the total is checkable", bridge.display() ); } if let Some(needle) = &opts.unit { return one_unit(&library, &catalogue, &megamek_bv, needle); } if opts.damaged { return print_damaged(&library, &catalogue, &bridge); } if opts.damaged { return print_damaged(&library, &catalogue, &bridge); } if opts.bench { return print_bench(&library, &catalogue); } if opts.rules { return print_rules(&library, &catalogue); } if opts.clusters { return print_clusters(&library, &catalogue, &megamek_bv, opts.label.as_deref()); } let report = helm_bv::Conformance::over(&library.units, &catalogue, &megamek_bv, &version); println!("{report}"); if report.is_clean() { Ok(()) } else { Err(format!( "{} of {} scored designs do not yet match MegaMek's battle value", report.in_scope() - report.battle_value().matched(), report.in_scope() )) } } /// Print one design's working beside MegaMek's, for chasing a disagreement. /// /// The conformance report says how many designs are wrong and by how much; it /// cannot say which term. This can, and it is the difference between a search /// and a bug report. fn one_unit( library: &helm_unitfile::Library, catalogue: &Catalogue, megamek: &std::collections::BTreeMap, needle: &str, ) -> Result<(), String> { let lower = needle.to_ascii_lowercase(); // An exact name wins outright. Twenty designs contain "Rhino" and one of // them is called it, so asking by full name should not be ambiguous. let matches: Vec<_> = library .units .iter() .filter(|u| u.display_name().eq_ignore_ascii_case(needle)) .collect(); let matches: Vec<_> = if matches.is_empty() { library .units .iter() .filter(|u| u.display_name().to_ascii_lowercase().contains(&lower)) .collect() } else { matches }; let unit = match matches.as_slice() { [] => return Err(format!("no design matches {needle:?}")), [only] => *only, many => { eprintln!("{} designs match {needle:?}:", many.len()); for u in many.iter().take(20) { eprintln!(" {}", u.display_name()); } return Err("be more specific".into()); } }; let name = unit.display_name(); println!("{name}"); // A platoon has no tonnage, no engine, no armour, no slots and no heat, // so neither the header line nor the Mek-shaped view below has anything // to say about one. Its own terms do. if unit .unit_type .as_deref() .is_some_and(|t| t.eq_ignore_ascii_case("infantry")) { let field = |key: &str| { unit.fields .iter() .find(|(k, _)| k.eq_ignore_ascii_case(key)) .map_or("?", |(_, v)| v.trim()) }; println!( " {} squads of {}, {}", field("squadn"), field("squad_size"), unit.motion_type.as_deref().unwrap_or("no motion type"), ); match helm_bv::terms_in(unit, catalogue, &helm_bv::Condition::undamaged()) { Ok(t) => { println!("\ndefensive"); println!(" troopers {:>10.1}", t.defensive.structure); println!(" x factor {:>10.2}", t.defensive.factor); println!(" = {:>22.1}", t.defensive.total); println!("\noffensive"); println!(" weapons {:>10.1}", t.offensive.weapons); println!(" field guns {:>10.1}", t.offensive.weight); println!(" ammo {:>10.1}", t.offensive.ammo); println!(" x speed {:>10.2}", t.offensive.speed_factor); println!(" = {:>22.1}", t.offensive.total); println!("\n x type {:>10.2}", t.scale); println!(" base {:>10}", t.base()); println!(" x skill {:>10.2}", t.skill); println!(" battle value {:>10}", t.battle_value); } Err(e) => println!("\nnot scored: {e}"), } print_megamek(megamek, &name); return Ok(()); } println!( " {} tons, {} walk, {} jump, {}, {}", unit.mass.unwrap_or_default(), unit.walk_mp.unwrap_or_default(), unit.jump_mp.unwrap_or_default(), unit.engine.as_deref().unwrap_or("no engine"), unit.heat_sinks.as_deref().unwrap_or("no heat sinks"), ); // `Machine::read` is the Mek reader, so a vehicle or a suit read through // it fails on a header line it does not have. Its terms are the same // terms, and they come from the gate every entry point goes through. let machine = helm_bv::Machine::read(unit, catalogue); if machine.is_err() && let Ok(t) = helm_bv::terms_in(unit, catalogue, &helm_bv::Condition::undamaged()) { println!("\ndefensive"); println!(" armour {:>10.1}", t.defensive.armor); println!(" structure {:>10.1}", t.defensive.structure); println!(" equipment {:>10.1}", t.defensive.equipment); println!(" explosive {:>10.1}", t.defensive.explosive); println!(" subtotal {:>10.1}", t.defensive.subtotal); println!(" x factor {:>10.2}", t.defensive.factor); println!(" = {:>22.1}", t.defensive.total); println!("\noffensive"); println!(" weapons {:>10.1}", t.offensive.weapons); println!(" equipment {:>10.1}", t.offensive.equipment); println!(" ammo {:>10.1}", t.offensive.ammo); println!(" weight {:>10.1}", t.offensive.weight); println!(" subtotal {:>10.1}", t.offensive.subtotal); println!(" x fire control {:>10.2}", t.offensive.fire_control); println!(" x speed {:>10.2}", t.offensive.speed_factor); println!(" = {:>22.1}", t.offensive.total); println!("\n x scale {:>10.2}", t.scale); println!(" battle value {:>10}", t.battle_value); print_megamek(megamek, &name); return Ok(()); } match machine .as_ref() .map_err(Clone::clone) .and_then(helm_bv::defensive) { Ok(d) => { println!("\ndefensive"); println!(" armour {:>10.1}", d.armor); println!(" structure {:>10.1}", d.structure); println!(" gyro {:>10.1}", d.gyro); println!(" equipment {:>10.1}", d.equipment); println!(" explosive {:>10.1}", d.explosive); println!(" subtotal {:>10.1}", d.subtotal); println!(" x factor {:>10.2}", d.factor); println!(" = {:>22.1}", d.total); } Err(e) => println!("\ndefensive: {e}"), } match machine .as_ref() .map_err(Clone::clone) .and_then(helm_bv::offensive) { Ok(o) => { println!("\noffensive"); println!(" weapons {:>10.1}", o.weapons); println!(" ammo {:>10.1}", o.ammo); println!(" weight {:>10.1}", o.weight); println!(" subtotal {:>10.1}", o.subtotal); println!(" x speed {:>10.2}", o.speed_factor); println!(" = {:>22.1}", o.total); println!( " (heat budget {}, weapons make {:.1})", o.heat_budget, o.heat_used ); } Err(e) => println!("\noffensive: {e}"), } let items = helm_bv::classify(unit, catalogue); if !items.is_empty() { println!("\nequipment"); for i in &items { println!( " {:<34} {:>3} slots bv {:>7.1} {}", i.name, i.slots, i.battle_value, i.role ); } } print_megamek(megamek, &name); Ok(()) } /// What MegaMek makes of the same design, or that it has never seen it. fn print_megamek(megamek: &std::collections::BTreeMap, name: &str) { let Some(theirs) = megamek.get(name) else { println!("\nmegamek has no record for this design"); return; }; println!("\nmegamek"); println!( " defensive {:>10.1}", theirs.defensive.unwrap_or_default() ); println!( " offensive {:>10.1}", theirs.offensive.unwrap_or_default() ); println!( " battle value {:>10}", theirs.battle_value.unwrap_or_default() ); } /// Group the designs that still disagree by what they have in common. /// /// The conformance report says how many are wrong; this says what kind. A /// label carrying five hundred wrong designs is an afternoon that fixes five /// hundred designs, and the ranked list of individual disagreements never /// shows that - it shows whichever three designs are strangest. fn print_clusters( library: &helm_unitfile::Library, catalogue: &Catalogue, megamek: &std::collections::BTreeMap, label: Option<&str>, ) -> Result<(), String> { let mut designs = Vec::new(); for unit in &library.units { let name = unit.display_name(); let Some(theirs) = megamek.get(&name).and_then(|b| b.battle_value) else { continue; }; let verdict = match helm_bv::battle_value(unit, catalogue) { Ok(ours) => Some(ours == theirs), // Declining by unit type is out of scope, not a gap. Err(helm_bv::Unsupported::UnitType(_)) => continue, Err(_) => None, }; designs.push((unit, verdict)); } let total = designs.len(); let matched = designs.iter().filter(|(_, v)| *v == Some(true)).count(); println!( "{matched} of {total} in-scope designs match. Grouping the {} that do not:\n", total - matched ); println!("{:>6} {:>6} {:>6} label", "wrong", "of", "rate"); // One label's failures, when that is what was asked for. A cluster says a // rule is missing; this says which designs and which half of the // calculation, which is the next question every time. if let Some(label) = label { let mut listed = 0; for (unit, verdict) in &designs { let carries = label == "*" || helm_bv::labels(unit, catalogue).iter().any(|l| l == label); // Only designs that produced an answer and got it wrong. One this // crate declines is reported by the summary under the reason it // declined, and listing it here as a disagreement made the two // counts differ by the number of LAMs. if *verdict != Some(false) || !carries { continue; } let name = unit.display_name(); let theirs = megamek.get(&name); let machine = helm_bv::Machine::read(unit, catalogue).ok(); let gap = |ours: Option, theirs: Option| match (ours, theirs) { (Some(a), Some(b)) if (a - b).abs() <= helm_core::RATING_TOLERANCE => { " ok".to_string() } (Some(a), Some(b)) => format!("{:+7.1}", a - b), _ => " -".to_string(), }; println!( " {name:<38} def {} off {}", gap( machine .as_ref() .and_then(|m| helm_bv::defensive(m).ok()) .map(|d| d.total), theirs.and_then(|b| b.defensive) ), gap( machine .as_ref() .and_then(|m| helm_bv::offensive(m).ok()) .map(|o| o.total), theirs.and_then(|b| b.offensive) ), ); listed += 1; } if label == "*" { println!("\n{listed} designs disagree."); } else { println!("\n{listed} designs carrying {label:?} disagree."); } return Ok(()); } // The plainest failures first. A design that is wrong while carrying // nothing unusual is wrong in arithmetic every other design shares, and // that is worth more than the strangest design in the library. let mut plain: Vec<_> = designs .iter() .filter(|(_, v)| *v != Some(true)) .map(|(unit, _)| (helm_bv::oddity(unit, catalogue), *unit)) .collect(); plain.sort_by(|a, b| { a.0.cmp(&b.0) .then_with(|| a.1.display_name().cmp(&b.1.display_name())) }); println!("\nthe most ordinary designs that still disagree"); for (oddity, unit) in plain.iter().take(15) { let name = unit.display_name(); let theirs = megamek .get(&name) .and_then(|b| b.battle_value) .unwrap_or_default(); let ours = helm_bv::battle_value(unit, catalogue) .map_or_else(|e| e.to_string(), |v| v.to_string()); let notable: Vec = helm_bv::labels(unit, catalogue) .into_iter() .filter(|l| !l.starts_with("config:") && !l.starts_with("tech:")) .collect(); println!( " {oddity:>2} odd {name:<34} ours {ours:>6} megamek {theirs:>6} {}", notable.join(", ") ); } println!(); for c in helm_bv::clusters(designs, catalogue) { let wrong = c.differed + c.unscored; // A label with nothing wrong is noise, and so is one carrying a // handful of designs - it cannot explain anything. if wrong == 0 || c.total() < 5 { continue; } let unscored = if c.unscored > 0 { format!(" ({} unscored)", c.unscored) } else { String::new() }; println!( "{:>6} {:>6} {:>5.0}% {:<34}{}", wrong, c.total(), 100.0 * c.rate(), c.label, unscored ); } Ok(()) } /// How many designs each design-level rule fires on. /// /// The question this answers is "does this rule do anything at all". A wrong /// flag name does not fail - the lookup matches nothing and the rule silently /// stops applying, which looks exactly like a rule nobody has written yet. /// A zero here says which. fn print_rules(library: &helm_unitfile::Library, catalogue: &Catalogue) -> Result<(), String> { // Every design the scorer can read, not every design the *Mek* reader // can: three of the rules are battle armour's, and reading a suit as a // Mek is what had them firing on nothing. let condition = helm_bv::Condition::undamaged(); let machines: Vec<_> = library .units .iter() .filter_map(|unit| helm_bv::readable(unit, catalogue, &condition).ok()) .collect(); let mut counts: std::collections::BTreeMap<&str, usize> = helm_bv::RULES.iter().map(|rule| (rule.name, 0)).collect(); for machine in &machines { for name in helm_bv::fired(machine) { *counts.entry(name).or_default() += 1; } } println!("{} designs read\n", machines.len()); println!("{:>7} rule", "designs"); let mut rows: Vec<_> = counts.into_iter().collect(); rows.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0))); for (name, n) in &rows { let note = if *n == 0 { " <- fires on nothing" } else { "" }; println!("{n:>7} {name}{note}"); } if rows.iter().any(|(_, n)| *n == 0) { return Err("a rule fires on no design at all; check its flag name".into()); } Ok(()) } /// How helm scores designs that have been shot at, against MegaMek's own /// answers for the same states. /// /// A `.mtf` describes a design as it leaves the factory, so this is the only /// way to check the rules that only damage reaches - and it is the report the /// conformance test asserts on, printed for a human. fn print_damaged( library: &helm_unitfile::Library, catalogue: &Catalogue, bridge: &std::path::Path, ) -> Result<(), String> { let path = bridge.join("damaged.jsonl"); let rows = helm_bridge::read_damaged(&path).map_err(|e| format!("{}: {e}", path.display()))?; if rows.is_empty() { return Err(format!( "{} holds no damaged designs; run bridge/dump.sh to make some", path.display() )); } let mut checked = 0; let mut wrong = 0; let mut last = String::new(); for row in &rows { let Some(unit) = library.units.iter().find(|u| u.display_name() == row.name) else { continue; }; let condition = helm_bv::Condition { armor: row.armor.clone(), structure: row.structure.clone(), destroyed: row.destroyed.clone(), empty_ammo: row.empty_ammo.clone(), loaded: Default::default(), }; checked += 1; if row.name != last { println!("{}", row.name); last = row.name.clone(); } match helm_bv::battle_value_in(unit, catalogue, &condition) { Ok(ours) if ours == row.battle_value => { println!(" {:<18} {:>6}", row.scenario, ours); } Ok(ours) => { wrong += 1; println!( " {:<18} {:>6} megamek {} ({:+})", row.scenario, ours, row.battle_value, ours - row.battle_value ); } Err(why) => { wrong += 1; println!(" {:<18} {why}", row.scenario); } } } println!("\n{} of {checked} states agree", checked - wrong); if wrong > 0 { return Err(format!("{wrong} damaged states disagree")); } Ok(()) } /// Time the operations a force-building screen actually performs. /// /// The question this answers is whether a browser can recompute battle value /// as somebody drags a slider, and the useful answer is not one number. The /// three things a player does cost wildly different amounts: /// /// * **Reseating a pilot** changes no part of the design. Battle value is the /// design's figure times a number from a 9x9 table, so nothing is /// recomputed at all. /// * **Repairing or damaging** changes the armour, so the ratings have to be /// worked out again - but the loadout has not moved, so the expensive part /// does not have to be redone. /// * **Changing the loadout** invalidates everything, including reading the /// design's critical slots against the catalogue, which is where most of the /// time goes. fn print_bench(library: &helm_unitfile::Library, catalogue: &Catalogue) -> Result<(), String> { use std::time::Instant; let meks: Vec<&helm_core::Unit> = library .units .iter() .filter(|u| helm_bv::Machine::read(u, catalogue).is_ok()) .collect(); if meks.is_empty() { return Err("no scoreable designs; is the catalogue loaded?".into()); } // Enough repetitions that the clock is not the thing being measured. let rounds = 20; let total = meks.len() * rounds; let per = |elapsed: std::time::Duration| elapsed.as_secs_f64() * 1e9 / total as f64; // Everything: read the design and score it. What a loadout change costs. let start = Instant::now(); let mut sink = 0i64; for _ in 0..rounds { for unit in &meks { sink += helm_bv::battle_value(unit, catalogue).unwrap_or(0); } } let whole = per(start.elapsed()); // Reading alone, which is the part a repair does not have to repeat. let start = Instant::now(); for _ in 0..rounds { for unit in &meks { sink += helm_bv::Machine::read(unit, catalogue).map_or(0, |m| m.tons as i64); } } let reading = per(start.elapsed()); // Scoring a design already read. What a repair costs. let prepared: Vec> = meks .iter() .filter_map(|u| helm_bv::Machine::read(u, catalogue).ok()) .collect(); let start = Instant::now(); for _ in 0..rounds { for machine in &prepared { sink += helm_bv::defensive(machine).map_or(0, |d| d.total as i64) + helm_bv::offensive(machine).map_or(0, |o| o.total as i64); } } let scoring = per(start.elapsed()); // Reseating a pilot: a table lookup and a multiply, over a design whose // own battle value has not changed. let start = Instant::now(); for round in 0..rounds { for (i, _) in meks.iter().enumerate() { let pilot = helm_force::Pilot::new((i % 8) as u8, (round % 8) as u8); sink += (2000.0 * pilot.battle_value_multiplier()) as i64; } } let reseating = per(start.elapsed()); println!("{} designs, {rounds} rounds each\n", meks.len()); println!("{:>12} what changed", "ns/unit"); println!("{reseating:>12.0} the pilot (a table lookup; nothing is recomputed)"); println!("{scoring:>12.0} armour or damage (both ratings, loadout already read)"); println!("{reading:>12.0} reading the design (resolving every critical slot)"); println!("{whole:>12.0} the loadout (reading and scoring together)"); println!("\nwhat that means for a screen, at these rates:"); for (units, label) in [(4, "a lance"), (12, "a company"), (36, "a battalion")] { println!( " {label:<12} {:>7.3} ms to rescore after a repair, {:>7.3} ms from scratch", scoring * units as f64 / 1e6, whole * units as f64 / 1e6 ); } // Keep the optimiser from deleting the work being measured. if sink == i64::MIN { println!("(unreachable)"); } Ok(()) }