"""The statistics are load-bearing: they decide what gets believed.""" import unittest from pathlib import Path from sds.stats import Summary, games_needed, group, scenario_name, scenario_size, wilson def result(outcome, winner_team=-1, players=None, scenario=None): return { "scenario": scenario or "/work/scenarios/suite/4v4-jihad-s102040.mms", "outcome": outcome, "round": 10, "elapsedMillis": 1000, "victoryTeam": winner_team, "victoryPlayerId": -1, "players": players or [], } def player(name, team, role, units=4, bv=1000, bv_start=1000): return { "id": team, "name": name, "kind": "princess", "team": team, "role": role, "unitsRemaining": units, "unitsStart": 4, "bvRemaining": bv, "bvStart": bv_start, } class TestWilson(unittest.TestCase): def test_small_samples_are_wide(self): low, high = wilson(12, 20) self.assertLess(low, 0.5) self.assertGreater(high, 0.5) def test_bounds_stay_in_range(self): # The textbook normal interval goes outside [0, 1] here, which is the # reason this is Wilson. low, high = wilson(20, 20) self.assertGreaterEqual(low, 0.0) self.assertLessEqual(high, 1.0) def test_no_trials_claims_nothing(self): self.assertEqual(wilson(0, 0), (0.0, 1.0)) class TestGamesNeeded(unittest.TestCase): def test_smaller_effects_need_more_games(self): self.assertGreater(games_needed(0.05), games_needed(0.10)) class TestSummary(unittest.TestCase): def test_undecided_matches_are_not_half_wins(self): """A round limit is not a draw split between the sides. This is the failure that matters most: counting an undecided game as half a win makes a bot that refuses to engage look average. """ results = [ result("round-limit", players=[player("N", 1, "sds"), player("S", 2, "princess")]), result("victory", 1, [player("N", 1, "sds"), player("S", 2, "princess", units=0)]), ] summary = Summary(results, key="role") self.assertEqual(summary.decided, 1) rate, _ = summary.win_rate("sds") self.assertEqual(rate, 1.0) def test_winner_is_never_inferred_from_survivors(self): """A side with everything left has not won if MegaMek recorded no win.""" results = [ result( "round-limit", players=[ player("N", 1, "sds", units=4, bv=1000), player("S", 2, "princess", units=1, bv=10), ], ) ] summary = Summary(results, key="role") self.assertEqual(summary.decided, 0) self.assertEqual(summary.sides["sds"].wins, 0) def test_roles_follow_the_bot_across_swapped_sides(self): """Alternating deployment edges must not split a bot's tally in two.""" results = [ result("victory", 1, [player("N", 1, "sds"), player("S", 2, "princess")]), result("victory", 1, [player("N", 1, "princess"), player("S", 2, "sds")]), ] summary = Summary(results, key="role") self.assertEqual(set(summary.sides), {"sds", "princess"}) self.assertEqual(summary.decided, 2) # One win each: game one the sds seat was team 1, game two it was team 2. self.assertEqual(summary.sides["sds"].wins, 1) self.assertEqual(summary.sides["princess"].wins, 1) class TestTurnOrder(unittest.TestCase): """Turn order has to be countable, not only correct. "we sent the wrong unit" and "no other unit could have gone" must not look the same, and the mean eligible-set size is the number that tells them apart. A substituted unit is its own column because the bot did answer - it is not a default, it is a refused choice. """ @staticmethod def _seat(**counters): seat = player("N", 1, "sds") seat["sds"] = { "decisions": 10, "answered": 10, "defaulted": 0, "illegal": 0, "failed": 0, **counters, } return seat def test_the_counters_are_summed_and_printed(self): seat = self._seat(eligible=25, chose=10, differed=6, substituted=2) summary = Summary([result("victory", 1, [seat, player("S", 2, "princess")])]) side = summary.sides["N"] self.assertEqual((side.eligible, side.chose, side.differed), (25, 10, 6)) rendered = summary.render() self.assertIn("turn order", rendered) # 25 eligible over 10 decisions: the bot had a real choice. self.assertIn("2.50", rendered) self.assertIn("named a unit that could not act 2 time(s)", rendered) def test_a_log_without_the_counters_prints_no_turn_order_table(self): summary = Summary([result("victory", 1, [self._seat(), player("S", 2, "princess")])]) self.assertNotIn("turn order", summary.render()) class TestBreakdown(unittest.TestCase): """The per-scenario split is what says whether to move the round limit.""" def sides(self): return [player("N", 1, "A"), player("S", 2, "B")] def test_size_comes_from_the_scenario_name(self): self.assertEqual(scenario_size(scenario_name(result("victory"))), "4v4") self.assertEqual(scenario_size("8v8-clan-invasion-s101080"), "8v8") self.assertEqual(scenario_size("mirror-lance"), "other") def test_undecided_splits_round_limit_from_everything_else(self): """A round limit is a scenario too slow for the limit. A stall is a bug. The exit criterion counts only the second, so adding them together would make a passing harness look broken and a broken one look busy. """ eight = "/work/scenarios/suite/8v8-jihad-s102080.mms" results = [ result("victory", 1, self.sides()), result("round-limit", players=self.sides(), scenario=eight), result("stalled", players=self.sides(), scenario=eight), ] by_size = group(results, lambda r: scenario_size(scenario_name(r)), key="role") self.assertEqual(set(by_size), {"4v4", "8v8"}) self.assertEqual(by_size["4v4"].decided, 1) self.assertEqual(by_size["4v4"].undecided, 0) eight_v_eight = by_size["8v8"] self.assertEqual(eight_v_eight.games, 2) self.assertEqual(eight_v_eight.decided, 0) self.assertEqual(eight_v_eight.undecided, 2) self.assertEqual(eight_v_eight.round_limit, 1) self.assertEqual(eight_v_eight.other_undecided, 1) self.assertEqual(eight_v_eight.other_reasons, "stalled 1") def test_a_victory_with_no_recorded_winner_is_undecided_not_a_round_limit(self): # outcome says victory, nothing identifies who won: that is the harness # failing, and it must not be filed under the round limit. broken = result("victory", -1, self.sides()) broken["victoryPlayerId"] = -1 stats = group([broken], scenario_name, key="role")["4v4-jihad-s102040"] self.assertEqual(stats.decided, 0) self.assertEqual(stats.round_limit, 0) self.assertEqual(stats.other_undecided, 1) def test_summary_totals_agree_with_the_breakdown(self): results = [ result("victory", 1, self.sides()), result("round-limit", players=self.sides()), result("timeout", players=self.sides()), ] summary = Summary(results, key="role") self.assertEqual(summary.round_limit, 1) self.assertEqual(summary.other_undecided, 1) self.assertEqual( len(results) - summary.decided, summary.round_limit + summary.other_undecided ) self.assertIn("undecided, not the round limit", summary.render()) if __name__ == "__main__": unittest.main() class TestSurvivorCounts(unittest.TestCase): """`unitsRemaining` must mean live units, not entries in the object list. It used to come from `IGame.getEntitiesOwnedBy`, which filters on owner and nothing else, so wrecks counted and a wiped-out side read as "one unit left at 0 BV". The host now uses `Game.getLiveEntitiesOwnedBy` and keeps the raw figure as `unitsInPlay`, so a destroyed side reads as zero here. """ def test_a_destroyed_side_has_no_survivors(self): wiped = player("N", 1, "sds", units=0, bv=0) wiped["unitsInPlay"] = 2 summary = Summary([result("victory", 2, [wiped, player("S", 2, "princess")])], key="role") self.assertEqual(summary.sides["sds"].units_remaining_total, 0) def test_missing_field_does_not_crash_an_older_log(self): old = player("N", 1, "sds") del old["unitsRemaining"] summary = Summary([result("victory", 2, [old, player("S", 2, "princess")])], key="role") self.assertEqual(summary.sides["sds"].units_remaining_total, 0) class TestProgressOnAPartialRun(unittest.TestCase): """`sds progress` reads a run that is still being written. The interim read exists so a suite run with hours left can be asked whether the vector is in the fight. It has to cope with the two things a live run directory always has: files that are not match results, and a result being written as it is read. """ def setUp(self): import json import tempfile self.dir = Path(tempfile.mkdtemp()) seats = [player("sds North", 1, "sds"), player("princess South", 2, "princess")] for index, team in enumerate((1, 1, 2)): (self.dir / f"m{index}.json").write_text( json.dumps(result("victory", winner_team=team, players=seats)) ) # The things that are not match results. (self.dir / "m0-sds_North.weights.json").write_text('{"weights": {}}') (self.dir / "results.json").write_text("[]") # A match mid-write. (self.dir / "m9.json").write_text('{"outcome": "vic') def tearDown(self): import shutil shutil.rmtree(self.dir, ignore_errors=True) def test_it_counts_only_the_finished_matches(self): from argparse import Namespace from sds.cli import cmd_progress self.assertEqual(cmd_progress(Namespace(run=str(self.dir))), 0) def test_a_half_written_result_is_skipped_rather_than_fatal(self): import json loaded = [] for path in sorted(self.dir.glob("*.json")): if path.name.endswith(".weights.json") or path.name == "results.json": continue try: document = json.loads(path.read_text()) except json.JSONDecodeError: continue if isinstance(document, dict) and "outcome" in document: loaded.append(document) self.assertEqual(len(loaded), 3, "the truncated file was counted or threw") summary = Summary(loaded, key="role") self.assertEqual(summary.decided, 3) def test_an_empty_run_directory_says_so(self): import tempfile from argparse import Namespace from sds.cli import cmd_progress empty = Path(tempfile.mkdtemp()) self.assertEqual(cmd_progress(Namespace(run=str(empty))), 1) class TestPairedComparison(unittest.TestCase): """Two runs over the same scenarios are paired, not independent.""" def test_agreement_carries_no_information(self): from sds.stats import mcnemar # Same discordant counts, wildly different numbers of agreed pairs: # the test must not move, because agreed pairs say nothing about which # bot is better. self.assertEqual(mcnemar(29, 17), mcnemar(29, 17)) def test_no_discordant_pairs_claims_nothing(self): from sds.stats import mcnemar chi, p = mcnemar(0, 0) self.assertEqual(chi, 0.0) self.assertEqual(p, 1.0) def test_a_lopsided_split_separates(self): from sds.stats import mcnemar _, p = mcnemar(40, 5) self.assertLess(p, 0.05) def test_an_even_split_does_not(self): from sds.stats import mcnemar _, p = mcnemar(20, 20) self.assertGreater(p, 0.05) def test_pairing_beats_the_unpaired_interval_on_the_same_games(self): """The measurement this was written for. r6 against r7: 66-45 and 54-56 unpaired gives an interval spanning zero from -23.0% to +2.7%. Paired, the same games are 29 against 17 discordant, p=0.105 - still not significant, and several times tighter. """ from sds.stats import mcnemar _, p = mcnemar(29, 17) self.assertLess(p, 0.2) self.assertGreater(p, 0.05) class TestPairedSampleSize(unittest.TestCase): """How many pairs a paired comparison needs.""" def test_a_bigger_effect_needs_fewer_pairs(self): from sds.stats import paired_games_needed self.assertLess(paired_games_needed(0.70), paired_games_needed(0.63)) self.assertLess(paired_games_needed(0.63), paired_games_needed(0.55)) def test_no_effect_is_not_a_sample_size_problem(self): """A split at 50% says the two are the same; more games will not help.""" from sds.stats import paired_games_needed self.assertEqual(paired_games_needed(0.5), 0) self.assertEqual(paired_games_needed(0.3), 0) def test_the_observed_effect_was_nearly_within_reach(self): """r6 against r7 was 29-17 discordant of 108 decided pairs. The run needed about 133 and had 108, which is the fact that makes a slightly longer run worth spending rather than a much longer one. """ from sds.stats import paired_games_needed needed = paired_games_needed(29 / 46, discordant_rate=46 / 108) self.assertGreater(needed, 108) self.assertLess(needed, 160) def test_a_rarely_discordant_pair_of_bots_needs_more_games(self): from sds.stats import paired_games_needed self.assertGreater( paired_games_needed(0.63, discordant_rate=0.10), paired_games_needed(0.63, discordant_rate=0.50), )