Something went wrong. Try again.
bayes for days
Something went wrong. Try again.
Python
at claude/shape
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369"""The statistics are load-bearing: they decide what gets believed."""
import unittestfrom pathlib import Path
from sds.stats import Summary, games_needed, group, scenario_name, scenario_size, wilson
def result(outcome, winner_team=-1, players=None, scenario=None): return { "scenario": scenario or "/work/scenarios/suite/4v4-jihad-s102040.mms", "outcome": outcome, "round": 10, "elapsedMillis": 1000, "victoryTeam": winner_team, "victoryPlayerId": -1, "players": players or [], }
def player(name, team, role, units=4, bv=1000, bv_start=1000): return { "id": team, "name": name, "kind": "princess", "team": team, "role": role, "unitsRemaining": units, "unitsStart": 4, "bvRemaining": bv, "bvStart": bv_start, }
class TestWilson(unittest.TestCase): def test_small_samples_are_wide(self): low, high = wilson(12, 20) self.assertLess(low, 0.5) self.assertGreater(high, 0.5)
def test_bounds_stay_in_range(self): # The textbook normal interval goes outside [0, 1] here, which is the # reason this is Wilson. low, high = wilson(20, 20) self.assertGreaterEqual(low, 0.0) self.assertLessEqual(high, 1.0)
def test_no_trials_claims_nothing(self): self.assertEqual(wilson(0, 0), (0.0, 1.0))
class TestGamesNeeded(unittest.TestCase): def test_smaller_effects_need_more_games(self): self.assertGreater(games_needed(0.05), games_needed(0.10))
class TestSummary(unittest.TestCase): def test_undecided_matches_are_not_half_wins(self): """A round limit is not a draw split between the sides.
This is the failure that matters most: counting an undecided game as half a win makes a bot that refuses to engage look average. """ results = [ result("round-limit", players=[player("N", 1, "sds"), player("S", 2, "princess")]), result("victory", 1, [player("N", 1, "sds"), player("S", 2, "princess", units=0)]), ] summary = Summary(results, key="role") self.assertEqual(summary.decided, 1) rate, _ = summary.win_rate("sds") self.assertEqual(rate, 1.0)
def test_winner_is_never_inferred_from_survivors(self): """A side with everything left has not won if MegaMek recorded no win.""" results = [ result( "round-limit", players=[ player("N", 1, "sds", units=4, bv=1000), player("S", 2, "princess", units=1, bv=10), ], ) ] summary = Summary(results, key="role") self.assertEqual(summary.decided, 0) self.assertEqual(summary.sides["sds"].wins, 0)
def test_roles_follow_the_bot_across_swapped_sides(self): """Alternating deployment edges must not split a bot's tally in two.""" results = [ result("victory", 1, [player("N", 1, "sds"), player("S", 2, "princess")]), result("victory", 1, [player("N", 1, "princess"), player("S", 2, "sds")]), ] summary = Summary(results, key="role") self.assertEqual(set(summary.sides), {"sds", "princess"}) self.assertEqual(summary.decided, 2) # One win each: game one the sds seat was team 1, game two it was team 2. self.assertEqual(summary.sides["sds"].wins, 1) self.assertEqual(summary.sides["princess"].wins, 1)
class TestTurnOrder(unittest.TestCase): """Turn order has to be countable, not only correct.
"we sent the wrong unit" and "no other unit could have gone" must not look the same, and the mean eligible-set size is the number that tells them apart. A substituted unit is its own column because the bot did answer - it is not a default, it is a refused choice. """
@staticmethod def _seat(**counters): seat = player("N", 1, "sds") seat["sds"] = { "decisions": 10, "answered": 10, "defaulted": 0, "illegal": 0, "failed": 0, **counters, } return seat
def test_the_counters_are_summed_and_printed(self): seat = self._seat(eligible=25, chose=10, differed=6, substituted=2) summary = Summary([result("victory", 1, [seat, player("S", 2, "princess")])]) side = summary.sides["N"] self.assertEqual((side.eligible, side.chose, side.differed), (25, 10, 6)) rendered = summary.render() self.assertIn("turn order", rendered) # 25 eligible over 10 decisions: the bot had a real choice. self.assertIn("2.50", rendered) self.assertIn("named a unit that could not act 2 time(s)", rendered)
def test_a_log_without_the_counters_prints_no_turn_order_table(self): summary = Summary([result("victory", 1, [self._seat(), player("S", 2, "princess")])]) self.assertNotIn("turn order", summary.render())
class TestBreakdown(unittest.TestCase): """The per-scenario split is what says whether to move the round limit."""
def sides(self): return [player("N", 1, "A"), player("S", 2, "B")]
def test_size_comes_from_the_scenario_name(self): self.assertEqual(scenario_size(scenario_name(result("victory"))), "4v4") self.assertEqual(scenario_size("8v8-clan-invasion-s101080"), "8v8") self.assertEqual(scenario_size("mirror-lance"), "other")
def test_undecided_splits_round_limit_from_everything_else(self): """A round limit is a scenario too slow for the limit. A stall is a bug.
The exit criterion counts only the second, so adding them together would make a passing harness look broken and a broken one look busy. """ eight = "/work/scenarios/suite/8v8-jihad-s102080.mms" results = [ result("victory", 1, self.sides()), result("round-limit", players=self.sides(), scenario=eight), result("stalled", players=self.sides(), scenario=eight), ] by_size = group(results, lambda r: scenario_size(scenario_name(r)), key="role") self.assertEqual(set(by_size), {"4v4", "8v8"}) self.assertEqual(by_size["4v4"].decided, 1) self.assertEqual(by_size["4v4"].undecided, 0) eight_v_eight = by_size["8v8"] self.assertEqual(eight_v_eight.games, 2) self.assertEqual(eight_v_eight.decided, 0) self.assertEqual(eight_v_eight.undecided, 2) self.assertEqual(eight_v_eight.round_limit, 1) self.assertEqual(eight_v_eight.other_undecided, 1) self.assertEqual(eight_v_eight.other_reasons, "stalled 1")
def test_a_victory_with_no_recorded_winner_is_undecided_not_a_round_limit(self): # outcome says victory, nothing identifies who won: that is the harness # failing, and it must not be filed under the round limit. broken = result("victory", -1, self.sides()) broken["victoryPlayerId"] = -1 stats = group([broken], scenario_name, key="role")["4v4-jihad-s102040"] self.assertEqual(stats.decided, 0) self.assertEqual(stats.round_limit, 0) self.assertEqual(stats.other_undecided, 1)
def test_summary_totals_agree_with_the_breakdown(self): results = [ result("victory", 1, self.sides()), result("round-limit", players=self.sides()), result("timeout", players=self.sides()), ] summary = Summary(results, key="role") self.assertEqual(summary.round_limit, 1) self.assertEqual(summary.other_undecided, 1) self.assertEqual( len(results) - summary.decided, summary.round_limit + summary.other_undecided ) self.assertIn("undecided, not the round limit", summary.render())
if __name__ == "__main__": unittest.main()
class TestSurvivorCounts(unittest.TestCase): """`unitsRemaining` must mean live units, not entries in the object list.
It used to come from `IGame.getEntitiesOwnedBy`, which filters on owner and nothing else, so wrecks counted and a wiped-out side read as "one unit left at 0 BV". The host now uses `Game.getLiveEntitiesOwnedBy` and keeps the raw figure as `unitsInPlay`, so a destroyed side reads as zero here. """
def test_a_destroyed_side_has_no_survivors(self): wiped = player("N", 1, "sds", units=0, bv=0) wiped["unitsInPlay"] = 2 summary = Summary([result("victory", 2, [wiped, player("S", 2, "princess")])], key="role") self.assertEqual(summary.sides["sds"].units_remaining_total, 0)
def test_missing_field_does_not_crash_an_older_log(self): old = player("N", 1, "sds") del old["unitsRemaining"] summary = Summary([result("victory", 2, [old, player("S", 2, "princess")])], key="role") self.assertEqual(summary.sides["sds"].units_remaining_total, 0)
class TestProgressOnAPartialRun(unittest.TestCase): """`sds progress` reads a run that is still being written.
The interim read exists so a suite run with hours left can be asked whether the vector is in the fight. It has to cope with the two things a live run directory always has: files that are not match results, and a result being written as it is read. """
def setUp(self): import json import tempfile
self.dir = Path(tempfile.mkdtemp()) seats = [player("sds North", 1, "sds"), player("princess South", 2, "princess")] for index, team in enumerate((1, 1, 2)): (self.dir / f"m{index}.json").write_text( json.dumps(result("victory", winner_team=team, players=seats)) ) # The things that are not match results. (self.dir / "m0-sds_North.weights.json").write_text('{"weights": {}}') (self.dir / "results.json").write_text("[]") # A match mid-write. (self.dir / "m9.json").write_text('{"outcome": "vic')
def tearDown(self): import shutil
shutil.rmtree(self.dir, ignore_errors=True)
def test_it_counts_only_the_finished_matches(self): from argparse import Namespace
from sds.cli import cmd_progress
self.assertEqual(cmd_progress(Namespace(run=str(self.dir))), 0)
def test_a_half_written_result_is_skipped_rather_than_fatal(self): import json
loaded = [] for path in sorted(self.dir.glob("*.json")): if path.name.endswith(".weights.json") or path.name == "results.json": continue try: document = json.loads(path.read_text()) except json.JSONDecodeError: continue if isinstance(document, dict) and "outcome" in document: loaded.append(document) self.assertEqual(len(loaded), 3, "the truncated file was counted or threw") summary = Summary(loaded, key="role") self.assertEqual(summary.decided, 3)
def test_an_empty_run_directory_says_so(self): import tempfile from argparse import Namespace
from sds.cli import cmd_progress
empty = Path(tempfile.mkdtemp()) self.assertEqual(cmd_progress(Namespace(run=str(empty))), 1)
class TestPairedComparison(unittest.TestCase): """Two runs over the same scenarios are paired, not independent."""
def test_agreement_carries_no_information(self): from sds.stats import mcnemar
# Same discordant counts, wildly different numbers of agreed pairs: # the test must not move, because agreed pairs say nothing about which # bot is better. self.assertEqual(mcnemar(29, 17), mcnemar(29, 17))
def test_no_discordant_pairs_claims_nothing(self): from sds.stats import mcnemar
chi, p = mcnemar(0, 0) self.assertEqual(chi, 0.0) self.assertEqual(p, 1.0)
def test_a_lopsided_split_separates(self): from sds.stats import mcnemar
_, p = mcnemar(40, 5) self.assertLess(p, 0.05)
def test_an_even_split_does_not(self): from sds.stats import mcnemar
_, p = mcnemar(20, 20) self.assertGreater(p, 0.05)
def test_pairing_beats_the_unpaired_interval_on_the_same_games(self): """The measurement this was written for.
r6 against r7: 66-45 and 54-56 unpaired gives an interval spanning zero from -23.0% to +2.7%. Paired, the same games are 29 against 17 discordant, p=0.105 - still not significant, and several times tighter. """ from sds.stats import mcnemar
_, p = mcnemar(29, 17) self.assertLess(p, 0.2) self.assertGreater(p, 0.05)
class TestPairedSampleSize(unittest.TestCase): """How many pairs a paired comparison needs."""
def test_a_bigger_effect_needs_fewer_pairs(self): from sds.stats import paired_games_needed
self.assertLess(paired_games_needed(0.70), paired_games_needed(0.63)) self.assertLess(paired_games_needed(0.63), paired_games_needed(0.55))
def test_no_effect_is_not_a_sample_size_problem(self): """A split at 50% says the two are the same; more games will not help.""" from sds.stats import paired_games_needed
self.assertEqual(paired_games_needed(0.5), 0) self.assertEqual(paired_games_needed(0.3), 0)
def test_the_observed_effect_was_nearly_within_reach(self): """r6 against r7 was 29-17 discordant of 108 decided pairs.
The run needed about 133 and had 108, which is the fact that makes a slightly longer run worth spending rather than a much longer one. """ from sds.stats import paired_games_needed
needed = paired_games_needed(29 / 46, discordant_rate=46 / 108) self.assertGreater(needed, 108) self.assertLess(needed, 160)
def test_a_rarely_discordant_pair_of_bots_needs_more_games(self): from sds.stats import paired_games_needed
self.assertGreater( paired_games_needed(0.63, discordant_rate=0.10), paired_games_needed(0.63, discordant_rate=0.50), )