Something went wrong. Try again.
bayes for days
Something went wrong. Try again.
Python
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223"""The comparison decides what gets merged. It has to be hard to fool."""
import unittest
from sds.baseline import Baseline, Incomparable, compare, difference_intervalfrom sds.epoch import EPOCH, describe
def baseline(name="main", wins=10, decided=20, **overrides): fields = dict( name=name, bot="sds-bot", scenario="mirror-lance.mms", games=decided, decided=decided, wins=wins, outcomes={"victory": decided}, bv_left=0.3, bv_left_opponent=0.3, bv_destroyed_percent=70, max_rounds=40, opponent="princess", commit="abc1234", recorded="2026-08-19T00:00:00Z", ) fields.update(overrides) return Baseline(**fields)
class TestDifferenceInterval(unittest.TestCase): def test_identical_runs_straddle_zero(self): low, high = difference_interval(10, 20, 10, 20) self.assertLess(low, 0) self.assertGreater(high, 0)
def test_a_large_clear_difference_excludes_zero(self): low, _ = difference_interval(95, 100, 20, 100) self.assertGreater(low, 0)
def test_stays_within_minus_one_to_one(self): low, high = difference_interval(100, 100, 0, 100) self.assertGreaterEqual(low, -1.0) self.assertLessEqual(high, 1.0)
def test_no_decided_games_claims_nothing(self): self.assertEqual(difference_interval(0, 0, 3, 10), (-1.0, 1.0))
class TestCompare(unittest.TestCase): def test_refuses_a_baseline_measured_under_other_rules(self): """The failure this is built against: same shape, different meaning.
A baseline recorded with a different victory condition produces numbers that look comparable and are not, and nothing in a diff would show it. """ candidate = baseline(name="candidate", bv_destroyed_percent=50) with self.assertRaises(Incomparable): compare(candidate, baseline())
def test_refuses_a_different_scenario(self): candidate = baseline(name="candidate", scenario="other.mms") with self.assertRaises(Incomparable): compare(candidate, baseline())
def test_a_small_improvement_is_reported_as_no_evidence(self): """12-8 against 10-10 is the trap. It must not read as progress.""" report = compare(baseline(name="candidate", wins=12), baseline(wins=10)) self.assertIn("does not show a difference", report)
def test_a_real_improvement_is_called_one(self): candidate = baseline(name="candidate", wins=95, decided=100, games=100) report = compare(candidate, baseline(wins=20, decided=100, games=100)) self.assertIn("is an improvement", report)
def test_a_real_regression_is_called_one(self): candidate = baseline(name="candidate", wins=20, decided=100, games=100) report = compare(candidate, baseline(wins=95, decided=100, games=100)) self.assertIn("is a regression", report)
def test_a_mostly_undecided_run_is_flagged_as_weak(self): candidate = baseline(name="candidate", wins=6, decided=10, games=40) report = compare(candidate, baseline(wins=10, decided=10, games=40)) self.assertIn("Treat the comparison as weak", report)
if __name__ == "__main__": unittest.main()
class TestEpoch(unittest.TestCase): """An epoch is the catch-all for rule changes no field names.
The ejected-pilots rule changed every result in the suite and no baseline field mentioned it. Enumerating settings cannot catch that class of change; a version number bumped by hand can. """
def test_different_epochs_are_incomparable(self): old = baseline(name="old", epoch=1) new = baseline(name="new", epoch=2) with self.assertRaises(Incomparable) as caught: compare(new, old) self.assertIn("epoch", str(caught.exception))
def test_the_error_says_what_changed(self): with self.assertRaises(Incomparable) as caught: compare(baseline(name="new", epoch=2), baseline(name="old", epoch=1)) self.assertIn(describe(2), str(caught.exception))
def test_same_epoch_compares(self): compare(baseline(name="new", epoch=2), baseline(name="old", epoch=2))
def test_every_epoch_up_to_current_has_a_note(self): for e in range(1, EPOCH + 1): self.assertNotEqual(describe(e), "unrecorded", f"epoch {e} has no note")
class TestTwoSeatRunRecordsItsWins(unittest.TestCase): """A benchmark alternates seats, so there is no side called "sds".
`from_run` looked one up by that name, got None, and wrote `wins: 0`. Four stored baselines read 0.0% for as long as they had existed, and the comparison machinery that is supposed to refuse misleading comparisons was quietly comparing against zero. """
def summary(self): from sds.stats import Summary
def player(name, team, role, kind, bv_remaining): return { "name": name, "team": team, "role": role, "kind": kind, "bvStart": 1000, "bvRemaining": bv_remaining, "unitsRemaining": 2, }
results = [] # Four matches: the bot wins two, one from each seat. for winner_team, north_kind in ((1, "sds"), (2, "sds"), (1, "princess"), (2, "princess")): south_kind = "princess" if north_kind == "sds" else "sds" results.append( { "scenario": "/work/scenarios/suite/4v4-jihad-s1.mms", "outcome": "victory", "round": 10, "elapsedMillis": 1000, "victoryTeam": winner_team, "players": [ player("North", 1, f"{north_kind} North", north_kind, 400), player("South", 2, f"{south_kind} South", south_kind, 300), ], } ) return results, Summary(results, key="role")
def test_the_wins_are_summed_across_both_seats(self): from sds.baseline import from_run
results, summary = self.summary() self.assertNotIn("sds", summary.sides, "the fixture has to have no side named 'sds'") record = from_run( name="two-seat", bot="sds-bot", results=results, summary=summary, scenario="suite:labels2-suite:abc", bv_destroyed_percent=70, max_rounds=40, ) expected = sum(s.wins for s in summary.sides.values() if s.kind == "sds") self.assertEqual(record.wins, expected) self.assertGreater(record.wins, 0, "a two-seat run recorded no wins at all") self.assertGreater(record.bv_left, 0.0) self.assertGreater(record.bv_left_opponent, 0.0)
class TestRefusingAZeroWinArtefact(unittest.TestCase): """Comparing against a baseline that never tallied anything.
The module's job is to refuse comparisons that mislead, and comparing a candidate against zero wins is the most misleading one available: every candidate is a triumph. """
def record(self, name, wins, bv, bv_opponent, decided=150): return Baseline( name=name, bot="sds-bot", scenario="suite.mms", games=160, decided=decided, wins=wins, outcomes={"victory": decided}, bv_left=bv, bv_left_opponent=bv_opponent, bv_destroyed_percent=70, max_rounds=40, )
def test_the_artefact_is_refused(self): from sds.baseline import compare, unrecorded
broken = self.record("overnight-4v4", 0, 0.0, 0.0) self.assertTrue(unrecorded(broken)) with self.assertRaises(Incomparable) as caught: compare(self.record("candidate", 70, 0.2, 0.2), broken) self.assertIn("unrecorded", str(caught.exception))
def test_a_bot_that_really_lost_every_game_is_not_refused(self): """It still leaves BV on the board, so it is a result and comparable.""" from sds.baseline import unrecorded
self.assertFalse(unrecorded(self.record("hopeless", 0, 0.05, 0.9)))
def test_a_run_with_nothing_decided_is_not_the_artefact(self): from sds.baseline import unrecorded
self.assertFalse(unrecorded(self.record("empty", 0, 0.0, 0.0, decided=0)))