Something went wrong. Try again.
bayes for days
Something went wrong. Try again.
Python
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403"""The generator feeds every benchmark. A bad scenario is a wasted run."""
import jsonimport tempfileimport unittestfrom pathlib import Path
from sds.scenario import MAPS, SIZES, generate, write_suitefrom sds.units import ERAS, NoDatabase, pool
MM_HOME = Path.home() / ".cache" / "mul-build" / "megamek"
class TestGenerate(unittest.TestCase): def setUp(self): try: pool("succession-wars") except NoDatabase as error: self.skipTest(str(error))
def test_sides_are_mirrored(self): """The property the whole benchmark rests on.
If the two sides ever differ, a win rate stops measuring the bots and starts measuring the forces, and nothing in the output would say so. """ for size in SIZES: _, text, _meta = generate(size, 42, "succession-wars") north = [ line.split("=", 1)[1].split(",")[0] for line in text.splitlines() if line.startswith("Unit_North_") ] south = [ line.split("=", 1)[1].split(",")[0] for line in text.splitlines() if line.startswith("Unit_South_") ] self.assertEqual(north, south, f"{size}v{size} sides differ") self.assertEqual(len(north), size)
def test_same_seed_same_scenario(self): first = generate(4, 7, "clan-invasion") second = generate(4, 7, "clan-invasion") self.assertEqual(first, second)
def test_different_seeds_differ(self): names = {generate(4, seed, "succession-wars")[0] for seed in range(5)} self.assertEqual(len(names), 5)
def test_eight_a_side_gets_two_sheets(self): """Eight machines on one 16x17 start inside weapons range.
That deletes the approach, which is the part of a fight where movement decisions matter most - so a cramped 8v8 would measure gunnery. """ _, text, _meta = generate(8, 1, "succession-wars") self.assertIn("BoardWidth=2", text) _, small, _meta2 = generate(4, 1, "succession-wars") self.assertIn("BoardWidth=1", small)
def test_maps_named_are_maps_that_exist(self): boards = MM_HOME / "data" / "boards" if not boards.is_dir(): self.skipTest(f"no MegaMek boards under {boards}") for name in MAPS: self.assertTrue((boards / f"{name}.board").is_file(), f"missing board {name}")
def test_every_era_can_field_the_largest_fight(self): for era in ERAS: designs = pool(era) self.assertGreaterEqual(len(designs), 8, f"{era} cannot field 8v8")
def test_forces_are_not_all_one_role(self): """A lance of four missile boats is a real force and a poor test.
It also gives the hierarchy's role reasoning nothing to reason about. """ for seed in range(20): _, text, _meta = generate(4, seed, "succession-wars") roles = [ line.split(":", 1)[1].strip() for line in text.splitlines() if line.startswith("# Roles:") ] self.assertTrue(roles and "," in roles[0], f"seed {seed} drew one role")
if __name__ == "__main__": unittest.main()
class TestCommittedSuite(unittest.TestCase): """The suite in the repository is what a benchmark actually runs.
`sds validate` is the real check and needs MegaMek; these are the parts that can be checked without one, so a bad commit fails in CI rather than an hour into a run. """
SUITE = Path(__file__).resolve().parent.parent / "scenarios" / "suite"
def suite_files(self): found = sorted(self.SUITE.glob("*.mms")) self.assertTrue(found, f"no scenarios in {self.SUITE}") return found
def test_every_scenario_has_exactly_two_factions(self): for path in self.suite_files(): with self.subTest(scenario=path.name): line = next( line for line in path.read_text().splitlines() if line.startswith("Factions=") ) self.assertEqual(len(line.split("=", 1)[1].split(",")), 2)
def test_every_scenario_names_a_map(self): for path in self.suite_files(): with self.subTest(scenario=path.name): self.assertTrue(any(x.startswith("Maps=") for x in path.read_text().splitlines()))
def test_sides_field_the_same_machines(self): for path in self.suite_files(): with self.subTest(scenario=path.name): north, south = [], [] for line in path.read_text().splitlines(): if line.startswith("Unit_North_"): north.append(line.split("=", 1)[1].split(",")[0]) elif line.startswith("Unit_South_"): south.append(line.split("=", 1)[1].split(",")[0]) self.assertEqual(sorted(north), sorted(south)) self.assertTrue(north)
class TestBattleValueTiers(unittest.TestCase): """A tier is a budget, and battle value depends on the crew as much as the machine."""
def test_a_tiered_force_lands_near_its_tier(self): for tier in (4000, 6000, 8000, 10000): _, _, meta = generate(4, 1234 + tier, "jihad", tier) self.assertEqual(meta["target_bv"], tier) # Within a percent: the miss has to be small enough that two # scenarios in a tier are comparable fights. self.assertLess( abs(meta["bv_miss"]), tier * 0.01, f"tier {tier} missed by {meta['bv_miss']}" )
def test_the_tier_is_the_sum_of_the_crewed_ratings(self): """Not the sum of the designs' own ratings, which ignore the crew.""" _, _, meta = generate(4, 77, "clan-invasion", 8000) self.assertEqual(meta["bv"], sum(u["bv"] for u in meta["units"])) base = sum(u["bv_base"] for u in meta["units"]) self.assertNotEqual( base, meta["bv"], "no crew was away from regular, so nothing was priced" )
def test_crews_vary_when_a_tier_is_asked_for(self): seen = set() for seed in range(12): _, _, meta = generate(4, seed, "succession-wars", 6000) seen.update((u["gunnery"], u["piloting"]) for u in meta["units"]) self.assertGreater(len(seen), 8, f"only {len(seen)} distinct crews across 48 slots")
def test_without_a_tier_every_crew_is_regular(self): """The older behaviour, kept: skill is a knob, not a default.""" _, _, meta = generate(4, 3, "succession-wars") self.assertEqual({(u["gunnery"], u["piloting"]) for u in meta["units"]}, {(4, 5)}) self.assertIsNone(meta["target_bv"])
def test_both_sides_get_the_same_machines_and_the_same_crews(self): """Mirrored is what makes a win rate away from 50% the bots.""" _, text, _ = generate(4, 9, "jihad", 7000) north = [ line.split("=", 1)[1] for line in text.splitlines() if line.startswith("Unit_North") ] south = [ line.split("=", 1)[1] for line in text.splitlines() if line.startswith("Unit_South") ]
def crewed(rows): return [(r.split(",")[0], r.split(",")[2], r.split(",")[3]) for r in rows]
self.assertEqual(crewed(north), crewed(south))
def test_the_metadata_carries_what_a_correlation_needs(self): _, _, meta = generate(4, 11, "dark-age", 9000) self.assertEqual(len(meta["units"]), 4) for unit in meta["units"]: for field in ("mass", "role", "total_armor", "weapon_count", "walk_mp", "jump_mp"): self.assertIn(field, unit)
def test_forces_within_a_tier_are_not_all_the_same_shape(self): """Equal shares would put every machine at a quarter of the tier.""" spreads = [] for seed in range(10): _, _, meta = generate(4, seed, "clan-invasion", 8000) masses = [u["mass"] for u in meta["units"]] spreads.append(max(masses) - min(masses)) self.assertGreater(max(spreads), 40, "no scenario mixed light and heavy machines")
class TestCrewRules(unittest.TestCase): """The three rules a force organiser would recognise."""
def test_no_crew_is_more_than_two_ranks_apart(self): from sds.scenario import CREWS
for gunnery, piloting in CREWS: self.assertLessEqual(abs(gunnery - piloting), 2, f"{gunnery}/{piloting}") for seed in range(15): _, _, meta = generate(4, seed, "jihad", 8000) for unit in meta["units"]: self.assertLessEqual( abs(unit["gunnery"] - unit["piloting"]), 2, f"{unit['name']} in {meta['scenario']}", )
def test_succession_wars_does_not_get_the_top_tier(self): from sds.scenario import tiers_for
asked = (4000, 6000, 8000, 10000) self.assertEqual(tiers_for("succession-wars", asked), [4000, 6000, 8000]) self.assertEqual(tiers_for("jihad", asked), list(asked))
def test_a_suite_writes_no_scenario_an_era_cannot_field(self): with tempfile.TemporaryDirectory() as directory: write_suite( Path(directory), per_size=1, seed=3, eras=("succession-wars", "jihad"), sizes=(4,), bv_tiers=(6000, 10000), ) names = [p.name for p in Path(directory).glob("*.mms")] self.assertFalse( [n for n in names if n.startswith("4v4-succession-wars-bv10000")], f"succession-wars was given the 10000 tier: {names}", ) self.assertTrue([n for n in names if n.startswith("4v4-jihad-bv10000")])
def test_a_higher_tier_fields_heavier_forces_on_average(self): """On average, and only on average.
Machines are drawn first and crewed onto the tier afterwards, so the windows a tier will accept overlap: one lance can serve 4000 with a green crew or 8000 with a veteran one. That is how a force organiser balances, and it means the relation between tier and tonnage is a tendency rather than a rule. Asserting it per seed fails honestly - it did, at 285 tons against 275. """
def mean_tons(tier): forces = [generate(4, seed, "succession-wars", tier)[2] for seed in range(12)] return sum(u["mass"] for f in forces for u in f["units"]) / len(forces)
self.assertLess(mean_tons(4000), mean_tons(8000))
class TestDesignPool(unittest.TestCase): """Breadth against resolution.
The full pool answers "does the bot generalise". It cannot answer "which machines does it fly badly", because a thousand designs across a few hundred scenarios gives every design one or two matches. A narrow pool trades the first question for the second. """
def test_the_sample_is_the_same_every_time(self): from sds.scenario import design_pool
first = [d.name for d in design_pool("jihad", 30, 5)] second = [d.name for d in design_pool("jihad", 30, 5)] self.assertEqual(first, second) self.assertEqual(len(first), 30)
def test_a_different_seed_samples_different_machines(self): from sds.scenario import design_pool
one = {d.name for d in design_pool("jihad", 30, 5)} two = {d.name for d in design_pool("jihad", 30, 6)} self.assertNotEqual(one, two)
def test_asking_for_more_than_exists_keeps_everything(self): from sds.scenario import design_pool, pool
everything = design_pool("succession-wars", 10**6, 1) self.assertEqual(len(everything), len(pool("succession-wars")))
def test_a_pooled_suite_repeats_its_machines(self): with tempfile.TemporaryDirectory() as directory: write_suite( Path(directory), per_size=20, seed=3, eras=("jihad",), sizes=(4,), bv_tiers=(6000,), designs_per_era=25, ) names = [] for path in Path(directory).glob("*.meta.json"): names += [u["name"] for u in json.loads(path.read_text())["units"]] self.assertLessEqual(len(set(names)), 25) self.assertGreater(len(names) / len(set(names)), 2.0, "machines barely repeated")
class TestMapPool(unittest.TestCase): """The boards a suite draws from."""
def test_the_pool_is_wide(self): """Eleven boards let a bot be good at one kind of ground.""" self.assertGreater(len(MAPS), 60)
def test_no_board_the_bots_cannot_cross_is_in_it(self): """These four are 63-66% water banded across the middle.
`sds validate` refuses them, and a suite that contains one spends a game on a round limit at 100% BV a side. """ for board in ( "Map Set 4/16x17 River Delta 1", "Map Set 4/16x17 River Delta 2", "Map Set 7/16x17 Archipelago 1", "Map Set 7/16x17 Archipelago 2", ): self.assertNotIn(board, MAPS)
def test_every_board_is_sixteen_by_seventeen(self): """A suite mixing board sizes would vary the approach without saying so.""" for board in MAPS: self.assertIn("16x17", board)
def test_a_suite_spreads_over_many_boards(self): seen = set() for seed in range(60): _, _, meta = generate(4, seed, "jihad", 6000) seen.add(meta["map"]) self.assertGreater(len(seen), 25, f"only {len(seen)} boards in 60 scenarios")
class TestAsymmetricForces(unittest.TestCase): """Two sides drawn independently, with the battle value still matched.
Every scenario this generator has ever written gave both sides the same machines with the same pilots. That is what makes a win rate mean the bot rather than the force - and it also means no fit has ever seen a row where one side outranged or outran the other, so a matchup is not something the weights could have learned. There was nothing to learn from. """
def rosters(self, text: str) -> tuple[list[str], list[str]]: north, south = [], [] for line in text.splitlines(): if line.startswith("Unit_North_"): north.append(line.split("=", 1)[1]) elif line.startswith("Unit_South_"): south.append(line.split("=", 1)[1]) return north, south
def test_mirrored_is_still_the_default(self): # Every stored suite and every recorded baseline depends on it. _, text, meta = generate(4, 4242, "jihad", 8000) north, south = self.rosters(text) self.assertEqual([u.split(",")[0] for u in north], [u.split(",")[0] for u in south]) self.assertTrue(meta["mirrored"]) self.assertEqual(meta["gap"]["max_range"], 0.0)
def test_asymmetric_draws_two_different_forces(self): _, text, meta = generate(4, 4242, "jihad", 8000, mirrored=False) north, south = self.rosters(text) self.assertNotEqual(north, south) self.assertFalse(meta["mirrored"])
def test_the_battle_value_still_matches(self): # The whole case for not loosening the target: crew skill moves a force # by up to 2.29x, so two random draws both land on the tier. Measured # over 200 pairs an era the sides come out a median 0.2% apart. for seed in range(4243, 4253): _, _, meta = generate(4, seed, "jihad", 8000, mirrored=False) gap = abs(meta["forces"]["North"]["bv"] - meta["forces"]["South"]["bv"]) self.assertLess(gap / 8000, 0.02, f"seed {seed} is {gap} BV apart")
def test_the_composition_actually_varies(self): # A suite where the sides differ only on paper would teach a matchup # feature nothing, which is the failure this exists to avoid. gaps = [] for seed in range(4300, 4330): _, _, meta = generate(4, seed, "jihad", 8000, mirrored=False) gaps.append(abs(meta["gap"]["max_range"])) self.assertGreater(max(gaps), 3.0, "no draw separated the sides by 3 hexes of range")
def test_the_flat_keys_survive_for_old_readers(self): # `correlate` and every stored corpus read these. They describe North. _, _, meta = generate(4, 4242, "jihad", 8000, mirrored=False) for key in ("bv", "bv_miss", "mean_gunnery", "mean_piloting", "units"): self.assertIn(key, meta) self.assertEqual(meta["bv"], meta["forces"]["North"]["bv"])
def test_the_name_says_which_it_is(self): name, _, _ = generate(4, 4242, "jihad", 8000, mirrored=False) self.assertTrue(name.endswith("-vs"), name)