Something went wrong. Try again.
bayes for days
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289"""The rollup says what the run said, and says when it could not read it."""
from __future__ import annotations
import jsonimport tempfileimport unittestfrom pathlib import Path
from sds.runstats import render, rollup
RESULT = { "outcome": "victory", "players": [ { "name": "sds North", "kind": "sds", "sds": { "prone": 10, "pronePinned": 4, "pinnedGyro": 1, "pinnedNoMp": 1, "pinnedLimb": 2, "phases": { "movement": { "decisions": 20, "answered": 18, "defaulted": 1, "illegal": 1, "failed": 0, "eligible": 40, "differed": 8, } }, }, }, # Princess is MegaMek's own bot and counts nothing. It must not become # a seat with every counter at zero, which would read as a bot that # answered nothing. {"name": "princess South", "kind": "princess", "sds": {}}, ],}
# One unit's worth of the stderr the bot writes. Long lines, kept verbatim -# the parser exists to read what the bot actually prints, so a tidied-up# fixture would test the tidying rather than the format.UNIT = ( "[sds-bot] unit 0: 5 states (5 ground, 0 jump), 1 enemy positions " "(0 jumped, 0 unmoved), 2 exchanges, 5 surfaced, 1 range brackets, " "0 weapons not modelled, 100 los asked (90% hit), 0 of 1 enemies out of reach")LOG = "\n".join( [ UNIT, "[sds-bot] unit 0 volley cache: 200 asked, 50% hit (100 missed)", "[sds-bot] unit 0 options cache: 10 asked, 100% hit (0 missed, 5 stands held)", "[sds-bot] unit 0 timing: paths 1us, envelopes 2us, sweep 100us " "(volleys 90us), rank 3us, surface 4us", ])
class TheRollupReadsARun(unittest.TestCase): def setUp(self): self.run = Path(tempfile.mkdtemp()) (self.run / "a.json").write_text(json.dumps(RESULT)) (self.run / "a.log").write_text(LOG)
def test_sidecars_are_not_counted_as_matches(self): (self.run / "a-sds_North.weights.json").write_text(json.dumps({"weights": {}})) (self.run / "manifest.json").write_text(json.dumps({"name": "x"})) self.assertEqual(rollup(self.run).matches, 1)
def test_a_seat_that_counts_nothing_is_not_a_seat(self): out = rollup(self.run) self.assertEqual(sorted(out.seats), ["sds"], "princess keeps no tally")
def test_phase_counters_are_summed(self): movement = rollup(self.run).seats["sds"].phases["movement"] self.assertEqual(movement["decisions"], 20) self.assertEqual(movement["answered"], 18)
def test_the_prone_split_survives(self): counts = rollup(self.run).seats["sds"].seat_counts self.assertEqual(counts["pronePinned"], 4) self.assertEqual(counts["pinnedGyro"] + counts["pinnedNoMp"] + counts["pinnedLimb"], 4)
def test_volley_time_is_not_counted_twice(self): # The log reads "sweep 100us (volleys 90us)": volleys is inside sweep. # Summing all six would make the total 200us and every share wrong. text = render(rollup(self.run)) self.assertIn("of which volleys", text) self.assertIn("0.0s of unit sweeps", text) timing = rollup(self.run).timing_us stages = ("paths", "envelopes", "sweep", "rank", "surface") self.assertEqual(sum(timing[name] for name in stages), 110)
def test_caches_are_summed_from_the_logs(self): out = rollup(self.run) self.assertEqual((out.los.asked, out.los.hit), (100, 90)) self.assertEqual((out.volley.asked, out.volley.hit), (200, 100)) self.assertEqual(out.options.rate, 1.0)
def test_a_run_with_no_logs_says_so_rather_than_reporting_zero(self): (self.run / "a.log").unlink() text = render(rollup(self.run)) self.assertIn("no readable match logs", text) self.assertNotIn("0.0% hit", text)
if __name__ == "__main__": unittest.main()
DECISIONS = [ { "seat": "sds North", "seq": 1, "round": 1, "phase": "MOVEMENT", "actor": 0, "outcome": "answered", "candidates": [{"phi": {"a": 1.0}}], }, { "seat": "sds North", "seq": 2, "round": 1, "phase": "MOVEMENT", "actor": 1, "outcome": "defaulted", }, { "seat": "sds North", "seq": 3, "round": 2, "phase": "MOVEMENT", "actor": 0, "outcome": "defaulted", }, { "seat": "sds North", "seq": 4, "round": 2, "phase": "MOVEMENT", "actor": 1, "outcome": "defaulted", }, { "seat": "sds North", "seq": 5, "round": 9, "phase": "MOVEMENT", "actor": 0, "outcome": "answered", }, { "seat": "sds North", "seq": 6, "round": 1, "phase": "FIRING", "actor": 0, "outcome": "answered", },]
class TheRollupReadsDecisionsByRound(unittest.TestCase): """Where a deadline bites is a shape, and the shape is per round."""
def setUp(self): self.run = Path(tempfile.mkdtemp()) (self.run / "a.json").write_text(json.dumps(RESULT)) (self.run / "a-sds_North.decisions.jsonl").write_text( "\n".join(json.dumps(r) for r in DECISIONS) + "\n" )
def test_defaults_are_counted_against_the_round_they_happened_in(self): rounds = rollup(self.run).turns.by_phase["MOVEMENT"] self.assertEqual(rounds[1]["defaulted"], 1) self.assertEqual(rounds[1]["answered"], 1) self.assertEqual(rounds[2]["defaulted"], 2) self.assertEqual(rounds[9]["answered"], 1) self.assertEqual(rounds[9]["defaulted"], 0)
def test_phases_are_kept_apart(self): turns = rollup(self.run).turns self.assertEqual(turns.by_phase["FIRING"][1]["answered"], 1) self.assertNotIn("defaulted", {k: v for k, v in turns.by_phase["FIRING"][1].items() if v})
def test_the_worst_round_is_marked(self): text = render(rollup(self.run)) self.assertIn("[by round]", text) line = next(row for row in text.splitlines() if "round 2" in row and "<- worst" in row) self.assertIn("100.0% defaulted", line)
def test_a_huge_row_is_read_from_its_head_not_parsed(self): """A decision row runs to hundreds of kilobytes of candidate vectors.
The fast path reads round, phase and outcome off the first 512 bytes. This pins that it agrees with a full parse on a row far too big to be covered by the small fixtures above. """ big = dict(DECISIONS[1]) big["candidates"] = [{"phi": {f"f{i}": float(i)}} for i in range(20000)] path = self.run / "b-sds_North.decisions.jsonl" path.write_text(json.dumps(big) + "\n") self.assertGreater(path.stat().st_size, 400_000) rounds = rollup(self.run).turns.by_phase["MOVEMENT"] self.assertEqual(rounds[1]["defaulted"], 2)
TIMED = { "outcome": "victory", "players": [ { "name": "sds North", "kind": "sds", "turns": { "count": 3, "totalMillis": 600, "maxMillis": 400, "millis": [50, 150, 400], "byRound": {"1": {"millis": 200, "turns": 2}, "2": {"millis": 400, "turns": 1}}, }, "sds": {"phases": {"movement": {"decisions": 3, "answered": 3}}}, }, # Princess keeps no `sds` tally at all. If the rollup only looked at # seats that do, the comparison this exists for would be impossible. { "name": "princess South", "kind": "princess", "sds": {}, "turns": { "count": 3, "totalMillis": 3000, "maxMillis": 2000, "millis": [400, 600, 2000], "byRound": {"1": {"millis": 1000, "turns": 2}, "2": {"millis": 2000, "turns": 1}}, }, }, ],}
class ThinkingTimeIsComparedAcrossSides(unittest.TestCase): def setUp(self): self.run = Path(tempfile.mkdtemp()) (self.run / "a.json").write_text(json.dumps(TIMED))
def test_princess_is_measured_even_though_it_keeps_no_tally(self): seats = rollup(self.run).seats self.assertEqual(seats["princess"].latency.turns, 3) self.assertEqual(seats["princess"].latency.total_ms, 3000)
def test_percentiles_are_over_the_run_not_averaged_per_match(self): # Two matches of the same shape must not change p50; averaging each # match's p50 would be an average of percentiles, which is not one. (self.run / "b.json").write_text(json.dumps(TIMED)) lat = rollup(self.run).seats["sds"].latency self.assertEqual(len(lat.samples), 6) self.assertEqual(sorted(lat.samples), [50, 50, 150, 150, 400, 400])
def test_the_faster_side_is_named_with_a_factor(self): text = render(rollup(self.run)) self.assertIn("[thinking time]", text) self.assertIn("sds is 5.00x faster per turn than princess", text)
def test_the_shape_by_round_survives(self): by_round = rollup(self.run).seats["sds"].latency.by_round # Millis and turns, not millis alone: a round's entry is a total and the # count it is a total of. self.assertEqual(by_round[1], [200, 2]) self.assertEqual(by_round[2], [400, 1]) self.assertIn("by round, mean ms per turn", render(rollup(self.run)))
def test_by_round_is_per_turn_and_does_not_grow_with_the_run(self): # Round 1 is 200ms over 2 turns in every match, so the column reads 100 # however many matches there are. Summing the totals and dividing by the # number of matches - which is what this did - reads 200 for one match # and 200 for ten, and is a per-round total wearing a per-turn label. one = rollup(self.run).seats["sds"].latency.by_round self.assertEqual(one[1][0] / one[1][1], 100) (self.run / "b.json").write_text(json.dumps(TIMED)) (self.run / "c.json").write_text(json.dumps(TIMED)) three = rollup(self.run).seats["sds"].latency.by_round self.assertEqual(three[1][0] / three[1][1], 100) self.assertIn(" 100", render(rollup(self.run)))