From ff24072a0dcc4bd7dfec07bde9275e8327bdb7db Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Sun, 9 Aug 2026 11:27:24 -0400 Subject: [PATCH] test(perf): add render benchmarks that scale units on the board Every render measurement so far has used a 2-unit training scenario, and a 2-unit scenario says a match performs well. It does not scale like one. Three scenarios holding everything still except unit count and board area - 2v2 on two map sheets, 4v4 and 4x4 (16 units) on four - plus scripts/perf/ stats.py, which summarises Suramadu's per-frame stats out of the container log with percentiles rather than medians. Percentiles are the point. Across the three, total bytes for the same scripted sweep move 1%, and the p90 frame goes 20KB -> 197KB -> 299KB. A median hides that completely. --- Dockerfile | 8 ++++ megamek/scenarios/bench-2v2.mms | 19 +++++++++ megamek/scenarios/bench-4v4.mms | 23 +++++++++++ megamek/scenarios/bench-4x4.mms | 33 ++++++++++++++++ scripts/perf/stats.py | 68 +++++++++++++++++++++++++++++++++ 5 files changed, 151 insertions(+) create mode 100644 megamek/scenarios/bench-2v2.mms create mode 100644 megamek/scenarios/bench-4v4.mms create mode 100644 megamek/scenarios/bench-4x4.mms create mode 100755 scripts/perf/stats.py diff --git a/Dockerfile b/Dockerfile index c419944..8f22982 100644 --- a/Dockerfile +++ b/Dockerfile @@ -188,6 +188,14 @@ RUN /src/megamek/skin/install-skin.sh # that are still there and still pulled. See megamek/prune-data.sh - it is one # documented line per path, and it fails the build if a MegaMek bump moves one. # It runs after install-skin.sh so the skin's reference check sees the full tree. +# The render benchmarks. Not scenarios anyone plays: they hold everything still +# except unit count and board area, so two measurement runs differ only in the +# thing being measured. Installed under their own directory so a MegaMek bump +# cannot collide with them, and so `ls data/scenarios` still reads as upstream's +# library plus ours. +RUN mkdir -p "$MM_HOME/data/scenarios/lance-blue" \ + && cp /src/megamek/scenarios/*.mms "$MM_HOME/data/scenarios/lance-blue/" + RUN /src/megamek/prune-data.sh # The source tree and the platform launchers are build-time only. diff --git a/megamek/scenarios/bench-2v2.mms b/megamek/scenarios/bench-2v2.mms new file mode 100644 index 0000000..8c38b3b --- /dev/null +++ b/megamek/scenarios/bench-2v2.mms @@ -0,0 +1,19 @@ +# A lance.blue render benchmark, not a game to play. Unit counts and board area +# are the variables; everything else is held still so two runs differ only in +# the thing being measured. +# +# MegaMek Data (C) 2025 by The MegaMek Team is licensed under CC BY-NC-SA 4.0. +# https://creativecommons.org/licenses/by-nc-sa/4.0/ +MMSVersion=1 +Name=Bench 2v2 +Description=lance.blue render benchmark: 2 factions, 2 units each, 2 map sheets. +BoardWidth=2 +BoardHeight=1 +Maps=Map Set 2/16x17 BattleTech,Map Set 2/16x17 BattleTech +Factions=BenchA,BenchB +Location_BenchA=S +Location_BenchB=N +Unit_BenchA_1=Wasp WSP-1A,Bench A1,4,0 +Unit_BenchA_2=Griffin GRF-1N,Bench A2,4,0 +Unit_BenchB_1=Stinger STG-3R,Bench B1,4,0 +Unit_BenchB_2=Wolverine WVR-6R,Bench B2,4,0 diff --git a/megamek/scenarios/bench-4v4.mms b/megamek/scenarios/bench-4v4.mms new file mode 100644 index 0000000..1af5b5d --- /dev/null +++ b/megamek/scenarios/bench-4v4.mms @@ -0,0 +1,23 @@ +# A lance.blue render benchmark, not a game to play. Unit counts and board area +# are the variables; everything else is held still so two runs differ only in +# the thing being measured. +# +# MegaMek Data (C) 2025 by The MegaMek Team is licensed under CC BY-NC-SA 4.0. +# https://creativecommons.org/licenses/by-nc-sa/4.0/ +MMSVersion=1 +Name=Bench 4v4 +Description=lance.blue render benchmark: 2 factions, 4 units each, 4 map sheets. +BoardWidth=2 +BoardHeight=2 +Maps=Map Set 2/16x17 BattleTech,Map Set 2/16x17 BattleTech,Map Set 2/16x17 BattleTech,Map Set 2/16x17 BattleTech +Factions=BenchA,BenchB +Location_BenchA=S +Location_BenchB=N +Unit_BenchA_1=Wasp WSP-1A,Bench A1,4,0 +Unit_BenchA_2=Griffin GRF-1N,Bench A2,4,0 +Unit_BenchA_3=Blackjack BJ-1,Bench A3,4,0 +Unit_BenchA_4=Assassin ASN-21,Bench A4,4,0 +Unit_BenchB_1=Stinger STG-3R,Bench B1,4,0 +Unit_BenchB_2=Wolverine WVR-6R,Bench B2,4,0 +Unit_BenchB_3=Catapult CPLT-C1,Bench B3,4,0 +Unit_BenchB_4=Archer ARC-2R,Bench B4,4,0 diff --git a/megamek/scenarios/bench-4x4.mms b/megamek/scenarios/bench-4x4.mms new file mode 100644 index 0000000..93e95ff --- /dev/null +++ b/megamek/scenarios/bench-4x4.mms @@ -0,0 +1,33 @@ +# A lance.blue render benchmark, not a game to play. Unit counts and board area +# are the variables; everything else is held still so two runs differ only in +# the thing being measured. +# +# MegaMek Data (C) 2025 by The MegaMek Team is licensed under CC BY-NC-SA 4.0. +# https://creativecommons.org/licenses/by-nc-sa/4.0/ +MMSVersion=1 +Name=Bench 4x4 +Description=lance.blue render benchmark: 4 factions, 4 units each (16), 4 map sheets. +BoardWidth=2 +BoardHeight=2 +Maps=Map Set 2/16x17 BattleTech,Map Set 2/16x17 BattleTech,Map Set 2/16x17 BattleTech,Map Set 2/16x17 BattleTech +Factions=BenchA,BenchB,BenchC,BenchD +Location_BenchA=S +Location_BenchB=N +Location_BenchC=E +Location_BenchD=W +Unit_BenchA_1=Wasp WSP-1A,Bench A1,4,0 +Unit_BenchA_2=Griffin GRF-1N,Bench A2,4,0 +Unit_BenchA_3=Blackjack BJ-1,Bench A3,4,0 +Unit_BenchA_4=Assassin ASN-21,Bench A4,4,0 +Unit_BenchB_1=Stinger STG-3R,Bench B1,4,0 +Unit_BenchB_2=Wolverine WVR-6R,Bench B2,4,0 +Unit_BenchB_3=Catapult CPLT-C1,Bench B3,4,0 +Unit_BenchB_4=Archer ARC-2R,Bench B4,4,0 +Unit_BenchC_1=Atlas AS7-D,Bench C1,4,0 +Unit_BenchC_2=Awesome AWS-8Q,Bench C2,4,0 +Unit_BenchC_3=Banshee BNC-3E,Bench C3,4,0 +Unit_BenchC_4=BattleMaster BLR-1G,Bench C4,4,0 +Unit_BenchD_1=Axman AXM-1N,Bench D1,4,0 +Unit_BenchD_2=Black Knight BL-6-KNT,Bench D2,4,0 +Unit_BenchD_3=Caesar CES-4S,Bench D3,4,0 +Unit_BenchD_4=Cataphract CTF-3D,Bench D4,4,0 diff --git a/scripts/perf/stats.py b/scripts/perf/stats.py new file mode 100755 index 0000000..7723a60 --- /dev/null +++ b/scripts/perf/stats.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Summarise Suramadu's per-frame stats from a running match's container log. + + scripts/perf/stats.py [HH:MM:SS] [--session NAME] + +Suramadu logs one `stats,,,` line per metric per frame when +allowStatisticsLogging is on, which suramadu.config.template sets. Those lines are +the only per-frame server-side numbers there are, and reading them by eye does not +scale past a few frames. + +The percentiles matter more than the medians here. A board repaint is rare and +huge; a cursor move is constant and small. Anything that reports only a median +will say a 16-unit match performs like a 4-unit one, which is false - see +PERFORMANCE.md, "How a match scales with units on the board". +""" + +import argparse +import collections +import re +import statistics +import subprocess +import sys + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( + "since", + nargs="?", + default="00:00:00", + help="only lines at or after this UTC HH:MM:SS", + ) + ap.add_argument( + "--session", + default="perf", + help="ARENA_SESSION of the container to read (default: perf)", + ) + args = ap.parse_args() + + ids = subprocess.run( + ["docker", "ps", "-q", "-f", f"name=arena-{args.session}"], + capture_output=True, + text=True, + ).stdout.split() + if not ids: + sys.exit(f"no running container named arena-{args.session}") + out = subprocess.run(["docker", "logs", ids[0]], capture_output=True) + log = out.stdout.decode("utf8", "replace") + out.stderr.decode("utf8", "replace") + + vals = collections.defaultdict(list) + for line in log.splitlines(): + m = re.search(r"(\d\d:\d\d:\d\d)\.\d+ stats,[^,]*,([A-Za-z]+),(-?\d+)", line) + if m and m.group(1) >= args.since: + vals[m.group(2)].append(int(m.group(3))) + + if not vals: + sys.exit( + "no stats lines found - is allowStatisticsLogging on, and has anyone connected?" + ) + print(f"{'metric':26s} {'n':>5s} {'median':>10s} {'p90':>10s} {'max':>10s}") + for k in sorted(vals): + v = sorted(vals[k]) + p90 = v[min(len(v) - 1, int(len(v) * 0.9))] + print(f"{k:26s} {len(v):5d} {statistics.median(v):10.0f} {p90:10d} {v[-1]:10d}") + + +if __name__ == "__main__": + main() -- 2.51.2