From a534fd6b0063febcde7ee91137ea0d66d2cee8f4 Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Wed, 19 Aug 2026 13:35:54 -0400 Subject: [PATCH] feat: baselines and experiment comparison bench --save-baseline records an aggregate under baselines/; --against compares this run to one and writes a PR-ready comparison.md. The comparison refuses a baseline measured under different rules, and states in words when a difference is not supported. --- bots/random_bot.py | 4 +- docs/EXPERIMENTS.md | 81 ++++++++++++++ sds/baseline.py | 244 +++++++++++++++++++++++++++++++++++++++++ sds/cli.py | 66 ++++++++++- tests/test_baseline.py | 86 +++++++++++++++ 5 files changed, 477 insertions(+), 4 deletions(-) create mode 100644 docs/EXPERIMENTS.md create mode 100644 sds/baseline.py create mode 100644 tests/test_baseline.py diff --git a/bots/random_bot.py b/bots/random_bot.py index a9073e4..7b251b9 100755 --- a/bots/random_bot.py +++ b/bots/random_bot.py @@ -124,9 +124,7 @@ class Bot: best[target] = shot["toHit"] target = min(best, key=lambda t: best[t]) attacks = [ - {"weapon": s["weapon"], "target": target} - for s in shots - if s["target"] == target + {"weapon": s["weapon"], "target": target} for s in shots if s["target"] == target ] return {"kind": "fire", "attacks": attacks} diff --git a/docs/EXPERIMENTS.md b/docs/EXPERIMENTS.md new file mode 100644 index 0000000..2cfe4bd --- /dev/null +++ b/docs/EXPERIMENTS.md @@ -0,0 +1,81 @@ +# Running an experiment + +One idea, one branch, one comparison in the pull request. + + git checkout -b claude/flank-weighting origin/main + # change the bot + ./scripts/build.sh + ./sds.sh bench --games 60 --bot /work/target/release/sds-bot --against main + +The last command prints a comparison and writes `comparison.md` into the run +directory. That file is the PR body. + +## The baseline + +`baselines/*.json` are committed. They hold an aggregate — games, decided, wins, +BV left, the commit they were measured at — and never the match files, which are +large, derived, and regenerated by any run. + +Record one when `main` moves: + + ./sds.sh bench --games 60 --bot /work/target/release/sds-bot \ + --save-baseline main --notes "after the terrain-cost fix" + +Then commit it. A baseline that lives on one machine cannot be compared against +in a review. + +## What the comparison refuses to do + +**Compare across different rules.** Scenario, victory condition, round limit and +opponent travel with the baseline, and a mismatch is an error rather than a +warning. A baseline recorded at a 50% BV threshold and a run at 70% produce +numbers of the same shape that mean different things, and nothing in a diff +would show it. + +**Read a difference that is not there.** Every comparison carries a 95% interval +on the *difference* in win rate — Newcombe's method, which stays sane at rates +near 0 and 1 where the normal approximation does not — and states the verdict in +a sentence: + +> The interval spans zero, so this run does not show a difference. An effect this +> size would need roughly 380 decided games per side to separate from noise. + +That sentence is the point of the whole apparatus. 12–8 against 10–10 looks like +progress and is nothing, and a table of numbers invites reading the one you hoped +for. + +## How many games + +Rough, for a mirror match: + +| difference you want to see | decided games per side | +|---|---| +| 20 points | ~25 | +| 10 points | ~100 | +| 5 points | ~385 | + +A match is 1–4 minutes, and `--jobs` matches run at once. Sixty games is about +half an hour at `--jobs 3` and is enough to catch a large change; a subtle one +needs an overnight run and there is no way around that. + +Watch the undecided count. Games that hit the round limit are excluded from the +rate, so 60 games with 20 undecided is a 40-game experiment. + +## Before you believe any of it + + ./sds.sh control --games 40 + +Princess against itself must come out near 50/50. If it does not, the harness is +biased and every number it has printed is suspect. Run it after touching the +harness, not after being surprised by a result. + +## What belongs in the PR + +- `comparison.md`, as the body. +- What was changed and why you expected it to help — before the numbers, so the + reader can tell a prediction from a rationalisation. +- The `answered` / `decisions` split if the bot's own counters moved. A change + that improves the win rate by making the bot answer more of its decisions is a + bug fix wearing a strategy's clothes, and worth naming as one. +- Any run that was thrown away, and why. A benchmark you can re-roll is not a + benchmark. diff --git a/sds/baseline.py b/sds/baseline.py new file mode 100644 index 0000000..d3621d3 --- /dev/null +++ b/sds/baseline.py @@ -0,0 +1,244 @@ +"""Baselines, and comparing a candidate against one. + +An experiment is a branch: change the bot, run the bench, compare against the +baseline, put the comparison in the pull request. This module is the compare +step, and most of it exists to refuse comparisons that would mislead. + +The two failure modes it is built against: + +*Comparing across different rules.* A baseline measured with a different +scenario, victory condition or round limit is not a baseline for this run. The +numbers are the same shape and mean different things, and nothing in a diff +would show it. So the settings travel with the baseline and a mismatch is an +error, not a warning. + +*Reading a difference that is not there.* Twelve-to-eight against ten-to-ten +looks like progress and is nothing. Every comparison carries an interval on the +difference, and says in words whether the result supports a conclusion. +""" + +from __future__ import annotations + +import json +import math +import subprocess +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from pathlib import Path + +from .stats import Summary + +BASELINES = Path(__file__).resolve().parent.parent / "baselines" + +# Settings that change what a win rate means. Two runs that disagree on any of +# these are measuring different games and must not be compared. +COMPARABLE = ("scenario", "bv_destroyed_percent", "max_rounds", "opponent") + + +@dataclass +class Baseline: + """What a bot scored, and the exact conditions it scored it under.""" + + name: str + bot: str + scenario: str + games: int + decided: int + wins: int + outcomes: dict[str, int] + bv_left: float + bv_left_opponent: float + bv_destroyed_percent: int + max_rounds: int + opponent: str = "princess" + commit: str = "unknown" + recorded: str = "" + notes: str = "" + seeds: list[int] = field(default_factory=list) + + @property + def rate(self) -> float: + return self.wins / self.decided if self.decided else 0.0 + + def path(self) -> Path: + return BASELINES / f"{self.name}.json" + + def save(self) -> Path: + BASELINES.mkdir(parents=True, exist_ok=True) + path = self.path() + path.write_text(json.dumps(asdict(self), indent=2) + "\n") + return path + + @staticmethod + def load(name: str) -> Baseline: + path = BASELINES / f"{name}.json" + if not path.is_file(): + known = sorted(p.stem for p in BASELINES.glob("*.json")) + raise FileNotFoundError(f"no baseline '{name}'. Known: {', '.join(known) or 'none'}") + return Baseline(**json.loads(path.read_text())) + + +def head_commit() -> str: + try: + out = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + check=False, + cwd=BASELINES.parent, + ) + dirty = subprocess.run( + ["git", "status", "--porcelain"], + capture_output=True, + text=True, + check=False, + cwd=BASELINES.parent, + ) + sha = out.stdout.strip() or "unknown" + # A dirty tree is marked, because a baseline recorded from uncommitted + # code cannot be reproduced and should not be trusted later as if it + # could. + return f"{sha}-dirty" if dirty.stdout.strip() else sha + except Exception: + return "unknown" + + +def from_run( + name: str, + bot: str, + results: list[dict], + summary: Summary, + scenario: str, + bv_destroyed_percent: int, + max_rounds: int, + notes: str = "", +) -> Baseline: + side = summary.sides.get("sds") + opponent = summary.sides.get("princess") + games = len(results) + return Baseline( + name=name, + bot=bot, + scenario=Path(scenario).name, + games=games, + decided=summary.decided, + wins=side.wins if side else 0, + outcomes=dict(summary.outcomes), + bv_left=(side.bv_fraction_total / games) if side and games else 0.0, + bv_left_opponent=(opponent.bv_fraction_total / games) if opponent and games else 0.0, + bv_destroyed_percent=bv_destroyed_percent, + max_rounds=max_rounds, + commit=head_commit(), + recorded=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + notes=notes, + seeds=sorted(r.get("seed", 0) for r in results), + ) + + +def difference_interval( + wins_a: int, n_a: int, wins_b: int, n_b: int, z: float = 1.96 +) -> tuple[float, float]: + """A 95% interval on (rate_a - rate_b), by Newcombe's method. + + Newcombe rather than the normal approximation for the difference: with the + sample sizes a bot benchmark can afford, and rates that can sit near 0 or 1, + the normal interval is wrong in exactly the cases an experiment cares about. + Newcombe builds the difference interval out of each rate's Wilson interval, + which stays sane at the edges. + """ + if n_a == 0 or n_b == 0: + return (-1.0, 1.0) + + def wilson(wins: int, n: int) -> tuple[float, float]: + p = wins / n + denominator = 1 + z * z / n + centre = (p + z * z / (2 * n)) / denominator + spread = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denominator + return (max(0.0, centre - spread), min(1.0, centre + spread)) + + low_a, high_a = wilson(wins_a, n_a) + low_b, high_b = wilson(wins_b, n_b) + p_a, p_b = wins_a / n_a, wins_b / n_b + lower = (p_a - p_b) - math.sqrt((p_a - low_a) ** 2 + (high_b - p_b) ** 2) + upper = (p_a - p_b) + math.sqrt((high_a - p_a) ** 2 + (p_b - low_b) ** 2) + return (max(-1.0, lower), min(1.0, upper)) + + +class Incomparable(RuntimeError): + pass + + +def compare(candidate: Baseline, baseline: Baseline) -> str: + """A report a person can paste into a pull request.""" + mismatched = [ + field_name + for field_name in COMPARABLE + if getattr(candidate, field_name) != getattr(baseline, field_name) + ] + if mismatched: + detail = ", ".join( + f"{f}: {getattr(candidate, f)!r} vs baseline {getattr(baseline, f)!r}" + for f in mismatched + ) + raise Incomparable( + f"this run and baseline '{baseline.name}' were measured under " + f"different rules ({detail}). Re-record the baseline, or run this " + f"under the baseline's settings." + ) + + low, high = difference_interval( + candidate.wins, candidate.decided, baseline.wins, baseline.decided + ) + delta = candidate.rate - baseline.rate + + lines = [ + f"## {candidate.bot} vs {baseline.opponent}", + "", + f"Scenario `{candidate.scenario}`, a side is beaten at " + f"{candidate.bv_destroyed_percent}% BV destroyed, round limit " + f"{candidate.max_rounds}.", + "", + f"| | baseline `{baseline.name}` | this run |", + "|---|---|---|", + f"| commit | `{baseline.commit}` | `{candidate.commit}` |", + f"| games | {baseline.games} | {candidate.games} |", + f"| decided | {baseline.decided} | {candidate.decided} |", + f"| wins | {baseline.wins} | {candidate.wins} |", + f"| win rate | {baseline.rate:.1%} | {candidate.rate:.1%} |", + f"| own BV left | {baseline.bv_left:.1%} | {candidate.bv_left:.1%} |", + f"| opponent BV left | {baseline.bv_left_opponent:.1%} | " + f"{candidate.bv_left_opponent:.1%} |", + "", + f"**Win rate change: {delta:+.1%}**, 95% interval {low:+.1%} to {high:+.1%}.", + "", + ] + + # The verdict in words. A table of numbers invites reading the one you + # hoped for; a sentence that says "this proves nothing" is harder to skim + # past. + if low > 0: + lines.append("The interval is entirely above zero: this is an improvement.") + elif high < 0: + lines.append("The interval is entirely below zero: this is a regression.") + else: + needed = int(math.ceil(4 * 0.25 * (1.96 / max(abs(delta), 0.01)) ** 2)) + lines.append( + "The interval spans zero, so this run does not show a difference. " + f"An effect this size would need roughly {needed} decided games per " + "side to separate from noise." + ) + + undecided = candidate.games - candidate.decided + if undecided: + share = undecided / candidate.games + lines.append("") + lines.append( + f"{undecided} of {candidate.games} games were undecided ({share:.0%}) " + "and are excluded from the rate." + ) + if share > 0.3: + lines.append( + "That is a large share. Treat the comparison as weak until the " + "scenario or the victory condition resolves more games." + ) + return "\n".join(lines) diff --git a/sds/cli.py b/sds/cli.py index ad2506f..c06a543 100644 --- a/sds/cli.py +++ b/sds/cli.py @@ -16,6 +16,7 @@ import sys from datetime import UTC, datetime from pathlib import Path +from .baseline import Baseline, Incomparable, compare, from_run from .match import REPO, MatchError, MatchSpec, Seat, kill_all_matches, kill_stragglers, run from .stats import Summary, games_needed @@ -138,6 +139,16 @@ def _bench(args: argparse.Namespace, label: str, sds_bot: str | None) -> int: summary = Summary(results, key="role") print(summary.render()) + record = from_run( + name=args.save_baseline or "candidate", + bot=sds_bot or "princess", + results=results, + summary=summary, + scenario=str(scenario), + bv_destroyed_percent=args.bv_destroyed_percent, + max_rounds=args.max_rounds, + notes=getattr(args, "notes", "") or "", + ) if failures: print(f"\n{failures} matches failed and are not in the numbers above") # The planning number, always: it is the difference between "we measured @@ -149,6 +160,25 @@ def _bench(args: argparse.Namespace, label: str, sds_bot: str | None) -> int: ) (out_dir / "summary.txt").write_text(summary.render() + "\n") (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") + + if getattr(args, "save_baseline", None): + path = record.save() + print(f"\nrecorded baseline '{record.name}' -> {path}") + print( + "Commit it: a baseline that lives only on one machine cannot be " + "compared against in a review." + ) + + against = getattr(args, "against", None) + if against: + try: + report = compare(record, Baseline.load(against)) + except (Incomparable, FileNotFoundError) as error: + print(f"\ncannot compare: {error}", file=sys.stderr) + return 1 + print("\n" + report) + (out_dir / "comparison.md").write_text(report + "\n") + print(f"\nwritten to {out_dir / 'comparison.md'} - paste it into the PR.") return 0 @@ -161,6 +191,24 @@ def cmd_control(args: argparse.Namespace) -> int: return _bench(args, "control", None) +def cmd_baselines(args: argparse.Namespace) -> int: + from .baseline import BASELINES + + found = sorted(BASELINES.glob("*.json")) + if not found: + print("no baselines recorded. Make one with:") + print(" ./sds.sh bench --games 60 --bot ... --save-baseline main") + return 0 + print(f"{'name':<16} {'bot':<28} {'games':>6} {'decided':>8} {'rate':>7} commit") + for path in found: + b = Baseline.load(path.stem) + print( + f"{b.name:<16} {b.bot[:28]:<28} {b.games:>6} {b.decided:>8} {b.rate:>6.1%} {b.commit}" + ) + _ = args + return 0 + + def cmd_clean(args: argparse.Namespace) -> int: """Kill any match container left running by a harness that died.""" killed = kill_all_matches() @@ -181,7 +229,9 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--max-rounds", type=int, default=40) parser.add_argument("--timeout-ms", type=int, default=10_000) parser.add_argument( - "--bv-destroyed-percent", type=int, default=70, + "--bv-destroyed-percent", + type=int, + default=70, help="a side is beaten at this %% of its BV destroyed; 0 fights to the last unit", ) sub = parser.add_subparsers(dest="command", required=True) @@ -200,6 +250,17 @@ def main(argv: list[str] | None = None) -> int: # and this machine is shared with other agents; the sibling repos have all # learned this the expensive way. Raise it deliberately, having looked. bench.add_argument("--jobs", type=int, default=int(os.environ.get("SDS_JOBS", "2"))) + bench.add_argument( + "--save-baseline", + metavar="NAME", + help="record this run as a baseline under baselines/NAME.json", + ) + bench.add_argument( + "--against", + metavar="NAME", + help="compare this run against a recorded baseline and write comparison.md", + ) + bench.add_argument("--notes", default="", help="one line, stored with a baseline") bench.set_defaults(func=cmd_bench) control = sub.add_parser("control", help="Princess vs Princess: the harness's self-test") @@ -207,6 +268,9 @@ def main(argv: list[str] | None = None) -> int: control.add_argument("--jobs", type=int, default=int(os.environ.get("SDS_JOBS", "2"))) control.set_defaults(func=cmd_control) + baselines = sub.add_parser("baselines", help="list recorded baselines") + baselines.set_defaults(func=cmd_baselines) + clean = sub.add_parser( "clean", help="kill match containers left by a dead harness - NOT while a run is live", diff --git a/tests/test_baseline.py b/tests/test_baseline.py new file mode 100644 index 0000000..901bfc6 --- /dev/null +++ b/tests/test_baseline.py @@ -0,0 +1,86 @@ +"""The comparison decides what gets merged. It has to be hard to fool.""" + +import unittest + +from sds.baseline import Baseline, Incomparable, compare, difference_interval + + +def baseline(name="main", wins=10, decided=20, **overrides): + fields = dict( + name=name, + bot="sds-bot", + scenario="mirror-lance.mms", + games=decided, + decided=decided, + wins=wins, + outcomes={"victory": decided}, + bv_left=0.3, + bv_left_opponent=0.3, + bv_destroyed_percent=70, + max_rounds=40, + opponent="princess", + commit="abc1234", + recorded="2026-08-19T00:00:00Z", + ) + fields.update(overrides) + return Baseline(**fields) + + +class TestDifferenceInterval(unittest.TestCase): + def test_identical_runs_straddle_zero(self): + low, high = difference_interval(10, 20, 10, 20) + self.assertLess(low, 0) + self.assertGreater(high, 0) + + def test_a_large_clear_difference_excludes_zero(self): + low, _ = difference_interval(95, 100, 20, 100) + self.assertGreater(low, 0) + + def test_stays_within_minus_one_to_one(self): + low, high = difference_interval(100, 100, 0, 100) + self.assertGreaterEqual(low, -1.0) + self.assertLessEqual(high, 1.0) + + def test_no_decided_games_claims_nothing(self): + self.assertEqual(difference_interval(0, 0, 3, 10), (-1.0, 1.0)) + + +class TestCompare(unittest.TestCase): + def test_refuses_a_baseline_measured_under_other_rules(self): + """The failure this is built against: same shape, different meaning. + + A baseline recorded with a different victory condition produces numbers + that look comparable and are not, and nothing in a diff would show it. + """ + candidate = baseline(name="candidate", bv_destroyed_percent=50) + with self.assertRaises(Incomparable): + compare(candidate, baseline()) + + def test_refuses_a_different_scenario(self): + candidate = baseline(name="candidate", scenario="other.mms") + with self.assertRaises(Incomparable): + compare(candidate, baseline()) + + def test_a_small_improvement_is_reported_as_no_evidence(self): + """12-8 against 10-10 is the trap. It must not read as progress.""" + report = compare(baseline(name="candidate", wins=12), baseline(wins=10)) + self.assertIn("does not show a difference", report) + + def test_a_real_improvement_is_called_one(self): + candidate = baseline(name="candidate", wins=95, decided=100, games=100) + report = compare(candidate, baseline(wins=20, decided=100, games=100)) + self.assertIn("is an improvement", report) + + def test_a_real_regression_is_called_one(self): + candidate = baseline(name="candidate", wins=20, decided=100, games=100) + report = compare(candidate, baseline(wins=95, decided=100, games=100)) + self.assertIn("is a regression", report) + + def test_a_mostly_undecided_run_is_flagged_as_weak(self): + candidate = baseline(name="candidate", wins=6, decided=10, games=40) + report = compare(candidate, baseline(wins=10, decided=10, games=40)) + self.assertIn("Treat the comparison as weak", report) + + +if __name__ == "__main__": + unittest.main() -- 2.51.2