From a491bcc58dfe4757897698d5250da4f4dd9b73e2 Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Sat, 29 Aug 2026 22:36:26 -0400 Subject: [PATCH] fix(unit-types): refuse a --bot the match container cannot see The bot string reaches the seat inside the container verbatim, and the container mounts only the repository. A host path outside it did not fail: the seat never started, every decision defaulted, and the run still reported a win rate. It cost a whole 24-match baseline arm. Co-Authored-By: Claude Opus 5 (1M context) --- plan/unit-types.md | 15 +++++++++++++++ sds/cli.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/plan/unit-types.md b/plan/unit-types.md index 7a4b0bd..438a3db 100644 --- a/plan/unit-types.md +++ b/plan/unit-types.md @@ -110,6 +110,21 @@ that says so the day it changes. it comes up for a Mek down to 1 walk MP as well. Not a vehicle rule and not this branch's code, recorded here so it is not lost. +## Measuring an arm + +Two rules, both bought rather than reasoned: + +- **Smoke-test one game before committing an arm to 24.** Run one, read the + decision log, and confirm the seat answered with candidates - `outcomes + {'answered': 32}` and 9,617 candidates is a working bot; `{'defaulted': 162}` + and zero candidates is a seat that never started. A whole baseline arm was + lost to the second, and it printed weapon tables and a win rate while being a + benchmark of the inert default. +- **Report the per-scenario breakdown even when the pooled result is clean.** At + the sample sizes a suite of this size gives, one scenario can carry an entire + difference, and a single pooled share hides it. A summary standing in for the + thing it summarises is the same failure as every other one in this file. + ## Open questions for jmm - **The motive damage result table is not dumped.** MegaMek flags the roll and diff --git a/sds/cli.py b/sds/cli.py index abdddab..84e4906 100644 --- a/sds/cli.py +++ b/sds/cli.py @@ -388,9 +388,47 @@ def cmd_play(args: argparse.Namespace) -> int: return play_match(spec, _run_dir(Path(args.out), "play")) +def _check_bot_is_reachable(sds_bot: str | None) -> None: + """A custom `--bot` has to be a path the match container can see. + + The bot string is handed to the seat process inside the container verbatim, + and the container mounts only the repository, at `/work`. A host path + outside that mount does not fail: the seat never starts, every decision + takes the inert default, and the run completes and reports a win rate. + + That is exactly the shape this repository keeps paying for - an absence + wearing a legal value. It cost a whole 24-match baseline arm, which came + back with 162 decisions of 162 `defaulted` and not one candidate, and the + only reason it was caught is that the decision logs were three orders of + magnitude too small. + + **Why this is its own function rather than a line in `_check_bot`.** That + check - the one that refuses a binary older than the code it was built from + - does not run for a custom bot at all. So the case where a caller is most + likely to hand over something wrong was the one case with no coverage, while + the default path, where the binary is derived and can hardly be wrong, had + all of it. A guard that stands down exactly where the risk is concentrated + is worse than no guard: its presence reads as protection, and nobody looks + again. Any check gated on "unless the caller supplied their own" is worth + re-reading in that light. + """ + if sds_bot is None: + return + binary = sds_bot.split()[0] if sds_bot else sds_bot + if binary.startswith("/work/"): + return + raise MatchError( + f"--bot {binary} is not under /work, so the match container cannot see " + "it and every decision would silently default. Copy the binary into the " + "repository and pass its container path, e.g. " + "`--bot /work/target/release/sds-bot-baseline`." + ) + + def _bench(args: argparse.Namespace, label: str, sds_bot: str | None) -> int: sds_bot = _with_explore(sds_bot, args) _check_bot(sds_bot) + _check_bot_is_reachable(sds_bot) if getattr(args, "suite", None): scenarios, scenario_label = _suite(Path(args.suite).resolve()) print(f"suite: {len(scenarios)} scenarios, {scenario_label}", file=sys.stderr) -- 2.51.2