diff --git a/.gitignore b/.gitignore index e0ee783..887a9d8 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ target/ __pycache__/ *.pyc .ruff_cache/ + +# Validation reports, regenerated by `sds validate`. +results/ diff --git a/bridge/sds/SdsValidate.java b/bridge/sds/SdsValidate.java new file mode 100644 index 0000000..598204a --- /dev/null +++ b/bridge/sds/SdsValidate.java @@ -0,0 +1,165 @@ +package sds; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import megamek.common.Player; +import megamek.common.compute.Compute; +import megamek.common.game.Game; +import megamek.common.game.IGame; +import megamek.common.loaders.MekSummaryCache; +import megamek.common.scenario.Scenario; +import megamek.common.scenario.ScenarioLoader; +import megamek.common.units.Entity; +import megamek.common.board.Board; +import megamek.common.board.Coords; + +/** + * Load every scenario in a suite and say which ones a match could not play. + * + *

A broken map reference or a unit name the cache cannot resolve fails at + * the same point a real match would, but here it costs one JVM start for the + * whole suite instead of an hour of benchmark that reports the failures as + * matches which produced no result. + * + *

Runs the same checks the deployment phase would: the scenario parses, the + * game builds, both sides have units, and every unit has somewhere legal to + * stand. Nothing here plays a turn. + */ +public final class SdsValidate { + + public static void main(String[] argv) throws Exception { + List scenarios = new ArrayList<>(); + Path out = null; + for (int i = 0; i < argv.length; i++) { + if ("--out".equals(argv[i])) { + out = Path.of(argv[++i]); + } else { + scenarios.add(argv[i]); + } + } + if (scenarios.isEmpty()) { + System.err.println("usage: sds.SdsValidate [--out report.json] ..."); + System.exit(2); + } + + // The cache resolves unit names. Without it every scenario fails for + // the same uninteresting reason. + MekSummaryCache cache = MekSummaryCache.getInstance(); + while (!cache.isInitialized()) { + Thread.sleep(250L); + } + + ObjectMapper mapper = new ObjectMapper(); + ArrayNode report = mapper.createArrayNode(); + int failed = 0; + + for (String path : scenarios) { + ObjectNode entry = mapper.createObjectNode(); + entry.put("scenario", path); + ArrayNode problems = mapper.createArrayNode(); + try { + check(new File(path), entry, problems); + } catch (Exception e) { + problems.add(e.getClass().getSimpleName() + ": " + e.getMessage()); + } + entry.set("problems", problems); + entry.put("ok", problems.isEmpty()); + report.add(entry); + + String name = new File(path).getName(); + if (problems.isEmpty()) { + System.out.printf("ok %s (%d units)%n", name, entry.path("units").asInt()); + } else { + failed++; + System.out.printf("FAIL %s%n", name); + for (var problem : problems) { + System.out.printf(" %s%n", problem.asText()); + } + } + } + + System.out.printf("%n%d scenarios, %d failed%n", scenarios.size(), failed); + if (out != null) { + Files.createDirectories(out.toAbsolutePath().getParent()); + Files.writeString(out, mapper.writerWithDefaultPrettyPrinter().writeValueAsString(report)); + } + System.exit(failed == 0 ? 0 : 1); + } + + private static void check(File file, ObjectNode entry, ArrayNode problems) throws Exception { + Scenario scenario = new ScenarioLoader(file).load(); + IGame igame = scenario.createGame(); + if (!(igame instanceof Game game)) { + problems.add("scenario did not produce a Total Warfare game"); + return; + } + + int units = 0; + for (Entity entity : game.getEntitiesVector()) { + units++; + } + entry.put("units", units); + + List players = game.getPlayersList(); + entry.put("players", players.size()); + if (players.size() < 2) { + problems.add("needs two players, has " + players.size()); + } + for (Player player : players) { + if (game.getEntitiesOwnedBy(player) == 0) { + problems.add("player '" + player.getName() + "' has no units"); + } + } + + for (Entity entity : game.getEntitiesVector()) { + Board board = game.getBoard(entity); + if (board == null) { + problems.add(entity.getShortName() + ": no board"); + continue; + } + if (entity.getWeaponList().isEmpty()) { + problems.add(entity.getShortName() + ": carries no weapons"); + } + if (!hasLegalDeployment(game, board, entity)) { + problems.add(entity.getShortName() + ": no legal deployment hex in its zone"); + } + } + } + + /** + * The same test {@link SdsClient} applies in the deployment phase, stopped + * at the first hex that passes. A unit with none of these hangs a match in + * DEPLOYMENT, which is the expensive way to learn the map and the start + * zone disagree. + */ + private static boolean hasLegalDeployment(Game game, Board board, Entity unit) { + for (int y = 0; y < board.getHeight(); y++) { + for (int x = 0; x < board.getWidth(); x++) { + Coords candidate = new Coords(x, y); + if (!board.isLegalDeployment(candidate, unit)) { + continue; + } + if (unit.isLocationProhibited(candidate, unit.getBoardId(), 0)) { + continue; + } + if (Compute.stackingViolation(game, unit, candidate, 0, candidate, null, + unit.climbMode(), true) != null) { + continue; + } + return true; + } + } + return false; + } + + private SdsValidate() { + } +} diff --git a/sds/cli.py b/sds/cli.py index d249db7..1fe54ef 100644 --- a/sds/cli.py +++ b/sds/cli.py @@ -3,6 +3,7 @@ sds one one match, verbose, for looking at sds bench many matches, aggregated, for deciding with sds control the harness's self-test: Princess against itself +sds validate every scenario in a suite loads and can deploy sds view what the bot believed and chose, as HTML """ @@ -111,6 +112,17 @@ def _bench(args: argparse.Namespace, label: str, sds_bot: str | None) -> int: if len(factions) != 2: raise MatchError(f"{path} has {len(factions)} factions; bench wants exactly 2") + # Five seconds against the possibility of an hour. A scenario that cannot + # be loaded fails once per game it is drawn for, as "match produced no + # result", and a round-robin draws each one many times. + if not getattr(args, "no_validate", False): + from .match import validate + + code, report = validate(scenarios) + if code != 0: + broken = [entry["scenario"] for entry in report if not entry["ok"]] + raise MatchError(f"{len(broken)} scenarios will not load; nothing was run") + out_dir = _run_dir(Path(args.out), label) plans = [] for game in range(args.games): @@ -236,6 +248,19 @@ def cmd_scenarios(args: argparse.Namespace) -> int: return 0 +def cmd_validate(args: argparse.Namespace) -> int: + """Load every scenario in a suite and report the ones a match could not play.""" + from .match import validate + + if args.suite: + scenarios, label = _suite(Path(args.suite).resolve()) + print(f"suite: {len(scenarios)} scenarios, {label}", file=sys.stderr) + else: + scenarios = [Path(args.scenario or DEFAULT_SCENARIO).resolve()] + code, _report = validate(scenarios) + return code + + def cmd_baselines(args: argparse.Namespace) -> int: from .baseline import BASELINES @@ -343,12 +368,22 @@ def main(argv: list[str] | None = None) -> int: metavar="DIR", help="run games round-robin across every .mms in DIR instead of one scenario", ) + bench.add_argument( + "--no-validate", + action="store_true", + help="skip the scenario preflight", + ) bench.set_defaults(func=cmd_bench) control = sub.add_parser("control", help="Princess vs Princess: the harness's self-test") control.add_argument("--games", type=int, default=20) control.add_argument("--jobs", type=int, default=int(os.environ.get("SDS_JOBS", "2"))) control.add_argument("--suite", metavar="DIR", help="control across a whole suite") + control.add_argument( + "--no-validate", + action="store_true", + help="skip the scenario preflight", + ) control.set_defaults(func=cmd_control) scenarios = sub.add_parser("scenarios", help="generate a benchmark suite") @@ -357,6 +392,17 @@ def main(argv: list[str] | None = None) -> int: scenarios.add_argument("--suite-seed", type=int, default=1) scenarios.set_defaults(func=cmd_scenarios) + validate = sub.add_parser( + "validate", + help="load every scenario in a suite and report the broken ones", + description="Parses each scenario, builds its game, and checks both " + "sides have units with somewhere legal to stand. One JVM for the whole " + "suite: a broken map reference costs seconds here rather than an hour " + "of benchmark reporting matches that produced no result.", + ) + validate.add_argument("--suite", metavar="DIR", help="validate every .mms in DIR") + validate.set_defaults(func=cmd_validate) + baselines = sub.add_parser("baselines", help="list recorded baselines") baselines.set_defaults(func=cmd_baselines) diff --git a/sds/match.py b/sds/match.py index 977642b..39517aa 100644 --- a/sds/match.py +++ b/sds/match.py @@ -245,3 +245,65 @@ def run(spec: MatchSpec, out_dir: Path) -> dict: result["resultPath"] = str(result_path) result["exitCode"] = completed.returncode return result + + +def validate(scenarios: list[Path], report: Path | None = None) -> tuple[int, list[dict]]: + """Load every scenario in one JVM and report the ones a match could not play. + + A broken map reference costs one JVM start here instead of an hour of + benchmark reporting "match produced no result". Returns the exit code and + the parsed report. + """ + if shutil.which("docker") is None: + raise MatchError("docker is not on PATH") + mm_home = DEFAULT_MM_HOME + if not (mm_home / "MegaMek.jar").is_file(): + raise MatchError(f"no MegaMek.jar under {mm_home}; set MM_HOME") + if not (REPO / "bridge" / "build" / "sds.jar").is_file(): + raise MatchError("bridge/build/sds.jar is missing; run ./scripts/build.sh") + + report = report or (REPO / "results" / "validate.json") + report.parent.mkdir(parents=True, exist_ok=True) + container = f"sds-validate-{uuid.uuid4().hex[:8]}" + command = [ + "docker", + "run", + "--rm", + "--name", + container, + "--user", + f"{os.getuid()}:{os.getgid()}", + "-v", + f"{REPO}:/work", + "-v", + f"{mm_home}:/mm", + "-w", + "/mm", + DEFAULT_IMAGE, + "java", + "-Dlog4j2.configurationFile=/work/bridge/log4j2-quiet.xml", + "-Dsentry.dsn=", + "-Djava.awt.headless=true", + *ADD_OPENS, + "-cp", + "MegaMek.jar:lib/*:/work/bridge/build/sds.jar", + "sds.SdsValidate", + "--out", + f"/work/{report.relative_to(REPO)}", + ] + command += [f"/work/{path.resolve().relative_to(REPO)}" for path in scenarios] + + _track(container) + try: + completed = subprocess.run(command, check=False) + finally: + subprocess.run( + ["docker", "kill", container], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + _untrack(container) + + parsed = json.loads(report.read_text()) if report.is_file() else [] + return completed.returncode, parsed