From 088e3cac7de9c3d791a79d4eb0d9fa40381ae7dd Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Thu, 20 Aug 2026 11:04:35 -0400 Subject: [PATCH] feat(training): fingerprint the feature basis a corpus was recorded against The manifest compares feature name lists, which cannot see a feature that keeps its name and changes what it measures. `p_kill` did exactly that: it was overconfident by 3.5x to 1144x depending on the volley's shape, and every weight fitted against the old one is void. Hashes each feature's name, normalisation and one-sentence description, since a feature whose meaning changed needs its sentence changed too. --- plan/training.md | 33 +++++++++++++++++++++ sds/cli.py | 3 +- sds/corpus.py | 70 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_corpus.py | 48 ++++++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 1 deletion(-) diff --git a/plan/training.md b/plan/training.md index 2a2d824..ae4b1f6 100644 --- a/plan/training.md +++ b/plan/training.md @@ -519,3 +519,36 @@ engagement terms are trustworthy has an obvious degenerate optimum: land one good blow and run everything off the map. That scores well on every label above and is not the game. Withdrawal waits until a fitted `M` beats the hand-authored one on a fight it is supposed to win. + +## The manifest cannot see a feature that changed its mind + +A corpus manifest records the sorted list of feature names it was recorded +against, which catches a feature added, removed or renamed. It cannot catch the +case that has already happened once: + +`p_kill` asked `P(total damage >= centre torso health)`. That ignored that +damage is distributed across hit locations, and it was overconfident by between +3.5x and 1144x depending on the volley's shape - worst for many small packets, +because a threshold needing `k` packets on one location falls off like +`share^k` rather than like `share`. Its replacement asks `P(a lethal location is +destroyed)`. Same name, same normalisation, a different quantity, and **every +weight ever fitted against the old one is void**. + +A name list sees none of that. So the manifest also records a **basis +fingerprint**: a hash over each feature's name, normalisation and one-sentence +description. A feature whose meaning changed needs its sentence changed too, and +a test already fails the build when a description is missing or is not one +sentence, so the sentence is a usable proxy for the meaning. + +`sds train` refuses a corpus recorded against a different basis, with +`--stale-ok` to override. + +**What it still does not catch:** a computation changed while its sentence was +left alone. Nothing short of a hand-bumped constant beside the code can, and +that belongs in `sds-core` rather than in the Python. Left open rather than +pretended away. + +Not in [epoch](../sds/epoch.py) on purpose: the epoch versions what a *match* +is - rules, victory conditions, what the bots experience. This versions what the +bot's *scoring* means. A change can move one without the other, and collapsing +them would make both vaguer. diff --git a/sds/cli.py b/sds/cli.py index 0b4e0cf..7ae226c 100644 --- a/sds/cli.py +++ b/sds/cli.py @@ -475,7 +475,7 @@ def cmd_train(args: argparse.Namespace) -> int: bot carrying it, `--against` the same baseline. """ from .baseline import head_commit - from .corpus import Stale, check, check_data, resolve + from .corpus import Stale, check, check_basis, check_data, current_basis, resolve from .train import Label, TrainingError, fit, read_corpus, report, save # A saved corpus resolves to its data directory and brings its manifest; @@ -514,6 +514,7 @@ def cmd_train(args: argparse.Namespace) -> int: found = sorted({name for d in corpus.decisions for c in d.candidates for name in c.phi}) try: check_data(manifest, found) + check_basis(manifest, current_basis()) except Stale as error: if not args.stale_ok: print(f"error: {error}", file=sys.stderr) diff --git a/sds/corpus.py b/sds/corpus.py index b090dda..2aea3fc 100644 --- a/sds/corpus.py +++ b/sds/corpus.py @@ -32,6 +32,7 @@ footnote. from __future__ import annotations +import hashlib import json import os import re @@ -140,6 +141,11 @@ class Manifest: # Every feature name that appears in any candidate's `phi`, sorted. The # field the staleness check is about. features: list[str] = field(default_factory=list) + # Fingerprint of the feature basis this was recorded against. See + # `basis_fingerprint`: a name list cannot see a feature that keeps its name + # and changes what it measures. + basis: str = "" + # The same names split by normalisation class, as the log declared them. A # `local` one can never be fitted; see `train.feature_names`. learnable: list[str] = field(default_factory=list) @@ -305,6 +311,7 @@ def save(run_dir: Path, name: str, notes: str = "", replace: bool = False) -> Ma recorded=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), commit=head_commit(), epoch=EPOCH, + basis=current_basis(), scenario=_scenario_of(run_dir), bot=bot, weights=weights, @@ -358,6 +365,19 @@ def resolve(target: str | Path) -> tuple[Path, Manifest | None]: return path, None +def current_basis() -> str: + """This checkout's feature basis, from the binary that defines it.""" + try: + from .catalogue import from_bot + + return basis_fingerprint(from_bot()["features"]) + except Exception: + # No binary built, or an older one with no --print-catalogue. A corpus + # saved without a basis simply does not get the check; it is not worth + # refusing to save over. + return "" + + def reference_features() -> list[str]: """The features this checkout's baseline `M` is written against. @@ -476,6 +496,56 @@ def check( return review +def basis_fingerprint(entries: list[dict]) -> str: + """A short hash over what every feature is called, normalised as, and says. + + The manifest already compares feature **name lists**, which catches a + feature added or removed. It cannot catch the case that is about to matter + most: a feature that keeps its name and changes what it computes. + + `p_kill` is the example. It asked `P(total damage >= centre torso health)`, + which ignored that damage is distributed across hit locations and was + overconfident by between 3.5x and 1144x depending on the volley's shape. The + replacement asks `P(a lethal location is destroyed)`. Same name, same + normalisation, a different quantity - and every weight ever fitted against + the old one is meaningless. + + Hashing the description alongside the name and normalisation catches it, + because a feature whose meaning changed needs its sentence changed too, and + a test already fails the build when a description is missing or is not one + sentence. + + **What this does not catch**: a computation changed while its sentence was + left alone. Nothing short of a hand-bumped constant beside the code can, and + that belongs in `sds-core` rather than here. Recorded in + `plan/training.md` rather than pretended away. + + Deliberately not in `sds/epoch.py`: the epoch versions what a *match* is - + rules, victory conditions, what the bots experience. This versions what the + bot's *scoring* means. A run can change one without the other. + """ + material = "\n".join( + f"{e['name']}|{e['normalisation']}|{e['description']}" + for e in sorted(entries, key=lambda e: e["name"]) + ) + return hashlib.sha256(material.encode()).hexdigest()[:12] + + +def check_basis(manifest: Manifest, current: str) -> None: + """Refuse a corpus recorded against a different feature basis.""" + if not manifest.basis or manifest.basis == current: + return + raise Stale( + f"corpus '{manifest.name}' was recorded against feature basis " + f"{manifest.basis}; this checkout is {current}. A feature has been " + "added, removed, renamed, re-normalised, or has changed what it " + "measures. Weights fitted across that boundary are not comparable and " + "may be meaningless - `p_kill` once kept its name while its answer " + "moved by three orders of magnitude. Record the corpus again, or pass " + "--stale-ok if you know why this one is still worth fitting." + ) + + def check_data(manifest: Manifest, found: list[str]) -> None: """Refuse a corpus whose data is not what its manifest describes. diff --git a/tests/test_corpus.py b/tests/test_corpus.py index d6924cb..a031d12 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -20,7 +20,9 @@ from sds.corpus import ( Manifest, Review, Stale, + basis_fingerprint, check, + check_basis, check_data, corpora_root, listing, @@ -254,3 +256,49 @@ class TestListing(CorpusTest): if __name__ == "__main__": unittest.main() + + +class TestBasisFingerprint(unittest.TestCase): + """A name list cannot see a feature that keeps its name and changes meaning. + + `p_kill` asked `P(total damage >= centre torso health)` and was overconfident + by between 3.5x and 1144x depending on the volley's shape. Its replacement + asks `P(a lethal location is destroyed)`. Same name, same normalisation, a + different quantity, and every weight fitted against the old one is void. + """ + + def entry(self, name="p_kill", norm="bounded", description="The chance of a kill."): + return {"name": name, "normalisation": norm, "description": description} + + def test_the_same_basis_hashes_the_same(self): + a = [self.entry(), self.entry("overkill")] + self.assertEqual(basis_fingerprint(a), basis_fingerprint(list(reversed(a)))) + + def test_a_changed_description_changes_the_basis(self): + before = basis_fingerprint([self.entry(description="P(total >= CT health).")]) + after = basis_fingerprint([self.entry(description="P(a lethal location dies).")]) + self.assertNotEqual(before, after) + + def test_a_changed_normalisation_changes_the_basis(self): + self.assertNotEqual( + basis_fingerprint([self.entry(norm="bounded")]), + basis_fingerprint([self.entry(norm="rank")]), + ) + + def test_an_added_feature_changes_the_basis(self): + self.assertNotEqual( + basis_fingerprint([self.entry()]), + basis_fingerprint([self.entry(), self.entry("p_breach")]), + ) + + def test_a_mismatched_basis_is_refused_and_says_why(self): + manifest = Manifest(name="old", basis="aaaaaaaaaaaa") + with self.assertRaises(Stale) as caught: + check_basis(manifest, "bbbbbbbbbbbb") + message = str(caught.exception) + self.assertIn("aaaaaaaaaaaa", message) + self.assertIn("bbbbbbbbbbbb", message) + + def test_a_corpus_with_no_basis_is_not_refused(self): + # Recorded before the fingerprint existed. Not worth refusing over. + check_basis(Manifest(name="old", basis=""), "bbbbbbbbbbbb") -- 2.51.2