diff --git a/sds/train.py b/sds/train.py index c7c7040..e383d2e 100644 --- a/sds/train.py +++ b/sds/train.py @@ -423,6 +423,7 @@ class Fit: weights: dict[str, float] features: list[str] excluded_local: list[str] + excluded_dead: list[str] decisions: int matches: int rows: int @@ -456,6 +457,7 @@ class Fit: "r_squared": self.r_squared, "agreement": self.agreement, "excluded_local": self.excluded_local, + "excluded_dead": self.excluded_dead, }, } @@ -504,6 +506,24 @@ def _logged_argmax(corpus: Corpus) -> float | None: return agreed / seen if seen else None +def dead_columns(design: list[list[float]], names: list[str]) -> list[str]: + """Features that never distinguished a chosen candidate from a rejected one. + + A feature the bot cannot compute reads as the same number everywhere, so its + column of `chosen - rejected` is all zeros. Ridge then answers zero for it, + which is indistinguishable in the report from "fitted, and it does not + matter". Two features are in this state today - `has_been_fired_upon`, since + no attack event reaches the bot, and `honour_broken`, which waits on + doctrine - and any feature that quietly stops being computed joins them. + + Named and dropped rather than regularised to zero, so a dead column is a + line in the report instead of a weight somebody trusts. + """ + return [ + name for index, name in enumerate(names) if all(abs(row[index]) <= 1e-12 for row in design) + ] + + def fit(corpus: Corpus, ridge: float = 1e-3, features: list[str] | None = None) -> Fit: """Fit one weight per learnable feature, on candidate differences. @@ -523,6 +543,16 @@ def fit(corpus: Corpus, ridge: float = 1e-3, features: list[str] | None = None) "nothing to compare against. A corpus of forced moves teaches " "nothing." ) + dead = dead_columns(design, names) + if dead: + names = [name for name in names if name not in dead] + if not names: + raise TrainingError( + "every learnable feature was constant across every decision's " + "candidates: " + ", ".join(dead) + ". There is nothing to fit." + ) + design, target, sample_weights = difference_rows(corpus, names) + solved = least_squares(design, target, sample_weights, ridge) weights = dict(zip(names, solved, strict=True)) @@ -549,6 +579,7 @@ def fit(corpus: Corpus, ridge: float = 1e-3, features: list[str] | None = None) weights=weights, features=names, excluded_local=sorted(corpus.local_features), + excluded_dead=dead, decisions=len(rows), matches=len({d.tag for d, _ in rows}), rows=len(design), @@ -625,6 +656,17 @@ def report(result: Fit) -> str: + ", ".join(f"`{n}`" for n in result.excluded_local) + ".", ] + if result.excluded_dead: + lines += [ + "", + "**Dropped as constant:** " + + ", ".join(f"`{n}`" for n in result.excluded_dead) + + ". These never differed between a chosen candidate and one it " + "beat, so nothing in this corpus says what they are worth. That is " + "usually a feature the bot cannot compute yet rather than a feature " + "that does not matter - the two look identical in a weight of zero, " + "which is why they are named here instead.", + ] if result.suspect_signs: lines += ["", "**Signs that contradict the feature's own description:**", ""] for name, weight, expected in result.suspect_signs: diff --git a/tests/test_train.py b/tests/test_train.py index 738531b..8ced3d7 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -7,6 +7,7 @@ import unittest from pathlib import Path from sds.train import ( + Candidate, Corpus, Decision, LocalFeatureRefused, @@ -370,3 +371,77 @@ class TestWeightsFile(unittest.TestCase): if __name__ == "__main__": unittest.main() + + +class TestDeadColumns(unittest.TestCase): + """A feature the bot cannot compute must be named, not weighted zero. + + `has_been_fired_upon` and `honour_broken` have setters and no callers: no + attack event reaches the bot, and doctrine does not exist. Their column of + `chosen - rejected` is all zeros, ridge answers zero, and the report would + otherwise read exactly like a feature that was fitted and found not to + matter. + """ + + def corpus_with_a_constant(self): + rows = [] + for i in range(20): + rows.append( + Decision( + tag=f"m{i}", + seat="N", + round=1, + phase="MOVEMENT", + unit=1, + candidates=( + Candidate("a", {"expected_damage": 0.9, "never_set": 0.0}, 1.0), + Candidate("b", {"expected_damage": 0.1, "never_set": 0.0}, 0.1), + ), + chosen=0, + learnable=("expected_damage", "never_set"), + local=(), + ) + ) + return Corpus( + decisions=rows, + labels={(f"m{i}", "N"): 0.5 for i in range(20)}, + matches={f"m{i}" for i in range(20)}, + ) + + def test_a_constant_feature_is_named_not_weighted(self): + result = fit(self.corpus_with_a_constant()) + self.assertEqual(result.excluded_dead, ["never_set"]) + self.assertNotIn("never_set", result.weights) + self.assertIn("expected_damage", result.weights) + + def test_the_report_says_why(self): + result = fit(self.corpus_with_a_constant()) + self.assertIn("never_set", report(result)) + self.assertIn("constant", report(result).lower()) + + def test_a_corpus_of_only_constants_refuses(self): + rows = [] + for i in range(5): + rows.append( + Decision( + tag=f"m{i}", + seat="N", + round=1, + phase="MOVEMENT", + unit=1, + candidates=( + Candidate("a", {"never_set": 0.0}, 1.0), + Candidate("b", {"never_set": 0.0}, 0.1), + ), + chosen=0, + learnable=("never_set",), + local=(), + ) + ) + corpus = Corpus( + decisions=rows, + labels={(f"m{i}", "N"): 0.5 for i in range(5)}, + matches={f"m{i}" for i in range(5)}, + ) + with self.assertRaises(TrainingError): + fit(corpus)