diff --git a/plan/training.md b/plan/training.md index 3a59654..da1f34d 100644 --- a/plan/training.md +++ b/plan/training.md @@ -1956,12 +1956,8 @@ and played indistinguishably from the hand-assembled vector it was trained from explanations were tried and both failed: dropping the thermometer's components moved R2 by 0.001, and the label was reached for before the weights were read. -- [ ] **19.6% of rows are displaced decisions read as preferences.** The fit - report already counts them: on policy, and still not the candidate the - weights ranked first, because another unit claimed the ground before they - reconciled. `chosen` is what was left rather than what was preferred. This - is the only mechanical explanation still standing and it costs no matches - to test - refit excluding those rows and see whether R2 moves +- [x] **19.6% of rows are displaced decisions read as preferences.** + `--drop-displaced` leaves them out. Measured below - [ ] **Try a label that is not one-sided.** `inflicted_bv_per_round` has printed its own warning on every fit: it counts only what was taken off the enemy, so mutual disengagement scores the same as a good trade, and @@ -2009,3 +2005,53 @@ attribution error the team tier cannot avoid. - [ ] Fit at `unit` and at `force` and compare. **Informative only** - keep the learning restricted to the team tier until a tiered fit has been benched and shown better. Cheap: no new matches, one refit each + +## Dropping the displaced rows moves the fit, and not where expected + +`--drop-displaced` leaves out the decisions where the weights' first choice was +taken by another unit before the force reconciled. Over the 300-match corpus +that is 3721 of 18980 decisions, 19.6%, and 1.37M of 4.99M difference rows. + +| | kept | dropped | +| --- | --- | --- | +| decisions | 18980 | 15259 | +| rows | 4989629 | 3623485 | +| `r_squared` | 0.2518 | 0.2309 | +| agreement | 0.506 | 0.443 | + +**The two `r_squared` figures are not comparable and neither is the agreement.** +Dropping 27% of the rows changes the denominator both are measured against, so +the pair is not a ranking, and the report now says so rather than inviting the +reading. What the numbers can say is that the fit *moved*: the two vectors +correlate at r = +0.93 with five sign flips, which is a different vector rather +than a nudge. + +Where it moved is the interesting part, and it is the opposite of a nuisance +correction: + +| | kept | dropped | +| --- | --- | --- | +| `expected_damage` | +0.0868 | +0.0636 | +| `incoming_damage` | -0.0334 | -0.0720 | +| `level_tmm` | -0.0287 | -0.0897 | +| `enemy_centroid` | -0.0376 | -0.0081 | + +Summing the terms that pull us in against those that hold us off, the balance +goes from **1.69:1 in favour of closing to 0.40:1** - defence outweighs offence +once the displaced rows are gone. That is the same quantity +`plan/features.md` calls "offence is priced exactly and defence is not priced at +all", arrived at from a completely different direction, and it says a +substantial part of that imbalance was rows where the bot did not get the hex it +asked for being read as a preference for the hex it settled for. + +Two cautions before this is believed. `level_tmm` going to -0.0897 is a *larger* +violation of "more defence is never worse", not a smaller one. And the heat +group's suspect signs got worse rather than better - `heat_shutdown_risk` joins +`heat_ammo_explosion_risk` and `heat_mp_penalty` on the wrong side, all three +mutually collinear above 0.7, where the report's own rule says the split of +weight between them is arbitrary. + +- [ ] Bench `fitted-r12-nodisp` against `fitted-r11`. The diagnostics cannot + rank these two and the sign warnings say do not spend a benchmark on a + vector whose signs are unexplained - so explain the heat group first, or + bench with it excluded diff --git a/sds/cli.py b/sds/cli.py index e287d68..9d3bda6 100644 --- a/sds/cli.py +++ b/sds/cli.py @@ -1321,7 +1321,12 @@ def cmd_train(args: argparse.Namespace) -> int: label = Label(kind=args.label, gamma=args.gamma, credit=args.credit, tier=args.tier) # Streamed rather than read in: a real corpus is tens of gigabytes of # JSON, and the fit needs only the running totals `scan_run` keeps. - corpus, normals = scan_run(target, imitation=args.imitation, label=label) + corpus, normals = scan_run( + target, + imitation=args.imitation, + label=label, + drop_displaced=getattr(args, "drop_displaced", False), + ) if manifest is not None: try: check_data(manifest, sorted(corpus.phi_names)) @@ -1769,6 +1774,15 @@ def main(argv: list[str] | None = None) -> int: metavar="NAME", help="fit only these features; naming a local one is an error", ) + train.add_argument( + "--drop-displaced", + action="store_true", + help="leave out decisions where the weights' first choice was taken by " + "another unit before the force reconciled. On such a row `chosen` is " + "what was left rather than what was preferred, so every difference it " + "contributes states a preference about a comparison nobody made. It was " + "19.6%% of rows on the corpus this was measured against", + ) train.add_argument( "--label", default=TRAIN_LABEL, diff --git a/sds/train.py b/sds/train.py index 07e7722..ffad2f0 100644 --- a/sds/train.py +++ b/sds/train.py @@ -2266,6 +2266,11 @@ class Normals: self.explored = 0 # On policy, but the ground was taken first. See `Decision.displaced`. self.displaced = 0 + # Set by the caller before any `observe`. Off by default: the rows are an + # honest record of moves that were played, and whether a fit should read + # one as a preference is the open question. + self.drop_displaced = False + self.dropped_displaced = 0 # ---- accumulating @@ -2325,7 +2330,17 @@ class Normals: Each decision contributes total weight 1 however many candidates it had, so a unit with twenty options does not outvote nineteen units with two. + + A displaced decision is skipped when `drop_displaced` is set. On such a + row `chosen` is what was left after another unit claimed the ground, not + what the weights preferred, so every difference it contributes says + "this was better than that" about a comparison nobody made. It is 19.6% + of rows on the corpus this was measured against - enough to be worth + asking about, and the answer is a refit rather than an argument. """ + if self.drop_displaced and decision.displaced: + self.dropped_displaced += 1 + return if not math.isfinite(label): raise TrainingError( f"a label came out {label}, which cannot be fitted. A ratio " @@ -2565,6 +2580,10 @@ class Fit: # Per suspect-sign feature, the columns it is most collinear with. A sign # flip with a +0.9 partner is the partner's weight wearing this one's name. suspect_partners: dict[str, list[tuple[str, float]]] + # How many displaced decisions were left out of the solve rather than + # fitted. Zero unless `--drop-displaced`; the denominator changes with it, + # so the report has to say which of the two it is describing. + dropped_displaced: int = 0 def to_document(self, commit: str = "unknown") -> dict: return { @@ -2797,6 +2816,7 @@ def _fitted( logged_argmax=normals.logged_argmax, explored_decisions=normals.explored, displaced_decisions=normals.displaced, + dropped_displaced=normals.dropped_displaced, suspect_signs=suspect, suspect_partners={name: normals.partners(name, names) for name, _, _ in suspect}, ) @@ -2823,7 +2843,10 @@ def fit( def scan_run( - target: Path, imitation: bool = False, label: Label | None = None + target: Path, + imitation: bool = False, + label: Label | None = None, + drop_displaced: bool = False, ) -> tuple[Corpus, Normals]: """Read a run directory into the normal equations, holding no rows. @@ -2841,6 +2864,7 @@ def scan_run( """ corpus = Corpus() normals = Normals() + normals.drop_displaced = drop_displaced for decision, y in stream_corpus(target, corpus, imitation=imitation, label=label): normals.note(decision) if y is not None: @@ -2883,9 +2907,12 @@ def fit_run( ridge: float = 1e-3, features: list[str] | None = None, label: Label | None = None, + drop_displaced: bool = False, ) -> tuple[Corpus, Fit]: """Read a run directory and fit it without ever holding it.""" - corpus, normals = scan_run(target, imitation=imitation, label=label) + corpus, normals = scan_run( + target, imitation=imitation, label=label, drop_displaced=drop_displaced + ) return corpus, fit_scan( target, corpus, normals, imitation=imitation, ridge=ridge, features=features, label=label ) @@ -3034,7 +3061,22 @@ def report(result: Fit) -> str: "a corpus with none of them can only confirm what it was generated " "from." ) - if result.displaced_decisions: + if result.dropped_displaced: + total = result.decisions + result.dropped_displaced + share = result.dropped_displaced / total if total else 0.0 + lines.append( + f"{result.dropped_displaced} of {total} decisions ({share:.1%}) were " + "**left out**: on policy, and still not the candidate the weights " + "ranked first, because another unit claimed the ground before they " + "reconciled. `chosen` on such a row is what was left rather than " + "what was preferred, so every difference it contributes states a " + "preference about a comparison nobody made. The rows are an honest " + "record of moves that were played, and dropping them is a question " + "being asked rather than a defect being fixed - so `r_squared` here " + "is over a different row set and is not comparable with a fit that " + "kept them. Bench the two; do not rank them by the diagnostic." + ) + elif result.displaced_decisions: share = result.displaced_decisions / result.decisions if result.decisions else 0.0 lines.append( f"{result.displaced_decisions} of {result.decisions} decisions " @@ -3042,10 +3084,10 @@ def report(result: Fit) -> str: "the weights ranked first: another unit claimed the ground before " "they reconciled. The row is an honest record of a move that was " "played, but `chosen` on it is what was left rather than what was " - "preferred, and the fit reads it as a preference. A corpus recorded " - "before `record_training` wrote the played move reports zero here " - "because the old code recomputed the argmax, not because nothing " - "was displaced." + "preferred, and the fit reads it as a preference. `--drop-displaced` " + "leaves them out. A corpus recorded before `record_training` wrote " + "the played move reports zero here because the old code recomputed " + "the argmax, not because nothing was displaced." ) if result.logged_argmax is not None and result.logged_argmax < 0.999: lines.append( diff --git a/tests/test_train.py b/tests/test_train.py index d504d3d..e81ee4a 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -1737,3 +1737,62 @@ class AFittedVectorRecordsItsBasis(unittest.TestCase): if not document.get("basis"): missing.append(path.name) self.assertEqual(missing, [], "these weight vectors do not say what basis they are for") + + +class TestDroppedDisplacedIsReportedHonestly(unittest.TestCase): + """A fit that left rows out must not describe them as rows it fitted. + + The first version of `--drop-displaced` kept the old paragraph, which says + the fit "reads it as a preference" - about rows it had just excluded - and + divided the dropped count by the *kept* decisions, reporting 24.4% where the + real share of the corpus was 19.6%. A diagnostic that describes the opposite + of what happened is worse than none. + """ + + def make(self, decisions: int, displaced: int, dropped: int): + from sds.train import Fit + + return Fit( + weights={}, + features=[], + excluded_local=[], + excluded_dead=[], + excluded_thin={}, + decisions=decisions, + matches=300, + rows=0, + labelled_seats=0, + unlabelled_matches=0, + unscoped_decisions=0, + r_squared=0.0, + label_mean=0.0, + label_kind="inflicted_bv_per_round", + label=Label(), + imitation_rows=0, + ridge=1e-3, + framing="difference", + agreement=0.0, + logged_argmax=None, + explored_decisions=0, + displaced_decisions=displaced, + suspect_signs=[], + suspect_partners={}, + dropped_displaced=dropped, + ) + + def test_the_share_is_over_the_whole_corpus(self): + text = report(self.make(15259, 3721, 3721)) + self.assertIn("3721 of 18980", text) + self.assertIn("19.6%", text) + self.assertIn("left out", text) + self.assertNotIn("the fit reads it as a preference", text) + + def test_it_warns_the_r_squared_is_not_comparable(self): + # Dropping a quarter of the rows changes the denominator, so the two + # numbers are not a ranking. Somebody will try to read them as one. + self.assertIn("not comparable", report(self.make(100, 25, 25))) + + def test_keeping_them_still_says_the_fit_reads_them(self): + text = report(self.make(18980, 3721, 0)) + self.assertIn("the fit reads it as a preference", text) + self.assertIn("--drop-displaced", text)