diff --git a/crates/sds-bot/src/force.rs b/crates/sds-bot/src/force.rs index c3135d7..84ae2ad 100644 --- a/crates/sds-bot/src/force.rs +++ b/crates/sds-bot/src/force.rs @@ -8,15 +8,17 @@ //! 2. **Commit.** Move its stance toward what the appraisal wants, but only if //! the case is strong enough to overcome its grit. See //! [`sds_core::stance::Commitment`]. -//! 3. **Choose.** For each unit, take the proposal worth most under the stance -//! the lance is actually holding - not the one worth most in the abstract. +//! 3. **Reconcile.** Turn the menus into one set of orders. Not an argmax per +//! unit: see [`sds_core::reconcile`]. Two of ours never take the same hex, +//! and two of ours do not both empty into a machine one of them was going +//! to finish. //! //! Everything considered is kept in the decision, including the rejected //! proposals. A lance that chose badly and a lance offered nothing good look //! identical in a record that keeps only the winner, and telling those apart is //! most of debugging a hierarchy. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::Mutex; use anyhow::Result; @@ -24,6 +26,7 @@ use async_trait::async_trait; use sds_core::explore::Explore; use sds_core::features::Weights; use sds_core::plan::{Chosen, ForceDecision, UnitThought}; +use sds_core::reconcile::{reconcile_exploring, Doctrine}; use sds_core::stance::{Appraisal, Commitment, Grit, Intent, Stance, INTENTS}; use sds_node::{Request, Response, Thinker}; @@ -81,8 +84,9 @@ impl Thinker for ForceThinker { stance, grit, round, + remaining, } => Ok(Response::Decision(Box::new( - self.command(force, thoughts, stance, grit, round), + self.command(force, thoughts, stance, grit, round, &remaining), ))), Request::Propose { unit, .. } => { anyhow::bail!("force thinker was asked to propose for unit {unit}") @@ -99,6 +103,7 @@ impl ForceThinker { from_above: Stance, grit: Grit, round: i32, + remaining: &BTreeMap, ) -> ForceDecision { let mut appraisal = combine(&thoughts); @@ -128,49 +133,53 @@ impl ForceThinker { }; drop(commitments); - // Choose by the dot product, one weight set across every unit. A unit - // no longer says what its move is *for* and is no longer believed about - // it; the force measures every menu the same way, which is the whole - // difference between a self-report and a measurement. + // One set of orders out of many menus, not an argmax down each one. + // One weight set across every unit: a unit no longer says what its + // move is *for* and is no longer believed about it, the force measures + // every menu the same way. Menus are read in unit id order and every + // reduction is over a `BTreeMap`, so nothing here depends on which + // unit finished first. // - // Ties go to the earlier proposal, and a unit offers its proposals in a - // fixed order, so nothing here depends on what finished first. - // - // One argmax per unit, coordinating with nothing: two units can pick - // the same hex and can both empty their guns into a machine one of them - // was going to finish. `sds_core::reconcile` is the replacement and is - // not wired in here yet - adopting it changes play, so it arrives with - // a control run. - let chosen: Vec = thoughts + // Exploration happens inside the reconciliation rather than after it. + // A draw taken afterwards can land on a hex another unit has already + // claimed, which is the collision this replaced an argmax to stop. + let orders = reconcile_exploring( + &thoughts, + remaining, + &self.weights, + &Doctrine::default(), + &self.explore, + round.max(0) as u32, + ); + let stats = orders.stats; + if !orders.stalled.is_empty() { + eprintln!( + "[sds-bot] {force}: {} unit(s) had every hex claimed and got no order: {:?}", + orders.stalled.len(), + orders.stalled + ); + } + eprintln!( + "[sds-bot] {force} round {round}: {} units, {} proposals, {} scored, \ + {} blocked, {} moved off their own pick, {} explored, {} stalled", + stats.units, + stats.considered, + stats.evaluations, + stats.blocked, + stats.moved, + stats.explored, + orders.stalled.len() + ); + let stalled = orders.stalled.clone(); + let chosen: Vec = orders + .orders .iter() - .filter_map(|thought| { - let mut best = 0usize; - let mut best_value = f32::NEG_INFINITY; - for (index, proposal) in thought.proposals.iter().enumerate() { - let value = proposal.value(&self.weights); - if value > best_value { - best = index; - best_value = value; - } - } - // Sometimes take one the weights ranked lower, and say so. - // A corpus where the argmax was taken every time cannot teach - // which choices were good, only which ones we make. See - // `sds_core::explore`. - let (taken, policy) = self.explore.choose( - best, - thought.proposals.len(), - round.max(0) as u32, - thought.unit, - ); - let proposal = thought.proposals.get(taken)?; - Some(Chosen { - unit: thought.unit, - proposal: taken, - label: proposal.label.clone(), - value: proposal.value(&self.weights), - policy: policy.name().to_string(), - }) + .map(|order| Chosen { + unit: order.unit, + proposal: order.proposal, + label: order.label.clone(), + value: order.value, + policy: order.policy.name().to_string(), }) .collect(); @@ -182,6 +191,7 @@ impl ForceThinker { changed_because, weights: self.weights.clone(), chosen, + stalled, considered: thoughts, } } diff --git a/crates/sds-bot/src/main.rs b/crates/sds-bot/src/main.rs index 34f212b..0b92561 100644 --- a/crates/sds-bot/src/main.rs +++ b/crates/sds-bot/src/main.rs @@ -15,7 +15,7 @@ //! `SDS_LANCE_NODE=tcp:10.0.0.4:7000` moves lance command to another machine //! without changing a line here. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; use anyhow::{Context, Result}; @@ -60,6 +60,11 @@ struct Bot { grit: Grit, /// Orders chosen this round, by unit. Cleared when the round turns. orders: HashMap, + /// Units the force could not give a hex to this round: every stand they + /// offered was in a hex another order had already claimed. They pass, and + /// the host counts a pass as defaulted. A stand-still nobody chose must not + /// be reported as an answer - see the `answered`/`decisions` invariant. + stalled: HashSet, planned_round: Option, /// What each lance decided, kept for the rationale on the wire. decisions: Vec, @@ -150,6 +155,7 @@ impl Bot { company_node: registry.resolve(&company)?, grit, orders: HashMap::new(), + stalled: HashSet::new(), planned_round: None, decisions: Vec::new(), training: HashMap::new(), @@ -195,12 +201,22 @@ impl Bot { return Ok(()); } self.orders.clear(); + self.stalled.clear(); self.decisions.clear(); self.training.clear(); self.declared.clear(); self.latches.observe(observation); let mine = Self::mine(observation); + // What the other side has left, so a force can price finishing a + // machine above hurting two. Built from the bot client's observation + // and nothing else. + let enemies: Vec<&sds_core::wire::Unit> = observation + .units + .iter() + .filter(|u| !u.friendly && !u.destroyed) + .collect(); + let remaining = sds_core::reconcile::remaining(&enemies); if mine.is_empty() { self.planned_round = Some(observation.round); return Ok(()); @@ -257,6 +273,7 @@ impl Bot { stance: Stance::default(), grit: self.grit, round: observation.round, + remaining: remaining.clone(), }) .await?; let decision = answer.into_decision()?; @@ -278,6 +295,7 @@ impl Bot { stance: company_stance, grit: self.grit, round: observation.round, + remaining: remaining.clone(), }) .await?; let decision = answer.into_decision()?; @@ -288,6 +306,7 @@ impl Bot { } } } + self.stalled.extend(decision.stalled.iter().copied()); self.record_training(&decision); self.decisions.push(decision); } @@ -317,10 +336,16 @@ impl Bot { async fn movement(&mut self, observation: &Observation) -> Result { self.plan(observation).await?; let actor = observation.actor.unwrap_or(-1); - Ok(self - .orders - .remove(&actor) - .unwrap_or(Action::Move { steps: Vec::new() })) + if let Some(action) = self.orders.remove(&actor) { + return Ok(action); + } + // A unit whose every stand was in a hex somebody else claimed has no + // answer, and saying "stand still" would be counted as one. Pass: the + // host stands it still and counts the decision defaulted. + if self.stalled.contains(&actor) { + return Ok(Action::Pass); + } + Ok(Action::Move { steps: Vec::new() }) } /// Firing: build a menu of allocations, measure each, take the best. diff --git a/crates/sds-core/src/plan.rs b/crates/sds-core/src/plan.rs index b2a7b28..475d18c 100644 --- a/crates/sds-core/src/plan.rs +++ b/crates/sds-core/src/plan.rs @@ -111,6 +111,12 @@ pub struct ForceDecision { pub weights: Weights, /// The chosen proposal per unit, by index into that unit's proposals. pub chosen: Vec, + /// Units the force could not give a hex to: every proposal they had stood + /// somewhere another order had already claimed. Recorded rather than given + /// a hex they cannot have, and reported as unanswered rather than answered + /// with a stand-still nobody chose. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub stalled: Vec, /// Everything considered, kept so a bad call can be told apart from a bad /// menu. pub considered: Vec, @@ -183,6 +189,7 @@ mod tests { value: proposal.value(&Weights::hand_authored()), policy: "argmax".into(), }], + stalled: vec![], considered: vec![UnitThought { unit: 12, role: "SNIPER".into(), diff --git a/crates/sds-core/src/reconcile.rs b/crates/sds-core/src/reconcile.rs index aca9689..dca22e5 100644 --- a/crates/sds-core/src/reconcile.rs +++ b/crates/sds-core/src/reconcile.rs @@ -45,11 +45,22 @@ //! reduction is over a `BTreeMap`, so a force that hears from its units in a //! different sequence gets the same orders. Nothing here reads a clock. //! -//! Nothing in `sds-bot` calls this yet: `ForceThinker::command` still takes an -//! independent argmax, play is unchanged, and no control run applies. +//! **Exploration happens inside, or not at all.** [`crate::explore`] sometimes +//! takes a candidate the weights ranked lower so the corpus can teach which +//! choices were good rather than only which ones we make. Run outside this +//! module that draw can land on a hex another unit has already claimed, which +//! is the collision this exists to stop. So [`reconcile_exploring`] draws at +//! the moment a unit is served, over the options still open to it. The key is +//! unchanged - `(seed, game round, unit)` - so whether a unit explores at all +//! is the same draw it always was; only the set it lands in is narrowed to +//! what the unit can actually have. +//! +//! `ForceThinker::command` calls this. It replaced an independent argmax per +//! unit, so it changes play and arrived with a control run. use std::collections::{BTreeMap, BTreeSet}; +use crate::explore::{Explore, Policy}; use crate::features::Weights; use crate::plan::{Proposal, UnitThought}; use crate::wire::{Coord, Unit}; @@ -141,6 +152,9 @@ pub struct Order { /// Which round of the auction served this unit. 0 is the unit that had the /// most to lose by waiting. pub round: usize, + /// Whether the weights picked this or [`crate::explore`] overruled them. + /// Recorded on the row so a fit can tell the two apart. + pub policy: Policy, } impl Order { @@ -164,6 +178,8 @@ pub struct Stats { pub blocked: u64, /// Orders that differ from what the unit would have taken alone. pub moved: u64, + /// Orders where [`crate::explore`] overruled the weights. + pub explored: u64, } /// One coherent set of orders, and everything it did not take. @@ -322,6 +338,34 @@ pub fn reconcile( remaining: &BTreeMap, weights: &Weights, doctrine: &Doctrine, +) -> Reconciliation { + reconcile_exploring( + thoughts, + remaining, + weights, + doctrine, + &Explore { + epsilon: 0.0, + seed: 0, + }, + 0, + ) +} + +/// The same, sometimes taking an option the weights ranked lower. +/// +/// `game_round` is the match's round, not the auction's: it is half the key +/// [`Explore::choose`] draws on and has to mean the same thing it meant when +/// the draw happened outside this module. +/// +/// With `epsilon` at 0 this is [`reconcile`] and draws nothing. +pub fn reconcile_exploring( + thoughts: &[UnitThought], + remaining: &BTreeMap, + weights: &Weights, + doctrine: &Doctrine, + explore: &Explore, + game_round: u32, ) -> Reconciliation { // Unit id order, so the sequence the force heard from its units in cannot // reach the answer. Stable, so two menus with one id would at least be @@ -384,27 +428,41 @@ pub fn reconcile( }; let menu = pending[position]; let thought = menus[menu]; - let proposal = &thought.proposals[bid.at]; + // Overrule the weights here rather than before or after, so the draw + // can only land on a hex this unit may still have. + let (at, policy) = explored(thought, &taken, bid.at, explore, game_round); + let proposal = &thought.proposals[at]; + // Measured against the orders already given, so it has to be read + // before this proposal's own damage joins them. + let marginal = if at == bid.at { + bid.marginal + } else { + marginal(proposal, &dealt, remaining, weights, doctrine) + }; if let Some(end) = proposal.end { taken.insert(end); } for (target, damage) in &proposal.damage_by_target { *dealt.entry(*target).or_insert(0.0) += *damage; } - let alone = alone[menu].unwrap_or(bid.at); - if alone != bid.at { + let alone = alone[menu].unwrap_or(at); + if alone != at { stats.moved += 1; } + if policy == Policy::Explore { + stats.explored += 1; + } orders.push(Order { unit: thought.unit, - proposal: bid.at, + proposal: at, label: proposal.label.clone(), end: proposal.end, value: proposal.value(weights), - marginal: bid.marginal, + marginal, alone, regret: bid.regret, round, + policy, }); done.push(position); done.sort_unstable(); @@ -423,6 +481,38 @@ pub fn reconcile( } } +/// Which proposal this unit actually takes, once [`crate::explore`] has had its +/// say. +/// +/// `best` is what the auction picked. The alternatives offered are the ones +/// still open - a proposal standing in a claimed hex is not one this unit may +/// have, so it is not one exploration may draw. The key is `(seed, game round, +/// unit)` exactly as it was when this ran over the whole menu, so the decision +/// *whether* to explore is the same draw it always was. +fn explored( + thought: &UnitThought, + taken: &BTreeSet, + best: usize, + explore: &Explore, + game_round: u32, +) -> (usize, Policy) { + if explore.epsilon <= 0.0 { + return (best, Policy::Argmax); + } + let open: Vec = thought + .proposals + .iter() + .enumerate() + .filter(|(_, proposal)| !proposal.end.is_some_and(|end| taken.contains(&end))) + .map(|(at, _)| at) + .collect(); + let Some(here) = open.iter().position(|at| *at == best) else { + return (best, Policy::Argmax); + }; + let (pick, policy) = explore.choose(here, open.len(), game_round, thought.unit); + (open[pick], policy) +} + /// One unit's best remaining option, and what it would give up by waiting. /// /// `None` when every proposal it has stands in a claimed hex. @@ -821,6 +911,148 @@ mod tests { assert_eq!(doctrine.term(0.0, 5.0), 0.0); } + /// Exploration used to run after the argmax, over the unit's whole menu, + /// and could put a second unit into a hex the first had taken. This is the + /// case that found it: two units, one menu each, every stand shared. + #[test] + fn exploring_never_lands_on_a_claimed_hex() { + let build = |id: i32| { + menu( + id, + (0..6) + .map(|k| { + offer( + id, + &format!("stand {k}"), + hex(11, k), + 0.9 - k as f32 * 0.1, + &[(11, 6.0)], + ) + }) + .collect(), + ) + }; + let health = BTreeMap::from([(11, 40.0)]); + for seed in 0..40u64 { + let thoughts = vec![build(1), build(2)]; + let orders = reconcile_exploring( + &thoughts, + &health, + &weights(), + &Doctrine::default(), + &Explore { epsilon: 1.0, seed }, + 1, + ); + let hexes: BTreeSet = orders.orders.iter().filter_map(|o| o.end).collect(); + assert_eq!( + hexes.len(), + orders.orders.len(), + "seed {seed} put two units in one hex" + ); + assert_eq!(orders.orders.len(), 2, "seed {seed} lost a unit"); + assert!( + orders.orders.iter().all(|o| o.policy == Policy::Explore), + "seed {seed} did not explore at epsilon 1" + ); + } + } + + /// Exploring is still keyed on `(seed, round, unit)` and still ignores the + /// order the menus arrived in. + #[test] + fn exploring_is_reproducible_and_order_independent() { + let build = |id: i32, base: f32| { + menu( + id, + (0..5) + .map(|k| { + offer( + id, + &format!("{id}-{k}"), + hex(k, id % 2), + base + k as f32 * 0.05, + &[(11, 5.0 + k as f32)], + ) + }) + .collect(), + ) + }; + let health = BTreeMap::from([(11, 30.0)]); + let explore = Explore { + epsilon: 0.5, + seed: 17, + }; + let forward = vec![build(1, 0.2), build(2, 0.3), build(3, 0.25)]; + let mut backward = forward.clone(); + backward.reverse(); + + let one = reconcile_exploring( + &forward, + &health, + &weights(), + &Doctrine::default(), + &explore, + 4, + ); + let two = reconcile_exploring( + &backward, + &health, + &weights(), + &Doctrine::default(), + &explore, + 4, + ); + assert_eq!(one.orders, two.orders); + assert_eq!(one.stats, two.stats); + // A different match round is a different key, so the draws move. + let later = reconcile_exploring( + &forward, + &health, + &weights(), + &Doctrine::default(), + &explore, + 5, + ); + assert_ne!(later.picks(), one.picks(), "the round is not in the key"); + } + + /// With exploration off this is the plain reconciliation, draw for draw. + #[test] + fn zero_epsilon_changes_nothing() { + let thoughts = vec![ + menu( + 1, + vec![ + offer(1, "a", hex(4, 4), 0.9, &[(11, 10.0)]), + offer(1, "b", hex(4, 5), 0.5, &[(11, 6.0)]), + ], + ), + menu( + 2, + vec![ + offer(2, "c", hex(4, 4), 0.8, &[(11, 9.0)]), + offer(2, "d", hex(6, 5), 0.4, &[(11, 5.0)]), + ], + ), + ]; + let health = BTreeMap::from([(11, 40.0)]); + let plain = reconcile(&thoughts, &health, &weights(), &Doctrine::default()); + let off = reconcile_exploring( + &thoughts, + &health, + &weights(), + &Doctrine::default(), + &Explore { + epsilon: 0.0, + seed: 99, + }, + 3, + ); + assert_eq!(plain, off); + assert!(plain.orders.iter().all(|o| o.policy == Policy::Argmax)); + assert_eq!(plain.stats.explored, 0); + } + /// The assignment is what the force-level features are measured over. #[test] fn the_orders_read_as_a_joint_assignment() { diff --git a/crates/sds-node/src/lib.rs b/crates/sds-node/src/lib.rs index cb06fe9..71d69ef 100644 --- a/crates/sds-node/src/lib.rs +++ b/crates/sds-node/src/lib.rs @@ -70,6 +70,16 @@ pub enum Request { stance: Stance, grit: Grit, round: i32, + /// What each enemy has left, by unit id: armour plus structure. The + /// denominator `sds_core::reconcile` normalises its target terms by, + /// and the reason two units can decide that one of them finishes a + /// machine rather than both emptying into it. + /// + /// Empty when the caller did not say. A target the map does not name + /// is left out of the coordination terms rather than assumed healthy: + /// an unknown denominator is not a large one. + #[serde(default)] + remaining: std::collections::BTreeMap, }, } diff --git a/plan/hierarchy.md b/plan/hierarchy.md index 1980de0..bcac68a 100644 --- a/plan/hierarchy.md +++ b/plan/hierarchy.md @@ -23,6 +23,8 @@ Built already: two-level planning, the barrier, stance/grit, node transports `sds-core/src/reconcile.rs`. Sequential greedy with max regret, one hex to one unit, every proposal still in the record. See "Reconciliation" below +- [x] `ForceThinker::command` calls it, exploration draws inside it, and a + unit it cannot place is counted unanswered - [ ] Company level does something with more than one lance beyond passing a stance down - [ ] Two-pass planning: cheap appraisal first so the force can pick a stance, @@ -92,5 +94,30 @@ at 100 candidates evaluate 500 and 1,000 proposals, which a test pins. About 0.6 us a proposal, against 0.27 ms a proposal to generate one - the reconciliation is under 1% of the turn. -**Nothing in `sds-bot` calls it.** `ForceThinker::command` still takes an -independent argmax, play is unchanged, and no control run applies. +### Wired in + +`ForceThinker::command` calls it. The independent argmax is gone, so the +force's selection changes every match, flagged or not. + +What it fixes, from a real match: round 1, two of our units both routed to +(11, 2). The reachability sweep blocks on where units *are* and nothing +deconflicted where they were *going*. Two orders can no longer name one hex. + +`Request::Command` carries `remaining` - what each enemy has left, armour plus +structure - so the target terms have a denominator. A target the map does not +name is left out of them rather than assumed healthy. + +Exploration moved inside. `sds-core/src/explore.rs` sometimes takes a candidate +the weights ranked lower so the corpus can teach which choices were good; run +after the reconciliation that draw can land on a claimed hex and put the +collision back. `reconcile_exploring` draws at the moment a unit is served, +over the options still open to it. The key is unchanged - `(seed, round, unit)` +- so *whether* a unit explores is the same draw it always was, and only the set +it lands in is narrowed to what the unit may actually have. + +A unit whose every stand was in a claimed hex gets no order. It passes, and the +host counts a pass as `defaulted`: a stand-still nobody chose must not be +reported as an answer. + +No control run. jmm dropped it: this branch goes straight to a bench against +Princess with the hand-authored weights.