diff --git a/CLAUDE.md b/CLAUDE.md index d19fb2c..1ea10ef 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,10 +37,10 @@ Most of the suite is ordinary `cargo test`. Three things are not: `cargo build -p helm-wasm --target wasm32-unknown-unknown --release`. - `crates/helm-wasm/facets.mjs` is the filtering half of the same question: `helm index --megamek --bridge-dir --out /units.json - --filters crates/helm-wasm/filters.json` writes the index a page filters, - the loadout beside it and what this build selects from the two, and - `node crates/helm-wasm/facets.mjs /units.json /loadout.json - /answers.json` re-runs every filter through wasm and fails on any + --filters crates/helm-wasm/filters.json` writes the spine, every chunk and + what this build selects from them, and `node crates/helm-wasm/facets.mjs + /units.json /answers.json` re-runs every filter through wasm - + fetching the chunks each one reads, as a page does - and fails on any disagreement. About a second. - `crates/helm-wasm/wire.mjs` is the same boundary over the whole library rather than one design: `helm wire-check --megamek --bridge-dir @@ -61,10 +61,15 @@ and puts them in the assets bucket under `//`, then writes `releases.helm` in infra. It applies nothing — the apply is infra's, and jmm runs it. -`units.json` and `loadout.json` carry the same `build` block and are joined by +The index is a spine and six chunks - `art`, `figures`, `paperwork`, `combat`, +`quirks`, `loadout` - and `units.json` names them all in `chunks`. One command +writes the whole set: a prefix holding a spine and half its columns would have +a page filter on a library that quietly answers a narrower question. + +Every one of those files carries the same `build` block and they are joined by position, so a page that mixes two prefixes is refused rather than answered. -`catalogue.jsonl` has no identity of its own and is keyed only by the prefix -it sits under. +`catalogue.jsonl` has no identity of its own and is keyed only by the prefix it +sits under. - A prefix is written once. Never overwrite one, and never reach for a CloudFront invalidation: a rebuild is a new ref, which is what makes the diff --git a/Cargo.lock b/Cargo.lock index 3acee67..5a26c2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -349,6 +349,7 @@ dependencies = [ "helm-query", "helm-unitfile", "rusqlite", + "serde", "serde_json", ] diff --git a/TODO.md b/TODO.md index f8f7611..0364be3 100644 --- a/TODO.md +++ b/TODO.md @@ -83,11 +83,14 @@ to helm. in `helm-query`, which helm-mcp and the boundary both read. What is left is headquarters': `web/src/units.ts` still has a filter table of its own, and that is the second implementation the epic warns about. -15. **Drop the index's duplicated header.** `units.json` writes `megamek` and - `helm` at the top level as well as inside `build`, because - headquarters-api refuses an index it cannot find a top-level `megamek` in - and falls back to silhouettes. Delete both once - `services/api/src/units.rs` reads `build`. +15. **headquarters-api reads the spine and the art chunk.** That service + deserialises `units.json` as `{megamek, sprite_base, units: [{name, + sprite}]}` and warns "no unit carries a sprite" when it cannot, falling + back to silhouettes for every design. `sprite` now lives in `art.json`, + and `megamek` is written at the top level as well as inside `build` only + until that service reads both. Both are helm's to publish and + headquarters' to read: the change is `services/api/src/units.rs`, and this + entry is here so it is not discovered by a deploy. 16. **An identity for the catalogue.** `units.json` and `loadout.json` carry a `build` block and refuse to be joined across runs; `catalogue.jsonl` is JSON Lines with nowhere to put one, and is keyed only by the prefix it is diff --git a/crates/helm-cli/Cargo.toml b/crates/helm-cli/Cargo.toml index 020820c..6b493f2 100644 --- a/crates/helm-cli/Cargo.toml +++ b/crates/helm-cli/Cargo.toml @@ -14,6 +14,7 @@ path = "src/main.rs" helm-core.workspace = true helm-unitfile.workspace = true helm-bridge.workspace = true +serde.workspace = true serde_json.workspace = true helm-bv.workspace = true helm-facet.workspace = true diff --git a/crates/helm-cli/src/build.rs b/crates/helm-cli/src/build.rs index 037f87e..3f1d033 100644 --- a/crates/helm-cli/src/build.rs +++ b/crates/helm-cli/src/build.rs @@ -421,60 +421,46 @@ pub(crate) fn index(args: &[String]) -> Result<(), String> { units: units.len(), ..Default::default() }; - let mut rows = Vec::with_capacity(units.len()); - for unit in &units { - let stats = by_name.get(&unit.name); - let sprite = set.art_for(unit).map(|art| art.path.to_string()); - if sprite.is_none() { - counts.without_sprite += 1; - } - if stats.and_then(|s| s.battle_value).is_none() { - counts.without_battle_value += 1; - } - rows.push(helm_query::IndexRow { - facets: helm_query::Row(helm_facet::UnitFacets::from_unit( - unit, - stats, - catalogue.as_ref(), - )), - sprite, - total_armor: Some(unit.total_armor()), - }); - } + let rows: Vec<(helm_facet::UnitFacets, helm_query::Extras)> = units + .iter() + .map(|unit| { + let stats = by_name.get(&unit.name); + let sprite = set.art_for(unit).map(|art| art.path.to_string()); + if sprite.is_none() { + counts.without_sprite += 1; + } + if stats.and_then(|s| s.battle_value).is_none() { + counts.without_battle_value += 1; + } + ( + helm_facet::UnitFacets::from_unit(unit, stats, catalogue.as_ref()), + helm_query::Extras { + sprite, + total_armor: Some(unit.total_armor()), + }, + ) + }) + .collect(); - let mut document = helm_query::Index { + let document = helm_query::Index { megamek: build.megamek.clone(), helm: build.helm.clone(), build: build.clone(), sprite_base: "data/images/units".to_string(), counts, + chunks: helm_query::Chunk::ALL.iter().map(|c| c.file()).collect(), units: rows, }; - // Answered while the two halves are still one, so what is recorded is - // what a page gets after it has joined them. + // Answered against the whole thing, so what is recorded is what a page + // gets once it has joined every chunk it needs. let answers = match &opts.filters { Some(corpus) => Some(answer(&document, corpus)?), None => None, }; - // Moved out of the rows rather than copied beside them: two files holding - // the same list is two chances for one of them to be stale. - let carried: Vec> = document - .units - .iter_mut() - .map(|row| std::mem::take(&mut row.facets.0.loadout)) - .collect(); - - let text = serde_json::to_string(&document).map_err(|e| e.to_string())?; - std::fs::write(&out, format!("{text}\n")).map_err(|e| format!("{}: {e}", out.display()))?; - println!( - "wrote {} - {} units, {:.0} KB, build {}", - out.display(), - document.units.len(), - text.len() as f64 / 1024.0, - build.id - ); + write_json(&out, &document, "units")?; + println!(" build {}", build.id); if document.counts.without_battle_value > 0 || document.counts.without_sprite > 0 { println!( " {} without a battle value, {} without art", @@ -482,20 +468,42 @@ pub(crate) fn index(args: &[String]) -> Result<(), String> { ); } - let beside = out.with_file_name("loadout.json"); - let equipment = helm_query::Loadout { - build, - loadout: carried, - }; - let text = serde_json::to_string(&equipment).map_err(|e| e.to_string())?; - std::fs::write(&beside, format!("{text}\n")) - .map_err(|e| format!("{}: {e}", beside.display()))?; - println!( - "wrote {} - {} entries, {:.0} KB", - beside.display(), - equipment.loadout.iter().map(Vec::len).sum::(), - text.len() as f64 / 1024.0 - ); + // Every chunk, every time. A prefix holding a spine and half its columns + // is worse than one holding neither: a page would fetch what is there and + // filter on a library that quietly answers a narrower question. + for chunk in helm_query::Chunk::ALL { + let beside = out.with_file_name(chunk.file()); + match chunk { + helm_query::Chunk::Art => { + write_json(&beside, &document.part::(), chunk.name()) + } + helm_query::Chunk::Figures => write_json( + &beside, + &document.part::(), + chunk.name(), + ), + helm_query::Chunk::Paperwork => write_json( + &beside, + &document.part::(), + chunk.name(), + ), + helm_query::Chunk::Combat => write_json( + &beside, + &document.part::(), + chunk.name(), + ), + helm_query::Chunk::Quirks => write_json( + &beside, + &document.part::(), + chunk.name(), + ), + helm_query::Chunk::Loadout => write_json( + &beside, + &document.part::(), + chunk.name(), + ), + }?; + } if let Some(answers) = answers { let beside = out.with_file_name("answers.json"); @@ -506,6 +514,18 @@ pub(crate) fn index(args: &[String]) -> Result<(), String> { Ok(()) } +/// Write one document and say what it cost. +fn write_json(path: &Path, value: &T, what: &str) -> Result<(), String> { + let text = serde_json::to_string(value).map_err(|e| e.to_string())?; + std::fs::write(path, format!("{text}\n")).map_err(|e| format!("{}: {e}", path.display()))?; + println!( + "wrote {} - {what}, {:.0} KB", + path.display(), + text.len() as f64 / 1024.0 + ); + Ok(()) +} + /// What every filter in a corpus selects, as this build answers it. /// /// The other half of `crates/helm-wasm/facets.mjs`: the script runs the same diff --git a/crates/helm-query/src/facets.rs b/crates/helm-query/src/facets.rs index 40a0448..8041f3b 100644 --- a/crates/helm-query/src/facets.rs +++ b/crates/helm-query/src/facets.rs @@ -1,116 +1,352 @@ //! The units themselves, as they travel. //! //! [`helm_facet::UnitFacets`] is what a predicate reads, and it is built in -//! three places already: from a parsed design, from a database row, and - once -//! a browser is doing the filtering - from a document fetched over HTTP. +//! three places: from a parsed design, from a database row, and - once a +//! browser is doing the filtering - from the documents helm publishes. //! -//! serde's remote derive is what writes that third one. The mirror below is -//! field for field against the real struct and the compiler says so: add a -//! facet and forget this file, and helm-query stops building rather than -//! quietly shipping a browser that filters on one fewer column than the server -//! does. +//! # Why it is more than one document //! -//! Absent is the common case - two thirds of the columns are `None` for a -//! design with no computed record - so every optional field is skipped when it -//! is empty and defaulted when it is missing. +//! A screen that draws a match report wants a picture and a name. A force +//! builder wants every column there is. A tournament wants the year, the tech +//! base and the tonnage. Published as one file, the first of those pays 382KB +//! to answer a question worth 60. +//! +//! So the index is a spine and a set of chunks. The spine is what identifies a +//! design - the columns nothing can be said without - and each chunk is one +//! group of columns, in the spine's order, as an array with no keys in it. +//! +//! # Why the chunks carry no names +//! +//! Because names are the expensive column. `chassis` and `model` are long +//! strings that gzip cannot do much with: 44KB of the spine's 98KB is names +//! alone. A chunk that repeated them would cost more in keys than it carries +//! in values - art is 23KB as an array and 64KB as a map - and every chunk +//! after the first would pay that toll again. +//! +//! The price is that a chunk is meaningless without the spine, and that a +//! chunk from another run of helm lines up row for row and means the wrong +//! designs. [`Build`] is what makes that a refusal rather than a wrong answer, +//! and one command writes the whole set so there is never half of one. use crate::Build; use helm_facet::UnitFacets; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; + +/// One group of columns, published as its own file. +/// +/// The set is closed and the compiler enforces it: every `match` over a chunk +/// is exhaustive, so a group added here fails to build until the writer writes +/// it and the reader reads it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Chunk { + /// Which picture each design is drawn with. + Art, + /// What it weighs, what it is worth, and what it costs. + Figures, + /// Where it comes from: the year, the tech base, the books. + Paperwork, + /// How it fights: movement, firepower, reach, heat, plate. + Combat, + /// MegaMek's per-design quirks. + Quirks, + /// What it carries, as MegaMek lists it. + Loadout, +} + +impl Chunk { + /// Every chunk, in the order one command writes them. + pub const ALL: [Chunk; 6] = [ + Chunk::Art, + Chunk::Figures, + Chunk::Paperwork, + Chunk::Combat, + Chunk::Quirks, + Chunk::Loadout, + ]; + + /// What it is called, in a filter's `needs` and in the spine's manifest. + pub fn name(self) -> &'static str { + match self { + Chunk::Art => "art", + Chunk::Figures => "figures", + Chunk::Paperwork => "paperwork", + Chunk::Combat => "combat", + Chunk::Quirks => "quirks", + Chunk::Loadout => "loadout", + } + } + + /// The file it is published as, beside the spine. + pub fn file(self) -> String { + format!("{}.json", self.name()) + } + + /// The chunk of that name, for a caller reading one back. + pub fn named(name: &str) -> Option { + Chunk::ALL.into_iter().find(|c| c.name() == name) + } +} + +/// One group of columns for every design, in the spine's order. +/// +/// `chunk` is written so that a file cannot be mistaken for another chunk of +/// the same shape - art and quirks are both one value per design, and reading +/// one as the other would be silent. +#[derive(Serialize, Deserialize)] +pub struct Part { + pub build: Build, + pub chunk: Chunk, + pub rows: Vec, +} + +/// One group of columns, projected out of a unit and put back onto one. +/// +/// Every facet belongs to exactly one chunk. Nothing here enforces that on its +/// own; `every_column_survives_the_split` does, and it cannot be written +/// without naming all of them. +pub trait Columns: Serialize + DeserializeOwned + Sized { + /// Which chunk these columns are published in. + const CHUNK: Chunk; -/// The wire form of one unit's facets. + /// Read them off a unit. + fn take(unit: &UnitFacets, extra: &Extras) -> Self; + + /// Put them back onto one. + fn put(self, unit: &mut UnitFacets, extra: &mut Extras); +} + +/// The two columns a screen draws that no predicate reads. /// -/// Never constructed directly: it exists so serde can read and write a -/// `UnitFacets`, which belongs to a crate that does not know what serde is. -#[derive(Default, Serialize, Deserialize)] -#[serde(remote = "UnitFacets", default, deny_unknown_fields)] -struct UnitFacetsDef { +/// They travel with the facets rather than beside them because they are per +/// design and positional like everything else, and a third parallel structure +/// for two fields would be worse than a struct with two fields in it. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Extras { + /// Where the design's picture is, relative to the release's art tree. + pub sprite: Option, + /// Plate as the file declares it. `armor_pct` is what a filter asks + /// about; this is what a row prints. + pub total_armor: Option, +} + +/// The columns nothing can be said without: what this design is. +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Spine { #[serde(skip_serializing_if = "String::is_empty")] - name: String, + pub name: String, #[serde(skip_serializing_if = "String::is_empty")] - chassis: String, + pub chassis: String, #[serde(skip_serializing_if = "String::is_empty")] - model: String, + pub model: String, #[serde(skip_serializing_if = "Option::is_none")] - unit_type: Option, + pub unit_type: Option, #[serde(skip_serializing_if = "Option::is_none")] - tech_base: Option, + pub config: Option, #[serde(skip_serializing_if = "Option::is_none")] - role: Option, + pub mul_id: Option, +} + +/// Which picture each design is drawn with. +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Art(pub Option); + +impl Columns for Art { + const CHUNK: Chunk = Chunk::Art; + + fn take(_unit: &UnitFacets, extra: &Extras) -> Self { + Art(extra.sprite.clone()) + } + + fn put(self, _unit: &mut UnitFacets, extra: &mut Extras) { + extra.sprite = self.0; + } +} + +/// What it weighs, what it is worth, and what it costs. +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Figures { + #[serde(rename = "mass", skip_serializing_if = "Option::is_none")] + pub tons: Option, + #[serde(rename = "bv", skip_serializing_if = "Option::is_none")] + pub battle_value: Option, #[serde(skip_serializing_if = "Option::is_none")] - source: Option, + pub cost: Option, #[serde(skip_serializing_if = "Option::is_none")] - weight_class: Option, + pub weight_class: Option, #[serde(skip_serializing_if = "Option::is_none")] - year: Option, - #[serde(rename = "mass", skip_serializing_if = "Option::is_none")] - tons: Option, - #[serde(rename = "bv", skip_serializing_if = "Option::is_none")] - battle_value: Option, + pub total_armor: Option, #[serde(skip_serializing_if = "Option::is_none")] - walk_mp: Option, + pub canon: Option, #[serde(skip_serializing_if = "Option::is_none")] - jump_mp: Option, + pub invalid: Option, #[serde(skip_serializing_if = "Option::is_none")] - canon: Option, + pub omni: Option, #[serde(skip_serializing_if = "Option::is_none")] - invalid: Option, + pub clan: Option, +} + +impl Columns for Figures { + const CHUNK: Chunk = Chunk::Figures; + + fn take(unit: &UnitFacets, extra: &Extras) -> Self { + Figures { + tons: unit.tons, + battle_value: unit.battle_value, + cost: unit.cost, + weight_class: unit.weight_class, + total_armor: extra.total_armor, + canon: unit.canon, + invalid: unit.invalid, + omni: unit.omni, + clan: unit.clan, + } + } + + fn put(self, unit: &mut UnitFacets, extra: &mut Extras) { + unit.tons = self.tons; + unit.battle_value = self.battle_value; + unit.cost = self.cost; + unit.weight_class = self.weight_class; + extra.total_armor = self.total_armor; + unit.canon = self.canon; + unit.invalid = self.invalid; + unit.omni = self.omni; + unit.clan = self.clan; + } +} + +/// Where a design comes from. +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Paperwork { #[serde(skip_serializing_if = "Option::is_none")] - omni: Option, + pub year: Option, #[serde(skip_serializing_if = "Option::is_none")] - clan: Option, - #[serde(skip_serializing_if = "Vec::is_empty")] - loadout: Vec<(String, i64)>, + pub tech_base: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option, #[serde(skip_serializing_if = "Option::is_none")] - firepower: Option, + pub rules_level: Option, #[serde(skip_serializing_if = "Option::is_none")] - max_range: Option, + pub source: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub sources: Vec, +} + +impl Columns for Paperwork { + const CHUNK: Chunk = Chunk::Paperwork; + + fn take(unit: &UnitFacets, _extra: &Extras) -> Self { + Paperwork { + year: unit.year, + tech_base: unit.tech_base.clone(), + role: unit.role.clone(), + rules_level: unit.rules_level.clone(), + source: unit.source.clone(), + sources: unit.sources.clone(), + } + } + + fn put(self, unit: &mut UnitFacets, _extra: &mut Extras) { + unit.year = self.year; + unit.tech_base = self.tech_base; + unit.role = self.role; + unit.rules_level = self.rules_level; + unit.source = self.source; + unit.sources = self.sources; + } +} + +/// How a design fights. +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Combat { #[serde(skip_serializing_if = "Option::is_none")] - heat: Option, + pub walk_mp: Option, #[serde(skip_serializing_if = "Option::is_none")] - dissipation: Option, + pub run_mp: Option, #[serde(skip_serializing_if = "Option::is_none")] - heat_efficiency: Option, + pub jump_mp: Option, #[serde(skip_serializing_if = "Option::is_none")] - armor_pct: Option, + pub firepower: Option, #[serde(skip_serializing_if = "Option::is_none")] - rules_level: Option, + pub max_range: Option, #[serde(skip_serializing_if = "Option::is_none")] - cost: Option, + pub heat: Option, #[serde(skip_serializing_if = "Option::is_none")] - config: Option, + pub dissipation: Option, #[serde(skip_serializing_if = "Option::is_none")] - run_mp: Option, + pub heat_efficiency: Option, #[serde(skip_serializing_if = "Option::is_none")] - mul_id: Option, - #[serde(skip_serializing_if = "Vec::is_empty")] - quirks: Vec, - #[serde(skip_serializing_if = "Vec::is_empty")] - sources: Vec, + pub armor_pct: Option, } -/// One unit, in the document. -#[derive(Debug, Serialize, Deserialize)] -pub struct Row(#[serde(with = "UnitFacetsDef")] pub UnitFacets); +impl Columns for Combat { + const CHUNK: Chunk = Chunk::Combat; -/// One design as the index carries it: what a filter reads, and the two -/// things only a screen wants. -/// -/// The facets are flattened rather than nested because this file is read by a -/// page that draws a list as well as by one that filters it, and a row of -/// twenty-five columns with two of them behind a `facets` key is a shape -/// nobody would choose to write against. -#[derive(Debug, Serialize, Deserialize)] -pub struct IndexRow { - #[serde(flatten)] - pub facets: Row, - /// Where the design's picture is, relative to the release's art tree. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub sprite: Option, - /// Plate as the file declares it. `armor_pct` is what a filter asks - /// about; this is what a row prints. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub total_armor: Option, + fn take(unit: &UnitFacets, _extra: &Extras) -> Self { + Combat { + walk_mp: unit.walk_mp, + run_mp: unit.run_mp, + jump_mp: unit.jump_mp, + firepower: unit.firepower, + max_range: unit.max_range, + heat: unit.heat, + dissipation: unit.dissipation, + heat_efficiency: unit.heat_efficiency, + armor_pct: unit.armor_pct, + } + } + + fn put(self, unit: &mut UnitFacets, _extra: &mut Extras) { + unit.walk_mp = self.walk_mp; + unit.run_mp = self.run_mp; + unit.jump_mp = self.jump_mp; + unit.firepower = self.firepower; + unit.max_range = self.max_range; + unit.heat = self.heat; + unit.dissipation = self.dissipation; + unit.heat_efficiency = self.heat_efficiency; + unit.armor_pct = self.armor_pct; + } +} + +/// MegaMek's per-design quirks. +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Quirks(pub Vec); + +impl Columns for Quirks { + const CHUNK: Chunk = Chunk::Quirks; + + fn take(unit: &UnitFacets, _extra: &Extras) -> Self { + Quirks(unit.quirks.clone()) + } + + fn put(self, unit: &mut UnitFacets, _extra: &mut Extras) { + unit.quirks = self.0; + } +} + +/// What each design carries, as MegaMek lists it. +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Loadout(pub Vec<(String, i64)>); + +impl Columns for Loadout { + const CHUNK: Chunk = Chunk::Loadout; + + fn take(unit: &UnitFacets, _extra: &Extras) -> Self { + Loadout(unit.loadout.clone()) + } + + fn put(self, unit: &mut UnitFacets, _extra: &mut Extras) { + unit.loadout = self.0; + } } /// How much of the library is answered, which is not the same as how large it @@ -125,15 +361,7 @@ pub struct Counts { pub without_sprite: usize, } -/// Every unit a page draws or filters, and which build wrote them. -/// -/// One document rather than two. An index of what to show and an index of -/// what to filter on are ninety percent the same columns, and publishing both -/// means a page fetches a megabyte twice to answer one screen. -/// -/// What each design *carries* is a document of its own, [`Loadout`], because -/// a screen filtering by tonnage and year needs none of it and it is a -/// quarter of the bytes. +/// The spine: which designs there are, and where the rest of them is. #[derive(Serialize, Deserialize)] pub struct Index { pub build: Build, @@ -153,55 +381,90 @@ pub struct Index { /// a consumer needs no second convention to turn one into an address. pub sprite_base: String, pub counts: Counts, - pub units: Vec, + /// The files published beside this one, so a reader discovers them rather + /// than being told about them by whoever wrote its fetching code. + pub chunks: Vec, + /// The designs, in the order every chunk repeats. + #[serde(with = "spine")] + pub units: Vec<(UnitFacets, Extras)>, } -impl Index { - /// The units, as the predicate wants them. - pub fn facets(&self) -> Vec<&UnitFacets> { - self.units.iter().map(|row| &row.facets.0).collect() +/// Only the spine's own columns cross, which is what makes a chunk worth +/// having: the rest of each unit is filled in when its chunk arrives. +mod spine { + use super::{Extras, Spine, UnitFacets}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub fn serialize( + units: &[(UnitFacets, Extras)], + out: S, + ) -> Result { + let rows: Vec = units + .iter() + .map(|(unit, _)| Spine { + name: unit.name.clone(), + chassis: unit.chassis.clone(), + model: unit.model.clone(), + unit_type: unit.unit_type.clone(), + config: unit.config.clone(), + mul_id: unit.mul_id, + }) + .collect(); + rows.serialize(out) } -} -/// What each design carries, in the order [`Facets`] wrote them. -/// -/// Joined by position and checked by [`Build::joins`]. The alternative was a -/// key per row, and there is not one: 479 designs in 0.51.0 share a display -/// name with another, and nothing in a `.mtf` distinguishes them. -/// -/// Kept apart from the facets because it is the expensive half - 60,881 -/// entries over 8,666 designs - and because most filtering never asks. A page -/// fetches it when somebody first filters by equipment, and until then the -/// question it answers is not on the screen. -#[derive(Serialize, Deserialize)] -pub struct Loadout { - pub build: Build, - /// One list per design: `(equipment name, quantity)`, MegaMek's own - /// spelling, invented entries included. - pub loadout: Vec>, + pub fn deserialize<'de, D: Deserializer<'de>>( + input: D, + ) -> Result, D::Error> { + Ok(Vec::::deserialize(input)? + .into_iter() + .map(|row| { + ( + UnitFacets { + name: row.name, + chassis: row.chassis, + model: row.model, + unit_type: row.unit_type, + config: row.config, + mul_id: row.mul_id, + ..Default::default() + }, + Extras::default(), + ) + }) + .collect()) + } } -/// Why a loadout document could not be put onto a set of facets. +/// Why a chunk could not be put onto a spine. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Mismatch { /// The two files are about different runs of helm. DifferentBuild { index: Box, - loadout: Box, + chunk: Box, }, /// The same run, and one of the files is truncated. - DifferentLength { index: usize, loadout: usize }, + DifferentLength { index: usize, chunk: usize }, + /// A file of the right shape and the wrong contents: art read as quirks. + DifferentChunk { wanted: Chunk, got: Chunk }, } impl std::fmt::Display for Mismatch { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Mismatch::DifferentBuild { index, loadout } => { - write!(f, "this index is {index} and that loadout is {loadout}") + Mismatch::DifferentBuild { index, chunk } => { + write!(f, "this index is {index} and that chunk is {chunk}") } - Mismatch::DifferentLength { index, loadout } => write!( + Mismatch::DifferentLength { index, chunk } => write!( + f, + "{index} designs are indexed and {chunk} are in the chunk" + ), + Mismatch::DifferentChunk { wanted, got } => write!( f, - "{index} designs are indexed and {loadout} have a loadout" + "that is the {} chunk, not the {}", + got.name(), + wanted.name() ), } } @@ -210,27 +473,56 @@ impl std::fmt::Display for Mismatch { impl std::error::Error for Mismatch {} impl Index { - /// Put each design's equipment onto it, or say why the two files are not - /// about the same designs. + /// The units, as the predicate wants them. + pub fn facets(&self) -> Vec<&UnitFacets> { + self.units.iter().map(|(unit, _)| unit).collect() + } + + /// What a screen draws that no predicate reads. + pub fn extras(&self) -> Vec<&Extras> { + self.units.iter().map(|(_, extra)| extra).collect() + } + + /// The chunk this run published, for one group of columns. + pub fn part(&self) -> Part { + Part { + build: self.build.clone(), + chunk: C::CHUNK, + rows: self + .units + .iter() + .map(|(unit, extra)| C::take(unit, extra)) + .collect(), + } + } + + /// Put one group of columns onto the designs it belongs to, or say why the + /// two files are not about the same run. /// /// A positional join is right or it is nonsense: nothing about the fifth /// row of one file says whether it is the fifth design of the other. So /// this is the only way the two are put together. - pub fn join(&mut self, loadout: Loadout) -> Result<(), Mismatch> { - if !self.build.joins(&loadout.build) { + pub fn join(&mut self, part: Part) -> Result<(), Mismatch> { + if part.chunk != C::CHUNK { + return Err(Mismatch::DifferentChunk { + wanted: C::CHUNK, + got: part.chunk, + }); + } + if !self.build.joins(&part.build) { return Err(Mismatch::DifferentBuild { index: Box::new(self.build.clone()), - loadout: Box::new(loadout.build), + chunk: Box::new(part.build), }); } - if self.units.len() != loadout.loadout.len() { + if self.units.len() != part.rows.len() { return Err(Mismatch::DifferentLength { index: self.units.len(), - loadout: loadout.loadout.len(), + chunk: part.rows.len(), }); } - for (row, carried) in self.units.iter_mut().zip(loadout.loadout) { - row.facets.0.loadout = carried; + for ((unit, extra), columns) in self.units.iter_mut().zip(part.rows) { + columns.put(unit, extra); } Ok(()) } @@ -250,104 +542,163 @@ mod tests { ) } - fn document(names: &[&str]) -> Index { + /// One design with every column set to something distinguishable, written + /// out in full. + /// + /// The literal is the point: it names all of them, so a facet added to + /// `UnitFacets` fails to compile here until somebody has decided which + /// chunk publishes it. That is the whole guarantee that splitting the + /// index does not quietly drop a column. + fn a_unit() -> (UnitFacets, Extras) { + ( + UnitFacets { + name: "Atlas AS7-D".into(), + chassis: "Atlas".into(), + model: "AS7-D".into(), + unit_type: Some("Mek".into()), + config: Some("Biped".into()), + mul_id: Some(87), + tech_base: Some("Inner Sphere".into()), + role: Some("Juggernaut".into()), + source: Some("TR:3039,TR:SW".into()), + sources: vec!["TR:3039".into(), "TR:SW".into()], + rules_level: Some("Standard".into()), + year: Some(2755), + weight_class: Some(4), + tons: Some(100.0), + battle_value: Some(1897), + cost: Some(9_000_000), + canon: Some(true), + invalid: Some(false), + omni: Some(false), + clan: Some(false), + walk_mp: Some(3), + run_mp: Some(5), + jump_mp: Some(0), + firepower: Some(43.5), + max_range: Some(21), + heat: Some(28), + dissipation: Some(20), + heat_efficiency: Some(-8), + armor_pct: Some(93), + quirks: vec!["battle_fists".into()], + loadout: vec![("Autocannon/20".into(), 1), ("Medium Laser".into(), 4)], + }, + Extras { + sprite: Some("meks/atlas.png".into()), + total_armor: Some(307), + }, + ) + } + + fn index(units: Vec<(UnitFacets, Extras)>) -> Index { + let names: Vec<&str> = units.iter().map(|(u, _)| u.name.as_str()).collect(); Index { - build: build(names), + build: build(&names), megamek: "0.51.0".into(), helm: None, sprite_base: "data/images/units".into(), counts: Counts::default(), - units: names - .iter() - .map(|name| IndexRow { - facets: Row(UnitFacets { - name: (*name).into(), - ..Default::default() - }), - sprite: None, - total_armor: None, - }) - .collect(), + chunks: Chunk::ALL.iter().map(|c| c.file()).collect(), + units, } } - /// The join the split exists for: the equipment arrives later, from its - /// own file, and lands on the design it belongs to. + /// Split the whole set out and put it all back: what comes out the far + /// side is what went in, column for column. #[test] - fn a_loadout_lands_on_the_design_it_belongs_to() { - let mut facets = document(&["Atlas AS7-D", "Locust LCT-1V"]); - let loadout = Loadout { - build: build(&["Atlas AS7-D", "Locust LCT-1V"]), - loadout: vec![ - vec![("ISGaussRifle".into(), 1)], - vec![("Machine Gun".into(), 2)], - ], - }; - facets.join(loadout).expect("the same designs"); - assert_eq!( - facets.units[1].facets.0.loadout, - [("Machine Gun".to_string(), 2)] + fn every_column_survives_the_split() { + let whole = index(vec![a_unit()]); + let (art, figures, paperwork, combat, quirks, loadout) = ( + whole.part::(), + whole.part::(), + whole.part::(), + whole.part::(), + whole.part::(), + whole.part::(), ); + + // Nothing but the spine crosses in the index itself. + let text = serde_json::to_string(&whole).expect("write"); + let mut read: Index = serde_json::from_str(&text).expect("read"); + assert_eq!(read.units[0].0.battle_value, None, "a chunk column crossed"); + assert_eq!(read.units[0].0.name, "Atlas AS7-D"); + + for part in [ + serde_json::to_string(&art).unwrap(), + serde_json::to_string(&figures).unwrap(), + serde_json::to_string(&paperwork).unwrap(), + serde_json::to_string(&combat).unwrap(), + serde_json::to_string(&quirks).unwrap(), + serde_json::to_string(&loadout).unwrap(), + ] { + let chunk: serde_json::Value = serde_json::from_str(&part).unwrap(); + match Chunk::named(chunk["chunk"].as_str().unwrap()).unwrap() { + Chunk::Art => read.join(serde_json::from_str::>(&part).unwrap()), + Chunk::Figures => read.join(serde_json::from_str::>(&part).unwrap()), + Chunk::Paperwork => { + read.join(serde_json::from_str::>(&part).unwrap()) + } + Chunk::Combat => read.join(serde_json::from_str::>(&part).unwrap()), + Chunk::Quirks => read.join(serde_json::from_str::>(&part).unwrap()), + Chunk::Loadout => read.join(serde_json::from_str::>(&part).unwrap()), + } + .expect("the same run"); + } + + assert_eq!(read.units[0], a_unit()); } /// Two files from different runs line up row for row and mean different /// designs. Refused, because the alternative is a page telling somebody a /// Locust carries a Gauss rifle. #[test] - fn a_loadout_from_another_run_is_refused() { - let mut facets = document(&["Atlas AS7-D", "Locust LCT-1V"]); - let loadout = Loadout { - build: build(&["Atlas AS7-D", "Banshee BNC-3E"]), - loadout: vec![vec![], vec![]], + fn a_chunk_from_another_run_is_refused() { + let mut whole = index(vec![a_unit()]); + let part = Part { + build: build(&["Locust LCT-1V"]), + chunk: Chunk::Loadout, + rows: vec![Loadout(vec![("ISGaussRifle".into(), 1)])], }; - let err = facets.join(loadout).expect_err("a different run"); + let err = whole.join(part).expect_err("a different run"); assert!(matches!(err, Mismatch::DifferentBuild { .. }), "{err}"); - assert!( - facets - .units - .iter() - .all(|row| row.facets.0.loadout.is_empty()) - ); + assert_eq!(whole.units[0].0.loadout, a_unit().0.loadout); } - /// A round trip through the document is the whole contract: what the - /// server filtered is what the browser filters. + /// Nothing about the shape of a chunk says which chunk it is - art and + /// quirks are both one value per design, and today they are told apart by + /// luck rather than by design. The name written in the file is what makes + /// that a refusal instead of a silent misreading. #[test] - fn a_unit_survives_the_document() { - let facets = UnitFacets { - name: "Atlas AS7-D".into(), - chassis: "Atlas".into(), - model: "AS7-D".into(), - battle_value: Some(1897), - tons: Some(100.0), - loadout: vec![("ISGaussRifle".into(), 1), ("Medium Laser".into(), 4)], - quirks: vec!["Battle Fists".into()], - ..Default::default() + fn a_chunk_read_as_another_is_refused() { + let mut whole = index(vec![a_unit()]); + let mislabelled = Part { + build: whole.build.clone(), + chunk: Chunk::Art, + rows: vec![Quirks(vec!["battle_fists".into()])], }; - let text = serde_json::to_string(&Row(facets.clone())).expect("write"); - let read: Row = serde_json::from_str(&text).expect("read"); - assert_eq!(read.0, facets); + let err = whole + .join(mislabelled) + .expect_err("that says it is the art"); + assert_eq!( + err.to_string(), + "that is the art chunk, not the quirks", + "{err}" + ); } - /// Absent is the common case and an empty column is not worth its name: - /// a design with no computed record writes three fields, not thirty-one. + /// Absent is the common case and an empty column is not worth its name: a + /// design with no computed record writes three fields, not thirty-one. #[test] fn an_empty_column_is_not_written() { - let text = serde_json::to_string(&Row(UnitFacets { - name: "Locust LCT-1V".into(), - ..Default::default() - })) - .expect("write"); - assert_eq!(text, r#"{"name":"Locust LCT-1V"}"#); - } - - /// A column this build does not know is a document from a newer helm, and - /// filtering it on what is understood would quietly answer a different - /// question. - #[test] - fn a_column_from_the_future_is_refused() { - let err = - serde_json::from_str::(r#"{"name":"x","spice":3}"#).expect_err("not one of ours"); - assert!(err.to_string().contains("spice"), "{err}"); + let whole = index(vec![( + UnitFacets { + name: "Locust LCT-1V".into(), + ..Default::default() + }, + Extras::default(), + )]); + let text = serde_json::to_string(&whole.part::()).unwrap(); + assert!(text.ends_with(r#""rows":[{}]}"#), "{text}"); } } diff --git a/crates/helm-query/src/lib.rs b/crates/helm-query/src/lib.rs index ea82c65..e009497 100644 --- a/crates/helm-query/src/lib.rs +++ b/crates/helm-query/src/lib.rs @@ -19,7 +19,10 @@ mod build; mod facets; pub use build::Build; -pub use facets::{Counts, Index, IndexRow, Loadout, Mismatch, Row}; +pub use facets::{ + Art, Chunk, Columns, Combat, Counts, Extras, Figures, Index, Loadout, Mismatch, Paperwork, + Part, Quirks, Spine, +}; use std::collections::BTreeSet; @@ -265,21 +268,63 @@ impl Vocabulary { } impl Filter { - /// Whether this filter asks what a design carries. + /// Which chunks of the index this filter reads. /// - /// A library with no loadout joined to it answers those with silence - - /// every design carrying nothing - which reads exactly like an honest - /// empty result. So a caller that can go and fetch the loadout is told to - /// rather than shown one. - pub fn asks_about_equipment(&self) -> bool { - [ - &self.carries, - &self.lacks, - &self.carries_class, - &self.lacks_class, - ] - .into_iter() - .any(|list| list.as_ref().is_some_and(|l| !l.is_empty())) + /// A library that has not joined one of them answers those terms with + /// silence - every design having nothing - which reads exactly like an + /// honest empty result. So a caller is told what to go and fetch instead + /// of being shown one. + /// + /// The spine's own columns are not a chunk: a library that has been loaded + /// at all has them, and art is a chunk nothing filters on. + pub fn needs(&self) -> Vec { + let some = |list: &Option>| list.as_ref().is_some_and(|l| !l.is_empty()); + let mut needs = Vec::new(); + if some(&self.carries) + || some(&self.lacks) + || some(&self.carries_class) + || some(&self.lacks_class) + { + needs.push(Chunk::Loadout); + } + if some(&self.has_quirks) { + needs.push(Chunk::Quirks); + } + if some(&self.tech_base) + || some(&self.role) + || some(&self.rules_level) + || some(&self.source) + || self.year_min.is_some() + || self.year_max.is_some() + { + needs.push(Chunk::Paperwork); + } + if some(&self.weight_class) + || self.tons_min.is_some() + || self.tons_max.is_some() + || self.bv_min.is_some() + || self.bv_max.is_some() + || self.cost_min.is_some() + || self.cost_max.is_some() + || self.omni.is_some() + { + needs.push(Chunk::Figures); + } + if self.walk_mp_min.is_some() + || self.jump_mp_min.is_some() + || self.run_mp_min.is_some() + || self.firepower_min.is_some() + || self.firepower_max.is_some() + || self.max_range_min.is_some() + || self.max_range_max.is_some() + || self.heat_efficiency_min.is_some() + || self.heat_efficiency_max.is_some() + || self.armor_pct_min.is_some() + || self.armor_pct_max.is_some() + { + needs.push(Chunk::Combat); + } + needs } /// The predicate this filter means, or the first thing in it the library @@ -477,6 +522,33 @@ mod tests { ); } + /// Every chunk a filter can read is asked for by some term, and an empty + /// filter asks for none. A term whose chunk is not named here filters on a + /// column the browser may never have fetched. + #[test] + fn every_chunk_is_asked_for_by_some_term() { + assert!(Filter::default().needs().is_empty()); + let everything = Filter { + carries: Some(vec!["ISGaussRifle".into()]), + has_quirks: Some(vec!["easy_maintain".into()]), + year_max: Some(3025), + bv_max: Some(2000), + firepower_min: Some(10), + ..Default::default() + }; + let mut asked: Vec<&str> = everything.needs().iter().map(|c| c.name()).collect(); + asked.sort_unstable(); + // Art is the exception, and the only one: it is a column a screen + // draws and no filter reads. + let mut filterable: Vec<&str> = Chunk::ALL + .iter() + .map(|c| c.name()) + .filter(|name| *name != "art") + .collect(); + filterable.sort_unstable(); + assert_eq!(asked, filterable); + } + /// A field nobody knows is a typo in a filter, and a filter that silently /// ignores one selects the whole library while looking narrow. #[test] diff --git a/crates/helm-wasm/facets.mjs b/crates/helm-wasm/facets.mjs index 4cbe516..9e2c50e 100644 --- a/crates/helm-wasm/facets.mjs +++ b/crates/helm-wasm/facets.mjs @@ -3,8 +3,7 @@ // // helm index --megamek --bridge-dir --out /units.json \ // --filters crates/helm-wasm/filters.json -// node crates/helm-wasm/facets.mjs /units.json /loadout.json \ -// /answers.json +// node crates/helm-wasm/facets.mjs /units.json /answers.json // // Why this exists: `unit-search` says one filter definition has to answer the // same way in three places - an agent over MCP, the server, and the browser - @@ -19,11 +18,12 @@ // bound, a battle value the library does not have - of which there are // several. import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; import { Helm } from "./helm.mjs"; -const [index, loadout, answers] = process.argv.slice(2); -if (!index || !loadout || !answers) { - console.error("usage: node facets.mjs "); +const [index, answers] = process.argv.slice(2); +if (!index || !answers) { + console.error("usage: node facets.mjs "); process.exit(2); } @@ -32,14 +32,19 @@ const wasm = new URL( import.meta.url, ); +// The chunks sit beside the spine, which is how a page finds them: the +// document names them rather than the fetching code knowing them. +const beside = (name) => + readFileSync(new URL(name, `file://${resolve(index)}`), "utf8"); + const helm = await Helm.load(readFileSync(wasm)); const started = Date.now(); const library = helm.library(readFileSync(index, "utf8")); const indexed = Date.now() - started; -// What a page does before anybody has filtered by equipment: a filter that -// asks what a design carries is refused rather than answered against designs -// that carry nothing. +// What a page does before it has fetched anything: a filter that reads a +// chunk which is not loaded is refused rather than answered against columns +// every design is missing. let unarmed = null; try { library.select({ carries: ["ISGaussRifle"] }); @@ -47,9 +52,21 @@ try { unarmed = error; } -const joining = Date.now(); -library.loadout(readFileSync(loadout, "utf8")); -const joined = Date.now() - joining; +// And what it does instead: ask for the chunks the filter reads, then answer. +let fetched = []; +const fetching = Date.now(); +await library.selectAsync({ carries: ["ISGaussRifle"], bv_max: 3000 }, (name) => { + fetched.push(name); + return beside(`${name}.json`); +}); +const joined = Date.now() - fetching; + +// Everything else the corpus reads, so the run below is measuring filtering +// rather than fetching. +for (const chunk of JSON.parse(readFileSync(index, "utf8")).chunks) { + const name = chunk.replace(/\.json$/, ""); + if (!fetched.includes(name)) library.chunk(beside(chunk)); +} const native = JSON.parse(readFileSync(answers, "utf8")); const wrong = []; @@ -77,14 +94,19 @@ for (const row of native) { const took = Date.now() - filtering; console.log( - `${library.size} units in ${indexed}ms, loadout joined in ${joined}ms, ` + + `${library.size} units in ${indexed}ms, ` + + `${fetched.join(" and ")} fetched on demand in ${joined}ms, ` + `${native.length} filters selecting ${selected} rows in ${took}ms, ` + `${wrong.length} disagreeing`, ); -if (!unarmed) { - console.log(" an equipment filter was answered before the loadout was joined"); +if (!unarmed?.needs?.includes("loadout")) { + console.log(" an equipment filter was answered before the loadout was loaded"); wrong.push("unjoined equipment filter"); } +if (!fetched.includes("loadout") || !fetched.includes("figures")) { + console.log(` a filter reading two chunks fetched ${fetched.join(", ") || "nothing"}`); + wrong.push("wrong chunks fetched"); +} for (const line of wrong.slice(0, 20)) { console.log(` ${line}`); } @@ -102,13 +124,13 @@ if (!refused?.valid?.length) { wrong.push("unknown role"); } -// A loadout from another run lines up row for row and means different -// designs, so it is refused rather than joined. -const forged = JSON.parse(readFileSync(loadout, "utf8")); +// A chunk from another run lines up row for row and means different designs, +// so it is refused rather than joined. +const forged = JSON.parse(beside("loadout.json")); forged.build.id = "0000000000000000"; let refusedJoin = null; try { - helm.library(readFileSync(index, "utf8")).loadout(JSON.stringify(forged)); + helm.library(readFileSync(index, "utf8")).chunk(JSON.stringify(forged)); } catch (error) { refusedJoin = error; } diff --git a/crates/helm-wasm/helm.mjs b/crates/helm-wasm/helm.mjs index 77bf22c..4caff8c 100644 --- a/crates/helm-wasm/helm.mjs +++ b/crates/helm-wasm/helm.mjs @@ -20,7 +20,7 @@ // including when the call throws. /** The boundary this wrapper was written against; the module must agree. */ -const ABI_VERSION = 4n; +const ABI_VERSION = 5n; const ERRORS = new Map([ [-1n, "no equipment catalogue is loaded"], @@ -481,19 +481,19 @@ export class Library { } /** - * Put what each design carries onto it, from the loadout document. + * Put one chunk of the index onto it. * - * Its own file because most filtering never asks, and it is a quarter of - * the bytes: fetch it the first time somebody filters by equipment. Until - * then a filter that asks what a design carries is refused rather than - * answered with silence. + * The index is a spine and a set of chunks, because most filtering never + * reads most of them. Pass a fetcher to `select` and this happens on its + * own; call it directly to load one up front - what a screen does for the + * columns it draws rather than filters on. * - * Throws if the loadout is about another run of helm. The two are joined by + * Throws if the chunk is about another run of helm. The two are joined by * position, so a mismatch is not a smaller answer - it is the wrong one. */ - loadout(loadout) { - const answer = this.#withBytes(loadout, (ptr, len) => { - const packed = this.#api.helm_library_loadout(this.#live, ptr, len); + chunk(document) { + const answer = this.#withBytes(document, (ptr, len) => { + const packed = this.#api.helm_library_chunk(this.#live, ptr, len); return packed === 0n ? null : this.#take(packed); }); if (answer !== null) { @@ -511,11 +511,13 @@ export class Library { * quirk or an equipment class it does not. */ select(filter) { - const answer = JSON.parse( - this.#withBytes(JSON.stringify(filter ?? {}), (ptr, len) => - this.#take(this.#api.helm_library_select(this.#live, ptr, len)), - ), - ); + const answer = this.#ask(filter); + if (answer.needs) { + const error = new HelmError(0n); + error.message = `this filter reads ${answer.needs.join(", ")}, which is not loaded`; + Object.assign(error, answer); + throw error; + } if (answer.error) { const error = new HelmError(0n); error.message = answer.error; @@ -525,6 +527,36 @@ export class Library { return answer.units; } + /** + * The same, fetching whatever the filter reads and this library has not got. + * + * `fetch` is given a chunk's name - "loadout", "figures" - and returns the + * document's text. Nothing is fetched twice: once a chunk is on, it stays + * on for the life of the library. + * + * One round of fetching, then the filter is asked again. If it still wants + * something, that is a chunk the fetcher would not produce, and the answer + * is an error rather than a shorter list: a filter quietly answered against + * columns nobody could load is worse than one that says so. + */ + async selectAsync(filter, fetch) { + const answer = this.#ask(filter); + if (!answer.needs) return this.select(filter); + for (const name of answer.needs) { + this.chunk(await fetch(name)); + } + return this.select(filter); + } + + /** One question across the boundary, answered as it came back. */ + #ask(filter) { + return JSON.parse( + this.#withBytes(JSON.stringify(filter ?? {}), (ptr, len) => + this.#take(this.#api.helm_library_select(this.#live, ptr, len)), + ), + ); + } + /** Let it go. Filtering afterwards throws rather than answering. */ free() { check(this.#api.helm_library_free(this.#live)); diff --git a/crates/helm-wasm/src/lib.rs b/crates/helm-wasm/src/lib.rs index 06b7b7a..0d577c1 100644 --- a/crates/helm-wasm/src/lib.rs +++ b/crates/helm-wasm/src/lib.rs @@ -481,9 +481,9 @@ fn with_force_mut(handle: i64, f: impl FnOnce(&mut helm_unitfile::Mul) -> i64) - struct Filterable { index: helm_query::Index, vocabulary: helm_query::Vocabulary, - /// Whether the loadout document has been joined on. Until it has, what - /// each design carries is not in memory at all. - carrying: bool, + /// The chunks joined on so far. Until one is, the columns it holds are + /// not in memory at all. + joined: Vec, } /// Read a library of units a filter can select from, and keep it. @@ -510,7 +510,7 @@ pub unsafe extern "C" fn helm_library_load(ptr: *const u8, len: usize) -> i64 { let held = Filterable { index, vocabulary, - carrying: false, + joined: Vec::new(), }; let handle = match libraries.iter().position(|slot| slot.is_none()) { Some(free) => { @@ -551,40 +551,64 @@ pub extern "C" fn helm_library_len(handle: i64) -> i64 { with_library(handle, |held| held.index.units.len() as i64) } -/// Put what each design carries onto a library already loaded. +/// Put one chunk of the index onto a library already loaded. /// -/// The loadout is its own document because most filtering never asks what a -/// design carries, and it is a quarter of the bytes. A page fetches it the -/// first time somebody filters by equipment. +/// The index is a spine and a set of chunks - what each design carries, what +/// it is worth, where it came from - because most filtering never reads most +/// of them, and the loadout alone is a quarter of the bytes. A page fetches a +/// chunk the first time a filter needs it, which `helm_library_select` says. /// -/// The two files are joined by position and checked by the build each carries. -/// A loadout from another run of helm lines up row for row and means different -/// designs, so it is refused with what the two builds are. +/// The chunk names itself, and the two files are joined by position and +/// checked by the build each carries: a chunk from another run of helm lines +/// up row for row and means different designs, so it is refused with what the +/// two builds are. /// /// # Safety /// `ptr` must point to `len` readable bytes. #[unsafe(no_mangle)] -pub unsafe extern "C" fn helm_library_loadout(handle: i64, ptr: *const u8, len: usize) -> i64 { +pub unsafe extern "C" fn helm_library_chunk(handle: i64, ptr: *const u8, len: usize) -> i64 { let Some(json) = (unsafe { text(ptr, len) }) else { return ERR_NOT_UTF8; }; - let Ok(loadout) = serde_json::from_str::(json) else { + // Which chunk it is, before it is read as one: every chunk is an array of + // something, and two of them are an array of the same something. + let Ok(head) = serde_json::from_str::(json) else { return ERR_UNREADABLE; }; - with_library_mut(handle, |held| match held.index.join(loadout) { - Ok(()) => { - held.carrying = true; - 0 + let Some(chunk) = head["chunk"].as_str().and_then(helm_query::Chunk::named) else { + return ERR_UNREADABLE; + }; + + with_library_mut(handle, |held| { + let joined = match chunk { + helm_query::Chunk::Art => join_part::(held, json), + helm_query::Chunk::Figures => join_part::(held, json), + helm_query::Chunk::Paperwork => join_part::(held, json), + helm_query::Chunk::Combat => join_part::(held, json), + helm_query::Chunk::Quirks => join_part::(held, json), + helm_query::Chunk::Loadout => join_part::(held, json), + }; + match joined { + Ok(()) => { + held.joined.push(chunk); + // What a filter may name can only widen once a chunk that + // holds a vocabulary column has arrived. + held.vocabulary = helm_query::Vocabulary::from_facets( + &held.index.facets().into_iter().cloned().collect::>(), + ); + 0 + } + Err(why) => give(serde_json::json!({ "error": why }).to_string()), } - Err(mismatch) => give( - serde_json::json!({ - "error": mismatch.to_string(), - }) - .to_string(), - ), }) } +/// Read one chunk and put it on, or say what was wrong with it. +fn join_part(held: &mut Filterable, json: &str) -> Result<(), String> { + let part = serde_json::from_str::>(json).map_err(|e| e.to_string())?; + held.index.join(part).map_err(|e| e.to_string()) +} + /// The units a filter selects, by their place in the library. /// /// The filter is the same document an agent sends over MCP - `bv_max`, @@ -609,14 +633,17 @@ pub unsafe extern "C" fn helm_library_select(handle: i64, ptr: *const u8, len: u Err(_) => return ERR_BAD_STATE, }; with_library(handle, |held| { - if filter.asks_about_equipment() && !held.carrying { - return give( - serde_json::json!({ - "error": "this filter asks what a design carries and no loadout is loaded", - "field": "loadout", - }) - .to_string(), - ); + // What this filter reads that is not here yet. Answering anyway would + // filter on columns every design is missing, which reads exactly like + // an honest empty result. + let missing: Vec<&str> = filter + .needs() + .into_iter() + .filter(|chunk| !held.joined.contains(chunk)) + .map(helm_query::Chunk::name) + .collect(); + if !missing.is_empty() { + return give(serde_json::json!({ "needs": missing }).to_string()); } let query = match filter.to_query(&held.vocabulary) { Ok(query) => query, @@ -693,9 +720,13 @@ fn with_library_mut(handle: i64, f: impl FnOnce(&mut Filterable) -> i64) -> i64 /// 4 splits what a design carries into its own document, joined on with /// `helm_library_loadout`. Not additive: a page on 3 handed a 4 index would /// filter by equipment against designs that carry nothing. +/// +/// 5 makes that general - the index is a spine and six chunks, joined on with +/// `helm_library_chunk` - and answers a filter that reads a chunk which is +/// not loaded with what it needs rather than refusing it. #[unsafe(no_mangle)] pub extern "C" fn helm_abi_version() -> i64 { - 4 + 5 } /// How many equipment entries are loaded, so a caller can tell an empty diff --git a/scripts/deploy.sh b/scripts/deploy.sh index de5bd17..9da6f6c 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -15,8 +15,8 @@ # Three files, under one prefix named after this commit: # # /helm_wasm.wasm the rules, compiled for a browser -# //units.json the index a force-building screen filters -# //loadout.json what each of those designs carries +# //units.json which designs there are, and where the rest is +# //.json one group of columns each, named by units.json # //catalogue.jsonl the equipment the rules read # # https://lance.blue/assets/helm///units.json @@ -182,7 +182,14 @@ mm="$(grep -o '"megamek":"[^"]*"' "$out/units.json" | head -1 | cut -d'"' -f4)" cp target/wasm32-unknown-unknown/release/helm_wasm.wasm "$out/helm_wasm.wasm" echo -for file in units.json loadout.json catalogue.jsonl helm_wasm.wasm; do +# Every chunk the index names, so a chunk added in helm is published without +# this script learning its name. +chunks="$(jq -r '.chunks[]' "$out/units.json")" +for file in units.json $chunks catalogue.jsonl helm_wasm.wasm; do + [ -f "$out/$file" ] || { + echo "deploy: units.json names $file and helm did not write it" >&2 + exit 1 + } printf ' %-16s %6s KB\n' "$file" "$(( $(wc -c <"$out/$file") / 1024 ))" done echo @@ -202,8 +209,10 @@ aws s3 cp "$out/helm_wasm.wasm" "s3://$bucket/$ref/helm_wasm.wasm" \ --content-type application/wasm --cache-control "$cache" --no-progress aws s3 cp "$out/units.json" "s3://$bucket/$ref/$mm/units.json" \ --content-type application/json --cache-control "$cache" --no-progress -aws s3 cp "$out/loadout.json" "s3://$bucket/$ref/$mm/loadout.json" \ - --content-type application/json --cache-control "$cache" --no-progress +for file in $chunks; do + aws s3 cp "$out/$file" "s3://$bucket/$ref/$mm/$file" \ + --content-type application/json --cache-control "$cache" --no-progress +done aws s3 cp "$out/catalogue.jsonl" "s3://$bucket/$ref/$mm/catalogue.jsonl" \ --content-type application/x-ndjson --cache-control "$cache" --no-progress @@ -216,7 +225,9 @@ echo echo "Published under $ref/$mm:" echo " https://lance.blue/assets/helm/$ref/helm_wasm.wasm" echo " https://lance.blue/assets/helm/$ref/$mm/units.json" -echo " https://lance.blue/assets/helm/$ref/$mm/loadout.json" +for file in $chunks; do + echo " https://lance.blue/assets/helm/$ref/$mm/$file" +done echo " https://lance.blue/assets/helm/$ref/$mm/catalogue.jsonl" echo echo "Wrote releases.helm = $ref"