diff --git a/CLAUDE.md b/CLAUDE.md
index d19fb2c..1ea10ef 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -37,10 +37,10 @@ Most of the suite is ordinary `cargo test`. Three things are not:
`cargo build -p helm-wasm --target wasm32-unknown-unknown --release`.
- `crates/helm-wasm/facets.mjs` is the filtering half of the same question:
`helm index --megamek
--bridge-dir --out /units.json
- --filters crates/helm-wasm/filters.json` writes the index a page filters,
- the loadout beside it and what this build selects from the two, and
- `node crates/helm-wasm/facets.mjs /units.json /loadout.json
- /answers.json` re-runs every filter through wasm and fails on any
+ --filters crates/helm-wasm/filters.json` writes the spine, every chunk and
+ what this build selects from them, and `node crates/helm-wasm/facets.mjs
+ /units.json /answers.json` re-runs every filter through wasm -
+ fetching the chunks each one reads, as a page does - and fails on any
disagreement. About a second.
- `crates/helm-wasm/wire.mjs` is the same boundary over the whole library
rather than one design: `helm wire-check --megamek --bridge-dir
@@ -61,10 +61,15 @@ and puts them in the assets bucket under `[//`, then writes
`releases.helm` in infra. It applies nothing — the apply is infra's, and jmm
runs it.
-`units.json` and `loadout.json` carry the same `build` block and are joined by
+The index is a spine and six chunks - `art`, `figures`, `paperwork`, `combat`,
+`quirks`, `loadout` - and `units.json` names them all in `chunks`. One command
+writes the whole set: a prefix holding a spine and half its columns would have
+a page filter on a library that quietly answers a narrower question.
+
+Every one of those files carries the same `build` block and they are joined by
position, so a page that mixes two prefixes is refused rather than answered.
-`catalogue.jsonl` has no identity of its own and is keyed only by the prefix
-it sits under.
+`catalogue.jsonl` has no identity of its own and is keyed only by the prefix it
+sits under.
- A prefix is written once. Never overwrite one, and never reach for a
CloudFront invalidation: a rebuild is a new ref, which is what makes the
diff --git a/Cargo.lock b/Cargo.lock
index 3acee67..5a26c2b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -349,6 +349,7 @@ dependencies = [
"helm-query",
"helm-unitfile",
"rusqlite",
+ "serde",
"serde_json",
]
diff --git a/TODO.md b/TODO.md
index f8f7611..0364be3 100644
--- a/TODO.md
+++ b/TODO.md
@@ -83,11 +83,14 @@ to helm.
in `helm-query`, which helm-mcp and the boundary both read. What is left
is headquarters': `web/src/units.ts` still has a filter table of its own,
and that is the second implementation the epic warns about.
-15. **Drop the index's duplicated header.** `units.json` writes `megamek` and
- `helm` at the top level as well as inside `build`, because
- headquarters-api refuses an index it cannot find a top-level `megamek` in
- and falls back to silhouettes. Delete both once
- `services/api/src/units.rs` reads `build`.
+15. **headquarters-api reads the spine and the art chunk.** That service
+ deserialises `units.json` as `{megamek, sprite_base, units: [{name,
+ sprite}]}` and warns "no unit carries a sprite" when it cannot, falling
+ back to silhouettes for every design. `sprite` now lives in `art.json`,
+ and `megamek` is written at the top level as well as inside `build` only
+ until that service reads both. Both are helm's to publish and
+ headquarters' to read: the change is `services/api/src/units.rs`, and this
+ entry is here so it is not discovered by a deploy.
16. **An identity for the catalogue.** `units.json` and `loadout.json` carry a
`build` block and refuse to be joined across runs; `catalogue.jsonl` is
JSON Lines with nowhere to put one, and is keyed only by the prefix it is
diff --git a/crates/helm-cli/Cargo.toml b/crates/helm-cli/Cargo.toml
index 020820c..6b493f2 100644
--- a/crates/helm-cli/Cargo.toml
+++ b/crates/helm-cli/Cargo.toml
@@ -14,6 +14,7 @@ path = "src/main.rs"
helm-core.workspace = true
helm-unitfile.workspace = true
helm-bridge.workspace = true
+serde.workspace = true
serde_json.workspace = true
helm-bv.workspace = true
helm-facet.workspace = true
diff --git a/crates/helm-cli/src/build.rs b/crates/helm-cli/src/build.rs
index 037f87e..3f1d033 100644
--- a/crates/helm-cli/src/build.rs
+++ b/crates/helm-cli/src/build.rs
@@ -421,60 +421,46 @@ pub(crate) fn index(args: &[String]) -> Result<(), String> {
units: units.len(),
..Default::default()
};
- let mut rows = Vec::with_capacity(units.len());
- for unit in &units {
- let stats = by_name.get(&unit.name);
- let sprite = set.art_for(unit).map(|art| art.path.to_string());
- if sprite.is_none() {
- counts.without_sprite += 1;
- }
- if stats.and_then(|s| s.battle_value).is_none() {
- counts.without_battle_value += 1;
- }
- rows.push(helm_query::IndexRow {
- facets: helm_query::Row(helm_facet::UnitFacets::from_unit(
- unit,
- stats,
- catalogue.as_ref(),
- )),
- sprite,
- total_armor: Some(unit.total_armor()),
- });
- }
+ let rows: Vec<(helm_facet::UnitFacets, helm_query::Extras)> = units
+ .iter()
+ .map(|unit| {
+ let stats = by_name.get(&unit.name);
+ let sprite = set.art_for(unit).map(|art| art.path.to_string());
+ if sprite.is_none() {
+ counts.without_sprite += 1;
+ }
+ if stats.and_then(|s| s.battle_value).is_none() {
+ counts.without_battle_value += 1;
+ }
+ (
+ helm_facet::UnitFacets::from_unit(unit, stats, catalogue.as_ref()),
+ helm_query::Extras {
+ sprite,
+ total_armor: Some(unit.total_armor()),
+ },
+ )
+ })
+ .collect();
- let mut document = helm_query::Index {
+ let document = helm_query::Index {
megamek: build.megamek.clone(),
helm: build.helm.clone(),
build: build.clone(),
sprite_base: "data/images/units".to_string(),
counts,
+ chunks: helm_query::Chunk::ALL.iter().map(|c| c.file()).collect(),
units: rows,
};
- // Answered while the two halves are still one, so what is recorded is
- // what a page gets after it has joined them.
+ // Answered against the whole thing, so what is recorded is what a page
+ // gets once it has joined every chunk it needs.
let answers = match &opts.filters {
Some(corpus) => Some(answer(&document, corpus)?),
None => None,
};
- // Moved out of the rows rather than copied beside them: two files holding
- // the same list is two chances for one of them to be stale.
- let carried: Vec> = document
- .units
- .iter_mut()
- .map(|row| std::mem::take(&mut row.facets.0.loadout))
- .collect();
-
- let text = serde_json::to_string(&document).map_err(|e| e.to_string())?;
- std::fs::write(&out, format!("{text}\n")).map_err(|e| format!("{}: {e}", out.display()))?;
- println!(
- "wrote {} - {} units, {:.0} KB, build {}",
- out.display(),
- document.units.len(),
- text.len() as f64 / 1024.0,
- build.id
- );
+ write_json(&out, &document, "units")?;
+ println!(" build {}", build.id);
if document.counts.without_battle_value > 0 || document.counts.without_sprite > 0 {
println!(
" {} without a battle value, {} without art",
@@ -482,20 +468,42 @@ pub(crate) fn index(args: &[String]) -> Result<(), String> {
);
}
- let beside = out.with_file_name("loadout.json");
- let equipment = helm_query::Loadout {
- build,
- loadout: carried,
- };
- let text = serde_json::to_string(&equipment).map_err(|e| e.to_string())?;
- std::fs::write(&beside, format!("{text}\n"))
- .map_err(|e| format!("{}: {e}", beside.display()))?;
- println!(
- "wrote {} - {} entries, {:.0} KB",
- beside.display(),
- equipment.loadout.iter().map(Vec::len).sum::(),
- text.len() as f64 / 1024.0
- );
+ // Every chunk, every time. A prefix holding a spine and half its columns
+ // is worse than one holding neither: a page would fetch what is there and
+ // filter on a library that quietly answers a narrower question.
+ for chunk in helm_query::Chunk::ALL {
+ let beside = out.with_file_name(chunk.file());
+ match chunk {
+ helm_query::Chunk::Art => {
+ write_json(&beside, &document.part::(), chunk.name())
+ }
+ helm_query::Chunk::Figures => write_json(
+ &beside,
+ &document.part::(),
+ chunk.name(),
+ ),
+ helm_query::Chunk::Paperwork => write_json(
+ &beside,
+ &document.part::(),
+ chunk.name(),
+ ),
+ helm_query::Chunk::Combat => write_json(
+ &beside,
+ &document.part::(),
+ chunk.name(),
+ ),
+ helm_query::Chunk::Quirks => write_json(
+ &beside,
+ &document.part::(),
+ chunk.name(),
+ ),
+ helm_query::Chunk::Loadout => write_json(
+ &beside,
+ &document.part::(),
+ chunk.name(),
+ ),
+ }?;
+ }
if let Some(answers) = answers {
let beside = out.with_file_name("answers.json");
@@ -506,6 +514,18 @@ pub(crate) fn index(args: &[String]) -> Result<(), String> {
Ok(())
}
+/// Write one document and say what it cost.
+fn write_json(path: &Path, value: &T, what: &str) -> Result<(), String> {
+ let text = serde_json::to_string(value).map_err(|e| e.to_string())?;
+ std::fs::write(path, format!("{text}\n")).map_err(|e| format!("{}: {e}", path.display()))?;
+ println!(
+ "wrote {} - {what}, {:.0} KB",
+ path.display(),
+ text.len() as f64 / 1024.0
+ );
+ Ok(())
+}
+
/// What every filter in a corpus selects, as this build answers it.
///
/// The other half of `crates/helm-wasm/facets.mjs`: the script runs the same
diff --git a/crates/helm-query/src/facets.rs b/crates/helm-query/src/facets.rs
index 40a0448..8041f3b 100644
--- a/crates/helm-query/src/facets.rs
+++ b/crates/helm-query/src/facets.rs
@@ -1,116 +1,352 @@
//! The units themselves, as they travel.
//!
//! [`helm_facet::UnitFacets`] is what a predicate reads, and it is built in
-//! three places already: from a parsed design, from a database row, and - once
-//! a browser is doing the filtering - from a document fetched over HTTP.
+//! three places: from a parsed design, from a database row, and - once a
+//! browser is doing the filtering - from the documents helm publishes.
//!
-//! serde's remote derive is what writes that third one. The mirror below is
-//! field for field against the real struct and the compiler says so: add a
-//! facet and forget this file, and helm-query stops building rather than
-//! quietly shipping a browser that filters on one fewer column than the server
-//! does.
+//! # Why it is more than one document
//!
-//! Absent is the common case - two thirds of the columns are `None` for a
-//! design with no computed record - so every optional field is skipped when it
-//! is empty and defaulted when it is missing.
+//! A screen that draws a match report wants a picture and a name. A force
+//! builder wants every column there is. A tournament wants the year, the tech
+//! base and the tonnage. Published as one file, the first of those pays 382KB
+//! to answer a question worth 60.
+//!
+//! So the index is a spine and a set of chunks. The spine is what identifies a
+//! design - the columns nothing can be said without - and each chunk is one
+//! group of columns, in the spine's order, as an array with no keys in it.
+//!
+//! # Why the chunks carry no names
+//!
+//! Because names are the expensive column. `chassis` and `model` are long
+//! strings that gzip cannot do much with: 44KB of the spine's 98KB is names
+//! alone. A chunk that repeated them would cost more in keys than it carries
+//! in values - art is 23KB as an array and 64KB as a map - and every chunk
+//! after the first would pay that toll again.
+//!
+//! The price is that a chunk is meaningless without the spine, and that a
+//! chunk from another run of helm lines up row for row and means the wrong
+//! designs. [`Build`] is what makes that a refusal rather than a wrong answer,
+//! and one command writes the whole set so there is never half of one.
use crate::Build;
use helm_facet::UnitFacets;
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
+
+/// One group of columns, published as its own file.
+///
+/// The set is closed and the compiler enforces it: every `match` over a chunk
+/// is exhaustive, so a group added here fails to build until the writer writes
+/// it and the reader reads it.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Chunk {
+ /// Which picture each design is drawn with.
+ Art,
+ /// What it weighs, what it is worth, and what it costs.
+ Figures,
+ /// Where it comes from: the year, the tech base, the books.
+ Paperwork,
+ /// How it fights: movement, firepower, reach, heat, plate.
+ Combat,
+ /// MegaMek's per-design quirks.
+ Quirks,
+ /// What it carries, as MegaMek lists it.
+ Loadout,
+}
+
+impl Chunk {
+ /// Every chunk, in the order one command writes them.
+ pub const ALL: [Chunk; 6] = [
+ Chunk::Art,
+ Chunk::Figures,
+ Chunk::Paperwork,
+ Chunk::Combat,
+ Chunk::Quirks,
+ Chunk::Loadout,
+ ];
+
+ /// What it is called, in a filter's `needs` and in the spine's manifest.
+ pub fn name(self) -> &'static str {
+ match self {
+ Chunk::Art => "art",
+ Chunk::Figures => "figures",
+ Chunk::Paperwork => "paperwork",
+ Chunk::Combat => "combat",
+ Chunk::Quirks => "quirks",
+ Chunk::Loadout => "loadout",
+ }
+ }
+
+ /// The file it is published as, beside the spine.
+ pub fn file(self) -> String {
+ format!("{}.json", self.name())
+ }
+
+ /// The chunk of that name, for a caller reading one back.
+ pub fn named(name: &str) -> Option {
+ Chunk::ALL.into_iter().find(|c| c.name() == name)
+ }
+}
+
+/// One group of columns for every design, in the spine's order.
+///
+/// `chunk` is written so that a file cannot be mistaken for another chunk of
+/// the same shape - art and quirks are both one value per design, and reading
+/// one as the other would be silent.
+#[derive(Serialize, Deserialize)]
+pub struct Part {
+ pub build: Build,
+ pub chunk: Chunk,
+ pub rows: Vec,
+}
+
+/// One group of columns, projected out of a unit and put back onto one.
+///
+/// Every facet belongs to exactly one chunk. Nothing here enforces that on its
+/// own; `every_column_survives_the_split` does, and it cannot be written
+/// without naming all of them.
+pub trait Columns: Serialize + DeserializeOwned + Sized {
+ /// Which chunk these columns are published in.
+ const CHUNK: Chunk;
-/// The wire form of one unit's facets.
+ /// Read them off a unit.
+ fn take(unit: &UnitFacets, extra: &Extras) -> Self;
+
+ /// Put them back onto one.
+ fn put(self, unit: &mut UnitFacets, extra: &mut Extras);
+}
+
+/// The two columns a screen draws that no predicate reads.
///
-/// Never constructed directly: it exists so serde can read and write a
-/// `UnitFacets`, which belongs to a crate that does not know what serde is.
-#[derive(Default, Serialize, Deserialize)]
-#[serde(remote = "UnitFacets", default, deny_unknown_fields)]
-struct UnitFacetsDef {
+/// They travel with the facets rather than beside them because they are per
+/// design and positional like everything else, and a third parallel structure
+/// for two fields would be worse than a struct with two fields in it.
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct Extras {
+ /// Where the design's picture is, relative to the release's art tree.
+ pub sprite: Option,
+ /// Plate as the file declares it. `armor_pct` is what a filter asks
+ /// about; this is what a row prints.
+ pub total_armor: Option,
+}
+
+/// The columns nothing can be said without: what this design is.
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct Spine {
#[serde(skip_serializing_if = "String::is_empty")]
- name: String,
+ pub name: String,
#[serde(skip_serializing_if = "String::is_empty")]
- chassis: String,
+ pub chassis: String,
#[serde(skip_serializing_if = "String::is_empty")]
- model: String,
+ pub model: String,
#[serde(skip_serializing_if = "Option::is_none")]
- unit_type: Option,
+ pub unit_type: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- tech_base: Option,
+ pub config: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- role: Option,
+ pub mul_id: Option,
+}
+
+/// Which picture each design is drawn with.
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct Art(pub Option);
+
+impl Columns for Art {
+ const CHUNK: Chunk = Chunk::Art;
+
+ fn take(_unit: &UnitFacets, extra: &Extras) -> Self {
+ Art(extra.sprite.clone())
+ }
+
+ fn put(self, _unit: &mut UnitFacets, extra: &mut Extras) {
+ extra.sprite = self.0;
+ }
+}
+
+/// What it weighs, what it is worth, and what it costs.
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct Figures {
+ #[serde(rename = "mass", skip_serializing_if = "Option::is_none")]
+ pub tons: Option,
+ #[serde(rename = "bv", skip_serializing_if = "Option::is_none")]
+ pub battle_value: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- source: Option,
+ pub cost: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- weight_class: Option,
+ pub weight_class: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- year: Option,
- #[serde(rename = "mass", skip_serializing_if = "Option::is_none")]
- tons: Option,
- #[serde(rename = "bv", skip_serializing_if = "Option::is_none")]
- battle_value: Option,
+ pub total_armor: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- walk_mp: Option,
+ pub canon: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- jump_mp: Option,
+ pub invalid: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- canon: Option,
+ pub omni: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- invalid: Option,
+ pub clan: Option,
+}
+
+impl Columns for Figures {
+ const CHUNK: Chunk = Chunk::Figures;
+
+ fn take(unit: &UnitFacets, extra: &Extras) -> Self {
+ Figures {
+ tons: unit.tons,
+ battle_value: unit.battle_value,
+ cost: unit.cost,
+ weight_class: unit.weight_class,
+ total_armor: extra.total_armor,
+ canon: unit.canon,
+ invalid: unit.invalid,
+ omni: unit.omni,
+ clan: unit.clan,
+ }
+ }
+
+ fn put(self, unit: &mut UnitFacets, extra: &mut Extras) {
+ unit.tons = self.tons;
+ unit.battle_value = self.battle_value;
+ unit.cost = self.cost;
+ unit.weight_class = self.weight_class;
+ extra.total_armor = self.total_armor;
+ unit.canon = self.canon;
+ unit.invalid = self.invalid;
+ unit.omni = self.omni;
+ unit.clan = self.clan;
+ }
+}
+
+/// Where a design comes from.
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct Paperwork {
#[serde(skip_serializing_if = "Option::is_none")]
- omni: Option,
+ pub year: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- clan: Option,
- #[serde(skip_serializing_if = "Vec::is_empty")]
- loadout: Vec<(String, i64)>,
+ pub tech_base: Option,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub role: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- firepower: Option,
+ pub rules_level: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- max_range: Option,
+ pub source: Option,
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ pub sources: Vec,
+}
+
+impl Columns for Paperwork {
+ const CHUNK: Chunk = Chunk::Paperwork;
+
+ fn take(unit: &UnitFacets, _extra: &Extras) -> Self {
+ Paperwork {
+ year: unit.year,
+ tech_base: unit.tech_base.clone(),
+ role: unit.role.clone(),
+ rules_level: unit.rules_level.clone(),
+ source: unit.source.clone(),
+ sources: unit.sources.clone(),
+ }
+ }
+
+ fn put(self, unit: &mut UnitFacets, _extra: &mut Extras) {
+ unit.year = self.year;
+ unit.tech_base = self.tech_base;
+ unit.role = self.role;
+ unit.rules_level = self.rules_level;
+ unit.source = self.source;
+ unit.sources = self.sources;
+ }
+}
+
+/// How a design fights.
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct Combat {
#[serde(skip_serializing_if = "Option::is_none")]
- heat: Option,
+ pub walk_mp: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- dissipation: Option,
+ pub run_mp: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- heat_efficiency: Option,
+ pub jump_mp: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- armor_pct: Option,
+ pub firepower: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- rules_level: Option,
+ pub max_range: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- cost: Option,
+ pub heat: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- config: Option,
+ pub dissipation: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- run_mp: Option,
+ pub heat_efficiency: Option,
#[serde(skip_serializing_if = "Option::is_none")]
- mul_id: Option,
- #[serde(skip_serializing_if = "Vec::is_empty")]
- quirks: Vec,
- #[serde(skip_serializing_if = "Vec::is_empty")]
- sources: Vec,
+ pub armor_pct: Option,
}
-/// One unit, in the document.
-#[derive(Debug, Serialize, Deserialize)]
-pub struct Row(#[serde(with = "UnitFacetsDef")] pub UnitFacets);
+impl Columns for Combat {
+ const CHUNK: Chunk = Chunk::Combat;
-/// One design as the index carries it: what a filter reads, and the two
-/// things only a screen wants.
-///
-/// The facets are flattened rather than nested because this file is read by a
-/// page that draws a list as well as by one that filters it, and a row of
-/// twenty-five columns with two of them behind a `facets` key is a shape
-/// nobody would choose to write against.
-#[derive(Debug, Serialize, Deserialize)]
-pub struct IndexRow {
- #[serde(flatten)]
- pub facets: Row,
- /// Where the design's picture is, relative to the release's art tree.
- #[serde(default, skip_serializing_if = "Option::is_none")]
- pub sprite: Option,
- /// Plate as the file declares it. `armor_pct` is what a filter asks
- /// about; this is what a row prints.
- #[serde(default, skip_serializing_if = "Option::is_none")]
- pub total_armor: Option,
+ fn take(unit: &UnitFacets, _extra: &Extras) -> Self {
+ Combat {
+ walk_mp: unit.walk_mp,
+ run_mp: unit.run_mp,
+ jump_mp: unit.jump_mp,
+ firepower: unit.firepower,
+ max_range: unit.max_range,
+ heat: unit.heat,
+ dissipation: unit.dissipation,
+ heat_efficiency: unit.heat_efficiency,
+ armor_pct: unit.armor_pct,
+ }
+ }
+
+ fn put(self, unit: &mut UnitFacets, _extra: &mut Extras) {
+ unit.walk_mp = self.walk_mp;
+ unit.run_mp = self.run_mp;
+ unit.jump_mp = self.jump_mp;
+ unit.firepower = self.firepower;
+ unit.max_range = self.max_range;
+ unit.heat = self.heat;
+ unit.dissipation = self.dissipation;
+ unit.heat_efficiency = self.heat_efficiency;
+ unit.armor_pct = self.armor_pct;
+ }
+}
+
+/// MegaMek's per-design quirks.
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct Quirks(pub Vec);
+
+impl Columns for Quirks {
+ const CHUNK: Chunk = Chunk::Quirks;
+
+ fn take(unit: &UnitFacets, _extra: &Extras) -> Self {
+ Quirks(unit.quirks.clone())
+ }
+
+ fn put(self, unit: &mut UnitFacets, _extra: &mut Extras) {
+ unit.quirks = self.0;
+ }
+}
+
+/// What each design carries, as MegaMek lists it.
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct Loadout(pub Vec<(String, i64)>);
+
+impl Columns for Loadout {
+ const CHUNK: Chunk = Chunk::Loadout;
+
+ fn take(unit: &UnitFacets, _extra: &Extras) -> Self {
+ Loadout(unit.loadout.clone())
+ }
+
+ fn put(self, unit: &mut UnitFacets, _extra: &mut Extras) {
+ unit.loadout = self.0;
+ }
}
/// How much of the library is answered, which is not the same as how large it
@@ -125,15 +361,7 @@ pub struct Counts {
pub without_sprite: usize,
}
-/// Every unit a page draws or filters, and which build wrote them.
-///
-/// One document rather than two. An index of what to show and an index of
-/// what to filter on are ninety percent the same columns, and publishing both
-/// means a page fetches a megabyte twice to answer one screen.
-///
-/// What each design *carries* is a document of its own, [`Loadout`], because
-/// a screen filtering by tonnage and year needs none of it and it is a
-/// quarter of the bytes.
+/// The spine: which designs there are, and where the rest of them is.
#[derive(Serialize, Deserialize)]
pub struct Index {
pub build: Build,
@@ -153,55 +381,90 @@ pub struct Index {
/// a consumer needs no second convention to turn one into an address.
pub sprite_base: String,
pub counts: Counts,
- pub units: Vec,
+ /// The files published beside this one, so a reader discovers them rather
+ /// than being told about them by whoever wrote its fetching code.
+ pub chunks: Vec,
+ /// The designs, in the order every chunk repeats.
+ #[serde(with = "spine")]
+ pub units: Vec<(UnitFacets, Extras)>,
}
-impl Index {
- /// The units, as the predicate wants them.
- pub fn facets(&self) -> Vec<&UnitFacets> {
- self.units.iter().map(|row| &row.facets.0).collect()
+/// Only the spine's own columns cross, which is what makes a chunk worth
+/// having: the rest of each unit is filled in when its chunk arrives.
+mod spine {
+ use super::{Extras, Spine, UnitFacets};
+ use serde::{Deserialize, Deserializer, Serialize, Serializer};
+
+ pub fn serialize(
+ units: &[(UnitFacets, Extras)],
+ out: S,
+ ) -> Result {
+ let rows: Vec = units
+ .iter()
+ .map(|(unit, _)| Spine {
+ name: unit.name.clone(),
+ chassis: unit.chassis.clone(),
+ model: unit.model.clone(),
+ unit_type: unit.unit_type.clone(),
+ config: unit.config.clone(),
+ mul_id: unit.mul_id,
+ })
+ .collect();
+ rows.serialize(out)
}
-}
-/// What each design carries, in the order [`Facets`] wrote them.
-///
-/// Joined by position and checked by [`Build::joins`]. The alternative was a
-/// key per row, and there is not one: 479 designs in 0.51.0 share a display
-/// name with another, and nothing in a `.mtf` distinguishes them.
-///
-/// Kept apart from the facets because it is the expensive half - 60,881
-/// entries over 8,666 designs - and because most filtering never asks. A page
-/// fetches it when somebody first filters by equipment, and until then the
-/// question it answers is not on the screen.
-#[derive(Serialize, Deserialize)]
-pub struct Loadout {
- pub build: Build,
- /// One list per design: `(equipment name, quantity)`, MegaMek's own
- /// spelling, invented entries included.
- pub loadout: Vec>,
+ pub fn deserialize<'de, D: Deserializer<'de>>(
+ input: D,
+ ) -> Result, D::Error> {
+ Ok(Vec::::deserialize(input)?
+ .into_iter()
+ .map(|row| {
+ (
+ UnitFacets {
+ name: row.name,
+ chassis: row.chassis,
+ model: row.model,
+ unit_type: row.unit_type,
+ config: row.config,
+ mul_id: row.mul_id,
+ ..Default::default()
+ },
+ Extras::default(),
+ )
+ })
+ .collect())
+ }
}
-/// Why a loadout document could not be put onto a set of facets.
+/// Why a chunk could not be put onto a spine.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Mismatch {
/// The two files are about different runs of helm.
DifferentBuild {
index: Box,
- loadout: Box,
+ chunk: Box,
},
/// The same run, and one of the files is truncated.
- DifferentLength { index: usize, loadout: usize },
+ DifferentLength { index: usize, chunk: usize },
+ /// A file of the right shape and the wrong contents: art read as quirks.
+ DifferentChunk { wanted: Chunk, got: Chunk },
}
impl std::fmt::Display for Mismatch {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
- Mismatch::DifferentBuild { index, loadout } => {
- write!(f, "this index is {index} and that loadout is {loadout}")
+ Mismatch::DifferentBuild { index, chunk } => {
+ write!(f, "this index is {index} and that chunk is {chunk}")
}
- Mismatch::DifferentLength { index, loadout } => write!(
+ Mismatch::DifferentLength { index, chunk } => write!(
+ f,
+ "{index} designs are indexed and {chunk} are in the chunk"
+ ),
+ Mismatch::DifferentChunk { wanted, got } => write!(
f,
- "{index} designs are indexed and {loadout} have a loadout"
+ "that is the {} chunk, not the {}",
+ got.name(),
+ wanted.name()
),
}
}
@@ -210,27 +473,56 @@ impl std::fmt::Display for Mismatch {
impl std::error::Error for Mismatch {}
impl Index {
- /// Put each design's equipment onto it, or say why the two files are not
- /// about the same designs.
+ /// The units, as the predicate wants them.
+ pub fn facets(&self) -> Vec<&UnitFacets> {
+ self.units.iter().map(|(unit, _)| unit).collect()
+ }
+
+ /// What a screen draws that no predicate reads.
+ pub fn extras(&self) -> Vec<&Extras> {
+ self.units.iter().map(|(_, extra)| extra).collect()
+ }
+
+ /// The chunk this run published, for one group of columns.
+ pub fn part(&self) -> Part {
+ Part {
+ build: self.build.clone(),
+ chunk: C::CHUNK,
+ rows: self
+ .units
+ .iter()
+ .map(|(unit, extra)| C::take(unit, extra))
+ .collect(),
+ }
+ }
+
+ /// Put one group of columns onto the designs it belongs to, or say why the
+ /// two files are not about the same run.
///
/// A positional join is right or it is nonsense: nothing about the fifth
/// row of one file says whether it is the fifth design of the other. So
/// this is the only way the two are put together.
- pub fn join(&mut self, loadout: Loadout) -> Result<(), Mismatch> {
- if !self.build.joins(&loadout.build) {
+ pub fn join(&mut self, part: Part) -> Result<(), Mismatch> {
+ if part.chunk != C::CHUNK {
+ return Err(Mismatch::DifferentChunk {
+ wanted: C::CHUNK,
+ got: part.chunk,
+ });
+ }
+ if !self.build.joins(&part.build) {
return Err(Mismatch::DifferentBuild {
index: Box::new(self.build.clone()),
- loadout: Box::new(loadout.build),
+ chunk: Box::new(part.build),
});
}
- if self.units.len() != loadout.loadout.len() {
+ if self.units.len() != part.rows.len() {
return Err(Mismatch::DifferentLength {
index: self.units.len(),
- loadout: loadout.loadout.len(),
+ chunk: part.rows.len(),
});
}
- for (row, carried) in self.units.iter_mut().zip(loadout.loadout) {
- row.facets.0.loadout = carried;
+ for ((unit, extra), columns) in self.units.iter_mut().zip(part.rows) {
+ columns.put(unit, extra);
}
Ok(())
}
@@ -250,104 +542,163 @@ mod tests {
)
}
- fn document(names: &[&str]) -> Index {
+ /// One design with every column set to something distinguishable, written
+ /// out in full.
+ ///
+ /// The literal is the point: it names all of them, so a facet added to
+ /// `UnitFacets` fails to compile here until somebody has decided which
+ /// chunk publishes it. That is the whole guarantee that splitting the
+ /// index does not quietly drop a column.
+ fn a_unit() -> (UnitFacets, Extras) {
+ (
+ UnitFacets {
+ name: "Atlas AS7-D".into(),
+ chassis: "Atlas".into(),
+ model: "AS7-D".into(),
+ unit_type: Some("Mek".into()),
+ config: Some("Biped".into()),
+ mul_id: Some(87),
+ tech_base: Some("Inner Sphere".into()),
+ role: Some("Juggernaut".into()),
+ source: Some("TR:3039,TR:SW".into()),
+ sources: vec!["TR:3039".into(), "TR:SW".into()],
+ rules_level: Some("Standard".into()),
+ year: Some(2755),
+ weight_class: Some(4),
+ tons: Some(100.0),
+ battle_value: Some(1897),
+ cost: Some(9_000_000),
+ canon: Some(true),
+ invalid: Some(false),
+ omni: Some(false),
+ clan: Some(false),
+ walk_mp: Some(3),
+ run_mp: Some(5),
+ jump_mp: Some(0),
+ firepower: Some(43.5),
+ max_range: Some(21),
+ heat: Some(28),
+ dissipation: Some(20),
+ heat_efficiency: Some(-8),
+ armor_pct: Some(93),
+ quirks: vec!["battle_fists".into()],
+ loadout: vec![("Autocannon/20".into(), 1), ("Medium Laser".into(), 4)],
+ },
+ Extras {
+ sprite: Some("meks/atlas.png".into()),
+ total_armor: Some(307),
+ },
+ )
+ }
+
+ fn index(units: Vec<(UnitFacets, Extras)>) -> Index {
+ let names: Vec<&str> = units.iter().map(|(u, _)| u.name.as_str()).collect();
Index {
- build: build(names),
+ build: build(&names),
megamek: "0.51.0".into(),
helm: None,
sprite_base: "data/images/units".into(),
counts: Counts::default(),
- units: names
- .iter()
- .map(|name| IndexRow {
- facets: Row(UnitFacets {
- name: (*name).into(),
- ..Default::default()
- }),
- sprite: None,
- total_armor: None,
- })
- .collect(),
+ chunks: Chunk::ALL.iter().map(|c| c.file()).collect(),
+ units,
}
}
- /// The join the split exists for: the equipment arrives later, from its
- /// own file, and lands on the design it belongs to.
+ /// Split the whole set out and put it all back: what comes out the far
+ /// side is what went in, column for column.
#[test]
- fn a_loadout_lands_on_the_design_it_belongs_to() {
- let mut facets = document(&["Atlas AS7-D", "Locust LCT-1V"]);
- let loadout = Loadout {
- build: build(&["Atlas AS7-D", "Locust LCT-1V"]),
- loadout: vec![
- vec![("ISGaussRifle".into(), 1)],
- vec![("Machine Gun".into(), 2)],
- ],
- };
- facets.join(loadout).expect("the same designs");
- assert_eq!(
- facets.units[1].facets.0.loadout,
- [("Machine Gun".to_string(), 2)]
+ fn every_column_survives_the_split() {
+ let whole = index(vec![a_unit()]);
+ let (art, figures, paperwork, combat, quirks, loadout) = (
+ whole.part::(),
+ whole.part::(),
+ whole.part::(),
+ whole.part::(),
+ whole.part::(),
+ whole.part::(),
);
+
+ // Nothing but the spine crosses in the index itself.
+ let text = serde_json::to_string(&whole).expect("write");
+ let mut read: Index = serde_json::from_str(&text).expect("read");
+ assert_eq!(read.units[0].0.battle_value, None, "a chunk column crossed");
+ assert_eq!(read.units[0].0.name, "Atlas AS7-D");
+
+ for part in [
+ serde_json::to_string(&art).unwrap(),
+ serde_json::to_string(&figures).unwrap(),
+ serde_json::to_string(&paperwork).unwrap(),
+ serde_json::to_string(&combat).unwrap(),
+ serde_json::to_string(&quirks).unwrap(),
+ serde_json::to_string(&loadout).unwrap(),
+ ] {
+ let chunk: serde_json::Value = serde_json::from_str(&part).unwrap();
+ match Chunk::named(chunk["chunk"].as_str().unwrap()).unwrap() {
+ Chunk::Art => read.join(serde_json::from_str::>(&part).unwrap()),
+ Chunk::Figures => read.join(serde_json::from_str::>(&part).unwrap()),
+ Chunk::Paperwork => {
+ read.join(serde_json::from_str::>(&part).unwrap())
+ }
+ Chunk::Combat => read.join(serde_json::from_str::>(&part).unwrap()),
+ Chunk::Quirks => read.join(serde_json::from_str::>(&part).unwrap()),
+ Chunk::Loadout => read.join(serde_json::from_str::>(&part).unwrap()),
+ }
+ .expect("the same run");
+ }
+
+ assert_eq!(read.units[0], a_unit());
}
/// Two files from different runs line up row for row and mean different
/// designs. Refused, because the alternative is a page telling somebody a
/// Locust carries a Gauss rifle.
#[test]
- fn a_loadout_from_another_run_is_refused() {
- let mut facets = document(&["Atlas AS7-D", "Locust LCT-1V"]);
- let loadout = Loadout {
- build: build(&["Atlas AS7-D", "Banshee BNC-3E"]),
- loadout: vec![vec![], vec![]],
+ fn a_chunk_from_another_run_is_refused() {
+ let mut whole = index(vec![a_unit()]);
+ let part = Part {
+ build: build(&["Locust LCT-1V"]),
+ chunk: Chunk::Loadout,
+ rows: vec![Loadout(vec![("ISGaussRifle".into(), 1)])],
};
- let err = facets.join(loadout).expect_err("a different run");
+ let err = whole.join(part).expect_err("a different run");
assert!(matches!(err, Mismatch::DifferentBuild { .. }), "{err}");
- assert!(
- facets
- .units
- .iter()
- .all(|row| row.facets.0.loadout.is_empty())
- );
+ assert_eq!(whole.units[0].0.loadout, a_unit().0.loadout);
}
- /// A round trip through the document is the whole contract: what the
- /// server filtered is what the browser filters.
+ /// Nothing about the shape of a chunk says which chunk it is - art and
+ /// quirks are both one value per design, and today they are told apart by
+ /// luck rather than by design. The name written in the file is what makes
+ /// that a refusal instead of a silent misreading.
#[test]
- fn a_unit_survives_the_document() {
- let facets = UnitFacets {
- name: "Atlas AS7-D".into(),
- chassis: "Atlas".into(),
- model: "AS7-D".into(),
- battle_value: Some(1897),
- tons: Some(100.0),
- loadout: vec![("ISGaussRifle".into(), 1), ("Medium Laser".into(), 4)],
- quirks: vec!["Battle Fists".into()],
- ..Default::default()
+ fn a_chunk_read_as_another_is_refused() {
+ let mut whole = index(vec![a_unit()]);
+ let mislabelled = Part {
+ build: whole.build.clone(),
+ chunk: Chunk::Art,
+ rows: vec![Quirks(vec!["battle_fists".into()])],
};
- let text = serde_json::to_string(&Row(facets.clone())).expect("write");
- let read: Row = serde_json::from_str(&text).expect("read");
- assert_eq!(read.0, facets);
+ let err = whole
+ .join(mislabelled)
+ .expect_err("that says it is the art");
+ assert_eq!(
+ err.to_string(),
+ "that is the art chunk, not the quirks",
+ "{err}"
+ );
}
- /// Absent is the common case and an empty column is not worth its name:
- /// a design with no computed record writes three fields, not thirty-one.
+ /// Absent is the common case and an empty column is not worth its name: a
+ /// design with no computed record writes three fields, not thirty-one.
#[test]
fn an_empty_column_is_not_written() {
- let text = serde_json::to_string(&Row(UnitFacets {
- name: "Locust LCT-1V".into(),
- ..Default::default()
- }))
- .expect("write");
- assert_eq!(text, r#"{"name":"Locust LCT-1V"}"#);
- }
-
- /// A column this build does not know is a document from a newer helm, and
- /// filtering it on what is understood would quietly answer a different
- /// question.
- #[test]
- fn a_column_from_the_future_is_refused() {
- let err =
- serde_json::from_str::](r#"{"name":"x","spice":3}"#).expect_err("not one of ours");
- assert!(err.to_string().contains("spice"), "{err}");
+ let whole = index(vec![(
+ UnitFacets {
+ name: "Locust LCT-1V".into(),
+ ..Default::default()
+ },
+ Extras::default(),
+ )]);
+ let text = serde_json::to_string(&whole.part::()).unwrap();
+ assert!(text.ends_with(r#""rows":[{}]}"#), "{text}");
}
}
diff --git a/crates/helm-query/src/lib.rs b/crates/helm-query/src/lib.rs
index ea82c65..e009497 100644
--- a/crates/helm-query/src/lib.rs
+++ b/crates/helm-query/src/lib.rs
@@ -19,7 +19,10 @@ mod build;
mod facets;
pub use build::Build;
-pub use facets::{Counts, Index, IndexRow, Loadout, Mismatch, Row};
+pub use facets::{
+ Art, Chunk, Columns, Combat, Counts, Extras, Figures, Index, Loadout, Mismatch, Paperwork,
+ Part, Quirks, Spine,
+};
use std::collections::BTreeSet;
@@ -265,21 +268,63 @@ impl Vocabulary {
}
impl Filter {
- /// Whether this filter asks what a design carries.
+ /// Which chunks of the index this filter reads.
///
- /// A library with no loadout joined to it answers those with silence -
- /// every design carrying nothing - which reads exactly like an honest
- /// empty result. So a caller that can go and fetch the loadout is told to
- /// rather than shown one.
- pub fn asks_about_equipment(&self) -> bool {
- [
- &self.carries,
- &self.lacks,
- &self.carries_class,
- &self.lacks_class,
- ]
- .into_iter()
- .any(|list| list.as_ref().is_some_and(|l| !l.is_empty()))
+ /// A library that has not joined one of them answers those terms with
+ /// silence - every design having nothing - which reads exactly like an
+ /// honest empty result. So a caller is told what to go and fetch instead
+ /// of being shown one.
+ ///
+ /// The spine's own columns are not a chunk: a library that has been loaded
+ /// at all has them, and art is a chunk nothing filters on.
+ pub fn needs(&self) -> Vec {
+ let some = |list: &Option>| list.as_ref().is_some_and(|l| !l.is_empty());
+ let mut needs = Vec::new();
+ if some(&self.carries)
+ || some(&self.lacks)
+ || some(&self.carries_class)
+ || some(&self.lacks_class)
+ {
+ needs.push(Chunk::Loadout);
+ }
+ if some(&self.has_quirks) {
+ needs.push(Chunk::Quirks);
+ }
+ if some(&self.tech_base)
+ || some(&self.role)
+ || some(&self.rules_level)
+ || some(&self.source)
+ || self.year_min.is_some()
+ || self.year_max.is_some()
+ {
+ needs.push(Chunk::Paperwork);
+ }
+ if some(&self.weight_class)
+ || self.tons_min.is_some()
+ || self.tons_max.is_some()
+ || self.bv_min.is_some()
+ || self.bv_max.is_some()
+ || self.cost_min.is_some()
+ || self.cost_max.is_some()
+ || self.omni.is_some()
+ {
+ needs.push(Chunk::Figures);
+ }
+ if self.walk_mp_min.is_some()
+ || self.jump_mp_min.is_some()
+ || self.run_mp_min.is_some()
+ || self.firepower_min.is_some()
+ || self.firepower_max.is_some()
+ || self.max_range_min.is_some()
+ || self.max_range_max.is_some()
+ || self.heat_efficiency_min.is_some()
+ || self.heat_efficiency_max.is_some()
+ || self.armor_pct_min.is_some()
+ || self.armor_pct_max.is_some()
+ {
+ needs.push(Chunk::Combat);
+ }
+ needs
}
/// The predicate this filter means, or the first thing in it the library
@@ -477,6 +522,33 @@ mod tests {
);
}
+ /// Every chunk a filter can read is asked for by some term, and an empty
+ /// filter asks for none. A term whose chunk is not named here filters on a
+ /// column the browser may never have fetched.
+ #[test]
+ fn every_chunk_is_asked_for_by_some_term() {
+ assert!(Filter::default().needs().is_empty());
+ let everything = Filter {
+ carries: Some(vec!["ISGaussRifle".into()]),
+ has_quirks: Some(vec!["easy_maintain".into()]),
+ year_max: Some(3025),
+ bv_max: Some(2000),
+ firepower_min: Some(10),
+ ..Default::default()
+ };
+ let mut asked: Vec<&str> = everything.needs().iter().map(|c| c.name()).collect();
+ asked.sort_unstable();
+ // Art is the exception, and the only one: it is a column a screen
+ // draws and no filter reads.
+ let mut filterable: Vec<&str> = Chunk::ALL
+ .iter()
+ .map(|c| c.name())
+ .filter(|name| *name != "art")
+ .collect();
+ filterable.sort_unstable();
+ assert_eq!(asked, filterable);
+ }
+
/// A field nobody knows is a typo in a filter, and a filter that silently
/// ignores one selects the whole library while looking narrow.
#[test]
diff --git a/crates/helm-wasm/facets.mjs b/crates/helm-wasm/facets.mjs
index 4cbe516..9e2c50e 100644
--- a/crates/helm-wasm/facets.mjs
+++ b/crates/helm-wasm/facets.mjs
@@ -3,8 +3,7 @@
//
// helm index --megamek --bridge-dir --out /units.json \
// --filters crates/helm-wasm/filters.json
-// node crates/helm-wasm/facets.mjs /units.json /loadout.json \
-// /answers.json
+// node crates/helm-wasm/facets.mjs /units.json /answers.json
//
// Why this exists: `unit-search` says one filter definition has to answer the
// same way in three places - an agent over MCP, the server, and the browser -
@@ -19,11 +18,12 @@
// bound, a battle value the library does not have - of which there are
// several.
import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
import { Helm } from "./helm.mjs";
-const [index, loadout, answers] = process.argv.slice(2);
-if (!index || !loadout || !answers) {
- console.error("usage: node facets.mjs ");
+const [index, answers] = process.argv.slice(2);
+if (!index || !answers) {
+ console.error("usage: node facets.mjs ");
process.exit(2);
}
@@ -32,14 +32,19 @@ const wasm = new URL(
import.meta.url,
);
+// The chunks sit beside the spine, which is how a page finds them: the
+// document names them rather than the fetching code knowing them.
+const beside = (name) =>
+ readFileSync(new URL(name, `file://${resolve(index)}`), "utf8");
+
const helm = await Helm.load(readFileSync(wasm));
const started = Date.now();
const library = helm.library(readFileSync(index, "utf8"));
const indexed = Date.now() - started;
-// What a page does before anybody has filtered by equipment: a filter that
-// asks what a design carries is refused rather than answered against designs
-// that carry nothing.
+// What a page does before it has fetched anything: a filter that reads a
+// chunk which is not loaded is refused rather than answered against columns
+// every design is missing.
let unarmed = null;
try {
library.select({ carries: ["ISGaussRifle"] });
@@ -47,9 +52,21 @@ try {
unarmed = error;
}
-const joining = Date.now();
-library.loadout(readFileSync(loadout, "utf8"));
-const joined = Date.now() - joining;
+// And what it does instead: ask for the chunks the filter reads, then answer.
+let fetched = [];
+const fetching = Date.now();
+await library.selectAsync({ carries: ["ISGaussRifle"], bv_max: 3000 }, (name) => {
+ fetched.push(name);
+ return beside(`${name}.json`);
+});
+const joined = Date.now() - fetching;
+
+// Everything else the corpus reads, so the run below is measuring filtering
+// rather than fetching.
+for (const chunk of JSON.parse(readFileSync(index, "utf8")).chunks) {
+ const name = chunk.replace(/\.json$/, "");
+ if (!fetched.includes(name)) library.chunk(beside(chunk));
+}
const native = JSON.parse(readFileSync(answers, "utf8"));
const wrong = [];
@@ -77,14 +94,19 @@ for (const row of native) {
const took = Date.now() - filtering;
console.log(
- `${library.size} units in ${indexed}ms, loadout joined in ${joined}ms, ` +
+ `${library.size} units in ${indexed}ms, ` +
+ `${fetched.join(" and ")} fetched on demand in ${joined}ms, ` +
`${native.length} filters selecting ${selected} rows in ${took}ms, ` +
`${wrong.length} disagreeing`,
);
-if (!unarmed) {
- console.log(" an equipment filter was answered before the loadout was joined");
+if (!unarmed?.needs?.includes("loadout")) {
+ console.log(" an equipment filter was answered before the loadout was loaded");
wrong.push("unjoined equipment filter");
}
+if (!fetched.includes("loadout") || !fetched.includes("figures")) {
+ console.log(` a filter reading two chunks fetched ${fetched.join(", ") || "nothing"}`);
+ wrong.push("wrong chunks fetched");
+}
for (const line of wrong.slice(0, 20)) {
console.log(` ${line}`);
}
@@ -102,13 +124,13 @@ if (!refused?.valid?.length) {
wrong.push("unknown role");
}
-// A loadout from another run lines up row for row and means different
-// designs, so it is refused rather than joined.
-const forged = JSON.parse(readFileSync(loadout, "utf8"));
+// A chunk from another run lines up row for row and means different designs,
+// so it is refused rather than joined.
+const forged = JSON.parse(beside("loadout.json"));
forged.build.id = "0000000000000000";
let refusedJoin = null;
try {
- helm.library(readFileSync(index, "utf8")).loadout(JSON.stringify(forged));
+ helm.library(readFileSync(index, "utf8")).chunk(JSON.stringify(forged));
} catch (error) {
refusedJoin = error;
}
diff --git a/crates/helm-wasm/helm.mjs b/crates/helm-wasm/helm.mjs
index 77bf22c..4caff8c 100644
--- a/crates/helm-wasm/helm.mjs
+++ b/crates/helm-wasm/helm.mjs
@@ -20,7 +20,7 @@
// including when the call throws.
/** The boundary this wrapper was written against; the module must agree. */
-const ABI_VERSION = 4n;
+const ABI_VERSION = 5n;
const ERRORS = new Map([
[-1n, "no equipment catalogue is loaded"],
@@ -481,19 +481,19 @@ export class Library {
}
/**
- * Put what each design carries onto it, from the loadout document.
+ * Put one chunk of the index onto it.
*
- * Its own file because most filtering never asks, and it is a quarter of
- * the bytes: fetch it the first time somebody filters by equipment. Until
- * then a filter that asks what a design carries is refused rather than
- * answered with silence.
+ * The index is a spine and a set of chunks, because most filtering never
+ * reads most of them. Pass a fetcher to `select` and this happens on its
+ * own; call it directly to load one up front - what a screen does for the
+ * columns it draws rather than filters on.
*
- * Throws if the loadout is about another run of helm. The two are joined by
+ * Throws if the chunk is about another run of helm. The two are joined by
* position, so a mismatch is not a smaller answer - it is the wrong one.
*/
- loadout(loadout) {
- const answer = this.#withBytes(loadout, (ptr, len) => {
- const packed = this.#api.helm_library_loadout(this.#live, ptr, len);
+ chunk(document) {
+ const answer = this.#withBytes(document, (ptr, len) => {
+ const packed = this.#api.helm_library_chunk(this.#live, ptr, len);
return packed === 0n ? null : this.#take(packed);
});
if (answer !== null) {
@@ -511,11 +511,13 @@ export class Library {
* quirk or an equipment class it does not.
*/
select(filter) {
- const answer = JSON.parse(
- this.#withBytes(JSON.stringify(filter ?? {}), (ptr, len) =>
- this.#take(this.#api.helm_library_select(this.#live, ptr, len)),
- ),
- );
+ const answer = this.#ask(filter);
+ if (answer.needs) {
+ const error = new HelmError(0n);
+ error.message = `this filter reads ${answer.needs.join(", ")}, which is not loaded`;
+ Object.assign(error, answer);
+ throw error;
+ }
if (answer.error) {
const error = new HelmError(0n);
error.message = answer.error;
@@ -525,6 +527,36 @@ export class Library {
return answer.units;
}
+ /**
+ * The same, fetching whatever the filter reads and this library has not got.
+ *
+ * `fetch` is given a chunk's name - "loadout", "figures" - and returns the
+ * document's text. Nothing is fetched twice: once a chunk is on, it stays
+ * on for the life of the library.
+ *
+ * One round of fetching, then the filter is asked again. If it still wants
+ * something, that is a chunk the fetcher would not produce, and the answer
+ * is an error rather than a shorter list: a filter quietly answered against
+ * columns nobody could load is worse than one that says so.
+ */
+ async selectAsync(filter, fetch) {
+ const answer = this.#ask(filter);
+ if (!answer.needs) return this.select(filter);
+ for (const name of answer.needs) {
+ this.chunk(await fetch(name));
+ }
+ return this.select(filter);
+ }
+
+ /** One question across the boundary, answered as it came back. */
+ #ask(filter) {
+ return JSON.parse(
+ this.#withBytes(JSON.stringify(filter ?? {}), (ptr, len) =>
+ this.#take(this.#api.helm_library_select(this.#live, ptr, len)),
+ ),
+ );
+ }
+
/** Let it go. Filtering afterwards throws rather than answering. */
free() {
check(this.#api.helm_library_free(this.#live));
diff --git a/crates/helm-wasm/src/lib.rs b/crates/helm-wasm/src/lib.rs
index 06b7b7a..0d577c1 100644
--- a/crates/helm-wasm/src/lib.rs
+++ b/crates/helm-wasm/src/lib.rs
@@ -481,9 +481,9 @@ fn with_force_mut(handle: i64, f: impl FnOnce(&mut helm_unitfile::Mul) -> i64) -
struct Filterable {
index: helm_query::Index,
vocabulary: helm_query::Vocabulary,
- /// Whether the loadout document has been joined on. Until it has, what
- /// each design carries is not in memory at all.
- carrying: bool,
+ /// The chunks joined on so far. Until one is, the columns it holds are
+ /// not in memory at all.
+ joined: Vec,
}
/// Read a library of units a filter can select from, and keep it.
@@ -510,7 +510,7 @@ pub unsafe extern "C" fn helm_library_load(ptr: *const u8, len: usize) -> i64 {
let held = Filterable {
index,
vocabulary,
- carrying: false,
+ joined: Vec::new(),
};
let handle = match libraries.iter().position(|slot| slot.is_none()) {
Some(free) => {
@@ -551,40 +551,64 @@ pub extern "C" fn helm_library_len(handle: i64) -> i64 {
with_library(handle, |held| held.index.units.len() as i64)
}
-/// Put what each design carries onto a library already loaded.
+/// Put one chunk of the index onto a library already loaded.
///
-/// The loadout is its own document because most filtering never asks what a
-/// design carries, and it is a quarter of the bytes. A page fetches it the
-/// first time somebody filters by equipment.
+/// The index is a spine and a set of chunks - what each design carries, what
+/// it is worth, where it came from - because most filtering never reads most
+/// of them, and the loadout alone is a quarter of the bytes. A page fetches a
+/// chunk the first time a filter needs it, which `helm_library_select` says.
///
-/// The two files are joined by position and checked by the build each carries.
-/// A loadout from another run of helm lines up row for row and means different
-/// designs, so it is refused with what the two builds are.
+/// The chunk names itself, and the two files are joined by position and
+/// checked by the build each carries: a chunk from another run of helm lines
+/// up row for row and means different designs, so it is refused with what the
+/// two builds are.
///
/// # Safety
/// `ptr` must point to `len` readable bytes.
#[unsafe(no_mangle)]
-pub unsafe extern "C" fn helm_library_loadout(handle: i64, ptr: *const u8, len: usize) -> i64 {
+pub unsafe extern "C" fn helm_library_chunk(handle: i64, ptr: *const u8, len: usize) -> i64 {
let Some(json) = (unsafe { text(ptr, len) }) else {
return ERR_NOT_UTF8;
};
- let Ok(loadout) = serde_json::from_str::(json) else {
+ // Which chunk it is, before it is read as one: every chunk is an array of
+ // something, and two of them are an array of the same something.
+ let Ok(head) = serde_json::from_str::(json) else {
return ERR_UNREADABLE;
};
- with_library_mut(handle, |held| match held.index.join(loadout) {
- Ok(()) => {
- held.carrying = true;
- 0
+ let Some(chunk) = head["chunk"].as_str().and_then(helm_query::Chunk::named) else {
+ return ERR_UNREADABLE;
+ };
+
+ with_library_mut(handle, |held| {
+ let joined = match chunk {
+ helm_query::Chunk::Art => join_part::(held, json),
+ helm_query::Chunk::Figures => join_part::(held, json),
+ helm_query::Chunk::Paperwork => join_part::(held, json),
+ helm_query::Chunk::Combat => join_part::(held, json),
+ helm_query::Chunk::Quirks => join_part::(held, json),
+ helm_query::Chunk::Loadout => join_part::(held, json),
+ };
+ match joined {
+ Ok(()) => {
+ held.joined.push(chunk);
+ // What a filter may name can only widen once a chunk that
+ // holds a vocabulary column has arrived.
+ held.vocabulary = helm_query::Vocabulary::from_facets(
+ &held.index.facets().into_iter().cloned().collect::>(),
+ );
+ 0
+ }
+ Err(why) => give(serde_json::json!({ "error": why }).to_string()),
}
- Err(mismatch) => give(
- serde_json::json!({
- "error": mismatch.to_string(),
- })
- .to_string(),
- ),
})
}
+/// Read one chunk and put it on, or say what was wrong with it.
+fn join_part(held: &mut Filterable, json: &str) -> Result<(), String> {
+ let part = serde_json::from_str::>(json).map_err(|e| e.to_string())?;
+ held.index.join(part).map_err(|e| e.to_string())
+}
+
/// The units a filter selects, by their place in the library.
///
/// The filter is the same document an agent sends over MCP - `bv_max`,
@@ -609,14 +633,17 @@ pub unsafe extern "C" fn helm_library_select(handle: i64, ptr: *const u8, len: u
Err(_) => return ERR_BAD_STATE,
};
with_library(handle, |held| {
- if filter.asks_about_equipment() && !held.carrying {
- return give(
- serde_json::json!({
- "error": "this filter asks what a design carries and no loadout is loaded",
- "field": "loadout",
- })
- .to_string(),
- );
+ // What this filter reads that is not here yet. Answering anyway would
+ // filter on columns every design is missing, which reads exactly like
+ // an honest empty result.
+ let missing: Vec<&str> = filter
+ .needs()
+ .into_iter()
+ .filter(|chunk| !held.joined.contains(chunk))
+ .map(helm_query::Chunk::name)
+ .collect();
+ if !missing.is_empty() {
+ return give(serde_json::json!({ "needs": missing }).to_string());
}
let query = match filter.to_query(&held.vocabulary) {
Ok(query) => query,
@@ -693,9 +720,13 @@ fn with_library_mut(handle: i64, f: impl FnOnce(&mut Filterable) -> i64) -> i64
/// 4 splits what a design carries into its own document, joined on with
/// `helm_library_loadout`. Not additive: a page on 3 handed a 4 index would
/// filter by equipment against designs that carry nothing.
+///
+/// 5 makes that general - the index is a spine and six chunks, joined on with
+/// `helm_library_chunk` - and answers a filter that reads a chunk which is
+/// not loaded with what it needs rather than refusing it.
#[unsafe(no_mangle)]
pub extern "C" fn helm_abi_version() -> i64 {
- 4
+ 5
}
/// How many equipment entries are loaded, so a caller can tell an empty
diff --git a/scripts/deploy.sh b/scripts/deploy.sh
index de5bd17..9da6f6c 100755
--- a/scripts/deploy.sh
+++ b/scripts/deploy.sh
@@ -15,8 +15,8 @@
# Three files, under one prefix named after this commit:
#
# [/helm_wasm.wasm the rules, compiled for a browser
-# ][//units.json the index a force-building screen filters
-# ][//loadout.json what each of those designs carries
+# ][//units.json which designs there are, and where the rest is
+# ][//.json one group of columns each, named by units.json
# ][//catalogue.jsonl the equipment the rules read
#
# https://lance.blue/assets/helm/][//units.json
@@ -182,7 +182,14 @@ mm="$(grep -o '"megamek":"[^"]*"' "$out/units.json" | head -1 | cut -d'"' -f4)"
cp target/wasm32-unknown-unknown/release/helm_wasm.wasm "$out/helm_wasm.wasm"
echo
-for file in units.json loadout.json catalogue.jsonl helm_wasm.wasm; do
+# Every chunk the index names, so a chunk added in helm is published without
+# this script learning its name.
+chunks="$(jq -r '.chunks[]' "$out/units.json")"
+for file in units.json $chunks catalogue.jsonl helm_wasm.wasm; do
+ [ -f "$out/$file" ] || {
+ echo "deploy: units.json names $file and helm did not write it" >&2
+ exit 1
+ }
printf ' %-16s %6s KB\n' "$file" "$(( $(wc -c <"$out/$file") / 1024 ))"
done
echo
@@ -202,8 +209,10 @@ aws s3 cp "$out/helm_wasm.wasm" "s3://$bucket/$ref/helm_wasm.wasm" \
--content-type application/wasm --cache-control "$cache" --no-progress
aws s3 cp "$out/units.json" "s3://$bucket/$ref/$mm/units.json" \
--content-type application/json --cache-control "$cache" --no-progress
-aws s3 cp "$out/loadout.json" "s3://$bucket/$ref/$mm/loadout.json" \
- --content-type application/json --cache-control "$cache" --no-progress
+for file in $chunks; do
+ aws s3 cp "$out/$file" "s3://$bucket/$ref/$mm/$file" \
+ --content-type application/json --cache-control "$cache" --no-progress
+done
aws s3 cp "$out/catalogue.jsonl" "s3://$bucket/$ref/$mm/catalogue.jsonl" \
--content-type application/x-ndjson --cache-control "$cache" --no-progress
@@ -216,7 +225,9 @@ echo
echo "Published under $ref/$mm:"
echo " https://lance.blue/assets/helm/$ref/helm_wasm.wasm"
echo " https://lance.blue/assets/helm/$ref/$mm/units.json"
-echo " https://lance.blue/assets/helm/$ref/$mm/loadout.json"
+for file in $chunks; do
+ echo " https://lance.blue/assets/helm/$ref/$mm/$file"
+done
echo " https://lance.blue/assets/helm/$ref/$mm/catalogue.jsonl"
echo
echo "Wrote releases.helm = $ref"
]