From 2bd57cc38a6d39a9fd9cd44f826dc736e25527a7 Mon Sep 17 00:00:00 2001 From: @permadeath.com Date: Sat, 22 Aug 2026 14:24:31 +0000 Subject: [PATCH] feat(unit-search)!: one unit index, and what each design carries beside it The index a screen draws and the columns a filter reads were ninety percent the same, published twice; they are one document now, and `helm facets` is gone. What each design carries moves out into `loadout.json`, a quarter of the bytes that most filtering never asks for. The two are joined by position and checked by the build each carries: `helm_library_loadout` refuses a loadout from another run, and until one is joined a filter that asks what a design carries is refused rather than answered against designs that carry nothing. Change-Id: I7d15dad9c2926085462f9b38f7274e3e936a6ff4 --- CLAUDE.md | 22 +++++++++++++++------- TODO.md | 28 +++++++++++++++++----------- scripts/deploy.sh | 8 +++++++- crates/helm-wasm/facets.mjs | 50 ++++++++++++++++++++++++++++++++++++++++++-------- crates/helm-wasm/helm.mjs | 32 ++++++++++++++++++++++++++++---- crates/helm-cli/src/build.rs | 372 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- crates/helm-cli/src/main.rs | 3 +-- crates/helm-cli/src/options.rs | 4 ++-- crates/helm-query/src/facets.rs | 213 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------- crates/helm-query/src/lib.rs | 19 ++++++++++++++++++- crates/helm-wasm/src/lib.rs | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------- 11 file(s) changed, 563 insertion(s)(+), 276 deletion(s)(-) diff --git a/CLAUDE.md b/CLAUDE.md --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,10 +36,12 @@ the module first: `cargo build -p helm-wasm --target wasm32-unknown-unknown --release`. - `crates/helm-wasm/facets.mjs` is the filtering half of the same question: - `helm facets --megamek --bridge-dir --out /facets.json - --filters crates/helm-wasm/filters.json` writes the units a page would - filter and what this build selects from them, and the script re-runs every - filter through wasm and fails on any disagreement. About a second. + `helm index --megamek --bridge-dir --out /units.json + --filters crates/helm-wasm/filters.json` writes the index a page filters, + the loadout beside it and what this build selects from the two, and + `node crates/helm-wasm/facets.mjs /units.json /loadout.json + /answers.json` re-runs every filter through wasm and fails on any + disagreement. About a second. - `crates/helm-wasm/wire.mjs` is the same boundary over the whole library rather than one design: `helm wire-check --megamek --bridge-dir --out ` writes what a page would be handed and what this build says @@ -54,9 +56,15 @@ ## Deploying -`scripts/deploy.sh` builds the wasm, the index and the catalogue and puts them -in the assets bucket under `//`, then writes `releases.helm` in -infra. It applies nothing — the apply is infra's, and jmm runs it. +`scripts/deploy.sh` builds the wasm, the index, the loadout and the catalogue +and puts them in the assets bucket under `//`, then writes +`releases.helm` in infra. It applies nothing — the apply is infra's, and jmm +runs it. + +`units.json` and `loadout.json` carry the same `build` block and are joined by +position, so a page that mixes two prefixes is refused rather than answered. +`catalogue.jsonl` has no identity of its own and is keyed only by the prefix +it sits under. - A prefix is written once. Never overwrite one, and never reach for a CloudFront invalidation: a rebuild is a new ref, which is what makes the diff --git a/TODO.md b/TODO.md --- a/TODO.md +++ b/TODO.md @@ -17,7 +17,7 @@ ## What to do next, in order -Twenty things, ranked. The tiers are the argument: the first finishes what +Twenty-one things, ranked. The tiers are the argument: the first finishes what `plan/unit-rules.md` already promises, the second widens what helm can score at all, and the third is what makes any of it reach a player. Everything below appears again in its own section with the detail; this is the order to take @@ -75,31 +75,37 @@ 13. **Decide how a design reaches the page, and build it.** The one open decision, and everything in `plan/unit-search.md` waits behind it. 14. **Compile `helm-facet` to wasm and filter with it in the browser.** Done - on this side: `helm facets` writes the columns a predicate reads, - `helm_library_load` and `helm_library_select` run the same predicate over - them in wasm, and `facets.mjs` checks 28 filters over 8,666 designs - against what the native build selects. The filter itself is one definition + on this side: `helm index` writes the columns a predicate reads and the + loadout beside them, `helm_library_load`, `helm_library_loadout` and + `helm_library_select` run the same predicate over them in wasm, and + `facets.mjs` checks 28 filters over 8,666 designs against what the native + build selects. The filter itself is one definition in `helm-query`, which helm-mcp and the boundary both read. What is left is headquarters': `web/src/units.ts` still has a filter table of its own, and that is the second implementation the epic warns about. -15. **The redistribution question.** MegaMek's data is CC BY-NC-SA and serving +15. **An identity for the catalogue.** `units.json` and `loadout.json` carry a + `build` block and refuse to be joined across runs; `catalogue.jsonl` is + JSON Lines with nowhere to put one, and is keyed only by the prefix it is + published under. Two dumps of one MegaMek release are not interchangeable + - see the note in CLAUDE.md - so the prefix is not enough. +16. **The redistribution question.** MegaMek's data is CC BY-NC-SA and serving an index of it to browsers is redistribution. It needs a decision and an attribution mechanism *before* 13 ships. -16. **Provenance on every record.** Which MegaMek, which producer, which +17. **Provenance on every record.** Which MegaMek, which producer, which `RULES_VERSION`. The constants exist and nothing writes them into the database or into an ATProto record, so a stored figure cannot be told from one computed against a different MegaMek. -17. **The attribution and the repair list on a match page.** +18. **The attribution and the repair list on a match page.** `plan/after-action.md`. The pieces are built and cross the wasm boundary; what is missing is the screen that reads them. -18. **Damage that persists between matches.** `plan/campaign.md`. The +19. **Damage that persists between matches.** `plan/campaign.md`. The mechanical half exists; storing a force's condition as a record and reading it back is the rest, and the construct-write-read round trip supports it. -19. **Formation validation.** `list_formations` and `validate_formation` +20. **Formation validation.** `list_formations` and `validate_formation` against the Campaign Operations blueprints - "is this a legal Fire Lance". Validation before generation, and it needs no faction data: it is role and count, both of which helm holds. -20. **Ingest `data/forcegenerator/`.** 40 era files of per-faction availability +21. **Ingest `data/forcegenerator/`.** 40 era files of per-faction availability weights, `factions.xml`, and 61 faction rulesets that are a small DSL of their own. This is the foundation for force generation, faction filtering and era-correct scenarios, and nothing above it should be attempted until diff --git a/scripts/deploy.sh b/scripts/deploy.sh --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -16,6 +16,7 @@ # # /helm_wasm.wasm the rules, compiled for a browser # //units.json the index a force-building screen filters +# //loadout.json what each of those designs carries # //catalogue.jsonl the equipment the rules read # # https://lance.blue/assets/helm///units.json @@ -168,6 +169,8 @@ --helm-version "$ref" \ --out "$out/units.json" +# From the build block, which is also what says this index and the loadout +# beside it are about the same designs. mm="$(grep -o '"megamek":"[^"]*"' "$out/units.json" | head -1 | cut -d'"' -f4)" [ -n "$mm" ] || { echo "deploy: the index names no MegaMek version" >&2 @@ -179,7 +182,7 @@ cp target/wasm32-unknown-unknown/release/helm_wasm.wasm "$out/helm_wasm.wasm" echo -for file in units.json catalogue.jsonl helm_wasm.wasm; do +for file in units.json loadout.json catalogue.jsonl helm_wasm.wasm; do printf ' %-16s %6s KB\n' "$file" "$(( $(wc -c <"$out/$file") / 1024 ))" done echo @@ -199,6 +202,8 @@ --content-type application/wasm --cache-control "$cache" --no-progress aws s3 cp "$out/units.json" "s3://$bucket/$ref/$mm/units.json" \ --content-type application/json --cache-control "$cache" --no-progress +aws s3 cp "$out/loadout.json" "s3://$bucket/$ref/$mm/loadout.json" \ + --content-type application/json --cache-control "$cache" --no-progress aws s3 cp "$out/catalogue.jsonl" "s3://$bucket/$ref/$mm/catalogue.jsonl" \ --content-type application/x-ndjson --cache-control "$cache" --no-progress @@ -211,6 +216,7 @@ echo "Published under $ref/$mm:" echo " https://lance.blue/assets/helm/$ref/helm_wasm.wasm" echo " https://lance.blue/assets/helm/$ref/$mm/units.json" +echo " https://lance.blue/assets/helm/$ref/$mm/loadout.json" echo " https://lance.blue/assets/helm/$ref/$mm/catalogue.jsonl" echo echo "Wrote releases.helm = $ref" diff --git a/crates/helm-wasm/facets.mjs b/crates/helm-wasm/facets.mjs --- a/crates/helm-wasm/facets.mjs +++ b/crates/helm-wasm/facets.mjs @@ -1,9 +1,10 @@ // Filter the whole library through the wasm boundary and check it against what // the native build selected. // -// helm facets --megamek --bridge-dir --out /facets.json \ +// helm index --megamek --bridge-dir --out /units.json \ // --filters crates/helm-wasm/filters.json -// node crates/helm-wasm/facets.mjs /facets.json /answers.json +// node crates/helm-wasm/facets.mjs /units.json /loadout.json \ +// /answers.json // // Why this exists: `unit-search` says one filter definition has to answer the // same way in three places - an agent over MCP, the server, and the browser - @@ -20,9 +21,9 @@ import { readFileSync } from "node:fs"; import { Helm } from "./helm.mjs"; -const [facets, answers] = process.argv.slice(2); -if (!facets || !answers) { - console.error("usage: node facets.mjs "); +const [index, loadout, answers] = process.argv.slice(2); +if (!index || !loadout || !answers) { + console.error("usage: node facets.mjs "); process.exit(2); } @@ -33,8 +34,22 @@ const helm = await Helm.load(readFileSync(wasm)); const started = Date.now(); -const library = helm.library(readFileSync(facets, "utf8")); -const loaded = Date.now() - started; +const library = helm.library(readFileSync(index, "utf8")); +const indexed = Date.now() - started; + +// What a page does before anybody has filtered by equipment: a filter that +// asks what a design carries is refused rather than answered against designs +// that carry nothing. +let unarmed = null; +try { + library.select({ carries: ["ISGaussRifle"] }); +} catch (error) { + unarmed = error; +} + +const joining = Date.now(); +library.loadout(readFileSync(loadout, "utf8")); +const joined = Date.now() - joining; const native = JSON.parse(readFileSync(answers, "utf8")); const wrong = []; @@ -62,10 +77,14 @@ const took = Date.now() - filtering; console.log( - `${library.size} units loaded in ${loaded}ms, ` + + `${library.size} units in ${indexed}ms, loadout joined in ${joined}ms, ` + `${native.length} filters selecting ${selected} rows in ${took}ms, ` + `${wrong.length} disagreeing`, ); +if (!unarmed) { + console.log(" an equipment filter was answered before the loadout was joined"); + wrong.push("unjoined equipment filter"); +} for (const line of wrong.slice(0, 20)) { console.log(` ${line}`); } @@ -81,6 +100,21 @@ if (!refused?.valid?.length) { console.log(" a filter naming an unknown role was not refused with the roles there are"); wrong.push("unknown role"); +} + +// A loadout from another run lines up row for row and means different +// designs, so it is refused rather than joined. +const forged = JSON.parse(readFileSync(loadout, "utf8")); +forged.build.id = "0000000000000000"; +let refusedJoin = null; +try { + helm.library(readFileSync(index, "utf8")).loadout(JSON.stringify(forged)); +} catch (error) { + refusedJoin = error; +} +if (!refusedJoin) { + console.log(" a loadout from another build was joined anyway"); + wrong.push("mismatched join"); } library.free(); diff --git a/crates/helm-wasm/helm.mjs b/crates/helm-wasm/helm.mjs --- a/crates/helm-wasm/helm.mjs +++ b/crates/helm-wasm/helm.mjs @@ -20,7 +20,7 @@ // including when the call throws. /** The boundary this wrapper was written against; the module must agree. */ -const ABI_VERSION = 3n; +const ABI_VERSION = 4n; const ERRORS = new Map([ [-1n, "no equipment catalogue is loaded"], @@ -150,16 +150,16 @@ } /** - * Read a library of units to filter, as `helm facets` writes it. + * Read a library of units to filter, as `helm index` writes it. * * The whole document is handed over once and kept on the far side. A page * filters as fast as somebody types, and copying ten thousand rows across * the boundary per keystroke is the one way to make a predicate that costs * under a millisecond feel slow. */ - library(facets) { + library(index) { const handle = check( - this.#withBytes(facets, (ptr, len) => this.#api.helm_library_load(ptr, len)), + this.#withBytes(index, (ptr, len) => this.#api.helm_library_load(ptr, len)), ); return new Library(this.#api, handle, (text, use) => this.#withBytes(text, use), (packed) => this.#take(packed), @@ -478,6 +478,30 @@ /** How many units it holds. */ get size() { return Number(check(this.#api.helm_library_len(this.#live))); + } + + /** + * Put what each design carries onto it, from the loadout document. + * + * Its own file because most filtering never asks, and it is a quarter of + * the bytes: fetch it the first time somebody filters by equipment. Until + * then a filter that asks what a design carries is refused rather than + * answered with silence. + * + * Throws if the loadout is about another run of helm. The two are joined by + * position, so a mismatch is not a smaller answer - it is the wrong one. + */ + loadout(loadout) { + const answer = this.#withBytes(loadout, (ptr, len) => { + const packed = this.#api.helm_library_loadout(this.#live, ptr, len); + return packed === 0n ? null : this.#take(packed); + }); + if (answer !== null) { + const { error } = JSON.parse(answer); + const failure = new HelmError(0n); + failure.message = error; + throw failure; + } } /** diff --git a/crates/helm-cli/src/build.rs b/crates/helm-cli/src/build.rs --- a/crates/helm-cli/src/build.rs +++ b/crates/helm-cli/src/build.rs @@ -332,6 +332,63 @@ }) } +/// The designs every published file is about, in the order every one of them +/// writes. +/// +/// The index and the loadout beside it are joined by position, so "which +/// designs, in what order" has to be one answer rather than two that happen +/// to agree today. A design MegaMek marks non-canon or unplayable is dropped +/// here, once. +fn fieldable<'a>( + library: &'a helm_unitfile::Library, + by_name: &std::collections::HashMap, + all_types: bool, +) -> Vec<&'a helm_core::Unit> { + // What a match can field and this repository can score: Meks, combat + // vehicles, battle armour and conventional infantry - the four `helm-bv` + // has a calculator for. + // + // `Unit`'s own predicates, not a second reading of `unit_type`: an `.mtf` + // declares no type at all, so a filter that trusts the field drops every + // Mek written in that format. + let mut units: Vec<&helm_core::Unit> = library + .units + .iter() + .filter(|u| { + all_types + || u.is_mek() + || u.is_combat_vehicle() + || u.is_battle_armor() + || u.is_conventional_infantry() + }) + // Canon and validity are MegaMek's judgement and arrive with its own + // figures. Without a producer nothing is dropped and a screen offers a + // unit MegaMek marks unplayable, which `scored` says out loud. + .filter(|u| { + !by_name + .get(&u.name) + .is_some_and(|s| s.canon == Some(false) || s.invalid == Some(true)) + }) + .collect(); + units.sort_by(|a, b| a.name.cmp(&b.name).then(a.path.cmp(&b.path))); + units +} + +/// The identity a published file carries, from the designs in it. +fn identity( + provenance: &Provenance, + helm_version: Option<&str>, + units: &[&helm_core::Unit], +) -> helm_query::Build { + helm_query::Build::new( + &provenance.megamek_version, + helm_version, + &provenance.stats_producer, + provenance.rules_version.as_deref(), + units.iter().map(|u| u.name.as_str()), + ) +} + pub(crate) fn index(args: &[String]) -> Result<(), String> { let opts = parse_opts(args)?; let FromInstall { @@ -349,126 +406,134 @@ provenance, } = scored(&opts, &library, &version)?; - // What a match can field and this repository can score: Meks, combat - // vehicles, battle armour and conventional infantry - the four `helm-bv` - // has a calculator for. Everything else is behind --all-types, which is - // 10,896 designs and 4.8MB, gun emplacements, buildings and handheld + // Everything outside the four scoreable types is behind --all-types, which + // is 10,896 designs and 4.8MB, gun emplacements, buildings and handheld // weapons among them, none of which a force offers. // // The list tracks what can be priced rather than what can be drawn, so a // design with no art still gets a row: art is a picture and battle value // is a fact a screen filters on. 93 platoons have no `mekset.txt` entry, // and MegaMek draws those with a default silhouette too. - // - // `Unit`'s own predicates, not a second reading of `unit_type` here: an - // `.mtf` declares no type at all, so a filter that trusts the field - // drops every Mek written in that format. - let all_types = opts.all_types; - let mut units: Vec<&helm_core::Unit> = library - .units - .iter() - .filter(|u| { - all_types - || u.is_mek() - || u.is_combat_vehicle() - || u.is_battle_armor() - || u.is_conventional_infantry() - }) - .collect(); - units.sort_by(|a, b| a.name.cmp(&b.name).then(a.path.cmp(&b.path))); + let units = fieldable(&library, &by_name, opts.all_types); + let build = identity(&provenance, opts.helm_version.as_deref(), &units); + let mut counts = helm_query::Counts { + units: units.len(), + ..Default::default() + }; let mut rows = Vec::with_capacity(units.len()); - let mut without_sprite = 0usize; - let mut without_bv = 0usize; - for unit in units { + for unit in &units { let stats = by_name.get(&unit.name); - - // Canon and validity are MegaMek's judgement and arrive with the - // computed values, so a library read without a producer cannot filter - // on them. Without one, everything is offered and the caller is told. - if let Some(s) = stats - && (s.canon == Some(false) || s.invalid == Some(true)) - { - continue; - } - let sprite = set.art_for(unit).map(|art| art.path.to_string()); if sprite.is_none() { - without_sprite += 1; + counts.without_sprite += 1; } - let bv = stats.and_then(|s| s.battle_value); - if bv.is_none() { - without_bv += 1; + if stats.and_then(|s| s.battle_value).is_none() { + counts.without_battle_value += 1; } - let metrics = catalogue - .as_ref() - .map(|cat| helm_core::CombatMetrics::compute(unit, cat)); - - rows.push(serde_json::json!({ - "chassis": unit.chassis, - "model": unit.model, - "name": unit.name, - "mul_id": unit.mul_id, - "unit_type": unit.unit_type, - "config": unit.config, - "tech_base": unit.tech_base, - "rules_level": unit.rules_level, - "role": unit.role, - "source": unit.source, - "year": unit.year, - "mass": unit.mass, - "walk_mp": unit.walk_mp, - "jump_mp": unit.jump_mp, - "run_mp": stats.and_then(|s| s.run_mp).or(unit.run_mp), - "total_armor": unit.total_armor(), - "bv": bv, - "cost": stats.and_then(|s| s.cost), - "weight_class": stats.and_then(|s| s.weight_class), - "omni": stats.and_then(|s| s.omni), - "clan": stats.and_then(|s| s.clan), - "sprite": sprite, - "firepower": metrics.as_ref().map(|m| m.firepower), - "max_range": metrics.as_ref().map(|m| m.max_range), - "heat_efficiency": metrics.as_ref().map(helm_core::CombatMetrics::heat_efficiency), - })); + rows.push(helm_query::IndexRow { + facets: helm_query::Row(helm_facet::UnitFacets::from_unit( + unit, + stats, + catalogue.as_ref(), + )), + sprite, + total_armor: Some(unit.total_armor()), + }); } - let doc = serde_json::json!({ - "megamek": provenance.megamek_version, - // Which helm wrote this, whichever producer filled the battle values. - // The file is a function of both: the columns and the derived figures - // are this code's, the library and the figures are that MegaMek's, and - // an artifact keyed by one of them alone is one that can be quietly - // replaced by a different build. - "helm": opts.helm_version, - "stats_producer": provenance.stats_producer, - "rules_version": provenance.rules_version, - // Relative to the release, and the prefix a sprite path is joined to. - // The bucket holds the tree at this path under the same version, so a - // consumer needs no second convention to turn one into an address. - "sprite_base": "data/images/units", - "counts": { - "units": rows.len(), - "without_battle_value": without_bv, - "without_sprite": without_sprite, - }, - "units": rows, - }); + let mut document = helm_query::Index { + build: build.clone(), + sprite_base: "data/images/units".to_string(), + counts, + units: rows, + }; - let text = serde_json::to_string(&doc).map_err(|e| e.to_string())?; + // Answered while the two halves are still one, so what is recorded is + // what a page gets after it has joined them. + let answers = match &opts.filters { + Some(corpus) => Some(answer(&document, corpus)?), + None => None, + }; + + // Moved out of the rows rather than copied beside them: two files holding + // the same list is two chances for one of them to be stale. + let carried: Vec> = document + .units + .iter_mut() + .map(|row| std::mem::take(&mut row.facets.0.loadout)) + .collect(); + + let text = serde_json::to_string(&document).map_err(|e| e.to_string())?; std::fs::write(&out, format!("{text}\n")).map_err(|e| format!("{}: {e}", out.display()))?; - println!( - "wrote {} - {} units, {:.0} KB", + "wrote {} - {} units, {:.0} KB, build {}", out.display(), - rows.len(), + document.units.len(), + text.len() as f64 / 1024.0, + build.id + ); + if document.counts.without_battle_value > 0 || document.counts.without_sprite > 0 { + println!( + " {} without a battle value, {} without art", + document.counts.without_battle_value, document.counts.without_sprite + ); + } + + let beside = out.with_file_name("loadout.json"); + let equipment = helm_query::Loadout { + build, + loadout: carried, + }; + let text = serde_json::to_string(&equipment).map_err(|e| e.to_string())?; + std::fs::write(&beside, format!("{text}\n")) + .map_err(|e| format!("{}: {e}", beside.display()))?; + println!( + "wrote {} - {} entries, {:.0} KB", + beside.display(), + equipment.loadout.iter().map(Vec::len).sum::(), text.len() as f64 / 1024.0 ); - if without_bv > 0 || without_sprite > 0 { - println!(" {without_bv} without a battle value, {without_sprite} without art"); + + if let Some(answers) = answers { + let beside = out.with_file_name("answers.json"); + std::fs::write(&beside, format!("{answers}\n")) + .map_err(|e| format!("{}: {e}", beside.display()))?; + println!("wrote {} - what this build answers", beside.display()); } Ok(()) } + +/// What every filter in a corpus selects, as this build answers it. +/// +/// The other half of `crates/helm-wasm/facets.mjs`: the script runs the same +/// corpus over the same documents through the wasm module, and a disagreement +/// is the one failure that matters here - the predicate compiled twice and +/// answering differently is exactly what running it in a browser is supposed +/// to rule out. +fn answer(document: &helm_query::Index, corpus: &Path) -> Result { + let text = std::fs::read_to_string(corpus).map_err(|e| format!("{}: {e}", corpus.display()))?; + let filters: Vec = serde_json::from_str(&text).map_err(|e| e.to_string())?; + let units = document.facets(); + let vocabulary = helm_query::Vocabulary::from_facets( + &units.iter().map(|u| (*u).clone()).collect::>(), + ); + + let mut out = Vec::with_capacity(filters.len()); + for value in filters { + let filter: helm_query::Filter = + serde_json::from_value(value.clone()).map_err(|e| format!("{value}: {e}"))?; + let query = filter + .to_query(&vocabulary) + .map_err(|e| format!("{value}: {e}"))?; + let hits: Vec = (0..units.len()) + .filter(|&i| query.matches(units[i])) + .collect(); + out.push(serde_json::json!({ "filter": value, "units": hits })); + } + serde_json::to_string(&out).map_err(|e| e.to_string()) +} + /// Write the equipment catalogue a browser needs, and nothing else. /// /// The bridge dumps every field MegaMek's `EquipmentType` carries, which is @@ -516,119 +581,4 @@ text.len() as f64 / 1_048_576.0 ); Ok(()) -} - -/// Write the columns a filter reads, for every design a match can field. -/// -/// The browser's half of `unit-search`. `index` writes what a screen *shows* - -/// a sprite path, a display name, the handful of numbers a row prints - and -/// this writes what a predicate *asks about*, which is a different list: the -/// loadout, the quirks, the books, and the figures nothing draws. -/// -/// One document, read by `helm_library_load`, filtered by the same -/// `helm-facet` predicate the server and the MCP server run. A page that -/// filtered these rows itself would be a second implementation of the -/// question, and the two would disagree on the first column with a surprise -/// in it - an unparseable bound, an absent battle value - of which there are -/// several. -pub(crate) fn facets(args: &[String]) -> Result<(), String> { - let opts = parse_opts(args)?; - let FromInstall { - out, - version, - library, - .. - } = from_install(&opts)?; - let Scored { - by_name, - catalogue, - provenance, - } = scored(&opts, &library, &version)?; - - // The same four types `index` offers, for the same reason: what a match - // can field and this repository can score. - let all_types = opts.all_types; - let mut units: Vec<&helm_core::Unit> = library - .units - .iter() - .filter(|u| { - all_types - || u.is_mek() - || u.is_combat_vehicle() - || u.is_battle_armor() - || u.is_conventional_infantry() - }) - .collect(); - units.sort_by(|a, b| a.name.cmp(&b.name).then(a.path.cmp(&b.path))); - - let mut rows = Vec::with_capacity(units.len()); - for unit in units { - let stats = by_name.get(&unit.name); - if let Some(s) = stats - && (s.canon == Some(false) || s.invalid == Some(true)) - { - continue; - } - rows.push(helm_query::Row(helm_facet::UnitFacets::from_unit( - unit, - stats, - catalogue.as_ref(), - ))); - } - - let document = helm_query::Facets { - megamek: provenance.megamek_version.clone(), - helm: opts.helm_version.clone(), - stats_producer: provenance.stats_producer.clone(), - rules_version: provenance.rules_version.clone(), - units: rows, - }; - let text = serde_json::to_string(&document).map_err(|e| e.to_string())?; - std::fs::write(&out, format!("{text}\n")).map_err(|e| format!("{}: {e}", out.display()))?; - - println!( - "wrote {} - {} units, {:.0} KB", - out.display(), - document.units.len(), - text.len() as f64 / 1024.0 - ); - - if let Some(corpus) = &opts.filters { - let answers = answer(&document, corpus)?; - let beside = out.with_file_name("answers.json"); - std::fs::write(&beside, format!("{answers}\n")) - .map_err(|e| format!("{}: {e}", beside.display()))?; - println!("wrote {} - what this build answers", beside.display()); - } - Ok(()) -} - -/// What every filter in a corpus selects, as this build answers it. -/// -/// The other half of `crates/helm-wasm/facets.mjs`: the script runs the same -/// corpus over the same document through the wasm module, and a disagreement -/// is the one failure that matters here - the predicate compiled twice and -/// answering differently is exactly what running it in a browser is supposed -/// to rule out. -fn answer(document: &helm_query::Facets, corpus: &Path) -> Result { - let text = std::fs::read_to_string(corpus).map_err(|e| format!("{}: {e}", corpus.display()))?; - let filters: Vec = serde_json::from_str(&text).map_err(|e| e.to_string())?; - let units = document.facets(); - let vocabulary = helm_query::Vocabulary::from_facets( - &units.iter().map(|u| (*u).clone()).collect::>(), - ); - - let mut out = Vec::with_capacity(filters.len()); - for value in filters { - let filter: helm_query::Filter = - serde_json::from_value(value.clone()).map_err(|e| format!("{value}: {e}"))?; - let query = filter - .to_query(&vocabulary) - .map_err(|e| format!("{value}: {e}"))?; - let hits: Vec = (0..units.len()) - .filter(|&i| query.matches(units[i])) - .collect(); - out.push(serde_json::json!({ "filter": value, "units": hits })); - } - serde_json::to_string(&out).map_err(|e| e.to_string()) } diff --git a/crates/helm-cli/src/main.rs b/crates/helm-cli/src/main.rs --- a/crates/helm-cli/src/main.rs +++ b/crates/helm-cli/src/main.rs @@ -24,7 +24,7 @@ mod report; mod wire; -use build::{art, build, catalogue, facets, index, stats}; +use build::{art, build, catalogue, index, stats}; use force::{check, force, repairs, wear}; use options::usage; use report::bv_report; @@ -38,7 +38,6 @@ Some("bv-report") => bv_report(&args[1..]), Some("art") => art(&args[1..]), Some("index") => index(&args[1..]), - Some("facets") => facets(&args[1..]), Some("catalogue") => catalogue(&args[1..]), Some("wire-check") => wire_check(&args[1..]), Some("force") => force(&args[1..]), diff --git a/crates/helm-cli/src/options.rs b/crates/helm-cli/src/options.rs --- a/crates/helm-cli/src/options.rs +++ b/crates/helm-cli/src/options.rs @@ -17,7 +17,7 @@ helm bv-report --megamek --bridge-dir helm art --megamek [--out ] helm index --megamek --bridge-dir --out [--helm-version ] - helm facets --megamek --bridge-dir --out [--helm-version ] + [--filters ] helm force --megamek --bridge-dir helm check --megamek --bridge-dir helm repairs --megamek --bridge-dir @@ -103,7 +103,7 @@ pub(crate) all_types: bool, /// `index`: the release id of the helm that built it. pub(crate) helm_version: Option, - /// `facets`: a corpus of filters to answer beside the document, for + /// `index`: a corpus of filters to answer beside the documents, for /// `facets.mjs` to check the wasm module against. pub(crate) filters: Option, pub(crate) label: Option, diff --git a/crates/helm-query/src/facets.rs b/crates/helm-query/src/facets.rs --- a/crates/helm-query/src/facets.rs +++ b/crates/helm-query/src/facets.rs @@ -14,6 +14,7 @@ //! design with no computed record - so every optional field is skipped when it //! is empty and defaulted when it is missing. +use crate::Build; use helm_facet::UnitFacets; use serde::{Deserialize, Serialize}; @@ -42,9 +43,9 @@ weight_class: Option, #[serde(skip_serializing_if = "Option::is_none")] year: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "mass", skip_serializing_if = "Option::is_none")] tons: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "bv", skip_serializing_if = "Option::is_none")] battle_value: Option, #[serde(skip_serializing_if = "Option::is_none")] walk_mp: Option, @@ -92,37 +93,209 @@ #[derive(Debug, Serialize, Deserialize)] pub struct Row(#[serde(with = "UnitFacetsDef")] pub UnitFacets); -/// Every unit a filter may select, and which build wrote them. +/// One design as the index carries it: what a filter reads, and the two +/// things only a screen wants. /// -/// The provenance travels with the units rather than beside them: a figure -/// computed against a different MegaMek is a different figure, and a document -/// that does not say which one it came from cannot be told from one that did. -#[derive(Serialize, Deserialize)] -pub struct Facets { - /// The MegaMek release the designs were read from. - pub megamek: String, - /// The helm build that wrote the file, where the writer knows its own - /// version. +/// The facets are flattened rather than nested because this file is read by a +/// page that draws a list as well as by one that filters it, and a row of +/// twenty-five columns with two of them behind a `facets` key is a shape +/// nobody would choose to write against. +#[derive(Debug, Serialize, Deserialize)] +pub struct IndexRow { + #[serde(flatten)] + pub facets: Row, + /// Where the design's picture is, relative to the release's art tree. #[serde(default, skip_serializing_if = "Option::is_none")] - pub helm: Option, - /// Who computed the battle values: MegaMek's own bridge, or helm-bv. - pub stats_producer: String, - /// Which rules produced them, where they are helm's. + pub sprite: Option, + /// Plate as the file declares it. `armor_pct` is what a filter asks + /// about; this is what a row prints. #[serde(default, skip_serializing_if = "Option::is_none")] - pub rules_version: Option, - pub units: Vec, + pub total_armor: Option, } -impl Facets { +/// How much of the library is answered, which is not the same as how large it +/// is. +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct Counts { + pub units: usize, + /// Designs no producer could score. A screen showing none of them is + /// showing a shorter library than it says. + pub without_battle_value: usize, + /// Designs MegaMek draws with a default silhouette. + pub without_sprite: usize, +} + +/// Every unit a page draws or filters, and which build wrote them. +/// +/// One document rather than two. An index of what to show and an index of +/// what to filter on are ninety percent the same columns, and publishing both +/// means a page fetches a megabyte twice to answer one screen. +/// +/// What each design *carries* is a document of its own, [`Loadout`], because +/// a screen filtering by tonnage and year needs none of it and it is a +/// quarter of the bytes. +#[derive(Serialize, Deserialize)] +pub struct Index { + pub build: Build, + /// Relative to the release, and the prefix a sprite path is joined to. + /// The bucket holds the art tree at this path under the same version, so + /// a consumer needs no second convention to turn one into an address. + pub sprite_base: String, + pub counts: Counts, + pub units: Vec, +} + +impl Index { /// The units, as the predicate wants them. pub fn facets(&self) -> Vec<&UnitFacets> { - self.units.iter().map(|row| &row.0).collect() + self.units.iter().map(|row| &row.facets.0).collect() + } +} + +/// What each design carries, in the order [`Facets`] wrote them. +/// +/// Joined by position and checked by [`Build::joins`]. The alternative was a +/// key per row, and there is not one: 479 designs in 0.51.0 share a display +/// name with another, and nothing in a `.mtf` distinguishes them. +/// +/// Kept apart from the facets because it is the expensive half - 60,881 +/// entries over 8,666 designs - and because most filtering never asks. A page +/// fetches it when somebody first filters by equipment, and until then the +/// question it answers is not on the screen. +#[derive(Serialize, Deserialize)] +pub struct Loadout { + pub build: Build, + /// One list per design: `(equipment name, quantity)`, MegaMek's own + /// spelling, invented entries included. + pub loadout: Vec>, +} + +/// Why a loadout document could not be put onto a set of facets. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Mismatch { + /// The two files are about different runs of helm. + DifferentBuild { + index: Box, + loadout: Box, + }, + /// The same run, and one of the files is truncated. + DifferentLength { index: usize, loadout: usize }, +} + +impl std::fmt::Display for Mismatch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Mismatch::DifferentBuild { index, loadout } => { + write!(f, "this index is {index} and that loadout is {loadout}") + } + Mismatch::DifferentLength { index, loadout } => write!( + f, + "{index} designs are indexed and {loadout} have a loadout" + ), + } + } +} + +impl std::error::Error for Mismatch {} + +impl Index { + /// Put each design's equipment onto it, or say why the two files are not + /// about the same designs. + /// + /// A positional join is right or it is nonsense: nothing about the fifth + /// row of one file says whether it is the fifth design of the other. So + /// this is the only way the two are put together. + pub fn join(&mut self, loadout: Loadout) -> Result<(), Mismatch> { + if !self.build.joins(&loadout.build) { + return Err(Mismatch::DifferentBuild { + index: Box::new(self.build.clone()), + loadout: Box::new(loadout.build), + }); + } + if self.units.len() != loadout.loadout.len() { + return Err(Mismatch::DifferentLength { + index: self.units.len(), + loadout: loadout.loadout.len(), + }); + } + for (row, carried) in self.units.iter_mut().zip(loadout.loadout) { + row.facets.0.loadout = carried; + } + Ok(()) } } #[cfg(test)] mod tests { use super::*; + + fn build(names: &[&str]) -> Build { + Build::new( + "0.51.0", + None, + "megamek-bridge", + None, + names.iter().copied(), + ) + } + + fn document(names: &[&str]) -> Index { + Index { + build: build(names), + sprite_base: "data/images/units".into(), + counts: Counts::default(), + units: names + .iter() + .map(|name| IndexRow { + facets: Row(UnitFacets { + name: (*name).into(), + ..Default::default() + }), + sprite: None, + total_armor: None, + }) + .collect(), + } + } + + /// The join the split exists for: the equipment arrives later, from its + /// own file, and lands on the design it belongs to. + #[test] + fn a_loadout_lands_on_the_design_it_belongs_to() { + let mut facets = document(&["Atlas AS7-D", "Locust LCT-1V"]); + let loadout = Loadout { + build: build(&["Atlas AS7-D", "Locust LCT-1V"]), + loadout: vec![ + vec![("ISGaussRifle".into(), 1)], + vec![("Machine Gun".into(), 2)], + ], + }; + facets.join(loadout).expect("the same designs"); + assert_eq!( + facets.units[1].facets.0.loadout, + [("Machine Gun".to_string(), 2)] + ); + } + + /// Two files from different runs line up row for row and mean different + /// designs. Refused, because the alternative is a page telling somebody a + /// Locust carries a Gauss rifle. + #[test] + fn a_loadout_from_another_run_is_refused() { + let mut facets = document(&["Atlas AS7-D", "Locust LCT-1V"]); + let loadout = Loadout { + build: build(&["Atlas AS7-D", "Banshee BNC-3E"]), + loadout: vec![vec![], vec![]], + }; + let err = facets.join(loadout).expect_err("a different run"); + assert!(matches!(err, Mismatch::DifferentBuild { .. }), "{err}"); + assert!( + facets + .units + .iter() + .all(|row| row.facets.0.loadout.is_empty()) + ); + } /// A round trip through the document is the whole contract: what the /// server filtered is what the browser filters. diff --git a/crates/helm-query/src/lib.rs b/crates/helm-query/src/lib.rs --- a/crates/helm-query/src/lib.rs +++ b/crates/helm-query/src/lib.rs @@ -19,7 +19,7 @@ mod facets; pub use build::Build; -pub use facets::{Facets, Row}; +pub use facets::{Counts, Index, IndexRow, Loadout, Mismatch, Row}; use std::collections::BTreeSet; @@ -265,6 +265,23 @@ } impl Filter { + /// Whether this filter asks what a design carries. + /// + /// A library with no loadout joined to it answers those with silence - + /// every design carrying nothing - which reads exactly like an honest + /// empty result. So a caller that can go and fetch the loadout is told to + /// rather than shown one. + pub fn asks_about_equipment(&self) -> bool { + [ + &self.carries, + &self.lacks, + &self.carries_class, + &self.lacks_class, + ] + .into_iter() + .any(|list| list.as_ref().is_some_and(|l| !l.is_empty())) + } + /// The predicate this filter means, or the first thing in it the library /// does not have. pub fn to_query(&self, vocabulary: &Vocabulary) -> Result { diff --git a/crates/helm-wasm/src/lib.rs b/crates/helm-wasm/src/lib.rs --- a/crates/helm-wasm/src/lib.rs +++ b/crates/helm-wasm/src/lib.rs @@ -479,8 +479,11 @@ /// query: it is four sets over ten thousand rows, and a page filters as fast /// as somebody types. struct Filterable { - units: Vec, + index: helm_query::Index, vocabulary: helm_query::Vocabulary, + /// Whether the loadout document has been joined on. Until it has, what + /// each design carries is not in memory at all. + carrying: bool, } /// Read a library of units a filter can select from, and keep it. @@ -496,14 +499,19 @@ let Some(json) = (unsafe { text(ptr, len) }) else { return ERR_NOT_UTF8; }; - let Ok(document) = serde_json::from_str::(json) else { + let Ok(index) = serde_json::from_str::(json) else { return ERR_UNREADABLE; }; - let units: Vec = document.units.into_iter().map(|row| row.0).collect(); - let vocabulary = helm_query::Vocabulary::from_facets(&units); + let vocabulary = helm_query::Vocabulary::from_facets( + &index.facets().into_iter().cloned().collect::>(), + ); LIBRARIES.with(|l| { let mut libraries = l.borrow_mut(); - let held = Filterable { units, vocabulary }; + let held = Filterable { + index, + vocabulary, + carrying: false, + }; let handle = match libraries.iter().position(|slot| slot.is_none()) { Some(free) => { libraries[free] = Some(held); @@ -540,7 +548,41 @@ /// missing one. #[unsafe(no_mangle)] pub extern "C" fn helm_library_len(handle: i64) -> i64 { - with_library(handle, |held| held.units.len() as i64) + with_library(handle, |held| held.index.units.len() as i64) +} + +/// Put what each design carries onto a library already loaded. +/// +/// The loadout is its own document because most filtering never asks what a +/// design carries, and it is a quarter of the bytes. A page fetches it the +/// first time somebody filters by equipment. +/// +/// The two files are joined by position and checked by the build each carries. +/// A loadout from another run of helm lines up row for row and means different +/// designs, so it is refused with what the two builds are. +/// +/// # Safety +/// `ptr` must point to `len` readable bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn helm_library_loadout(handle: i64, ptr: *const u8, len: usize) -> i64 { + let Some(json) = (unsafe { text(ptr, len) }) else { + return ERR_NOT_UTF8; + }; + let Ok(loadout) = serde_json::from_str::(json) else { + return ERR_UNREADABLE; + }; + with_library_mut(handle, |held| match held.index.join(loadout) { + Ok(()) => { + held.carrying = true; + 0 + } + Err(mismatch) => give( + serde_json::json!({ + "error": mismatch.to_string(), + }) + .to_string(), + ), + }) } /// The units a filter selects, by their place in the library. @@ -567,12 +609,22 @@ Err(_) => return ERR_BAD_STATE, }; with_library(handle, |held| { + if filter.asks_about_equipment() && !held.carrying { + return give( + serde_json::json!({ + "error": "this filter asks what a design carries and no loadout is loaded", + "field": "loadout", + }) + .to_string(), + ); + } let query = match filter.to_query(&held.vocabulary) { Ok(query) => query, Err(invalid) => return give(refused(&invalid).to_string()), }; - let hits: Vec = (0..held.units.len()) - .filter(|&i| query.matches(&held.units[i])) + let units = held.index.facets(); + let hits: Vec = (0..units.len()) + .filter(|&i| query.matches(units[i])) .collect(); give(serde_json::json!({ "units": hits }).to_string()) }) @@ -608,6 +660,20 @@ }) } +/// The same, to put a loadout onto one. +fn with_library_mut(handle: i64, f: impl FnOnce(&mut Filterable) -> i64) -> i64 { + LIBRARIES.with(|l| { + let mut libraries = l.borrow_mut(); + match usize::try_from(handle - 1) + .ok() + .and_then(|i| libraries.get_mut(i)) + { + Some(Some(held)) => f(held), + _ => ERR_NO_LIBRARY, + } + }) +} + /// Which shape of boundary this module speaks. /// /// Bumped when an exported function changes its arguments or their meaning, @@ -623,9 +689,13 @@ /// 3 adds `helm_library_load` and the filtering beside it. Additive again, and /// a page that means to filter here rather than by hand has to be able to tell /// that this module can. +/// +/// 4 splits what a design carries into its own document, joined on with +/// `helm_library_loadout`. Not additive: a page on 3 handed a 4 index would +/// filter by equipment against designs that carry nothing. #[unsafe(no_mangle)] pub extern "C" fn helm_abi_version() -> i64 { - 3 + 4 } /// How many equipment entries are loaded, so a caller can tell an empty -- tangled.sh