Something went wrong. Try again.
Data management tools for playing BattleTech
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637//! The commands that write something: the database, the browser's index, the//! art mapping and the trimmed catalogue.//!//! Each reads a MegaMek install and a bridge dump and produces a file//! somebody else consumes, which is what separates them from the reports.
use std::collections::BTreeSet;use std::path::Path;
use helm_core::{Catalogue, ComputedStats, Provenance};use helm_db::Inputs;use helm_unitfile::mekset::Via;
use crate::inputs::{FromInstall, from_install, read_library, resolve_version};use crate::options::parse_opts;
pub(crate) fn build(args: &[String]) -> Result<(), String> { let opts = parse_opts(args)?; let FromInstall { out, version, library, .. } = from_install(&opts)?;
// Choosing the producer of ComputedStats is this function's job and // nothing else's. When helm-bv exists it is chosen here, beside the // bridge, and no other crate changes. let mut computed: Vec<ComputedStats> = Vec::new(); let mut catalogue: Option<Catalogue> = None; let mut provenance = Provenance::none_yet(&version);
if let Some(dir) = &opts.bridge_dir { let units = dir.join("units.jsonl"); let equipment = dir.join("equipment.jsonl"); if !units.is_file() && !equipment.is_file() { return Err(format!( "--bridge-dir {} holds neither units.jsonl nor equipment.jsonl", dir.display() )); } if units.is_file() { computed = helm_bridge::read_units(&units).map_err(|e| e.to_string())?; } if equipment.is_file() { catalogue = Some(helm_bridge::read_catalogue(&equipment).map_err(|e| e.to_string())?); } provenance = Provenance::from_bridge(&version); eprintln!( "bridge: {} unit records, {} equipment types", computed.len(), catalogue.as_ref().map_or(0, Catalogue::len) ); } else { eprintln!("no --bridge-dir: computed columns will be null"); }
let had_producer = !computed.is_empty(); let inputs = Inputs { stats: &computed, catalogue: catalogue.as_ref(), }; let stats = helm_db::build(&library, &inputs, &provenance, &out) .map_err(|e| format!("writing {}: {e}", out.display()))?;
eprintln!( "wrote {} - {} units, {} mounts, {} loadout rows, {} quirks, {} crit slots, \ {} catalogue entries", out.display(), stats.units, stats.equipment_rows, stats.loadout_rows, stats.quirk_rows, stats.critical_rows, stats.catalogue_rows ); if had_producer { let missed = stats.units.saturating_sub(stats.with_stats); eprintln!( " {} units carry computed values, {missed} matched no record ({})", stats.with_stats, provenance.stats_producer ); } if !stats.fts { eprintln!(" note: this SQLite has no FTS5, so units_fts was not built"); } Ok(())}pub(crate) fn stats(args: &[String]) -> Result<(), String> { let path = args.first().ok_or("stats needs a database path")?; let db = rusqlite::Connection::open(Path::new(path)).map_err(|e| e.to_string())?;
let mut stmt = db .prepare("SELECT key, value FROM meta ORDER BY key") .map_err(|e| e.to_string())?; let rows = stmt .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))) .map_err(|e| e.to_string())?; for row in rows { let (k, v) = row.map_err(|e| e.to_string())?; println!("{k:20} {v}"); }
let mut stmt = db .prepare("SELECT unit_type, COUNT(*) FROM units GROUP BY unit_type ORDER BY 2 DESC") .map_err(|e| e.to_string())?; let rows = stmt .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?))) .map_err(|e| e.to_string())?; println!("\nby unit type:"); for row in rows { let (t, n) = row.map_err(|e| e.to_string())?; println!(" {n:6} {t}"); } Ok(())}/// `helm art` - which picture belongs to which design.////// The art tree is 6,995 files and says nothing about which design uses which/// of them; `mekset.txt` is the mapping, and reading it is the only part of/// serving MegaMek's unit art that is not a file copy. So this writes the/// mapping and leaves the bytes alone: the sync that puts the tree in a bucket/// is `aws s3 sync data/images/units/`, and what needs generating is this.////// `files` is what that sync is expected to carry, and `missing_files` is what/// the set file names and the release does not ship - a sync that came out/// short can be told from a release that was short to begin with.pub(crate) fn art(args: &[String]) -> Result<(), String> { let opts = parse_opts(args)?; let megamek = opts .megamek .ok_or("--megamek is required: point it at a MegaMek install")?; let version = resolve_version(&megamek, opts.version); let units_dir = megamek.join("data/images/units"); if !units_dir.join("mekset.txt").is_file() { return Err(format!( "{} holds no mekset.txt - point --megamek at an install, not at its data directory", units_dir.display() )); }
let set = helm_unitfile::read_mekset(&units_dir).map_err(|e| e.to_string())?; let (exact, chassis) = set.counts(); eprintln!("mekset: {exact} exact entries, {chassis} chassis entries");
let library = read_library(&megamek)?;
let mut rows = Vec::new(); let mut unmatched = Vec::new(); let mut files: BTreeSet<String> = BTreeSet::new(); let mut by_via = [0usize; 3];
let mut units: Vec<&helm_core::Unit> = library.units.iter().collect(); units.sort_by(|a, b| a.name.cmp(&b.name).then(a.path.cmp(&b.path)));
for unit in units { match set.art_for(unit) { Some(art) => { let via = match art.via { Via::Exact => "exact", Via::Chassis => "chassis", Via::Generic => "generic", }; by_via[match art.via { Via::Exact => 0, Via::Chassis => 1, Via::Generic => 2, }] += 1; files.insert(art.path.to_string()); rows.push(serde_json::json!({ "chassis": unit.chassis, "model": unit.model, "name": unit.name, "unit_type": unit.unit_type, "sprite": art.path, "via": via, })); } None => unmatched.push(unit.name.clone()), } }
// Named by the set file and not in the release. Reported rather than // dropped: it is the one failure here that is upstream's rather than // ours, and a sync cannot tell the two apart on its own. let missing: Vec<&String> = files .iter() .filter(|path| !units_dir.join(path).is_file()) .collect();
let doc = serde_json::json!({ "megamek": version, "source": "data/images/units/mekset.txt", "prefix": "data/images/units", "counts": { "units": rows.len(), "exact": by_via[0], "chassis": by_via[1], "generic": by_via[2], "unmatched": unmatched.len(), "files": files.len(), "missing_files": missing.len(), }, "units": rows, "files": files, "missing_files": missing, "unmatched": unmatched, });
let text = serde_json::to_string_pretty(&doc).map_err(|e| e.to_string())?; match &opts.out { Some(out) => { std::fs::write(out, format!("{text}\n")) .map_err(|e| format!("{}: {e}", out.display()))?; println!("wrote {}", out.display()); } None => println!("{text}"), }
println!( "{} units: {} exact, {} chassis, {} generic, {} with no art", rows.len(), by_via[0], by_via[1], by_via[2], unmatched.len() ); println!( "{} distinct files{}", files.len(), if missing.is_empty() { String::new() } else { format!(", {} of them not in this release", missing.len()) } ); Ok(())}/// `helm index` - the browser's view of the library.////// The database is the server's and the agent's: 112MB of SQLite, indexed for/// arbitrary queries. A force-building screen wants none of that and cannot/// carry it. What it wants is every design it may offer, with the columns a/// person filters and sorts on, small enough to fetch while somebody reads the/// page.////// So this is a *reduction* of the same read rather than a second extract with/// its own truth: the same library, the same producer of computed values, the/// same `mekset.txt` resolution `helm art` uses. A figure here and a figure in/// the database cannot disagree, because neither is computed twice.////// Every artefact is stamped with the MegaMek it came from and with what/// produced its battle values. That is not bookkeeping: the art is served from/// a bucket keyed by the same version, so an index that does not say which/// release it describes is one whose sprites may or may not be there./// The figures a screen filters on, and who produced them.////// The same choice `build` makes, and made in the same place: MegaMek's own/// figures when a bridge dump is at hand, ours when it is not. A screen/// showing a battle value the game will not agree with is worse than one/// showing none, so which producer answered is recorded rather than implied.struct Scored { by_name: std::collections::HashMap<String, ComputedStats>, catalogue: Option<Catalogue>, provenance: Provenance,}
fn scored( opts: &crate::options::Opts, library: &helm_unitfile::Library, version: &str,) -> Result<Scored, String> { let mut by_name: std::collections::HashMap<String, ComputedStats> = std::collections::HashMap::new(); let mut catalogue: Option<Catalogue> = None; let mut provenance = Provenance::none_yet(version);
if let Some(dir) = &opts.bridge_dir { let equipment = dir.join("equipment.jsonl"); if equipment.is_file() { catalogue = Some(helm_bridge::read_catalogue(&equipment).map_err(|e| e.to_string())?); } let units = dir.join("units.jsonl"); if units.is_file() { for stats in helm_bridge::read_units(&units).map_err(|e| e.to_string())? { by_name.insert(stats.name.clone(), stats); } provenance = Provenance::from_bridge(version); } }
let native = by_name.is_empty(); if native { let Some(cat) = &catalogue else { return Err( "battle value needs the equipment catalogue: pass --bridge-dir with \ equipment.jsonl in it" .to_string(), ); }; for unit in &library.units { if let Ok(bv) = helm_bv::battle_value(unit, cat) { let mut stats = ComputedStats::new(unit.name.clone()); stats.battle_value = Some(bv); by_name.insert(unit.name.clone(), stats); } } provenance = Provenance { megamek_version: version.to_string(), stats_producer: "helm-bv".to_string(), rules_version: Some(env!("CARGO_PKG_VERSION").to_string()), }; } eprintln!( "battle value: {} of {} designs, from {}", by_name.len(), library.units.len(), provenance.stats_producer ); if native { // Canon and validity are MegaMek's judgement and arrive with its own // figures. Without them nothing here can drop the 15 designs it would // otherwise have dropped, and a screen would offer a unit MegaMek // marks unplayable. eprintln!("no units.jsonl: canon and validity are unknown, so nothing is filtered on them"); }
Ok(Scored { by_name, catalogue, provenance, })}
/// The designs every published file is about, in the order every one of them/// writes.////// The index and the loadout beside it are joined by position, so "which/// designs, in what order" has to be one answer rather than two that happen/// to agree today. A design MegaMek marks non-canon or unplayable is dropped/// here, once.fn fieldable<'a>( library: &'a helm_unitfile::Library, by_name: &std::collections::HashMap<String, ComputedStats>, all_types: bool,) -> Vec<&'a helm_core::Unit> { // What a match can field and this repository can score: Meks, combat // vehicles, battle armour and conventional infantry - the four `helm-bv` // has a calculator for. // // `Unit`'s own predicates, not a second reading of `unit_type`: an `.mtf` // declares no type at all, so a filter that trusts the field drops every // Mek written in that format. let mut units: Vec<&helm_core::Unit> = library .units .iter() .filter(|u| { all_types || u.is_mek() || u.is_combat_vehicle() || u.is_battle_armor() || u.is_conventional_infantry() }) // Canon and validity are MegaMek's judgement and arrive with its own // figures. Without a producer nothing is dropped and a screen offers a // unit MegaMek marks unplayable, which `scored` says out loud. .filter(|u| { !by_name .get(&u.name) .is_some_and(|s| s.canon == Some(false) || s.invalid == Some(true)) }) .collect(); units.sort_by(|a, b| a.name.cmp(&b.name).then(a.path.cmp(&b.path))); units}
/// The identity a published file carries, from the designs in it.fn identity( provenance: &Provenance, helm_version: Option<&str>, units: &[&helm_core::Unit],) -> helm_query::Build { helm_query::Build::new( &provenance.megamek_version, helm_version, &provenance.stats_producer, provenance.rules_version.as_deref(), units.iter().map(|u| u.name.as_str()), )}
pub(crate) fn index(args: &[String]) -> Result<(), String> { let opts = parse_opts(args)?; let FromInstall { megamek, out, version, library, } = from_install(&opts)?; let set = helm_unitfile::read_mekset(&megamek.join("data/images/units")) .map_err(|e| e.to_string())?;
let Scored { by_name, catalogue, provenance, } = scored(&opts, &library, &version)?;
// Everything outside the four scoreable types is behind --all-types, which // is 10,896 designs and 4.8MB, gun emplacements, buildings and handheld // weapons among them, none of which a force offers. // // The list tracks what can be priced rather than what can be drawn, so a // design with no art still gets a row: art is a picture and battle value // is a fact a screen filters on. 93 platoons have no `mekset.txt` entry, // and MegaMek draws those with a default silhouette too. let units = fieldable(&library, &by_name, opts.all_types); let build = identity(&provenance, opts.helm_version.as_deref(), &units);
let mut counts = helm_query::Counts { units: units.len(), ..Default::default() }; let rows: Vec<(helm_facet::UnitFacets, helm_query::Extras)> = units .iter() .map(|unit| { let stats = by_name.get(&unit.name); let sprite = set.art_for(unit).map(|art| art.path.to_string()); if sprite.is_none() { counts.without_sprite += 1; } if stats.and_then(|s| s.battle_value).is_none() { counts.without_battle_value += 1; } ( helm_facet::UnitFacets::from_unit(unit, stats, catalogue.as_ref()), helm_query::Extras { sprite, total_armor: Some(unit.total_armor()), }, ) }) .collect();
let document = helm_query::Index { megamek: build.megamek.clone(), helm: build.helm.clone(), build: build.clone(), sprite_base: "data/images/units".to_string(), counts, chunks: helm_query::Chunk::ALL.iter().map(|c| c.file()).collect(), units: rows, };
let names = helm_query::Names::of( build.clone(), catalogue .as_ref() .unwrap_or(&helm_core::Catalogue::default()), );
// Answered against the whole thing, so what is recorded is what a page // gets once it has joined every chunk it needs. let answers = match &opts.filters { Some(corpus) => Some(answer(&document, &names, corpus)?), None => None, };
write_json(&out, &document, "units")?; println!(" build {}", build.id); if document.counts.without_battle_value > 0 || document.counts.without_sprite > 0 { println!( " {} without a battle value, {} without art", document.counts.without_battle_value, document.counts.without_sprite ); }
// Every chunk, every time. A prefix holding a spine and half its columns // is worse than one holding neither: a page would fetch what is there and // filter on a library that quietly answers a narrower question. for chunk in helm_query::Chunk::ALL { let beside = out.with_file_name(chunk.file()); match chunk { helm_query::Chunk::Art => { write_json(&beside, &document.part::<helm_query::Art>(), chunk.name()) } helm_query::Chunk::Figures => write_json( &beside, &document.part::<helm_query::Figures>(), chunk.name(), ), helm_query::Chunk::Paperwork => write_json( &beside, &document.part::<helm_query::Paperwork>(), chunk.name(), ), helm_query::Chunk::Combat => write_json( &beside, &document.part::<helm_query::Combat>(), chunk.name(), ), helm_query::Chunk::Curve => { write_json(&beside, &document.part::<helm_query::Curve>(), chunk.name()) } helm_query::Chunk::Quirks => write_json( &beside, &document.part::<helm_query::Quirks>(), chunk.name(), ), helm_query::Chunk::Loadout => write_json( &beside, &document.part::<helm_query::Loadout>(), chunk.name(), ), }?; }
// The names a search resolves through. Its own document rather than a // chunk: it is per item rather than per design, and a page wants it only // once somebody types a name. let beside = out.with_file_name("equipment.json"); write_json(&beside, &names, "equipment names")?;
if let Some(answers) = answers { let beside = out.with_file_name("answers.json"); std::fs::write(&beside, format!("{answers}\n")) .map_err(|e| format!("{}: {e}", beside.display()))?; println!("wrote {} - what this build answers", beside.display()); } Ok(())}
/// Write one document and say what it cost.fn write_json<T: serde::Serialize>(path: &Path, value: &T, what: &str) -> Result<(), String> { let text = serde_json::to_string(value).map_err(|e| e.to_string())?; std::fs::write(path, format!("{text}\n")).map_err(|e| format!("{}: {e}", path.display()))?; println!( "wrote {} - {what}, {:.0} KB", path.display(), text.len() as f64 / 1024.0 ); Ok(())}
/// What every filter in a corpus selects, as this build answers it.////// The other half of `crates/helm-wasm/facets.mjs`: the script runs the same/// corpus over the same documents through the wasm module, and a disagreement/// is the one failure that matters here - the predicate compiled twice and/// answering differently is exactly what running it in a browser is supposed/// to rule out.fn answer( document: &helm_query::Index, names: &helm_query::Names, corpus: &Path,) -> Result<String, String> { let text = std::fs::read_to_string(corpus).map_err(|e| format!("{}: {e}", corpus.display()))?; let filters: Vec<serde_json::Value> = serde_json::from_str(&text).map_err(|e| e.to_string())?; let units = document.facets(); let vocabulary = helm_query::Vocabulary::from_facets( &units.iter().map(|u| (*u).clone()).collect::<Vec<_>>(), ) .with_names(names.clone());
let mut out = Vec::with_capacity(filters.len()); for value in filters { let filter: helm_query::Filter = serde_json::from_value(value.clone()).map_err(|e| format!("{value}: {e}"))?; let query = filter .to_query(&vocabulary) .map_err(|e| format!("{value}: {e}"))?; let mut hits: Vec<usize> = (0..units.len()) .filter(|&i| query.matches(units[i])) .collect(); if let Some(band) = filter.band().map_err(|e| format!("{value}: {e}"))? { hits.sort_by(|a, b| { let (x, y) = ( helm_query::Filter::firepower_at(band, units[*a]), helm_query::Filter::firepower_at(band, units[*b]), ); y.total_cmp(&x).then(a.cmp(b)) }); } out.push(serde_json::json!({ "filter": value, "units": hits })); } serde_json::to_string(&out).map_err(|e| e.to_string())}
/// Write the equipment catalogue a browser needs, and nothing else.////// The bridge dumps every field MegaMek's `EquipmentType` carries, which is/// six megabytes for four thousand entries - a `namesVector` per row, a/// `techAdvancement` per row, and several static tables repeated on every one/// of them. The rules read about twenty of those fields.////// Same format, same reader, a third of the size: 1.7MB rather than 6.0MB, and/// 88ms rather than 216ms to parse in a browser. That is a page load, so it is/// worth the one command.pub(crate) fn catalogue(args: &[String]) -> Result<(), String> { let opts = parse_opts(args)?; let bridge = opts .bridge_dir .ok_or("--bridge-dir is required: the catalogue is trimmed from equipment.jsonl")?; let out = opts .out .ok_or("--out is required: nothing is written without somewhere to write it")?;
let text = std::fs::read_to_string(bridge.join("equipment.jsonl")) .map_err(|e| format!("equipment.jsonl: {e}"))?;
let mut kept = String::with_capacity(text.len() / 3); let mut entries = 0; for line in text.lines().filter(|l| !l.trim().is_empty()) { let value: serde_json::Value = serde_json::from_str(line).map_err(|e| format!("equipment.jsonl: {e}"))?; let Some(object) = value.as_object() else { continue; }; let slim: serde_json::Map<String, serde_json::Value> = object .iter() .filter(|(k, _)| helm_bridge::CATALOGUE_FIELDS.contains(&k.as_str())) .map(|(k, v)| (k.clone(), v.clone())) .collect(); kept.push_str(&serde_json::Value::Object(slim).to_string()); kept.push('\n'); entries += 1; } std::fs::write(&out, &kept).map_err(|e| format!("{}: {e}", out.display()))?; println!( "wrote {} ({entries} entries, {:.1} MB from {:.1} MB)", out.display(), kept.len() as f64 / 1_048_576.0, text.len() as f64 / 1_048_576.0 ); Ok(())}