diff --git a/crates/harness/src/bin/text_metrics.rs b/crates/harness/src/bin/text_metrics.rs new file mode 100644 index 00000000..fd809138 --- /dev/null +++ b/crates/harness/src/bin/text_metrics.rs @@ -0,0 +1,493 @@ +//! Measures whether Chromium, Firefox and WebKit agree on text geometry closely enough +//! to vote on (`specs/text-metrics-oracle.md`). Runs every case in +//! `harness::text_cases` through the oracle's text probe in each pinned font, votes +//! every measurement by the rule the geometry vote already uses (`Consensus` at +//! `dump::TOLERANCE_PX` = 1/60 px), and prints, per font and per category, the share of +//! measurements that are unanimous, 2-1 and split, plus the largest difference between +//! any two engines. +//! +//! It builds no layout code and scores nothing. A negative result -- the engines do not +//! agree closely enough -- is the answer, not a failure. The written result is +//! `docs/text-metrics.md`. +//! +//! Usage: +//! +//! ```text +//! cargo run --release -p harness --bin text_metrics +//! cargo run --release -p harness --bin text_metrics -- --random 60 --seed 0x7e47 +//! ``` +//! +//! Raw responses are written under `corpus/text-metrics//.json` (gitignored) +//! so a rerun on other engine versions can be diffed against this one. + +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Context, Result}; +use harness::compare::{self, Consensus}; +use harness::dump::{TextDump, TOLERANCE_PX}; +use harness::oracle::Oracle; +use harness::text_cases::{self, Case, TextCategory}; +use harness::trace::vote_text; + +/// The pinned fonts, in the order the report presents them: the one most likely to +/// agree first, then the one that says whether agreement survives real advances, +/// kerning and hinting. +const FONTS: &[&str] = &["ahem", "lato"]; + +/// What one font-and-category cell of the report holds. +#[derive(Debug, Default, Clone)] +struct Tally { + unanimous: usize, + majority: usize, + split: usize, + /// The largest difference between any two engines on a single measurement, in CSS + /// pixels, over every measurement in this cell that is measured in pixels. Fragment + /// counts are excluded: a line-count difference of one is not a distance, and mixing + /// it in would put a hard `1.000000` floor under every cell. + max_spread: f64, + /// Which measurement that largest difference came from, for naming it in the report. + max_spread_at: String, + /// Every pixel measurement's spread, kept so the report can say what tolerance the + /// engines would agree at, rather than only whether they agree at 1/60 px. A + /// measurement is unanimous at tolerance T exactly when its spread is at most T, so + /// this list answers the question for every T at once. + spreads: Vec, + /// Measurements of how many fragments a box or text node broke into that were not + /// unanimous -- a line-breaking difference, counted on its own because it is a + /// different kind of disagreement from a sub-pixel one and a worse one: it means the + /// engines did not lay out the same lines, so there is nothing to compare within + /// them. + fragment_count_disputes: usize, + /// How often each engine was the named outlier on a 2-1 measurement. + outliers: BTreeMap, + /// Cases whose probe failed outright in at least one engine. + failed_cases: usize, +} + +impl Tally { + fn total(&self) -> usize { + self.unanimous + self.majority + self.split + } + + /// `in_px` is false for a fragment count, which is a number of lines rather than a + /// distance and so does not belong in the largest-difference column. + fn record(&mut self, consensus: &Consensus, spread: f64, in_px: bool, at: &str) { + match consensus { + Consensus::Unanimous => self.unanimous += 1, + Consensus::Majority { outlier } => { + self.majority += 1; + *self.outliers.entry(outlier.clone()).or_default() += 1; + } + Consensus::Split => self.split += 1, + } + if !in_px { + if !matches!(consensus, Consensus::Unanimous) { + self.fragment_count_disputes += 1; + } + return; + } + self.spreads.push(spread); + if spread > self.max_spread { + self.max_spread = spread; + self.max_spread_at = at.to_string(); + } + } + + fn share(&self, n: usize) -> String { + if self.total() == 0 { + return "-".to_string(); + } + format!("{:.1}%", 100.0 * n as f64 / self.total() as f64) + } + + fn named_outliers(&self) -> String { + if self.outliers.is_empty() { + return "none".to_string(); + } + self.outliers + .iter() + .map(|(engine, count)| format!("{engine} x{count}")) + .collect::>() + .join(", ") + } +} + +struct Args { + random: usize, + seed: u64, +} + +fn parse_args() -> Result { + let mut args = Args { + random: 0, + seed: 0x7e47, + }; + let mut argv = std::env::args().skip(1); + while let Some(flag) = argv.next() { + match flag.as_str() { + "--random" => { + let value = argv.next().context("--random needs a count")?; + args.random = value.parse().context("--random takes a whole number")?; + } + "--seed" => { + let value = argv.next().context("--seed needs a value")?; + let parsed = value + .strip_prefix("0x") + .map(|hex| u64::from_str_radix(hex, 16)) + .unwrap_or_else(|| value.parse()); + args.seed = parsed.context("--seed takes a number, decimal or 0x-prefixed")?; + } + other => bail!("unknown flag: {other}"), + } + } + Ok(args) +} + +/// Stores one case's raw per-engine responses, so a rerun on other engine versions can +/// be compared against this one rather than re-argued. +fn store(out_dir: &Path, font: &str, case: &Case, results: &BTreeMap) -> Result<()> { + let dir = out_dir.join(font); + fs::create_dir_all(&dir)?; + let path = dir.join(format!("{}.json", case.id)); + let body = serde_json::json!({ + "case": case.id, + "font": font, + "categories": case.categories.iter().map(|c| c.name()).collect::>(), + "html": case.html, + "results": results, + }); + fs::write(&path, serde_json::to_string_pretty(&body)?) + .with_context(|| format!("writing {}", path.display()))?; + Ok(()) +} + +/// One voted measurement, flattened so the report can fold it into a per-category and a +/// per-quantity view without voting twice. +struct Observation { + /// Which quantity this is, as the report labels it. Element and text-fragment + /// quantities are kept apart: they answer different questions about the same case. + quantity: &'static str, + consensus: Consensus, + spread: f64, + /// False for a fragment count, which is a number of lines rather than a distance. + in_px: bool, + at: String, +} + +/// The order quantities appear in the per-quantity table: element boxes first, then the +/// text fragments themselves. +const QUANTITIES: &[&str] = &[ + "element x", + "element y", + "element width", + "element height", + "element fragment count", + "element fragment width", + "element fragment height", + "text fragment count", + "text fragment x", + "text fragment y", + "text fragment width", + "text fragment height", +]; + +fn element_quantity(metric: compare::Metric) -> &'static str { + match metric { + compare::Metric::OffsetX => "element x", + compare::Metric::OffsetY => "element y", + compare::Metric::Width => "element width", + compare::Metric::Height => "element height", + compare::Metric::FragmentCount => "element fragment count", + compare::Metric::FragmentWidth(_) => "element fragment width", + compare::Metric::FragmentHeight(_) => "element fragment height", + } +} + +fn text_quantity(metric: harness::trace::TextMetric) -> &'static str { + use harness::trace::TextMetric; + match metric { + TextMetric::FragmentCount => "text fragment count", + TextMetric::FragmentX(_) => "text fragment x", + TextMetric::FragmentY(_) => "text fragment y", + TextMetric::FragmentWidth(_) => "text fragment width", + TextMetric::FragmentHeight(_) => "text fragment height", + } +} + +/// Votes one case in one font, flattening every measurement into an [`Observation`] the +/// caller folds into both views of the report. +fn measure(case: &Case, results: &BTreeMap) -> Result> { + let mut observations = Vec::new(); + + // Element bounding boxes go through the geometry vote unchanged: the element half of + // a text-probe response is an ordinary dump, so there is no second implementation of + // the same rule for it. + let geometry: BTreeMap = results + .iter() + .map(|(name, dump)| (name.clone(), dump.geometry())) + .collect(); + match compare::vote(&geometry) { + Ok(report) => { + for m in &report.measurements { + let values: Vec = m.values.values().copied().collect(); + observations.push(Observation { + quantity: element_quantity(m.metric), + consensus: m.consensus.clone(), + spread: spread_of(&values), + in_px: !matches!(m.metric, compare::Metric::FragmentCount), + at: format!("{} element {} {:?}", case.id, m.node_index, m.metric), + }); + } + } + Err(e) => bail!("{}: element vote unavailable: {e:?}", case.id), + } + + // Text-node line fragments. A text node that produced no line box in any engine -- + // the contents of `\n\n\n{body}\n\n\n" + ) +} + +fn make_case(name: &str, seed: u64, categories: &[TextCategory], css: &str, body: &str) -> Case { + let html = document(css, body); + // Counted from the markup rather than carried by hand, so the metadata cannot drift + // from the document. ``, ``, ``, ``; + const head = /]*>/i.exec(html); + if (!head) return style + html; + const at = head.index + head[0].length; + return html.slice(0, at) + style + html.slice(at); +} + function parseEngineList(argv) { const flag = argv.find((a) => a.startsWith('--engines=')); if (!flag) return ALL_ENGINE_NAMES.slice(); @@ -378,6 +452,87 @@ function stackingDump({ points }) { }; } +// Runs in the page, for a text-probe request, before the geometry walk. Forces the probe +// face to load, waits for font loading to settle, and reports what actually happened, so +// the caller can refuse to measure a page that fell back to a system font. Kept +// self-contained for the same reason domDump is. +async function fontProbe({ family }) { + const spec = `16px "${family}"`; + try { + await document.fonts.load(spec); + } catch (err) { + // A load failure is reported through the face status below rather than thrown here, + // so the response says which family failed instead of only that something did. + } + await document.fonts.ready; + + const faces = []; + document.fonts.forEach((face) => { + // Engines differ on whether FontFace.family keeps the source's quotes. + const name = face.family.replace(/^["']|["']$/g, ''); + faces.push({ family: name, status: face.status }); + }); + const ours = faces.filter((f) => f.family === family); + + return { + font_family: family, + font_faces: faces, + // True only when a face for this family exists in the document and every such face + // finished loading. `document.fonts.check` alone cannot carry this: for a family with + // no matching @font-face it reports the system fallback as available, so it answers + // "true" exactly in the case that must fail. + font_loaded: ours.length > 0 && ours.every((f) => f.status === 'loaded'), + fonts_check: document.fonts.check(spec), + }; +} + +// Runs in the page, for a text-probe request, after domDump. Reports every text node in +// document order with its parent element's index and one rect per line fragment. +// +// The parent index is domDump's own node index. It is recomputed here rather than passed +// in because the two walks are separate page.evaluate calls with no shared scope; what is +// duplicated is only the ordering rule (pre-order from document.documentElement, children +// in order), not the dump itself. +// +// `text` is truncated: the only measured quantities are the rects, and a text probe's +// document carries the font's ~850 KB data: URL as a text node inside
AA
'; + const textRes = await oracle.send({ op: 'textProbe', html: textHtml, font: 'ahem', engines: active }); + check( + 'textProbe reports no per-engine error', + !textRes.errors || Object.keys(textRes.errors).length === 0, + JSON.stringify(textRes.errors), + ); + for (const engine of active) { + const r = textRes.results && textRes.results[engine]; + check(`${engine}: text probe answered`, !!r, JSON.stringify(textRes.errors)); + if (!r) continue; + + check( + `${engine}: text probe reports the font key and a sha256 of its bytes`, + r.font === 'ahem' && typeof r.font_sha256 === 'string' && /^[0-9a-f]{64}$/.test(r.font_sha256), + JSON.stringify({ font: r.font, font_sha256: r.font_sha256 }), + ); + check( + `${engine}: text probe carries the element dump alongside the text nodes`, + Array.isArray(r.nodes) && r.nodes.length >= 4 && Array.isArray(r.texts), + JSON.stringify({ nodes: r.nodes && r.nodes.length, texts: r.texts && r.texts.length }), + ); + + const div = findNode(r, (n) => n.tag === 'DIV'); + const aa = r.texts.find((t) => t.text === 'AA'); + check(`${engine}: the text node is reported`, !!aa, JSON.stringify(r.texts.map((t) => t.text))); + if (!aa || !div) continue; + + check( + `${engine}: the text node names its parent element's index`, + aa.parent === div.index, + JSON.stringify({ parent: aa.parent, div: div.index }), + ); + check( + `${engine}: one line fragment, so one rect`, + aa.rects.length === 1, + JSON.stringify(aa.rects), + ); + // The load gate is the oracle's own: a face that did not load fails the request, so + // reaching here at all means it loaded. This measures the loaded bytes to prove the + // gate is checking the right thing. + check( + `${engine}: the pinned Ahem loaded, not a fallback (2 glyphs at 100px = 200px)`, + Math.abs(aa.rects[0].w - 200) < 1 / 60, + JSON.stringify(aa.rects[0]), + ); + } + + // A font key with no file behind it is a request-level error naming the fetch command, + // not a silent measurement against something else. + const badFontRes = await oracle.send({ op: 'textProbe', html: textHtml, font: 'no-such-font', engines: active }); + check( + 'an unknown font key fails the request', + typeof badFontRes.error === 'string' && badFontRes.error.includes('no-such-font'), + JSON.stringify(badFontRes.error), + ); + await oracle.send({ op: 'shutdown' }); oracle.stop(); diff --git a/roadmap.md b/roadmap.md index 990e6a6a..916ee1b3 100644 --- a/roadmap.md +++ b/roadmap.md @@ -211,6 +211,14 @@ Oracles for what the geometry vote cannot see (`docs/oracle-coverage.md`): numeric execution (`docs/webgpu-cts.md`). - **Trace oracle** — script-observable behaviour voted across engines. Done for hand-written probes (`docs/dump-format.md` → "Trace protocol"). +- **Text metrics** — whether the three engines agree on text geometry closely enough for + the 1/60 px vote to run once glyphs are involved. Every other corpus here is text-free + by construction and the held-out WPT filter rejects any page with a text node, so this + had never been measured; until it was, no text-bearing generated function could be + scored. A text probe embeds a pinned font from the WPT checkout as a `data:` URL, + refuses to measure a page that fell back to a system font, and reports each text node's + line-fragment rects (`docs/dump-format.md` → "Text probe"). The result, per font and + per category, is `docs/text-metrics.md`. ### Phase 2 — Generation pipeline