From e94f04b8f2b02ff09e3acc13dc561a8914c7c781 Mon Sep 17 00:00:00 2001 From: Pierre Le Fevre Date: Fri, 22 May 2026 05:18:04 +0200 Subject: [PATCH] Add --real-web runner with summary report (isu issue 260) Implements the Phase 23 orchestrator that walks every `crates/e2e/scenarios/real-web/*.we` scenario, runs each through the standard scenario engine, captures pass/fail/panic/xfail status with per-scenario wall-clock time, peak RSS, and the maximum screenshot diff percentage observed during `assert_screenshot_matches`, and writes a deterministic `report.json` under `/real-web/`. Per-scenario artifacts (PNGs, DOM dumps, console captures, `perf.json`, `panics/L.txt`) land under `/real-web/runs//`. The runner survives per-scenario panics: each scenario already runs under `panic::catch`, so one buggy site cannot abort a batch. Exit code is non-zero on any `fail` or `panic`, `0` otherwise. `--real-web-online` re-fetches every `goto_as ` target through the live network before running each scenario, overwriting the on-disk snapshot for ad-hoc re-baselining. No commits are issued. Scenarios that need to be marked expected-to-fail can include a `# xfail` comment near the top of the `.we` file; xfailed scenarios are counted separately and do not fail the overall run. --- .isu/issues.json | 2 +- CLAUDE.md | 57 ++- crates/e2e/src/lib.rs | 1 + crates/e2e/src/main.rs | 78 +++- crates/e2e/src/real_web.rs | 649 ++++++++++++++++++++++++++++ crates/e2e/tests/real_web_runner.rs | 165 +++++++ 6 files changed, 935 insertions(+), 17 deletions(-) create mode 100644 crates/e2e/src/real_web.rs create mode 100644 crates/e2e/tests/real_web_runner.rs diff --git a/.isu/issues.json b/.isu/issues.json index fa4e260..b56dd7a 100644 --- a/.isu/issues.json +++ b/.isu/issues.json @@ -3141,7 +3141,7 @@ ], "assigned": [], "author": "piefev", - "state": "open", + "state": "closed", "created_at": "2026-05-22T02:23:27Z" }, { diff --git a/CLAUDE.md b/CLAUDE.md index 5e88439..0510d67 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -177,7 +177,7 @@ Use it after any change that could affect rendering, layout, parsing, scripting, or the UA stylesheet. The screenshot diff is the cheapest way to catch regressions you would otherwise miss until a user reports them. -### Two modes +### Three modes ```sh # Single-shot screenshot. @@ -186,6 +186,11 @@ cargo run -p we-e2e -- --url --out \ # Run a scenario (recommended). One scenario per logical test. cargo run -p we-e2e -- --scenario [--out-dir DIR] + +# Run the full real-web suite and write a summary report. +cargo run -p we-e2e -- --real-web [--out-dir DIR] \ + [--real-web-scenarios-dir DIR] \ + [--real-web-online] ``` ### Scenario DSL @@ -246,11 +251,51 @@ at `crates/e2e/real-web/snapshots//` and golden PNGs at `crates/e2e/scenarios/real-web/..expected.png`. Each scenario forces `network offline`, loads the snapshot via `goto_as`, and asserts a screenshot match against the committed golden so every run is -deterministic. Re-baselining against the live site is an ad-hoc step (curl -the page, replace the snapshot, re-capture the golden, commit) until the -Phase 23 `--real-web-online` runner lands. The legacy `realweb_*.we` -scenarios that hit the live network have been removed in favour of these -offline equivalents. +deterministic. The legacy `realweb_*.we` scenarios that hit the live network +have been removed in favour of these offline equivalents. + +The `--real-web` runner walks every `*.we` under +`crates/e2e/scenarios/real-web/` (alphabetical for deterministic order), +runs each through the standard scenario engine, and writes an aggregate +summary to `/real-web/report.json`. Per-scenario artifacts (PNGs, +DOM dumps, console captures, `perf.json`, `panics/L.txt`) land in +`/real-web/runs//`. The runner survives per-scenario +panics so one bad site does not abort a batch, and emits a one-line summary +to stdout (`real-web: 14/17 passed (2 failed, 1 panic) in 132.4s`). Exit +code is `0` only if every scenario is `pass` or `xfail`. + +```sh +# Default: run every committed offline scenario. +cargo run -p we-e2e -- --real-web --out-dir crates/e2e/artifacts + +# Restrict to a different scenarios directory (handy for tests). +cargo run -p we-e2e -- --real-web \ + --real-web-scenarios-dir path/to/scenarios \ + --out-dir crates/e2e/artifacts +``` + +Mark a scenario as expected-to-fail by adding a `# xfail` line (typically +with a tracking-issue pointer such as `# xfail: see isu issue 999`) as a +comment near the top of the `.we` file. Xfailed scenarios are counted in +the `xfail` total and do not fail the overall run. + +#### `--real-web-online` for re-baselining + +When the live web drifts away from a committed snapshot, pass +`--real-web-online` to re-fetch every `goto_as ` target through +the live network before running the scenario. The fetched body overwrites +the on-disk snapshot at `` (still relative to the scenario file's +directory). No commits are issued; the working tree is updated so a +maintainer can inspect the diff, regenerate the golden PNG, and decide +whether the change is a real regression or upstream drift. + +```sh +cargo run -p we-e2e -- --real-web --real-web-online \ + --out-dir crates/e2e/artifacts +``` + +Re-baselining is intentionally manual: review the snapshot diff, recapture +the golden screenshot, and commit page + scenario + golden in one revision. ### Adding a new scenario diff --git a/crates/e2e/src/lib.rs b/crates/e2e/src/lib.rs index a4fa0a1..c1fa67e 100644 --- a/crates/e2e/src/lib.rs +++ b/crates/e2e/src/lib.rs @@ -15,6 +15,7 @@ pub mod dom_dump; pub mod panic; pub mod perf; pub mod png_out; +pub mod real_web; pub mod render; pub mod scenario; pub mod screenshot_diff; diff --git a/crates/e2e/src/main.rs b/crates/e2e/src/main.rs index bead193..2f104bd 100644 --- a/crates/e2e/src/main.rs +++ b/crates/e2e/src/main.rs @@ -1,15 +1,20 @@ //! `we-e2e` — headless harness CLI. //! -//! Two modes: +//! Three modes: //! 1. Single-shot: render a URL and write a screenshot. //! `we-e2e --url URL --out path.png [--width W --height H]` //! 2. Scenario file: run a `.we` scenario. //! `we-e2e --scenario path.we [--out-dir DIR]` +//! 3. Real-web suite: walk every scenario under +//! `crates/e2e/scenarios/real-web/` and write a summary report. +//! `we-e2e --real-web [--out-dir DIR] [--real-web-scenarios-dir DIR] +//! [--real-web-online]` use std::path::{Path, PathBuf}; use std::process::ExitCode; use we_e2e::perf::PerfBudget; +use we_e2e::real_web::{self, RealWebOptions}; fn main() -> ExitCode { let args: Vec = std::env::args().skip(1).collect(); @@ -25,6 +30,17 @@ fn main() -> ExitCode { match opts.mode { Mode::SingleShot { url, out } => single_shot(&url, opts.width, opts.height, &out), Mode::Scenario { path } => run_scenario(&path, &opts.out_dir, opts.perf_budget), + Mode::RealWeb { + scenarios_dir, + online_resnapshot, + } => { + let rw_opts = RealWebOptions { + scenarios_dir, + budget: opts.perf_budget, + online_resnapshot, + }; + real_web::run_cli(&rw_opts, &opts.out_dir) + } } } @@ -37,8 +53,17 @@ struct Opts { } enum Mode { - SingleShot { url: String, out: PathBuf }, - Scenario { path: PathBuf }, + SingleShot { + url: String, + out: PathBuf, + }, + Scenario { + path: PathBuf, + }, + RealWeb { + scenarios_dir: PathBuf, + online_resnapshot: bool, + }, } fn parse_args(args: &[String]) -> Result { @@ -50,6 +75,9 @@ fn parse_args(args: &[String]) -> Result { let mut out_dir: Option = None; let mut perf_wall_ms: Option = None; let mut perf_rss_mb: Option = None; + let mut real_web_flag = false; + let mut real_web_scenarios_dir: Option = None; + let mut real_web_online = false; let mut i = 0; while i < args.len() { @@ -109,6 +137,19 @@ fn parse_args(args: &[String]) -> Result { .map_err(|_| "--perf-budget-rss-mb: not an integer")?, ); } + "--real-web" => { + real_web_flag = true; + } + "--real-web-scenarios-dir" => { + i += 1; + real_web_scenarios_dir = Some(PathBuf::from( + args.get(i) + .ok_or("--real-web-scenarios-dir: missing value")?, + )); + } + "--real-web-online" => { + real_web_online = true; + } "-h" | "--help" => { print_usage(); std::process::exit(0); @@ -118,13 +159,27 @@ fn parse_args(args: &[String]) -> Result { i += 1; } - let mode = match (url, out, scenario) { - (Some(u), Some(o), None) => Mode::SingleShot { url: u, out: o }, - (None, None, Some(p)) => Mode::Scenario { path: p }, - (Some(_), None, _) => return Err("--url requires --out".into()), - (None, Some(_), _) => return Err("--out requires --url".into()), - (Some(_), _, Some(_)) => return Err("--url and --scenario are mutually exclusive".into()), - (None, None, None) => return Err("specify --url/--out or --scenario".into()), + if !real_web_flag && (real_web_scenarios_dir.is_some() || real_web_online) { + return Err("--real-web-scenarios-dir / --real-web-online require --real-web".into()); + } + + let mode = match (url, out, scenario, real_web_flag) { + (None, None, None, true) => Mode::RealWeb { + scenarios_dir: real_web_scenarios_dir + .unwrap_or_else(|| PathBuf::from(real_web::DEFAULT_SCENARIOS_DIR)), + online_resnapshot: real_web_online, + }, + (Some(u), Some(o), None, false) => Mode::SingleShot { url: u, out: o }, + (None, None, Some(p), false) => Mode::Scenario { path: p }, + (Some(_), None, _, _) => return Err("--url requires --out".into()), + (None, Some(_), _, _) => return Err("--out requires --url".into()), + (Some(_), _, Some(_), _) => { + return Err("--url and --scenario are mutually exclusive".into()) + } + (_, _, _, true) => return Err("--real-web cannot be combined with --url/--scenario".into()), + (None, None, None, false) => { + return Err("specify --url/--out, --scenario, or --real-web".into()) + } }; Ok(Opts { @@ -144,6 +199,9 @@ fn print_usage() { "usage:\n \ we-e2e --url --out [--width W --height H]\n \ we-e2e --scenario [--out-dir DIR] \ + [--perf-budget-wall-ms N] [--perf-budget-rss-mb N]\n \ + we-e2e --real-web [--out-dir DIR] \ + [--real-web-scenarios-dir DIR] [--real-web-online] \ [--perf-budget-wall-ms N] [--perf-budget-rss-mb N]" ); } diff --git a/crates/e2e/src/real_web.rs b/crates/e2e/src/real_web.rs new file mode 100644 index 0000000..1b4f1e3 --- /dev/null +++ b/crates/e2e/src/real_web.rs @@ -0,0 +1,649 @@ +//! `--real-web` runner: walk every scenario under +//! `crates/e2e/scenarios/real-web/`, run each offline through the standard +//! scenario engine, and write an aggregate summary to +//! `/real-web/report.json`. +//! +//! The runner is the orchestrator that Phase 23 exit-criteria are checked +//! against: it collates per-scenario pass/fail/panic/xfail status, captures +//! peak RSS, wall-clock time, and the maximum screenshot diff percentage +//! observed during `assert_screenshot_matches`, and emits a deterministic +//! report that CI can diff against. +//! +//! The runner survives per-scenario panics: each scenario already runs under +//! [`crate::panic::catch`], so one buggy site cannot abort a batch. The +//! optional `--real-web-online` mode re-snapshots each domain's `goto_as` +//! targets against the live network before running, for ad-hoc rebaselining +//! against the live web. No commits are issued; snapshots are only rewritten +//! on the working tree. + +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use crate::perf::PerfBudget; +use crate::scenario::{self, Cmd, Scenario}; + +/// Default location of real-web scenarios within the workspace. +pub const DEFAULT_SCENARIOS_DIR: &str = "crates/e2e/scenarios/real-web"; + +/// Per-scenario outcome captured by the runner. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ScenarioStatus { + /// Scenario ran cleanly and every assertion passed. + Pass, + /// At least one assertion failed (no panic). + Fail, + /// Scenario or one of its commands panicked. + Panic, + /// Scenario is marked `# xfail` and (as expected) did not pass. + Xfail, +} + +impl ScenarioStatus { + pub fn as_str(&self) -> &'static str { + match self { + Self::Pass => "pass", + Self::Fail => "fail", + Self::Panic => "panic", + Self::Xfail => "xfail", + } + } +} + +/// Per-scenario record written into the summary report. +#[derive(Debug, Clone)] +pub struct ScenarioRecord { + /// Domain-style scenario name, e.g. `bing.com`. + pub name: String, + /// Status bucket for the totals. + pub status: ScenarioStatus, + /// Total scenario wall-clock time (ms). + pub wall_ms: u64, + /// Peak resident-set size observed during the run (bytes). + pub peak_rss_bytes: u64, + /// Maximum `assert_screenshot_matches` diff percentage observed during + /// the run, or `None` if the scenario did not call that assertion. + pub screenshot_diff_pct: Option, + /// Human-readable failure messages from failing scenario steps. + pub failures: Vec, + /// One-line panic summary if the scenario panicked, else `None`. + pub panic: Option, +} + +/// Aggregate totals across all scenarios. +#[derive(Debug, Clone, Default)] +pub struct Totals { + pub pass: usize, + pub fail: usize, + pub panic: usize, + pub xfail: usize, +} + +/// Final summary written to `report.json`. +#[derive(Debug, Clone)] +pub struct Report { + pub started_at: String, + pub duration_ms: u64, + pub scenarios: Vec, + pub totals: Totals, +} + +impl Report { + pub fn to_json(&self) -> String { + let mut s = String::with_capacity(256); + s.push('{'); + s.push_str("\"started_at\":"); + push_json_string(&mut s, &self.started_at); + s.push(','); + push_kv_u64(&mut s, "duration_ms", self.duration_ms); + s.push(','); + s.push_str("\"scenarios\":["); + for (i, sc) in self.scenarios.iter().enumerate() { + if i > 0 { + s.push(','); + } + s.push('{'); + s.push_str("\"name\":"); + push_json_string(&mut s, &sc.name); + s.push(','); + s.push_str("\"status\":"); + push_json_string(&mut s, sc.status.as_str()); + s.push(','); + push_kv_u64(&mut s, "wall_ms", sc.wall_ms); + s.push(','); + push_kv_u64(&mut s, "peak_rss_bytes", sc.peak_rss_bytes); + s.push(','); + s.push_str("\"screenshot_diff_pct\":"); + match sc.screenshot_diff_pct { + Some(v) => push_json_f64(&mut s, v), + None => s.push_str("null"), + } + s.push(','); + s.push_str("\"failures\":["); + for (j, f) in sc.failures.iter().enumerate() { + if j > 0 { + s.push(','); + } + push_json_string(&mut s, f); + } + s.push(']'); + s.push(','); + s.push_str("\"panic\":"); + match &sc.panic { + Some(msg) => push_json_string(&mut s, msg), + None => s.push_str("null"), + } + s.push('}'); + } + s.push(']'); + s.push(','); + s.push_str("\"totals\":{"); + push_kv_u64(&mut s, "pass", self.totals.pass as u64); + s.push(','); + push_kv_u64(&mut s, "fail", self.totals.fail as u64); + s.push(','); + push_kv_u64(&mut s, "panic", self.totals.panic as u64); + s.push(','); + push_kv_u64(&mut s, "xfail", self.totals.xfail as u64); + s.push('}'); + s.push('}'); + s + } +} + +/// Runner configuration. +#[derive(Debug, Clone)] +pub struct RealWebOptions { + /// Directory containing `*.we` scenarios. Defaults to + /// [`DEFAULT_SCENARIOS_DIR`]. + pub scenarios_dir: PathBuf, + /// Default perf budget passed to each scenario; scenarios may override it + /// via `perf_budget`. + pub budget: PerfBudget, + /// When set, re-fetch every `goto_as` URL/path pair through the live + /// network before running the scenario, overwriting the on-disk snapshot. + pub online_resnapshot: bool, +} + +impl Default for RealWebOptions { + fn default() -> Self { + Self { + scenarios_dir: PathBuf::from(DEFAULT_SCENARIOS_DIR), + budget: PerfBudget::default(), + online_resnapshot: false, + } + } +} + +/// CLI entry point: run the real-web suite and return a process exit code. +/// +/// Exits with `0` when every scenario is `pass` or `xfail`. Any `fail` or +/// `panic` produces a non-zero exit code so CI fails closed. +pub fn run_cli(opts: &RealWebOptions, out_dir: &Path) -> ExitCode { + match run(opts, out_dir) { + Ok(report) => { + print_summary(&report); + let any_bad = report.totals.fail > 0 || report.totals.panic > 0; + if any_bad { + ExitCode::from(1) + } else { + ExitCode::SUCCESS + } + } + Err(e) => { + eprintln!("real-web: {e}"); + ExitCode::from(2) + } + } +} + +/// Run the real-web suite, writing `report.json` under `/real-web/`. +pub fn run(opts: &RealWebOptions, out_dir: &Path) -> Result { + let scenarios = discover_scenarios(&opts.scenarios_dir)?; + let started_at_iso = now_iso8601_utc(); + let started = Instant::now(); + let mut records = Vec::with_capacity(scenarios.len()); + let mut totals = Totals::default(); + + let real_web_out_dir = out_dir.join("real-web"); + let _ = std::fs::create_dir_all(&real_web_out_dir); + + for path in &scenarios { + let record = run_one(path, &real_web_out_dir, opts); + match record.status { + ScenarioStatus::Pass => totals.pass += 1, + ScenarioStatus::Fail => totals.fail += 1, + ScenarioStatus::Panic => totals.panic += 1, + ScenarioStatus::Xfail => totals.xfail += 1, + } + records.push(record); + } + + let duration_ms = started.elapsed().as_millis().min(u64::MAX as u128) as u64; + let report = Report { + started_at: started_at_iso, + duration_ms, + scenarios: records, + totals, + }; + + let report_path = real_web_out_dir.join("report.json"); + std::fs::write(&report_path, report.to_json()) + .map_err(|e| format!("real-web: write report {}: {e}", report_path.display()))?; + + Ok(report) +} + +/// Discover `*.we` files under `dir`, sorted alphabetically. Files are +/// considered case-sensitively so the order is deterministic across platforms. +pub fn discover_scenarios(dir: &Path) -> Result, String> { + let entries = + std::fs::read_dir(dir).map_err(|e| format!("read scenarios dir {}: {e}", dir.display()))?; + let mut out = Vec::new(); + for entry in entries { + let entry = entry.map_err(|e| format!("scenarios dir entry: {e}"))?; + let path = entry.path(); + if path.extension().and_then(|s| s.to_str()) == Some("we") { + out.push(path); + } + } + out.sort(); + Ok(out) +} + +/// Run a single scenario file and produce its [`ScenarioRecord`]. +fn run_one(path: &Path, real_web_out_dir: &Path, opts: &RealWebOptions) -> ScenarioRecord { + let name = scenario_name(path); + let parsed = match scenario::parse(path) { + Ok(s) => s, + Err(e) => { + return ScenarioRecord { + name, + status: ScenarioStatus::Fail, + wall_ms: 0, + peak_rss_bytes: 0, + screenshot_diff_pct: None, + failures: vec![format!("scenario parse: {e}")], + panic: None, + }; + } + }; + + let xfail = scenario_is_xfail(path); + + if opts.online_resnapshot { + if let Err(e) = resnapshot_scenario(&parsed) { + // Resnapshot failure should not silently mask the run; record it + // and continue with the on-disk snapshot. + eprintln!("real-web: resnapshot {name}: {e}"); + } + } + + let scenario_out_dir = real_web_out_dir.join("runs").join(&name); + let _ = std::fs::create_dir_all(&scenario_out_dir); + + let started = Instant::now(); + let outcome = + crate::panic::catch(|| scenario::run_with_budget(&parsed, &scenario_out_dir, opts.budget)); + let wall_ms = started.elapsed().as_millis().min(u64::MAX as u128) as u64; + + match outcome { + Err(report) => ScenarioRecord { + name, + status: ScenarioStatus::Panic, + wall_ms, + peak_rss_bytes: 0, + screenshot_diff_pct: None, + failures: Vec::new(), + panic: Some(report.summary()), + }, + Ok(run_report) => { + let mut failures: Vec = Vec::new(); + let mut panic_summary: Option = None; + for (lineno, ok, msg) in &run_report.steps { + if *ok { + continue; + } + if msg.starts_with("sev-crash") { + panic_summary.get_or_insert_with(|| msg.clone()); + } + failures.push(format!("L{lineno}: {msg}")); + } + let diff_pct = max_screenshot_diff_pct(&run_report.steps); + + let status = if panic_summary.is_some() { + if xfail { + ScenarioStatus::Xfail + } else { + ScenarioStatus::Panic + } + } else if run_report.passed { + ScenarioStatus::Pass + } else if xfail { + ScenarioStatus::Xfail + } else { + ScenarioStatus::Fail + }; + + ScenarioRecord { + name, + status, + wall_ms: run_report.wall_ms.max(wall_ms), + peak_rss_bytes: run_report.peak_rss_bytes, + screenshot_diff_pct: diff_pct, + failures, + panic: panic_summary, + } + } + } +} + +/// Derive a short scenario name from its file path. `foo/bing.com.we` -> +/// `bing.com`. +fn scenario_name(path: &Path) -> String { + path.file_stem() + .and_then(|s| s.to_str()) + .map(|s| s.to_string()) + .unwrap_or_else(|| path.display().to_string()) +} + +/// Inspect the scenario file for an `# xfail` directive. Considered present +/// when any line (trimmed) starts with `# xfail` — typically used in the +/// header comments next to a tracking issue. +fn scenario_is_xfail(path: &Path) -> bool { + let Ok(text) = std::fs::read_to_string(path) else { + return false; + }; + text.lines().any(|line| { + let trimmed = line.trim_start(); + if !trimmed.starts_with('#') { + return false; + } + let after_hash = trimmed.trim_start_matches('#').trim_start(); + after_hash.starts_with("xfail") + }) +} + +/// Find the maximum `assert_screenshot_matches` diff percentage in `steps`. +/// Diff percentages are parsed from step messages of the form +/// `screenshot_matches: 99.98% match (...)`. +fn max_screenshot_diff_pct(steps: &[(usize, bool, String)]) -> Option { + let mut max: Option = None; + for (_, _, msg) in steps { + if let Some(pct) = parse_match_pct(msg) { + let diff = (100.0 - pct).max(0.0); + max = Some(match max { + Some(prev) => prev.max(diff), + None => diff, + }); + } + } + max +} + +/// Parse the match-percentage number out of a `screenshot_matches:` message. +fn parse_match_pct(msg: &str) -> Option { + let rest = msg.strip_prefix("screenshot_matches:")?.trim_start(); + let end = rest.find('%')?; + rest[..end].trim().parse::().ok() +} + +/// Print the one-line summary `cargo run` prints to stdout. +pub fn print_summary(report: &Report) { + let total = report.scenarios.len(); + let passed = report.totals.pass + report.totals.xfail; + let duration_s = (report.duration_ms as f64) / 1000.0; + println!( + "real-web: {passed}/{total} passed ({} failed, {} panic) in {duration_s:.1}s", + report.totals.fail, report.totals.panic + ); +} + +/// Re-snapshot a scenario's `goto_as` targets against the live network. +/// Walks the scenario commands, fetches each `goto_as ` over HTTP, +/// and overwrites the on-disk snapshot file with the live response body. +/// +/// Paths are resolved relative to the scenario's containing directory so they +/// match the offline lookup convention. Errors are reported but do not abort +/// the rest of the scenario list. +pub fn resnapshot_scenario(scenario: &Scenario) -> Result<(), String> { + let scenario_dir = scenario + .source + .parent() + .map(|p| p.to_path_buf()) + .unwrap_or_else(|| PathBuf::from(".")); + let mut client = we_net::client::HttpClient::new(); + let mut any_err: Option = None; + + for (_, cmd) in &scenario.commands { + let Cmd::GotoAs { url, path } = cmd else { + continue; + }; + let url = match we_url::Url::parse(url) { + Ok(u) => u, + Err(e) => { + any_err.get_or_insert_with(|| format!("invalid URL {url}: {e}")); + continue; + } + }; + let target = if path.is_absolute() { + path.clone() + } else { + scenario_dir.join(path) + }; + match client.get(&url) { + Ok(resp) => { + if let Some(parent) = target.parent() { + let _ = std::fs::create_dir_all(parent); + } + if let Err(e) = std::fs::write(&target, &resp.body) { + any_err.get_or_insert_with(|| format!("write {}: {e}", target.display())); + } + } + Err(e) => { + any_err.get_or_insert_with(|| format!("fetch {}: {e}", url.serialize())); + } + } + } + + match any_err { + Some(e) => Err(e), + None => Ok(()), + } +} + +/// Format a [`SystemTime::now`] timestamp as ISO 8601 UTC (no fractional +/// seconds), e.g. `2026-05-22T15:30:45Z`. +fn now_iso8601_utc() -> String { + let secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + iso8601_utc_from_unix(secs) +} + +/// Convert a UNIX timestamp in seconds to an ISO 8601 UTC string. +/// Algorithm: Howard Hinnant's days_from_civil / civil_from_days +/// (https://howardhinnant.github.io/date_algorithms.html). +fn iso8601_utc_from_unix(secs: u64) -> String { + let days = (secs / 86_400) as i64; + let time_of_day = secs % 86_400; + let hour = (time_of_day / 3600) as u32; + let minute = ((time_of_day / 60) % 60) as u32; + let second = (time_of_day % 60) as u32; + + // Days since 1970-01-01 -> y/m/d. + let z = days + 719_468; // shift epoch to 0000-03-01 + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097) as u64; // [0, 146096] + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; // [0, 399] + let mut y = (yoe as i64) + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365] + let mp = (5 * doy + 2) / 153; // [0, 11] + let day = (doy - (153 * mp + 2) / 5 + 1) as u32; // [1, 31] + let month = if mp < 10 { + (mp + 3) as u32 + } else { + (mp - 9) as u32 + }; + if month <= 2 { + y += 1; + } + format!("{y:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}Z") +} + +fn push_kv_u64(s: &mut String, key: &str, value: u64) { + s.push('"'); + s.push_str(key); + s.push_str("\":"); + s.push_str(&value.to_string()); +} + +fn push_json_f64(s: &mut String, value: f64) { + if !value.is_finite() { + s.push_str("null"); + return; + } + // Stable formatting: at most 6 fractional digits, no trailing zeros. + let formatted = format!("{value:.6}"); + let trimmed = formatted.trim_end_matches('0').trim_end_matches('.'); + if trimmed.is_empty() || trimmed == "-" { + s.push('0'); + } else { + s.push_str(trimmed); + } +} + +fn push_json_string(s: &mut String, value: &str) { + s.push('"'); + for c in value.chars() { + match c { + '"' => s.push_str("\\\""), + '\\' => s.push_str("\\\\"), + '\n' => s.push_str("\\n"), + '\r' => s.push_str("\\r"), + '\t' => s.push_str("\\t"), + c if (c as u32) < 0x20 => { + use std::fmt::Write; + let _ = write!(s, "\\u{:04x}", c as u32); + } + c => s.push(c), + } + } + s.push('"'); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn iso8601_epoch_is_1970_01_01() { + assert_eq!(iso8601_utc_from_unix(0), "1970-01-01T00:00:00Z"); + } + + #[test] + fn iso8601_known_dates() { + assert_eq!(iso8601_utc_from_unix(1_705_276_800), "2024-01-15T00:00:00Z"); + assert_eq!(iso8601_utc_from_unix(1_577_836_800), "2020-01-01T00:00:00Z"); + assert_eq!(iso8601_utc_from_unix(946_684_799), "1999-12-31T23:59:59Z"); + } + + #[test] + fn iso8601_handles_leap_year_feb_29() { + // 2024-02-29T00:00:00Z = 1709164800 + assert_eq!(iso8601_utc_from_unix(1_709_164_800), "2024-02-29T00:00:00Z"); + } + + #[test] + fn parse_match_pct_extracts_percentage() { + let msg = "screenshot_matches: 99.98% match (3/16 px differ, tol=4, max_diff=0.1000%)"; + let pct = parse_match_pct(msg).expect("pct"); + assert!((pct - 99.98).abs() < 1e-6); + } + + #[test] + fn parse_match_pct_returns_none_for_other_messages() { + assert!(parse_match_pct("goto about:blank ok").is_none()); + assert!(parse_match_pct("screenshot_matches: no number here%").is_none()); + } + + #[test] + fn max_screenshot_diff_pct_picks_largest() { + let steps = vec![ + (1usize, true, "goto ok".to_string()), + ( + 2, + true, + "screenshot_matches: 99.98% match (3/16 px differ, tol=4, max_diff=0.1000%)" + .to_string(), + ), + ( + 3, + false, + "screenshot_matches: 80.00% match (4/16 px differ, tol=4, max_diff=0.1000%)" + .to_string(), + ), + ]; + let pct = max_screenshot_diff_pct(&steps).expect("diff pct"); + // 100 - 80 == 20 should be picked over 100 - 99.98 == 0.02 + assert!((pct - 20.0).abs() < 1e-6); + } + + #[test] + fn max_screenshot_diff_pct_returns_none_without_assert() { + let steps = vec![(1usize, true, "goto ok".to_string())]; + assert!(max_screenshot_diff_pct(&steps).is_none()); + } + + #[test] + fn report_to_json_round_trips_minimal_shape() { + let r = Report { + started_at: "2026-05-22T00:00:00Z".to_string(), + duration_ms: 1234, + scenarios: vec![ScenarioRecord { + name: "foo.com".to_string(), + status: ScenarioStatus::Pass, + wall_ms: 42, + peak_rss_bytes: 5_000_000, + screenshot_diff_pct: Some(0.5), + failures: Vec::new(), + panic: None, + }], + totals: Totals { + pass: 1, + fail: 0, + panic: 0, + xfail: 0, + }, + }; + let json = r.to_json(); + assert!(json.contains("\"started_at\":\"2026-05-22T00:00:00Z\"")); + assert!(json.contains("\"duration_ms\":1234")); + assert!(json.contains("\"name\":\"foo.com\"")); + assert!(json.contains("\"status\":\"pass\"")); + assert!(json.contains("\"screenshot_diff_pct\":0.5")); + assert!(json.contains("\"panic\":null")); + assert!(json.contains("\"totals\":{\"pass\":1,\"fail\":0,\"panic\":0,\"xfail\":0}")); + } + + #[test] + fn scenario_is_xfail_detects_top_comment() { + let dir = std::env::temp_dir().join(format!( + "we-e2e-xfail-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + let _ = std::fs::create_dir_all(&dir); + let path = dir.join("xfail.we"); + std::fs::write(&path, "# xfail: see isu 999\ngoto about:blank\n").unwrap(); + assert!(scenario_is_xfail(&path)); + + let path2 = dir.join("normal.we"); + std::fs::write(&path2, "# regular comment\ngoto about:blank\n").unwrap(); + assert!(!scenario_is_xfail(&path2)); + } +} diff --git a/crates/e2e/tests/real_web_runner.rs b/crates/e2e/tests/real_web_runner.rs new file mode 100644 index 0000000..cc091af --- /dev/null +++ b/crates/e2e/tests/real_web_runner.rs @@ -0,0 +1,165 @@ +//! Integration tests for the `--real-web` runner. +//! +//! These exercise the runner against a synthetic real-web scenarios directory +//! so the test does not depend on the live network or on the committed +//! popular-sites snapshots. The runner is invoked through the library API +//! so the test does not need to spawn a CLI process — it asserts the +//! resulting `report.json` directly. + +use std::fs; +use std::path::PathBuf; + +use we_e2e::perf::PerfBudget; +use we_e2e::real_web::{self, RealWebOptions, Report, ScenarioRecord, ScenarioStatus}; + +fn tmp_dir(name: &str) -> PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!( + "we-e2e-real-web-{name}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + let _ = fs::remove_dir_all(&p); + fs::create_dir_all(&p).expect("create tmp dir"); + p +} + +fn write_scenario(dir: &std::path::Path, name: &str, contents: &str) { + fs::write(dir.join(name), contents).expect("write scenario"); +} + +fn record<'a>(report: &'a Report, name: &str) -> &'a ScenarioRecord { + report + .scenarios + .iter() + .find(|r| r.name == name) + .unwrap_or_else(|| panic!("no scenario {name:?} in report")) +} + +#[test] +fn run_aggregates_synthetic_scenarios_into_report() { + let root = tmp_dir("aggregate"); + let scenarios_dir = root.join("scenarios"); + fs::create_dir_all(&scenarios_dir).unwrap(); + let out_dir = root.join("artifacts"); + fs::create_dir_all(&out_dir).unwrap(); + + write_scenario( + &scenarios_dir, + "alpha.com.we", + "goto about:blank\nassert_dom_contains \"\"\n", + ); + write_scenario( + &scenarios_dir, + "broken.com.we", + "goto about:blank\nassert_dom_contains \"this-string-will-never-be-in-about-blank-dom\"\n", + ); + write_scenario( + &scenarios_dir, + "panic.com.we", + "goto about:blank\npanic_now \"deliberate crash for runner test\"\n", + ); + write_scenario( + &scenarios_dir, + "xfail.com.we", + "# xfail: see isu issue 99999\ngoto about:blank\nassert_dom_contains \"definitely-not-present\"\n", + ); + + let opts = RealWebOptions { + scenarios_dir: scenarios_dir.clone(), + budget: PerfBudget::default(), + online_resnapshot: false, + }; + let report = real_web::run(&opts, &out_dir).expect("run real-web suite"); + + // Scenarios sorted alphabetically by file name. + let names: Vec<&str> = report.scenarios.iter().map(|r| r.name.as_str()).collect(); + assert_eq!( + names, + vec!["alpha.com", "broken.com", "panic.com", "xfail.com"] + ); + + let alpha = record(&report, "alpha.com"); + assert_eq!(alpha.status, ScenarioStatus::Pass, "alpha was {alpha:?}"); + assert!(alpha.panic.is_none()); + + let broken = record(&report, "broken.com"); + assert_eq!(broken.status, ScenarioStatus::Fail); + assert!(broken + .failures + .iter() + .any(|s| s.contains("dom does NOT contain"))); + + let panicked = record(&report, "panic.com"); + assert_eq!(panicked.status, ScenarioStatus::Panic); + assert!(panicked + .panic + .as_ref() + .is_some_and(|s| s.contains("deliberate crash for runner test"))); + + let xfailed = record(&report, "xfail.com"); + assert_eq!( + xfailed.status, + ScenarioStatus::Xfail, + "expected xfail for marked scenario" + ); + + assert_eq!(report.totals.pass, 1); + assert_eq!(report.totals.fail, 1); + assert_eq!(report.totals.panic, 1); + assert_eq!(report.totals.xfail, 1); + + // report.json was written under /real-web/. + let report_path = out_dir.join("real-web").join("report.json"); + assert!( + report_path.exists(), + "report.json missing at {}", + report_path.display() + ); + let body = fs::read_to_string(&report_path).unwrap(); + assert!(body.contains("\"name\":\"alpha.com\"")); + assert!(body.contains("\"status\":\"pass\"")); + assert!(body.contains("\"status\":\"fail\"")); + assert!(body.contains("\"status\":\"panic\"")); + assert!(body.contains("\"status\":\"xfail\"")); + assert!(body.contains("\"totals\":{\"pass\":1,\"fail\":1,\"panic\":1,\"xfail\":1}")); + assert!(body.starts_with("{\"started_at\":\"")); +} + +#[test] +fn run_writes_report_even_for_empty_directory() { + let root = tmp_dir("empty"); + let scenarios_dir = root.join("scenarios"); + fs::create_dir_all(&scenarios_dir).unwrap(); + let out_dir = root.join("artifacts"); + + let opts = RealWebOptions { + scenarios_dir: scenarios_dir.clone(), + budget: PerfBudget::default(), + online_resnapshot: false, + }; + let report = real_web::run(&opts, &out_dir).expect("empty run"); + assert!(report.scenarios.is_empty()); + assert_eq!(report.totals.pass, 0); + assert_eq!(report.totals.fail, 0); + + let report_path = out_dir.join("real-web").join("report.json"); + assert!(report_path.exists()); + let body = fs::read_to_string(&report_path).unwrap(); + assert!(body.contains("\"scenarios\":[]")); +} + +#[test] +fn missing_scenarios_directory_is_an_error() { + let root = tmp_dir("missing"); + let opts = RealWebOptions { + scenarios_dir: root.join("does-not-exist"), + budget: PerfBudget::default(), + online_resnapshot: false, + }; + let err = real_web::run(&opts, &root.join("artifacts")).expect_err("missing dir"); + assert!(err.contains("read scenarios dir"), "err was {err:?}"); +} -- 2.51.2