//! `--real-web` runner: walk every scenario under //! `crates/e2e/scenarios/real-web/`, run each offline through the standard //! scenario engine, and write an aggregate summary to //! `/real-web/report.json`. //! //! The runner is the orchestrator that Phase 23 exit-criteria are checked //! against: it collates per-scenario pass/fail/panic/xfail status, captures //! peak RSS, wall-clock time, and the maximum screenshot diff percentage //! observed during `assert_screenshot_matches`, and emits a deterministic //! report that CI can diff against. //! //! The runner survives per-scenario panics: each scenario already runs under //! [`crate::panic::catch`], so one buggy site cannot abort a batch. The //! optional `--real-web-online` mode re-snapshots each domain's `goto_as` //! targets against the live network before running, for ad-hoc rebaselining //! against the live web. No commits are issued; snapshots are only rewritten //! on the working tree. use std::path::{Path, PathBuf}; use std::process::ExitCode; use std::time::{Instant, SystemTime, UNIX_EPOCH}; use crate::perf::PerfBudget; use crate::scenario::{self, Cmd, Scenario}; /// Default location of real-web scenarios within the workspace. pub const DEFAULT_SCENARIOS_DIR: &str = "crates/e2e/scenarios/real-web"; /// Per-scenario outcome captured by the runner. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ScenarioStatus { /// Scenario ran cleanly and every assertion passed. Pass, /// At least one assertion failed (no panic). Fail, /// Scenario or one of its commands panicked. Panic, /// Scenario is marked `# xfail` and (as expected) did not pass. Xfail, } impl ScenarioStatus { pub fn as_str(&self) -> &'static str { match self { Self::Pass => "pass", Self::Fail => "fail", Self::Panic => "panic", Self::Xfail => "xfail", } } } /// Per-scenario record written into the summary report. #[derive(Debug, Clone)] pub struct ScenarioRecord { /// Domain-style scenario name, e.g. `bing.com`. pub name: String, /// Status bucket for the totals. pub status: ScenarioStatus, /// Total scenario wall-clock time (ms). pub wall_ms: u64, /// Peak resident-set size observed during the run (bytes). pub peak_rss_bytes: u64, /// Maximum `assert_screenshot_matches` diff percentage observed during /// the run, or `None` if the scenario did not call that assertion. pub screenshot_diff_pct: Option, /// Human-readable failure messages from failing scenario steps. pub failures: Vec, /// One-line panic summary if the scenario panicked, else `None`. pub panic: Option, } /// Aggregate totals across all scenarios. #[derive(Debug, Clone, Default)] pub struct Totals { pub pass: usize, pub fail: usize, pub panic: usize, pub xfail: usize, } /// Final summary written to `report.json`. #[derive(Debug, Clone)] pub struct Report { pub started_at: String, pub duration_ms: u64, pub scenarios: Vec, pub totals: Totals, } impl Report { pub fn to_json(&self) -> String { let mut s = String::with_capacity(256); s.push('{'); s.push_str("\"started_at\":"); push_json_string(&mut s, &self.started_at); s.push(','); push_kv_u64(&mut s, "duration_ms", self.duration_ms); s.push(','); s.push_str("\"scenarios\":["); for (i, sc) in self.scenarios.iter().enumerate() { if i > 0 { s.push(','); } s.push('{'); s.push_str("\"name\":"); push_json_string(&mut s, &sc.name); s.push(','); s.push_str("\"status\":"); push_json_string(&mut s, sc.status.as_str()); s.push(','); push_kv_u64(&mut s, "wall_ms", sc.wall_ms); s.push(','); push_kv_u64(&mut s, "peak_rss_bytes", sc.peak_rss_bytes); s.push(','); s.push_str("\"screenshot_diff_pct\":"); match sc.screenshot_diff_pct { Some(v) => push_json_f64(&mut s, v), None => s.push_str("null"), } s.push(','); s.push_str("\"failures\":["); for (j, f) in sc.failures.iter().enumerate() { if j > 0 { s.push(','); } push_json_string(&mut s, f); } s.push(']'); s.push(','); s.push_str("\"panic\":"); match &sc.panic { Some(msg) => push_json_string(&mut s, msg), None => s.push_str("null"), } s.push('}'); } s.push(']'); s.push(','); s.push_str("\"totals\":{"); push_kv_u64(&mut s, "pass", self.totals.pass as u64); s.push(','); push_kv_u64(&mut s, "fail", self.totals.fail as u64); s.push(','); push_kv_u64(&mut s, "panic", self.totals.panic as u64); s.push(','); push_kv_u64(&mut s, "xfail", self.totals.xfail as u64); s.push('}'); s.push('}'); s } } /// Runner configuration. #[derive(Debug, Clone)] pub struct RealWebOptions { /// Directory containing `*.we` scenarios. Defaults to /// [`DEFAULT_SCENARIOS_DIR`]. pub scenarios_dir: PathBuf, /// Default perf budget passed to each scenario; scenarios may override it /// via `perf_budget`. pub budget: PerfBudget, /// When set, re-fetch every `goto_as` URL/path pair through the live /// network before running the scenario, overwriting the on-disk snapshot. pub online_resnapshot: bool, } impl Default for RealWebOptions { fn default() -> Self { Self { scenarios_dir: PathBuf::from(DEFAULT_SCENARIOS_DIR), budget: PerfBudget::default(), online_resnapshot: false, } } } /// CLI entry point: run the real-web suite and return a process exit code. /// /// Exits with `0` when every scenario is `pass` or `xfail`. Any `fail` or /// `panic` produces a non-zero exit code so CI fails closed. pub fn run_cli(opts: &RealWebOptions, out_dir: &Path) -> ExitCode { match run(opts, out_dir) { Ok(report) => { print_summary(&report); let any_bad = report.totals.fail > 0 || report.totals.panic > 0; if any_bad { ExitCode::from(1) } else { ExitCode::SUCCESS } } Err(e) => { eprintln!("real-web: {e}"); ExitCode::from(2) } } } /// Run the real-web suite, writing `report.json` under `/real-web/`. pub fn run(opts: &RealWebOptions, out_dir: &Path) -> Result { let scenarios = discover_scenarios(&opts.scenarios_dir)?; let started_at_iso = now_iso8601_utc(); let started = Instant::now(); let mut records = Vec::with_capacity(scenarios.len()); let mut totals = Totals::default(); let real_web_out_dir = out_dir.join("real-web"); let _ = std::fs::create_dir_all(&real_web_out_dir); for path in &scenarios { let record = run_one(path, &real_web_out_dir, opts); match record.status { ScenarioStatus::Pass => totals.pass += 1, ScenarioStatus::Fail => totals.fail += 1, ScenarioStatus::Panic => totals.panic += 1, ScenarioStatus::Xfail => totals.xfail += 1, } records.push(record); } let duration_ms = started.elapsed().as_millis().min(u64::MAX as u128) as u64; let report = Report { started_at: started_at_iso, duration_ms, scenarios: records, totals, }; let report_path = real_web_out_dir.join("report.json"); std::fs::write(&report_path, report.to_json()) .map_err(|e| format!("real-web: write report {}: {e}", report_path.display()))?; Ok(report) } /// Discover `*.we` files under `dir`, sorted alphabetically. Files are /// considered case-sensitively so the order is deterministic across platforms. pub fn discover_scenarios(dir: &Path) -> Result, String> { let entries = std::fs::read_dir(dir).map_err(|e| format!("read scenarios dir {}: {e}", dir.display()))?; let mut out = Vec::new(); for entry in entries { let entry = entry.map_err(|e| format!("scenarios dir entry: {e}"))?; let path = entry.path(); if path.extension().and_then(|s| s.to_str()) == Some("we") { out.push(path); } } out.sort(); Ok(out) } /// Run a single scenario file and produce its [`ScenarioRecord`]. fn run_one(path: &Path, real_web_out_dir: &Path, opts: &RealWebOptions) -> ScenarioRecord { let name = scenario_name(path); let parsed = match scenario::parse(path) { Ok(s) => s, Err(e) => { return ScenarioRecord { name, status: ScenarioStatus::Fail, wall_ms: 0, peak_rss_bytes: 0, screenshot_diff_pct: None, failures: vec![format!("scenario parse: {e}")], panic: None, }; } }; let xfail = scenario_is_xfail(path); if opts.online_resnapshot { if let Err(e) = resnapshot_scenario(&parsed) { // Resnapshot failure should not silently mask the run; record it // and continue with the on-disk snapshot. eprintln!("real-web: resnapshot {name}: {e}"); } } let scenario_out_dir = real_web_out_dir.join("runs").join(&name); let _ = std::fs::create_dir_all(&scenario_out_dir); let started = Instant::now(); let outcome = crate::panic::catch(|| scenario::run_with_budget(&parsed, &scenario_out_dir, opts.budget)); let wall_ms = started.elapsed().as_millis().min(u64::MAX as u128) as u64; match outcome { Err(report) => ScenarioRecord { name, status: ScenarioStatus::Panic, wall_ms, peak_rss_bytes: 0, screenshot_diff_pct: None, failures: Vec::new(), panic: Some(report.summary()), }, Ok(run_report) => { let mut failures: Vec = Vec::new(); let mut panic_summary: Option = None; for (lineno, ok, msg) in &run_report.steps { if *ok { continue; } if msg.starts_with("sev-crash") { panic_summary.get_or_insert_with(|| msg.clone()); } failures.push(format!("L{lineno}: {msg}")); } let diff_pct = max_screenshot_diff_pct(&run_report.steps); let status = if panic_summary.is_some() { if xfail { ScenarioStatus::Xfail } else { ScenarioStatus::Panic } } else if run_report.passed { ScenarioStatus::Pass } else if xfail { ScenarioStatus::Xfail } else { ScenarioStatus::Fail }; ScenarioRecord { name, status, wall_ms: run_report.wall_ms.max(wall_ms), peak_rss_bytes: run_report.peak_rss_bytes, screenshot_diff_pct: diff_pct, failures, panic: panic_summary, } } } } /// Derive a short scenario name from its file path. `foo/bing.com.we` -> /// `bing.com`. fn scenario_name(path: &Path) -> String { path.file_stem() .and_then(|s| s.to_str()) .map(|s| s.to_string()) .unwrap_or_else(|| path.display().to_string()) } /// Inspect the scenario file for an `# xfail` directive. Considered present /// when any line (trimmed) starts with `# xfail` — typically used in the /// header comments next to a tracking issue. fn scenario_is_xfail(path: &Path) -> bool { let Ok(text) = std::fs::read_to_string(path) else { return false; }; text.lines().any(|line| { let trimmed = line.trim_start(); if !trimmed.starts_with('#') { return false; } let after_hash = trimmed.trim_start_matches('#').trim_start(); after_hash.starts_with("xfail") }) } /// Find the maximum `assert_screenshot_matches` diff percentage in `steps`. /// Diff percentages are parsed from step messages of the form /// `screenshot_matches: 99.98% match (...)`. fn max_screenshot_diff_pct(steps: &[(usize, bool, String)]) -> Option { let mut max: Option = None; for (_, _, msg) in steps { if let Some(pct) = parse_match_pct(msg) { let diff = (100.0 - pct).max(0.0); max = Some(match max { Some(prev) => prev.max(diff), None => diff, }); } } max } /// Parse the match-percentage number out of a `screenshot_matches:` message. fn parse_match_pct(msg: &str) -> Option { let rest = msg.strip_prefix("screenshot_matches:")?.trim_start(); let end = rest.find('%')?; rest[..end].trim().parse::().ok() } /// Print the one-line summary `cargo run` prints to stdout. pub fn print_summary(report: &Report) { let total = report.scenarios.len(); let passed = report.totals.pass + report.totals.xfail; let duration_s = (report.duration_ms as f64) / 1000.0; println!( "real-web: {passed}/{total} passed ({} failed, {} panic) in {duration_s:.1}s", report.totals.fail, report.totals.panic ); } /// Re-snapshot a scenario's `goto_as` targets against the live network. /// Walks the scenario commands, fetches each `goto_as ` over HTTP, /// and overwrites the on-disk snapshot file with the live response body. /// /// Paths are resolved relative to the scenario's containing directory so they /// match the offline lookup convention. Errors are reported but do not abort /// the rest of the scenario list. pub fn resnapshot_scenario(scenario: &Scenario) -> Result<(), String> { let scenario_dir = scenario .source .parent() .map(|p| p.to_path_buf()) .unwrap_or_else(|| PathBuf::from(".")); let mut client = we_net::client::HttpClient::new(); let mut any_err: Option = None; for (_, cmd) in &scenario.commands { let Cmd::GotoAs { url, path } = cmd else { continue; }; let url = match we_url::Url::parse(url) { Ok(u) => u, Err(e) => { any_err.get_or_insert_with(|| format!("invalid URL {url}: {e}")); continue; } }; let target = if path.is_absolute() { path.clone() } else { scenario_dir.join(path) }; match client.get(&url) { Ok(resp) => { if let Some(parent) = target.parent() { let _ = std::fs::create_dir_all(parent); } if let Err(e) = std::fs::write(&target, &resp.body) { any_err.get_or_insert_with(|| format!("write {}: {e}", target.display())); } } Err(e) => { any_err.get_or_insert_with(|| format!("fetch {}: {e}", url.serialize())); } } } match any_err { Some(e) => Err(e), None => Ok(()), } } /// Format a [`SystemTime::now`] timestamp as ISO 8601 UTC (no fractional /// seconds), e.g. `2026-05-22T15:30:45Z`. fn now_iso8601_utc() -> String { let secs = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0); iso8601_utc_from_unix(secs) } /// Convert a UNIX timestamp in seconds to an ISO 8601 UTC string. /// Algorithm: Howard Hinnant's days_from_civil / civil_from_days /// (https://howardhinnant.github.io/date_algorithms.html). fn iso8601_utc_from_unix(secs: u64) -> String { let days = (secs / 86_400) as i64; let time_of_day = secs % 86_400; let hour = (time_of_day / 3600) as u32; let minute = ((time_of_day / 60) % 60) as u32; let second = (time_of_day % 60) as u32; // Days since 1970-01-01 -> y/m/d. let z = days + 719_468; // shift epoch to 0000-03-01 let era = z.div_euclid(146_097); let doe = z.rem_euclid(146_097) as u64; // [0, 146096] let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; // [0, 399] let mut y = (yoe as i64) + era * 400; let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365] let mp = (5 * doy + 2) / 153; // [0, 11] let day = (doy - (153 * mp + 2) / 5 + 1) as u32; // [1, 31] let month = if mp < 10 { (mp + 3) as u32 } else { (mp - 9) as u32 }; if month <= 2 { y += 1; } format!("{y:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}Z") } fn push_kv_u64(s: &mut String, key: &str, value: u64) { s.push('"'); s.push_str(key); s.push_str("\":"); s.push_str(&value.to_string()); } fn push_json_f64(s: &mut String, value: f64) { if !value.is_finite() { s.push_str("null"); return; } // Stable formatting: at most 6 fractional digits, no trailing zeros. let formatted = format!("{value:.6}"); let trimmed = formatted.trim_end_matches('0').trim_end_matches('.'); if trimmed.is_empty() || trimmed == "-" { s.push('0'); } else { s.push_str(trimmed); } } fn push_json_string(s: &mut String, value: &str) { s.push('"'); for c in value.chars() { match c { '"' => s.push_str("\\\""), '\\' => s.push_str("\\\\"), '\n' => s.push_str("\\n"), '\r' => s.push_str("\\r"), '\t' => s.push_str("\\t"), c if (c as u32) < 0x20 => { use std::fmt::Write; let _ = write!(s, "\\u{:04x}", c as u32); } c => s.push(c), } } s.push('"'); } #[cfg(test)] mod tests { use super::*; #[test] fn iso8601_epoch_is_1970_01_01() { assert_eq!(iso8601_utc_from_unix(0), "1970-01-01T00:00:00Z"); } #[test] fn iso8601_known_dates() { assert_eq!(iso8601_utc_from_unix(1_705_276_800), "2024-01-15T00:00:00Z"); assert_eq!(iso8601_utc_from_unix(1_577_836_800), "2020-01-01T00:00:00Z"); assert_eq!(iso8601_utc_from_unix(946_684_799), "1999-12-31T23:59:59Z"); } #[test] fn iso8601_handles_leap_year_feb_29() { // 2024-02-29T00:00:00Z = 1709164800 assert_eq!(iso8601_utc_from_unix(1_709_164_800), "2024-02-29T00:00:00Z"); } #[test] fn parse_match_pct_extracts_percentage() { let msg = "screenshot_matches: 99.98% match (3/16 px differ, tol=4, max_diff=0.1000%)"; let pct = parse_match_pct(msg).expect("pct"); assert!((pct - 99.98).abs() < 1e-6); } #[test] fn parse_match_pct_returns_none_for_other_messages() { assert!(parse_match_pct("goto about:blank ok").is_none()); assert!(parse_match_pct("screenshot_matches: no number here%").is_none()); } #[test] fn max_screenshot_diff_pct_picks_largest() { let steps = vec![ (1usize, true, "goto ok".to_string()), ( 2, true, "screenshot_matches: 99.98% match (3/16 px differ, tol=4, max_diff=0.1000%)" .to_string(), ), ( 3, false, "screenshot_matches: 80.00% match (4/16 px differ, tol=4, max_diff=0.1000%)" .to_string(), ), ]; let pct = max_screenshot_diff_pct(&steps).expect("diff pct"); // 100 - 80 == 20 should be picked over 100 - 99.98 == 0.02 assert!((pct - 20.0).abs() < 1e-6); } #[test] fn max_screenshot_diff_pct_returns_none_without_assert() { let steps = vec![(1usize, true, "goto ok".to_string())]; assert!(max_screenshot_diff_pct(&steps).is_none()); } #[test] fn report_to_json_round_trips_minimal_shape() { let r = Report { started_at: "2026-05-22T00:00:00Z".to_string(), duration_ms: 1234, scenarios: vec![ScenarioRecord { name: "foo.com".to_string(), status: ScenarioStatus::Pass, wall_ms: 42, peak_rss_bytes: 5_000_000, screenshot_diff_pct: Some(0.5), failures: Vec::new(), panic: None, }], totals: Totals { pass: 1, fail: 0, panic: 0, xfail: 0, }, }; let json = r.to_json(); assert!(json.contains("\"started_at\":\"2026-05-22T00:00:00Z\"")); assert!(json.contains("\"duration_ms\":1234")); assert!(json.contains("\"name\":\"foo.com\"")); assert!(json.contains("\"status\":\"pass\"")); assert!(json.contains("\"screenshot_diff_pct\":0.5")); assert!(json.contains("\"panic\":null")); assert!(json.contains("\"totals\":{\"pass\":1,\"fail\":0,\"panic\":0,\"xfail\":0}")); } #[test] fn scenario_is_xfail_detects_top_comment() { let dir = std::env::temp_dir().join(format!( "we-e2e-xfail-{}-{}", std::process::id(), SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_nanos()) .unwrap_or(0) )); let _ = std::fs::create_dir_all(&dir); let path = dir.join("xfail.we"); std::fs::write(&path, "# xfail: see isu 999\ngoto about:blank\n").unwrap(); assert!(scenario_is_xfail(&path)); let path2 = dir.join("normal.we"); std::fs::write(&path2, "# regular comment\ngoto about:blank\n").unwrap(); assert!(!scenario_is_xfail(&path2)); } }