Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
3.8 kB · 119 lines
JavaScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120#!/usr/bin/env node// Builds a per-case agreement table between two completed runs (default:// dawn.node on lavapipe vs. Chromium), broken down by top-level CTS area.// Reads every results/<run>/*/*.json the same contamination-tolerant way// aggregate-dawn-node.mjs does (the real results object is the last// `{"version": ...}` blob in the file), maps each individual case name to// its status, then for every case name present in either run reports// whether both agree (same status) or not, and which cases exist in only// one run.//// Usage: node agreement.mjs [runA=dawn-node] [runB=browser-chromium]import { readdirSync, readFileSync, statSync } from 'node:fs';import { join } from 'node:path';
const SELF_DIR = new URL('.', import.meta.url).pathname;const RESULTS_ROOT = join(SELF_DIR, 'results');
function parseResultsJSON(text) { const marker = '{\n "version"'; const lastStart = text.lastIndexOf(marker); if (lastStart === -1) return undefined; try { return JSON.parse(text.slice(lastStart)); } catch { return undefined; }}
function areaOf(testName) { const afterSuite = testName.split(':')[1] ?? ''; return afterSuite.split(',').slice(0, 2).join(',') || '(unknown)';}
/** @returns {Map<string, string>} test name -> status */function loadRun(runName) { const root = join(RESULTS_ROOT, runName); const byCase = new Map(); let shardDirs; try { shardDirs = readdirSync(root).filter(name => /^\d+$/.test(name)); } catch (e) { console.error(`no results under ${root}: ${e.message}`); process.exit(1); } for (const shard of shardDirs) { const shardDir = join(root, shard); let entries; try { entries = readdirSync(shardDir); } catch { continue; } for (const entry of entries) { if (!entry.endsWith('.json')) continue; const path = join(shardDir, entry); if (statSync(path).size === 0) continue; const parsed = parseResultsJSON(readFileSync(path, 'utf8')); if (parsed === undefined) continue; for (const [testName, result] of parsed.results ?? []) { byCase.set(testName, result.status ?? '(no status)'); } } } return byCase;}
const runA = process.argv[2] || 'dawn-node';const runB = process.argv[3] || 'browser-chromium';
const a = loadRun(runA);const b = loadRun(runB);
const allNames = new Set([...a.keys(), ...b.keys()]);
// area -> { agree, disagree, onlyA, onlyB }const byArea = new Map();function bump(area, key) { if (!byArea.has(area)) byArea.set(area, { agree: 0, disagree: 0, onlyA: 0, onlyB: 0 }); byArea.get(area)[key]++;}
const disagreements = [];for (const name of allNames) { const area = areaOf(name); const sa = a.get(name); const sb = b.get(name); if (sa === undefined) { bump(area, 'onlyB'); } else if (sb === undefined) { bump(area, 'onlyA'); } else if (sa === sb) { bump(area, 'agree'); } else { bump(area, 'disagree'); disagreements.push([name, sa, sb]); }}
console.log(`${runA}: ${a.size} cases; ${runB}: ${b.size} cases; ${allNames.size} case names total`);console.log('');console.log(`by area (${runA} vs ${runB}):`);for (const [area, counts] of [...byArea.entries()].sort()) { const total = counts.agree + counts.disagree + counts.onlyA + counts.onlyB; console.log( `${area}: total=${total} agree=${counts.agree} disagree=${counts.disagree} ` + `only_${runA}=${counts.onlyA} only_${runB}=${counts.onlyB}` );}
if (disagreements.length) { console.log(''); console.log(`disagreements (${disagreements.length}):`); for (const [name, sa, sb] of disagreements.slice(0, 50)) { console.log(` ${runA}=${sa} ${runB}=${sb} ${name}`); } if (disagreements.length > 50) { console.log(` ... and ${disagreements.length - 50} more`); }}