Automerge backed Markdown editor for the Atmosphere
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652#!/usr/bin/env node// scripts/check-mutation-baseline.mjs//// Deterministic mutation-baseline regression check for audit-flow.//// Reads two Stryker-style JSON reports (or floor-map.json + a Stryker JSON)// and verifies that NO file regressed:// - HARD FAIL: any file's `survived` count INCREASED// - HARD FAIL: any file's score DROPPED below the floor (60% by default)// - HARD FAIL: a baseline file is missing from the post-refactor report// without explicit deletion acknowledgement// - PASS: score may stay flat OR improve; survived may stay flat OR drop//// Why this exists// ---------------// The pre-audit-flow spike (commit eca4b4f, branch audit-flow-pre-audit)// established a mutation-test baseline at `.audit-flow/floor-map.json` with// committed Stryker JSON reports at `reports/mutation/by-file/<source>.json`.// Adversary review (BLOCKER #6 / #13) flagged that percentage-only floor// gates allow regression to hide behind code shrinkage: 100 trivially-killed// mutants removed + 50 genuine new bugs introduced can RAISE the percentage// while LOWERING coverage. The v2 floor-map schema records absolute survivor// counts to fix this — but the schema is theater until something reads it.// This script is what reads it.//// Usage// -----// # Audit-flow per-issue verification (canonical use):// node scripts/check-mutation-baseline.mjs \// --baseline .audit-flow/<issue-id>/mutation-baseline.json \// --post reports/mutation/mutation.json//// # Pre-audit spike sanity check (against floor-map):// node scripts/check-mutation-baseline.mjs \// --baseline .audit-flow/floor-map.json \// --post reports/mutation/by-file/workspace.json//// # Optional: scope to a specific dep-map's files_in_scope:// node scripts/check-mutation-baseline.mjs \// --baseline ... --post ... \// --dep-map .audit-flow/<issue-id>/dep-map.json//// # Allow specific files to be deleted (refactor that removes a file):// node scripts/check-mutation-baseline.mjs ... \// --allow-deleted src/plugin/old-thing.ts,src/plugin/another.ts//// Inputs// ------// --baseline (required): path to a JSON file in one of two shapes:// (a) Stryker mutation.json — raw report (used when one Stryker run was// captured before refactoring as the baseline)// (b) floor-map.json (v2) — `.files[path].{score,killed,timed_out,// survived,total_mutants}` — used when comparing against the spike// baseline// (c) audit-flow mutation-baseline.json (per-issue) —// `.files[path].{score,killed,survived,total}` per// `subagents/prompt-mutation-baseline.md`// The script auto-detects which shape via top-level keys.//// --post (required): path to a JSON file in one of:// (a) Stryker mutation.json// (b) An object whose keys are file paths and values are// `{score,killed,timed_out,survived,total_mutants}` (same as// floor-map's .files entry — for stitching multiple per-file reports)// The script auto-detects.//// --dep-map (optional): path to dep-map.json with `files_in_scope: [...]`.// When present, the comparison is restricted to those files. Files in// the baseline but NOT in scope are reported as INFO (not failures).//// --allow-deleted (optional): comma-separated list of paths that may// appear in baseline but are missing from post (planned deletions per// the dep map / plan).//// --floor (optional, default 60): the minimum acceptable score.//// --report (optional): write a human-readable Markdown report to this// path in addition to stderr. Default: print only.//// --json (optional): emit machine-readable JSON verdict to stdout// (in addition to the human report on stderr). Useful for CI.//// Exit codes// ----------// 0 — PASS (no regressions)// 1 — FAIL (one or more regressions found)// 2 — usage / IO / parse error//// Notes on Stryker's `Timeout` status// -----------------------------------// Stryker counts Timeout as a kill in its score percentage, but a timeout// is weak evidence (the mutant produced a hang, not a semantically-detected// change). This script's `survived` regression check uses ONLY Survived// status (excluding Timeout) — that matches floor-map.json v2's `survived`// field. The score check uses Stryker's percentage, which DOES count// timeouts as kills. This is intentional asymmetry: the score lets// timeout-counted kills protect the floor, but the survived-count gate// catches regressions that flip a timeout into a clean survival.
import { readFileSync, writeFileSync, existsSync } from 'node:fs'import { resolve } from 'node:path'import process from 'node:process'
// ---------- argv parsing ----------
function parseArgs(argv) { const args = { _: [] } for (let i = 2; i < argv.length; i++) { const a = argv[i] if (a === '--help' || a === '-h') { args.help = true } else if (a.startsWith('--')) { const key = a.slice(2) const next = argv[i + 1] if (next !== undefined && !next.startsWith('--')) { args[key] = next i++ } else { args[key] = true } } else { args._.push(a) } } return args}
const HELP = `Usage: node scripts/check-mutation-baseline.mjs [options]
Required: --baseline <path> Path to baseline JSON (Stryker, floor-map, or mutation-baseline) --post <path> Path to post-refactor JSON (Stryker mutation.json)
Optional: --dep-map <path> Restrict comparison to files_in_scope from this dep-map --scope-files <csv> Comma-separated paths to compare; overrides --dep-map. Use when Stryker only mutated a subset of baselines (e.g. when only one source file was passed to --mutate) to suppress BASELINE_FILE_MISSING_FROM_POST. --allow-deleted <csv> Comma-separated paths that may be missing from post --floor <n> Minimum score (default: 60) --report <path> Write Markdown report to this file --json Emit JSON verdict on stdout (for CI) --help, -h Show this message
Exit codes: 0 PASS, 1 FAIL, 2 usage/IO error`
// ---------- JSON shape detection + normalization ----------
/** * Normalize any of the three accepted baseline shapes to a uniform internal * shape: { files: { <path>: { score, killed, survived, total, source } } } * where `source` is one of 'stryker' | 'floor-map' | 'mutation-baseline' * for trace messages. * * Stryker's per-mutant `status` is one of: * Killed | Survived | Timeout | NoCoverage | RuntimeError | CompileError | Ignored * For our purposes: * - `killed` = Killed + Timeout (matches Stryker's score numerator) * - `survived` = Survived ONLY (excludes Timeout — see file header note) * - `total` = Killed + Survived + Timeout + RuntimeError (excludes Ignored * and CompileError, which Stryker excludes from the denominator) * - `score` = (killed_total / valid_total) * 100, where killed_total is * Killed+Timeout and valid_total excludes Ignored/CompileError. We * prefer the score from the source JSON when it has one; otherwise * compute from mutants. */function normalizeReport(json, sourcePath) { // Stryker mutation.json has a top-level `schemaVersion` and `files` if (json && typeof json === 'object' && 'schemaVersion' in json && 'files' in json) { return normalizeStryker(json, sourcePath) } // floor-map.json (v2) has `version`, `floor`, and `files` with score/killed/survived/total_mutants if (json && typeof json === 'object' && 'version' in json && 'floor' in json && 'files' in json) { return normalizeFloorMap(json, sourcePath) } // audit-flow mutation-baseline.json has `issue_id`, `floor`, `files` with score/killed/survived/total if (json && typeof json === 'object' && 'files' in json && ('issue_id' in json || 'floor' in json)) { return normalizeMutationBaseline(json, sourcePath) } // Bare object whose keys are paths and values are floor-map-shape entries // (used for stitching multiple per-file Stryker reports). Heuristic: top-level // keys look like file paths. if (json && typeof json === 'object' && !Array.isArray(json)) { const keys = Object.keys(json) if (keys.length > 0 && keys.every(k => k.includes('/') && k.endsWith('.ts') || k.endsWith('.tsx'))) { return normalizeBarePathMap(json, sourcePath) } } throw new Error( `Could not detect JSON shape for ${sourcePath}. ` + `Expected: Stryker mutation.json (with schemaVersion), ` + `floor-map.json (with version+floor+files), ` + `or mutation-baseline.json (with issue_id+files). ` + `Got top-level keys: ${Object.keys(json || {}).join(', ')}`, )}
function normalizeStryker(json, sourcePath) { const files = {} for (const [path, entry] of Object.entries(json.files ?? {})) { const counts = { Killed: 0, Survived: 0, Timeout: 0, NoCoverage: 0, RuntimeError: 0, CompileError: 0, Ignored: 0 } for (const m of entry.mutants ?? []) { const status = m.status if (status in counts) counts[status]++ } // Stryker's score = (Killed + Timeout) / (Killed + Survived + Timeout + NoCoverage + RuntimeError) * 100 // (Ignored and CompileError are excluded from both numerator and denominator.) const validTotal = counts.Killed + counts.Survived + counts.Timeout + counts.NoCoverage + counts.RuntimeError const killedNumerator = counts.Killed + counts.Timeout const score = validTotal === 0 ? 0 : (killedNumerator / validTotal) * 100 files[path] = { score: round2(score), killed: counts.Killed, timed_out: counts.Timeout, survived: counts.Survived, no_coverage: counts.NoCoverage, errored: counts.RuntimeError, total: validTotal, source: 'stryker', } } return { files, sourcePath, sourceShape: 'stryker' }}
function normalizeFloorMap(json, sourcePath) { const files = {} for (const [path, entry] of Object.entries(json.files ?? {})) { // floor-map records {score, killed, timed_out, survived, total_mutants} // total_mutants is the *valid* total (Stryker's denominator + survivors). if (entry.score === undefined) { // pre-v2 entries (no absolute counts) — can only check score, not survived. // We still include them but mark `survived` as null so survived-regression // check is skipped for that file (with a warning). files[path] = { score: 0, killed: null, timed_out: null, survived: null, no_coverage: null, errored: null, total: null, source: 'floor-map (v1, no absolute counts)', } continue } files[path] = { score: round2(entry.score), killed: entry.killed ?? null, timed_out: entry.timed_out ?? 0, survived: entry.survived ?? null, no_coverage: null, errored: null, total: entry.total_mutants ?? null, source: 'floor-map', status: entry.status, // 'at_floor' | 'exempted' | etc. } } return { files, sourcePath, sourceShape: 'floor-map' }}
function normalizeMutationBaseline(json, sourcePath) { // audit-flow per-issue mutation-baseline.json: // { files: { <path>: { score, killed, survived, total } } } const files = {} for (const [path, entry] of Object.entries(json.files ?? {})) { files[path] = { score: round2(entry.score ?? 0), killed: entry.killed ?? null, timed_out: entry.timed_out ?? 0, survived: entry.survived ?? null, no_coverage: null, errored: null, total: entry.total ?? null, source: 'mutation-baseline', } } return { files, sourcePath, sourceShape: 'mutation-baseline' }}
function normalizeBarePathMap(json, sourcePath) { // Same shape as floor-map's `.files` entries — used for stitching multiple // single-file Stryker JSONs. return normalizeFloorMap({ version: 2, floor: 60, files: json }, sourcePath)}
function round2(n) { return Math.round(n * 100) / 100}
// ---------- Comparison logic ----------
/** * Compare two normalized reports. Returns: * { verdict: 'PASS' | 'FAIL', * issues: [...], // hard failures * warnings: [...], // non-fatal observations * perFile: [...] } // detailed per-file delta rows */function compare(baseline, post, opts) { const { floor, scopeFiles, allowDeleted } = opts const issues = [] const warnings = [] const perFile = []
const baselineFiles = baseline.files const postFiles = post.files
const allFilesUnion = new Set([ ...Object.keys(baselineFiles), ...Object.keys(postFiles), ])
// Filter to scope if a dep-map was provided. Files outside scope are not // compared (they are NOT failures — they may have intentionally moved). const filesToCompare = scopeFiles ? Array.from(allFilesUnion).filter(p => scopeFiles.has(p)) : Array.from(allFilesUnion)
// Detect baseline files that are entirely missing from post. for (const path of filesToCompare) { const b = baselineFiles[path] const p = postFiles[path]
// Skip exempted files — adversary's HIGH finding #5/#6 explicitly // accepts that exempted files have known surviving mutants. The // `survived` count for an exempted file is allowed to stay at the // exempted baseline value but NOT to grow. const isExempted = b?.status === 'exempted'
if (b && !p) { // Baseline says this file should be tested, but post-refactor doesn't // include it. Possibilities: // 1. The file was deleted by the refactor (planned per dep map). // 2. Stryker forgot to mutate it (config drift bug — repeat of e815530). // 3. The post report was for a different scope. if (allowDeleted.has(path)) { warnings.push({ kind: 'PLANNED_DELETION', path, message: `${path} was in baseline but is missing from post — accepted via --allow-deleted.`, }) perFile.push({ path, baseline: b, post: null, delta: null, status: 'DELETED' }) } else { issues.push({ kind: 'BASELINE_FILE_MISSING_FROM_POST', path, message: `${path} was in baseline (score: ${b.score}, survived: ${b.survived}) but is missing from post-refactor report. ` + `If this file was intentionally deleted, pass --allow-deleted ${path}. ` + `Otherwise this likely indicates Stryker config drift (repeat of e815530 incident — ` + `check vitest.config.stryker.ts include[] and stryker.config.mjs mutate[]).`, }) perFile.push({ path, baseline: b, post: null, delta: null, status: 'MISSING' }) } continue }
if (!b && p) { // Post adds a file not in baseline. A refactor may legitimately // introduce a new file — but the new file must STILL meet the floor. // Otherwise audit-flow could "improve" coverage by deleting a high- // scoring file and replacing it with a low-scoring one of the same // name pattern. Apply the floor gate; warn (not fail) if the file // is exempted via convention (we don't have a way to know yet). if (p.score < floor) { issues.push({ kind: 'NEW_FILE_BELOW_FLOOR', path, message: `${path} is new in post-refactor (not in baseline) but scored ${p.score}% < ${floor}% floor. ` + `New files introduced by audit-flow must meet the same floor as existing files. ` + `Either add tests to raise the score, or pre-exempt this file in floor-map.json with a documented reason.`, }) perFile.push({ path, baseline: null, post: p, delta: null, status: 'NEW_FLOOR_VIOLATION' }) } else { warnings.push({ kind: 'NEW_FILE', path, message: `${path} appears in post but not baseline. Score: ${p.score}%, survived: ${p.survived}. ` + `Add it to floor-map.json after this audit so future runs can detect regressions.`, }) perFile.push({ path, baseline: null, post: p, delta: null, status: 'NEW' }) } continue }
// Both present — the regular case. Run the two gates. if (!b || !p) continue // unreachable, but TypeScript-style guard
const scoreDelta = round2(p.score - b.score) const survivedDelta = (b.survived !== null && p.survived !== null) ? p.survived - b.survived : null
const row = { path, baseline: b, post: p, delta: { score: scoreDelta, survived: survivedDelta }, status: 'PASS', }
// Gate 1: floor compliance. Exempted files are allowed below floor at // their original score (the spike accepted them); enforce only for // non-exempted files. if (!isExempted && p.score < floor) { issues.push({ kind: 'FLOOR_VIOLATION', path, message: `${path} dropped below floor: ${p.score}% < ${floor}% (was ${b.score}%, Δ ${scoreDelta >= 0 ? '+' : ''}${scoreDelta}).`, }) row.status = 'FLOOR_VIOLATION' }
// Gate 2: survived must not increase. This is the regression gate that // catches score-stable-but-coverage-degraded refactors. if (survivedDelta !== null && survivedDelta > 0) { const exemptedNote = isExempted ? ' (file is EXEMPTED but survived is still allowed only to stay flat or drop)' : '' issues.push({ kind: 'SURVIVED_INCREASED', path, message: `${path}: surviving mutants grew from ${b.survived} to ${p.survived} (+${survivedDelta})${exemptedNote}. ` + `Score change: ${b.score}% → ${p.score}% (Δ ${scoreDelta >= 0 ? '+' : ''}${scoreDelta}). ` + `Even if score is flat or up, increased survivors mean the refactor introduced behavior the test suite no longer covers.`, }) row.status = row.status === 'PASS' ? 'SURVIVED_INCREASED' : row.status }
// Soft warning: score dropped but didn't violate floor and survived didn't grow // (means timeouts shifted around — fragile, worth noting but not a fail). if (scoreDelta < 0 && row.status === 'PASS') { warnings.push({ kind: 'SCORE_DROPPED_NO_REGRESSION', path, message: `${path}: score dropped (${b.score}% → ${p.score}%, Δ ${scoreDelta}) ` + `but survived count did NOT increase (b: ${b.survived} → p: ${p.survived}). ` + `Likely cause: Stryker timeouts shifted (timeouts count toward score but not survived). ` + `Schema-v2 absolute-count gate passes, but worth a manual look.`, }) }
// Survived stayed flat or dropped: legitimately a PASS. perFile.push(row) }
// Survived-count check needs absolute counts. If baseline came from a // floor-map v1 entry without survived, warn so the user knows we couldn't // enforce the gate for that file. for (const row of perFile) { if (row.baseline && row.baseline.survived === null && row.post) { warnings.push({ kind: 'SURVIVED_GATE_SKIPPED', path: row.path, message: `${row.path}: baseline lacks absolute survived count (likely floor-map v1). ` + `Score-only check applied; survived-regression gate skipped. ` + `Re-baseline this file under v2 to enable the full gate.`, }) } }
return { verdict: issues.length === 0 ? 'PASS' : 'FAIL', issues, warnings, perFile, }}
// ---------- Output formatting ----------
function formatMarkdown(result, opts) { const { baselinePath, postPath, floor } = opts const lines = [] lines.push(`# Mutation-baseline regression check`) lines.push(``) lines.push(`**Verdict:** ${result.verdict === 'PASS' ? '✅ PASS' : '❌ FAIL'}`) lines.push(`**Floor:** ${floor}%`) lines.push(`**Baseline:** \`${baselinePath}\``) lines.push(`**Post:** \`${postPath}\``) lines.push(``)
// Per-file table lines.push(`## Per-file`) lines.push(``) lines.push(`| File | Baseline score | Post score | Δ score | Baseline survived | Post survived | Δ survived | Status |`) lines.push(`|------|---------------:|-----------:|--------:|------------------:|--------------:|-----------:|--------|`) const sortedRows = [...result.perFile].sort((a, b) => statusRank(b.status) - statusRank(a.status) || a.path.localeCompare(b.path)) for (const row of sortedRows) { const bScore = row.baseline?.score ?? '—' const pScore = row.post?.score ?? '—' const dScore = row.delta?.score !== undefined && row.delta?.score !== null ? (row.delta.score >= 0 ? `+${row.delta.score}` : `${row.delta.score}`) : '—' const bSurv = row.baseline?.survived ?? '—' const pSurv = row.post?.survived ?? '—' const dSurv = row.delta?.survived !== undefined && row.delta?.survived !== null ? (row.delta.survived >= 0 ? `+${row.delta.survived}` : `${row.delta.survived}`) : '—' const statusEmoji = row.status === 'PASS' ? '✅' : (row.status === 'NEW' || row.status === 'DELETED' ? 'ℹ️' : '❌') lines.push(`| \`${row.path}\` | ${bScore} | ${pScore} | ${dScore} | ${bSurv} | ${pSurv} | ${dSurv} | ${statusEmoji} ${row.status} |`) } lines.push(``)
if (result.issues.length > 0) { lines.push(`## Failures (${result.issues.length})`) lines.push(``) for (const issue of result.issues) { lines.push(`### ${issue.kind}: \`${issue.path}\``) lines.push(``) lines.push(issue.message) lines.push(``) } }
if (result.warnings.length > 0) { lines.push(`## Warnings (${result.warnings.length})`) lines.push(``) for (const w of result.warnings) { lines.push(`- **${w.kind}** \`${w.path}\`: ${w.message}`) } lines.push(``) }
return lines.join('\n')}
function statusRank(status) { if (status === 'FLOOR_VIOLATION' || status === 'SURVIVED_INCREASED' || status === 'MISSING') return 3 if (status === 'NEW' || status === 'DELETED') return 2 return 1}
// ---------- Main ----------
function main() { const args = parseArgs(process.argv)
if (args.help) { process.stdout.write(HELP) process.exit(0) }
if (!args.baseline || !args.post) { process.stderr.write(`Error: --baseline and --post are required.\n\n`) process.stderr.write(HELP) process.exit(2) }
const baselinePath = resolve(args.baseline) const postPath = resolve(args.post) const floor = args.floor !== undefined ? Number(args.floor) : 60 if (Number.isNaN(floor) || floor < 0 || floor > 100) { process.stderr.write(`Error: --floor must be a number between 0 and 100.\n`) process.exit(2) } const allowDeleted = new Set( typeof args['allow-deleted'] === 'string' ? args['allow-deleted'].split(',').map(s => s.trim()).filter(Boolean) : [], )
// Load JSON. let baselineJson, postJson try { if (!existsSync(baselinePath)) throw new Error(`File not found: ${baselinePath}`) if (!existsSync(postPath)) throw new Error(`File not found: ${postPath}`) baselineJson = JSON.parse(readFileSync(baselinePath, 'utf-8')) postJson = JSON.parse(readFileSync(postPath, 'utf-8')) } catch (e) { process.stderr.write(`Error reading inputs: ${e.message}\n`) process.exit(2) }
let depMapScope = null if (typeof args['dep-map'] === 'string') { try { const depMap = JSON.parse(readFileSync(resolve(args['dep-map']), 'utf-8')) const scope = depMap?.files_in_scope if (!Array.isArray(scope)) { throw new Error(`dep-map at ${args['dep-map']} has no files_in_scope array`) } depMapScope = new Set(scope) } catch (e) { process.stderr.write(`Error reading dep-map: ${e.message}\n`) process.exit(2) } }
// --scope-files: comma-separated list of file paths to limit the comparison // to. Useful when a Stryker run only mutates a subset of the baseline files // (e.g. running `stryker run --mutate src/plugin/workspace.ts`); without // scoping, the unmutated files trigger BASELINE_FILE_MISSING_FROM_POST // failures even though they weren't part of this verification run. // Mutually-compatible with --dep-map: when both are present, --scope-files // takes precedence (use it to override the dep-map's files_in_scope). if (typeof args['scope-files'] === 'string') { const explicitScope = new Set( args['scope-files'].split(',').map(s => s.trim()).filter(Boolean), ) if (explicitScope.size === 0) { process.stderr.write(`Error: --scope-files passed but parsed empty.\n`) process.exit(2) } depMapScope = explicitScope }
let baseline, post try { baseline = normalizeReport(baselineJson, baselinePath) post = normalizeReport(postJson, postPath) } catch (e) { process.stderr.write(`Error normalizing report: ${e.message}\n`) process.exit(2) }
const result = compare(baseline, post, { floor, scopeFiles: depMapScope, allowDeleted, })
// Human-readable to stderr (so --json on stdout stays parseable). const md = formatMarkdown(result, { baselinePath, postPath, floor }) process.stderr.write(md + '\n')
if (typeof args.report === 'string') { try { writeFileSync(resolve(args.report), md, 'utf-8') } catch (e) { process.stderr.write(`Warning: could not write report to ${args.report}: ${e.message}\n`) } }
if (args.json) { process.stdout.write(JSON.stringify({ verdict: result.verdict, floor, baseline_source: baseline.sourceShape, post_source: post.sourceShape, issues: result.issues, warnings: result.warnings, per_file: result.perFile, }, null, 2) + '\n') }
process.exit(result.verdict === 'PASS' ? 0 : 1)}
main()