// Runner: evaluates the skip contract, executes checks, aggregates verdicts. import { Context, type ProvisionFn } from './context.js' import { deriveCapabilities } from './discover.js' import { adminAuthConfigured } from './xrpc.js' import type { Check, CheckResult, Mode, Profile, TargetProfile, Tier } from './types.js' export interface RunOptions { mode: Mode // highest mode allowed tiers?: Tier[] only?: string[] // check id prefixes provision?: ProvisionFn /** Target profile declaring deliberate divergences. Waived expectations are annotated * as acknowledged (fail→warn), never silently silenced. */ profile?: Profile } // Consent ranking: a check runs only if its mode is at or below the run's mode. // Sequence reads as increasing target impact: reads are safe, mutations write to a test // account, lifecycle transitions (deactivate/migrate) change account state. const MODE_RANK: Record = { readonly: 0, mutate: 1, lifecycle: 2 } function skipReason(check: Check, profile: TargetProfile, opts: RunOptions): string | null { if (MODE_RANK[check.mode] > MODE_RANK[opts.mode]) { return `mode-${check.mode}-not-enabled` } if (opts.tiers && !opts.tiers.includes(check.tier)) return 'tier-not-selected' if (opts.only && !opts.only.some((p) => check.id.startsWith(p))) return 'not-selected' const requiredEndpoints = check.requires?.endpoints ?? [] if (requiredEndpoints.length > 0 && !profile.endpoints) { // No inventory, no skip decision. Running an endpoint-gated check on unknown data // turns a target-side absence into a check-side error (a raw 404 assertion // failure) — fail closed instead. discover() populates the inventory, so this only // fires when a profile was hand-built outside the standard pipeline. return 'endpoint-inventory-unavailable' } for (const ep of requiredEndpoints) { if (!profile.endpoints!.implemented.includes(ep)) { return `endpoint-not-implemented:${ep}` } } if ((check.requires?.accounts?.length ?? 0) > 0 && !opts.provision) { return 'account-provisioning-unavailable' } // Capabilities are derived at the runAll boundary. A bare runCheck call has no // derivation and therefore no capability enforcement; only runAll derives → runs. for (const cap of check.requires?.capabilities ?? []) { if (!profile.capabilities?.has(cap)) { return `capability-absent:${cap}` } } // Admin-auth checks need an admin password for the target (PDSUITE_ADMIN_PASSWORD, or // the oracle's known password in e2e). Without one the check skips — never fails. if (check.requires?.auth === 'admin' && !adminAuthConfigured()) { return 'admin-auth-unavailable' } return null } export async function runCheck(check: Check, profile: TargetProfile, opts: RunOptions): Promise { const started = Date.now() const skip = skipReason(check, profile, opts) if (skip) { return { checkId: check.id, verdict: 'skip', skipReason: skip, assertions: [], durationMs: 0 } } const ctx = new Context(profile.url, profile, check, opts.provision) try { const timeout = check.timeoutMs ?? 30_000 await Promise.race([ check.run(ctx), new Promise((_, rej) => setTimeout(() => rej(new Error(`timeout after ${timeout}ms`)), timeout)), ]) } catch (err) { return { checkId: check.id, verdict: 'error', errorMessage: err instanceof Error ? err.message : String(err), assertions: ctx.assertions, durationMs: Date.now() - started, } } // Unasserted expectations are failures of the check itself — a pass must be earned. const missing = check.expectations.filter((e) => !ctx.assertions.some((a) => a.expectation.id === e.id)) if (missing.length > 0) { return { checkId: check.id, verdict: 'error', errorMessage: `expectations never asserted: ${missing.map((e) => e.id).join(', ')}`, assertions: ctx.assertions, durationMs: Date.now() - started, } } const verdict = ctx.assertions.some((a) => a.verdict === 'fail') ? 'fail' : ctx.assertions.some((a) => a.verdict === 'warn') ? 'warn' : 'pass' const result: CheckResult = { checkId: check.id, verdict, assertions: ctx.assertions, durationMs: Date.now() - started } applyWaivers(check, result, opts.profile) return result } /** Annotate assertions that a target profile acknowledges as deliberate divergences. * A waived fail is downgraded to warn and carries the target's note + link; a waived warn * is annotated in place. The divergence stays visible in the report — it is owned, not hidden. */ function applyWaivers(check: Check, result: CheckResult, profile?: Profile): void { if (!profile) return for (const a of result.assertions) { if (a.verdict === 'pass') continue const key = `${check.id}#${a.expectation.id}` const w = profile.divergences.find((d) => d.expectation === key) if (!w) continue a.acknowledged = { note: w.note, link: w.link } if (a.verdict === 'fail') a.verdict = 'warn' } // Recompute the check verdict after downgrades. result.verdict = result.assertions.some((a) => a.verdict === 'fail') ? 'fail' : result.assertions.some((a) => a.verdict === 'warn') ? 'warn' : 'pass' } export async function runAll(checks: Check[], profile: TargetProfile, opts: RunOptions): Promise { // Derive capabilities once, here. discover() has already populated the endpoint // inventory by construction, so the derivation always sees a complete profile. // Skips are not failures — a capability the target legitimately lacks must surface as // a neutral skip, never as a failed assertion. profile.capabilities = deriveCapabilities(profile, opts.provision !== undefined) // Sequential on purpose: checks share provisioned accounts and the firehose, so // concurrent execution would interleave mutations and make sequencing assertions flaky. const results: CheckResult[] = [] for (const check of checks) { results.push(await runCheck(check, profile, opts)) } return results }