import type { ThoughtEvent } from "../events/types.js"; import type { AgentRun } from "../store/types.js"; const terminalStatuses = new Set(["completed", "failed", "blocked", "abandoned"]); const terminalTypes = new Set([ "stream.thought.agent.run.completed", "stream.thought.agent.run.failed", "stream.thought.agent.run.blocked", "stream.thought.agent.run.abandoned", ]); export interface RunEvidenceIssue { code: | "missing-terminal-event" | "terminal-status-mismatch" | "terminal-event-on-nonterminal-run" | "duplicate-terminal-events" | "missing-output-event" | "completed-without-output" | "terminal-output-mismatch" | "nonterminal-has-output"; message: string; eventId?: string; } export interface RunEvidenceReport { runId: string; status: AgentRun["status"]; consistent: boolean; terminalEventIds: string[]; issues: RunEvidenceIssue[]; } export function auditRunEvidence(runs: AgentRun[], events: ThoughtEvent[]): RunEvidenceReport[] { const eventsById = new Map(events.map((event) => [event.id, event])); const terminalByRun = new Map(); for (const event of events) { if (!terminalTypes.has(event.type) || typeof event.payload.runId !== "string") continue; const bucket = terminalByRun.get(event.payload.runId) ?? []; bucket.push(event); terminalByRun.set(event.payload.runId, bucket); } return runs.map((run) => { const issues: RunEvidenceIssue[] = []; const terminalEvents = terminalByRun.get(run.id) ?? []; const rowIsTerminal = terminalStatuses.has(run.status); const expectedType = rowIsTerminal ? `stream.thought.agent.run.${run.status}` : undefined; if (rowIsTerminal && terminalEvents.length === 0) { issues.push({ code: "missing-terminal-event", message: `Run row is ${run.status}, but no terminal event exists`, }); } if (!rowIsTerminal && terminalEvents.length > 0) { for (const event of terminalEvents) { issues.push({ code: "terminal-event-on-nonterminal-run", message: `Run row is ${run.status}, but ${event.type} exists`, eventId: event.id, }); } } if (terminalEvents.length > 1) { issues.push({ code: "duplicate-terminal-events", message: `Run has ${terminalEvents.length} terminal events`, }); } if (expectedType) { for (const event of terminalEvents.filter((candidate) => candidate.type !== expectedType)) { issues.push({ code: "terminal-status-mismatch", message: `Run row is ${run.status}, but terminal receipt is ${event.type}`, eventId: event.id, }); } } if (run.status === "completed" && run.outputEventIds.length === 0) { issues.push({ code: "completed-without-output", message: "Completed run row references no output event" }); } if (!rowIsTerminal && run.outputEventIds.length > 0) { issues.push({ code: "nonterminal-has-output", message: "Nonterminal run row already references accepted output" }); } for (const outputEventId of run.outputEventIds) { if (!eventsById.has(outputEventId)) { issues.push({ code: "missing-output-event", message: `Run row references missing output event ${outputEventId}`, }); } } for (const event of terminalEvents) { const outputEventId = event.payload.outputEventId; if (typeof outputEventId === "string" && !run.outputEventIds.includes(outputEventId)) { issues.push({ code: "terminal-output-mismatch", message: `Terminal receipt references ${outputEventId}, which is absent from the run row`, eventId: event.id, }); } if (typeof outputEventId === "string" && !eventsById.has(outputEventId)) { issues.push({ code: "missing-output-event", message: `Terminal receipt references missing output event ${outputEventId}`, eventId: event.id, }); } } return { runId: run.id, status: run.status, consistent: issues.length === 0, terminalEventIds: terminalEvents.map((event) => event.id), issues, }; }); }