Something went wrong. Try again.
fork of prime agent with changes i want but are not upstreamable probqbly (and some fixes)
Something went wrong. Try again.
34 kB · 955 lines
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956import type { AgentMessage } from "@earendil-works/pi-agent-core";import type { AssistantMessage, ImageContent } from "@earendil-works/pi-ai";import { afterEach, describe, expect, it, vi } from "vitest";import type { AgentAutonomousStatus } from "../src/core/autonomous.js";import { createCompactionOutcomeMessage, createCustomMessage, createRefinementOutcomeMessage, createSessionSlashCommandResultMessage,} from "../src/core/messages.js";import type { SessionShutdownEvent } from "../src/index.js";import { selectHeadlessTerminalResult } from "../src/modes/headless-completion.js";import { runPrintMode } from "../src/modes/print-mode.js";
const output = vi.hoisted(() => ({ write: vi.fn(), flush: vi.fn(async () => {}) }));vi.mock("../src/core/output-guard.js", () => ({ writeRawStdout: output.write, flushRawStdout: output.flush,}));vi.mock("../src/utils/shell.js", () => ({ killTrackedDetachedChildren: vi.fn(),}));
type EmitEvent = SessionShutdownEvent;
type FakeExtensionRunner = { hasHandlers: (eventType: string) => boolean; emit: ReturnType<typeof vi.fn<(event: EmitEvent) => Promise<void>>>;};
type FakeSession = { sessionManager: { getHeader: () => object | undefined }; agent: { waitForIdle: ReturnType<typeof vi.fn<() => Promise<void>>> }; waitForIdle: ReturnType<typeof vi.fn<() => Promise<void>>>; waitForHeadlessIdle: ReturnType<typeof vi.fn<() => Promise<void>>>; state: { messages: AgentMessage[] }; messages: AgentMessage[]; extensionRunner: FakeExtensionRunner; bindExtensions: ReturnType<typeof vi.fn>; subscribe: ReturnType<typeof vi.fn>; prompt: ReturnType<typeof vi.fn>; promptAndWait: ReturnType<typeof vi.fn>; reload: ReturnType<typeof vi.fn>; getAutonomousStatus: ReturnType<typeof vi.fn>; recordHostAutonomousContinuation: ReturnType<typeof vi.fn>; refreshAutonomousGates: ReturnType<typeof vi.fn>;};
type FakeRuntimeHost = { session: FakeSession; newSession: ReturnType<typeof vi.fn>; fork: ReturnType<typeof vi.fn>; switchSession: ReturnType<typeof vi.fn>; dispose: ReturnType<typeof vi.fn>; setRebindSession: ReturnType<typeof vi.fn>;};
function createAssistantMessage(options?: { text?: string; stopReason?: AssistantMessage["stopReason"]; errorMessage?: string;}): AssistantMessage { return { role: "assistant", content: options?.text ? [{ type: "text", text: options.text }] : [], api: "openai-responses", provider: "openai", model: "gpt-4o-mini", usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }, stopReason: options?.stopReason ?? "stop", errorMessage: options?.errorMessage, timestamp: Date.now(), };}
function createRuntimeHost( assistantMessage: AgentMessage | AgentMessage[], autonomousStatus: AgentAutonomousStatus = { enabled: false, continuationsUsed: 0, turnsUsed: 0, tokensUsed: 0, limits: { maxContinuations: 3, maxTurns: 12, maxTokens: 80_000, timeoutMs: 1_800_000 }, gates: { commands: [], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: {}, },): FakeRuntimeHost { const extensionRunner: FakeExtensionRunner = { hasHandlers: (eventType: string) => eventType === "session_shutdown", emit: vi.fn(async () => {}), };
const state = { messages: Array.isArray(assistantMessage) ? assistantMessage : [assistantMessage] }; const waitForIdle = vi.fn(async () => {});
const session: FakeSession = { sessionManager: { getHeader: () => undefined }, agent: { waitForIdle: vi.fn(async () => {}) }, waitForIdle, waitForHeadlessIdle: waitForIdle, state, messages: state.messages, extensionRunner, bindExtensions: vi.fn(async () => {}), subscribe: vi.fn(() => () => {}), prompt: vi.fn(async () => {}), promptAndWait: vi.fn(async () => {}), reload: vi.fn(async () => {}), getAutonomousStatus: vi.fn(() => autonomousStatus), recordHostAutonomousContinuation: vi.fn(), refreshAutonomousGates: vi.fn(), };
return { session, newSession: vi.fn(async () => undefined), fork: vi.fn(async () => ({ selectedText: "" })), switchSession: vi.fn(async () => undefined), dispose: vi.fn(async () => { await session.extensionRunner.emit({ type: "session_shutdown", reason: "quit" }); }), setRebindSession: vi.fn(), };}
afterEach(() => { vi.restoreAllMocks();});
describe("runPrintMode", () => { it("emits session_shutdown in text mode", async () => { const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "done" })); const { session } = runtimeHost; const images: ImageContent[] = [{ type: "image", mimeType: "image/png", data: "abc" }];
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", initialMessage: "Say done", initialImages: images, });
expect(exitCode).toBe(0); expect(session.promptAndWait).toHaveBeenCalledWith("Say done", { images }); expect(session.extensionRunner.emit).toHaveBeenCalledTimes(1); expect(session.extensionRunner.emit).toHaveBeenCalledWith({ type: "session_shutdown", reason: "quit" }); });
it("disposes the connection before exiting on SIGINT", async () => { const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "done" })); const { session } = runtimeHost; let resolvePrompt: (() => void) | undefined; session.promptAndWait.mockImplementation( () => new Promise<void>((resolve) => { resolvePrompt = resolve; }), ); const onSpy = vi.spyOn(process, "on"); const exitSpy = vi.spyOn(process, "exit").mockImplementation((() => undefined) as typeof process.exit);
const runPromise = runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", initialMessage: "Wait", }); await vi.waitFor(() => expect(session.promptAndWait).toHaveBeenCalled()); const handler = onSpy.mock.calls.find(([event]) => event === "SIGINT")?.[1]; if (typeof handler !== "function") throw new Error("SIGINT handler was not registered");
handler();
await vi.waitFor(() => expect(exitSpy).toHaveBeenCalledWith(130)); expect(runtimeHost.dispose).toHaveBeenCalledTimes(1); expect(session.extensionRunner.emit).toHaveBeenCalledWith({ type: "session_shutdown", reason: "quit" }); resolvePrompt?.(); await expect(runPromise).resolves.toBe(0); });
it("prints successful session command results in text mode", async () => { const result = createSessionSlashCommandResultMessage("No active goal.", { command: { name: "goal", args: "status", text: "/goal status" }, success: true, severity: "info", }); const runtimeHost = createRuntimeHost(result); output.write.mockClear();
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(0); expect(output.write).toHaveBeenCalledWith("No active goal.\n"); });
it("prints a session command result before a trailing compaction outcome", async () => { const result = createSessionSlashCommandResultMessage("No active goal.", { command: { name: "goal", args: "status", text: "/goal status" }, success: true, severity: "info", }); const outcome = createCompactionOutcomeMessage("Requested compaction skipped", { reason: "requested", outcome: "skipped", }); const runtimeHost = createRuntimeHost([result, outcome]); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); output.write.mockClear();
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(0); expect(output.write).toHaveBeenCalledWith("No active goal.\n"); expect(errorSpy).toHaveBeenCalledWith("Requested compaction skipped"); });
it("skips malformed terminal outcomes without hiding earlier valid failures", () => { const failed = createCompactionOutcomeMessage("Compaction failed", { reason: "requested", outcome: "failed", }); const malformed = { ...failed, details: { reason: "unknown", outcome: "failed" } } as AgentMessage; const assistant = createAssistantMessage({ text: "done" });
expect(selectHeadlessTerminalResult([assistant, failed, malformed])).toEqual({ primary: assistant, compactionOutcomes: [failed], }); });
it("does not select a result across a message barrier", () => { const outcome = createCompactionOutcomeMessage("Requested compaction skipped", { reason: "requested", outcome: "skipped", }); const barriers: AgentMessage[] = [ { role: "user", content: "next request", timestamp: Date.now() }, createCustomMessage("extension.notice", "unrelated", true, undefined, new Date().toISOString()), ];
for (const barrier of barriers) { expect(selectHeadlessTerminalResult([createAssistantMessage({ text: "stale" }), barrier, outcome])).toEqual({ primary: undefined, compactionOutcomes: [outcome], }); } });
it("returns non-zero for failed session command results in text mode", async () => { const result = createSessionSlashCommandResultMessage("Command failed: bad arguments", { command: { name: "refine", args: "rollback", text: "/refine rollback" }, success: false, severity: "error", error: "bad arguments", }); const runtimeHost = createRuntimeHost(result); output.write.mockClear();
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(1); expect(output.write).toHaveBeenCalledWith("Command failed: bad arguments\n"); });
it("emits session_shutdown in json mode", async () => { const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "done" })); const { session } = runtimeHost;
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "json", messages: ["hello"], });
expect(exitCode).toBe(0); expect(session.promptAndWait).toHaveBeenCalledWith("hello", {}); expect(session.extensionRunner.emit).toHaveBeenCalledTimes(1); expect(session.extensionRunner.emit).toHaveBeenCalledWith({ type: "session_shutdown", reason: "quit" }); });
it("emits session_shutdown and returns non-zero on assistant error", async () => { const runtimeHost = createRuntimeHost( createAssistantMessage({ stopReason: "error", errorMessage: "provider failure" }), ); const { session } = runtimeHost; const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(1); expect(errorSpy).toHaveBeenCalledWith("provider failure"); expect(session.extensionRunner.emit).toHaveBeenCalledTimes(1); expect(session.extensionRunner.emit).toHaveBeenCalledWith({ type: "session_shutdown", reason: "quit" }); });
it("prints assistant output and reports a trailing compaction outcome", async () => { const outcome = createCompactionOutcomeMessage("Auto-compaction skipped: nothing to compact", { reason: "threshold", outcome: "skipped", }); const runtimeHost = createRuntimeHost([createAssistantMessage({ text: "done" }), outcome]); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); output.write.mockClear();
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(0); expect(output.write).toHaveBeenCalledWith("done\n"); expect(errorSpy).toHaveBeenCalledWith("Auto-compaction skipped: nothing to compact"); });
it("prints assistant output past a trailing refinement outcome", async () => { const outcome = createRefinementOutcomeMessage({ id: "refine-1", summary: "Added a local memory.", rationale: "", expectedOutcome: "", appliedEdits: [], harnessStatePath: "/tmp/harness/state.json", scope: "local", }); const runtimeHost = createRuntimeHost([createAssistantMessage({ text: "done" }), outcome]); output.write.mockClear();
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(0); expect(output.write).toHaveBeenCalledWith("done\n"); });
it("reports an outcome-only failure and exits non-zero", async () => { const outcome = createCompactionOutcomeMessage("Context overflow recovery failed", { reason: "overflow", outcome: "failed", }); const runtimeHost = createRuntimeHost(outcome); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); output.write.mockClear();
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(1); expect(output.write).not.toHaveBeenCalled(); expect(errorSpy).toHaveBeenCalledWith("Context overflow recovery failed"); });
it("stops host-driven gate retries once gate maxRetries is exhausted", async () => { const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), { enabled: true, continuationsUsed: 1, turnsUsed: 2, tokensUsed: 100, startedAt: Date.now(), limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 4 }, lastGateFailure: { command: "verify-public", attempt: 4, exitText: "not rerun: workspace unchanged since previous failed gate", output: "edit source files before attempting to finish again", }, }); const { session } = runtimeHost; const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(1); expect(session.prompt).not.toHaveBeenCalled(); expect(session.recordHostAutonomousContinuation).not.toHaveBeenCalled(); expect(errorSpy).toHaveBeenCalledWith( "Autonomous quality gate still failing after attempt 4/3: not rerun: workspace unchanged since previous failed gate", ); });
it("refreshes autonomous gates after host-driven gate retries", async () => { const failingStatus: AgentAutonomousStatus = { enabled: true, continuationsUsed: 0, turnsUsed: 1, tokensUsed: 100, startedAt: Date.now(), limits: { maxContinuations: 3, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 1 }, lastGateFailure: { command: "verify-public", attempt: 1, exitText: "exited 1", output: "0/9", }, }; const passingStatus: AgentAutonomousStatus = { ...failingStatus, continuationsUsed: 1, turnsUsed: 2, tokensUsed: 200, gateAttempts: { "verify-public": 0 }, lastGateFailure: undefined, }; const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "fixed the gate" }), failingStatus); const { session } = runtimeHost; let currentStatus = failingStatus; session.getAutonomousStatus.mockImplementation(() => currentStatus); session.refreshAutonomousGates.mockImplementation(() => { currentStatus = passingStatus; });
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(0); expect(session.prompt).toHaveBeenCalledTimes(1); expect(session.recordHostAutonomousContinuation).toHaveBeenCalledTimes(1); expect(session.refreshAutonomousGates).toHaveBeenCalledTimes(1); });
it("keeps autonomous gate prompting after a transient assistant error while limits remain", async () => { const statuses: AgentAutonomousStatus[] = [ { enabled: true, continuationsUsed: 1, turnsUsed: 2, tokensUsed: 100, startedAt: Date.now(), limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 1 }, lastGateFailure: { command: "verify-public", attempt: 1, exitText: "exited 1", output: "0/9", }, }, { enabled: true, continuationsUsed: 1, turnsUsed: 3, tokensUsed: 200, startedAt: Date.now(), limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 2 }, lastGateFailure: { command: "verify-public", attempt: 2, exitText: "exited 1", output: "0/9", }, }, { enabled: true, continuationsUsed: 1, turnsUsed: 4, tokensUsed: 300, startedAt: Date.now(), limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 2 }, lastGateFailure: { command: "verify-public", attempt: 2, exitText: "exited 1", output: "0/9", }, }, { enabled: true, continuationsUsed: 1, turnsUsed: 5, tokensUsed: 400, startedAt: Date.now(), limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 2 }, }, ]; const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), statuses[0]); const { session } = runtimeHost; let statusIndex = 0; session.getAutonomousStatus.mockImplementation( () => statuses[Math.min(statusIndex++, statuses.length - 1)] as AgentAutonomousStatus, ); session.prompt.mockImplementationOnce(async () => { session.state.messages = [ createAssistantMessage({ stopReason: "error", errorMessage: "provider down" }), createCompactionOutcomeMessage("Auto-compaction failed", { reason: "threshold", outcome: "failed" }), ]; }); session.prompt.mockImplementationOnce(async () => { session.state.messages = [createAssistantMessage({ text: "still failing" })]; }); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(0); expect(session.prompt).toHaveBeenCalledTimes(2); expect(session.recordHostAutonomousContinuation).toHaveBeenCalledTimes(2); expect(errorSpy).not.toHaveBeenCalledWith("provider down"); expect(session.prompt.mock.calls[0][0]).toContain("Autonomous quality gate failed"); expect(session.prompt.mock.calls[1][0]).toContain("Autonomous quality gate failed"); });
it("waits for a queued follow-up turn before evaluating transient assistant errors", async () => { const statuses: AgentAutonomousStatus[] = [ { enabled: true, continuationsUsed: 0, turnsUsed: 1, tokensUsed: 100, startedAt: Date.now(), limits: { maxContinuations: 1, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 1 }, lastGateFailure: { command: "verify-public", attempt: 1, exitText: "exited 1", output: "0/9", }, }, { enabled: true, continuationsUsed: 1, turnsUsed: 2, tokensUsed: 200, startedAt: Date.now(), limits: { maxContinuations: 1, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 1 }, }, ]; const runtimeHost = createRuntimeHost( createAssistantMessage({ stopReason: "error", errorMessage: "provider down" }), statuses[0], ); const { session } = runtimeHost; let statusIndex = 0; session.getAutonomousStatus.mockImplementation( () => statuses[Math.min(statusIndex++, statuses.length - 1)] as AgentAutonomousStatus, ); let waitCount = 0; session.waitForIdle.mockImplementation(async () => { waitCount++; if (waitCount === 2) { session.state.messages = [createAssistantMessage({ text: "queued retry completed" })]; } }); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(0); expect(session.prompt).toHaveBeenCalledTimes(1); expect(session.recordHostAutonomousContinuation).toHaveBeenCalledTimes(1); expect(session.waitForIdle).toHaveBeenCalledTimes(3); expect(errorSpy).not.toHaveBeenCalledWith("provider down"); });
it("does not issue host-driven gate prompts once maxContinuations is reached", async () => { const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), { enabled: true, continuationsUsed: 3, turnsUsed: 2, tokensUsed: 100, startedAt: Date.now(), limits: { maxContinuations: 3, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 1 }, lastGateFailure: { command: "verify-public", attempt: 1, exitText: "exited 1", output: "0/9", }, }); const { session } = runtimeHost; const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(1); expect(session.prompt).not.toHaveBeenCalled(); expect(session.recordHostAutonomousContinuation).not.toHaveBeenCalled(); expect(errorSpy).toHaveBeenCalledWith( "Autonomous quality gate still failing after attempt 1/3: exited 1; autonomous limit reached: maxContinuations reached (3/3)", ); });
it("returns non-zero when autonomous gates are still failing", async () => { const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), { enabled: true, continuationsUsed: 999, turnsUsed: 92, tokensUsed: 215_535, limits: { maxContinuations: 999, maxTurns: 1000, maxTokens: 2_000_000, timeoutMs: 1_800_000 }, gates: { commands: ["verify-public"], maxRetries: 999, timeoutMs: 3_600_000 }, gateAttempts: { "verify-public": 34 }, lastGateFailure: { command: "verify-public", attempt: 34, exitText: "exited 1", output: "0/43", }, }); const { session } = runtimeHost; const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(1); expect(errorSpy).toHaveBeenCalledWith( "Autonomous quality gate still failing after attempt 34/999: exited 1; autonomous limit reached: maxContinuations reached (999/999)", ); expect(session.extensionRunner.emit).toHaveBeenCalledWith({ type: "session_shutdown", reason: "quit" }); });
it("reports the exact autonomous limit that stopped a still-failing gate", async () => { const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), { enabled: true, continuationsUsed: 34, turnsUsed: 92, tokensUsed: 2_000_000, limits: { maxContinuations: 999, maxTurns: 1000, maxTokens: 2_000_000, timeoutMs: 1_800_000 }, gates: { commands: ["verify-public"], maxRetries: 999, timeoutMs: 3_600_000 }, gateAttempts: { "verify-public": 34 }, lastGateFailure: { command: "verify-public", attempt: 34, exitText: "exited 1", output: "0/43", }, }); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(1); expect(errorSpy).toHaveBeenCalledWith( "Autonomous quality gate still failing after attempt 34/999: exited 1; autonomous limit reached: maxTokens reached (2000000/2000000)", ); });
it("returns non-zero when ungated autonomous runs stop at a limit", async () => { const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "I still need more work." }), { enabled: true, continuationsUsed: 3, turnsUsed: 4, tokensUsed: 100, startedAt: Date.now(), limits: { maxContinuations: 3, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: [], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: {}, }); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(1); expect(errorSpy).toHaveBeenCalledWith( "Autonomous run stopped before terminal evidence; maxContinuations reached (3/3)", ); });
it("keeps prompting while autonomous gates fail below retry limits", async () => { const statuses: AgentAutonomousStatus[] = [ { enabled: true, continuationsUsed: 1, turnsUsed: 2, tokensUsed: 100, startedAt: Date.now(), limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 1 }, lastGateFailure: { command: "verify-public", attempt: 1, exitText: "exited 1", output: "0/9", }, }, { enabled: true, continuationsUsed: 2, turnsUsed: 3, tokensUsed: 200, startedAt: Date.now(), limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 2 }, lastGateFailure: { command: "verify-public", attempt: 2, exitText: "exited 1", output: "0/9 summary", }, }, { enabled: true, continuationsUsed: 2, turnsUsed: 4, tokensUsed: 250, startedAt: Date.now(), limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 2 }, }, ]; const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still working" }), statuses[0]); const { session } = runtimeHost; let statusIndex = 0; session.getAutonomousStatus.mockImplementation( () => statuses[Math.min(statusIndex++, statuses.length - 1)] as AgentAutonomousStatus, );
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(0); expect(session.waitForIdle).toHaveBeenCalledBefore(session.prompt); expect(session.prompt).toHaveBeenCalledTimes(2); expect(session.prompt.mock.calls[0][0]).toContain("Autonomous quality gate failed (attempt 1/3)"); expect(session.prompt.mock.calls[0][0]).toContain("0/9"); expect(session.prompt.mock.calls[0][1]).toEqual({ streamingBehavior: "followUp", internalPrompt: true, suppressAutonomousContinuation: true, }); expect(session.prompt.mock.calls[1][0]).toContain("Autonomous quality gate failed (attempt 2/3)"); expect(session.prompt.mock.calls[1][0]).toContain("0/9 summary"); expect(session.prompt.mock.calls[1][1]).toEqual({ streamingBehavior: "followUp", internalPrompt: true, suppressAutonomousContinuation: true, }); });
it("continues prompting when gate attempts do not advance but autonomous usage does", async () => { const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), { enabled: true, continuationsUsed: 1, turnsUsed: 2, tokensUsed: 100, startedAt: Date.now(), limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 1 }, lastGateFailure: { command: "verify-public", attempt: 1, exitText: "exited 1", output: "0/9", }, }); const { session } = runtimeHost; const statuses: AgentAutonomousStatus[] = [ { enabled: true, continuationsUsed: 1, turnsUsed: 2, tokensUsed: 100, startedAt: Date.now(), limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 1 }, lastGateFailure: { command: "verify-public", attempt: 1, exitText: "exited 1", output: "0/9" }, }, { enabled: true, continuationsUsed: 2, turnsUsed: 3, tokensUsed: 150, startedAt: Date.now(), limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 1 }, lastGateFailure: { command: "verify-public", attempt: 1, exitText: "not rerun: workspace unchanged since previous failed gate", output: "edit source files before attempting to finish again", }, }, { enabled: true, continuationsUsed: 3, turnsUsed: 4, tokensUsed: 150, startedAt: Date.now(), limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 1 }, lastGateFailure: { command: "verify-public", attempt: 1, exitText: "not rerun: workspace unchanged since previous failed gate", output: "edit source files before attempting to finish again", }, }, { enabled: true, continuationsUsed: 10, turnsUsed: 5, tokensUsed: 150, startedAt: Date.now(), limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 1 }, lastGateFailure: { command: "verify-public", attempt: 1, exitText: "not rerun: workspace unchanged since previous failed gate", output: "edit source files before attempting to finish again", }, }, ]; let statusIndex = 0; session.getAutonomousStatus.mockImplementation( () => statuses[Math.min(statusIndex++, statuses.length - 1)] as AgentAutonomousStatus, );
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(1); expect(session.waitForIdle).toHaveBeenCalledTimes(7); expect(session.prompt).toHaveBeenCalledTimes(3); expect(session.recordHostAutonomousContinuation).toHaveBeenCalledTimes(3); expect(session.prompt.mock.calls[1][0]).toContain("workspace unchanged"); expect(session.prompt.mock.calls[2][0]).toContain("workspace unchanged"); });
it("keeps prompting on repeated gate progress until autonomous limits stop the run", async () => { const startedAt = Date.now(); const statuses: AgentAutonomousStatus[] = [ { enabled: true, continuationsUsed: 7, turnsUsed: 8, tokensUsed: 100, startedAt, limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 20, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 7 }, lastGateFailure: { command: "verify-public", attempt: 7, exitText: "not rerun: workspace unchanged since previous failed gate", output: "edit source files before attempting to finish again", }, }, { enabled: true, continuationsUsed: 8, turnsUsed: 9, tokensUsed: 100, startedAt, limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 20, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 7 }, lastGateFailure: { command: "verify-public", attempt: 7, exitText: "not rerun: workspace unchanged since previous failed gate", output: "edit source files before attempting to finish again", }, }, { enabled: true, continuationsUsed: 9, turnsUsed: 10, tokensUsed: 100, startedAt, limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 20, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 7 }, lastGateFailure: { command: "verify-public", attempt: 7, exitText: "not rerun: workspace unchanged since previous failed gate", output: "edit source files before attempting to finish again", }, }, { enabled: true, continuationsUsed: 10, turnsUsed: 11, tokensUsed: 100, startedAt, limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, gates: { commands: ["verify-public"], maxRetries: 20, timeoutMs: 300_000 }, gateAttempts: { "verify-public": 7 }, lastGateFailure: { command: "verify-public", attempt: 7, exitText: "not rerun: workspace unchanged since previous failed gate", output: "edit source files before attempting to finish again", }, }, ]; const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), statuses[0]); const { session } = runtimeHost; let statusIndex = 0; session.getAutonomousStatus.mockImplementation( () => statuses[Math.min(statusIndex++, statuses.length - 1)] as AgentAutonomousStatus, );
const exitCode = await runPrintMode(runtimeHost as unknown as Parameters<typeof runPrintMode>[0], { mode: "text", });
expect(exitCode).toBe(1); expect(session.prompt).toHaveBeenCalledTimes(3); expect(session.prompt.mock.calls[0][0]).toContain("workspace unchanged"); expect(session.prompt.mock.calls[2][0]).toContain("workspace unchanged"); });});