diff --git a/packages/coding-agent/.changes/prompt-cache-miss-minimization.md b/packages/coding-agent/.changes/prompt-cache-miss-minimization.md new file mode 100644 index 000000000..9a2a4b876 --- /dev/null +++ b/packages/coding-agent/.changes/prompt-cache-miss-minimization.md @@ -0,0 +1,2 @@ +- Added an `auxiliaryModel` setting (`"provider/id"`) that routes refinement LLM passes (auto-refine review and refinement planning) to a different model. These passes use their own prompt prefixes, so running them on the session model evicts the provider's prompt-cache entry for the session and forces a full context re-read on the next session request; the setting isolates those passes while falling back to the session model when unset or unusable. +- Prime Inference `anthropic/*` models now send anthropic-style `cache_control` markers (system prompt, last tool, last conversation message) so prompt caching engages on gateways that pass them through to the upstream Anthropic API, matching the Anthropic cache pricing the catalog already applies to those entries. diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 7a854eb06..e11e32560 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -8785,23 +8785,59 @@ export class AgentSession { } } + /** + * Refinement passes (review and planning) run with their own prompts, so + * issuing them on the session model evicts the provider's prefix-cache entry + * for the session and forces a full context re-read on the next session + * request. Route them to the configured auxiliary model when it is set and + * usable; fall back to the session model otherwise. + */ + private async _resolveRefinementModel(): Promise< + { model: Model; apiKey: string; headers?: Record } | undefined + > { + const sessionModel = this.model; + if (!sessionModel) { + return undefined; + } + const selector = this.settingsManager.getAuxiliaryModel()?.trim().toLowerCase(); + if (!selector || `${sessionModel.provider}/${sessionModel.id}`.toLowerCase() === selector) { + const { apiKey, headers, requestModel } = await this._getRequiredRequestAuth(sessionModel); + return { model: requestModel, apiKey, headers }; + } + try { + const model = (await this._authenticatedRlmModels()).find( + (candidate) => `${candidate.provider}/${candidate.id}`.toLowerCase() === selector, + ); + if (!model) { + throw new Error(`model "${selector}" is unavailable, unauthenticated, or expired`); + } + const { apiKey, headers, requestModel } = await this._getRequiredRequestAuth(model); + return { model: requestModel, apiKey, headers }; + } catch { + // Error details from the auth stack can embed credential material, so only + // the selector is logged (CodeQL js/clear-text-logging). + console.warn(`Warning: auxiliaryModel "${selector}" unusable for refinement; using the session model.`); + const { apiKey, headers, requestModel } = await this._getRequiredRequestAuth(sessionModel); + return { model: requestModel, apiKey, headers }; + } + } + private async _reviewAutoRefine(context: AutoRefineReviewRequest, signal?: AbortSignal): Promise { if (this._autoRefineReviewer) { return this._autoRefineReviewer(context, signal); } - const model = this.model; - if (!model) { + const refinementModel = await this._resolveRefinementModel(); + if (!refinementModel) { return { shouldRefine: false, rationale: "No model selected." }; } - const { apiKey, headers, requestModel } = await this._getRequiredRequestAuth(model); return reviewAutoRefine( this.agent.state.messages, this._loadMergedHarnessState(), this._loadRefinementHistory(), - requestModel, - apiKey, + refinementModel.model, + refinementModel.apiKey, context, - headers, + refinementModel.headers, signal, this.thinkingLevel, providerRetryPolicy(this.settingsManager), @@ -9040,7 +9076,10 @@ export class AgentSession { throw new Error(formatNoModelSelectedMessage()); } - const { apiKey, headers, requestModel: model } = await this._getRequiredRequestAuth(this.model); + const refinementModel = await this._resolveRefinementModel(); + if (!refinementModel) { + throw new Error(formatNoModelSelectedMessage()); + } const globalHarnessStateDir = getGlobalHarnessStateDir(); const localHarnessStateDir = this._localHarnessStateDir(); const requestedScope = options.global ? "global" : "local"; @@ -9102,10 +9141,10 @@ export class AgentSession { this.agent.state.messages, planningState, history, - model, - apiKey, + refinementModel.model, + refinementModel.apiKey, { ...options, retry: providerRetryPolicy(this.settingsManager) }, - headers, + refinementModel.headers, signal, this.thinkingLevel, this.sessionId, diff --git a/packages/coding-agent/src/core/prime-inference-model-catalog.ts b/packages/coding-agent/src/core/prime-inference-model-catalog.ts index 8c9c2f672..2b3e2861a 100644 --- a/packages/coding-agent/src/core/prime-inference-model-catalog.ts +++ b/packages/coding-agent/src/core/prime-inference-model-catalog.ts @@ -48,6 +48,15 @@ export function buildPrimeInferenceModels( if (!template && (!entry.contextWindow || !entry.maxTokens || entry.reasoning === undefined)) continue; const contextWindow = entry.contextWindow ?? template?.contextWindow ?? 0; const maxTokens = Math.min(entry.maxTokens ?? template?.maxTokens ?? 0, contextWindow); + const compat = structuredClone(template?.compat ?? DEFAULT_COMPAT); + // Anthropic models cache with explicit breakpoints, not automatic + // server-side prefix caching; cacheControlFormat makes the provider add + // anthropic-style cache_control markers for these entries. The catalog + // already prices anthropic/* with Anthropic cache economics (10% cache + // reads, 125% cache writes), so the wire format follows the pricing. + if (entry.id.toLowerCase().startsWith("anthropic/")) { + compat.cacheControlFormat = "anthropic"; + } models.push({ id: entry.id, name: entry.name ?? template?.name ?? entry.id, @@ -61,7 +70,7 @@ export function buildPrimeInferenceModels( contextWindow, maxTokens, ...(template?.featured ? { featured: true } : {}), - compat: structuredClone(template?.compat ?? DEFAULT_COMPAT), + compat, }); } const minimumModels = options.minimumModels ?? Math.ceil(bundledModels.length * MIN_CATALOG_COVERAGE); diff --git a/packages/coding-agent/src/core/settings-manager.ts b/packages/coding-agent/src/core/settings-manager.ts index f5e08a14c..35e623754 100644 --- a/packages/coding-agent/src/core/settings-manager.ts +++ b/packages/coding-agent/src/core/settings-manager.ts @@ -145,6 +145,11 @@ export interface Settings { defaultProvider?: string; defaultModel?: string; recentModels?: string[]; // "provider/id" keys, most-recently-used first + // "provider/id" for background LLM passes (refinement review and planning); + // unset falls back to the session model. Routing these to a different model + // keeps their different prompt prefixes from evicting the session's provider + // prefix-cache entry. + auxiliaryModel?: string; defaultThinkingLevel?: "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; defaultServiceTier?: ServiceTier; rlmMaxDepth?: number; // default for new sessions; unset falls through to RLM_MAX_DEPTH, then 2 @@ -746,6 +751,13 @@ export class SettingsManager { this.save(); } + getAuxiliaryModel(): string | undefined { + // Hand-edited or corrupt settings files can persist non-string values; treat + // anything malformed as unset so refinement falls back to the session model. + const value = this.settings.auxiliaryModel; + return typeof value === "string" ? value : undefined; + } + getRecentModels(): string[] { return this.settings.recentModels ?? []; } diff --git a/packages/coding-agent/test/prime-inference-model-catalog.test.ts b/packages/coding-agent/test/prime-inference-model-catalog.test.ts index e5a033a00..f461a247d 100644 --- a/packages/coding-agent/test/prime-inference-model-catalog.test.ts +++ b/packages/coding-agent/test/prime-inference-model-catalog.test.ts @@ -87,6 +87,18 @@ describe("Prime Inference model catalog", () => { expect(live).not.toHaveProperty("headers"); }); + test("marks anthropic models for anthropic-style cache control and leaves others untouched", () => { + const models = + buildPrimeInferenceModels( + [model("anthropic/claude-opus-4.8"), model("vendor/model")], + [entry("anthropic/claude-opus-4.8"), entry("vendor/model")], + ) ?? []; + const anthropicModel = models.find((candidate) => candidate.id === "anthropic/claude-opus-4.8"); + const vendorModel = models.find((candidate) => candidate.id === "vendor/model"); + expect(anthropicModel?.compat?.cacheControlFormat).toBe("anthropic"); + expect(vendorModel?.compat?.cacheControlFormat).toBeUndefined(); + }); + test("adds complete new models and skips incomplete unknown models", () => { const models = buildPrimeInferenceModels( diff --git a/packages/coding-agent/test/settings-manager.test.ts b/packages/coding-agent/test/settings-manager.test.ts index ffef82212..9079f851d 100644 --- a/packages/coding-agent/test/settings-manager.test.ts +++ b/packages/coding-agent/test/settings-manager.test.ts @@ -276,6 +276,22 @@ describe("SettingsManager", () => { }); }); + describe("auxiliaryModel", () => { + it("returns a valid persisted selector", () => { + writeFileSync(join(agentDir, "settings.json"), JSON.stringify({ auxiliaryModel: "faux/aux-model" })); + const manager = SettingsManager.create(projectDir, agentDir); + + expect(manager.getAuxiliaryModel()).toBe("faux/aux-model"); + }); + + it("treats malformed persisted values as unset", () => { + writeFileSync(join(agentDir, "settings.json"), JSON.stringify({ auxiliaryModel: 42 })); + const manager = SettingsManager.create(projectDir, agentDir); + + expect(manager.getAuxiliaryModel()).toBeUndefined(); + }); + }); + describe("recentModels", () => { it("records most-recently-used first, dedupes, and persists", async () => { const manager = SettingsManager.create(projectDir, agentDir); diff --git a/packages/coding-agent/test/suite/agent-session-refine-auxiliary-model.test.ts b/packages/coding-agent/test/suite/agent-session-refine-auxiliary-model.test.ts new file mode 100644 index 000000000..13aafab92 --- /dev/null +++ b/packages/coding-agent/test/suite/agent-session-refine-auxiliary-model.test.ts @@ -0,0 +1,109 @@ +import type * as PiAi from "@earendil-works/pi-ai"; +import type { AssistantMessage } from "@earendil-works/pi-ai"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const { completeSimpleMock } = vi.hoisted(() => ({ + completeSimpleMock: vi.fn(), +})); + +vi.mock("@earendil-works/pi-ai", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + completeSimple: completeSimpleMock, + }; +}); + +import { createHarness, type Harness } from "./harness.js"; + +function assistantText(text: string): AssistantMessage { + return { + role: "assistant", + content: [{ type: "text", text }], + api: "faux", + provider: "faux", + model: "faux", + usage: { + input: 1, + output: 1, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 2, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }; +} + +function emptyProposalResponse(): AssistantMessage { + return assistantText( + JSON.stringify({ summary: "no-op", rationale: "nothing to record", expectedOutcome: "none", edits: [] }), + ); +} + +describe("AgentSession refinement auxiliary model", () => { + const harnesses: Harness[] = []; + + beforeEach(() => { + completeSimpleMock.mockReset(); + completeSimpleMock.mockResolvedValue(emptyProposalResponse()); + }); + + afterEach(() => { + for (const harness of harnesses.splice(0)) harness.cleanup(); + }); + + async function createRefineHarness(settings?: { auxiliaryModel?: string }): Promise { + const harness = await createHarness({ + models: [ + { id: "session-model", name: "Session Model" }, + { id: "aux-model", name: "Aux Model" }, + ], + settings, + persistSession: true, + }); + harnesses.push(harness); + return harness; + } + + it("routes refinement planning to the configured auxiliary model", async () => { + const harness = await createRefineHarness({ auxiliaryModel: "faux/aux-model" }); + await harness.session.refine({ instructions: "record a note" }); + expect(completeSimpleMock).toHaveBeenCalledTimes(1); + expect(completeSimpleMock.mock.calls[0][0]).toMatchObject({ + provider: "faux", + id: "aux-model", + }); + }); + + it("falls back to the session model when no auxiliary model is configured", async () => { + const harness = await createRefineHarness(); + await harness.session.refine({ instructions: "record a note" }); + expect(completeSimpleMock).toHaveBeenCalledTimes(1); + expect(completeSimpleMock.mock.calls[0][0]).toMatchObject({ + provider: "faux", + id: "session-model", + }); + }); + + it("falls back to the session model when the auxiliary model is unusable", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + const harness = await createRefineHarness({ auxiliaryModel: "faux/missing-model" }); + await harness.session.refine({ instructions: "record a note" }); + expect(completeSimpleMock).toHaveBeenCalledTimes(1); + expect(completeSimpleMock.mock.calls[0][0]).toMatchObject({ + provider: "faux", + id: "session-model", + }); + expect(warnSpy).toHaveBeenCalledTimes(1); + const [message] = warnSpy.mock.calls[0]; + expect(message).toContain('auxiliaryModel "faux/missing-model" unusable for refinement'); + // Caught error details can embed credential material, so they must not be logged. + expect(message).not.toContain("unavailable, unauthenticated, or expired"); + } finally { + warnSpy.mockRestore(); + } + }); +});