Something went wrong. Try again.
opencode v2 mirror
Something went wrong. Try again.
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166// Apply an `LLMRequest.cache` policy by injecting `CacheHint`s onto the parts// the policy designates. Runs once at compile time, before the per-protocol// body builder, so the existing inline-hint lowering path handles the rest.//// The default `"auto"` shape places breakpoints at the last tool definition,// the first and last distinct system parts, and the conversation tail. This// exposes reusable tool, base-agent, project, and session prefixes while// advancing the tail after each tool result keeps recent conversation prefixes// reusable during long agent runs.//// Manual `cache: CacheHint` placements on individual parts are preserved and// count against the four-breakpoint budget; auto only fills remaining slots.import { CacheHint, type CachePolicy, type CachePolicyObject } from "./schema/options.js"import { LLMRequest, Message, ToolDefinition, type ContentPart, type ToolEntry } from "./schema/messages.js"import { effortUpdate } from "./effort-updates.js"
const AUTO: CachePolicyObject = { tools: true, system: true, messages: { tail: 1 },}
const NONE: CachePolicyObject = {}const BREAKPOINT_CAP = 4
// Resolution rules:// - undefined → "auto" — caching is on by default.// - "auto" → tools + first/last system + final message boundary.// - "none" → no auto placement; manual `CacheHint`s still flow.// - object form → exactly what the caller asked for.const resolve = (policy: CachePolicy | undefined): CachePolicyObject => { if (policy === undefined || policy === "auto") return AUTO if (policy === "none") return NONE return policy}
// Protocols whose wire format ignores inline cache markers (OpenAI's implicit// prefix caching, Gemini's implicit + out-of-band CachedContent). Skip the// whole policy pass for these — emitting hints would be harmless but pointless.const RESPECTS_INLINE_HINTS = new Set([ "anthropic-messages", "google-vertex-messages", "bedrock-converse", "openrouter",])
const makeHint = (ttlSeconds: number | undefined): CacheHint => ttlSeconds !== undefined ? new CacheHint({ type: "ephemeral", ttlSeconds }) : new CacheHint({ type: "ephemeral" })
interface Budget { remaining: number}
const markLastTool = (tools: ReadonlyArray<ToolEntry>, hint: CacheHint, budget: Budget): ReadonlyArray<ToolEntry> => { const target = tools.at(-1) if (target === undefined) return tools if (target.type === "namespace") { const nested = markLastTool(target.tools, hint, budget) return nested === target.tools ? tools : [...tools.slice(0, -1), { ...target, tools: nested }] } if (target.cache || budget.remaining === 0) return tools budget.remaining -= 1 return [...tools.slice(0, -1), new ToolDefinition({ ...target, cache: hint })]}
const countToolHints = (tools: ReadonlyArray<ToolEntry>): number => tools.reduce( (count, tool) => count + (tool.type === "tool" ? (tool.cache === undefined ? 0 : 1) : countToolHints(tool.tools)), 0, )
const markSystemBoundaries = (system: LLMRequest["system"], hint: CacheHint, budget: Budget): LLMRequest["system"] => { if (system.length === 0) return system let changed = false const next = system.map((part, index) => { if ((index !== 0 && index !== system.length - 1) || part.cache || budget.remaining === 0) return part budget.remaining -= 1 changed = true return { ...part, cache: hint } }) return changed ? next : system}
const lastIndexOfRole = (messages: ReadonlyArray<Message>, role: Message["role"]): number => messages.findLastIndex((m) => m.role === role)
// Mark the last text part of `messages[index]`. If no text part exists, mark// the last content part regardless of type — that's the breakpoint position// in tool-result-only messages too.const markMessageAt = ( messages: ReadonlyArray<Message>, index: number, hint: CacheHint, budget: Budget,): ReadonlyArray<Message> => { if (index < 0 || index >= messages.length) return messages const target = messages[index]! if (target.content.length === 0) return messages const lastTextIndex = target.content.findLastIndex((part) => part.type === "text") const markAt = lastTextIndex >= 0 ? lastTextIndex : target.content.length - 1 const existing = target.content[markAt]! if (("cache" in existing && existing.cache) || budget.remaining === 0) return messages budget.remaining -= 1 const nextContent = target.content.map((part, i) => (i === markAt ? ({ ...part, cache: hint } as ContentPart) : part)) const next = new Message({ ...target, content: nextContent }) // Single pass over `messages`, substituting the one updated entry. Long // conversations call this on every request, so avoid `.map()` here — its // closure dispatch and identity copies show up in profiling. const result = messages.slice() result[index] = next return result}
const markMessages = ( messages: ReadonlyArray<Message>, strategy: NonNullable<CachePolicyObject["messages"]>, hint: CacheHint, budget: Budget,): ReadonlyArray<Message> => { if (messages.length === 0) return messages if (strategy === "latest-user-message") return markMessageAt(messages, lastIndexOfRole(messages, "user"), hint, budget) if (strategy === "latest-assistant") return markMessageAt(messages, lastIndexOfRole(messages, "assistant"), hint, budget) let start = messages.length let remaining = strategy.tail while (remaining > 0 && start > 0) { start -= 1 if (effortUpdate(messages[start]!) === undefined) remaining -= 1 } let next = messages for (let i = start; i < messages.length; i++) if (effortUpdate(messages[i]!) === undefined) next = markMessageAt(next, i, hint, budget) return next}
const countHints = (request: LLMRequest) => countToolHints(request.tools) + request.system.reduce((count, part) => count + (part.cache === undefined ? 0 : 1), 0) + request.messages.reduce( (count, message) => count + message.content.reduce( (contentCount, part) => contentCount + ("cache" in part && part.cache !== undefined ? 1 : 0), 0, ), 0, )
export const applyCachePolicy = (request: LLMRequest): LLMRequest => { if (!RESPECTS_INLINE_HINTS.has(request.model.route.id)) return request if (request.model.route.id === "openrouter" && (request.cache === undefined || request.cache === "auto")) return request const policy = resolve(request.cache) if (!policy.tools && !policy.system && !policy.messages) return request
const hint = makeHint(policy.ttlSeconds) const budget = { remaining: Math.max(0, BREAKPOINT_CAP - countHints(request)) } const tools = policy.tools ? markLastTool(request.tools, hint, budget) : request.tools const system = policy.system ? markSystemBoundaries(request.system, hint, budget) : request.system const messages = policy.messages ? markMessages(request.messages, policy.messages, hint, budget) : request.messages
if (tools === request.tools && system === request.system && messages === request.messages) return request return LLMRequest.update(request, { tools, system, messages })}