diff --git a/src/executor.ts b/src/executor.ts index 47c2f25..1c03157 100644 --- a/src/executor.ts +++ b/src/executor.ts @@ -27,10 +27,28 @@ export interface ExecSpec { command: string; /** The conversation whose persistent sandbox to run in; omitted → one-off. */ session?: string; - /** Wall-clock cap; falls back to the executor's configured default. */ + /** Wall-clock cap; falls back to the configured default, clamped to the max. */ timeoutMs?: number; } +/** + * What the sandbox actually is, for whoever has to describe it. + * + * The model's only picture of the sandbox is the `run_shell` description, and a + * description that hardcodes what config decides is a description that lies: + * with `network: false` (the default) a cheerful "you can `apk add` things" sends + * the model to spend a turn on a command that cannot work. So the executor + * reports its own shape and the tool text is written from it. + */ +export interface SandboxInfo { + image: string; + network: boolean; + defaultTimeoutMs: number; + maxTimeoutMs: number; + memory: string; + cpus: string; +} + export interface ExecResult { stdout: string; stderr: string; @@ -49,6 +67,8 @@ export interface HarvestedFile { export interface Executor { /** A short tag for logs/UI ("docker", "kata"). */ readonly kind: string; + /** The shape of the sandbox, so tool text can describe it truthfully. */ + readonly info: SandboxInfo; run(spec: ExecSpec, signal?: AbortSignal): Promise; /** * Materialize bytes inside the sandbox, creating parent directories. This is @@ -79,6 +99,28 @@ function shq(s: string): string { return `'${s.replace(/'/g, "'\\''")}'`; } +/** What a killed command reports back. coreutils `timeout` uses 124 already. */ +const TIMEOUT_EXIT = 124; + +/** + * Wrap a command so the container kills it on time. + * + * Two wrinkles, both from the image being the deployment's choice. `timeout` may + * not exist at all, and a missing binary must not turn every command into "not + * found" — so its absence falls back to running bare, with the client-side timer + * as the only cap. And the two implementations disagree on the exit code: GNU + * reports 124, busybox reports 143 (128 + SIGTERM). Normalizing here means the + * caller reads one number. + */ +function withTimeout(command: string, timeoutMs: number): string { + const secs = Math.max(1, Math.ceil(timeoutMs / 1000)); + const inner = shq(command); + return ( + `if command -v timeout >/dev/null 2>&1; then timeout ${secs} sh -c ${inner}; c=$?; ` + + `[ "$c" = 143 ] && c=${TIMEOUT_EXIT}; exit $c; else sh -c ${inner}; fi` + ); +} + const MAX_OUTPUT = 100_000; // cap each stream so a runaway command can't flood the transcript function clamp(s: string): string { return s.length > MAX_OUTPUT ? s.slice(0, MAX_OUTPUT) + "\n…[output truncated]" : s; @@ -117,6 +159,15 @@ interface Session { lastUsed: number; } +/** The sandbox section of config; `maxTimeoutMs` is optional for older callers. */ +type SandboxConfig = Omit & { maxTimeoutMs?: number }; + +const MEMORY = "512m"; +const CPUS = "1"; + +/** Created up front so writing to the outbox never needs a `mkdir -p` first. */ +const WORKSPACE_DIRS = ["/workspace", "/workspace/inputs", "/workspace/outputs"]; + /** * Runs commands in docker with cpu/memory/pids caps and (by default) no network. * A session-scoped call runs in a long-lived container per conversation — one @@ -127,33 +178,37 @@ interface Session { */ export class LocalDockerExecutor implements Executor { readonly kind: string; + readonly info: SandboxInfo; private readonly image: string; private readonly defaultTimeoutMs: number; + private readonly maxTimeoutMs: number; private readonly network: boolean; private readonly idleMs: number; private readonly runtime: string | undefined; private readonly env: DockerEnv; private readonly sessions = new Map(); - constructor( - image: string, - defaultTimeoutMs: number, - network: boolean, - idleMs: number, - runtime?: string, - dockerHost?: string, - ) { - this.image = image; - this.defaultTimeoutMs = defaultTimeoutMs; - this.network = network; - this.idleMs = idleMs; - this.runtime = runtime; + constructor(cfg: SandboxConfig) { + this.image = cfg.image; + this.defaultTimeoutMs = cfg.timeoutMs; + this.maxTimeoutMs = Math.max(cfg.maxTimeoutMs ?? cfg.timeoutMs, cfg.timeoutMs); + this.network = cfg.network; + this.idleMs = cfg.idleMs; + this.runtime = cfg.runtime; // "kata" et al. are more informative in logs/UI than a bare "docker". - this.kind = runtime ?? "docker"; + this.kind = cfg.runtime ?? "docker"; + this.info = { + image: this.image, + network: this.network, + defaultTimeoutMs: this.defaultTimeoutMs, + maxTimeoutMs: this.maxTimeoutMs, + memory: MEMORY, + cpus: CPUS, + }; // A remote daemon (e.g. a KVM box over the tailnet) is selected purely via // DOCKER_HOST; the rest of the env is inherited. Undefined → local daemon. - this.env = dockerHost - ? { ...(process.env as Record), DOCKER_HOST: dockerHost } + this.env = cfg.dockerHost + ? { ...(process.env as Record), DOCKER_HOST: cfg.dockerHost } : undefined; // Reap containers a prior run left behind (a crash skips teardown), then // sweep idle ones periodically. Both best-effort; unref'd so neither pins @@ -177,9 +232,9 @@ export class LocalDockerExecutor implements Executor { "--network", this.network ? "bridge" : "none", "--cpus", - "1", + CPUS, "--memory", - "512m", + MEMORY, "--pids-limit", "512", "--workdir", @@ -201,7 +256,7 @@ export class LocalDockerExecutor implements Executor { this.image, "sh", "-c", - "mkdir -p /workspace && exec tail -f /dev/null", + `mkdir -p ${WORKSPACE_DIRS.join(" ")} && exec tail -f /dev/null`, ]; const r = await dockerRun(argv, this.env); if (r.exitCode !== 0) @@ -323,8 +378,17 @@ export class LocalDockerExecutor implements Executor { } async run(spec: ExecSpec, signal?: AbortSignal): Promise { - const timeoutMs = spec.timeoutMs ?? this.defaultTimeoutMs; - const t = this.timed(timeoutMs, signal); + // A caller may ask for longer than the default, never longer than the max — + // the ceiling belongs to the deployment, not to the command. + const timeoutMs = Math.min(spec.timeoutMs ?? this.defaultTimeoutMs, this.maxTimeoutMs); + // The client-side timer only ever abandons the docker CLI; the process it + // started keeps running inside the container, so a `while true` would burn + // the session's cpu until the container was swept. The cap is therefore + // enforced INSIDE, where something can actually kill it, and the outer timer + // stays on as a backstop for a hung daemon — a couple of seconds later, so + // the in-container kill is the one that normally fires. + const command = withTimeout(spec.command, timeoutMs); + const t = this.timed(timeoutMs + 2_000, signal); try { let argv: string[]; if (spec.session) { @@ -335,7 +399,7 @@ export class LocalDockerExecutor implements Executor { this.sessions.delete(spec.session); // let the next call retry a fresh container throw e; } - argv = ["exec", "--workdir", "/workspace", s.name, "sh", "-c", spec.command]; + argv = ["exec", "--workdir", "/workspace", s.name, "sh", "-c", command]; } else { argv = [ "run", @@ -348,15 +412,16 @@ export class LocalDockerExecutor implements Executor { this.image, "sh", "-c", - spec.command, + command, ]; } const r = await dockerRun(argv, this.env, t.signal); + const timedOut = t.state.timedOut || r.exitCode === TIMEOUT_EXIT; return { stdout: clamp(r.stdout), stderr: clamp(r.stderr), - exitCode: t.state.timedOut ? -1 : r.exitCode, - timedOut: t.state.timedOut, + exitCode: timedOut ? -1 : r.exitCode, + timedOut, }; } finally { t.cleanup(); @@ -376,20 +441,13 @@ export function resetExecutor(): void { } /** The configured executor, or null when the sandbox is disabled/unimplemented. */ -export function createExecutor(cfg: Config["sandbox"] = getConfig().sandbox): Executor | null { +export function createExecutor(cfg: SandboxConfig = getConfig().sandbox): Executor | null { if (!cfg.enabled) return null; switch (cfg.backend) { case "docker": // Local dev or remote microVM — the difference is entirely config: // `dockerHost` (which daemon) and `runtime` (e.g. "kata" for a VM). - return new LocalDockerExecutor( - cfg.image, - cfg.timeoutMs, - cfg.network, - cfg.idleMs, - cfg.runtime, - cfg.dockerHost, - ); + return new LocalDockerExecutor(cfg); default: return null; } @@ -398,7 +456,13 @@ export function createExecutor(cfg: Config["sandbox"] = getConfig().sandbox): Ex /** Fold an exec result into the text a tool hands back to the model. */ export function formatExecResult(r: ExecResult): string { const parts: string[] = []; - if (r.timedOut) parts.push("[command timed out]"); + // A bare "[timed out]" reads as a failure of the sandbox; saying what to do + // about it turns a dead end into the next step. + if (r.timedOut) + parts.push( + "[command timed out and was killed — output above is partial. Re-run something " + + "smaller, or pass a larger timeout_seconds if it genuinely needs the time.]", + ); parts.push(`exit code: ${r.exitCode}`); parts.push("stdout:\n" + (r.stdout.trim() || "(empty)")); if (r.stderr.trim()) parts.push("stderr:\n" + r.stderr.trim()); diff --git a/src/prompt.tpl b/src/prompt.tpl index 7a20f5c..05e673e 100644 --- a/src/prompt.tpl +++ b/src/prompt.tpl @@ -47,9 +47,11 @@ You are warm, direct, and unpretentious. You talk like a sharp person rather tha {{if .Tools}} -Reach for tools rather than speculate. Search before you assume, read before you edit. When several tool calls are independent, run them together. +Reach for tools rather than speculate. Search before you assume, read before you edit. When several tool calls are independent, run them together. Each tool's own description carries the details; this is the inventory. {{.Tools}} - +{{if .Sandbox}} +You have a sandbox, so prefer computing an answer to estimating one: run the numbers, parse the file, check the output. It is a scratch Linux container private to this chat — build up state across several small commands rather than one long one, and write anything worth keeping to /workspace/outputs/, which hands it to the user as a document. +{{end}} {{end}} {{/* ---- ENV + MEMORY (mirrors the Crush layout) ---- */}} diff --git a/src/prompt.ts b/src/prompt.ts index 9daadd2..e6036b5 100644 --- a/src/prompt.ts +++ b/src/prompt.ts @@ -153,16 +153,26 @@ function formatDate(d: Date): string { }); } -/** The `` body: one line per exposed tool. Empty → the section is dropped. */ +/** + * The `` body: one line per exposed tool. Empty → the section is dropped. + * + * A summary, not a manual. The provider already sends each tool's full + * description alongside its schema, so repeating it here buys nothing and costs + * it twice — and the cost grows every time a description earns another paragraph + * (run_shell's runs to several). What the prompt needs is the inventory: what + * exists, so the model knows there is something to reach for. + */ function toolsBlock(tools: ToolSet): string { const names = Object.keys(tools); if (names.length === 0) return ""; - return names - .map((n) => { - const desc = (tools[n] as { description?: string }).description ?? ""; - return `- ${n}: ${desc}`; - }) - .join("\n"); + return names.map((n) => `- ${n}: ${summarize(tools[n] as { description?: string })}`).join("\n"); +} + +/** A tool's first sentence, which by convention says what it is for. */ +function summarize(t: { description?: string }): string { + const first = (t.description ?? "").split("\n", 1)[0]!.trim(); + const stop = first.search(/\.(\s|$)/); + return stop === -1 ? first : first.slice(0, stop + 1); } /** Reads the configured context files into `{Path, Content}` for the block. */ @@ -207,6 +217,7 @@ export function buildSystemPrompt(opts: { Boundaries: p.boundaries ?? "", Platform: p.platform ?? "", Tools: toolsBlock(opts.tools), + Sandbox: "run_shell" in opts.tools, ContextFiles: contextFiles(p.contextFiles).concat(projectFiles), }; // Collapse the blank-line runs Go-style block actions leave behind (an diff --git a/src/settings.ts b/src/settings.ts index a1c4909..6711d96 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -118,8 +118,15 @@ const SandboxSchema = v.object({ * Omit to use the local daemon. */ dockerHost: v.optional(v.string()), - /** Per-command wall-clock cap. */ + /** Per-command wall-clock cap when the caller doesn't ask for one. */ timeoutMs: v.optional(v.pipe(v.number(), v.integer(), v.minValue(1)), 30_000), + /** + * Ceiling on a command's own timeout request. The default is tight enough to + * keep an ordinary command honest, but installing a package or crunching a + * file legitimately takes minutes — so a command may ask for longer, up to + * here, and nothing may exceed it. + */ + maxTimeoutMs: v.optional(v.pipe(v.number(), v.integer(), v.minValue(1)), 300_000), /** Idle time before a conversation's persistent sandbox is torn down. */ idleMs: v.optional(v.pipe(v.number(), v.integer(), v.minValue(1)), 10 * 60_000), /** docker: give the container network access (off = `--network none`). */ diff --git a/src/tools.ts b/src/tools.ts index cd9689c..b7b6916 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -1,7 +1,7 @@ import { jsonSchema, type LanguageModel, type Tool, type ToolSet, tool } from "ai"; import type { BlobStore } from "./blobs"; import type { ArtifactRef } from "./events"; -import { type Executor, formatExecResult, getExecutor } from "./executor"; +import { type Executor, formatExecResult, getExecutor, type SandboxInfo } from "./executor"; import { createFetchProvider, type FetchProvider } from "./fetch"; import { contextToText, @@ -282,37 +282,85 @@ function getAttachment(store: Store, blobs: BlobStore, executor: Executor, conve }); } +/** + * Describe the sandbox the way it actually is. + * + * Everything the model knows about this environment it learns here, so the text + * is assembled from the executor's own `info` rather than written once and hoped + * over: a static "you can `apk add` things" is a lie under the default + * `network: false`, and a model that doesn't know its wall clock will keep + * writing commands that get killed at 30 seconds and reading the corpse as a bug. + * + * The other half is what is NOT claimed. The image is whatever the deployment + * configured, so promising an interpreter is guesswork; a base alpine has + * busybox and little else. Better to say so and point at `command -v` than to + * send the model chasing a python3 that isn't there. + */ +export function sandboxDescription(info: SandboxInfo, hasAttachments: boolean): string { + const secs = (ms: number) => Math.round(ms / 1000); + const net = info.network + ? "It HAS network access, so `apk add …` (or pip/npm) works and installed packages persist for the rest of the chat." + : "It has NO network access: downloads, package installs, and any command that reaches out will fail. Work with what is in the image and what you are given."; + return [ + "Run a shell command in an isolated sandbox — a Linux container private to this conversation. " + + "Returns the exit code, stdout, and stderr. It is NOT the user's machine: it cannot reach their " + + "filesystem, their network, or any service they run, and it is torn down when the chat goes idle.", + "", + `Environment: the \`${info.image}\` image, ${info.cpus} cpu, ${info.memory} memory, no root outside the container. ` + + "Assume a minimal image — `sh` (busybox) and core utilities, not a full toolchain. " + + "Check with `command -v python3 || command -v node` before building on an interpreter rather than assuming one. " + + net, + "", + `Time: a command is killed at ${secs(info.defaultTimeoutMs)}s by default. If it will legitimately take longer ` + + `(a build, a large file), pass timeout_seconds — up to ${secs(info.maxTimeoutMs)}s. Long-running or interactive ` + + "commands have nowhere to go: nothing can answer a prompt, so pass `-y`/`--yes` and redirect input from /dev/null.", + "", + "State: /workspace is the working directory and it PERSISTS across calls in this chat, so build things up step by " + + "step rather than cramming a session into one command. It is still scratch — it dies with the container. " + + "To KEEP a file, write it to /workspace/outputs/ (already created): anything there is saved as a document the " + + "user can open and download, and is then removed from the directory, so do not expect to find it again — a " + + "second write of the same name becomes a new version." + + (hasAttachments + ? " Files the user attached, and documents from earlier turns, arrive at /workspace/inputs/ via get_attachment." + : ""), + ].join("\n"); +} + // A shell command in the sandbox executor (docker locally, a spindle microVM on // the homelab later). Offered only when a sandbox is configured. Marked // `sandbox` so the (future) durable loop routes it to the executor rather than // running it in-process; today its `execute` calls the executor directly. function runShell(executor: Executor, ctx: ToolContext) { const session = ctx.conversationId; + const hasAttachments = Boolean( + ctx.store && ctx.conversationId && ctx.store.listFiles(ctx.conversationId).length, + ); + const maxSeconds = Math.round(executor.info.maxTimeoutMs / 1000); return tool({ - description: - "Run a shell command in an isolated sandbox — a Linux container private to this " + - "conversation. `/workspace` (the working directory) and anything you install " + - "(e.g. `apk add git`) PERSIST across calls in this chat, so you can build up state " + - "step by step. Returns the exit code, stdout, and stderr. No network unless the " + - "deployment enables it. It is NOT the user's machine — it can't reach their real " + - "filesystem or services, and it's torn down when the conversation ends.\n\n" + - "The workspace is scratch: it dies with the container. To KEEP a file, write it " + - "to /workspace/outputs/ — anything there is saved as a document the user can " + - "open and download, and is then removed from the directory. Files the user " + - "attached, and documents from earlier turns, arrive via get_attachment.", - inputSchema: jsonSchema<{ command: string }>({ + description: sandboxDescription(executor.info, hasAttachments), + inputSchema: jsonSchema<{ command: string; timeout_seconds?: number }>({ type: "object", properties: { command: { type: "string", description: "The shell command line to run (via sh -c), starting in /workspace.", }, + timeout_seconds: { + type: "number", + description: `Wall-clock cap for this command, up to ${maxSeconds}. Omit for the default.`, + }, }, required: ["command"], additionalProperties: false, }), - execute: async ({ command }, { abortSignal }) => { - const result = formatExecResult(await executor.run({ command, session }, abortSignal)); + execute: async ({ command, timeout_seconds }, { abortSignal }) => { + const timeoutMs = + typeof timeout_seconds === "number" && timeout_seconds > 0 + ? Math.round(timeout_seconds * 1000) + : undefined; + const result = formatExecResult( + await executor.run({ command, session, timeoutMs }, abortSignal), + ); const artifacts = session ? await promoteOutputs(executor, ctx, session, abortSignal) : []; // Shape only shifts when there IS something to carry, so the ordinary // command keeps returning the plain transcript it always did. diff --git a/tests/executor.test.ts b/tests/executor.test.ts index c3881e8..3f64029 100644 --- a/tests/executor.test.ts +++ b/tests/executor.test.ts @@ -12,6 +12,16 @@ async function dockerAvailable(): Promise { } } const HAS_DOCKER = await dockerAvailable(); + +const SANDBOX = { + enabled: true, + backend: "docker", + image: "alpine:3.20", + timeoutMs: 30_000, + maxTimeoutMs: 300_000, + idleMs: 600_000, + network: false, +} as const; const liveTest = HAS_DOCKER ? test : test.skip; test("createExecutor returns null when the sandbox is disabled", () => { @@ -33,6 +43,7 @@ test("createExecutor builds a docker executor when enabled", () => { backend: "docker", image: "alpine:3.20", timeoutMs: 30_000, + maxTimeoutMs: 300_000, idleMs: 600_000, network: false, }); @@ -47,6 +58,7 @@ test("createExecutor reflects the configured runtime in kind (kata → microVM)" runtime: "kata", dockerHost: "ssh://kloe@prattle", timeoutMs: 30_000, + maxTimeoutMs: 300_000, idleMs: 600_000, network: false, }); @@ -63,7 +75,7 @@ test("formatExecResult surfaces exit code, stdout, and stderr", () => { liveTest( "docker executor runs a command and captures stdout + exit 0", async () => { - const e = new LocalDockerExecutor("alpine:3.20", 30_000, false, 600_000); + const e = new LocalDockerExecutor(SANDBOX); const r = await e.run({ command: "echo hi from sandbox" }); expect(r.exitCode).toBe(0); expect(r.stdout).toContain("hi from sandbox"); @@ -72,10 +84,70 @@ liveTest( 60_000, ); +liveTest( + "a runaway command is killed inside the container, on time", + async () => { + const e = new LocalDockerExecutor(SANDBOX); + const session = "test-" + Math.random().toString(36).slice(2); + try { + const started = Date.now(); + const r = await e.run({ command: "sleep 60", session, timeoutMs: 2_000 }); + expect(r.timedOut).toBe(true); + // The in-container `timeout` fires before the client-side backstop, so the + // process is actually dead rather than merely abandoned by the CLI. + expect(Date.now() - started).toBeLessThan(4_000); + // `[s]leep` so this probe's own command line doesn't match itself. + const still = await e.run({ command: "ps -o args= | grep -c '[s]leep 60'", session }); + expect(still.stdout.trim()).toBe("0"); + } finally { + e.disposeSession(session); + } + }, + 90_000, +); + +liveTest( + "a session's timeout may be raised, never past the configured max", + async () => { + const e = new LocalDockerExecutor({ ...SANDBOX, timeoutMs: 1_000, maxTimeoutMs: 3_000 }); + const session = "test-" + Math.random().toString(36).slice(2); + try { + // Longer than the default and under the max: honored. + expect((await e.run({ command: "sleep 2", session, timeoutMs: 5_000 })).timedOut).toBe(false); + // Asking past the max is clamped to it, so this still dies. + expect((await e.run({ command: "sleep 10", session, timeoutMs: 60_000 })).timedOut).toBe( + true, + ); + } finally { + e.disposeSession(session); + } + }, + 90_000, +); + +liveTest( + "a session starts with the inbox and outbox already there", + async () => { + const e = new LocalDockerExecutor(SANDBOX); + const session = "test-" + Math.random().toString(36).slice(2); + try { + // No `mkdir -p` first: writing a file to keep is a one-liner, as the tool + // description promises. + const r = await e.run({ command: "echo kept > outputs/note.txt && ls inputs", session }); + expect(r.exitCode).toBe(0); + const got = await e.harvest(session, "/workspace/outputs", { maxFiles: 5, maxBytes: 1e6 }); + expect(got.map((f) => f.path)).toEqual(["note.txt"]); + } finally { + e.disposeSession(session); + } + }, + 90_000, +); + liveTest( "docker executor reports a nonzero exit code", async () => { - const e = new LocalDockerExecutor("alpine:3.20", 30_000, false, 600_000); + const e = new LocalDockerExecutor(SANDBOX); const r = await e.run({ command: "exit 7" }); expect(r.exitCode).toBe(7); }, @@ -85,7 +157,7 @@ liveTest( liveTest( "a session's /workspace persists across calls; disposeSession clears it", async () => { - const e = new LocalDockerExecutor("alpine:3.20", 30_000, false, 600_000); + const e = new LocalDockerExecutor(SANDBOX); const session = "test-" + Math.random().toString(36).slice(2); try { const w = await e.run({ command: "echo persisted > /workspace/note.txt", session }); @@ -105,7 +177,7 @@ liveTest( liveTest( "a blob written in comes back out byte-identical, including binary", async () => { - const e = new LocalDockerExecutor("alpine:3.20", 30_000, false, 600_000); + const e = new LocalDockerExecutor(SANDBOX); const session = "test-" + Math.random().toString(36).slice(2); // Bytes that would not survive a text round-trip: NUL, high bytes, a lone // 0xff. `docker exec -i` pipes stdin through untouched, which is the whole @@ -143,7 +215,7 @@ liveTest( liveTest( "harvesting a workspace with no outbox is empty, not an error", async () => { - const e = new LocalDockerExecutor("alpine:3.20", 30_000, false, 600_000); + const e = new LocalDockerExecutor(SANDBOX); const session = "test-" + Math.random().toString(36).slice(2); try { await e.run({ command: "true", session }); diff --git a/tests/prompt.test.ts b/tests/prompt.test.ts index 6aef276..3bbd924 100644 --- a/tests/prompt.test.ts +++ b/tests/prompt.test.ts @@ -48,6 +48,30 @@ test("buildSystemPrompt includes the date and a tools section only when tools ex expect(withTools).toContain("web_search: Search the web."); }); +test("the tools section is an inventory: first sentence only, and no sandbox line without one", () => { + cfg({}); + const now = new Date("2026-08-04T12:00:00Z"); + // The provider already sends the full description next to the schema, so the + // prompt carries the summary and stops there. + const out = buildSystemPrompt({ + tools: { + web_search: { description: "Search the web.\n\nA second paragraph of detail." }, + } as unknown as ToolSet, + now, + }); + expect(out).toContain("web_search: Search the web."); + expect(out).not.toContain("second paragraph"); + expect(out).not.toContain("/workspace/outputs"); + + const sandboxed = buildSystemPrompt({ + tools: { + run_shell: { description: "Run a shell command in an isolated sandbox — a container." }, + } as unknown as ToolSet, + now, + }); + expect(sandboxed).toContain("/workspace/outputs"); +}); + test("buildSystemPrompt injects a project's context files as blocks", () => { cfg({}); const out = buildSystemPrompt({ diff --git a/tests/tools.test.ts b/tests/tools.test.ts index 4b7ac26..c8a5ad8 100644 --- a/tests/tools.test.ts +++ b/tests/tools.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "bun:test"; import type { ToolSet } from "ai"; -import { harden } from "../src/tools"; +import { harden, sandboxDescription } from "../src/tools"; // harden() only reads/replaces each tool's `execute`, so a minimal tool-shaped // object exercises it without the `tool()` builder's generic gymnastics. @@ -30,3 +30,36 @@ test("harden leaves a succeeding execute's result untouched", async () => { const out = await run(tools); expect(out).toEqual({ ok: true, n: 42 }); }); + +// The sandbox description is the model's whole picture of the environment, and +// the half that config decides is the half that used to be wrong. +const INFO = { + image: "alpine:3.20", + network: false, + defaultTimeoutMs: 30_000, + maxTimeoutMs: 300_000, + memory: "512m", + cpus: "1", +}; + +test("sandboxDescription tells the truth about the network, both ways", () => { + const off = sandboxDescription(INFO, false); + expect(off).toContain("NO network access"); + expect(off).not.toContain("apk add …"); + + const on = sandboxDescription({ ...INFO, network: true }, false); + expect(on).toContain("apk add"); + expect(on).not.toContain("NO network access"); +}); + +test("sandboxDescription states the image and both timeouts in seconds", () => { + const d = sandboxDescription(INFO, false); + expect(d).toContain("alpine:3.20"); + expect(d).toContain("killed at 30s"); + expect(d).toContain("up to 300s"); +}); + +test("sandboxDescription mentions get_attachment only when that tool is offered", () => { + expect(sandboxDescription(INFO, true)).toContain("get_attachment"); + expect(sandboxDescription(INFO, false)).not.toContain("get_attachment"); +});