import { spawnSync } from "node:child_process"; import { existsSync } from "node:fs"; import { homedir } from "node:os"; import { join, resolve } from "node:path"; import { describe, expect, test } from "vitest"; import { buildRlmPrompt } from "../src/core/prompts/index.js"; import { formatHarnessStateForPrompt, type HarnessState } from "../src/core/refinement/index.js"; import type { Skill } from "../src/core/skills.js"; import { buildSystemPrompt } from "../src/core/system-prompt.js"; import { createIpythonToolDefinition } from "../src/core/tools/ipython.js"; function skill(name: string): Skill { return { name, description: `${name} description`, filePath: `/skills/${name}/SKILL.md`, baseDir: `/skills/${name}`, sourceInfo: { source: "local", path: `/skills/${name}/SKILL.md`, scope: "project", origin: "top-level", }, disableModelInvocation: false, kind: "markdown", }; } function pythonSkill(name: string, importName = name.replaceAll("-", "_")): Skill { const base = skill(name); return { ...base, kind: "python", python: { importName, packagePath: `/skills/${name}`, pyprojectPath: `/skills/${name}/pyproject.toml`, }, }; } describe("buildRlmPrompt", () => { test("defaults omitted activeTools to ipython guidance", () => { const prompt = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", installedSkills: ["websearch"], }); expect(prompt).toContain("Installed Python skill modules (pre-imported): `websearch`."); expect(prompt).toContain("An `rlm` object is already in your global namespace"); expect(prompt).toContain("persistent Python REPL"); expect(prompt).toContain("Python is the orchestration language"); }); test("guides subagents via models() discovery before spawn()", () => { const prompt = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", activeTools: ["ipython"], }); expect(prompt).toContain("await rlm.spawn('sub-task', name='worker', model='...')"); expect(prompt).toContain("Before calling `rlm.spawn()`, you MUST call `await rlm.models()`"); expect(prompt).toContain("`name` and `model` are required"); }); test("only documents ipython shell prefixes when ipython is active", () => { const prompt = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", activeTools: ["bash"], allowRecursion: false, }); expect(prompt).not.toContain("persistent Python REPL"); }); test("keeps shell skill command guidance when ipython is inactive", () => { const prompt = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", installedSkills: ["websearch"], activeTools: ["bash"], allowRecursion: false, }); expect(prompt).toContain("Installed skills available as shell commands: `websearch`."); expect(prompt).toContain("Each skill is also available as a shell command"); expect(prompt).toContain("` --help`"); expect(prompt).not.toContain("Installed Python skill modules (pre-imported)"); expect(prompt).not.toContain("Read each skill's SKILL.md for its API"); }); test("gates agent messaging and observation doctrine on installed Python skills", () => { const withoutCapabilities = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/session.jsonl", activeTools: ["ipython"], allowRecursion: true, depth: 1, }); expect(withoutCapabilities).not.toContain("agent_message.send"); expect(withoutCapabilities).not.toContain("agent_observe"); const systemPromptWithoutCapabilities = buildSystemPrompt({ selectedTools: ["ipython"], contextFiles: [], skills: [], cwd: "/repo", }); expect(systemPromptWithoutCapabilities).not.toContain("agent_message.send"); expect(systemPromptWithoutCapabilities).not.toContain("agent_observe"); const withCapabilities = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/session.jsonl", installedSkills: ["agent_message", "agent_observe"], activeTools: ["ipython"], allowRecursion: true, depth: 1, }); expect(withCapabilities).toContain("agent_message.send"); expect(withCapabilities).toContain("agent_observe.list_agents()"); expect(withCapabilities).toContain("agent_observe"); expect(withCapabilities).toContain("restricted to your parent, siblings, and direct children"); }); test("does not prescribe kernel-only child replies without ipython", () => { const prompt = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/session.jsonl", installedSkills: ["agent_message"], activeTools: ["bash"], depth: 1, }); expect(prompt).toContain("You are a child agent"); expect(prompt).not.toContain("When a task calls for an answer, reply explicitly with `await agent_message.send"); }); test("exposes the automatic child registry independently of observation skills", () => { const withoutObserve = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", activeTools: ["ipython"], }); const withObserve = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", installedSkills: ["agent_observe"], activeTools: ["ipython"], }); for (const prompt of [withoutObserve, withObserve]) { expect(prompt).toContain("await rlm.list_subagents()"); expect(prompt).toContain("await rlm.delete_subagent(child)"); expect(prompt).toContain("recover direct child handles"); expect(prompt).not.toContain("Write a small disk registry"); } }); test("documents the bash() orchestration contract when ipython is active", () => { const prompt = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", activeTools: ["ipython"], allowRecursion: false, }); expect(prompt).toContain("Use `bash()` to invoke programs, not to write shell programs"); expect(prompt).toContain("A `bash()` handle left running beyond its creating cell sends a completion follow-up"); }); test("documents preferring Python for reading and searching files when ipython is active", () => { const prompt = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", activeTools: ["ipython"], allowRecursion: false, }); expect(prompt).toContain("Use Python for reading, searching, and editing files"); expect(prompt).toContain("Always assign read/search results to named variables"); }); test("publishes the generated skill-surface digest instead of the hand-written inspection line", () => { const digest = "# skill surface (generated)\ngit.status(cwd?) -> str"; const withDigest = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", installedSkills: ["git"], activeTools: ["ipython"], allowRecursion: false, skillDigest: `${digest}\n`, }); expect(withDigest).toContain(digest); expect(withDigest).toContain("If the digest disagrees with the installed code, the code wins"); expect(withDigest).not.toContain("Read each skill's SKILL.md for its API"); const withoutDigest = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", installedSkills: ["git"], activeTools: ["ipython"], allowRecursion: false, }); expect(withoutDigest).toContain("Read each skill's SKILL.md for its API"); expect(withoutDigest).not.toContain("# skill surface"); }); test("includes the edit skill guidance only when the edit skill is installed", () => { const withEdit = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", installedSkills: ["edit"], activeTools: ["ipython"], allowRecursion: false, }); expect(withEdit).toContain('await edit(path="pkg/file.py", old_str=old, new_str=new)'); expect(withEdit).toContain("triple double quotes"); const withoutEdit = buildRlmPrompt({ cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", installedSkills: ["websearch"], activeTools: ["ipython"], allowRecursion: false, }); expect(withoutEdit).not.toContain("await edit(path="); }); }); describe("buildSystemPrompt", () => { test("includes remote host in system prompt when remoteHost is provided", () => { const prompt = buildSystemPrompt({ cwd: "/test", selectedTools: ["ipython"], contextFiles: [], skills: [], remoteHost: "chernobog", }); expect(prompt).toContain("Remote execution host: chernobog"); expect(prompt).toContain("The `ipython` tool and `bash()` execute on remote host chernobog over SSH."); }); test("formats the harness digest with refine guidance for REPL sessions", () => { const harnessState: HarnessState = { schema: 1, entries: { prompt: { focused_edits: { id: "focused_edits", kind: "prompt", title: "Focused edits", content: "Prefer small prompt, memory, skill, or subagent updates over broad rewrites.", path: "policy", reference: {}, arguments: {}, metadata: {}, source: "refine", created_at: "2026-06-08T00:00:00.000Z", updated_at: "2026-06-08T00:00:00.000Z", version: 1, }, }, memory: { validation: { id: "validation", kind: "memory", title: "Validation", content: "Run `npm run check` after PrimeAgent code changes.", path: "repo/prime-agent", reference: {}, arguments: {}, metadata: {}, source: "refine", created_at: "2026-06-08T00:00:00.000Z", updated_at: "2026-06-08T00:00:00.000Z", version: 2, }, }, skill: { review_refinement: { id: "review_refinement", kind: "skill", title: "Review refinement", content: "Check requested edit coverage, rollback safety, and validation commands.", path: "quality", reference: { type: "python", import: "agent_skills.review_refinement", callable: "review_refinement", call_pattern: "await review_refinement(task=...)", }, arguments: { task: { type: "string", required: true, description: "Review task to perform." }, }, metadata: {}, source: "refine", created_at: "2026-06-08T00:00:00.000Z", updated_at: "2026-06-08T00:00:00.000Z", version: 1, }, }, subagent: { refinement_reviewer: { id: "refinement_reviewer", kind: "subagent", title: "Refinement reviewer", content: "Review proposed harness edits for scope, evidence, and unintended behavior.", path: "review", reference: {}, arguments: {}, metadata: {}, source: "refine", created_at: "2026-06-08T00:00:00.000Z", updated_at: "2026-06-08T00:00:00.000Z", version: 1, }, }, }, refinements: [ { id: "refine_1", trigger: "Observed validation miss", changes: ["create memory:validation"], evidence: "manual test", outcome: "Future runs should name npm run check.", created_at: "2026-06-08T00:00:00.000Z", }, ], }; const prompt = formatHarnessStateForPrompt(harnessState, { includeIpythonExamples: true, includeShellExamples: false, includeRefineExamples: true, }); expect(prompt).toContain("# Continual Harness State"); expect(prompt).toContain("Local continual harness entries belong to this Prime Agent session"); expect(prompt).toContain("The continual harness entries below are compact summaries, not full descriptions"); expect(prompt).toContain("Use global continual harness refinement only for stable cross-session lessons"); expect(prompt).toContain("When to call `await refine.run()`"); expect(prompt).toContain("Call contract: read each installed Python skill's SKILL.md"); expect(prompt).toContain("Continual harness skill entries are Python REPL skills"); expect(prompt).toContain("Spawn a continual harness subagent spec by composing a concise task prompt"); expect(prompt).toContain("handle = await rlm.spawn('sub-task', name='worker', model='')"); const spawnMatch = prompt.match(/rlm\.spawn\(([^)]+)\)/); expect(spawnMatch).not.toBeNull(); const documentedArgs = [...spawnMatch![1].matchAll(/([a-zA-Z_]\w*)\s*=/g)].map((m) => m[1]); const python = [ process.env.PRIME_AGENT_KERNEL_PYTHON, resolve(import.meta.dirname, "../../..", "prime-agent-runtime", ".venv", "bin", "python"), join(homedir(), ".prime", "agent", "kernel-venv", "bin", "python"), ].find((p) => p && existsSync(p)); expect(python).toBeDefined(); const sigCheck = spawnSync( python as string, [ "-c", "import inspect, json, rlm; sig = inspect.signature(rlm.spawn); print(json.dumps({'params': list(sig.parameters.keys()), 'required': [k for k, v in sig.parameters.items() if v.default is inspect.Parameter.empty and k != 'prompt']}))", ], { encoding: "utf8" }, ); expect(sigCheck.status).toBe(0); const { params, required } = JSON.parse(sigCheck.stdout) as { params: string[]; required: string[] }; for (const arg of documentedArgs) { expect(params).toContain(arg); } for (const req of required) { expect(documentedArgs).toContain(req); } expect(prompt).toContain("admission returns immediately"); expect(prompt).toContain("never the child's answer"); expect(prompt).toContain("receiver_role='parent'"); expect(prompt).toContain("await rlm.list_subagents()"); expect(prompt).toContain("receiver_role='child'"); expect(prompt).not.toContain("asyncio.create_task(rlm('sub-task'))"); expect(prompt).not.toContain("asyncio.gather(rlm('task1'), rlm('task2'))"); expect(prompt).toContain("after a repeated failure"); expect(prompt).toContain("a reusable tactic emerges"); expect(prompt).toContain("a repeated delegation role should become a subagent spec"); expect(prompt).toContain("a repeated procedure should become a skill"); expect(prompt).toContain("a durable fact/preference should become a memory"); expect(prompt).toContain("a narrow behavioral policy should become a prompt addendum"); expect(prompt).toContain("validation shows a continual harness entry is wrong"); expect(prompt).toContain("[global:focused_edits] Focused edits (policy, v1)"); expect(prompt).toContain("[global:validation] Validation (repo/prime-agent, v2): Run `npm run check`"); expect(prompt).toContain("[global:review_refinement] Review refinement (quality, v1)"); expect(prompt).toContain("[global:refinement_reviewer] Refinement reviewer (review, v1)"); expect(prompt).toContain("recent refinements: 1"); expect(prompt).toContain("[refine_1] Observed validation miss: create memory:validation"); }); test("system prompts no longer embed harness state", () => { const rlmPrompt = buildSystemPrompt({ selectedTools: ["ipython"], contextFiles: [], skills: [pythonSkill("refine")], cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", }); const customPrompt = buildSystemPrompt({ customPrompt: "custom body", selectedTools: ["ipython"], contextFiles: [], skills: [], cwd: "/repo", }); expect(rlmPrompt).not.toContain("# Continual Harness State"); expect(customPrompt).not.toContain("# Continual Harness State"); }); test("keeps the harness digest compact", () => { const longContent = "x".repeat(500); const memoryEntries: HarnessState["entries"]["memory"] = {}; for (let i = 0; i < 8; i++) { memoryEntries[`memory_${i}`] = { id: `memory_${i}`, kind: "memory", title: `Memory ${i}`, content: longContent, path: "overflow", reference: {}, arguments: {}, metadata: {}, source: "refine", created_at: "2026-06-08T00:00:00.000Z", updated_at: "2026-06-08T00:00:00.000Z", version: 1, }; } const harnessState: HarnessState = { schema: 1, entries: { prompt: {}, memory: memoryEntries, skill: {}, subagent: {}, }, refinements: [], }; const prompt = formatHarnessStateForPrompt(harnessState); expect(prompt).toContain("memory: 8"); expect(prompt).toContain("- [global:memory_6] Memory 6 (overflow, v1)\n"); expect(prompt).toContain("- [global:memory_7] Memory 7 (overflow, v1)\n"); expect(prompt).not.toContain("- +2 more memory entries"); expect(prompt).toContain(`${"x".repeat(177)}...`); expect(prompt).not.toContain(longContent); }); test("uses the model-agnostic rlm harness prompt", () => { const prompt = buildSystemPrompt({ selectedTools: ["ipython"], contextFiles: [], skills: [pythonSkill("refine"), pythonSkill("agent-message"), pythonSkill("agent-observe")], cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", }); expect(prompt).toContain("You are a general purpose agent that uses code to solve tasks."); expect(prompt).toContain("Working directory: /repo"); expect(prompt).toContain("Conversation log: /repo/.pi/sessions/session.jsonl"); expect(prompt).toContain("await rlm.spawn('sub-task', name='worker', model='...')"); expect(prompt).toContain("returns at admission, not completion"); expect(prompt).toContain("Results arrive only through an available messaging capability or files"); expect(prompt).toContain("recover direct child handles"); expect(prompt).toContain("kernel restart or compaction"); expect(prompt).toContain("rlm.list_subagents"); expect(prompt).toContain("rlm.delete_subagent"); expect(prompt).toContain("rlm_child_id"); expect(prompt).toContain("name='worker'"); expect(prompt).toContain("session_dir"); expect(prompt).toContain("agent_observe"); expect(prompt).toContain("restricted to your parent, siblings, and direct children"); expect(prompt).toContain("Children write deliverables atomically"); expect(prompt).toContain("The parent must re-run the child's one-command gate"); }); test("omits ipython-only subagent guidance when ipython is inactive", () => { const harnessState: HarnessState = { schema: 1, entries: { prompt: {}, memory: {}, skill: {}, subagent: { worker: { id: "worker", kind: "subagent", title: "Worker", content: "Review a self-contained task and report findings.", path: "review", reference: {}, arguments: {}, metadata: {}, source: "refine", created_at: "2026-06-08T00:00:00.000Z", updated_at: "2026-06-08T00:00:00.000Z", version: 1, }, }, }, refinements: [], }; const prompt = buildSystemPrompt({ selectedTools: ["bash"], contextFiles: [], skills: [], cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", }); const digest = formatHarnessStateForPrompt(harnessState, { includeIpythonExamples: false, includeShellExamples: true, }); expect(prompt).toContain("You are a general purpose agent that uses code to solve tasks."); expect(prompt).not.toContain("persistent Python REPL"); expect(prompt).not.toContain("Default to non-blocking subagents"); expect(prompt).not.toContain("agent_observe.list_agents"); expect(digest).toContain("# Continual Harness State"); expect(digest).toContain("Call contract: use installed skills as shell commands"); expect(digest).toContain("subagent: 1"); expect(digest).not.toContain("asyncio.create_task"); expect(digest).not.toContain("await "); expect(digest).not.toContain("await refine.run()"); }); test("omits shell guidance from harness state when shell is inactive", () => { const harnessState: HarnessState = { schema: 1, entries: { prompt: {}, memory: {}, skill: {}, subagent: { worker: { id: "worker", kind: "subagent", title: "Worker", content: "Review a self-contained task and report findings.", path: "review", reference: {}, arguments: {}, metadata: {}, source: "refine", created_at: "2026-06-08T00:00:00.000Z", updated_at: "2026-06-08T00:00:00.000Z", version: 1, }, }, }, refinements: [], }; const digest = formatHarnessStateForPrompt(harnessState, { includeIpythonExamples: false, includeShellExamples: false, }); expect(digest).toContain("# Continual Harness State"); expect(digest).toContain("without the Python REPL or shell access"); expect(digest).not.toContain("use installed skills as shell commands"); expect(digest).not.toContain(" ..."); expect(digest).not.toContain("asyncio.create_task"); expect(digest).not.toContain("await "); expect(digest).not.toContain("await refine.run()"); }); test("custom prompt override bypasses the rlm harness body", () => { const prompt = buildSystemPrompt({ customPrompt: "custom body", selectedTools: ["ipython"], appendSystemPrompt: "custom append", contextFiles: [], skills: [], cwd: "/repo", }); expect(prompt).toContain("custom body"); expect(prompt).not.toContain("# IPython Kernel Guidance"); expect(prompt).not.toContain("You are a general purpose agent that uses code to solve tasks."); expect(prompt.indexOf("Current working directory: /repo")).toBeLessThan(prompt.indexOf("custom append")); }); test("adds child reply doctrine to custom prompts when messaging is available", () => { const prompt = buildSystemPrompt({ customPrompt: "custom body", selectedTools: ["ipython"], contextFiles: [], skills: [pythonSkill("agent-message")], cwd: "/repo", rlmDepth: 1, rlmParentAgent: "orchestrator", }); expect(prompt).toContain("You are a child agent spawned by orchestrator"); expect(prompt).toContain('await agent_message.send(message, receiver_role="parent")'); expect(prompt).not.toContain("You are a general purpose agent that uses code to solve tasks."); }); test("gates custom-prompt child reply doctrine on IPython and agent messaging", () => { const build = (selectedTools: string[], skills: Skill[]) => buildSystemPrompt({ customPrompt: "custom body", selectedTools, contextFiles: [], skills, cwd: "/repo", rlmDepth: 1, }); expect(build(["ipython"], [])).toContain("You are a child agent spawned by your parent agent"); expect(build(["ipython"], [])).not.toContain("agent_message.send"); expect(build(["bash"], [pythonSkill("agent-message")])).not.toContain("agent_message.send"); }); test("append system prompt content is included after the rlm harness prompt", () => { const prompt = buildSystemPrompt({ selectedTools: ["ipython"], appendSystemPrompt: "extra instruction", contextFiles: [], skills: [], cwd: "/repo", }); expect(prompt.indexOf("Treat harness refinement as a small, evidence-backed update")).toBeLessThan( prompt.indexOf("extra instruction"), ); expect(prompt).not.toContain("Call at most one built-in tool per turn."); }); test("project context files are appended", () => { const prompt = buildSystemPrompt({ selectedTools: ["ipython"], contextFiles: [{ path: "AGENTS.md", content: "project rules" }], skills: [], cwd: "/repo", }); expect(prompt).toContain("# Project Context"); expect(prompt).toContain("## AGENTS.md\n\nproject rules"); }); test("markdown skills are included in rlm harness prompts without Python pre-imports", () => { const prompt = buildSystemPrompt({ selectedTools: ["ipython"], contextFiles: [], skills: [skill("websearch")], cwd: "/repo", }); expect(prompt).not.toContain("Installed Python skill modules (pre-imported)"); expect(prompt).toContain(""); expect(prompt).toContain("websearch"); expect(prompt).toContain("markdown"); expect(prompt).toContain("/skills/websearch/SKILL.md"); }); test("carries the runtime skill digest into the rlm prompt", () => { const digest = "# skill surface (generated)\nbash(command, input?, *, cwd?, display_command?) -> BashHandle"; const prompt = buildSystemPrompt({ selectedTools: ["ipython"], contextFiles: [], skills: [pythonSkill("git")], cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", skillDigest: digest, }); expect(prompt).toContain(digest); expect(prompt).not.toContain("Read each skill's SKILL.md for its API"); }); test("Python skills are configured for IPython and included in skill metadata", () => { const prompt = buildSystemPrompt({ selectedTools: ["ipython"], contextFiles: [], skills: [pythonSkill("web-search")], cwd: "/repo", }); expect(prompt).toContain("Installed Python skill modules (pre-imported): `web_search`."); expect(prompt).toContain("web-search"); expect(prompt).toContain("python"); expect(prompt).toContain("web_search"); }); test("prompt guidelines are appended and deduplicated", () => { const prompt = buildSystemPrompt({ selectedTools: ["ipython", "dynamic_tool"], promptGuidelines: ["Use dynamic_tool for summaries.", " Use dynamic_tool for summaries. ", " "], contextFiles: [], skills: [], cwd: "/repo", }); expect(prompt).toContain("# Additional Guidance"); expect(prompt.match(/- Use dynamic_tool for summaries\./g)).toHaveLength(1); }); }); describe("createIpythonToolDefinition", () => { test("describes project checks as target-environment work", () => { const tool = createIpythonToolDefinition("/repo"); expect(tool.description).toContain("persistent Python REPL"); expect(tool.description).toContain("target project's own environment"); expect(tool.promptSnippet).toContain("bash() orchestration"); const codeSchema = tool.parameters.properties.code; const codeDescription = "description" in codeSchema && typeof codeSchema.description === "string" ? codeSchema.description : ""; expect(codeDescription).toContain("target project's own environment"); }); });