From bd3c425217eb978fc5d81faedd52f02271d5d2cd Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Mon, 8 Jun 2026 19:13:21 -0600 Subject: [PATCH] feat(cogitate): plumb access tiers into run config + per-tier tool/policy gating Make a cogitate talent's access_tier a resolved, enforced per-run property distinct from the model tier, and gate the tool surface + command policy by it. - cogitate_contract: AccessCapabilities + tier->capability mapping keyed off COGITATE_ACCESS_TIERS (module-load drift guard; named ValueError on unknown tier) - talent: shared _validate_access_tier (default normal, loud reject), wired into both get_talent_configs (discovery) and get_talent (run path) - openhands.run_cogitate: register sol/read tools from the resolved tier's capabilities instead of unconditionally; finalization stays expects_emit_final-driven - cogitate_policy: tier-aware; deny support send-verbs (create/reply/attach/feedback) for normal/system-read, allow for outbound; reason names both tiers; chaining/substitution backstop scoped to support commands - assign tiers: steward=system-read, support+exec=outbound; all others resolve to normal - docs/COGITATE.md reconciled (system-read surface == normal today; outbound auto-submit via --yes window documented as the deferred next lode) This only narrows WHO may submit; the per-send human-approval gate is the next lode. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/COGITATE.md | 14 +-- solstone/apps/support/talent/support.md | 1 + solstone/talent/exec.md | 1 + solstone/talent/steward.md | 1 + solstone/think/cogitate_contract.py | 36 +++++++ solstone/think/cogitate_policy.py | 62 ++++++++++-- solstone/think/providers/openhands.py | 47 +++++---- solstone/think/talent.py | 38 ++++++- tests/test_cogitate_contract.py | 29 ++++++ tests/test_cogitate_policy.py | 129 +++++++++++++++++++++++- tests/test_openhands_provider.py | 121 +++++++++++++++++++++- tests/test_openhands_sol_tool.py | 2 +- tests/test_talent.py | 112 ++++++++++++++++++++ 13 files changed, 553 insertions(+), 40 deletions(-) diff --git a/docs/COGITATE.md b/docs/COGITATE.md index 62b902edb..4e8c720fa 100644 --- a/docs/COGITATE.md +++ b/docs/COGITATE.md @@ -98,10 +98,9 @@ cost — the denylist and caps are the safety bound, not a permission gate. An optional per-talent `read_scope` is a **narrowing hint** only; it is not the default gate, and a talent never fails for reading an undeclared journal file. -> The raw-read **tools** are built to this contract and are currently registered -> on every cogitate run. Per-tier gating is a later milestone; check the run's -> actual tool schema (`journal talent show --prompt`) for what a specific -> talent receives. +> The raw-read **tools** are built to this contract and registered per resolved +> access tier. Check the run's actual tool schema +> (`journal talent show --prompt`) for what a specific talent receives. ## Writes: only through `sol` domain commands @@ -124,8 +123,11 @@ enforcement are layered on top of it. | Tier | Purpose | Surface | |---|---|---| | `normal` | default cogitate talents | the `sol` tool (`sol` / `sol call`), the bounded raw-read tier, a finalization tool | -| `system-read` | diagnostics that also read scoped operational evidence (e.g. `steward`) | `normal` plus scoped reads of health / talent / log evidence | -| `outbound` | comms-like talents that may submit something that leaves the machine (e.g. `support`) | `normal` reads / drafts; **submit is gated on an explicit, real per-send human approval supplied to the run** — never an agent self-grant, a static template, or a scheduled inject | +| `system-read` | diagnostics boundary for scoped operational evidence (e.g. `steward`) | today its tool surface equals `normal`; the tier is the declared diagnostics boundary and extension point, with scoped evidence arriving through the talent pre-hook rather than an extra model read tool | +| `outbound` | comms-like talents that may submit something that leaves the machine (e.g. `support`) | `normal` reads / drafts plus submit-capable support commands; until the per-send human-approval lode lands, an outbound run can still auto-submit via `--yes` — this tier only narrows who may submit | + +Policy denies support send verbs (`create`, `reply`, `attach`, `feedback`) for +`normal` / `system-read` runs and allows them for `outbound` runs. **There is no `repair` tier in cogitate.** Health fact-gathering *and* repair are a deterministic supervisor/system workflow (explicitly launched and logged) — they do diff --git a/solstone/apps/support/talent/support.md b/solstone/apps/support/talent/support.md index cbfe28647..41ad7b0c4 100644 --- a/solstone/apps/support/talent/support.md +++ b/solstone/apps/support/talent/support.md @@ -1,5 +1,6 @@ { "type": "cogitate", + "access_tier": "outbound", "title": "Support", "description": "Files and monitors support requests with sol pbc — consent-gated, never sends data without explicit owner approval", "color": "#0288d1" diff --git a/solstone/talent/exec.md b/solstone/talent/exec.md index 3cdfbf66b..391827614 100644 --- a/solstone/talent/exec.md +++ b/solstone/talent/exec.md @@ -1,5 +1,6 @@ { "type": "cogitate", + "access_tier": "outbound", "title": "Exec", "description": "Sol — the journal itself, as a conversational partner", "hook": {"pre": "exec_context"} diff --git a/solstone/talent/steward.md b/solstone/talent/steward.md index eefedba6c..f43d9e006 100644 --- a/solstone/talent/steward.md +++ b/solstone/talent/steward.md @@ -4,6 +4,7 @@ "title": "Steward", "description": "Synthesizes federated health signals into identity/health.md.", "schedule": "daily", + "access_tier": "system-read", "priority": 45, "hook": {"pre": "steward", "post": "steward"}, "max_output_tokens": 900, diff --git a/solstone/think/cogitate_contract.py b/solstone/think/cogitate_contract.py index 9a557ab36..6f98911cc 100644 --- a/solstone/think/cogitate_contract.py +++ b/solstone/think/cogitate_contract.py @@ -12,6 +12,8 @@ re-typing the contract. from __future__ import annotations +from dataclasses import dataclass + COGITATE_RUNTIME_PREAMBLE = """\ You are a solstone cogitate talent running inside the live system. This runtime contract is authoritative; do not assume capabilities beyond it. @@ -36,9 +38,43 @@ FUTURE_ACCESS_TIERS = ("code-agent",) # The finalization mode a talent uses to signal completion. TALENT_FINALIZATION_MODES = ("emit_final", "FinishTool", "quiet") + +@dataclass(frozen=True) +class AccessCapabilities: + sol: bool + reads: bool + submit: bool + + +_ACCESS_TIER_CAPABILITIES: dict[str, AccessCapabilities] = { + "normal": AccessCapabilities(sol=True, reads=True, submit=False), + "system-read": AccessCapabilities(sol=True, reads=True, submit=False), + "outbound": AccessCapabilities(sol=True, reads=True, submit=True), +} + +_missing_access_tiers = set(COGITATE_ACCESS_TIERS) - set(_ACCESS_TIER_CAPABILITIES) +if _missing_access_tiers: + missing = ", ".join(sorted(_missing_access_tiers)) + raise RuntimeError(f"missing access tier capability mapping for: {missing}") + +_stray_access_tiers = set(_ACCESS_TIER_CAPABILITIES) - set(COGITATE_ACCESS_TIERS) +if _stray_access_tiers: + stray = ", ".join(sorted(_stray_access_tiers)) + raise RuntimeError(f"stray access tier capability mapping for: {stray}") + + +def capabilities_for_access_tier(tier: str) -> AccessCapabilities: + try: + return _ACCESS_TIER_CAPABILITIES[tier] + except KeyError as exc: + raise ValueError(f"unknown access_tier: {tier}") from exc + + __all__ = [ + "AccessCapabilities", "COGITATE_RUNTIME_PREAMBLE", "COGITATE_ACCESS_TIERS", "FUTURE_ACCESS_TIERS", "TALENT_FINALIZATION_MODES", + "capabilities_for_access_tier", ] diff --git a/solstone/think/cogitate_policy.py b/solstone/think/cogitate_policy.py index fa79a3b70..275d739b9 100644 --- a/solstone/think/cogitate_policy.py +++ b/solstone/think/cogitate_policy.py @@ -9,14 +9,23 @@ from datetime import date, datetime, timedelta from pathlib import Path from typing import Any +from solstone.think.cogitate_contract import ( + COGITATE_ACCESS_TIERS, + capabilities_for_access_tier, +) + MAX_TURNS = 60 DEFAULT_READ_CALL_BUDGET = 200 _SOL_INVOCATION_RE = re.compile(r"(^sol\s|\bsol call\b)") _JOURNAL_COMMANDS = {"identity", "routines", "health", "talent"} -_SHELL_CONTROL_TOKENS = {";", "&&", "||", "|", ">", ">>", "<", "<<"} +_SHELL_CONTROL_TOKENS = {";", "&&", "||", "|", ">", ">>", "<", "<<", "$(", "`"} _WRITE_TOOLS = {"write_file", "replace"} _READ_TOOLS = {"read_file", "glob", "list_directory", "grep_search"} +_SUBMIT_TIERS = tuple( + tier for tier in COGITATE_ACCESS_TIERS if capabilities_for_access_tier(tier).submit +) +_SUPPORT_SEND_VERBS = {"create", "reply", "attach", "feedback"} def _is_approved_journal_invocation(command: str) -> bool: @@ -42,10 +51,12 @@ class MaxTurnsExhausted(RuntimeError): class CogitatePolicy: """In-process policy gate for cogitate tool calls.""" - def __init__(self, *, allowed_roots: list[Path]) -> None: + def __init__(self, *, allowed_roots: list[Path], access_tier: str) -> None: self.allowed_roots = [ Path(root).expanduser().resolve() for root in allowed_roots ] + self.access_tier = access_tier + self.submit_allowed = capabilities_for_access_tier(access_tier).submit def check(self, tool: str, args: dict[str, Any]) -> tuple[bool, str]: if tool in _WRITE_TOOLS: @@ -53,15 +64,29 @@ class CogitatePolicy: if tool == "run_shell_command": command = str(args.get("command", "")) - if not ( - _SOL_INVOCATION_RE.search(command) - or _is_approved_journal_invocation(command) - ): + is_sol_invocation = _SOL_INVOCATION_RE.search(command) is not None + is_approved_journal_invocation = _is_approved_journal_invocation(command) + if not (is_sol_invocation or is_approved_journal_invocation): return ( False, "policy_deny: run_shell_command restricted to sol" " or approved journal invocations", ) + if is_sol_invocation and not self.submit_allowed: + send_verb = _support_send_verb(command) + if send_verb: + required = " or ".join(_SUBMIT_TIERS) + return ( + False, + f"policy_deny: 'sol call support {send_verb}' requires " + f"access_tier {required!r}; this run is {self.access_tier!r}", + ) + if _support_command_has_shell_control(command): + return ( + False, + "policy_deny: support commands may not be chained or wrapped " + "with shell-control operators (run them one at a time)", + ) return True, "ok" if tool in _READ_TOOLS: @@ -70,6 +95,31 @@ class CogitatePolicy: return True, "ok" +def _command_tokens(command: str) -> list[str]: + try: + return shlex.split(command) + except ValueError: + return command.split() + + +def _support_send_verb(command: str) -> str | None: + tokens = _command_tokens(command) + for index in range(len(tokens) - 3): + if tokens[index : index + 3] != ["sol", "call", "support"]: + continue + verb = tokens[index + 3] + if verb in _SUPPORT_SEND_VERBS: + return verb + return None + + +def _support_command_has_shell_control(command: str) -> bool: + lower_command = command.lower() + if "support" not in lower_command: + return False + return any(token in command for token in _SHELL_CONTROL_TOKENS) + + def _normalize_day(day: date | str) -> str: if isinstance(day, date): return day.strftime("%Y%m%d") diff --git a/solstone/think/providers/openhands.py b/solstone/think/providers/openhands.py index 126e7953e..d3918f652 100644 --- a/solstone/think/providers/openhands.py +++ b/solstone/think/providers/openhands.py @@ -24,6 +24,7 @@ from importlib import import_module from pathlib import Path from typing import Any +from solstone.think.cogitate_contract import capabilities_for_access_tier from solstone.think.cogitate_policy import ( DEFAULT_READ_CALL_BUDGET, MAX_TURNS, @@ -787,35 +788,39 @@ async def run_cogitate( sol_tool_name="sol", ) allowed_roots = _resolve_allowed_roots(config) - policy = CogitatePolicy(allowed_roots=allowed_roots) + access_tier = str(config.get("access_tier", "normal")) + caps = capabilities_for_access_tier(access_tier) + policy = CogitatePolicy(allowed_roots=allowed_roots, access_tier=access_tier) read_call_budget = int( config.get("read_call_budget", DEFAULT_READ_CALL_BUDGET) or 0 ) journal = Path(get_journal()) llm = _build_llm(provider, model) usage_start = _usage_snapshot(llm) - sol_tools, _executor = _build_sol_tools( - policy=policy, - callback=callback, - read_call_budget=read_call_budget, - ) - # openhands-sdk v1.23 resolves Agent.tools by spec name via the - # registry; passing ToolDefinition instances directly fails pydantic - # validation. Re-register the per-run SolTool instance (its executor - # closure captures this run's policy / callback / budget) and - # reference it by name. - register_tool("sol", sol_tools[0]) - tool_specs = [Tool(name="sol")] + tool_specs = [] + if caps.sol: + sol_tools, _executor = _build_sol_tools( + policy=policy, + callback=callback, + read_call_budget=read_call_budget, + ) + # openhands-sdk v1.23 resolves Agent.tools by spec name via the + # registry; passing ToolDefinition instances directly fails pydantic + # validation. Re-register the per-run SolTool instance (its executor + # closure captures this run's policy / callback / budget) and + # reference it by name. + register_tool("sol", sol_tools[0]) + tool_specs.append(Tool(name="sol")) from .read_tools import build_read_tools - # Per-tier gating is later; bounded reads register for every run today. - read_tools = build_read_tools( - journal=journal, - read_call_budget=read_call_budget, - ) - for read_tool in read_tools: - register_tool(read_tool.name, read_tool) - tool_specs.append(Tool(name=read_tool.name)) + if caps.reads: + read_tools = build_read_tools( + journal=journal, + read_call_budget=read_call_budget, + ) + for read_tool in read_tools: + register_tool(read_tool.name, read_tool) + tool_specs.append(Tool(name=read_tool.name)) default_tools = ["FinishTool"] if expects_emit_final: from .emit_final_tool import build_emit_final_tools diff --git a/solstone/think/talent.py b/solstone/think/talent.py index 2e679a2ef..f42ffd615 100644 --- a/solstone/think/talent.py +++ b/solstone/think/talent.py @@ -29,6 +29,7 @@ from typing import Any, Callable import frontmatter from jsonschema import Draft202012Validator, SchemaError +from solstone.think.cogitate_contract import COGITATE_ACCESS_TIERS from solstone.think.facets import get_facets # Import core prompt utilities from solstone.think.prompts @@ -85,6 +86,27 @@ def _validate_write(raw_write: Any, talent_type: Any, key: str) -> None: ) +def _validate_access_tier(raw: Any, talent_type: Any, key: str) -> str | None: + """Validate and normalize the optional cogitate access tier.""" + if talent_type == "cogitate": + if raw is None: + return "normal" + if raw in COGITATE_ACCESS_TIERS: + return raw + raise ValueError( + f"Prompt '{key}' has invalid 'access_tier' value '{raw}' " + f"(must be one of {COGITATE_ACCESS_TIERS})" + ) + + if raw is not None: + raise ValueError( + f"Prompt '{key}' sets 'access_tier' but access_tier is only valid " + "for type: cogitate" + ) + + return None + + def key_to_context(key: str) -> str: """Convert talent config key to context pattern. @@ -333,9 +355,16 @@ def get_talent_configs( f'(activity types to match, or ["*"] for all types).' ) - # Validate: cwd is only valid for cogitate prompts and defaults there + # Validate: cogitate-only runtime fields and defaults for key, info in configs.items(): _validate_write(info.get("write"), info.get("type"), key) + normalized_access_tier = _validate_access_tier( + info.get("access_tier"), info.get("type"), key + ) + if normalized_access_tier is None: + info.pop("access_tier", None) + else: + info["access_tier"] = normalized_access_tier normalized_cwd = _validate_cwd(info.get("cwd"), info.get("type"), key) if normalized_cwd is None: info.pop("cwd", None) @@ -595,6 +624,13 @@ def get_talent( post = frontmatter.load(md_path) config = dict(post.metadata) if post.metadata else {} _validate_write(config.get("write"), config.get("type"), name) + normalized_access_tier = _validate_access_tier( + config.get("access_tier"), config.get("type"), name + ) + if normalized_access_tier is None: + config.pop("access_tier", None) + else: + config["access_tier"] = normalized_access_tier normalized_cwd = _validate_cwd(config.get("cwd"), config.get("type"), name) if normalized_cwd is None: config.pop("cwd", None) diff --git a/tests/test_cogitate_contract.py b/tests/test_cogitate_contract.py index 86ba8bd84..a863cdc8d 100644 --- a/tests/test_cogitate_contract.py +++ b/tests/test_cogitate_contract.py @@ -3,6 +3,9 @@ from pathlib import Path +import pytest + +from solstone.think import cogitate_contract from solstone.think.cogitate_contract import ( COGITATE_ACCESS_TIERS, COGITATE_RUNTIME_PREAMBLE, @@ -68,6 +71,32 @@ def test_cogitate_vocabulary_lock(): assert "repair" not in FUTURE_ACCESS_TIERS +def test_access_tier_capability_mapping_matches_vocabulary(): + assert set(cogitate_contract._ACCESS_TIER_CAPABILITIES) == set( + COGITATE_ACCESS_TIERS + ) + + +@pytest.mark.parametrize( + ("tier", "expected"), + [ + ("normal", (True, True, False)), + ("system-read", (True, True, False)), + ("outbound", (True, True, True)), + ], +) +def test_capabilities_for_access_tier_real_tiers(tier, expected): + caps = cogitate_contract.capabilities_for_access_tier(tier) + + assert (caps.sol, caps.reads, caps.submit) == expected + + +@pytest.mark.parametrize("tier", ["repair", "code-agent", "bogus"]) +def test_capabilities_for_access_tier_unknown_names_tier(tier): + with pytest.raises(ValueError, match=tier): + cogitate_contract.capabilities_for_access_tier(tier) + + def test_cogitate_runtime_preamble_content_guard(): assert "sol call ..." in COGITATE_RUNTIME_PREAMBLE assert "journal root" in COGITATE_RUNTIME_PREAMBLE diff --git a/tests/test_cogitate_policy.py b/tests/test_cogitate_policy.py index ab2fe7938..cfeb46c8f 100644 --- a/tests/test_cogitate_policy.py +++ b/tests/test_cogitate_policy.py @@ -10,6 +10,12 @@ import pytest from solstone.think import cogitate_policy +def _policy(tmp_path, access_tier: str = "normal"): + return cogitate_policy.CogitatePolicy( + allowed_roots=[tmp_path], access_tier=access_tier + ) + + def test_resolve_read_scope_defaults_to_current_day_chronicle(): assert cogitate_policy.resolve_read_scope({}, "20260427") == ["chronicle/20260427"] @@ -29,7 +35,7 @@ def test_resolve_read_scope_span_is_inclusive(): def test_policy_denies_write_tools(tmp_path): - policy = cogitate_policy.CogitatePolicy(allowed_roots=[tmp_path]) + policy = _policy(tmp_path) allowed, reason = policy.check("write_file", {"file_path": "x"}) @@ -50,7 +56,7 @@ def test_policy_denies_write_tools(tmp_path): ], ) def test_policy_allows_approved_journal_invocations(tmp_path, command): - policy = cogitate_policy.CogitatePolicy(allowed_roots=[tmp_path]) + policy = _policy(tmp_path) allowed, reason = policy.check("run_shell_command", {"command": command}) @@ -70,7 +76,7 @@ def test_policy_allows_approved_journal_invocations(tmp_path, command): ], ) def test_policy_denies_unapproved_journal_invocations(tmp_path, command): - policy = cogitate_policy.CogitatePolicy(allowed_roots=[tmp_path]) + policy = _policy(tmp_path) allowed, reason = policy.check("run_shell_command", {"command": command}) @@ -78,6 +84,123 @@ def test_policy_denies_unapproved_journal_invocations(tmp_path, command): assert reason.startswith("policy_deny:") +@pytest.mark.parametrize("access_tier", ["normal", "system-read"]) +@pytest.mark.parametrize("verb", ["create", "reply", "attach", "feedback"]) +def test_policy_denies_support_send_verbs_without_submit_tier( + tmp_path, access_tier, verb +): + policy = _policy(tmp_path, access_tier) + + allowed, reason = policy.check( + "run_shell_command", + {"command": f"sol call support {verb} --subject x --description y"}, + ) + + assert allowed is False + assert reason.startswith("policy_deny:") + assert "outbound" in reason + assert access_tier in reason + + +@pytest.mark.parametrize("verb", ["create", "reply", "attach", "feedback"]) +def test_policy_allows_support_send_verbs_for_outbound(tmp_path, verb): + policy = _policy(tmp_path, "outbound") + + allowed, reason = policy.check( + "run_shell_command", + {"command": f"sol call support {verb} --subject x --description y"}, + ) + + assert allowed is True + assert reason == "ok" + + +@pytest.mark.parametrize("access_tier", ["normal", "system-read", "outbound"]) +@pytest.mark.parametrize( + "command", + [ + "sol call support register", + "sol call support search foo", + "sol call support article getting-started", + "sol call support list", + "sol call support show 42", + "sol call support announcements", + "sol call support diagnose", + ], +) +def test_policy_allows_support_read_verbs_for_all_tiers(tmp_path, access_tier, command): + policy = _policy(tmp_path, access_tier) + + allowed, reason = policy.check("run_shell_command", {"command": command}) + + assert allowed is True + assert reason == "ok" + + +def test_policy_denies_chained_support_send_for_normal(tmp_path): + policy = _policy(tmp_path, "normal") + + allowed, reason = policy.check( + "run_shell_command", + { + "command": ( + "sol call support search foo && sol call support create " + "--subject x --description y" + ) + }, + ) + + assert allowed is False + assert "requires access_tier" in reason + + +@pytest.mark.parametrize( + "command", + [ + "echo $(sol call support create --subject x)", + "sol call support search foo | grep bar", + ], +) +def test_policy_denies_wrapped_or_chained_support_command_for_normal(tmp_path, command): + policy = _policy(tmp_path, "normal") + + allowed, reason = policy.check("run_shell_command", {"command": command}) + + assert allowed is False + assert reason == ( + "policy_deny: support commands may not be chained or wrapped with " + "shell-control operators (run them one at a time)" + ) + + +def test_policy_allows_non_support_chain_for_normal(tmp_path): + policy = _policy(tmp_path, "normal") + + allowed, reason = policy.check( + "run_shell_command", + {"command": "sol call activities list && sol call todos list"}, + ) + + assert allowed is True + assert reason == "ok" + + +def test_policy_allows_chained_support_send_for_outbound(tmp_path): + policy = _policy(tmp_path, "outbound") + + allowed, reason = policy.check( + "run_shell_command", + { + "command": ( + "sol call support search foo && sol call support create --subject x" + ) + }, + ) + + assert allowed is True + assert reason == "ok" + + def test_cogitate_toml_removed_and_build_policy_import_fails(): # AC 19: TOML policy generation is removed. policy_path = ( diff --git a/tests/test_openhands_provider.py b/tests/test_openhands_provider.py index 555b818b2..7a9bc0017 100644 --- a/tests/test_openhands_provider.py +++ b/tests/test_openhands_provider.py @@ -9,11 +9,12 @@ from types import SimpleNamespace import pytest +from solstone.think.cogitate_contract import AccessCapabilities from solstone.think.cogitate_policy import MAX_TURNS, MaxTurnsExhausted from solstone.think.providers import openhands from solstone.think.providers.shared import USAGE_KEYS, JSONEventCallback -from solstone.think.talent import get_talent_configs -from tests.openhands_fakes import install_fake_openhands +from solstone.think.talent import get_talent, get_talent_configs +from tests.openhands_fakes import _REGISTERED_TOOLS, install_fake_openhands @pytest.fixture @@ -49,6 +50,24 @@ def _run_config(monkeypatch, tmp_path, **overrides): return config +def _real_talent_config(monkeypatch, tmp_path, name: str, **overrides): + config = get_talent(name) + config.update(_run_config(monkeypatch, tmp_path, **overrides)) + return config + + +def _run_and_capture_tool_state(fake_openhands, config: dict, events: list[dict]): + fake_openhands.Conversation.instances = [] + fake_openhands.Conversation.arun_impl = None + _REGISTERED_TOOLS.clear() + + result = asyncio.run(openhands.run_cogitate(config, events.append)) + conversation = fake_openhands.Conversation.instances[0] + agent_tool_names = {tool.name for tool in conversation.agent.tools} + registered_tool_names = set(_REGISTERED_TOOLS) + return result, conversation, agent_tool_names, registered_tool_names + + def _emit_final_action(fake_openhands, content: str): return fake_openhands.ActionEvent( reasoning_content=None, @@ -523,6 +542,104 @@ def test_run_cogitate_uses_emit_final_branch_for_daily_no_output( ] == ["error"] +def test_run_cogitate_skips_read_tool_registration_when_tier_caps_disable_reads( + fake_openhands, + monkeypatch, + tmp_path, +): + monkeypatch.setattr( + openhands, + "capabilities_for_access_tier", + lambda _tier: AccessCapabilities(sol=True, reads=False, submit=False), + ) + config = _run_config(monkeypatch, tmp_path, access_tier="normal") + events: list[dict] = [] + + _result, conversation, agent_tool_names, registered_tool_names = ( + _run_and_capture_tool_state(fake_openhands, config, events) + ) + + read_tool_names = {"read_file", "list_directory", "glob", "grep_search"} + assert "sol" in agent_tool_names + assert "sol" in registered_tool_names + assert agent_tool_names.isdisjoint(read_tool_names) + assert registered_tool_names.isdisjoint(read_tool_names) + assert conversation.agent.include_default_tools == ["FinishTool"] + + +@pytest.mark.parametrize( + ("name", "expected_access_tier", "expected_agent_tools", "expected_default_tools"), + [ + ( + "steward", + "system-read", + {"sol", "read_file", "list_directory", "glob", "grep_search", "emit_final"}, + [], + ), + ( + "support:support", + "outbound", + {"sol", "read_file", "list_directory", "glob", "grep_search"}, + ["FinishTool"], + ), + ( + "exec", + "outbound", + {"sol", "read_file", "list_directory", "glob", "grep_search"}, + ["FinishTool"], + ), + ( + "reflection", + "normal", + {"sol", "read_file", "list_directory", "glob", "grep_search"}, + ["FinishTool"], + ), + ], +) +def test_run_cogitate_real_talent_access_tiers_register_expected_tools( + fake_openhands, + monkeypatch, + tmp_path, + name, + expected_access_tier, + expected_agent_tools, + expected_default_tools, +): + config = _real_talent_config(monkeypatch, tmp_path, name) + events: list[dict] = [] + + _result, conversation, agent_tool_names, registered_tool_names = ( + _run_and_capture_tool_state(fake_openhands, config, events) + ) + + assert config["access_tier"] == expected_access_tier + assert agent_tool_names == expected_agent_tools + assert registered_tool_names == expected_agent_tools + assert conversation.agent.include_default_tools == expected_default_tools + + +def test_run_cogitate_exec_outbound_tool_surface_matches_normal_talent( + fake_openhands, + monkeypatch, + tmp_path, +): + exec_config = _real_talent_config(monkeypatch, tmp_path, "exec") + normal_config = _real_talent_config(monkeypatch, tmp_path, "reflection") + + exec_events: list[dict] = [] + _result, _conversation, exec_tool_names, _registered = _run_and_capture_tool_state( + fake_openhands, exec_config, exec_events + ) + normal_events: list[dict] = [] + _result, _conversation, normal_tool_names, _registered = ( + _run_and_capture_tool_state(fake_openhands, normal_config, normal_events) + ) + + assert exec_config["access_tier"] == "outbound" + assert normal_config["access_tier"] == "normal" + assert exec_tool_names == normal_tool_names + + def test_schedule_gated_cogitate_prompts_use_emit_final(): old_tool_name = "emit" + "_output" configs = get_talent_configs(type="cogitate") diff --git a/tests/test_openhands_sol_tool.py b/tests/test_openhands_sol_tool.py index 4cfbb6156..bf1633a0b 100644 --- a/tests/test_openhands_sol_tool.py +++ b/tests/test_openhands_sol_tool.py @@ -27,7 +27,7 @@ def _sol_tool_and_executor( events: list[dict], read_call_budget: int = 200, ): - policy = CogitatePolicy(allowed_roots=[tmp_path]) + policy = CogitatePolicy(allowed_roots=[tmp_path], access_tier="normal") tools, executor = openhands._build_sol_tools( policy=policy, callback=JSONEventCallback(events.append), diff --git a/tests/test_talent.py b/tests/test_talent.py index 03c90e99f..fef4f7e4e 100644 --- a/tests/test_talent.py +++ b/tests/test_talent.py @@ -10,8 +10,10 @@ import pytest import solstone.think.talent as talent_module from solstone.think.talent import ( + _validate_access_tier, _validate_cwd, get_talent, + get_talent_configs, get_talent_filter, source_is_enabled, source_is_required, @@ -105,6 +107,33 @@ def test_validate_cwd_rejects_invalid_value(): _validate_cwd("home", "cogitate", "test-agent") +def test_validate_access_tier_defaults_cogitate_to_normal(): + assert _validate_access_tier(None, "cogitate", "test-agent") == "normal" + + +def test_validate_access_tier_accepts_known_tier(): + assert _validate_access_tier("outbound", "cogitate", "test-agent") == "outbound" + + +def test_validate_access_tier_rejects_unknown_tier(): + with pytest.raises( + ValueError, + match="Prompt 'test-agent' has invalid 'access_tier' value 'repair'", + ): + _validate_access_tier("repair", "cogitate", "test-agent") + + +def test_validate_access_tier_rejects_generate_with_access_tier(): + with pytest.raises( + ValueError, + match=( + "Prompt 'test-agent' sets 'access_tier' but access_tier is only valid " + "for type: cogitate" + ), + ): + _validate_access_tier("normal", "generate", "test-agent") + + def test_get_agent_normalizes_cwd_for_cogitate(): config = get_talent("chat") assert config["type"] == "generate" @@ -126,6 +155,89 @@ def test_get_agent_rejects_cogitate_write_true(tmp_path, monkeypatch): get_talent("writer") +def test_get_talent_defaults_cogitate_access_tier_to_normal(tmp_path, monkeypatch): + monkeypatch.setattr(talent_module, "TALENT_DIR", tmp_path) + _write_talent_file(tmp_path, "agent", {"type": "cogitate"}) + + config = get_talent("agent") + + assert config["access_tier"] == "normal" + + +def test_get_talent_preserves_valid_access_tier(tmp_path, monkeypatch): + monkeypatch.setattr(talent_module, "TALENT_DIR", tmp_path) + _write_talent_file( + tmp_path, + "agent", + {"type": "cogitate", "access_tier": "system-read"}, + ) + + config = get_talent("agent") + + assert config["access_tier"] == "system-read" + + +def test_get_talent_rejects_invalid_access_tier(tmp_path, monkeypatch): + monkeypatch.setattr(talent_module, "TALENT_DIR", tmp_path) + _write_talent_file( + tmp_path, + "agent", + {"type": "cogitate", "access_tier": "code-agent"}, + ) + + with pytest.raises( + ValueError, + match="Prompt 'agent' has invalid 'access_tier' value 'code-agent'", + ): + get_talent("agent") + + +def test_get_talent_rejects_non_cogitate_access_tier(tmp_path, monkeypatch): + monkeypatch.setattr(talent_module, "TALENT_DIR", tmp_path) + _write_talent_file( + tmp_path, + "generator", + {"type": "generate", "output": "md", "access_tier": "normal"}, + ) + + with pytest.raises( + ValueError, + match=( + "Prompt 'generator' sets 'access_tier' but access_tier is only valid " + "for type: cogitate" + ), + ): + get_talent("generator") + + +def test_get_talent_configs_defaults_cogitate_access_tier_to_normal( + tmp_path, monkeypatch +): + monkeypatch.setattr(talent_module, "TALENT_DIR", tmp_path) + monkeypatch.setattr(talent_module, "APPS_DIR", tmp_path / "apps") + _write_talent_file(tmp_path, "agent", {"type": "cogitate"}) + + configs = get_talent_configs(include_disabled=True) + + assert configs["agent"]["access_tier"] == "normal" + + +def test_get_talent_configs_rejects_invalid_access_tier(tmp_path, monkeypatch): + monkeypatch.setattr(talent_module, "TALENT_DIR", tmp_path) + monkeypatch.setattr(talent_module, "APPS_DIR", tmp_path / "apps") + _write_talent_file( + tmp_path, + "agent", + {"type": "cogitate", "access_tier": "bogus"}, + ) + + with pytest.raises( + ValueError, + match="Prompt 'agent' has invalid 'access_tier' value 'bogus'", + ): + get_talent_configs(include_disabled=True) + + def test_get_talent_defaults_to_chat(): config = get_talent() assert config["name"] == "chat" -- 2.51.2