From fc74cb150d8798dbd21794c3a6845572bafdfbe4 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Tue, 16 Jun 2026 20:15:29 -0600 Subject: [PATCH] fix(chat): honest deterministic closer when an outbound support run fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the chat brain dispatches the outbound support talent and that run ends in a cogitate runtime failure (a talent_errored whose reason_code is one of DETERMINISTIC_FAILURE_REASON_CODES), the run never completed and nothing could have been sent. The closer now states that plainly — "nothing was sent" to solstone support — and offers to try again, instead of falling through to the read-framed generic "couldn't finish that lookup … rephrase the question?" copy that brushed the support trust rule. Selection is deterministic: (talent ∈ OUTBOUND_TALENTS) × (reason_code ∈ DETERMINISTIC_FAILURE_REASON_CODES), never model output. The terminating talent name and its reason_code are threaded from _on_cortex_error through _handle_talent_terminal_locked / _talent_terminal_trigger, persisted on the talent_errored chat event, and reconstructed in _trigger_from_stream_event so a process restart yields the same honest closer. Spawn-failure and watchdog-timeout talent errors carry no reason_code and keep the generic closer; loop_exhausted and all non-support behavior are unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- solstone/apps/chat/copy.py | 3 + solstone/convey/chat.py | 25 +++++++- solstone/convey/static/chat_copy.js | 1 + tests/test_chat_closer.py | 52 ++++++++++++++++ tests/test_chat_copy.py | 6 ++ tests/test_chat_runtime.py | 95 ++++++++++++++++++++++++++++- 6 files changed, 180 insertions(+), 2 deletions(-) diff --git a/solstone/apps/chat/copy.py b/solstone/apps/chat/copy.py index 4d9e27449..82c1e0afe 100644 --- a/solstone/apps/chat/copy.py +++ b/solstone/apps/chat/copy.py @@ -29,6 +29,9 @@ CHAT_CLOSER_LOOP_EXHAUSTED_PREFIX = "Here's what I have so far:" CHAT_CLOSER_DIFFERENT_ANGLE_SUFFIX = "Want me to try a different angle?" CHAT_CLOSER_TALENT_ERRORED_FORMAT = "I couldn't finish that lookup — {reason}. Want to try a different angle, or rephrase the question?" CHAT_CLOSER_TALENT_ERRORED_GENERIC = "I couldn't finish that lookup. Want to try a different angle, or rephrase the question?" +# Deterministic support-send-failure closer (backend-selected on an outbound +# talent_errored carrying a runtime-failure reason_code). Brand: "solstone support". +CHAT_CLOSER_SUPPORT_SEND_FAILED = "I couldn't finish reaching solstone support, so nothing was sent. Want me to try again?" # T2.4 — thinking summary surfaces (CPO LOCKED) CHAT_THINKING_EXPANDER_LABEL = "Show thinking" diff --git a/solstone/convey/chat.py b/solstone/convey/chat.py index 1f4660f2b..59ff45b51 100644 --- a/solstone/convey/chat.py +++ b/solstone/convey/chat.py @@ -25,6 +25,7 @@ from flask import Blueprint, jsonify, request from solstone.apps.chat.copy import ( CHAT_CLOSER_DIFFERENT_ANGLE_SUFFIX, CHAT_CLOSER_LOOP_EXHAUSTED_PREFIX, + CHAT_CLOSER_SUPPORT_SEND_FAILED, CHAT_CLOSER_TALENT_ERRORED_FORMAT, CHAT_CLOSER_TALENT_ERRORED_GENERIC, CHAT_OFFER_SUPPORT_DECLINE, @@ -51,6 +52,7 @@ from solstone.convey.sol_initiated import ( from solstone.convey.sol_initiated.copy import KIND_SOL_CHAT_REQUEST, SURFACE_CONVEY from solstone.convey.utils import error_response from solstone.think.callosum import CallosumConnection, callosum_send +from solstone.think.cogitate_policy import DETERMINISTIC_FAILURE_REASON_CODES from solstone.think.cortex_client import CortexSpawnUnavailable from solstone.think.utils import get_journal, now_ms @@ -470,7 +472,9 @@ def _on_cortex_finish(message: dict[str, Any]) -> None: message_text = _compose_terminal_closer( exit_mode, message_text, + talent_name=trigger.get("name"), talent_errored_reason=trigger.get("reason"), + talent_errored_reason_code=trigger.get("reason_code"), talent_finished_summary=trigger.get("summary"), ) requested_target = None @@ -630,12 +634,14 @@ def _on_cortex_error(message: dict[str, Any]) -> None: next_info = _clear_current_locked() elif use_id in _active_talents: reason = str(message.get("error") or "unknown") + reason_code = message.get("reason_code") or None _evict_thinking_locked(use_id) next_info = _handle_talent_terminal_locked( use_id, "talent_errored", "reason", reason, + reason_code=reason_code, ) elif _is_superseded_raw_use_id_locked(use_id): logger.debug( @@ -669,6 +675,7 @@ def _handle_talent_terminal_locked( result_field_name: str, result_value: str, *, + reason_code: str | None = None, terminal_message: dict[str, Any] | None = None, ) -> dict[str, Any] | None: _cancel_watchdog_locked(use_id) @@ -681,12 +688,15 @@ def _handle_talent_terminal_locked( talent_name, result_field_name, result_value, + reason_code=reason_code, ) event_fields: dict[str, Any] = { "use_id": use_id, "name": talent_name, result_field_name: result_value, } + if reason_code: + event_fields["reason_code"] = reason_code if kind == "talent_finished" and terminal_message is not None: thinking = _drain_thinking_locked(use_id, terminal_message) if thinking is not None: @@ -1373,7 +1383,9 @@ def _compose_terminal_closer( exit_mode: str, raw_message: str | None, *, + talent_name: str | None = None, talent_errored_reason: str | None = None, + talent_errored_reason_code: str | None = None, talent_finished_summary: str | None = None, ) -> str: if exit_mode == "loop_exhausted": @@ -1391,6 +1403,11 @@ def _compose_terminal_closer( ) if exit_mode == "talent_errored": + if ( + talent_name in OUTBOUND_TALENTS + and talent_errored_reason_code in DETERMINISTIC_FAILURE_REASON_CODES + ): + return CHAT_CLOSER_SUPPORT_SEND_FAILED reason = _clean_talent_errored_reason(talent_errored_reason) if reason: return CHAT_CLOSER_TALENT_ERRORED_FORMAT.format(reason=reason) @@ -1601,6 +1618,7 @@ def _trigger_from_stream_event(event: dict[str, Any]) -> dict[str, Any]: event.get("name", "exec"), "reason", event.get("reason", ""), + reason_code=event.get("reason_code"), ) raise ValueError(f"unsupported trigger event: {kind}") @@ -1611,13 +1629,18 @@ def _talent_terminal_trigger( name: Any, result_field_name: str, result_value: Any, + *, + reason_code: str | None = None, ) -> dict[str, Any]: - return { + trigger = { "type": kind, "use_id": use_id, "name": name, result_field_name: result_value, } + if reason_code: + trigger["reason_code"] = reason_code + return trigger def _read_talent_log(use_id: str) -> dict[str, Any] | None: diff --git a/solstone/convey/static/chat_copy.js b/solstone/convey/static/chat_copy.js index 812fa390d..110ce74ec 100644 --- a/solstone/convey/static/chat_copy.js +++ b/solstone/convey/static/chat_copy.js @@ -46,6 +46,7 @@ CHAT_CLOSER_DIFFERENT_ANGLE_SUFFIX: "Want me to try a different angle?", CHAT_CLOSER_TALENT_ERRORED_FORMAT: "I couldn't finish that lookup — {reason}. Want to try a different angle, or rephrase the question?", CHAT_CLOSER_TALENT_ERRORED_GENERIC: "I couldn't finish that lookup. Want to try a different angle, or rephrase the question?", + CHAT_CLOSER_SUPPORT_SEND_FAILED: "I couldn't finish reaching solstone support, so nothing was sent. Want me to try again?", CHAT_THINKING_EXPANDER_LABEL: "Show thinking", CHAT_THINKING_COLLAPSER_LABEL: "Hide thinking", CHAT_ERROR_DETAIL_EXPANDER_LABEL: "Show details", diff --git a/tests/test_chat_closer.py b/tests/test_chat_closer.py index 832f693e3..fd372cd9f 100644 --- a/tests/test_chat_closer.py +++ b/tests/test_chat_closer.py @@ -3,7 +3,9 @@ import logging +from solstone.apps.chat.copy import CHAT_CLOSER_SUPPORT_SEND_FAILED from solstone.convey import chat +from solstone.think.cogitate_policy import DETERMINISTIC_FAILURE_REASON_CODES CHAT_LOGGER = "solstone.convey.chat" FIXTURE_OPENERS = ( @@ -177,6 +179,56 @@ def test_talent_errored_reason_framing(): ) +def test_support_talent_errored_send_failed_closer(): + model_text = ( + "" + ) + for reason_code in DETERMINISTIC_FAILURE_REASON_CODES: + assert ( + chat._compose_terminal_closer( + "talent_errored", + model_text, + talent_name="support", + talent_errored_reason="Traceback (most recent call last)", + talent_errored_reason_code=reason_code, + ) + == CHAT_CLOSER_SUPPORT_SEND_FAILED + ) + + support_unknown = chat._compose_terminal_closer( + "talent_errored", + model_text, + talent_name="support", + talent_errored_reason="provider gave up", + talent_errored_reason_code=None, + ) + assert support_unknown != CHAT_CLOSER_SUPPORT_SEND_FAILED + assert ( + support_unknown + == "I couldn't finish that lookup — provider gave up. Want to try a different angle, or rephrase the question?" + ) + + support_unrecognized = chat._compose_terminal_closer( + "talent_errored", + model_text, + talent_name="support", + talent_errored_reason="provider gave up", + talent_errored_reason_code="some_other_code", + ) + assert support_unrecognized != CHAT_CLOSER_SUPPORT_SEND_FAILED + assert support_unrecognized == support_unknown + + non_support = chat._compose_terminal_closer( + "talent_errored", + model_text, + talent_name="exec", + talent_errored_reason="provider gave up", + talent_errored_reason_code="wall_clock_exceeded", + ) + assert non_support != CHAT_CLOSER_SUPPORT_SEND_FAILED + assert non_support == support_unknown + + def test_loop_exhausted_empty_empty_fallback(): assert ( chat._compose_terminal_closer( diff --git a/tests/test_chat_copy.py b/tests/test_chat_copy.py index 4adc64ae0..ae88e2cf0 100644 --- a/tests/test_chat_copy.py +++ b/tests/test_chat_copy.py @@ -162,6 +162,7 @@ def test_closer_constants_byte_parity(): "CHAT_CLOSER_DIFFERENT_ANGLE_SUFFIX": "Want me to try a different angle?", "CHAT_CLOSER_TALENT_ERRORED_FORMAT": "I couldn't finish that lookup — {reason}. Want to try a different angle, or rephrase the question?", "CHAT_CLOSER_TALENT_ERRORED_GENERIC": "I couldn't finish that lookup. Want to try a different angle, or rephrase the question?", + "CHAT_CLOSER_SUPPORT_SEND_FAILED": "I couldn't finish reaching solstone support, so nothing was sent. Want me to try again?", } for name, literal in expected.items(): @@ -169,6 +170,11 @@ def test_closer_constants_byte_parity(): assert literal in text assert "\u2014" in chat_copy.CHAT_CLOSER_TALENT_ERRORED_FORMAT + assert "solstone support" in chat_copy.CHAT_CLOSER_SUPPORT_SEND_FAILED + assert "sol pbc" not in chat_copy.CHAT_CLOSER_SUPPORT_SEND_FAILED + assert "live chat" not in chat_copy.CHAT_CLOSER_SUPPORT_SEND_FAILED + assert "lookup" not in chat_copy.CHAT_CLOSER_SUPPORT_SEND_FAILED + assert "try again" in chat_copy.CHAT_CLOSER_SUPPORT_SEND_FAILED.lower() def test_chat_placeholder_css_present(): diff --git a/tests/test_chat_runtime.py b/tests/test_chat_runtime.py index baa27ad8b..6eeed6512 100644 --- a/tests/test_chat_runtime.py +++ b/tests/test_chat_runtime.py @@ -10,6 +10,7 @@ from datetime import datetime import pytest from flask import Flask +from solstone.apps.chat.copy import CHAT_CLOSER_SUPPORT_SEND_FAILED from solstone.convey.chat_stream import append_chat_event, read_chat_events @@ -311,6 +312,73 @@ def test_post_talent_errored_request_is_forced_terminal(tmp_path, monkeypatch): assert chat._current_chat_use_id is None +def test_post_support_talent_errored_request_uses_send_failed_closer( + tmp_path, monkeypatch +): + import solstone.convey.chat as chat + + _setup_journal(tmp_path, monkeypatch) + _reset_chat_state(chat) + + actions: list[dict | None] = [] + finishes: list[tuple[str, str]] = [] + monkeypatch.setattr( + "solstone.convey.chat._run_next_action", lambda action: actions.append(action) + ) + monkeypatch.setattr( + "solstone.convey.chat._emit_finish", + lambda use_id, message: finishes.append((use_id, message)), + ) + monkeypatch.setattr( + "solstone.convey.chat._emit_error", + lambda *args, **kwargs: None, + ) + + with chat._state_lock: + chat._current_chat_use_id = "1713622150000" + chat._current_chat_state = { + "raw_use_id": "1713622150001", + "raw_use_ids_seen": {"1713622150001"}, + "trigger": { + "type": "talent_errored", + "name": "support", + "reason": "Traceback (most recent call last)", + "reason_code": "wall_clock_exceeded", + }, + "location": {"app": "sol", "path": "/app/sol", "facet": "work"}, + "retry_count": 0, + } + + chat._on_cortex_finish( + { + "use_id": "1713622150001", + "result": json.dumps( + { + "message": "I drafted a ticket and will file it via live chat.", + "notes": "blocked redispatch", + "talent_request": { + "target": "exec", + "task": "one more pass", + "context": json.dumps({}), + }, + } + ), + } + ) + + assert actions == [None] + events = read_chat_events(chat._today_day()) + sol_messages = [event for event in events if event["kind"] == "sol_message"] + assert len(sol_messages) == 1 + assert sol_messages[-1]["text"] == CHAT_CLOSER_SUPPORT_SEND_FAILED + assert sol_messages[-1]["requested_target"] is None + assert sol_messages[-1]["requested_task"] is None + assert finishes == [("1713622150000", CHAT_CLOSER_SUPPORT_SEND_FAILED)] + with chat._state_lock: + assert chat._current_chat_state is None + assert chat._current_chat_use_id is None + + def test_owner_message_direct_reply_keeps_raw_model_text(tmp_path, monkeypatch): import solstone.convey.chat as chat @@ -536,13 +604,38 @@ def test_cortex_finish_and_error_append_exec_terminal_events_by_use_id( "location": {"app": "sol", "path": "/app/sol", "facet": "work"}, } - chat._on_cortex_error({"use_id": "1713624000001", "error": "boom"}) + chat._on_cortex_error( + { + "use_id": "1713624000001", + "error": "boom", + "reason_code": "wall_clock_exceeded", + } + ) errored_events = [ e for e in read_chat_events(chat._today_day()) if e["kind"] == "talent_errored" ] assert errored_events[-1]["use_id"] == "1713624000001" assert actions[-1]["trigger"]["type"] == "talent_errored" assert actions[-1]["trigger"]["reason"] == "boom" + assert actions[-1]["trigger"]["reason_code"] == "wall_clock_exceeded" + + +def test_talent_errored_trigger_recovers_reason_code(): + import solstone.convey.chat as chat + + trigger = chat._trigger_from_stream_event( + { + "kind": "talent_errored", + "use_id": "1713624500001", + "name": "support", + "reason": "Traceback (most recent call last)", + "reason_code": "wall_clock_exceeded", + } + ) + + assert trigger["type"] == "talent_errored" + assert trigger["name"] == "support" + assert trigger["reason_code"] == "wall_clock_exceeded" @pytest.mark.parametrize( -- 2.51.2