diff --git a/solstone/apps/sol/tests/test_routes.py b/solstone/apps/sol/tests/test_routes.py index cac9df00f..8f7c07df2 100644 --- a/solstone/apps/sol/tests/test_routes.py +++ b/solstone/apps/sol/tests/test_routes.py @@ -152,6 +152,56 @@ def test_api_index_is_read_only(sol_env): assert _talents_snapshot(sol_env.journal) == before +def test_api_run_omits_non_responsive_raw_output(sol_env, monkeypatch): + use_id = "1700000000011" + raw_output = "I cannot describe this screen." + talent_dir = sol_env.journal / "talents" / "default" + talent_dir.mkdir(parents=True) + (talent_dir / f"{use_id}.jsonl").write_text( + "\n".join( + json.dumps(event) + for event in [ + { + "event": "request", + "ts": 1700000000011, + "use_id": use_id, + "name": "default", + "prompt": "Describe the screen", + "provider": "openai", + "day": "20260304", + }, + { + "event": "error", + "ts": 1700000000200, + "use_id": use_id, + "error": "The requested work was not completed.", + "reason_code": "non_responsive", + "terminal": True, + "raw": [ + { + "reason_code": "non_responsive", + "non_responsive_output": raw_output, + } + ], + }, + ] + ) + + "\n", + encoding="utf-8", + ) + monkeypatch.setattr( + "solstone.think.cortex_client.get_use_end_state", + lambda _use_id: "error", + ) + + response = sol_env.client.get(f"/app/sol/api/run/{use_id}") + + assert response.status_code == 200 + serialized = json.dumps(response.get_json()) + assert raw_output not in serialized + assert '"raw"' not in serialized + + def test_segment_summary_talent_meta_uses_declared_copy(): from solstone.apps.sol import routes as sol_routes diff --git a/tests/test_convey_chat.py b/tests/test_convey_chat.py index aed7270c7..caced272f 100644 --- a/tests/test_convey_chat.py +++ b/tests/test_convey_chat.py @@ -2770,6 +2770,47 @@ def test_talent_log_endpoint_returns_completed_run(chat_client, tmp_path): assert "raw" not in payload["events"][1] +def test_talent_log_endpoint_omits_non_responsive_raw_output(chat_client, tmp_path): + use_id = "1700000000010" + raw_output = "I cannot describe this screen." + _write_talent_log( + tmp_path / "journal", + "default", + f"{use_id}.jsonl", + [ + { + "event": "request", + "ts": 1700000000010, + "use_id": use_id, + "prompt": "Describe the screen", + "name": "default", + "provider": "openai", + }, + { + "event": "error", + "ts": 1700000000100, + "use_id": use_id, + "error": "The requested work was not completed.", + "reason_code": "non_responsive", + "terminal": True, + "raw": [ + { + "reason_code": "non_responsive", + "non_responsive_output": raw_output, + } + ], + }, + ], + ) + + response = chat_client.get(f"/api/chat/talent-log/{use_id}") + + assert response.status_code == 200 + serialized = json.dumps(response.get_json()) + assert raw_output not in serialized + assert '"raw"' not in serialized + + def test_talent_log_endpoint_returns_running_active_run(chat_client, tmp_path): use_id = "1700000000002" _write_talent_log( diff --git a/tests/test_importer_documents.py b/tests/test_importer_documents.py index e5e66ff0b..3115cc42d 100644 --- a/tests/test_importer_documents.py +++ b/tests/test_importer_documents.py @@ -11,6 +11,8 @@ import os import shutil import time from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock from solstone.observe.pdf_worker import ( PdfWorkerEncryptedError, @@ -19,6 +21,7 @@ from solstone.observe.pdf_worker import ( ) from solstone.think.importers.file_importer import FILE_IMPORTER_REGISTRY from solstone.think.models import NoBrainConfiguredError +from solstone.think.responsiveness import NON_RESPONSIVE_OUTPUT_MESSAGE from tests.pdf_worker_fixtures import ( IMAGE_TEXT_SENTINEL, MIXED_TEXT_SENTINEL, @@ -363,6 +366,44 @@ def test_vision_extraction_failure_marks_one_scanned_page_unavailable( assert (segment_dir / "pages" / "page-0002.png").is_file() +def test_non_responsive_document_description_uses_unavailable_marker_without_raw_refusal( + tmp_path, monkeypatch +): + import solstone.think.providers as providers_package + from solstone.think import models + + mod = _mod() + pdf = write_image_only_fixture(tmp_path / "scan.pdf") + _fixed_mtime(pdf) + provider_module = SimpleNamespace( + run_generate=MagicMock( + return_value={ + "text": "I cannot describe this screen.", + "model": "provider-model", + "finish_reason": "stop", + } + ) + ) + monkeypatch.setattr( + models, + "resolve_provider", + lambda _interface: ("fake", "provider-model"), + ) + monkeypatch.setattr( + providers_package, + "get_provider_module", + lambda _provider: provider_module, + ) + + result = _import_pdf(mod, pdf, tmp_path) + + transcript = _transcript(tmp_path, result) + assert "I cannot describe this screen." not in transcript + assert NON_RESPONSIVE_OUTPUT_MESSAGE in transcript + assert "2 model-extracted, 0 unavailable" not in transcript + assert "0 model-extracted, 2 unavailable" in transcript + + def test_description_failure_keeps_full_text_and_uses_description_marker( tmp_path, monkeypatch ): diff --git a/tests/test_importer_images.py b/tests/test_importer_images.py index 93e6bb442..2aeeb743f 100644 --- a/tests/test_importer_images.py +++ b/tests/test_importer_images.py @@ -4,6 +4,8 @@ import datetime as dt import os from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock import pytest from PIL import Image @@ -13,6 +15,7 @@ from solstone.think.importers.file_importer import ( FILE_IMPORTER_REGISTRY, get_file_importer, ) +from solstone.think.responsiveness import NonResponsiveOutputError def _configure_journal(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -163,6 +166,42 @@ def test_process_vision_failure_propagates_before_success_entry(tmp_path, monkey assert not list((tmp_path / "chronicle").glob("**/import.image")) +def test_non_responsive_image_import_leaves_no_artifacts(tmp_path, monkeypatch): + import solstone.think.providers as providers_package + from solstone.think import models + + mod = __import__("solstone.think.importers.images", fromlist=["importer"]) + _configure_journal(tmp_path, monkeypatch) + image_path = tmp_path / "shot.png" + _write_png(image_path) + import_id = "20260115_120000" + provider_module = SimpleNamespace( + run_generate=MagicMock( + return_value={ + "text": "I cannot describe this screen.", + "model": "provider-model", + "finish_reason": "stop", + } + ) + ) + monkeypatch.setattr( + models, + "resolve_provider", + lambda _interface: ("fake", "provider-model"), + ) + monkeypatch.setattr( + providers_package, + "get_provider_module", + lambda _provider: provider_module, + ) + + with pytest.raises(NonResponsiveOutputError): + mod.importer.process(image_path, tmp_path, import_id=import_id) + + assert not list((tmp_path / "chronicle").glob("**/import.image")) + assert not (tmp_path / "imports" / import_id / "content_manifest.jsonl").exists() + + def test_process_real_encoder_failure_leaves_no_image_artifacts(tmp_path, monkeypatch): mod = __import__("solstone.think.importers.images", fromlist=["importer"]) _configure_journal(tmp_path, monkeypatch) diff --git a/tests/test_responsiveness_wiring.py b/tests/test_responsiveness_wiring.py new file mode 100644 index 000000000..f6e549e5d --- /dev/null +++ b/tests/test_responsiveness_wiring.py @@ -0,0 +1,237 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +import ast +import asyncio +import io +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from PIL import Image + +import solstone.think.models as models_module +from solstone.think.responsiveness import ( + NON_RESPONSIVE_OUTPUT_MESSAGE, + NON_RESPONSIVE_REASON_CODE, + NonResponsiveOutputError, +) + +_REFUSAL = "I cannot describe this screen." +_USAGE = { + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, +} + + +def _png_bytes() -> bytes: + image_bytes = io.BytesIO() + Image.new("RGB", (8, 8), "white").save(image_bytes, format="PNG") + return image_bytes.getvalue() + + +def _describe_frame(frame_id: int, frame_bytes: bytes) -> dict: + return { + "frame_id": frame_id, + "timestamp": float(frame_id), + "frame_bytes": frame_bytes, + "aruco": None, + } + + +def _describe_processor(video_path: Path, frames: list[dict], monkeypatch) -> object: + from solstone.observe import describe as describe_module + + processor = describe_module.VideoProcessor.__new__(describe_module.VideoProcessor) + processor.video_path = video_path + processor.first_hash = None + processor.last_hash = None + processor.qualified_count = len(frames) + processor.qualified_frames = [] + monkeypatch.setattr(processor, "process", lambda: frames) + return processor + + +def _declining_result() -> dict: + return { + "text": _REFUSAL, + "model": "provider-model", + "finish_reason": "stop", + "usage": dict(_USAGE), + } + + +def _install_declining_provider(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace: + import solstone.think.providers as providers_package + from solstone.think import batch as batch_module + + provider_module = SimpleNamespace( + run_generate=MagicMock(return_value=_declining_result()), + run_agenerate=AsyncMock(return_value=_declining_result()), + ) + monkeypatch.setattr( + models_module, + "resolve_provider", + lambda _interface: ("fake", "provider-model"), + ) + monkeypatch.setattr( + providers_package, + "get_provider_module", + lambda _provider: provider_module, + ) + monkeypatch.setattr( + batch_module, + "resolve_provider", + lambda _interface: ("fake", "provider-model"), + ) + return provider_module + + +def _collect_provider_interface_call_sites() -> set[tuple[str, str, str]]: + root_paths = [Path("solstone"), Path("core"), Path("packages"), Path("observers")] + call_sites: set[tuple[str, str, str]] = set() + for root in root_paths: + if not root.exists(): + continue + for path in root.rglob("*.py"): + if "tests" in path.parts: + continue + tree = ast.parse(path.read_text(encoding="utf-8")) + parents: dict[ast.AST, ast.AST] = {} + for node in ast.walk(tree): + for child in ast.iter_child_nodes(node): + parents[child] = node + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + call_name = None + if isinstance(func, ast.Attribute): + call_name = func.attr + elif isinstance(func, ast.Name): + call_name = func.id + if call_name is None or not call_name.endswith( + ("run_generate", "run_agenerate") + ): + continue + owner = "module" + current = node + while current in parents: + current = parents[current] + if isinstance(current, ast.FunctionDef | ast.AsyncFunctionDef): + owner = current.name + break + if path == Path("solstone/think/models.py"): + continue + if owner in {"run_generate", "run_agenerate"}: + continue + call_sites.add((str(path), owner, call_name)) + return call_sites + + +def test_generate_provider_calls_outside_gate_are_only_known_probes(): + assert _collect_provider_interface_call_sites() == { + ("solstone/think/providers/openhands.py", "_probe", "_run_generate"), + ("solstone/think/providers/local.py", "validate_key", "run_generate"), + } + + +def test_no_second_non_responsive_signal_table(): + identifiers = [ + "_NON_RESPONSIVE_" + suffix for suffix in ("NEGATION_HEADS", "LEAD_INS") + ] + matches: set[Path] = set() + for path in Path("solstone").rglob("*.py"): + text = path.read_text(encoding="utf-8") + if any(identifier in text for identifier in identifiers): + matches.add(path) + + assert matches == {Path("solstone/think/responsiveness.py")} + + +def test_no_per_path_guard_non_responsive_property(tmp_path, monkeypatch): + _install_declining_provider(monkeypatch) + + import solstone.think.providers as providers_package + from solstone.observe import describe as describe_module + from solstone.think import talents + from solstone.think.importers import documents, images + + provider_module = providers_package.get_provider_module("fake") + + with pytest.raises(NonResponsiveOutputError): + images._describe_image(Image.new("RGB", (8, 8), "red")) + + raster_path = tmp_path / "page.png" + Image.new("RGB", (8, 8), "white").save(raster_path) + document_outcome = documents._generate_for_page( + prompt="describe this page", + raster_path=raster_path, + context="import.document.describe", + stats=documents._RenderStats(), + ) + assert document_outcome.text is None + assert document_outcome.reason + + monkeypatch.setattr(talents, "_read_runtime_fingerprint", lambda: None) + events: list[dict] = [] + config = { + "name": "responsiveness-property", + "output": "md", + "output_path": str(tmp_path / "talent.md"), + "prompt": "describe the screen", + } + asyncio.run(talents._execute_generate(config, events.append)) + assert not (tmp_path / "talent.md").exists() + assert [event.get("event") for event in events] == ["error"] + assert events[0]["reason_code"] == NON_RESPONSIVE_REASON_CODE + + provider_module.run_agenerate = AsyncMock( + side_effect=[ + _declining_result(), + { + "text": ( + '{"visual_description":"A code editor is open.",' + '"primary":"code","secondary":"none","overlap":true}' + ), + "model": "provider-model", + "finish_reason": "stop", + }, + ] + ) + video_path = ( + tmp_path / "chronicle" / "20250101" / "default" / "143022_300" / "screen.webm" + ) + video_path.parent.mkdir(parents=True) + video_path.write_text("video", encoding="utf-8") + frame_bytes = _png_bytes() + processor = _describe_processor( + video_path, + [_describe_frame(1, frame_bytes), _describe_frame(2, frame_bytes)], + monkeypatch, + ) + monkeypatch.setattr(describe_module, "callosum_send", lambda *args, **kwargs: True) + monkeypatch.setattr( + describe_module, + "select_frames_for_extraction", + lambda *_args, **_kwargs: [], + ) + output_path = video_path.with_suffix(".jsonl") + + asyncio.run( + processor.process_with_vision( + max_concurrent=1, + output_path=output_path, + work_key="20250101/143022_300/screen", + ) + ) + + rows = [ + line for line in output_path.read_text(encoding="utf-8").splitlines() if line + ] + assert provider_module.run_agenerate.await_count == 2 + assert any(NON_RESPONSIVE_OUTPUT_MESSAGE in row for row in rows[1:])