From 655d4318272a2379e3efe9647f5a59487ca60ea3 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Fri, 3 Jul 2026 21:12:06 -0600 Subject: [PATCH] fix(agents): honest outcomes for empty completions, schema violations, and hook failures - Gemini generate path returns "" on every empty completion (all finish reasons), matching Anthropic/OpenAI; deleted _format_completion_message + its 7 tests. No more fabricated "Completed." prose landing as model output. - generate/agenerate raise SchemaValidationError(ValueError) on schema-invalid (and now-unified empty) completions; every json_schema caller family degrades observably (None/[]/fallback/RuntimeError/req.error). generate_with_result unchanged (still non-raising; talent executor contract preserved). - detect_created feeds an explicit "exiftool metadata unavailable" sentinel to the prompt on exiftool failure instead of the exception string. - A raised pre/post talent hook yields a terminal hook_error outcome (not a clean completed): no day-log "ok" line, zero provider fallback on the cogitate path, error names hook + talent + underlying exception. Providers re-raise TalentHookError past their generic error handler so exactly one terminal error is emitted. Unified _terminal_error_evented gate also fixes the pre-existing no_output "ok" mislog. Pulse/steward self-caught skip_reason still records a clean skip. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../apps/timeline/tests/test_rollup_day.py | 29 ++++- .../apps/timeline/tests/test_rollup_master.py | 28 ++++- solstone/observe/transcribe/gemini.py | 26 ++-- solstone/think/detect_created.py | 57 +++++---- solstone/think/detect_transcript.py | 28 ++--- solstone/think/models.py | 27 ++++- solstone/think/providers/google.py | 51 +------- solstone/think/providers/local.py | 5 + solstone/think/providers/openhands.py | 5 + solstone/think/talents.py | 67 ++++++++++- tests/test_anthropic.py | 22 ++++ tests/test_batch.py | 27 ++++- tests/test_detect_created_schema.py | 41 +++++++ tests/test_detect_transcript_schema.py | 24 ++++ tests/test_enrich_schema.py | 22 ++++ tests/test_extract_schema.py | 22 ++++ tests/test_generate_full.py | 111 ++++++++++++++++++ tests/test_google.py | 108 +++++++++++------ tests/test_local.py | 34 ++++++ tests/test_models.py | 110 +++++++++++++++++ tests/test_openhands_errors.py | 26 ++++ tests/test_output_hooks.py | 67 ++++++++++- tests/test_talent_fallback.py | 86 +++++++++++++- tests/test_transcribe_gemini_schema.py | 27 +++++ 24 files changed, 897 insertions(+), 153 deletions(-) diff --git a/solstone/apps/timeline/tests/test_rollup_day.py b/solstone/apps/timeline/tests/test_rollup_day.py index 10668de8f..c62fc610a 100644 --- a/solstone/apps/timeline/tests/test_rollup_day.py +++ b/solstone/apps/timeline/tests/test_rollup_day.py @@ -10,7 +10,7 @@ import json from solstone.apps.timeline.maintenance import _rollup_day, run_rollup_day from solstone.apps.timeline.tests.conftest import write_json -from solstone.think.models import GEMINI_FLASH, GEMINI_LITE +from solstone.think.models import GEMINI_FLASH, GEMINI_LITE, SchemaValidationError DAY = "20260512" @@ -172,6 +172,33 @@ def test_rollup_day_hour_error_continues_picks_empty_with_error_field( assert len(payload["day_top"]) == 4 +def test_rollup_day_schema_error_continues_picks_empty_with_error_field( + timeline_journal, + mock_agenerate, +): + for i in range(5): + _write_segment(timeline_journal, DAY, f"12000{i}_60", f"Noon {i}") + _write_segment(timeline_journal, DAY, f"13000{i}_60", f"One {i}") + mock_agenerate( + SchemaValidationError( + [{"path": "", "constraint": "json_parse", "message": "empty"}], + "", + ), + {"picks": [0, 1, 2, 3], "rationale": "one pm"}, + ) + + asyncio.run( + _rollup_day(timeline_journal, DAY, top=4, jobs=5, dry_run=False, force=False) + ) + + payload = json.loads( + (timeline_journal / "chronicle" / DAY / "timeline.json").read_text() + ) + assert payload["hours"]["12"]["picks"] == [] + assert "schema validation" in payload["hours"]["12"]["error"] + assert len(payload["day_top"]) == 4 + + def test_rollup_day_final_error_skips_write_exits_zero( timeline_journal, mock_agenerate ): diff --git a/solstone/apps/timeline/tests/test_rollup_master.py b/solstone/apps/timeline/tests/test_rollup_master.py index 7e3f7726e..1eaf30852 100644 --- a/solstone/apps/timeline/tests/test_rollup_master.py +++ b/solstone/apps/timeline/tests/test_rollup_master.py @@ -12,7 +12,7 @@ import pytest from solstone.apps.timeline.maintenance import _rollup_master, run_rollup_master from solstone.apps.timeline.tests.conftest import write_json -from solstone.think.models import GEMINI_FLASH +from solstone.think.models import GEMINI_FLASH, SchemaValidationError def _write_day(journal, day, titles): @@ -230,3 +230,29 @@ def test_rollup_master_month_error_nonfatal(timeline_journal, mock_agenerate): assert payload["months"]["202605"]["month_top"] == [] assert payload["months"]["202605"]["month_rationale"] == "ERROR: month backend down" assert payload["year_top"] == [] + + +def test_rollup_master_schema_error_nonfatal(timeline_journal, mock_agenerate): + _write_day(timeline_journal, "20260510", ["A", "B", "C", "D", "E"]) + mock_agenerate( + SchemaValidationError( + [{"path": "", "constraint": "json_parse", "message": "empty"}], + "", + ) + ) + + asyncio.run( + _rollup_master( + timeline_journal, + top=4, + jobs=5, + dry_run=False, + force=False, + months_filter=None, + ) + ) + + payload = json.loads((timeline_journal / "timeline.json").read_text()) + assert payload["months"]["202605"]["month_top"] == [] + assert "schema validation" in payload["months"]["202605"]["month_rationale"] + assert payload["year_top"] == [] diff --git a/solstone/observe/transcribe/gemini.py b/solstone/observe/transcribe/gemini.py index 71f668e48..a1682ff15 100644 --- a/solstone/observe/transcribe/gemini.py +++ b/solstone/observe/transcribe/gemini.py @@ -34,6 +34,7 @@ from solstone.observe.utils import audio_to_flac_bytes from solstone.think.models import ( DEFAULT_PROVIDER_TIMEOUT_S, IncompleteJSONError, + SchemaValidationError, generate, ) from solstone.think.prompts import load_prompt @@ -334,16 +335,21 @@ def _transcribe_once( # Call Gemini via think.models.generate() # thinking_budget=0 disables thinking — transcription is extraction, not # reasoning, and Gemini's default thinking budget consumes output tokens. - response_text = generate( - contents=contents, - context="observe.transcribe.gemini", - temperature=0.3, - max_output_tokens=16384, - json_output=True, - thinking_budget=0, - json_schema=_SCHEMA, - timeout_s=timeout_s, - ) + try: + response_text = generate( + contents=contents, + context="observe.transcribe.gemini", + temperature=0.3, + max_output_tokens=16384, + json_output=True, + thinking_budget=0, + json_schema=_SCHEMA, + timeout_s=timeout_s, + ) + except SchemaValidationError as e: + logger.error("Gemini response failed schema validation: %s", e) + logger.debug("Response text: %s", e.preview) + raise RuntimeError(f"Gemini response failed schema validation: {e}") from e transcribe_time = time.perf_counter() - t0 logger.debug( diff --git a/solstone/think/detect_created.py b/solstone/think/detect_created.py index 6edc76d3b..3e0d958ab 100644 --- a/solstone/think/detect_created.py +++ b/solstone/think/detect_created.py @@ -63,8 +63,8 @@ def _extract_metadata(path: str) -> str: try: proc = subprocess.run(cmd, capture_output=True, text=True, check=True) return proc.stdout - except Exception as exc: # pragma: no cover - exiftool optional - return f"Error extracting metadata: {exc}" + except Exception: # pragma: no cover - exiftool optional + return "exiftool metadata unavailable" def _extract_metadata_json(path: str) -> dict: @@ -265,37 +265,36 @@ def detect_created( from solstone.think.models import generate - response_text = generate( - contents=markdown, - context="detect.created", - temperature=0.3, - max_output_tokens=256, - thinking_budget=4096, - system_instruction=_load_system_prompt(), - json_output=True, - json_schema=_SCHEMA, - ) - try: + response_text = generate( + contents=markdown, + context="detect.created", + temperature=0.3, + max_output_tokens=256, + thinking_budget=4096, + system_instruction=_load_system_prompt(), + json_output=True, + json_schema=_SCHEMA, + ) result = json.loads(response_text) + except (ValueError, json.JSONDecodeError): + return None - # Convert UTC to local time if needed - if result and result.get("utc") is True: - day = result.get("day") - time = result.get("time") + # Convert UTC to local time if needed + if result and result.get("utc") is True: + day = result.get("day") + time = result.get("time") - if day and time: - # Parse as UTC datetime - utc_dt = datetime.strptime(f"{day}{time}", "%Y%m%d%H%M%S") - utc_dt = utc_dt.replace(tzinfo=timezone.utc) + if day and time: + # Parse as UTC datetime + utc_dt = datetime.strptime(f"{day}{time}", "%Y%m%d%H%M%S") + utc_dt = utc_dt.replace(tzinfo=timezone.utc) - # Convert to local timezone - local_dt = utc_dt.astimezone() + # Convert to local timezone + local_dt = utc_dt.astimezone() - # Update result with local time - result["day"] = local_dt.strftime("%Y%m%d") - result["time"] = local_dt.strftime("%H%M%S") + # Update result with local time + result["day"] = local_dt.strftime("%Y%m%d") + result["time"] = local_dt.strftime("%H%M%S") - return result - except json.JSONDecodeError: - return None + return result diff --git a/solstone/think/detect_transcript.py b/solstone/think/detect_transcript.py index 265fbcedd..78f2b95b7 100644 --- a/solstone/think/detect_transcript.py +++ b/solstone/think/detect_transcript.py @@ -194,24 +194,24 @@ def detect_transcript_json(text: str, segment_start: str) -> Optional[dict]: from solstone.think.models import generate - response_text = generate( - contents=contents, - context="observe.detect.json", - temperature=0.3, - max_output_tokens=8192, - thinking_budget=8192, - system_instruction=_load_json_prompt(), - json_output=True, - json_schema=_JSON_SCHEMA, - ) - - logging.info(f"Received JSON conversion response: {response_text[:100]}") try: + response_text = generate( + contents=contents, + context="observe.detect.json", + temperature=0.3, + max_output_tokens=8192, + thinking_budget=8192, + system_instruction=_load_json_prompt(), + json_output=True, + json_schema=_JSON_SCHEMA, + ) + + logging.info(f"Received JSON conversion response: {response_text[:100]}") result = json.loads(response_text) logging.info("Successfully converted transcript to JSON") return result - except json.JSONDecodeError: - logging.error("Failed to parse JSON response from LLM") + except (ValueError, json.JSONDecodeError) as e: + logging.error(f"Failed to parse JSON response from LLM: {e}") return None diff --git a/solstone/think/models.py b/solstone/think/models.py index bbf44572f..70fa66de5 100644 --- a/solstone/think/models.py +++ b/solstone/think/models.py @@ -220,6 +220,25 @@ class IncompleteJSONError(ValueError): super().__init__(f"JSON response incomplete (reason: {reason})") +class SchemaValidationError(ValueError): + """Raised when JSON response text fails local schema validation. + + Attributes: + errors: The schema validation errors returned by _validate_schema. + text: The full offending response text. + preview: A short preview of the offending response text for error messages. + """ + + def __init__(self, errors: list[dict], text: str): + self.errors = errors + self.text = text + self.preview = text if len(text) <= 200 else text[:197] + "..." + super().__init__( + "JSON response failed schema validation " + f"({len(errors)} error(s); preview={self.preview!r})" + ) + + # --------------------------------------------------------------------------- # Prompt context discovery # @@ -1304,7 +1323,9 @@ def generate( _validate_json_response(result, json_output) if json_schema is not None: - _validate_schema(result["text"], json_schema) + validation = _validate_schema(result["text"], json_schema) + if validation["valid"] is False: + raise SchemaValidationError(validation["errors"], result["text"]) return result["text"] @@ -1623,7 +1644,9 @@ async def agenerate( _validate_json_response(result, json_output) if json_schema is not None: - _validate_schema(result["text"], json_schema) + validation = _validate_schema(result["text"], json_schema) + if validation["valid"] is False: + raise SchemaValidationError(validation["errors"], result["text"]) return result["text"] diff --git a/solstone/think/providers/google.py b/solstone/think/providers/google.py index bfc4a9735..8852ad4a0 100644 --- a/solstone/think/providers/google.py +++ b/solstone/think/providers/google.py @@ -325,8 +325,8 @@ def _build_generate_config( def _extract_response_text(response: Any) -> str: """Extract text from response. - Returns response.text if available, or a friendly completion message - if the response is empty. Raises on safety filter blocks. + Returns response.text if available, or "" if the response is empty. + Raises on safety filter blocks. Parameters ---------- @@ -341,13 +341,9 @@ def _extract_response_text(response: Any) -> str: if finish_reason and "SAFETY" in finish_reason.upper(): raise ValueError(f"Response blocked by safety filters: {finish_reason}") - # Extract text, or generate friendly message if empty + # Extract text; empty non-safety completions are represented honestly as empty. text = response.text if response.text else "" - if text: - return text - - # Empty text - generate user-friendly completion message - return _format_completion_message(finish_reason, had_tool_calls=False) + return text def _normalize_finish_reason(response: Any) -> str | None: @@ -444,45 +440,6 @@ def _extract_finish_reason(response: Any) -> str | None: return None -def _format_completion_message(finish_reason: str | None, had_tool_calls: bool) -> str: - """Create a user-friendly completion message based on finish reason. - - Parameters - ---------- - finish_reason - The finish_reason from the response (e.g., "STOP", "MAX_TOKENS"). - had_tool_calls - Whether tool calls were executed during this run. - - Returns - ------- - str - A concise, user-friendly completion message. - """ - if not finish_reason: - finish_reason = "UNKNOWN" - - # Normalize finish reason (handle both enum names and string values) - reason = finish_reason.upper().replace("FINISHREASON.", "") - - if reason == "STOP": - if had_tool_calls: - return "Completed via tools." - return "Completed." - elif reason == "MAX_TOKENS": - return "Reached token limit." - elif "SAFETY" in reason: - return "Blocked by safety filters." - elif reason == "RECITATION": - return "Stopped due to recitation." - elif "TOOL" in reason or "FUNCTION" in reason: - # UNEXPECTED_TOOL_CALL, MALFORMED_FUNCTION_CALL, etc. - return "Tool execution incomplete." - else: - # Unknown reason - include it for debugging - return f"Completed ({reason.lower()})." - - def _summarize_contents(contents: Any) -> str: """One-line, PII-free fingerprint of the contents passed to generate_content. diff --git a/solstone/think/providers/local.py b/solstone/think/providers/local.py index 9b8367f4f..8efc04995 100644 --- a/solstone/think/providers/local.py +++ b/solstone/think/providers/local.py @@ -381,6 +381,11 @@ async def run_cogitate( local_server.connect() return await openhands.run_cogitate(config, on_event=on_event) except Exception as exc: + from solstone.think.talents import TalentHookError + + if isinstance(exc, TalentHookError): + raise + reason_code = None if not endpoint.is_bundled: reason_code = classify_byo_cogitate_error(exc) or getattr( diff --git a/solstone/think/providers/openhands.py b/solstone/think/providers/openhands.py index 5b2384380..113d4f5ca 100644 --- a/solstone/think/providers/openhands.py +++ b/solstone/think/providers/openhands.py @@ -1289,6 +1289,11 @@ async def run_cogitate( except QuotaExhaustedError: raise except Exception as exc: + from solstone.think.talents import TalentHookError + + if isinstance(exc, TalentHookError): + raise + provider_exc = _unwrap_provider_exception(exc) reason_code = None local_endpoint = None diff --git a/solstone/think/talents.py b/solstone/think/talents.py index 033f46a01..ff0ee849b 100644 --- a/solstone/think/talents.py +++ b/solstone/think/talents.py @@ -84,6 +84,24 @@ MIN_INPUT_CHARS = 50 MIN_OUTPUT_TOKENS = 300 +class TalentHookError(RuntimeError): + """Raised when an invoked talent hook fails.""" + + def __init__( + self, + phase: str, + hook_name: str, + talent_name: str, + original: Exception, + ) -> None: + self.phase = phase + self.hook_name = hook_name + self.talent_name = talent_name + super().__init__( + f"{phase}-hook {hook_name!r} failed for talent {talent_name!r}: {original}" + ) + + def setup_logging(verbose: bool = False) -> logging.Logger: """Configure logging for agent CLI.""" level = logging.DEBUG if verbose else logging.INFO @@ -808,6 +826,8 @@ def _run_pre_hooks(config: dict) -> dict: if not pre_hook: return {} + hook_name = str(config.get("hook", {}).get("pre", "unknown")) + talent_name = str(config.get("name", "unknown")) try: modifications = pre_hook(config) if modifications: @@ -815,6 +835,7 @@ def _run_pre_hooks(config: dict) -> dict: return modifications except Exception as exc: LOG.error("Pre-hook failed: %s", exc) + raise TalentHookError("pre", hook_name, talent_name, exc) from exc return {} @@ -851,6 +872,8 @@ def _run_post_hooks(result: str, config: dict) -> str: if not post_hook: return result + hook_name = str(config.get("hook", {}).get("post", "unknown")) + talent_name = str(config.get("name", "unknown")) try: hook_result = post_hook(result, config) if hook_result is not None: @@ -858,6 +881,7 @@ def _run_post_hooks(result: str, config: dict) -> str: return hook_result except Exception as exc: LOG.error("Post-hook failed: %s", exc) + raise TalentHookError("post", hook_name, talent_name, exc) from exc return result @@ -1240,7 +1264,31 @@ def _build_dry_run_event(config: dict, before_values: dict) -> dict: return event +def _mark_terminal_error_evented(config: dict) -> None: + config["_terminal_error_evented"] = True + + +def _emit_terminal_hook_error( + config: dict, + emit_event: Callable[[dict], None], + exc: TalentHookError, +) -> None: + _mark_terminal_error_evented(config) + setattr(exc, "_evented", True) + emit_event( + { + "event": "error", + "error": str(exc), + "reason_code": "hook_error", + "provider": config.get("provider"), + "terminal": True, + "ts": now_ms(), + } + ) + + _NON_RETRYABLE_ERRORS = ( + TalentHookError, ValueError, json.JSONDecodeError, KeyError, @@ -1312,6 +1360,7 @@ async def _execute_with_tools( raw_result = data.get("result", "") result = _run_post_hooks(raw_result, config) if _expected_output_blank(config, raw_result, result): + _mark_terminal_error_evented(config) emit_event( { "event": "error", @@ -1366,6 +1415,9 @@ async def _execute_with_tools( try: await provider_mod.run_cogitate(config=config, on_event=talent_emit_event) + except TalentHookError as exc: + _emit_terminal_hook_error(config, emit_event, exc) + return except Exception as exc: if provider == "local": raise @@ -1575,8 +1627,13 @@ async def _execute_generate( usage_data = gen_result.get("usage") # Run post-hooks - result = _run_post_hooks(raw_result, config) + try: + result = _run_post_hooks(raw_result, config) + except TalentHookError as exc: + _emit_terminal_hook_error(config, emit_event, exc) + return if _expected_output_blank(config, raw_result, result): + _mark_terminal_error_evented(config) emit_event( { "event": "error", @@ -1713,7 +1770,11 @@ async def _run_talent( before_values["extra_context"] = config.get("extra_context", "") # Run pre-hooks - modifications = _run_pre_hooks(config) + try: + modifications = _run_pre_hooks(config) + except TalentHookError as exc: + _emit_terminal_hook_error(config, emit_event, exc) + return template_vars = modifications.pop("template_vars", None) for key, value in modifications.items(): config[key] = value @@ -1752,7 +1813,7 @@ async def _run_talent( await _execute_generate(config, emit_event) # Log completion - if config.get("day"): + if config.get("day") and not config.get("_terminal_error_evented"): day_log(config["day"], f"talent {name} ok") diff --git a/tests/test_anthropic.py b/tests/test_anthropic.py index 28760d3ac..b1be689bd 100644 --- a/tests/test_anthropic.py +++ b/tests/test_anthropic.py @@ -23,6 +23,28 @@ from solstone.think.models import ( ) +@pytest.fixture(autouse=True) +def _restore_talents_module(): + saved = sys.modules.pop("solstone.think.talents", None) + parent = sys.modules.get("solstone.think") + had_parent_attr = parent is not None and hasattr(parent, "talents") + saved_parent_attr = getattr(parent, "talents", None) if had_parent_attr else None + if had_parent_attr: + delattr(parent, "talents") + try: + yield + finally: + if saved is not None: + sys.modules["solstone.think.talents"] = saved + else: + sys.modules.pop("solstone.think.talents", None) + if parent is not None: + if had_parent_attr: + setattr(parent, "talents", saved_parent_attr) + elif hasattr(parent, "talents"): + delattr(parent, "talents") + + async def run_main(mod, argv, stdin_data=None): sys.argv = argv if stdin_data: diff --git a/tests/test_batch.py b/tests/test_batch.py index e68f84274..126e46d34 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -9,7 +9,7 @@ from unittest.mock import AsyncMock, patch import pytest from solstone.think.batch import Batch, BatchRequest -from solstone.think.models import GEMINI_FLASH, GEMINI_LITE +from solstone.think.models import GEMINI_FLASH, GEMINI_LITE, SchemaValidationError def test_batch_request_creation(): @@ -563,3 +563,28 @@ async def test_batch_passes_json_schema_to_agenerate(mock_agenerate): call_kwargs = mock_agenerate.call_args[1] assert call_kwargs["json_schema"] == {"type": "object"} + + +@pytest.mark.asyncio +@patch("solstone.think.batch.agenerate", new_callable=AsyncMock) +async def test_batch_schema_validation_error_populates_request_error(mock_agenerate): + mock_agenerate.side_effect = SchemaValidationError( + [{"path": "", "constraint": "json_parse", "message": "empty"}], + "", + ) + + batch = Batch(max_concurrent=5) + req = batch.create( + contents="Test prompt", + context="test.context", + json_schema={"type": "object"}, + ) + batch.add(req) + + results = [] + async for completed_req in batch.drain_batch(): + results.append(completed_req) + + assert len(results) == 1 + assert results[0].response is None + assert "schema validation" in results[0].error diff --git a/tests/test_detect_created_schema.py b/tests/test_detect_created_schema.py index 143f77378..7504002c3 100644 --- a/tests/test_detect_created_schema.py +++ b/tests/test_detect_created_schema.py @@ -85,3 +85,44 @@ def test_detect_created_passes_schema_to_generate(monkeypatch): "source": "QuickTime:CreateDate", "utc": False, } + + +def test_detect_created_schema_validation_error_returns_none(monkeypatch): + def fake_generate(**kwargs): + raise models.SchemaValidationError( + [{"path": "", "constraint": "json_parse", "message": "empty"}], + "", + ) + + monkeypatch.setattr(models, "generate", fake_generate) + monkeypatch.setattr( + detect_created_mod, + "_extract_metadata", + lambda path: "QuickTime Create Date : 2024:03:15 14:30:52", + ) + + assert detect_created_mod.detect_created("/dev/null") is None + + +def test_detect_created_metadata_failure_prompt_uses_sentinel(monkeypatch): + captured = {} + + def fail_run(*args, **kwargs): + raise FileNotFoundError("secret exiftool path") + + def fake_generate(**kwargs): + captured.update(kwargs) + return ( + '{"day": "20240315", "time": "143052", "confidence": "high", ' + '"source": "QuickTime:CreateDate", "utc": false}' + ) + + monkeypatch.setattr(detect_created_mod.subprocess, "run", fail_run) + monkeypatch.setattr(models, "generate", fake_generate) + + result = detect_created_mod.detect_created("/tmp/private-source.mov") + + assert result is not None + assert "exiftool metadata unavailable" in captured["contents"] + assert "secret exiftool path" not in captured["contents"] + assert "FileNotFoundError" not in captured["contents"] diff --git a/tests/test_detect_transcript_schema.py b/tests/test_detect_transcript_schema.py index 62ad882cc..230de0054 100644 --- a/tests/test_detect_transcript_schema.py +++ b/tests/test_detect_transcript_schema.py @@ -120,6 +120,18 @@ def test_detect_transcript_segment_passes_schema_to_generate(monkeypatch): assert all(isinstance(item, tuple) and len(item) == 2 for item in result) +def test_detect_transcript_segment_schema_validation_error_returns_empty(monkeypatch): + def fake_generate(**kwargs): + raise models.SchemaValidationError( + [{"path": "", "constraint": "json_parse", "message": "empty"}], + "", + ) + + monkeypatch.setattr(models, "generate", fake_generate) + + assert mod.detect_transcript_segment("01\n02\n", "12:00:00") == [] + + def test_detect_transcript_json_passes_schema_to_generate(monkeypatch): captured = {} @@ -140,3 +152,15 @@ def test_detect_transcript_json_passes_schema_to_generate(monkeypatch): "topics": "planning", "setting": "workplace", } + + +def test_detect_transcript_json_schema_validation_error_returns_none(monkeypatch): + def fake_generate(**kwargs): + raise models.SchemaValidationError( + [{"path": "", "constraint": "json_parse", "message": "empty"}], + "", + ) + + monkeypatch.setattr(models, "generate", fake_generate) + + assert mod.detect_transcript_json("some text", "12:00:00") is None diff --git a/tests/test_enrich_schema.py b/tests/test_enrich_schema.py index b456e8dc1..6356f4333 100644 --- a/tests/test_enrich_schema.py +++ b/tests/test_enrich_schema.py @@ -7,6 +7,8 @@ from pathlib import Path from jsonschema import Draft202012Validator +import solstone.think.models as models + enrich_mod = importlib.import_module("solstone.observe.enrich") _SCHEMA = json.loads( @@ -151,3 +153,23 @@ def test_enrich_transcript_passes_schema_to_generate(monkeypatch): "setting": "", "warning": "", } + + +def test_enrich_transcript_schema_validation_error_returns_none(monkeypatch): + def fake_generate(**kwargs): + raise models.SchemaValidationError( + [{"path": "", "constraint": "json_parse", "message": "empty"}], + "", + ) + + monkeypatch.setattr(enrich_mod, "generate", fake_generate) + + import numpy as np + + result = enrich_mod.enrich_transcript( + np.zeros(16000, dtype=np.float32), + 16000, + [{"id": 1, "start": 0.0, "end": 1.0, "text": "Hello world."}], + ) + + assert result is None diff --git a/tests/test_extract_schema.py b/tests/test_extract_schema.py index 06313986a..cbb4e309a 100644 --- a/tests/test_extract_schema.py +++ b/tests/test_extract_schema.py @@ -62,3 +62,25 @@ def test_ai_select_frames_passes_schema_to_generate(monkeypatch): assert captured["json_schema"] is extract_mod._SCHEMA assert result == [1] + + +def test_select_frames_schema_validation_error_uses_fallback(monkeypatch): + def fake_generate(**kwargs): + raise models.SchemaValidationError( + [{"path": "", "constraint": "json_parse", "message": "empty"}], + "", + ) + + monkeypatch.setattr(models, "generate", fake_generate) + + frames = [ + {"frame_id": 1, "timestamp": 1.0, "analysis": {"primary": "code"}}, + {"frame_id": 2, "timestamp": 2.0, "analysis": {"primary": "code"}}, + ] + categories = {"code": {"description": "Code editors"}} + + assert extract_mod.select_frames_for_extraction( + frames, + max_extractions=5, + categories=categories, + ) == [1, 2] diff --git a/tests/test_generate_full.py b/tests/test_generate_full.py index 6e49d6e3e..d6145781c 100644 --- a/tests/test_generate_full.py +++ b/tests/test_generate_full.py @@ -162,6 +162,57 @@ def test_execute_generate_blank_expected_output_emits_terminal_no_output( assert output_path.read_text(encoding="utf-8") == "old output" +def test_no_output_does_not_log_day_ok(tmp_path, monkeypatch): + mod = importlib.import_module("solstone.think.talents") + copy_day(tmp_path, monkeypatch) + + import solstone.think.talent as talent + + monkeypatch.setattr(talent, "TALENT_DIR", tmp_path) + _write_generator_file( + tmp_path, + "blank_day_gen", + { + "type": "generate", + "schedule": "daily", + "priority": 10, + "output": "md", + "load": {"transcripts": True, "percepts": True}, + }, + ) + + from solstone.think import models + + monkeypatch.setattr( + models, + "generate_with_result", + lambda *a, **k: {"text": " ", "usage": {"input_tokens": 1}}, + ) + monkeypatch.setenv("GOOGLE_API_KEY", "x") + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + + events = run_generator_with_config( + mod, + { + "name": "blank_day_gen", + "day": "20240101", + "output": "md", + "provider": "google", + "model": "gemini-2.0-flash", + }, + monkeypatch, + ) + + error_events = [e for e in events if e["event"] == "error"] + assert len(error_events) == 1 + assert error_events[0]["reason_code"] == "no_output" + assert [e for e in events if e["event"] == "finish"] == [] + + task_log = tmp_path / "chronicle" / "20240101" / "task_log.txt" + log_text = task_log.read_text(encoding="utf-8") if task_log.exists() else "" + assert "talent blank_day_gen ok" not in log_text + + def test_execute_generate_blank_without_output_path_still_finishes( tmp_path, monkeypatch ): @@ -553,6 +604,66 @@ def post_process(result, context): assert captured["has_hook"] is True # Frontmatter fields now directly in config +def test_generate_hook_error_emits_terminal_hook_error(tmp_path, monkeypatch): + mod = importlib.import_module("solstone.think.talents") + copy_day(tmp_path, monkeypatch) + + import solstone.think.talent as talent + + monkeypatch.setattr(talent, "TALENT_DIR", tmp_path) + _write_generator_file( + tmp_path, + "hook_error_gen", + { + "type": "generate", + "schedule": "daily", + "priority": 10, + "output": "md", + "hook": {"post": "hook_error_gen"}, + "load": {"transcripts": True, "percepts": True}, + }, + ) + + hook_file = tmp_path / "hook_error_gen.py" + hook_file.write_text(""" +def post_process(result, context): + raise RuntimeError("hook boom") +""") + + from solstone.think import models + + monkeypatch.setattr( + models, + "generate_with_result", + lambda *a, **k: MOCK_RESULT, + ) + monkeypatch.setenv("GOOGLE_API_KEY", "x") + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + + events = run_generator_with_config( + mod, + { + "name": "hook_error_gen", + "day": "20240101", + "output": "md", + "provider": "google", + "model": "gemini-2.0-flash", + }, + monkeypatch, + ) + + assert [e for e in events if e["event"] == "finish"] == [] + error_events = [e for e in events if e["event"] == "error"] + assert len(error_events) == 1 + assert error_events[0]["reason_code"] == "hook_error" + assert error_events[0]["terminal"] is True + assert ( + "post-hook 'hook_error_gen' failed for talent 'hook_error_gen'" + in (error_events[0]["error"]) + ) + assert "hook boom" in error_events[0]["error"] + + def test_generate_without_hook_succeeds(tmp_path, monkeypatch): """Test that generators without hooks still work correctly.""" mod = importlib.import_module("solstone.think.talents") diff --git a/tests/test_google.py b/tests/test_google.py index c865fc9f9..0976a1709 100644 --- a/tests/test_google.py +++ b/tests/test_google.py @@ -8,6 +8,7 @@ import sys from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock +import pytest from google.genai import types as genai_types from solstone.think.models import DEFAULT_PROVIDER_TIMEOUT_S, GEMINI_FLASH @@ -15,12 +16,33 @@ from solstone.think.providers import google as google_provider from solstone.think.providers.google import ( _extract_finish_reason, _extract_usage, - _format_completion_message, _resolved_model, ) from tests.conftest import setup_google_genai_stub +@pytest.fixture(autouse=True) +def _restore_talents_module(): + saved = sys.modules.pop("solstone.think.talents", None) + parent = sys.modules.get("solstone.think") + had_parent_attr = parent is not None and hasattr(parent, "talents") + saved_parent_attr = getattr(parent, "talents", None) if had_parent_attr else None + if had_parent_attr: + delattr(parent, "talents") + try: + yield + finally: + if saved is not None: + sys.modules["solstone.think.talents"] = saved + else: + sys.modules.pop("solstone.think.talents", None) + if parent is not None: + if had_parent_attr: + setattr(parent, "talents", saved_parent_attr) + elif hasattr(parent, "talents"): + delattr(parent, "talents") + + async def run_main(mod, argv, stdin_data=None): sys.argv = argv if stdin_data: @@ -244,52 +266,62 @@ def test_extract_finish_reason_no_candidates(): assert _extract_finish_reason(response) is None -def test_format_completion_message_stop_with_tools(): - """Test message for STOP with tool calls.""" - msg = _format_completion_message("STOP", had_tool_calls=True) - assert msg == "Completed via tools." - - -def test_format_completion_message_stop_no_tools(): - """Test message for STOP without tool calls.""" - msg = _format_completion_message("STOP", had_tool_calls=False) - assert msg == "Completed." - - -def test_format_completion_message_max_tokens(): - """Test message for MAX_TOKENS finish reason.""" - msg = _format_completion_message("MAX_TOKENS", had_tool_calls=False) - assert msg == "Reached token limit." - +def test_run_generate_empty_stop_completion_returns_empty_text(): + response = SimpleNamespace( + text="", + candidates=[SimpleNamespace(finish_reason="STOP", content=None)], + usage_metadata=None, + ) + client = SimpleNamespace( + models=SimpleNamespace(generate_content=MagicMock(return_value=response)) + ) -def test_format_completion_message_safety(): - """Test message for safety-related finish reasons.""" - msg = _format_completion_message("SAFETY", had_tool_calls=False) - assert msg == "Blocked by safety filters." + result = google_provider.run_generate( + contents="hello", + model=GEMINI_FLASH, + client=client, + ) - msg = _format_completion_message("PROHIBITED_SAFETY", had_tool_calls=False) - assert msg == "Blocked by safety filters." + assert result["text"] == "" + assert result["finish_reason"] == "stop" -def test_format_completion_message_tool_errors(): - """Test message for tool-related error finish reasons.""" - msg = _format_completion_message("UNEXPECTED_TOOL_CALL", had_tool_calls=True) - assert msg == "Tool execution incomplete." +def test_run_generate_empty_max_tokens_completion_returns_empty_text(): + response = SimpleNamespace( + text="", + candidates=[SimpleNamespace(finish_reason="MAX_TOKENS", content=None)], + usage_metadata=None, + ) + client = SimpleNamespace( + models=SimpleNamespace(generate_content=MagicMock(return_value=response)) + ) - msg = _format_completion_message("MALFORMED_FUNCTION_CALL", had_tool_calls=False) - assert msg == "Tool execution incomplete." + result = google_provider.run_generate( + contents="hello", + model=GEMINI_FLASH, + client=client, + ) + assert result["text"] == "" + assert result["finish_reason"] == "max_tokens" -def test_format_completion_message_unknown(): - """Test message for unknown finish reasons.""" - msg = _format_completion_message("SOME_NEW_REASON", had_tool_calls=False) - assert msg == "Completed (some_new_reason)." +def test_run_generate_safety_completion_still_raises(): + response = SimpleNamespace( + text="", + candidates=[SimpleNamespace(finish_reason="SAFETY")], + usage_metadata=None, + ) + client = SimpleNamespace( + models=SimpleNamespace(generate_content=MagicMock(return_value=response)) + ) -def test_format_completion_message_none(): - """Test message when finish_reason is None.""" - msg = _format_completion_message(None, had_tool_calls=False) - assert msg == "Completed (unknown)." + with pytest.raises(ValueError, match="blocked by safety filters"): + google_provider.run_generate( + contents="hello", + model=GEMINI_FLASH, + client=client, + ) class TestRunGenerateJsonSchema: diff --git a/tests/test_local.py b/tests/test_local.py index ec4348b62..82f4aa5eb 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -22,6 +22,7 @@ from solstone.think.models import ( TIER_PRO, get_model_provider, ) +from solstone.think.talents import TalentHookError def _provider(): @@ -704,6 +705,39 @@ def test_run_cogitate_byo_classified_error_uses_fixed_copy_and_redacts( assert token not in events[0]["trace"] +def test_run_cogitate_talent_hook_error_bypasses_local_error_event(monkeypatch): + provider = _provider() + events: list[dict] = [] + hook_exc = TalentHookError( + "post", + "broken_hook", + "chat", + RuntimeError("hook exploded"), + ) + + async def fail_cogitate(*_args, **_kwargs): + raise hook_exc + + monkeypatch.setattr(provider, "resolve_local_endpoint", _byo_endpoint) + monkeypatch.setattr( + "solstone.think.providers.local_server.connect", + lambda: (_ for _ in ()).throw(AssertionError("connect not expected")), + ) + monkeypatch.setattr( + "solstone.think.providers.openhands.run_cogitate", + fail_cogitate, + ) + + with pytest.raises(TalentHookError) as raised: + asyncio.run( + provider.run_cogitate({"model": LOCAL_MODEL}, on_event=events.append) + ) + + assert raised.value is hook_exc + assert events == [] + assert not getattr(hook_exc, "_evented", False) + + @pytest.mark.parametrize( ("credential", "expected_key"), [ diff --git a/tests/test_models.py b/tests/test_models.py index a894a372f..2cb3089a7 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -34,6 +34,7 @@ from solstone.think.models import ( TIER_PRO, TYPE_DEFAULTS, IncompleteJSONError, + SchemaValidationError, _Family, _find_pricing_fallback, _parse_family_anthropic, @@ -1510,6 +1511,37 @@ class TestGenerateJsonSchemaPlumbing: assert result["schema_validation"] == validation + def test_generate_with_result_returns_failed_schema_validation_without_raising( + self, + ): + provider_module = SimpleNamespace( + run_generate=MagicMock( + return_value={"text": '{"field": "bad"}', "finish_reason": "stop"} + ) + ) + validation = { + "valid": False, + "errors": [{"path": "/field", "constraint": "type", "message": "bad"}], + } + + with ( + patch( + "solstone.think.models.resolve_provider", return_value=("fake", "model") + ), + patch( + "solstone.think.providers.get_provider_module", + return_value=provider_module, + ), + patch("solstone.think.models._validate_schema", return_value=validation), + ): + result = generate_with_result( + "hello", + "test.context", + json_schema={"type": "object"}, + ) + + assert result["schema_validation"] == validation + def test_generate_with_result_omits_schema_validation_without_schema(self): provider_module = SimpleNamespace( run_generate=MagicMock(return_value={"text": "{}", "finish_reason": "stop"}) @@ -1579,6 +1611,84 @@ class TestGenerateJsonSchemaPlumbing: mock_validate_schema.assert_called_once() mock_async_validate.assert_called_once() + def test_generate_raises_schema_validation_error_with_structured_fields(self): + long_text = '{"field": "' + ("x" * 300) + '"}' + provider_module = SimpleNamespace( + run_generate=MagicMock( + return_value={"text": long_text, "finish_reason": "stop"} + ) + ) + schema = { + "type": "object", + "properties": {"field": {"type": "integer"}}, + } + + with ( + patch( + "solstone.think.models.resolve_provider", return_value=("fake", "model") + ), + patch( + "solstone.think.providers.get_provider_module", + return_value=provider_module, + ), + ): + with pytest.raises(SchemaValidationError) as exc_info: + generate("hello", "test.context", json_schema=schema) + + exc = exc_info.value + assert exc.text == long_text + assert exc.errors[0]["constraint"] == "type" + assert len(exc.preview) == 200 + assert exc.preview.endswith("...") + assert "x" * 250 not in str(exc) + + def test_agenerate_raises_schema_validation_error(self): + provider_module = SimpleNamespace( + run_agenerate=AsyncMock( + return_value={"text": '{"field": "bad"}', "finish_reason": "stop"} + ) + ) + schema = { + "type": "object", + "properties": {"field": {"type": "integer"}}, + } + + with ( + patch( + "solstone.think.models.resolve_provider", return_value=("fake", "model") + ), + patch( + "solstone.think.providers.get_provider_module", + return_value=provider_module, + ), + ): + with pytest.raises(SchemaValidationError) as exc_info: + asyncio.run(agenerate("hello", "test.context", json_schema=schema)) + + assert exc_info.value.text == '{"field": "bad"}' + assert exc_info.value.errors[0]["constraint"] == "type" + + def test_generate_empty_schema_response_raises_schema_validation_error(self): + provider_module = SimpleNamespace( + run_generate=MagicMock(return_value={"text": "", "finish_reason": "stop"}) + ) + + with ( + patch( + "solstone.think.models.resolve_provider", return_value=("fake", "model") + ), + patch( + "solstone.think.providers.get_provider_module", + return_value=provider_module, + ), + ): + with pytest.raises(SchemaValidationError) as exc_info: + generate("hello", "test.context", json_schema={"type": "object"}) + + assert exc_info.value.text == "" + assert exc_info.value.preview == "" + assert exc_info.value.errors[0]["constraint"] == "json_parse" + def test_truncation_raises_before_schema_validation(self): provider_module = SimpleNamespace( run_generate=MagicMock(return_value={"text": "{}", "finish_reason": "stop"}) diff --git a/tests/test_openhands_errors.py b/tests/test_openhands_errors.py index bba731c46..dfbf2b26f 100644 --- a/tests/test_openhands_errors.py +++ b/tests/test_openhands_errors.py @@ -17,6 +17,7 @@ from solstone.think.providers.local_endpoint import ( LOCAL_ENDPOINT_CONTRACT_COPY, LocalEndpoint, ) +from solstone.think.talents import TalentHookError from tests.openhands_fakes import install_fake_openhands @@ -129,6 +130,31 @@ def test_run_cogitate_generic_error_emits_event_and_marks_evented( assert events[0]["ts"] == 123456 +def test_run_cogitate_talent_hook_error_propagates_without_provider_event( + fake_openhands, + run_env, +): + hook_exc = TalentHookError( + "post", + "broken_hook", + "chat", + RuntimeError("hook exploded"), + ) + + async def fail(_conversation): + raise hook_exc + + fake_openhands.Conversation.arun_impl = fail + events: list[dict] = [] + + with pytest.raises(TalentHookError) as raised: + asyncio.run(openhands.run_cogitate(run_env, events.append)) + + assert raised.value is hook_exc + assert events == [] + assert not getattr(hook_exc, "_evented", False) + + def test_run_cogitate_error_before_usage_baseline_omits_usage( fake_openhands, run_env, diff --git a/tests/test_output_hooks.py b/tests/test_output_hooks.py index 1682d5c5c..b66299d36 100644 --- a/tests/test_output_hooks.py +++ b/tests/test_output_hooks.py @@ -291,8 +291,8 @@ def post_process(result, context): assert finish_events[0]["result"] == MOCK_RESULT["text"] -def test_output_hook_error_fallback(tmp_path, monkeypatch): - """Test that hook errors fall back to original result.""" +def test_output_hook_error_emits_terminal_hook_error(tmp_path, monkeypatch): + """Test that hook errors emit terminal hook_error events.""" mod = importlib.import_module("solstone.think.talents") copy_day(tmp_path, monkeypatch) @@ -330,12 +330,20 @@ def post_process(result, context): "model": "gemini-2.0-flash", } - # Should not raise, should fall back gracefully events = run_generator_with_config(mod, config, monkeypatch) finish_events = [e for e in events if e["event"] == "finish"] - assert len(finish_events) == 1 - assert finish_events[0]["result"] == MOCK_RESULT["text"] + assert finish_events == [] + + error_events = [e for e in events if e["event"] == "error"] + assert len(error_events) == 1 + assert error_events[0]["reason_code"] == "hook_error" + assert error_events[0]["terminal"] is True + assert ( + "post-hook 'broken_test' failed for talent 'broken_test'" + in error_events[0]["error"] + ) + assert "Hook exploded!" in error_events[0]["error"] # ============================================================================= @@ -455,7 +463,7 @@ def pre_process(context): "model": "gemini-2.0-flash", } - events = run_generator_with_config(mod, config, monkeypatch) + run_generator_with_config(mod, config, monkeypatch) # Verify pre-hook modified the prompt - check in contents contents = received_kwargs.get("contents", []) @@ -463,6 +471,53 @@ def pre_process(context): prompt_found = any("[pre-processed]" in str(c) for c in contents) assert prompt_found, f"Expected [pre-processed] in contents: {contents}" + +def test_pre_hook_skip_reason_dict_clean_skips(tmp_path, monkeypatch): + mod = importlib.import_module("solstone.think.talents") + copy_day(tmp_path, monkeypatch) + + import solstone.think.talent as talent + + monkeypatch.setattr(talent, "TALENT_DIR", tmp_path) + + prompt_file = tmp_path / "skip_test.md" + prompt_file.write_text( + '{\n "type": "generate",\n "title": "Skip",\n "schedule": "daily",\n "priority": 10,\n "output": "md",\n "hook": {"pre": "skip_test"},\n "load": {"transcripts": true, "percepts": true}\n}\n\nOriginal prompt' + ) + + hook_file = tmp_path / "skip_test.py" + hook_file.write_text(""" +def pre_process(context): + return {"skip_reason": "no_sources"} +""") + + from solstone.think import models + + monkeypatch.setattr( + models, + "generate_with_result", + lambda *a, **k: MOCK_RESULT, + ) + monkeypatch.setenv("GOOGLE_API_KEY", "x") + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + + events = run_generator_with_config( + mod, + { + "name": "skip_test", + "day": "20240101", + "output": "md", + "provider": "google", + "model": "gemini-2.0-flash", + }, + monkeypatch, + ) + + finish_events = [e for e in events if e["event"] == "finish"] + assert len(finish_events) == 1 + assert finish_events[0]["skipped"] == "no_sources" + assert [e for e in events if e["event"] == "error"] == [] + # Verify generator still completed successfully finish_events = [e for e in events if e["event"] == "finish"] assert len(finish_events) == 1 diff --git a/tests/test_talent_fallback.py b/tests/test_talent_fallback.py index 837494538..6cb79d7ff 100644 --- a/tests/test_talent_fallback.py +++ b/tests/test_talent_fallback.py @@ -19,7 +19,11 @@ from solstone.think.models import ( should_recheck_health, ) from solstone.think.providers.cli import QuotaExhaustedError -from solstone.think.talents import _is_retryable_error +from solstone.think.talents import ( + TalentHookError, + _is_retryable_error, + _should_fallback, +) from solstone.think.utils import now_ms @@ -754,6 +758,13 @@ def test_on_failure_no_retry_value_error(monkeypatch): assert not any(e.get("event") == "fallback" for e in events) +def test_talent_hook_error_is_non_retryable(): + exc = TalentHookError("post", "broken", "chat", RuntimeError("boom")) + + assert _is_retryable_error(exc) is False + assert _should_fallback(exc) is False + + def test_on_failure_both_fail_raises_original(monkeypatch): from solstone.think.talents import _execute_generate @@ -942,3 +953,76 @@ def test_main_async_no_duplicate_error_when_evented(monkeypatch, capsys): events = [json.loads(line) for line in lines] error_events = [event for event in events if event.get("event") == "error"] assert len(error_events) == 1 + + +def test_main_async_cogitate_hook_error_no_fallback_no_double_emit( + monkeypatch, + capsys, +): + from solstone.think import talents + + ndjson_input = json.dumps({"name": "chat", "prompt": "hello"}) + monkeypatch.setattr("sys.stdin", StringIO(ndjson_input)) + + provider_calls = [] + + async def fake_run_cogitate(config, on_event): + provider_calls.append(config["provider"]) + on_event({"event": "finish", "result": "provider result"}) + + def broken_post_hook(result, context): + raise RuntimeError("hook exploded") + + mock_args = MagicMock() + mock_args.verbose = False + mock_args.dry_run = False + mock_args.subcommand = None + + config = { + "type": "cogitate", + "name": "chat", + "provider": "google", + "model": "gemini-3-flash-preview", + "tier": "flash", + "prompt": "hello", + "hook": {"post": "broken_hook"}, + } + + monkeypatch.setattr("solstone.think.talents.setup_cli", lambda _parser: mock_args) + monkeypatch.setattr( + "solstone.think.talents.setup_logging", + lambda _verbose=False: MagicMock(), + ) + monkeypatch.setattr( + "solstone.think.talents.prepare_config", lambda _request: config + ) + monkeypatch.setattr("solstone.think.talents.validate_config", lambda _config: None) + monkeypatch.setattr( + "solstone.think.talents.load_post_hook", lambda _config: broken_post_hook + ) + monkeypatch.setattr( + "solstone.think.providers.PROVIDER_REGISTRY", + {"google": object(), "anthropic": object()}, + ) + monkeypatch.setattr( + "solstone.think.providers.get_provider_module", + lambda _provider: SimpleNamespace(run_cogitate=fake_run_cogitate), + ) + monkeypatch.setenv("ANTHROPIC_API_KEY", "x") + + asyncio.run(talents.main_async()) + + lines = [line for line in capsys.readouterr().out.splitlines() if line.strip()] + events = [json.loads(line) for line in lines] + error_events = [event for event in events if event.get("event") == "error"] + + assert provider_calls == ["google"] + assert [event for event in events if event.get("event") == "fallback"] == [] + assert [event for event in events if event.get("event") == "finish"] == [] + assert len(error_events) == 1 + assert error_events[0]["reason_code"] == "hook_error" + assert error_events[0]["terminal"] is True + assert ( + "post-hook 'broken_hook' failed for talent 'chat'" in error_events[0]["error"] + ) + assert "hook exploded" in error_events[0]["error"] diff --git a/tests/test_transcribe_gemini_schema.py b/tests/test_transcribe_gemini_schema.py index ce909e57f..d1d55fcc5 100644 --- a/tests/test_transcribe_gemini_schema.py +++ b/tests/test_transcribe_gemini_schema.py @@ -6,6 +6,7 @@ from pathlib import Path from types import SimpleNamespace import numpy as np +import pytest from jsonschema import Draft202012Validator import solstone.observe.transcribe.gemini as gemini_mod @@ -117,3 +118,29 @@ def test_transcribe_passes_schema_to_generate(monkeypatch): ) assert captured["json_schema"] is gemini_mod._SCHEMA + + +def test_transcribe_schema_validation_error_raises_runtime_error(monkeypatch): + def fake_generate(**kwargs): + raise gemini_mod.SchemaValidationError( + [{"path": "", "constraint": "json_parse", "message": "empty"}], + "", + ) + + monkeypatch.setattr(gemini_mod, "generate", fake_generate) + monkeypatch.setattr( + gemini_mod, "_build_chunk_contents", lambda *_args: ["contents"] + ) + monkeypatch.setattr( + gemini_mod, + "load_prompt", + lambda *_args, **_kwargs: SimpleNamespace(text="Prompt"), + ) + + with pytest.raises(RuntimeError, match="Gemini response failed schema validation"): + gemini_mod.transcribe( + np.zeros(16000, dtype=np.float32), + 16000, + {}, + [(0.0, 1.0)], + ) -- 2.51.2