diff --git a/solstone/think/providers/openhands.py b/solstone/think/providers/openhands.py index 8648aa010..e3ef0d04f 100644 --- a/solstone/think/providers/openhands.py +++ b/solstone/think/providers/openhands.py @@ -70,6 +70,7 @@ from solstone.think.providers.shared import ( classify_provider_error, exception_chain, is_cloud_model_not_found, + mark_cloud_model_request, safe_raw, validate_generate_result_strict, ) @@ -657,11 +658,15 @@ def _run_generate( thinking_budget=thinking_budget, responses_api=responses_api, ) - response = ( - llm.responses(messages, **call_kwargs) - if responses_api - else llm.completion(messages, **call_kwargs) - ) + try: + response = ( + llm.responses(messages, **call_kwargs) + if responses_api + else llm.completion(messages, **call_kwargs) + ) + except Exception as exc: + mark_cloud_model_request(exc) + raise return _generate_result(response, model) @@ -697,11 +702,15 @@ async def _run_agenerate( thinking_budget=thinking_budget, responses_api=responses_api, ) - response = ( - await llm.aresponses(messages, **call_kwargs) - if responses_api - else await llm.acompletion(messages, **call_kwargs) - ) + try: + response = ( + await llm.aresponses(messages, **call_kwargs) + if responses_api + else await llm.acompletion(messages, **call_kwargs) + ) + except Exception as exc: + mark_cloud_model_request(exc) + raise return _generate_result(response, model) diff --git a/solstone/think/providers/shared.py b/solstone/think/providers/shared.py index ead972651..ac0bf4986 100644 --- a/solstone/think/providers/shared.py +++ b/solstone/think/providers/shared.py @@ -168,6 +168,8 @@ _CONTEXT_WINDOW_PATTERNS = ( "longer than the model's context length", "context length exceeded", ) +_CLOUD_MODEL_REQUEST_ATTR = "_solstone_cloud_model_request" +_TRUSTED_MODEL_NOT_FOUND_MODULES = ("litellm.exceptions", "openai", "anthropic") def _status_code(exc: BaseException) -> int | None: @@ -203,12 +205,45 @@ def exception_chain(exc: BaseException) -> list[BaseException]: return chain -# Deliberately class-shape based: real cloud SDK missing-model 404s are -# NotFoundError-shaped classes (litellm, openai, anthropic), while a bare 404 -# can come from an unrelated endpoint. Do not broaden this to status alone. +def _chain_has_status_code(exc: BaseException, code: int) -> bool: + return any(_status_code(item) == code for item in exception_chain(exc)) + + +def mark_cloud_model_request(exc: BaseException) -> None: + for item in exception_chain(exc): + try: + setattr(item, _CLOUD_MODEL_REQUEST_ATTR, True) + except Exception: + # SDK exceptions that reject attributes must not replace the real failure. + continue + + +def has_cloud_model_request_mark(exc: BaseException) -> bool: + return any( + getattr(item, _CLOUD_MODEL_REQUEST_ATTR, False) is True + for item in exception_chain(exc) + ) + + +def _is_trusted_model_not_found(exc: BaseException) -> bool: + exc_type = type(exc) + if "notfound" not in exc_type.__name__.lower(): + return False + return any( + _module_matches(exc_type.__module__, module) + for module in _TRUSTED_MODEL_NOT_FOUND_MODULES + ) + + +# Missing-model classification first requires a built-in cloud provider. Trusted +# SDK NotFound-shaped exceptions identify provider model misses by class/module; +# otherwise a bare 404 is accepted only when narrow generate/probe transport +# provenance is present on the exception chain. Status alone is insufficient. def is_cloud_model_not_found(exc: BaseException, provider: str) -> bool: - return is_cloud_provider(provider) and any( - "notfound" in type(item).__name__.lower() for item in exception_chain(exc) + if not is_cloud_provider(provider): + return False + return any(_is_trusted_model_not_found(item) for item in exception_chain(exc)) or ( + has_cloud_model_request_mark(exc) and _chain_has_status_code(exc, 404) ) @@ -689,6 +724,8 @@ __all__ = [ "classify_canned_generate", "classify_provider_error", "exception_chain", + "has_cloud_model_request_mark", "is_cloud_model_not_found", + "mark_cloud_model_request", "safe_raw", ] diff --git a/tests/test_generate_full.py b/tests/test_generate_full.py index 32d15437a..040854a6a 100644 --- a/tests/test_generate_full.py +++ b/tests/test_generate_full.py @@ -171,6 +171,48 @@ def _write_schema_file(tmp_path: Path, name: str, schema: dict) -> None: (tmp_path / name).write_text(json.dumps(schema, indent=2), encoding="utf-8") +def _run_generate_failure( + tmp_path: Path, + monkeypatch, + side_effect: Exception, +) -> list[dict]: + mod = importlib.import_module("solstone.think.talents") + copy_day(tmp_path, monkeypatch) + _write_ready_brain_record(tmp_path) + + import solstone.think.talent as talent + + monkeypatch.setattr(talent, "TALENT_DIR", tmp_path) + _write_generator_file( + tmp_path, + "missing_model_day_gen", + { + "type": "generate", + "schedule": "daily", + "priority": 10, + "output": "md", + "load": {"transcripts": True, "percepts": True}, + }, + ) + provider_module = MagicMock() + provider_module.run_generate.side_effect = side_effect + monkeypatch.setattr( + "solstone.think.providers.get_provider_module", + lambda _provider: provider_module, + ) + monkeypatch.setenv("GOOGLE_API_KEY", "test-key") + + return run_generator_with_config( + mod, + { + "name": "missing_model_day_gen", + "day": "20240101", + "output": "md", + }, + monkeypatch, + ) + + def test_generate_output_ndjson(tmp_path, monkeypatch): """Test basic output generation via NDJSON protocol.""" mod = importlib.import_module("solstone.think.talents") @@ -309,44 +351,14 @@ def test_generate_model_not_found_records_runtime_failure(tmp_path, monkeypatch) from solstone.think.providers.brain_state import inspect_brain_state - mod = importlib.import_module("solstone.think.talents") - copy_day(tmp_path, monkeypatch) - _write_ready_brain_record(tmp_path) - - import solstone.think.talent as talent - - monkeypatch.setattr(talent, "TALENT_DIR", tmp_path) - _write_generator_file( + events = _run_generate_failure( tmp_path, - "missing_model_day_gen", - { - "type": "generate", - "schedule": "daily", - "priority": 10, - "output": "md", - "load": {"transcripts": True, "percepts": True}, - }, - ) - provider_module = MagicMock() - provider_module.run_generate.side_effect = NotFoundError( - "model not found", - model="gemini-3.5-flash", - llm_provider="gemini", - ) - monkeypatch.setattr( - "solstone.think.providers.get_provider_module", - lambda _provider: provider_module, - ) - monkeypatch.setenv("GOOGLE_API_KEY", "test-key") - - events = run_generator_with_config( - mod, - { - "name": "missing_model_day_gen", - "day": "20240101", - "output": "md", - }, monkeypatch, + NotFoundError( + "model not found", + model="gemini-3.5-flash", + llm_provider="gemini", + ), ) error_events = [event for event in events if event["event"] == "error"] @@ -362,6 +374,31 @@ def test_generate_model_not_found_records_runtime_failure(tmp_path, monkeypatch) assert record["evidence"]["generate"]["reason_code"] == "model_not_found" +def test_generate_marked_transport_404_records_runtime_failure(tmp_path, monkeypatch): + from solstone.think.providers.brain_state import inspect_brain_state + from solstone.think.providers.shared import mark_cloud_model_request + + class ProviderCatalogError(Exception): + status_code = 404 + + exc = ProviderCatalogError("missing model") + mark_cloud_model_request(exc) + + events = _run_generate_failure(tmp_path, monkeypatch, exc) + + error_events = [event for event in events if event["event"] == "error"] + assert len(error_events) == 1 + assert error_events[0]["reason_code"] == "model_not_found" + assert error_events[0]["provider"] == "google" + assert [event for event in events if event["event"] == "finish"] == [] + + inspection = inspect_brain_state(datetime.now(timezone.utc), journal_path=tmp_path) + record = inspection["record"] + assert record is not None + assert record["reason_code"] == "model_not_found" + assert record["evidence"]["generate"]["reason_code"] == "model_not_found" + + def test_execute_generate_provider_blank_rejected_when_config_switches_in_flight( tmp_path, monkeypatch, diff --git a/tests/test_openhands_generate.py b/tests/test_openhands_generate.py index 91b6403f3..a83f83ec6 100644 --- a/tests/test_openhands_generate.py +++ b/tests/test_openhands_generate.py @@ -15,10 +15,15 @@ from solstone.think.providers.cli import ProviderKeyMissingError from solstone.think.providers.shared import ( CANNED_GENERATE_MAX_OUTPUT_TOKENS, CANNED_GENERATE_NUM_RETRIES, + classify_provider_error, ) from tests.openhands_fakes import install_fake_openhands +class ProviderCatalogError(Exception): + status_code = 404 + + @pytest.fixture def fake_openhands(monkeypatch): return install_fake_openhands(monkeypatch) @@ -386,6 +391,35 @@ async def test_run_agenerate_transport_kwargs_do_not_shadow_llm_timeout( assert "timeout" not in getattr(llm, transport_attr) +@pytest.mark.parametrize( + ("provider", "model", "transport_attr"), + [ + ("google", "gemini-3.5-flash", "acompletion"), + ("openai", "gpt-5.5", "aresponses"), + ], +) +@pytest.mark.asyncio +async def test_run_agenerate_marks_generic_transport_404( + fake_openhands, + monkeypatch, + provider, + model, + transport_attr, +): + monkeypatch.setenv(openhands._API_KEY_ENV[provider], "test-key") + + async def missing_model(self, messages, **kwargs): + del self, messages, kwargs + raise ProviderCatalogError("missing model") + + monkeypatch.setattr(fake_openhands.LLM, transport_attr, missing_model) + + with pytest.raises(ProviderCatalogError) as raised: + await openhands.run_agenerate("hello", model, provider=provider) + + assert classify_provider_error(raised.value, provider) == "model_not_found" + + def test_validation_probe_uses_canned_generate_transport_contract(fake_openhands): assert openhands.validate_key("openai", "key") == {"valid": True} assert fake_openhands.LLM.instances[-1].max_output_tokens == ( @@ -407,15 +441,18 @@ def test_validation_probe_uses_canned_generate_transport_contract(fake_openhands def test_validation_uses_runtime_probe_and_classifies_results(monkeypatch): + from litellm.exceptions import NotFoundError + monkeypatch.setattr(openhands, "_probe", lambda *args: None) assert openhands.validate_key("google", "key") == {"valid": True} assert openhands.validate_model("openai", "gpt-5.5", "key") == {"valid": True} - class NotFoundError(RuntimeError): - status_code = 404 - def missing(*_args): - raise NotFoundError("missing") + raise NotFoundError( + "model not found", + model="gemini-3.5-flash", + llm_provider="gemini", + ) monkeypatch.setattr(openhands, "_probe", missing) assert openhands.validate_key("google", "key") == { @@ -443,6 +480,35 @@ def test_validation_uses_runtime_probe_and_classifies_results(monkeypatch): ) +@pytest.mark.parametrize( + ("provider", "model", "transport_attr"), + [ + ("google", "gemini-3.5-flash", "completion"), + ("openai", "gpt-5.5", "responses"), + ], +) +def test_validation_probe_marks_generic_transport_404( + fake_openhands, + monkeypatch, + provider, + model, + transport_attr, +): + def missing_model(self, messages, **kwargs): + del self, messages, kwargs + raise ProviderCatalogError("missing model") + + monkeypatch.setattr(fake_openhands.LLM, transport_attr, missing_model) + + assert openhands.validate_key(provider, "key") == { + "valid": True, + "probe_reason_code": "model_not_found", + } + assert openhands.validate_model(provider, model, "key")["reason_code"] == ( + "model_not_found" + ) + + def test_validation_model_not_found_uses_shared_cloud_predicate(monkeypatch): from litellm.exceptions import NotFoundError diff --git a/tests/test_provider_error_classification.py b/tests/test_provider_error_classification.py index 15f8456e5..836c181ba 100644 --- a/tests/test_provider_error_classification.py +++ b/tests/test_provider_error_classification.py @@ -10,6 +10,7 @@ from solstone.think.providers.cli import ProviderKeyMissingError, QuotaExhausted from solstone.think.providers.shared import ( _CONTEXT_WINDOW_PATTERNS, classify_provider_error, + mark_cloud_model_request, ) _OPENHANDS_BAD_REQUEST_GOOGLE_OUTAGE = ( @@ -177,6 +178,65 @@ def test_anthropic_not_found_shape_classifies_model_not_found_when_installed(): assert classify_provider_error(exc, "local") == "unknown" +class ProviderModelLookupError(Exception): + status_code = 404 + + +class ProviderWrapperError(Exception): + status_code = 500 + + +def test_marked_cloud_model_request_404_classifies_model_not_found(): + for provider in ("google", "openai", "anthropic"): + exc = ProviderModelLookupError("missing model") + mark_cloud_model_request(exc) + assert classify_provider_error(exc, provider) == "model_not_found" + + +def test_marked_cloud_model_request_uses_status_from_chain_after_unwrap(): + inner = ProviderModelLookupError("missing model") + outer = RuntimeError("transport wrapper") + outer.__cause__ = inner + + mark_cloud_model_request(outer) + + assert classify_provider_error(outer, "google") == "model_not_found" + assert classify_provider_error(inner, "google") == "model_not_found" + + +def test_marked_cloud_model_request_uses_inner_404_when_outer_has_status(): + inner = ProviderModelLookupError("missing model") + outer = ProviderWrapperError("transport wrapper") + outer.__cause__ = inner + + mark_cloud_model_request(outer) + + assert classify_provider_error(outer, "google") == "model_not_found" + assert classify_provider_error(inner, "google") == "model_not_found" + + +def test_unmarked_cloud_model_request_404_stays_unknown(): + for provider in ("google", "openai", "anthropic", "local"): + assert classify_provider_error( + ProviderModelLookupError("missing"), provider + ) == ("unknown") + + inner = ProviderModelLookupError("missing model") + outer = RuntimeError("transport wrapper") + outer.__cause__ = inner + + assert classify_provider_error(outer, "google") == "unknown" + + +def test_lookalike_not_found_name_without_trusted_identity_stays_unknown(): + class WidgetNotFoundError(Exception): + status_code = 404 + + assert classify_provider_error(WidgetNotFoundError("missing"), "google") == ( + "unknown" + ) + + def test_status_only_404_shapes_do_not_classify_model_not_found(): import httpx diff --git a/tests/test_talents_ndjson.py b/tests/test_talents_ndjson.py index 78988baa3..461296571 100644 --- a/tests/test_talents_ndjson.py +++ b/tests/test_talents_ndjson.py @@ -218,6 +218,46 @@ def test_ndjson_cogitate_model_not_found_records_runtime_failure( assert record["evidence"]["cogitate"]["reason_code"] == "model_not_found" +def test_ndjson_cogitate_generic_tool_404_stays_unknown( + mock_journal, monkeypatch, capsys +): + class ToolLookupError(Exception): + status_code = 404 + + _write_ready_brain(mock_journal) + + async def run_cogitate(config, on_event=None): + del config, on_event + raise ToolLookupError("tool endpoint missing") + + monkeypatch.setattr( + "solstone.think.providers.get_provider_module", + lambda _provider: SimpleNamespace(run_cogitate=run_cogitate), + ) + monkeypatch.setattr("solstone.think.talents.prepare_config", mock_prepare_config) + monkeypatch.setattr("sys.stdin", StringIO(json.dumps({"prompt": "use tools"}))) + mock_args = MagicMock() + mock_args.verbose = False + mock_args.dry_run = False + + from solstone.think.talents import main_async + + with patch("solstone.think.talents.setup_cli", return_value=mock_args): + asyncio.run(main_async()) + + captured = capsys.readouterr() + events = [json.loads(line) for line in captured.out.strip().split("\n") if line] + error_events = [event for event in events if event["event"] == "error"] + assert len(error_events) == 1 + assert error_events[0]["reason_code"] == "unknown" + assert error_events[0]["provider"] == "google" + + record = inspect_brain_state(NOW, journal_path=mock_journal)["record"] + assert record is not None + assert record["reason_code"] is None + assert record["evidence"]["cogitate"].get("reason_code") is None + + def test_ndjson_multiple_requests(mock_journal, monkeypatch, capsys): """Test processing multiple NDJSON requests from stdin.""" requests = [