From 79b3a080afefe19ea14c2931b9e3fbeb7ae380b8 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Sat, 25 Jul 2026 21:02:38 -0600 Subject: [PATCH] feat(providers): classify marked cloud model 404s as missing model Stamp model-request provenance only around the OpenHands llm.responses / llm.completion transport invocations and their async equivalents. Those helpers are reached by built-in cloud generate and the validation probe; local and BYO endpoints route through local.py instead, so they never receive this marker. A generic 404 now becomes model_not_found only when the marker is present in the exception chain and some chain item reports status 404. Trusted provider SDK NotFound-shaped classes keep the identity path, constrained to litellm.exceptions, openai, and anthropic namespaces, so lookalike local classes remain unknown. The chain status check uses membership semantics so an SDK or OpenHands wrapper with its own non-404 status cannot erase an inner 404. run_cogitate deliberately remains unmarked because the model call happens inside Conversation / Agent.astep with no seam we own that separates it from tool execution. Cloud tool 404s therefore stay out of model_not_found unless they also carry trusted SDK NotFound identity. Validation note: full make ci on this reviewed tree reported two failures outside this diff: tests/test_observer_client_bundle.py::test_observer_client_bundle_old_fixture_tree_absence_and_residue_do_not_affect_outputs and solstone/apps/speakers/tests/test_routes.py::test_api_owner_bootstrap_full_loop_from_stubbed_labels. Both passed when run individually; attribution is under separate investigation, so this commit does not claim the full gate is green. Co-Authored-By: Claude Opus 5 (1M context) --- solstone/think/providers/openhands.py | 29 ++++-- solstone/think/providers/shared.py | 47 ++++++++- tests/test_generate_full.py | 109 +++++++++++++------- tests/test_openhands_generate.py | 74 ++++++++++++- tests/test_provider_error_classification.py | 60 +++++++++++ tests/test_talents_ndjson.py | 40 +++++++ 6 files changed, 304 insertions(+), 55 deletions(-) diff --git a/solstone/think/providers/openhands.py b/solstone/think/providers/openhands.py index 8648aa010..e3ef0d04f 100644 --- a/solstone/think/providers/openhands.py +++ b/solstone/think/providers/openhands.py @@ -70,6 +70,7 @@ from solstone.think.providers.shared import ( classify_provider_error, exception_chain, is_cloud_model_not_found, + mark_cloud_model_request, safe_raw, validate_generate_result_strict, ) @@ -657,11 +658,15 @@ def _run_generate( thinking_budget=thinking_budget, responses_api=responses_api, ) - response = ( - llm.responses(messages, **call_kwargs) - if responses_api - else llm.completion(messages, **call_kwargs) - ) + try: + response = ( + llm.responses(messages, **call_kwargs) + if responses_api + else llm.completion(messages, **call_kwargs) + ) + except Exception as exc: + mark_cloud_model_request(exc) + raise return _generate_result(response, model) @@ -697,11 +702,15 @@ async def _run_agenerate( thinking_budget=thinking_budget, responses_api=responses_api, ) - response = ( - await llm.aresponses(messages, **call_kwargs) - if responses_api - else await llm.acompletion(messages, **call_kwargs) - ) + try: + response = ( + await llm.aresponses(messages, **call_kwargs) + if responses_api + else await llm.acompletion(messages, **call_kwargs) + ) + except Exception as exc: + mark_cloud_model_request(exc) + raise return _generate_result(response, model) diff --git a/solstone/think/providers/shared.py b/solstone/think/providers/shared.py index ead972651..ac0bf4986 100644 --- a/solstone/think/providers/shared.py +++ b/solstone/think/providers/shared.py @@ -168,6 +168,8 @@ _CONTEXT_WINDOW_PATTERNS = ( "longer than the model's context length", "context length exceeded", ) +_CLOUD_MODEL_REQUEST_ATTR = "_solstone_cloud_model_request" +_TRUSTED_MODEL_NOT_FOUND_MODULES = ("litellm.exceptions", "openai", "anthropic") def _status_code(exc: BaseException) -> int | None: @@ -203,12 +205,45 @@ def exception_chain(exc: BaseException) -> list[BaseException]: return chain -# Deliberately class-shape based: real cloud SDK missing-model 404s are -# NotFoundError-shaped classes (litellm, openai, anthropic), while a bare 404 -# can come from an unrelated endpoint. Do not broaden this to status alone. +def _chain_has_status_code(exc: BaseException, code: int) -> bool: + return any(_status_code(item) == code for item in exception_chain(exc)) + + +def mark_cloud_model_request(exc: BaseException) -> None: + for item in exception_chain(exc): + try: + setattr(item, _CLOUD_MODEL_REQUEST_ATTR, True) + except Exception: + # SDK exceptions that reject attributes must not replace the real failure. + continue + + +def has_cloud_model_request_mark(exc: BaseException) -> bool: + return any( + getattr(item, _CLOUD_MODEL_REQUEST_ATTR, False) is True + for item in exception_chain(exc) + ) + + +def _is_trusted_model_not_found(exc: BaseException) -> bool: + exc_type = type(exc) + if "notfound" not in exc_type.__name__.lower(): + return False + return any( + _module_matches(exc_type.__module__, module) + for module in _TRUSTED_MODEL_NOT_FOUND_MODULES + ) + + +# Missing-model classification first requires a built-in cloud provider. Trusted +# SDK NotFound-shaped exceptions identify provider model misses by class/module; +# otherwise a bare 404 is accepted only when narrow generate/probe transport +# provenance is present on the exception chain. Status alone is insufficient. def is_cloud_model_not_found(exc: BaseException, provider: str) -> bool: - return is_cloud_provider(provider) and any( - "notfound" in type(item).__name__.lower() for item in exception_chain(exc) + if not is_cloud_provider(provider): + return False + return any(_is_trusted_model_not_found(item) for item in exception_chain(exc)) or ( + has_cloud_model_request_mark(exc) and _chain_has_status_code(exc, 404) ) @@ -689,6 +724,8 @@ __all__ = [ "classify_canned_generate", "classify_provider_error", "exception_chain", + "has_cloud_model_request_mark", "is_cloud_model_not_found", + "mark_cloud_model_request", "safe_raw", ] diff --git a/tests/test_generate_full.py b/tests/test_generate_full.py index 32d15437a..040854a6a 100644 --- a/tests/test_generate_full.py +++ b/tests/test_generate_full.py @@ -171,6 +171,48 @@ def _write_schema_file(tmp_path: Path, name: str, schema: dict) -> None: (tmp_path / name).write_text(json.dumps(schema, indent=2), encoding="utf-8") +def _run_generate_failure( + tmp_path: Path, + monkeypatch, + side_effect: Exception, +) -> list[dict]: + mod = importlib.import_module("solstone.think.talents") + copy_day(tmp_path, monkeypatch) + _write_ready_brain_record(tmp_path) + + import solstone.think.talent as talent + + monkeypatch.setattr(talent, "TALENT_DIR", tmp_path) + _write_generator_file( + tmp_path, + "missing_model_day_gen", + { + "type": "generate", + "schedule": "daily", + "priority": 10, + "output": "md", + "load": {"transcripts": True, "percepts": True}, + }, + ) + provider_module = MagicMock() + provider_module.run_generate.side_effect = side_effect + monkeypatch.setattr( + "solstone.think.providers.get_provider_module", + lambda _provider: provider_module, + ) + monkeypatch.setenv("GOOGLE_API_KEY", "test-key") + + return run_generator_with_config( + mod, + { + "name": "missing_model_day_gen", + "day": "20240101", + "output": "md", + }, + monkeypatch, + ) + + def test_generate_output_ndjson(tmp_path, monkeypatch): """Test basic output generation via NDJSON protocol.""" mod = importlib.import_module("solstone.think.talents") @@ -309,44 +351,14 @@ def test_generate_model_not_found_records_runtime_failure(tmp_path, monkeypatch) from solstone.think.providers.brain_state import inspect_brain_state - mod = importlib.import_module("solstone.think.talents") - copy_day(tmp_path, monkeypatch) - _write_ready_brain_record(tmp_path) - - import solstone.think.talent as talent - - monkeypatch.setattr(talent, "TALENT_DIR", tmp_path) - _write_generator_file( + events = _run_generate_failure( tmp_path, - "missing_model_day_gen", - { - "type": "generate", - "schedule": "daily", - "priority": 10, - "output": "md", - "load": {"transcripts": True, "percepts": True}, - }, - ) - provider_module = MagicMock() - provider_module.run_generate.side_effect = NotFoundError( - "model not found", - model="gemini-3.5-flash", - llm_provider="gemini", - ) - monkeypatch.setattr( - "solstone.think.providers.get_provider_module", - lambda _provider: provider_module, - ) - monkeypatch.setenv("GOOGLE_API_KEY", "test-key") - - events = run_generator_with_config( - mod, - { - "name": "missing_model_day_gen", - "day": "20240101", - "output": "md", - }, monkeypatch, + NotFoundError( + "model not found", + model="gemini-3.5-flash", + llm_provider="gemini", + ), ) error_events = [event for event in events if event["event"] == "error"] @@ -362,6 +374,31 @@ def test_generate_model_not_found_records_runtime_failure(tmp_path, monkeypatch) assert record["evidence"]["generate"]["reason_code"] == "model_not_found" +def test_generate_marked_transport_404_records_runtime_failure(tmp_path, monkeypatch): + from solstone.think.providers.brain_state import inspect_brain_state + from solstone.think.providers.shared import mark_cloud_model_request + + class ProviderCatalogError(Exception): + status_code = 404 + + exc = ProviderCatalogError("missing model") + mark_cloud_model_request(exc) + + events = _run_generate_failure(tmp_path, monkeypatch, exc) + + error_events = [event for event in events if event["event"] == "error"] + assert len(error_events) == 1 + assert error_events[0]["reason_code"] == "model_not_found" + assert error_events[0]["provider"] == "google" + assert [event for event in events if event["event"] == "finish"] == [] + + inspection = inspect_brain_state(datetime.now(timezone.utc), journal_path=tmp_path) + record = inspection["record"] + assert record is not None + assert record["reason_code"] == "model_not_found" + assert record["evidence"]["generate"]["reason_code"] == "model_not_found" + + def test_execute_generate_provider_blank_rejected_when_config_switches_in_flight( tmp_path, monkeypatch, diff --git a/tests/test_openhands_generate.py b/tests/test_openhands_generate.py index 91b6403f3..a83f83ec6 100644 --- a/tests/test_openhands_generate.py +++ b/tests/test_openhands_generate.py @@ -15,10 +15,15 @@ from solstone.think.providers.cli import ProviderKeyMissingError from solstone.think.providers.shared import ( CANNED_GENERATE_MAX_OUTPUT_TOKENS, CANNED_GENERATE_NUM_RETRIES, + classify_provider_error, ) from tests.openhands_fakes import install_fake_openhands +class ProviderCatalogError(Exception): + status_code = 404 + + @pytest.fixture def fake_openhands(monkeypatch): return install_fake_openhands(monkeypatch) @@ -386,6 +391,35 @@ async def test_run_agenerate_transport_kwargs_do_not_shadow_llm_timeout( assert "timeout" not in getattr(llm, transport_attr) +@pytest.mark.parametrize( + ("provider", "model", "transport_attr"), + [ + ("google", "gemini-3.5-flash", "acompletion"), + ("openai", "gpt-5.5", "aresponses"), + ], +) +@pytest.mark.asyncio +async def test_run_agenerate_marks_generic_transport_404( + fake_openhands, + monkeypatch, + provider, + model, + transport_attr, +): + monkeypatch.setenv(openhands._API_KEY_ENV[provider], "test-key") + + async def missing_model(self, messages, **kwargs): + del self, messages, kwargs + raise ProviderCatalogError("missing model") + + monkeypatch.setattr(fake_openhands.LLM, transport_attr, missing_model) + + with pytest.raises(ProviderCatalogError) as raised: + await openhands.run_agenerate("hello", model, provider=provider) + + assert classify_provider_error(raised.value, provider) == "model_not_found" + + def test_validation_probe_uses_canned_generate_transport_contract(fake_openhands): assert openhands.validate_key("openai", "key") == {"valid": True} assert fake_openhands.LLM.instances[-1].max_output_tokens == ( @@ -407,15 +441,18 @@ def test_validation_probe_uses_canned_generate_transport_contract(fake_openhands def test_validation_uses_runtime_probe_and_classifies_results(monkeypatch): + from litellm.exceptions import NotFoundError + monkeypatch.setattr(openhands, "_probe", lambda *args: None) assert openhands.validate_key("google", "key") == {"valid": True} assert openhands.validate_model("openai", "gpt-5.5", "key") == {"valid": True} - class NotFoundError(RuntimeError): - status_code = 404 - def missing(*_args): - raise NotFoundError("missing") + raise NotFoundError( + "model not found", + model="gemini-3.5-flash", + llm_provider="gemini", + ) monkeypatch.setattr(openhands, "_probe", missing) assert openhands.validate_key("google", "key") == { @@ -443,6 +480,35 @@ def test_validation_uses_runtime_probe_and_classifies_results(monkeypatch): ) +@pytest.mark.parametrize( + ("provider", "model", "transport_attr"), + [ + ("google", "gemini-3.5-flash", "completion"), + ("openai", "gpt-5.5", "responses"), + ], +) +def test_validation_probe_marks_generic_transport_404( + fake_openhands, + monkeypatch, + provider, + model, + transport_attr, +): + def missing_model(self, messages, **kwargs): + del self, messages, kwargs + raise ProviderCatalogError("missing model") + + monkeypatch.setattr(fake_openhands.LLM, transport_attr, missing_model) + + assert openhands.validate_key(provider, "key") == { + "valid": True, + "probe_reason_code": "model_not_found", + } + assert openhands.validate_model(provider, model, "key")["reason_code"] == ( + "model_not_found" + ) + + def test_validation_model_not_found_uses_shared_cloud_predicate(monkeypatch): from litellm.exceptions import NotFoundError diff --git a/tests/test_provider_error_classification.py b/tests/test_provider_error_classification.py index 15f8456e5..836c181ba 100644 --- a/tests/test_provider_error_classification.py +++ b/tests/test_provider_error_classification.py @@ -10,6 +10,7 @@ from solstone.think.providers.cli import ProviderKeyMissingError, QuotaExhausted from solstone.think.providers.shared import ( _CONTEXT_WINDOW_PATTERNS, classify_provider_error, + mark_cloud_model_request, ) _OPENHANDS_BAD_REQUEST_GOOGLE_OUTAGE = ( @@ -177,6 +178,65 @@ def test_anthropic_not_found_shape_classifies_model_not_found_when_installed(): assert classify_provider_error(exc, "local") == "unknown" +class ProviderModelLookupError(Exception): + status_code = 404 + + +class ProviderWrapperError(Exception): + status_code = 500 + + +def test_marked_cloud_model_request_404_classifies_model_not_found(): + for provider in ("google", "openai", "anthropic"): + exc = ProviderModelLookupError("missing model") + mark_cloud_model_request(exc) + assert classify_provider_error(exc, provider) == "model_not_found" + + +def test_marked_cloud_model_request_uses_status_from_chain_after_unwrap(): + inner = ProviderModelLookupError("missing model") + outer = RuntimeError("transport wrapper") + outer.__cause__ = inner + + mark_cloud_model_request(outer) + + assert classify_provider_error(outer, "google") == "model_not_found" + assert classify_provider_error(inner, "google") == "model_not_found" + + +def test_marked_cloud_model_request_uses_inner_404_when_outer_has_status(): + inner = ProviderModelLookupError("missing model") + outer = ProviderWrapperError("transport wrapper") + outer.__cause__ = inner + + mark_cloud_model_request(outer) + + assert classify_provider_error(outer, "google") == "model_not_found" + assert classify_provider_error(inner, "google") == "model_not_found" + + +def test_unmarked_cloud_model_request_404_stays_unknown(): + for provider in ("google", "openai", "anthropic", "local"): + assert classify_provider_error( + ProviderModelLookupError("missing"), provider + ) == ("unknown") + + inner = ProviderModelLookupError("missing model") + outer = RuntimeError("transport wrapper") + outer.__cause__ = inner + + assert classify_provider_error(outer, "google") == "unknown" + + +def test_lookalike_not_found_name_without_trusted_identity_stays_unknown(): + class WidgetNotFoundError(Exception): + status_code = 404 + + assert classify_provider_error(WidgetNotFoundError("missing"), "google") == ( + "unknown" + ) + + def test_status_only_404_shapes_do_not_classify_model_not_found(): import httpx diff --git a/tests/test_talents_ndjson.py b/tests/test_talents_ndjson.py index 78988baa3..461296571 100644 --- a/tests/test_talents_ndjson.py +++ b/tests/test_talents_ndjson.py @@ -218,6 +218,46 @@ def test_ndjson_cogitate_model_not_found_records_runtime_failure( assert record["evidence"]["cogitate"]["reason_code"] == "model_not_found" +def test_ndjson_cogitate_generic_tool_404_stays_unknown( + mock_journal, monkeypatch, capsys +): + class ToolLookupError(Exception): + status_code = 404 + + _write_ready_brain(mock_journal) + + async def run_cogitate(config, on_event=None): + del config, on_event + raise ToolLookupError("tool endpoint missing") + + monkeypatch.setattr( + "solstone.think.providers.get_provider_module", + lambda _provider: SimpleNamespace(run_cogitate=run_cogitate), + ) + monkeypatch.setattr("solstone.think.talents.prepare_config", mock_prepare_config) + monkeypatch.setattr("sys.stdin", StringIO(json.dumps({"prompt": "use tools"}))) + mock_args = MagicMock() + mock_args.verbose = False + mock_args.dry_run = False + + from solstone.think.talents import main_async + + with patch("solstone.think.talents.setup_cli", return_value=mock_args): + asyncio.run(main_async()) + + captured = capsys.readouterr() + events = [json.loads(line) for line in captured.out.strip().split("\n") if line] + error_events = [event for event in events if event["event"] == "error"] + assert len(error_events) == 1 + assert error_events[0]["reason_code"] == "unknown" + assert error_events[0]["provider"] == "google" + + record = inspect_brain_state(NOW, journal_path=mock_journal)["record"] + assert record is not None + assert record["reason_code"] is None + assert record["evidence"]["cogitate"].get("reason_code") is None + + def test_ndjson_multiple_requests(mock_journal, monkeypatch, capsys): """Test processing multiple NDJSON requests from stdin.""" requests = [ -- 2.51.2