diff --git a/solstone/apps/stats/tests/test_workspace_template.py b/solstone/apps/stats/tests/test_workspace_template.py index 375d83be9..308cb2ddc 100644 --- a/solstone/apps/stats/tests/test_workspace_template.py +++ b/solstone/apps/stats/tests/test_workspace_template.py @@ -35,6 +35,7 @@ BACKLOG_COPY_KEYS = [ "BACKLOG_REASON_FAILING_STEP", "BACKLOG_REASON_MISSING_CONFIG", "BACKLOG_REASON_PROVIDER_DOWN", + "BACKLOG_REASON_PROVIDER_REFUSED", "BACKLOG_QUEUED_FEEDBACK", "BACKLOG_WHY_NEVER_ATTEMPTED", "BACKLOG_WHY_FAILED", @@ -81,6 +82,10 @@ BACKLOG_COPY_LITERALS = { "BACKLOG_REASON_FAILING_STEP": "a processing step keeps failing — try again", "BACKLOG_REASON_MISSING_CONFIG": "a setting's missing — check your journal's setup", "BACKLOG_REASON_PROVIDER_DOWN": "the AI provider was unreachable. try again", + "BACKLOG_REASON_PROVIDER_REFUSED": ( + "the AI provider refused a request sol sent — retrying won't help; " + "this is a defect in sol" + ), "BACKLOG_QUEUED_FEEDBACK": "queued, working on it now", "BACKLOG_WHY_NEVER_ATTEMPTED": "not looked at yet", "BACKLOG_WHY_FAILED": "couldn't finish — will retry", diff --git a/solstone/apps/stats/workspace.html b/solstone/apps/stats/workspace.html index 9e7b7f2c3..0eb1c84ac 100644 --- a/solstone/apps/stats/workspace.html +++ b/solstone/apps/stats/workspace.html @@ -794,6 +794,7 @@ h1 { REASON_FAILING_STEP: "a processing step keeps failing — try again", REASON_MISSING_CONFIG: "a setting's missing — check your journal's setup", REASON_PROVIDER_DOWN: "the AI provider was unreachable. try again", + REASON_PROVIDER_REFUSED: "the AI provider refused a request sol sent — retrying won't help; this is a defect in sol", WHY_NEVER_ATTEMPTED: "not looked at yet", WHY_FAILED: "couldn't finish — will retry", WHY_SENSED_NOT_THOUGHT: "taken in, not yet thought through", diff --git a/solstone/convey/backlog_copy.py b/solstone/convey/backlog_copy.py index 9af267790..5806772e4 100644 --- a/solstone/convey/backlog_copy.py +++ b/solstone/convey/backlog_copy.py @@ -34,6 +34,10 @@ BACKLOG_REASON_CORRUPT_RAW = "original raw media is missing or damaged — re-im BACKLOG_REASON_FAILING_STEP = "a processing step keeps failing — try again" BACKLOG_REASON_MISSING_CONFIG = "a setting's missing — check your journal's setup" BACKLOG_REASON_PROVIDER_DOWN = "the AI provider was unreachable. try again" +BACKLOG_REASON_PROVIDER_REFUSED = ( + "the AI provider refused a request sol sent — retrying won't help; " + "this is a defect in sol" +) BACKLOG_WHY_NEVER_ATTEMPTED = "not looked at yet" BACKLOG_WHY_FAILED = "couldn't finish — will retry" BACKLOG_WHY_SENSED_NOT_THOUGHT = "taken in, not yet thought through" @@ -57,6 +61,7 @@ __all__ = [ "BACKLOG_REASON_FAILING_STEP", "BACKLOG_REASON_MISSING_CONFIG", "BACKLOG_REASON_PROVIDER_DOWN", + "BACKLOG_REASON_PROVIDER_REFUSED", "BACKLOG_VERDICT_CANT_TELL", "BACKLOG_VERDICT_CAUGHT_UP", "BACKLOG_VERDICT_MIXED_PENDING_PLURAL", diff --git a/solstone/convey/backlog_view.py b/solstone/convey/backlog_view.py index a5ecef7d6..212b29a00 100644 --- a/solstone/convey/backlog_view.py +++ b/solstone/convey/backlog_view.py @@ -105,6 +105,8 @@ def _reason_copy(day: dict) -> str: # Follow-up: a future copy pass could add a "still starting up - try # again shortly" sentence and split startup off from provider here. return backlog_copy.BACKLOG_REASON_PROVIDER_DOWN + if category == "request": + return backlog_copy.BACKLOG_REASON_PROVIDER_REFUSED return backlog_copy.BACKLOG_REASON_FAILING_STEP diff --git a/solstone/convey/provider_readiness.py b/solstone/convey/provider_readiness.py index cc5d62b4e..98b4d03ca 100644 --- a/solstone/convey/provider_readiness.py +++ b/solstone/convey/provider_readiness.py @@ -56,6 +56,7 @@ PROVIDER_LEVEL_CODES = frozenset( "provider_key_missing", "provider_key_invalid", "provider_quota_exceeded", + "provider_request_rejected", "provider_unavailable", "network_unreachable", "local_endpoint_unreachable", @@ -244,6 +245,15 @@ _ENTRIES: dict[str, _Entry] = { detail="Wait for provider quota to reset or choose another provider.", recovery_action=None, ), + "provider_request_rejected": _Entry( + klass="request", + summary="the provider refused a request sol sent; this is a defect in sol", + detail=( + "The provider rejected the request before it could run. Retrying the " + "same work will not help because the request shape needs a code change." + ), + recovery_action=None, + ), "network_unreachable": _Entry( klass="generic", summary="I couldn't reach the network", diff --git a/solstone/convey/static/chat_reasons.js b/solstone/convey/static/chat_reasons.js index 85e5b3da8..6029594fa 100644 --- a/solstone/convey/static/chat_reasons.js +++ b/solstone/convey/static/chat_reasons.js @@ -106,6 +106,10 @@ "template": "your {provider} quota is spent", "action": null }, + "provider_request_rejected": { + "template": "the provider refused a request sol sent; this is a defect in sol", + "action": null + }, "network_unreachable": { "template": "I couldn't reach the network", "action": null diff --git a/solstone/think/cogitate_policy.py b/solstone/think/cogitate_policy.py index ec6bb473c..84e6ad2a5 100644 --- a/solstone/think/cogitate_policy.py +++ b/solstone/think/cogitate_policy.py @@ -42,6 +42,7 @@ DETERMINISTIC_FAILURE_REASON_CODES = frozenset( "context_window_exceeded", "max_turns_exhausted", "no_output", + "provider_request_rejected", "schema_invalid", "token_budget_exceeded", "wall_clock_exceeded", @@ -61,6 +62,7 @@ DETERMINISTIC_FAILURE_CAPS: dict[str, int] = { "context_window_exceeded": 2, "max_turns_exhausted": 2, "no_output": 2, + "provider_request_rejected": 1, "schema_invalid": 3, "token_budget_exceeded": 2, "wall_clock_exceeded": 2, diff --git a/solstone/think/providers/__init__.py b/solstone/think/providers/__init__.py index c63cc7998..c2cf02672 100644 --- a/solstone/think/providers/__init__.py +++ b/solstone/think/providers/__init__.py @@ -83,6 +83,13 @@ def managed_provider_env_keys() -> set[str]: return {m["env_key"] for m in PROVIDER_METADATA.values() if m.get("env_key")} +def is_cloud_provider(provider: str) -> bool: + """Return True when a registered provider uses a managed cloud API key.""" + + meta = PROVIDER_METADATA.get(provider) + return bool(meta and meta.get("env_key")) + + def get_provider_module(provider: str) -> ModuleType: """Get the provider module for the given provider name. @@ -247,5 +254,6 @@ __all__ = [ "build_provider_status", "validate_key", "validate_model", + "is_cloud_provider", "managed_provider_env_keys", ] diff --git a/solstone/think/providers/brain_state.py b/solstone/think/providers/brain_state.py index 175d43d46..2191efe8a 100644 --- a/solstone/think/providers/brain_state.py +++ b/solstone/think/providers/brain_state.py @@ -112,6 +112,7 @@ BrainReasonCode = Literal[ "provider_key_invalid", "model_not_found", "provider_quota_exceeded", + "provider_request_rejected", "provider_unavailable", "network_unreachable", "endpoint_unreachable", @@ -163,6 +164,7 @@ BRAIN_REASON_TO_AGGREGATE: dict[str, BrainAggregateState] = { "provider_key_invalid": "unhealthy", "model_not_found": "unhealthy", "provider_quota_exceeded": "unhealthy", + "provider_request_rejected": "unhealthy", "provider_unavailable": "unhealthy", "network_unreachable": "unhealthy", "endpoint_unreachable": "unhealthy", @@ -229,6 +231,7 @@ BRAIN_EVIDENCE_REASON_CODES: dict[str, frozenset[str]] = { "provider_key_invalid", "model_not_found", "provider_quota_exceeded", + "provider_request_rejected", "provider_unavailable", "network_unreachable", "endpoint_unreachable", @@ -245,6 +248,7 @@ BRAIN_EVIDENCE_REASON_CODES: dict[str, frozenset[str]] = { "provider_key_invalid", "model_not_found", "provider_quota_exceeded", + "provider_request_rejected", "provider_unavailable", "network_unreachable", "endpoint_unreachable", @@ -281,8 +285,8 @@ if set(BRAIN_REASON_TO_AGGREGATE) != BRAIN_REASON_CODES: _EVIDENCE_ALLOWED_REASON_CODES = frozenset().union( *BRAIN_EVIDENCE_REASON_CODES.values() ) -if len(_EVIDENCE_ALLOWED_REASON_CODES) != 31: - raise RuntimeError("brain evidence reason partition must contain 31 reasons") +if len(_EVIDENCE_ALLOWED_REASON_CODES) != 32: + raise RuntimeError("brain evidence reason partition must contain 32 reasons") if len(BRAIN_PROJECTION_ONLY_REASON_CODES) != 10: raise RuntimeError("brain projection-only reason partition must contain 10 reasons") if _EVIDENCE_ALLOWED_REASON_CODES & BRAIN_PROJECTION_ONLY_REASON_CODES: diff --git a/solstone/think/providers/local.py b/solstone/think/providers/local.py index 1c2125a88..ea85b2a2e 100644 --- a/solstone/think/providers/local.py +++ b/solstone/think/providers/local.py @@ -24,7 +24,6 @@ from typing import Any, Literal from solstone.think.models import LOCAL_MODEL from solstone.think.providers._image import encode_image_part, is_image_part from solstone.think.providers.local_endpoint import ( - ENDPOINT_ERROR_BODY_CAP_CHARS, LOCAL_ENDPOINT_CONTRACT_COPY, LOCAL_ENDPOINT_UNREACHABLE_COPY, classify_byo_cogitate_error, @@ -37,6 +36,7 @@ from solstone.think.providers.local_endpoint import ( ) from solstone.think.providers.shared import ( _CONTEXT_WINDOW_PATTERNS, + PROVIDER_ERROR_TEXT_CAP_CHARS, GenerateResult, _contains_any, classify_provider_error, @@ -505,7 +505,7 @@ def _classify_byo_generate_error( body_text = getattr(response, "text", None) if isinstance(body_text, str) and body_text: excerpt = redact_local_endpoint_credential( - body_text[:ENDPOINT_ERROR_BODY_CAP_CHARS], + body_text[:PROVIDER_ERROR_TEXT_CAP_CHARS], endpoint, ) if _contains_any(excerpt.lower(), _CONTEXT_WINDOW_PATTERNS): @@ -1299,7 +1299,7 @@ async def run_cogitate( error_text = redact_local_endpoint_credential(error_text, endpoint) trace_text = redact_local_endpoint_credential(trace_text, endpoint) if not fixed_copy: - error_text = error_text[:ENDPOINT_ERROR_BODY_CAP_CHARS] + error_text = error_text[:PROVIDER_ERROR_TEXT_CAP_CHARS] on_event( { "event": "error", diff --git a/solstone/think/providers/local_endpoint.py b/solstone/think/providers/local_endpoint.py index 477a9bf79..7d62b5437 100644 --- a/solstone/think/providers/local_endpoint.py +++ b/solstone/think/providers/local_endpoint.py @@ -13,7 +13,11 @@ from dataclasses import dataclass from typing import Any from solstone.think.journal_config import read_journal_config -from solstone.think.providers.shared import _CONTEXT_WINDOW_PATTERNS, _contains_any +from solstone.think.providers.shared import ( + _CONTEXT_WINDOW_PATTERNS, + PROVIDER_ERROR_TEXT_CAP_CHARS, + _contains_any, +) LOG = logging.getLogger(__name__) @@ -35,7 +39,6 @@ ENDPOINT_SERVED_CONTEXT_WINDOW_CONFIG_KEY = "served_context_window" ENDPOINT_SERVED_CONTEXT_WINDOW_MIN_TOKENS = 2048 ENDPOINT_SERVED_WINDOW_CACHE_TTL_S = 300.0 ENDPOINT_MODELS_TIMEOUT_S = 2.5 -ENDPOINT_ERROR_BODY_CAP_CHARS = 4096 _SERVED_WINDOW_CACHE: dict[tuple[str, str], tuple[float, int | None]] = {} @@ -345,7 +348,7 @@ def _payload_text(payload: Any, credential: str | None) -> str | None: text = str(redacted) if not text: return None - return text[:ENDPOINT_ERROR_BODY_CAP_CHARS] + return text[:PROVIDER_ERROR_TEXT_CAP_CHARS] def _candidate_exception_texts( @@ -521,7 +524,6 @@ def wrap_on_event_redacting( __all__ = [ - "ENDPOINT_ERROR_BODY_CAP_CHARS", "ENDPOINT_MODELS_TIMEOUT_S", "ENDPOINT_SERVED_CONTEXT_WINDOW_CONFIG_KEY", "ENDPOINT_SERVED_CONTEXT_WINDOW_MIN_TOKENS", diff --git a/solstone/think/providers/openhands.py b/solstone/think/providers/openhands.py index f2080eae6..fb1424144 100644 --- a/solstone/think/providers/openhands.py +++ b/solstone/think/providers/openhands.py @@ -63,6 +63,7 @@ from solstone.think.providers.shared import ( CANNED_GENERATE_PROMPT, CANNED_GENERATE_THINKING_BUDGET, CANNED_GENERATE_TIMEOUT_S, + PROVIDER_ERROR_TEXT_CAP_CHARS, USAGE_KEYS, GenerateResult, JSONEventCallback, @@ -1928,7 +1929,6 @@ async def run_cogitate( local_endpoint = None if provider == "local": from solstone.think.providers.local_endpoint import ( - ENDPOINT_ERROR_BODY_CAP_CHARS, classify_byo_cogitate_error, local_endpoint_reason_copy, redact_exception_credential, @@ -1942,14 +1942,12 @@ async def run_cogitate( setattr(exc, "reason_code", reason_code) setattr(provider_exc, "reason_code", reason_code) reason_code = reason_code or classify_provider_error(provider_exc, provider) - error_text = str(exc) + error_text = str(exc)[:PROVIDER_ERROR_TEXT_CAP_CHARS] trace_text = traceback.format_exc() if local_endpoint is not None: fixed_copy = local_endpoint_reason_copy(reason_code) if fixed_copy: error_text = fixed_copy - else: - error_text = error_text[:ENDPOINT_ERROR_BODY_CAP_CHARS] if reason_code == "provider_quota_exceeded": raise QuotaExhaustedError( str(provider_exc), _retry_delay_ms(provider_exc) diff --git a/solstone/think/providers/shared.py b/solstone/think/providers/shared.py index 4d8a8ef53..a8c3e81ad 100644 --- a/solstone/think/providers/shared.py +++ b/solstone/think/providers/shared.py @@ -19,6 +19,7 @@ from typing import Any, Callable, Literal, Mapping, Optional, Union from typing_extensions import Required, TypedDict +from solstone.think.providers import is_cloud_provider from solstone.think.utils import now_ms # --------------------------------------------------------------------------- @@ -204,6 +205,7 @@ RUNTIME_REASON_CODES = frozenset( "local_queue_timeout", "max_turns_exhausted", "network_unreachable", + "provider_request_rejected", "provider_unavailable", "provider_response_invalid", "incomplete_json_length", @@ -213,6 +215,9 @@ RUNTIME_REASON_CODES = frozenset( ) +PROVIDER_ERROR_TEXT_CAP_CHARS = 4096 + + def classify_provider_error(exc: BaseException, provider: str) -> str: """Return a chat reason code for a provider exception.""" try: @@ -358,6 +363,14 @@ def classify_provider_error(exc: BaseException, provider: str) -> str: if "internalservererror" in exc_name_lower or "servererror" in exc_name_lower: return "provider_unavailable" + if ( + is_cloud_provider(provider) + and _module_matches(exc_module, "litellm.exceptions") + and exc_name == "BadRequestError" + and status_code == 400 + ): + return "provider_request_rejected" + return "unknown" except Exception: return "unknown" @@ -639,6 +652,7 @@ __all__ = [ "Event", "GenerateResult", "JSONEventCallback", + "PROVIDER_ERROR_TEXT_CAP_CHARS", "RUNTIME_REASON_CODES", "ThinkingEvent", "USAGE_KEYS", diff --git a/solstone/think/talents.py b/solstone/think/talents.py index 7ce2081b3..33ace91bd 100644 --- a/solstone/think/talents.py +++ b/solstone/think/talents.py @@ -92,6 +92,7 @@ _BRAIN_INGRESS_REASONS: frozenset[str] = frozenset( "provider_key_invalid", "model_not_found", "provider_quota_exceeded", + "provider_request_rejected", "provider_unavailable", "network_unreachable", "endpoint_unreachable", diff --git a/tests/test_backlog_view.py b/tests/test_backlog_view.py index 5e47d371c..110f8e1f8 100644 --- a/tests/test_backlog_view.py +++ b/tests/test_backlog_view.py @@ -44,6 +44,7 @@ def _assert_single_stuck_reason(reason_code: str, expected: str) -> None: ("local_endpoint_unreachable", backlog_copy.BACKLOG_REASON_PROVIDER_DOWN), ("provider_quota_exceeded", backlog_copy.BACKLOG_REASON_PROVIDER_DOWN), ("provider_key_invalid", backlog_copy.BACKLOG_REASON_PROVIDER_DOWN), + ("provider_request_rejected", backlog_copy.BACKLOG_REASON_PROVIDER_REFUSED), ("catchup_backoff", backlog_copy.BACKLOG_REASON_FAILING_STEP), ("totally_made_up_code", backlog_copy.BACKLOG_REASON_FAILING_STEP), ], @@ -54,6 +55,23 @@ def test_stuck_rows_maps_reason_categories_to_backlog_copy( _assert_single_stuck_reason(reason_code, expected) +def test_stuck_rows_provider_request_rejected_copy_is_distinct_and_actionable(): + _assert_single_stuck_reason( + "provider_request_rejected", + backlog_copy.BACKLOG_REASON_PROVIDER_REFUSED, + ) + + reason = backlog_copy.BACKLOG_REASON_PROVIDER_REFUSED + assert "try again" not in reason + assert "unreachable" not in reason + assert reason not in { + backlog_copy.BACKLOG_REASON_CORRUPT_RAW, + backlog_copy.BACKLOG_REASON_FAILING_STEP, + backlog_copy.BACKLOG_REASON_MISSING_CONFIG, + backlog_copy.BACKLOG_REASON_PROVIDER_DOWN, + } + + def test_stuck_rows_maps_readiness_reasons_and_carries_operator_fields(): backlog = { "days": [ diff --git a/tests/test_brain_state.py b/tests/test_brain_state.py index fcf539c72..38fda3dd7 100644 --- a/tests/test_brain_state.py +++ b/tests/test_brain_state.py @@ -267,6 +267,7 @@ def test_vocabularies_and_reason_mapping_are_closed() -> None: "provider_key_invalid", "model_not_found", "provider_quota_exceeded", + "provider_request_rejected", "provider_unavailable", "network_unreachable", "endpoint_unreachable", @@ -316,7 +317,7 @@ def test_vocabularies_and_reason_mapping_are_closed() -> None: assert set(BRAIN_REASON_TO_AGGREGATE) == BRAIN_REASON_CODES assert set(BRAIN_REASON_TO_AGGREGATE.values()) <= BRAIN_AGGREGATE_STATES evidence_reasons = frozenset().union(*BRAIN_EVIDENCE_REASON_CODES.values()) - assert len(evidence_reasons) == 31 + assert len(evidence_reasons) == 32 assert len(BRAIN_PROJECTION_ONLY_REASON_CODES) == 10 assert evidence_reasons | BRAIN_PROJECTION_ONLY_REASON_CODES == BRAIN_REASON_CODES assert not (evidence_reasons & BRAIN_PROJECTION_ONLY_REASON_CODES) @@ -1545,6 +1546,38 @@ def test_runtime_failure_preserves_other_same_fingerprint_evidence( assert cogitate_record["evidence"]["cogitate"]["status"] == "failed" +def test_runtime_failure_provider_request_rejected_transitions_ready_to_unhealthy( + tmp_path: Path, +) -> None: + config = _cloud_config() + _write_ready_record(tmp_path, config) + assert ( + inspect_brain_state(NOW, journal_path=tmp_path)["projection"]["aggregate_state"] + == "ready" + ) + expected = _current_fingerprint(tmp_path, config) + + result = record_brain_runtime_failure( + "provider_request_rejected", + NOW, + expected_fingerprint_sha256=expected, + component="generate", + journal_path=tmp_path, + ) + + assert result["accepted"] is True + record = result["record"] + assert record is not None + assert record["aggregate_state"] == "unhealthy" + assert record["evidence"]["generate"] is not None + assert record["evidence"]["generate"]["status"] == "failed" + assert record["evidence"]["generate"]["reason_code"] == "provider_request_rejected" + assert ( + inspect_brain_state(NOW, journal_path=tmp_path)["projection"]["aggregate_state"] + == "unhealthy" + ) + + def test_runtime_failure_drops_evidence_from_prior_fingerprint(tmp_path: Path) -> None: config_f1 = _cloud_config(key="first") _write_ready_record(tmp_path, config_f1) diff --git a/tests/test_chat_reasons.py b/tests/test_chat_reasons.py index 143cbac1e..b3380ede0 100644 --- a/tests/test_chat_reasons.py +++ b/tests/test_chat_reasons.py @@ -35,6 +35,7 @@ EXPECTED_CODES = { "cuda_runtime_incomplete", "provider_key_invalid", "provider_quota_exceeded", + "provider_request_rejected", "network_unreachable", "provider_response_invalid", "provider_unavailable", diff --git a/tests/test_cogitate_policy.py b/tests/test_cogitate_policy.py index c55b177a5..f0286b877 100644 --- a/tests/test_cogitate_policy.py +++ b/tests/test_cogitate_policy.py @@ -50,6 +50,10 @@ def test_failure_capped_default_deterministic_cap_is_two(): assert cogitate_policy.failure_capped("context_window_exceeded", 2) is True +def test_failure_capped_provider_request_rejected_is_one(): + assert cogitate_policy.failure_capped("provider_request_rejected", 1) is True + + def test_deterministic_failure_caps_cover_reason_codes_exactly(): assert ( set(cogitate_policy.DETERMINISTIC_FAILURE_CAPS) diff --git a/tests/test_lane_failure_honesty.py b/tests/test_lane_failure_honesty.py index 9ddab6392..0d57c8559 100644 --- a/tests/test_lane_failure_honesty.py +++ b/tests/test_lane_failure_honesty.py @@ -625,6 +625,9 @@ def test_brain_runtime_failure_helper_records_only_allowed_ingress( ) talents_module._record_brain_runtime_failure("network_unreachable", "cogitate") + talents_module._record_brain_runtime_failure( + "provider_request_rejected", "generate" + ) talents_module._record_brain_runtime_failure("context_window_exceeded", "generate") talents_module._record_brain_runtime_failure("cogitate_terminal_error", "generate") @@ -634,10 +637,49 @@ def test_brain_runtime_failure_helper_records_only_allowed_ingress( "expected_fingerprint_sha256": "fingerprint", "component": "cogitate", "diagnostic": {}, - } + }, + { + "reason_code": "provider_request_rejected", + "expected_fingerprint_sha256": "fingerprint", + "component": "generate", + "diagnostic": {}, + }, ] +def test_bad_request_local_and_no_brain_do_not_record_brain_runtime_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.exceptions import BadRequestError + + from solstone.think.providers.shared import classify_provider_error + + recorded: list[dict[str, Any]] = [] + monkeypatch.setattr( + talents_module, + "_read_runtime_fingerprint", + lambda: "fingerprint", + ) + monkeypatch.setattr( + "solstone.think.providers.brain_state.record_brain_runtime_failure", + lambda reason_code, *_args, **kwargs: ( + recorded.append({"reason_code": reason_code, **kwargs}) + or {"accepted": True} + ), + ) + + for provider in ("local", "none"): + exc = BadRequestError( + "Invalid value for parameter 'temperature'", + model="local-model", + llm_provider=provider, + ) + reason_code = classify_provider_error(exc, provider) + talents_module._record_brain_runtime_failure(reason_code, "generate") + + assert recorded == [] + + @pytest.mark.parametrize( ("reason", "finish_reason", "expected"), [ diff --git a/tests/test_local.py b/tests/test_local.py index f519fab1d..dca787fd5 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -3335,7 +3335,7 @@ def test_run_cogitate_byo_classified_error_uses_fixed_copy_and_redacts( def test_run_cogitate_byo_context_error_event_caps_error_field(monkeypatch): - from solstone.think.providers.local_endpoint import ENDPOINT_ERROR_BODY_CAP_CHARS + from solstone.think.providers.shared import PROVIDER_ERROR_TEXT_CAP_CHARS provider = _provider() token = "SENTINEL-BYO-CONTEXT-CRED-219a" @@ -3371,7 +3371,7 @@ def test_run_cogitate_byo_context_error_event_caps_error_field(monkeypatch): ) assert events[0]["reason_code"] == "context_window_exceeded" - assert len(events[0]["error"]) <= ENDPOINT_ERROR_BODY_CAP_CHARS + assert len(events[0]["error"]) <= PROVIDER_ERROR_TEXT_CAP_CHARS assert token not in events[0]["error"] assert token not in events[0]["trace"] diff --git a/tests/test_local_endpoint.py b/tests/test_local_endpoint.py index 91d400bc9..e2ad105a2 100644 --- a/tests/test_local_endpoint.py +++ b/tests/test_local_endpoint.py @@ -11,6 +11,7 @@ from aiohttp import ClientConnectorError from aiohttp.client_reqrep import ConnectionKey from solstone.think.providers import local_endpoint +from solstone.think.providers.shared import PROVIDER_ERROR_TEXT_CAP_CHARS def _config(payload: dict) -> dict: @@ -335,7 +336,7 @@ def test_classify_byo_cogitate_error_contract_by_status_or_name(): def test_byo_exception_matches_context_window_caps_body_excerpt(): exc = BadRequestError("bad request") exc.body = ( - "x" * (local_endpoint.ENDPOINT_ERROR_BODY_CAP_CHARS + 1) + "x" * (PROVIDER_ERROR_TEXT_CAP_CHARS + 1) + "longer than the model's context length" ) assert local_endpoint.byo_exception_matches_context_window(exc) is False diff --git a/tests/test_openhands_errors.py b/tests/test_openhands_errors.py index 77b1319b4..a21c4ee76 100644 --- a/tests/test_openhands_errors.py +++ b/tests/test_openhands_errors.py @@ -16,11 +16,11 @@ from solstone.think.models import LOCAL_MODEL from solstone.think.providers import openhands from solstone.think.providers.cli import ProviderKeyMissingError, QuotaExhaustedError from solstone.think.providers.local_endpoint import ( - ENDPOINT_ERROR_BODY_CAP_CHARS, LOCAL_ENDPOINT_CONTRACT_COPY, LOCAL_ENDPOINT_UNREACHABLE_COPY, LocalEndpoint, ) +from solstone.think.providers.shared import PROVIDER_ERROR_TEXT_CAP_CHARS from solstone.think.talents import TalentHookError from tests.openhands_fakes import install_fake_openhands @@ -134,6 +134,27 @@ def test_run_cogitate_generic_error_emits_event_and_marks_evented( assert events[0]["ts"] == 123456 +def test_run_cogitate_cloud_error_event_caps_error_text( + fake_openhands, + run_env, +): + generic_exc = RuntimeError("x" * (PROVIDER_ERROR_TEXT_CAP_CHARS + 1)) + + async def fail(_conversation): + raise generic_exc + + fake_openhands.Conversation.arun_impl = fail + events: list[dict] = [] + + with pytest.raises(RuntimeError) as raised: + asyncio.run(openhands.run_cogitate(run_env, events.append)) + + assert raised.value is generic_exc + assert len(events) == 1 + assert len(events[0]["error"]) <= PROVIDER_ERROR_TEXT_CAP_CHARS + assert events[0]["error"] == str(generic_exc)[:PROVIDER_ERROR_TEXT_CAP_CHARS] + + def test_run_cogitate_talent_hook_error_propagates_without_provider_event( fake_openhands, run_env, @@ -290,7 +311,7 @@ def test_run_cogitate_local_byo_context_body_event_redacts( assert len(events) == 1 assert events[0]["reason_code"] == "context_window_exceeded" - assert len(events[0]["error"]) <= ENDPOINT_ERROR_BODY_CAP_CHARS + assert len(events[0]["error"]) <= PROVIDER_ERROR_TEXT_CAP_CHARS assert token not in json.dumps(events) assert token not in str(raised.value) diff --git a/tests/test_pipeline_health.py b/tests/test_pipeline_health.py index af0ddcf7b..22266076f 100644 --- a/tests/test_pipeline_health.py +++ b/tests/test_pipeline_health.py @@ -2198,6 +2198,53 @@ def test_read_backlog_view_provider_model_failure_remains_failing_step( assert not any("supervisor" in module for module in imported_modules) +def test_read_backlog_view_serializes_classified_provider_request_rejected( + pipeline_journal, +): + from litellm.exceptions import BadRequestError + + from solstone.think.journal_stats import _serialize_backlog_view + from solstone.think.providers.shared import classify_provider_error + + exc = BadRequestError( + "Invalid value for parameter 'temperature'", + model="gemini-test", + llm_provider="google", + ) + reason_code = classify_provider_error(exc, "google") + assert reason_code == "provider_request_rejected" + + day = "20990414" + segment = "131000_300" + _seed_screen_segment(pipeline_journal, day, segment) + _write_jsonl( + pipeline_journal / "chronicle" / day / "health" / "001_segment.jsonl", + [ + _sense_complete(segment, "active", 1, stream="default"), + _dispatch(segment, "documents", 2, stream="default"), + _complete(segment, "documents", 3, stream="default"), + _dispatch(segment, "entities", 4, stream="default"), + _fail(segment, "entities", 1000, stream="default"), + _fail(segment, "entities", 2000, stream="default"), + _fail( + segment, + "entities", + 3000, + stream="default", + reason_code=reason_code, + provider="google", + model="gemini-test", + ), + ], + ) + _touch_marker(pipeline_journal, day, "stream.updated", mtime_ms=3000) + + serialized = _serialize_backlog_view(read_backlog_view(window=1)) + + assert serialized["days"][0]["state"] == "stuck" + assert serialized["days"][0]["reason_code"] == reason_code + + def test_read_backlog_view_corrupt_raw_reason_wins_over_failing_step( pipeline_journal, ): diff --git a/tests/test_provider_error_classification.py b/tests/test_provider_error_classification.py index c20ddb61e..f697a1196 100644 --- a/tests/test_provider_error_classification.py +++ b/tests/test_provider_error_classification.py @@ -77,6 +77,77 @@ def test_classifies_litellm_bad_request_unrelated_unknown(): assert classify_provider_error(exc, "local") != "context_window_exceeded" +def test_classifies_litellm_bad_request_google_request_rejected(): + from litellm.exceptions import BadRequestError + + exc = BadRequestError( + "Invalid value for parameter 'temperature'", + model="gemini-test", + llm_provider="google", + ) + + assert exc.status_code == 400 + assert classify_provider_error(exc, "google") == "provider_request_rejected" + + +def test_preserves_existing_litellm_cloud_exception_classifications(): + from litellm.exceptions import ( + ContextWindowExceededError, + InternalServerError, + RateLimitError, + ServiceUnavailableError, + Timeout, + ) + + cases = [ + ( + Timeout("timeout", model="gemini-test", llm_provider="google"), + 408, + "chat_timeout", + ), + ( + InternalServerError( + "internal server error", + model="gemini-test", + llm_provider="google", + ), + 500, + "provider_unavailable", + ), + ( + RateLimitError( + "rate limit", + model="gemini-test", + llm_provider="google", + ), + 429, + "provider_quota_exceeded", + ), + ( + ContextWindowExceededError( + "context window exceeded", + model="gemini-test", + llm_provider="google", + ), + 400, + "context_window_exceeded", + ), + ( + ServiceUnavailableError( + "unavailable", + model="gemini-test", + llm_provider="google", + ), + 503, + "unknown", + ), + ] + + for exc, status_code, expected in cases: + assert exc.status_code == status_code + assert classify_provider_error(exc, "google") == expected + + def test_context_reason_codes_are_registered_with_existing_owner_copy(): from solstone.convey import provider_readiness from solstone.think.providers.shared import RUNTIME_REASON_CODES @@ -86,7 +157,11 @@ def test_context_reason_codes_are_registered_with_existing_owner_copy(): encoding="utf-8" ) - for reason_code in ("context_window_exceeded", "context_budget_exceeded"): + for reason_code in ( + "context_window_exceeded", + "context_budget_exceeded", + "provider_request_rejected", + ): assert reason_code in RUNTIME_REASON_CODES assert reason_code in provider_readiness.mapped_reason_codes() assert reason_code in projection diff --git a/tests/test_provider_readiness_presenter.py b/tests/test_provider_readiness_presenter.py index dea3b371f..b318d08bb 100644 --- a/tests/test_provider_readiness_presenter.py +++ b/tests/test_provider_readiness_presenter.py @@ -169,6 +169,7 @@ def test_blocking_reason_classification(): for code in ( "chat_timeout", "network_unreachable", + "provider_request_rejected", "provider_response_invalid", "incomplete_text_length", "no_output",