diff --git a/docs/design/local-provider-capacity-retry.md b/docs/design/local-provider-capacity-retry.md index 509695b2b..dcb4814d9 100644 --- a/docs/design/local-provider-capacity-retry.md +++ b/docs/design/local-provider-capacity-retry.md @@ -22,9 +22,10 @@ Ground truth from llama.cpp `b9291`: - Post-admission unified-KV exhaustion is HTTP 500 with `error.type == "server_error"` and message `Context size has been exceeded.`, with no numeric context fields. -- `local_budget.fit_contents()` makes admitted bundled requests fit the - advertised window before send, so a generic context-shaped error is capacity - pressure unless the server gives structured proof of prompt overflow. +- Bundled request prep fits text against the per-request window after image + reserve, then clamps completion to the remaining room before send. A generic + context-shaped error is capacity pressure unless the server gives structured + proof of prompt overflow. ## Decisions @@ -221,9 +222,9 @@ emit fallback events. Cloud fallback behavior stays in the non-local branch. - Add telemetry assertion for bundled generate failure: captured `record_local_inference` row has `reason_code == "local_capacity_exhausted"`. Failed on the pre-fix tree with `context_budget_exceeded`. -- Keep the no-POST fitter rejection test as-is: `local_budget.fit_contents()` - still raises `ContextBudgetExceeded` before any HTTP request when preserved - content cannot fit. +- Keep no-POST budget rejection coverage: bundled preparation raises + `ContextBudgetExceeded` before HTTP when preserved content, image reserve, or + the minimum completion floor cannot fit. ### `tests/test_local_admission.py` diff --git a/solstone/think/providers/local_budget.py b/solstone/think/providers/local_budget.py index 15a6a38b3..ee2b9c4ea 100644 --- a/solstone/think/providers/local_budget.py +++ b/solstone/think/providers/local_budget.py @@ -31,7 +31,6 @@ TRUNCATION_MARKER = "[earlier input truncated to fit the on-device model's conte class ContextWindowResolution: window_tokens: int slots: int - source: str def resolve_context_window() -> ContextWindowResolution: @@ -50,18 +49,17 @@ def resolve_context_window() -> ContextWindowResolution: if props.total_slots is not None else local_server._CAPABLE_TIER.parallel_slots ) - return ContextWindowResolution(props.n_ctx // slots, slots, "props") + return ContextWindowResolution(props.n_ctx // slots, slots) sidecar = local_server.read_local_context_window() if sidecar is not None and sidecar > 0: slots = ( local_server._slots_from_launched_tier(sidecar) or local_server._UNKNOWN_SLOTS ) - return ContextWindowResolution(sidecar, slots, "local_ctx") + return ContextWindowResolution(sidecar, slots) return ContextWindowResolution( local_server.LOCAL_MIN_CONTEXT_TOKENS, local_server._UNKNOWN_SLOTS, - "default", ) diff --git a/solstone/think/supervisor.py b/solstone/think/supervisor.py index d4ffa569a..0c7c8a075 100644 --- a/solstone/think/supervisor.py +++ b/solstone/think/supervisor.py @@ -2219,7 +2219,10 @@ def _log_context_assertion( "llama-server context assertion skipped: n_ctx unavailable from /props" ) else: - if n_ctx == expected_n_ctx and total_slots == plan.parallel_slots: + context_matches = n_ctx == expected_n_ctx and ( + total_slots is None or total_slots == plan.parallel_slots + ) + if context_matches: logging.info( "llama-server context OK: intended -c=%d parallel=%d actual n_ctx=%d", expected_n_ctx, diff --git a/tests/test_local.py b/tests/test_local.py index 888bf2f74..f519fab1d 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -3901,17 +3901,19 @@ def test_context_window_tokens_darwin_no_props_no_sidecar_uses_floor(monkeypatch @pytest.mark.parametrize( - ("props_total_slots", "capacity_slots"), - [(1, 1), (None, 2)], + ("props_total_slots", "capacity_slots", "expect_strict"), + [(1, 1, False), (None, 2, False), (None, 1, True)], ) def test_context_window_resolution_divisor_not_less_than_capacity( monkeypatch, props_total_slots, capacity_slots, + expect_strict, ): from solstone.think import utils from solstone.think.providers import local_budget, local_server + local_server.reset_parallel_slots_cache() monkeypatch.setattr(utils, "read_service_port", lambda service: 2468) monkeypatch.setattr( local_server, @@ -3928,7 +3930,10 @@ def test_context_window_resolution_divisor_not_less_than_capacity( ) resolution = local_budget.resolve_context_window() - assert resolution.slots >= local_server.read_server_capacity().parallel_slots + reported_slots = local_server.read_server_capacity().parallel_slots + assert resolution.slots >= reported_slots + if expect_strict: + assert resolution.slots > reported_slots def _select_local_provider(monkeypatch) -> None: diff --git a/tests/test_supervisor.py b/tests/test_supervisor.py index d05d9ca1c..509d5df75 100644 --- a/tests/test_supervisor.py +++ b/tests/test_supervisor.py @@ -3459,6 +3459,7 @@ def test_log_context_assertion(caplog): floor = plan_for(local_server._FLOOR_TIER) capable = plan_for(local_server._CAPABLE_TIER) + capable_n_ctx = capable.context_tokens * capable.parallel_slots with caplog.at_level(logging.INFO): mod._log_context_assertion(floor, 16384, 1) @@ -3466,15 +3467,22 @@ def test_log_context_assertion(caplog): caplog.clear() with caplog.at_level(logging.INFO): - mod._log_context_assertion(capable, 65536, 2) + mod._log_context_assertion(capable, capable_n_ctx, 2) assert not any(record.levelno >= logging.WARNING for record in caplog.records) caplog.clear() with caplog.at_level(logging.WARNING): - mod._log_context_assertion(capable, 65536, 1) + mod._log_context_assertion(capable, capable_n_ctx, 1) assert any("context MISMATCH" in record.message for record in caplog.records) assert any("slots MISMATCH" in record.message for record in caplog.records) + caplog.clear() + with caplog.at_level(logging.INFO): + mod._log_context_assertion(capable, capable_n_ctx, None) + assert not any(record.levelno >= logging.WARNING for record in caplog.records) + assert any("context OK" in record.message for record in caplog.records) + assert any("slot count not reported" in record.message for record in caplog.records) + caplog.clear() with caplog.at_level(logging.WARNING): mod._log_context_assertion(capable, 12345, 2)