From 68f848d2b6cd7cd80a39e15c6685446db0707fe5 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Sun, 19 Jul 2026 02:06:58 -0600 Subject: [PATCH] refactor(providers): make install truth manifest-backed and lease-coordinated Terminal installed now publishes at exactly 3 top-level installers, only after the readiness proof reports ready. This removes premature publication that marked llama.cpp installed before the model existed and parakeet installed before the Vulkan binary and model existed. All five install entry paths now converge on the nonblocking per-provider lease. This deletes parakeet's hand-rolled 1-hour blocking install lock and the supervisor's separate single-slot bootstrap bookkeeping. Interrupted classification is lease-based, so one process can no longer mark another process's live install failed. get_state() is read-only. Readiness consumers move to the typed outcome. Only missing-or-mismatched may initiate reacquisition, while proof-unavailable reports and preserves. providers.bundled no longer carries operational state. vulkan_device_index moves to providers.local as owner configuration, and progress coalescing moves into the status owner. The numbered migration, maint opt-ins, and architecture gate follow in the next commit within this lode. --- solstone/apps/thinking/install_copy.py | 2 - solstone/apps/thinking/local_bootstrap.py | 172 +++-- .../tests/test_local_bootstrap_routes.py | 399 ++++++++---- .../tests/test_providers_payload_extended.py | 51 +- .../tests/test_thinking_install_poll_js.py | 8 +- solstone/convey/provider_readiness.py | 6 + solstone/convey/static/chat_reasons.js | 4 + solstone/think/install_models.py | 56 +- solstone/think/install_provider.py | 100 ++- solstone/think/providers/artifact_proof.py | 4 + solstone/think/providers/install_state.py | 326 ++++++---- solstone/think/providers/local_install.py | 585 ++++++++++++------ solstone/think/providers/mlx_install.py | 431 ++++++++++--- solstone/think/providers/parakeet_install.py | 519 +++++++++++----- solstone/think/providers/state.py | 55 +- solstone/think/supervisor.py | 104 ++-- tests/test_chat_reasons.py | 1 + tests/test_install_models.py | 14 +- tests/test_install_provider.py | 120 +++- tests/test_install_state.py | 6 +- tests/test_journal_config_changed_contract.py | 59 +- tests/test_journal_config_transactions.py | 2 +- tests/test_local.py | 122 ++-- tests/test_local_install.py | 469 ++++++++------ tests/test_mlx_install.py | 60 +- tests/test_parakeet_install.py | 187 +++--- tests/test_provider_state.py | 180 ++++-- tests/test_supervisor.py | 67 +- tests/test_supervisor_parakeet.py | 113 +++- 29 files changed, 2879 insertions(+), 1343 deletions(-) diff --git a/solstone/apps/thinking/install_copy.py b/solstone/apps/thinking/install_copy.py index f2afac701..b8d5fc126 100644 --- a/solstone/apps/thinking/install_copy.py +++ b/solstone/apps/thinking/install_copy.py @@ -12,7 +12,6 @@ INSTALL_PHASE_INSTALLED = "Installed" INSTALL_PHASE_FAILED_PREFIX = "Install failed — " INSTALL_FAILED_FALLBACK = "try again" -INSTALL_FAILED_NO_PROGRESS = "no progress for 60 seconds — try again" INSTALL_FAILED_UV_MISSING = ( "uv not found — install uv (https://github.com/astral-sh/uv) and retry" ) @@ -64,7 +63,6 @@ __all__ = [ "INSTALL_PHASE_INSTALLED", "INSTALL_PHASE_FAILED_PREFIX", "INSTALL_FAILED_FALLBACK", - "INSTALL_FAILED_NO_PROGRESS", "INSTALL_FAILED_UV_MISSING", "INSTALL_BUTTON_INSTALL", "INSTALL_BUTTON_INSTALLING", diff --git a/solstone/apps/thinking/local_bootstrap.py b/solstone/apps/thinking/local_bootstrap.py index 6579b30ed..e3dee82da 100644 --- a/solstone/apps/thinking/local_bootstrap.py +++ b/solstone/apps/thinking/local_bootstrap.py @@ -18,10 +18,15 @@ from solstone.think.callosum import callosum_send from solstone.think.models import LOCAL_MODEL, QWEN_35_9B from solstone.think.providers import local_install, mlx_install from solstone.think.providers.fit_report import FitReport +from solstone.think.providers.install_lease import ( + InstallLease, + acquire_install_lease, + probe_install_lease_free, +) from solstone.think.providers.install_state import ( IN_FLIGHT_STATES, InstallStatus, - make_idle_status, + begin_or_replace_install_attempt, read_install_status, transition_state, write_install_status, @@ -107,7 +112,9 @@ def list_local_models() -> list[dict[str, object]]: def check_binary_present() -> bool: """Return whether the pinned llama-server binary is installed.""" try: - return bool(local_install.inspect_readiness(LOCAL_MODEL)["binary_installed"]) + return bool( + local_install.inspect_readiness(LOCAL_MODEL).artifacts["binary_installed"] + ) except Exception: return False @@ -116,7 +123,9 @@ def check_model_present(model: str) -> bool: """Return whether the pinned GGUF model is installed.""" try: model_id = normalize_model_id(model) - return bool(local_install.inspect_readiness(model_id)["model_installed"]) + return bool( + local_install.inspect_readiness(model_id).artifacts["model_installed"] + ) except Exception: return False @@ -146,17 +155,17 @@ def get_availability_payload(model: str) -> dict[str, bool | float | int | str | min_ram_gb = MLX_AVAILABLE_FLOOR_BYTES // 1024**3 memory_blocked = memory_verdict.severity == "blocked" available = bool( - readiness["platform_supported"] - and readiness["package_available"] + readiness.host["platform_supported"] + and readiness.host["package_available"] and not memory_blocked - and readiness["model_installed"] + and readiness.artifacts["model_installed"] ) warning = ( LOCAL_MLX_MEMORY_WARNING_UNKNOWN if memory_verdict.severity == "warning" else "" ) - if not readiness["platform_supported"]: + if not readiness.host["platform_supported"]: reason = "requires Apple Silicon macOS" elif memory_blocked: assert memory_verdict.available_bytes is not None @@ -165,20 +174,20 @@ def get_availability_payload(model: str) -> dict[str, bool | float | int | str | f"(need {gb_label(memory_verdict.required_bytes)} GB available, " f"have {gb_label(memory_verdict.available_bytes)} GB available)" ) - elif not readiness["package_available"]: + elif not readiness.host["package_available"]: reason = "mlx-vlm runtime is not installed" - elif not readiness["model_installed"]: + elif not readiness.artifacts["model_installed"]: reason = "local model files are not installed" else: reason = "" return { - "model": readiness["model_id"], - "platform_supported": readiness["platform_supported"], + "model": readiness.target["model_id"], + "platform_supported": readiness.host["platform_supported"], "total_memory_gb": gb(total_memory_bytes), "available_memory_gb": gb(memory_verdict.available_bytes), "min_ram_gb": min_ram_gb, - "binary_present": readiness["package_available"], - "model_present": readiness["model_installed"], + "binary_present": readiness.host["package_available"], + "model_present": readiness.artifacts["model_installed"], "available": available, "reason": reason, "warning": warning, @@ -186,8 +195,9 @@ def get_availability_payload(model: str) -> dict[str, bool | float | int | str | } spec = LOCAL_MODEL_SPECS[model_id] - binary_present = check_binary_present() - model_present = check_model_present(model_id) + readiness = local_install.inspect_readiness(model_id) + binary_present = bool(readiness.artifacts["binary_installed"]) + model_present = bool(readiness.artifacts["model_installed"]) platform_supported, reason = _platform_supported() total_memory_gb = gb(read_total_bytes()) memory_verdict = assess_memory(spec.min_ram_bytes, block_below_floor=False) @@ -227,20 +237,14 @@ def get_availability_payload(model: str) -> dict[str, bool | float | int | str | def _read_status() -> InstallStatus: - return read_install_status(scope="bundled", name=local_install.LOCAL_PROVIDER_NAME) + return read_install_status(name=local_install.LOCAL_PROVIDER_NAME) def _write_status(status: InstallStatus) -> InstallStatus: - write_install_status(status, scope="bundled") + write_install_status(status) return status -def _has_live_thread(model: str) -> bool: - with _INSTALL_LOCK: - thread = _INSTALL_THREADS.get(model) - return thread is not None and thread.is_alive() - - def _payload_for_status( _model: str, status: InstallStatus ) -> dict[str, int | str | None]: @@ -263,15 +267,24 @@ def _payload_for_status( } -def _normalize_stalled_status(_model: str, status: InstallStatus) -> InstallStatus: - return status +def _payload_for_read_status( + model: str, + status: InstallStatus, +) -> dict[str, int | str | None]: + if status["install_state"] in IN_FLIGHT_STATES and probe_install_lease_free( + local_install.LOCAL_PROVIDER_NAME + ): + payload = _payload_for_status(model, status) + payload["install_state"] = "failed" + payload["install_error"] = "install_interrupted" + return payload + return _payload_for_status(model, status) def get_state(model: str) -> dict[str, int | str | None]: - """Return the serialized bootstrap state, applying stall detection.""" + """Return the serialized bootstrap state without mutating on-disk state.""" model_id = _resolve_model_id(model) - status = _normalize_stalled_status(model_id, _read_status()) - return _payload_for_status(model_id, status) + return _payload_for_read_status(model_id, _read_status()) def start_bootstrap(model: str) -> tuple[dict[str, str], int]: @@ -281,28 +294,57 @@ def start_bootstrap(model: str) -> tuple[dict[str, str], int]: raise LocalBootstrapUnavailableError("BYO local endpoint is active") model_id = _resolve_model_id(model) - get_state(model_id) - status = _read_status() - if status["install_state"] == "installed": + readiness = ( + mlx_install.inspect_readiness(model_id) + if _is_mlx_backend() + else local_install.inspect_readiness(model_id) + ) + if readiness.ready: return {"install_state": "installed"}, 200 + if readiness.status in {"proof-unavailable", "host-ineligible"}: + raise LocalBootstrapUnavailableError(readiness.reason_code) availability = get_availability_payload(model_id) installed = bool(availability["binary_present"] and availability["model_present"]) + fingerprint = ( + mlx_install.target_fingerprint(model_id) + if _is_mlx_backend() + else local_install.target_fingerprint(model_id) + ) + from solstone.think.providers.install_state import ( + canonical_fingerprint, + fingerprint_sha256, + ) + + target_sha = fingerprint_sha256(canonical_fingerprint(fingerprint)) + lease = acquire_install_lease(local_install.LOCAL_PROVIDER_NAME) + if lease is None: + status = _read_status() + if ( + status["install_state"] in IN_FLIGHT_STATES + and status["target_fingerprint_sha256"] == target_sha + ): + return {"install_state": status["install_state"]}, 200 + return { + "install_state": status["install_state"], + "reason_code": "install_busy", + }, 409 + with _INSTALL_LOCK: status = _read_status() - if status["install_state"] == "installed": + if readiness.ready: + lease.release() return {"install_state": "installed"}, 200 if status["install_state"] == "idle" and installed: - _write_status( - transition_state( - make_idle_status(local_install.LOCAL_PROVIDER_NAME), - new_state="installed", - ) - ) + lease.release() return {"install_state": "installed"}, 200 - if status["install_state"] in IN_FLIGHT_STATES: + if ( + status["install_state"] in IN_FLIGHT_STATES + and status["target_fingerprint_sha256"] == target_sha + ): + lease.release() return {"install_state": status["install_state"]}, 200 # Only genuinely-missing artifacts reach here: build the host-fit report @@ -311,27 +353,36 @@ def start_bootstrap(model: str) -> tuple[dict[str, str], int]: report = _fit_report_for_model(model_id) blocked_reason = _blocked_reason(report) if blocked_reason: + lease.release() raise LocalBootstrapUnavailableError(blocked_reason) disk_reason = _disk_blocked_reason(report) if disk_reason: + lease.release() raise LocalBootstrapUnavailableError(disk_reason) try: worker = ( _mlx_bootstrap_worker if _is_mlx_backend() else _run_bootstrap_worker ) + attempt_status = begin_or_replace_install_attempt( + local_install.LOCAL_PROVIDER_NAME, + fingerprint, + initial_state="downloading", + owner={"entry": "thinking_bootstrap"}, + ) + ack = threading.Event() thread = threading.Thread( target=worker, - args=(model_id,), + args=(model_id, lease, attempt_status, ack), name=f"local-provider-bootstrap-{model_id}", daemon=True, ) except Exception as exc: + lease.release() _write_status(transition_state(status, new_state="failed", error=str(exc))) raise LocalBootstrapStartError(str(exc)) from exc - _write_status(transition_state(status, new_state="downloading")) _INSTALL_THREADS[model_id] = thread try: @@ -340,10 +391,17 @@ def start_bootstrap(model: str) -> tuple[dict[str, str], int]: with _INSTALL_LOCK: if _INSTALL_THREADS.get(model_id) is thread: _INSTALL_THREADS.pop(model_id, None) + lease.release() _write_status( transition_state(_read_status(), new_state="failed", error=str(exc)) ) raise LocalBootstrapStartError(str(exc)) from exc + if not ack.wait(timeout=5.0): + with _INSTALL_LOCK: + if _INSTALL_THREADS.get(model_id) is thread: + _INSTALL_THREADS.pop(model_id, None) + lease.release() + raise LocalBootstrapStartError("local bootstrap worker did not acknowledge") return {"install_state": "downloading"}, 202 @@ -371,13 +429,24 @@ def _disk_blocked_reason(report: FitReport) -> str: return "" -def _mlx_bootstrap_worker(model: str) -> None: +def _mlx_bootstrap_worker( + model: str, + lease: InstallLease, + attempt_status: InstallStatus, + ack: threading.Event, +) -> None: current_thread = threading.current_thread() + ack.set() try: - mlx_install.install_local_mlx(model) + mlx_install.install_local_mlx( + model, + lease=lease, + attempt_status=attempt_status, + ) except Exception: logger.exception("local MLX provider bootstrap failed") finally: + lease.release() with _INSTALL_LOCK: if _INSTALL_THREADS.get(model) is current_thread: _INSTALL_THREADS.pop(model, None) @@ -392,12 +461,20 @@ def _request_local_server_start() -> None: logger.exception("could not request local server start") -def _run_bootstrap_worker(model: str) -> None: +def _run_bootstrap_worker( + model: str, + lease: InstallLease, + attempt_status: InstallStatus, + ack: threading.Event, +) -> None: current_thread = threading.current_thread() + ack.set() try: - local_install.install_llama_server() - _write_status(transition_state(_read_status(), new_state="downloading")) - local_install.install_model(model) + local_install.install_local( + model, + lease=lease, + attempt_status=attempt_status, + ) except Exception as exc: logger.exception("local provider bootstrap failed") _write_status( @@ -407,6 +484,7 @@ def _run_bootstrap_worker(model: str) -> None: logger.info("local provider bootstrap complete; requesting local server start") _request_local_server_start() finally: + lease.release() with _INSTALL_LOCK: if _INSTALL_THREADS.get(model) is current_thread: _INSTALL_THREADS.pop(model, None) diff --git a/solstone/apps/thinking/tests/test_local_bootstrap_routes.py b/solstone/apps/thinking/tests/test_local_bootstrap_routes.py index 49b2e2a37..59f344e50 100644 --- a/solstone/apps/thinking/tests/test_local_bootstrap_routes.py +++ b/solstone/apps/thinking/tests/test_local_bootstrap_routes.py @@ -15,7 +15,8 @@ import pytest from solstone.apps.thinking import local_bootstrap from solstone.convey import create_app from solstone.think.models import LOCAL_MODEL, QWEN_35_9B -from solstone.think.providers import fit_report, memory +from solstone.think.providers import fit_report, local_cuda, local_vulkan, memory +from solstone.think.providers.artifact_proof import ReadinessOutcome from solstone.think.providers.install_state import ( InstallState, InstallStatus, @@ -64,15 +65,26 @@ class _FakeThread: def __init__(self, *args, **kwargs): type(self).init_count += 1 type(self).targets.append(kwargs.get("target")) + self.args = kwargs.get("args", ()) self.alive = True def start(self): type(self).start_count += 1 + if self.args and hasattr(self.args[-1], "set"): + self.args[-1].set() def is_alive(self): return self.alive +class _FakeLease: + def __init__(self) -> None: + self.released = False + + def release(self) -> None: + self.released = True + + def _write_local_status( state: InstallState, *, @@ -86,7 +98,7 @@ def _write_local_status( ) status["last_transition_at"] = "2026-05-23T00:00:00+00:00" status["last_progress_at"] = last_progress_at - return write_install_status(status, scope="bundled") + return write_install_status(status) def _old_progress_iso() -> str: @@ -98,22 +110,108 @@ def _fresh_progress_iso() -> str: def _mlx_readiness(**overrides): - readiness = { - "install_state": "idle", - "model_installed": True, - "snapshot_installed": True, - "variant_installed": True, - "ram_sufficient": True, - "platform_supported": True, - "package_available": True, - "model_id": QWEN_35_9B, - "snapshot_dir": "/tmp/qwen-snapshot", - "variant_dir": None, - "runtime_dir": "/tmp/qwen-snapshot", - "install_error": None, - } - readiness.update(overrides) - return readiness + install_state = overrides.pop("install_state", "idle") + model_installed = overrides.pop("model_installed", True) + snapshot_installed = overrides.pop("snapshot_installed", model_installed) + variant_installed = overrides.pop("variant_installed", True) + platform_supported = overrides.pop("platform_supported", True) + package_available = overrides.pop("package_available", True) + status = ( + "ready" + if model_installed and platform_supported and package_available + else "missing-or-mismatched" + ) + return ReadinessOutcome( + provider="local", + status=status, + reason_code="ready" if status == "ready" else "manifest_missing", + target={"model_id": overrides.pop("model_id", QWEN_35_9B)}, + install={ + "install_state": install_state, + "install_error": overrides.pop("install_error", None), + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": overrides.pop("ram_sufficient", True), + "platform_supported": platform_supported, + "package_available": package_available, + }, + artifacts={ + "model_installed": model_installed, + "snapshot_installed": snapshot_installed, + "variant_installed": variant_installed, + "snapshot_dir": overrides.pop("snapshot_dir", "/tmp/qwen-snapshot"), + "variant_dir": overrides.pop("variant_dir", None), + "runtime_dir": overrides.pop("runtime_dir", "/tmp/qwen-snapshot"), + }, + proof={ + "snapshot": { + "status": "ready" if snapshot_installed else "missing-or-mismatched", + "reason_code": "ready" if snapshot_installed else "manifest_missing", + "cache_hit": False, + }, + "variant": { + "status": "ready" if variant_installed else "missing-or-mismatched", + "reason_code": "ready" if variant_installed else "manifest_missing", + "cache_hit": False, + }, + }, + ) + + +def _local_readiness(**overrides) -> ReadinessOutcome: + binary_installed = overrides.pop("binary_installed", True) + model_installed = overrides.pop("model_installed", True) + status = ( + "ready" if binary_installed and model_installed else "missing-or-mismatched" + ) + return ReadinessOutcome( + provider="local", + status=status, + reason_code="ready" if status == "ready" else "manifest_missing", + target={"model_id": overrides.pop("model_id", LOCAL_MODEL)}, + install={ + "install_state": overrides.pop("install_state", "idle"), + "install_error": overrides.pop("install_error", None), + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": overrides.pop("ram_sufficient", True), + "gpu_available": overrides.pop("gpu_available", True), + "gpu_probe_ok": overrides.pop("gpu_probe_ok", True), + "backend": "vulkan", + "backend_reason": "test vulkan", + }, + artifacts={ + "binary_installed": binary_installed, + "model_installed": model_installed, + "binary_path": overrides.pop("binary_path", "/tmp/llama-server"), + "model_path": overrides.pop("model_path", "/tmp/model.gguf"), + "mmproj_path": None, + }, + proof={ + "binary": { + "status": "ready" if binary_installed else "missing-or-mismatched", + "reason_code": "ready" if binary_installed else "manifest_missing", + "cache_hit": False, + }, + "model": { + "status": "ready" if model_installed else "missing-or-mismatched", + "reason_code": "ready" if model_installed else "manifest_missing", + "cache_hit": False, + }, + }, + ) def _fit( @@ -153,8 +251,11 @@ def test_local_bootstrap_linux_contract_for_model_helpers(): def test_local_availability_payload_exact_shape(settings_env, monkeypatch): journal_path, _config = settings_env(_settings_config()) - monkeypatch.setattr(local_bootstrap, "check_binary_present", lambda: True) - monkeypatch.setattr(local_bootstrap, "check_model_present", lambda _model: True) + monkeypatch.setattr( + local_bootstrap.local_install, + "inspect_readiness", + lambda _model: _local_readiness(), + ) monkeypatch.setattr(local_bootstrap, "_platform_supported", lambda: (True, "")) monkeypatch.setattr( memory.psutil, @@ -325,8 +426,11 @@ def test_local_availability_warns_but_does_not_block_on_low_memory( settings_env, monkeypatch ): journal_path, _config = settings_env(_settings_config()) - monkeypatch.setattr(local_bootstrap, "check_binary_present", lambda: True) - monkeypatch.setattr(local_bootstrap, "check_model_present", lambda _model: True) + monkeypatch.setattr( + local_bootstrap.local_install, + "inspect_readiness", + lambda _model: _local_readiness(), + ) monkeypatch.setattr(local_bootstrap, "_platform_supported", lambda: (True, "")) monkeypatch.setattr( memory.psutil, @@ -405,9 +509,9 @@ def test_local_bootstrap_post_rejects_byo_endpoint(settings_env): @pytest.mark.parametrize( ("state", "expected_payload", "expected_status"), [ - ("installed", {"install_state": "installed"}, 200), - ("downloading", {"install_state": "downloading"}, 200), - ("verifying", {"install_state": "verifying"}, 200), + ("installed", {"install_state": "downloading"}, 202), + ("downloading", {"install_state": "downloading"}, 202), + ("verifying", {"install_state": "downloading"}, 202), ("idle", {"install_state": "downloading"}, 202), ("failed", {"install_state": "downloading"}, 202), ], @@ -455,6 +559,59 @@ def test_start_bootstrap_payload_for_canonical_states( ) +def test_start_bootstrap_repairs_stale_installed_without_manifest( + settings_env, monkeypatch +): + settings_env(_settings_config()) + _write_local_status("installed") + binary = local_bootstrap.local_install.binary_path_for_pin() + binary.parent.mkdir(parents=True, exist_ok=True) + binary.write_bytes(b"stale binary") + binary.chmod(0o755) + spec = LOCAL_MODEL_SPECS[LOCAL_MODEL] + model = local_bootstrap.local_install.model_path(LOCAL_MODEL) + model.parent.mkdir(parents=True, exist_ok=True) + model.write_bytes(b"stale model") + if spec.mmproj_filename: + mmproj = local_bootstrap.local_install.mmproj_path(LOCAL_MODEL) + assert mmproj is not None + mmproj.write_bytes(b"stale projector") + monkeypatch.setattr( + local_cuda, + "resolve_local_backend", + lambda _pin: local_cuda.BackendChoice("vulkan", "test vulkan"), + ) + monkeypatch.setattr(local_vulkan, "detect_gpus", lambda: []) + monkeypatch.setattr(local_vulkan, "gpu_probe_ok", lambda: True) + monkeypatch.setattr( + memory.psutil, + "virtual_memory", + lambda: SimpleNamespace(available=32 * 1024**3, total=32 * 1024**3), + ) + monkeypatch.setattr( + memory.shutil, + "disk_usage", + lambda _path: SimpleNamespace(free=100 * 1024**3), + ) + monkeypatch.setattr( + local_bootstrap, "_fit_report_for_model", lambda _model: _fit("ok") + ) + _FakeThread.init_count = 0 + _FakeThread.start_count = 0 + monkeypatch.setattr(local_bootstrap.threading, "Thread", _FakeThread) + + assert local_bootstrap.start_bootstrap(LOCAL_MODEL) == ( + {"install_state": "downloading"}, + 202, + ) + + assert _FakeThread.start_count == 1 + status = read_install_status(name="local") + assert status["install_state"] == "downloading" + assert binary.read_bytes() == b"stale binary" + assert model.read_bytes() == b"stale model" + + def test_start_bootstrap_low_memory_warning_does_not_block(settings_env, monkeypatch): settings_env(_settings_config()) _write_local_status("idle") @@ -525,20 +682,25 @@ def test_start_bootstrap_insufficient_disk_blocks_before_worker( local_bootstrap.start_bootstrap(LOCAL_MODEL) assert _FakeThread.init_count == 0 - status = read_install_status(scope="bundled", name="local") + status = read_install_status(name="local") assert status["install_state"] == "idle" def test_local_bootstrap_status_returns_canonical_shape(settings_env): journal_path, _config = settings_env(_settings_config()) _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) - status = read_install_status(scope="bundled", name="local") + status = read_install_status(name="local") status["progress_bytes_received"] = 12 status["progress_bytes_total"] = 24 - write_install_status(status, scope="bundled") + write_install_status(status) client = _client(journal_path) + lease = local_bootstrap.acquire_install_lease("local") + assert lease is not None - response = client.get("/app/thinking/api/local/bootstrap/status") + try: + response = client.get("/app/thinking/api/local/bootstrap/status") + finally: + lease.release() assert response.status_code == 200 payload = response.get_json() @@ -562,20 +724,23 @@ def test_local_bootstrap_lazy_stall_without_live_thread_stays_read_only(settings payload = local_bootstrap.get_state(LOCAL_MODEL) - assert payload["install_state"] == "downloading" - assert payload["install_error"] is None - persisted = read_install_status(scope="bundled", name="local") + assert payload["install_state"] == "failed" + assert payload["install_error"] == "install_interrupted" + persisted = read_install_status(name="local") assert persisted["install_state"] == "downloading" assert persisted["install_error"] is None -def test_local_bootstrap_lazy_stall_with_live_thread_stays_in_flight(settings_env): +def test_local_bootstrap_in_flight_with_held_lease_stays_in_flight(settings_env): settings_env(_settings_config()) _write_local_status("verifying", last_progress_at=_old_progress_iso()) - with local_bootstrap._INSTALL_LOCK: - local_bootstrap._INSTALL_THREADS[LOCAL_MODEL] = _FakeThread() + lease = local_bootstrap.acquire_install_lease("local") + assert lease is not None - payload = local_bootstrap.get_state(LOCAL_MODEL) + try: + payload = local_bootstrap.get_state(LOCAL_MODEL) + finally: + lease.release() assert payload["install_state"] == "verifying" assert payload["install_error"] is None @@ -609,16 +774,19 @@ def test_mlx_availability_ignores_install_state_when_snapshot_missing( assert payload["reason"] == "local model files are not installed" -def test_mlx_bootstrap_lazy_stall_with_live_thread_stays_in_flight( +def test_mlx_bootstrap_in_flight_with_held_lease_stays_in_flight( settings_env, monkeypatch ): settings_env(_settings_config()) _write_local_status("downloading", last_progress_at=_old_progress_iso()) monkeypatch.setattr(local_bootstrap, "_is_mlx_backend", lambda: True) - with local_bootstrap._INSTALL_LOCK: - local_bootstrap._INSTALL_THREADS[QWEN_35_9B] = _FakeThread() + lease = local_bootstrap.acquire_install_lease("local") + assert lease is not None - payload = local_bootstrap.get_state(QWEN_35_9B) + try: + payload = local_bootstrap.get_state(QWEN_35_9B) + finally: + lease.release() assert payload["install_state"] == "downloading" assert payload["install_error"] is None @@ -633,8 +801,8 @@ def test_mlx_bootstrap_lazy_stall_without_live_thread_stays_read_only( payload = local_bootstrap.get_state(QWEN_35_9B) - assert payload["install_state"] == "downloading" - assert payload["install_error"] is None + assert payload["install_state"] == "failed" + assert payload["install_error"] == "install_interrupted" @pytest.mark.parametrize("state", ["installed", "failed"]) @@ -649,20 +817,14 @@ def test_local_bootstrap_restart_terminal_states_have_no_bytes(settings_env, sta assert payload["progress_bytes_total"] is None -def test_local_bootstrap_migrates_preexisting_install_without_worker( +def test_local_bootstrap_ready_short_circuit_does_not_publish_terminal( settings_env, monkeypatch ): settings_env(_settings_config()) monkeypatch.setattr( local_bootstrap.local_install, "inspect_readiness", - lambda _model=None: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "binary_path": "/tmp/llama-server", - "model_path": "/tmp/model.gguf", - }, + lambda _model=None: _local_readiness(), ) monkeypatch.setattr(local_bootstrap, "_platform_supported", lambda: (True, "")) monkeypatch.setattr( @@ -687,8 +849,8 @@ def test_local_bootstrap_migrates_preexisting_install_without_worker( {"install_state": "installed"}, 200, ) - status = read_install_status(scope="bundled", name="local") - assert status["install_state"] == "installed" + status = read_install_status(name="local") + assert status["install_state"] == "idle" def test_mlx_start_bootstrap_dispatches_to_mlx_worker(settings_env, monkeypatch): @@ -733,41 +895,45 @@ def test_mlx_start_bootstrap_dispatches_to_mlx_worker(settings_env, monkeypatch) assert _FakeThread.targets == [local_bootstrap._mlx_bootstrap_worker] -def test_local_worker_resets_progress_between_binary_and_model( +def test_local_worker_delegates_to_top_level_installer_with_lease_and_attempt( settings_env, monkeypatch ): settings_env(_settings_config()) + attempt_status = _write_local_status( + "downloading", last_progress_at=_fresh_progress_iso() + ) + lease = _FakeLease() + ack = threading.Event() observed = {} - _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) - def fake_llama_server(): - status = read_install_status(scope="bundled", name="local") - write_install_status( - transition_state(status, new_state="installed"), - scope="bundled", + def fake_install_local(model, *, lease, attempt_status): + observed.update( + { + "model": model, + "lease": lease, + "attempt_status": attempt_status, + "ack_set": ack.is_set(), + } ) - - def fake_install_model(model): - observed.update(local_bootstrap.get_state(model)) - status = read_install_status(scope="bundled", name="local") + status = read_install_status(name="local") write_install_status( transition_state(status, new_state="installed"), - scope="bundled", ) monkeypatch.setattr( - local_bootstrap.local_install, "install_llama_server", fake_llama_server - ) - monkeypatch.setattr( - local_bootstrap.local_install, "install_model", fake_install_model + local_bootstrap.local_install, "install_local", fake_install_local ) monkeypatch.setattr(local_bootstrap, "callosum_send", Mock(return_value=True)) - local_bootstrap._run_bootstrap_worker(LOCAL_MODEL) + local_bootstrap._run_bootstrap_worker(LOCAL_MODEL, lease, attempt_status, ack) - assert observed["install_state"] == "downloading" - assert observed["progress_bytes_total"] is None - assert observed["progress_bytes_received"] is None + assert observed == { + "model": LOCAL_MODEL, + "lease": lease, + "attempt_status": attempt_status, + "ack_set": True, + } + assert lease.released is True @pytest.mark.parametrize( @@ -778,14 +944,17 @@ def test_local_worker_success_requests_local_server_start( settings_env, monkeypatch, send_behavior ): settings_env(_settings_config()) - _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) + attempt_status = _write_local_status( + "downloading", last_progress_at=_fresh_progress_iso() + ) + lease = _FakeLease() + ack = threading.Event() - def fake_install_model(model): + def fake_install_local(model, *, lease, attempt_status): assert model == LOCAL_MODEL - status = read_install_status(scope="bundled", name="local") + status = read_install_status(name="local") write_install_status( transition_state(status, new_state="installed"), - scope="bundled", ) if send_behavior == "raise": @@ -794,17 +963,16 @@ def test_local_worker_success_requests_local_server_start( callosum_send = Mock(return_value=send_behavior == "success") monkeypatch.setattr( - local_bootstrap.local_install, "install_llama_server", lambda: None - ) - monkeypatch.setattr( - local_bootstrap.local_install, "install_model", fake_install_model + local_bootstrap.local_install, "install_local", fake_install_local ) monkeypatch.setattr(local_bootstrap, "callosum_send", callosum_send) - local_bootstrap._run_bootstrap_worker(LOCAL_MODEL) + local_bootstrap._run_bootstrap_worker(LOCAL_MODEL, lease, attempt_status, ack) callosum_send.assert_called_once_with("supervisor", "start_local") - status = read_install_status(scope="bundled", name="local") + assert ack.is_set() + assert lease.released is True + status = read_install_status(name="local") assert status["install_state"] == "installed" assert status["install_error"] is None @@ -813,23 +981,26 @@ def test_local_worker_install_model_failure_does_not_request_local_server_start( settings_env, monkeypatch ): settings_env(_settings_config()) - _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) + attempt_status = _write_local_status( + "downloading", last_progress_at=_fresh_progress_iso() + ) + lease = _FakeLease() + ack = threading.Event() callosum_send = Mock(return_value=True) - monkeypatch.setattr( - local_bootstrap.local_install, "install_llama_server", lambda: None - ) monkeypatch.setattr( local_bootstrap.local_install, - "install_model", + "install_local", Mock(side_effect=RuntimeError("model download broke")), ) monkeypatch.setattr(local_bootstrap, "callosum_send", callosum_send) - local_bootstrap._run_bootstrap_worker(LOCAL_MODEL) + local_bootstrap._run_bootstrap_worker(LOCAL_MODEL, lease, attempt_status, ack) callosum_send.assert_not_called() - status = read_install_status(scope="bundled", name="local") + assert ack.is_set() + assert lease.released is True + status = read_install_status(name="local") assert status["install_state"] == "failed" assert status["install_error"] == "model download broke" @@ -839,24 +1010,24 @@ def test_local_worker_cleans_registered_thread(settings_env, monkeypatch): current = threading.current_thread() with local_bootstrap._INSTALL_LOCK: local_bootstrap._INSTALL_THREADS[LOCAL_MODEL] = current - _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) + attempt_status = _write_local_status( + "downloading", last_progress_at=_fresh_progress_iso() + ) + lease = _FakeLease() + ack = threading.Event() - def fake_install_model(_model): - status = read_install_status(scope="bundled", name="local") + def fake_install_local(_model, *, lease, attempt_status): + status = read_install_status(name="local") write_install_status( transition_state(status, new_state="installed"), - scope="bundled", ) monkeypatch.setattr( - local_bootstrap.local_install, "install_llama_server", lambda: None - ) - monkeypatch.setattr( - local_bootstrap.local_install, "install_model", fake_install_model + local_bootstrap.local_install, "install_local", fake_install_local ) monkeypatch.setattr(local_bootstrap, "callosum_send", Mock(return_value=True)) - local_bootstrap._run_bootstrap_worker(LOCAL_MODEL) + local_bootstrap._run_bootstrap_worker(LOCAL_MODEL, lease, attempt_status, ack) with local_bootstrap._INSTALL_LOCK: assert LOCAL_MODEL not in local_bootstrap._INSTALL_THREADS @@ -867,20 +1038,25 @@ def test_local_worker_cleans_registered_thread_after_failure(settings_env, monke current = threading.current_thread() with local_bootstrap._INSTALL_LOCK: local_bootstrap._INSTALL_THREADS[LOCAL_MODEL] = current - _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) + attempt_status = _write_local_status( + "downloading", last_progress_at=_fresh_progress_iso() + ) + lease = _FakeLease() + ack = threading.Event() monkeypatch.setattr( local_bootstrap.local_install, - "install_llama_server", - lambda: (_ for _ in ()).throw(RuntimeError("binary download broke")), + "install_local", + Mock(side_effect=RuntimeError("binary download broke")), ) - local_bootstrap._run_bootstrap_worker(LOCAL_MODEL) + local_bootstrap._run_bootstrap_worker(LOCAL_MODEL, lease, attempt_status, ack) with local_bootstrap._INSTALL_LOCK: thread = local_bootstrap._INSTALL_THREADS.get(LOCAL_MODEL) assert thread is None or not thread.is_alive() - status = read_install_status(scope="bundled", name="local") + assert lease.released is True + status = read_install_status(name="local") assert status["install_state"] == "failed" assert status["install_error"] == "binary download broke" @@ -892,14 +1068,17 @@ def test_mlx_worker_preserves_install_error_and_cleans_thread( current = threading.current_thread() with local_bootstrap._INSTALL_LOCK: local_bootstrap._INSTALL_THREADS[QWEN_35_9B] = current - _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) + attempt_status = _write_local_status( + "downloading", last_progress_at=_fresh_progress_iso() + ) + lease = _FakeLease() + ack = threading.Event() monkeypatch.setattr(local_bootstrap, "_is_mlx_backend", lambda: True) - def fake_install_mlx(_model): - status = read_install_status(scope="bundled", name="local") + def fake_install_mlx(_model, *, lease, attempt_status): + status = read_install_status(name="local") write_install_status( transition_state(status, new_state="failed", error="verify broke"), - scope="bundled", ) raise local_bootstrap.mlx_install.MLXVerificationError("verify broke") @@ -909,11 +1088,13 @@ def test_mlx_worker_preserves_install_error_and_cleans_thread( fake_install_mlx, ) - local_bootstrap._mlx_bootstrap_worker(QWEN_35_9B) + local_bootstrap._mlx_bootstrap_worker(QWEN_35_9B, lease, attempt_status, ack) with local_bootstrap._INSTALL_LOCK: assert QWEN_35_9B not in local_bootstrap._INSTALL_THREADS - status = read_install_status(scope="bundled", name="local") + assert ack.is_set() + assert lease.released is True + status = read_install_status(name="local") assert status["install_state"] == "failed" assert status["install_error"] == "verify broke" payload = local_bootstrap.get_state(QWEN_35_9B) diff --git a/solstone/apps/thinking/tests/test_providers_payload_extended.py b/solstone/apps/thinking/tests/test_providers_payload_extended.py index 3f8fdb24e..711122de9 100644 --- a/solstone/apps/thinking/tests/test_providers_payload_extended.py +++ b/solstone/apps/thinking/tests/test_providers_payload_extended.py @@ -13,6 +13,7 @@ from solstone.apps.thinking import routes from solstone.apps.thinking.local_bootstrap import LOCAL_MODEL_SPECS from solstone.apps.thinking.model_tiers import MODEL_TIERS from solstone.think.models import LOCAL_MODEL, NO_BRAIN_PROVIDER, resolve_provider +from solstone.think.providers.artifact_proof import ReadinessOutcome from solstone.think.providers.install_state import InstallState from solstone.think.providers.state import ProviderState @@ -65,6 +66,43 @@ def _assert_install_status(payload: dict) -> None: assert payload["install_state"] in CANONICAL_INSTALL_STATES +def _local_readiness_gpu_probe_failed() -> ReadinessOutcome: + return ReadinessOutcome( + provider="local", + status="ready", + reason_code="ready", + target={"model_id": LOCAL_MODEL}, + install={ + "install_state": "installed", + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": True, + "gpu_available": True, + "gpu_probe_ok": False, + "backend": "vulkan", + "backend_reason": "test vulkan", + }, + artifacts={ + "binary_installed": True, + "model_installed": True, + "binary_path": "/tmp/llama-server", + "model_path": "/tmp/model.gguf", + "model_id": LOCAL_MODEL, + }, + proof={ + "binary": {"status": "ready", "reason_code": "ready", "cache_hit": False}, + "model": {"status": "ready", "reason_code": "ready", "cache_hit": False}, + }, + ) + + def _patch_selected_providers(monkeypatch, *, provider: str = "google") -> None: monkeypatch.setattr( "solstone.think.models.resolve_provider", @@ -1158,18 +1196,7 @@ def test_get_providers_ai_readiness_surfaces_gpu_probe_failed_from_inspect( ): monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda _model=None: { - "install_state": "installed", - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": True, - "gpu_probe_ok": False, - "binary_path": "/tmp/llama-server", - "model_path": "/tmp/model.gguf", - "model_id": LOCAL_MODEL, - "install_error": None, - }, + lambda _model=None: _local_readiness_gpu_probe_failed(), ) monkeypatch.setattr( "solstone.think.providers.local_server.probe_state", diff --git a/solstone/apps/thinking/tests/test_thinking_install_poll_js.py b/solstone/apps/thinking/tests/test_thinking_install_poll_js.py index e9eb6f1d7..2af34e106 100644 --- a/solstone/apps/thinking/tests/test_thinking_install_poll_js.py +++ b/solstone/apps/thinking/tests/test_thinking_install_poll_js.py @@ -11,13 +11,13 @@ from pathlib import Path import pytest from solstone.apps.thinking import copy as thinking_copy -from solstone.apps.thinking.install_copy import INSTALL_FAILED_NO_PROGRESS from solstone.apps.thinking.tests.js_extract import ( extract_js_const, extract_js_function, ) STATIC = Path(__file__).resolve().parents[1] / "static" / "thinking.js" +SERVER_INSTALL_ERROR = "install_interrupted" def _node_script(body: str) -> str: @@ -35,7 +35,7 @@ def _node_script(body: str) -> str: extract_js_function(source, "startInstallPoll"), "function assert(condition, message) { if (!condition) throw new Error(message); }", f"const text = {json.dumps(thinking_copy.LOCAL_INSTALL)};", - f"const installFailedNoProgress = {json.dumps(INSTALL_FAILED_NO_PROGRESS)};", + f"const serverInstallError = {json.dumps(SERVER_INSTALL_ERROR)};", body, ] return "\n".join(parts) @@ -309,7 +309,7 @@ async function main() { install_state: 'failed', progress_bytes_received: null, progress_bytes_total: null, - install_error: installFailedNoProgress, + install_error: serverInstallError, }, ]; let fetchCalls = 0; @@ -342,7 +342,7 @@ async function main() { assert(result.install_state === 'failed', 'failed status should be terminal'); assert(fetchCalls === 2, 'poll should stop after failed'); assert(sleeps.length === 1, 'poll should sleep only before failed'); - assert(failed.message === installFailedNoProgress, 'server install_error should render verbatim'); + assert(failed.message === serverInstallError, 'server install_error should render verbatim'); assert(failed.bootstrap === true, 'failed install should offer retry'); assert(failed.bootstrapLabel === text.retry, 'retry label should come from copy'); assert(failed.tone === 'bad', 'failed install should render as an error'); diff --git a/solstone/convey/provider_readiness.py b/solstone/convey/provider_readiness.py index f532304e2..b9856a994 100644 --- a/solstone/convey/provider_readiness.py +++ b/solstone/convey/provider_readiness.py @@ -139,6 +139,12 @@ _ENTRIES: dict[str, _Entry] = { detail=_LOCAL_SETUP_DETAIL, recovery_action=_LOCAL_SETUP_ACTION, ), + "install_busy": _Entry( + klass="setup", + summary="local model setup is already running", + detail="A local provider install is already in progress. Try again shortly.", + recovery_action=None, + ), "local_model_installing": _Entry( klass="setup", summary="local model setup is finishing", diff --git a/solstone/convey/static/chat_reasons.js b/solstone/convey/static/chat_reasons.js index ffeaa97fd..2dabf10cf 100644 --- a/solstone/convey/static/chat_reasons.js +++ b/solstone/convey/static/chat_reasons.js @@ -42,6 +42,10 @@ "template": "local model setup is not finished", "action": {"label": "Open Local Model Setup", "href": "/app/thinking/#local-setup"} }, + "install_busy": { + "template": "local model setup is already running", + "action": null + }, "local_model_installing": { "template": "local model setup is finishing", "action": null diff --git a/solstone/think/install_models.py b/solstone/think/install_models.py index bafdfce4b..ddd175b61 100644 --- a/solstone/think/install_models.py +++ b/solstone/think/install_models.py @@ -386,12 +386,62 @@ def _install_rfdetr_model(*, check: bool, force: bool) -> int: def _install_linux_cpp(*, force: bool = False) -> int: from solstone.think.providers import parakeet_install + from solstone.think.providers.install_lease import acquire_install_lease + from solstone.think.providers.install_state import ( + IN_FLIGHT_STATES, + begin_or_replace_install_attempt, + canonical_fingerprint, + fingerprint_sha256, + observe_install_attempt, + read_install_status, + ) + + def progress_line(status: dict[str, Any]) -> None: + received = status.get("progress_bytes_received") + total = status.get("progress_bytes_total") + suffix = "" + if received is not None: + suffix = f" {received}" + if total is not None: + suffix += f"/{total}" + print(f"observing parakeet install: {status['install_state']}{suffix}") try: - if force: - parakeet_install.install_parakeet(force=True) + fingerprint = parakeet_install.target_fingerprint() + target_sha = fingerprint_sha256(canonical_fingerprint(fingerprint)) + lease = acquire_install_lease("parakeet") + if lease is None: + status = read_install_status(name="parakeet") + if ( + status["install_state"] not in IN_FLIGHT_STATES + or status["target_fingerprint_sha256"] != target_sha + ): + return _fail("parakeet install already running for a different target") + final = observe_install_attempt( + "parakeet", + target_fingerprint_sha256=target_sha, + timeout_s=60.0 * 60.0, + progress=progress_line, + ) + if final is None: + return _fail("timed out observing parakeet install") + if final["install_state"] != "installed": + return _fail(final.get("install_error") or "parakeet install failed") else: - parakeet_install.install_parakeet() + try: + attempt_status = begin_or_replace_install_attempt( + "parakeet", + fingerprint, + initial_state="resolving", + owner={"entry": "install_models"}, + ) + parakeet_install.install_parakeet( + force=force, + lease=lease, + attempt_status=attempt_status, + ) + finally: + lease.release() paths = _check_linux_cpp_ready() except Exception as exc: return _fail(f"parakeet install failed: {exc}") diff --git a/solstone/think/install_provider.py b/solstone/think/install_provider.py index eee38aa1d..ad381da0a 100644 --- a/solstone/think/install_provider.py +++ b/solstone/think/install_provider.py @@ -15,6 +15,15 @@ import sys from solstone.think.providers import local_install, parakeet_install from solstone.think.providers.fit_report import FitReport +from solstone.think.providers.install_lease import acquire_install_lease +from solstone.think.providers.install_state import ( + IN_FLIGHT_STATES, + begin_or_replace_install_attempt, + canonical_fingerprint, + fingerprint_sha256, + observe_install_attempt, + read_install_status, +) from solstone.think.utils import require_solstone PARAKEET_DOWNLOAD_DISCLOSURE = ( @@ -30,6 +39,48 @@ def _render_fit_report(report: FitReport) -> None: print(fit_report.render_fit_report(report), file=sys.stderr) +def _target_sha(fingerprint: dict) -> str: + return fingerprint_sha256(canonical_fingerprint(fingerprint)) + + +def _progress_line(status: dict) -> None: + received = status.get("progress_bytes_received") + total = status.get("progress_bytes_total") + progress = "" + if received is not None: + progress = f" {received}" + if total is not None: + progress += f"/{total}" + print( + f"observing {status['provider']} install: {status['install_state']}{progress}", + file=sys.stderr, + ) + + +def _observe_same_target(provider: str, target_sha: str) -> int: + status = read_install_status(name=provider) + if ( + status["install_state"] not in IN_FLIGHT_STATES + or status["target_fingerprint_sha256"] != target_sha + ): + print( + f"{provider} install already running for a different target", + file=sys.stderr, + ) + return 1 + final = observe_install_attempt( + provider, + target_fingerprint_sha256=target_sha, + timeout_s=60.0 * 60.0, + progress=_progress_line, + ) + if final is None: + print(f"timed out observing {provider} install", file=sys.stderr) + return 1 + print(json.dumps(final, indent=2)) + return 0 if final["install_state"] == "installed" else 1 + + def main() -> int: parser = argparse.ArgumentParser( prog="journal install-provider", @@ -50,34 +101,63 @@ def main() -> int: if args.name == "parakeet": print(PARAKEET_DOWNLOAD_DISCLOSURE, file=sys.stderr) readiness = parakeet_install.inspect_readiness() - installed = bool(readiness["binary_installed"] and readiness["model_installed"]) - if installed: + if readiness.ready: print("parakeet already installed", file=sys.stderr) - else: + print(json.dumps(read_install_status(name="parakeet"), indent=2)) + return 0 + fingerprint = parakeet_install.target_fingerprint() + target_sha = _target_sha(fingerprint) + lease = acquire_install_lease("parakeet") + if lease is None: + return _observe_same_target("parakeet", target_sha) + try: from solstone.think.providers import fit_report _render_fit_report(fit_report.build_parakeet_fit_report()) - try: - status = parakeet_install.install_parakeet() + attempt_status = begin_or_replace_install_attempt( + "parakeet", + fingerprint, + initial_state="resolving", + owner={"entry": "install_provider"}, + ) + status = parakeet_install.install_parakeet( + lease=lease, + attempt_status=attempt_status, + ) except parakeet_install.ParakeetProviderError as exc: print(str(exc), file=sys.stderr) return 1 + finally: + lease.release() print(json.dumps(status, indent=2)) return 0 readiness = local_install.inspect_readiness() - installed = bool(readiness["binary_installed"] and readiness["model_installed"]) - if installed: + if readiness.ready: print("local already installed", file=sys.stderr) - else: + print(json.dumps(read_install_status(name="local"), indent=2)) + return 0 + fingerprint = local_install.target_fingerprint() + target_sha = _target_sha(fingerprint) + lease = acquire_install_lease("local") + if lease is None: + return _observe_same_target("local", target_sha) + try: from solstone.think.providers import fit_report _render_fit_report(fit_report.build_local_fit_report(local_install.LOCAL_MODEL)) - try: - status = local_install.install_local() + attempt_status = begin_or_replace_install_attempt( + "local", + fingerprint, + initial_state="resolving", + owner={"entry": "install_provider"}, + ) + status = local_install.install_local(lease=lease, attempt_status=attempt_status) except local_install.LocalProviderError as exc: print(str(exc), file=sys.stderr) return 1 + finally: + lease.release() print(json.dumps(status, indent=2)) return 0 diff --git a/solstone/think/providers/artifact_proof.py b/solstone/think/providers/artifact_proof.py index 2ccf48cd9..8decc5cdb 100644 --- a/solstone/think/providers/artifact_proof.py +++ b/solstone/think/providers/artifact_proof.py @@ -51,6 +51,10 @@ class ReadinessOutcome: artifacts: dict[str, Any] proof: dict[str, Any] + @property + def ready(self) -> bool: + return self.status == "ready" + @dataclass(frozen=True) class ProofResult: diff --git a/solstone/think/providers/install_state.py b/solstone/think/providers/install_state.py index efe9d2c44..b89cc22b7 100644 --- a/solstone/think/providers/install_state.py +++ b/solstone/think/providers/install_state.py @@ -11,7 +11,7 @@ import time import uuid from datetime import datetime, timezone from pathlib import Path -from typing import Any, Literal, NotRequired, TypedDict, cast, get_args +from typing import Any, Callable, Literal, TypedDict, cast, get_args from solstone.think.journal_config import JournalConfigMutation, mutate_journal_config from solstone.think.journal_io.atomic import atomic_replace @@ -47,7 +47,6 @@ class InstallStatus(TypedDict): install_error: str | None error_code: str | None owner: dict[str, Any] | None - name: NotRequired[str] class InstallStateError(RuntimeError): @@ -71,7 +70,7 @@ IN_FLIGHT_STATES: frozenset[InstallState] = frozenset( TERMINAL_STATES: frozenset[InstallState] = frozenset({"idle", "installed", "failed"}) _INSTALL_STATES = frozenset(get_args(InstallState)) _STATUS_MODE = 0o600 -_LEGACY_STATUS_KEYS = frozenset( +_LEGACY_OPERATIONAL_KEYS = frozenset( { "install_state", "last_transition_at", @@ -79,6 +78,29 @@ _LEGACY_STATUS_KEYS = frozenset( "progress_bytes_received", "progress_bytes_total", "install_error", + "binary_artifact", + "binary_sha256", + "binary_path", + "model_id", + "model_path", + "model_sha256", + "mmproj_path", + "mmproj_sha256", + "mlx_model_id", + "mlx_revision", + "mlx_snapshot_dir", + "mlx_variant_dir", + "binary_artifact_cpu", + "binary_sha256_cpu", + "binary_path_cpu", + "binary_artifact_vulkan", + "binary_sha256_vulkan", + "binary_path_vulkan", + "model_repo", + "model_filename", + "model_revision", + "model_path", + "model_sha256", } ) _LAST_PROGRESS_WRITE_MONOTONIC: dict[tuple[str, str], float] = {} @@ -115,26 +137,24 @@ def provider_status_path( def make_idle_status(name: str) -> InstallStatus: provider = _validate_provider(name) - return _with_legacy_name( - { - "schema_version": SCHEMA_VERSION, - "provider": provider, - "revision": 0, - "install_state": "idle", - "attempt_id": None, - "target_fingerprint_json": None, - "target_fingerprint_sha256": None, - "started_at": None, - "last_transition_at": None, - "last_progress_at": None, - "completed_at": None, - "progress_bytes_received": None, - "progress_bytes_total": None, - "install_error": None, - "error_code": None, - "owner": None, - } - ) + return { + "schema_version": SCHEMA_VERSION, + "provider": provider, + "revision": 0, + "install_state": "idle", + "attempt_id": None, + "target_fingerprint_json": None, + "target_fingerprint_sha256": None, + "started_at": None, + "last_transition_at": None, + "last_progress_at": None, + "completed_at": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "install_error": None, + "error_code": None, + "owner": None, + } def begin_install_attempt( @@ -160,6 +180,91 @@ def begin_install_attempt( ) +def begin_or_replace_install_attempt( + provider: str, + fingerprint: dict[str, Any], + *, + initial_state: InstallState = "resolving", + owner: dict[str, Any] | None = None, + journal_path: str | Path | None = None, +) -> InstallStatus: + """Begin an attempt after the caller owns the provider lease.""" + current = read_install_status(name=provider, journal_path=journal_path) + if current["install_state"] in IN_FLIGHT_STATES: + record_interrupted_install( + provider, + attempt_id=str(current["attempt_id"]), + target_fingerprint_sha256=current["target_fingerprint_sha256"], + journal_path=journal_path, + ) + return begin_install_attempt( + provider, + fingerprint, + initial_state=initial_state, + owner=owner, + journal_path=journal_path, + ) + + +def assert_install_attempt_current( + status: InstallStatus, + *, + journal_path: str | Path | None = None, +) -> InstallStatus: + """Return current status if the same in-flight attempt still owns the slot.""" + attempt = _coerce_status(status) + current = read_install_status(name=attempt["provider"], journal_path=journal_path) + if current["install_state"] not in IN_FLIGHT_STATES: + raise InstallStatusConflictError("install attempt is no longer in-flight") + if current["attempt_id"] != attempt["attempt_id"]: + raise InstallStatusConflictError("install attempt id changed") + if current["target_fingerprint_sha256"] != attempt["target_fingerprint_sha256"]: + raise InstallStatusConflictError("install target fingerprint changed") + return current + + +def observe_install_attempt( + provider: str, + *, + target_fingerprint_sha256: str, + timeout_s: float, + poll_interval_s: float = 1.0, + progress_interval_s: float = 10.0, + progress: Callable[[InstallStatus], None] | None = None, + journal_path: str | Path | None = None, +) -> InstallStatus | None: + """Poll the status record for a same-target in-flight attempt to finish.""" + deadline = time.monotonic() + timeout_s + last_progress_emit = 0.0 + last_progress_key: tuple[Any, ...] | None = None + while True: + status = read_install_status(name=provider, journal_path=journal_path) + if status["target_fingerprint_sha256"] != target_fingerprint_sha256: + return status + if progress is not None: + progress_key = ( + status["install_state"], + status["progress_bytes_received"], + status["progress_bytes_total"], + status["install_error"], + status["error_code"], + ) + now = time.monotonic() + if ( + last_progress_key is None + or progress_key != last_progress_key + or now - last_progress_emit >= progress_interval_s + ): + progress(status) + last_progress_key = progress_key + last_progress_emit = now + if status["install_state"] in TERMINAL_STATES: + return status + if time.monotonic() >= deadline: + return None + time.sleep(poll_interval_s) + + def transition_state( status: InstallStatus, *, @@ -178,35 +283,33 @@ def transition_state( next_attempt_id = uuid.uuid4().hex is_terminal = new_state in TERMINAL_STATES - return _with_legacy_name( - { - "schema_version": SCHEMA_VERSION, - "provider": current["provider"], - "revision": current["revision"], - "install_state": new_state, - "attempt_id": None if new_state == "idle" else next_attempt_id, - "target_fingerprint_json": current["target_fingerprint_json"], - "target_fingerprint_sha256": current["target_fingerprint_sha256"], - "started_at": ( - timestamp - if current["install_state"] in TERMINAL_STATES - and new_state in IN_FLIGHT_STATES - else current["started_at"] - ), - "last_transition_at": timestamp, - "last_progress_at": timestamp if new_state in IN_FLIGHT_STATES else None, - "completed_at": timestamp if is_terminal and new_state != "idle" else None, - "progress_bytes_received": ( - None if is_terminal else current["progress_bytes_received"] - ), - "progress_bytes_total": ( - None if is_terminal else current["progress_bytes_total"] - ), - "install_error": error if new_state == "failed" else None, - "error_code": error_code if new_state == "failed" else None, - "owner": current["owner"], - } - ) + return { + "schema_version": SCHEMA_VERSION, + "provider": current["provider"], + "revision": current["revision"], + "install_state": new_state, + "attempt_id": None if new_state == "idle" else next_attempt_id, + "target_fingerprint_json": current["target_fingerprint_json"], + "target_fingerprint_sha256": current["target_fingerprint_sha256"], + "started_at": ( + timestamp + if current["install_state"] in TERMINAL_STATES + and new_state in IN_FLIGHT_STATES + else current["started_at"] + ), + "last_transition_at": timestamp, + "last_progress_at": timestamp if new_state in IN_FLIGHT_STATES else None, + "completed_at": timestamp if is_terminal and new_state != "idle" else None, + "progress_bytes_received": ( + None if is_terminal else current["progress_bytes_received"] + ), + "progress_bytes_total": None + if is_terminal + else current["progress_bytes_total"], + "install_error": error if new_state == "failed" else None, + "error_code": error_code if new_state == "failed" else None, + "owner": current["owner"], + } def bump_progress( @@ -218,32 +321,28 @@ def bump_progress( current = _coerce_status(status) if current["install_state"] not in IN_FLIGHT_STATES: raise ValueError("install progress can only be bumped for in-flight states") - return _with_legacy_name( - { - **current, - "last_progress_at": now_iso(), - "progress_bytes_received": ( - _nonnegative_int(received) - if received is not None - else current["progress_bytes_received"] - ), - "progress_bytes_total": ( - _nonnegative_int(total) - if total is not None - else current["progress_bytes_total"] - ), - } - ) + return { + **current, + "last_progress_at": now_iso(), + "progress_bytes_received": ( + _nonnegative_int(received) + if received is not None + else current["progress_bytes_received"] + ), + "progress_bytes_total": ( + _nonnegative_int(total) + if total is not None + else current["progress_bytes_total"] + ), + } def read_install_status( *, - scope: str = "bundled", name: str, journal_path: str | Path | None = None, ) -> InstallStatus: """Read provider install status; absent status is synthetic idle.""" - _validate_scope(scope) provider = _validate_provider(name) path = provider_status_path(provider, journal_path=journal_path) if not path.exists(): @@ -254,24 +353,22 @@ def read_install_status( raise InstallStatusMalformedError(f"malformed install status: {path}") from exc if not isinstance(data, dict): raise InstallStatusMalformedError(f"install status must be an object: {path}") - return _with_legacy_name(_coerce_status(data, provider=provider)) + return _coerce_status(data, provider=provider) def write_install_status( status: InstallStatus, *, - scope: str = "bundled", journal_path: str | Path | None = None, ) -> InstallStatus: """Write provider install status under a sidecar flock.""" - _validate_scope(scope) incoming = _coerce_status(status) path = provider_status_path(incoming["provider"], journal_path=journal_path) with hold_lock(path, mode=_STATUS_MODE): current = _read_current_unlocked(path, incoming["provider"]) accepted = _accept_transition(current, incoming) if accepted is current: - return _with_legacy_name(current) + return current stored = {**accepted, "revision": current["revision"] + 1} atomic_replace( path, @@ -279,7 +376,7 @@ def write_install_status( mode=_STATUS_MODE, ) _record_progress_write(stored) - return _with_legacy_name(stored) + return stored def record_interrupted_install( @@ -315,21 +412,35 @@ def migrate_legacy_provider_install_state( *, journal_path: str | Path | None = None, ) -> dict[str, int]: - """Remove legacy provider install status fields from journal config.""" + """Remove legacy provider install operational fields from journal config.""" def apply(config: dict[str, Any]) -> JournalConfigMutation[dict[str, int]]: removed = 0 + moved = 0 bundled = config.get("providers", {}).get("bundled") if isinstance(bundled, dict): + local_record = bundled.get("local") + if isinstance(local_record, dict) and "vulkan_device_index" in local_record: + providers = config.setdefault("providers", {}) + owner_config = providers.setdefault("local", {}) + if isinstance(owner_config, dict): + value = local_record.pop("vulkan_device_index") + if owner_config.get("vulkan_device_index") != value: + owner_config["vulkan_device_index"] = value + moved += 1 + removed += 1 for provider in PROVIDERS: record = bundled.get(provider) if not isinstance(record, dict): continue - for key in _LEGACY_STATUS_KEYS: + for key in _LEGACY_OPERATIONAL_KEYS: if key in record: record.pop(key, None) removed += 1 - return JournalConfigMutation(changed=removed > 0, value={"removed": removed}) + return JournalConfigMutation( + changed=removed > 0 or moved > 0, + value={"removed": removed, "moved": moved}, + ) return mutate_journal_config(apply, journal_path=journal_path).value @@ -343,7 +454,7 @@ def _read_current_unlocked(path: Path, provider: ProviderName) -> InstallStatus: raise InstallStatusMalformedError(f"malformed install status: {path}") from exc if not isinstance(data, dict): raise InstallStatusMalformedError(f"install status must be an object: {path}") - return _with_legacy_name(_coerce_status(data, provider=provider)) + return _coerce_status(data, provider=provider) def _accept_transition( @@ -429,7 +540,7 @@ def _coerce_status( *, provider: ProviderName | None = None, ) -> InstallStatus: - raw_provider = provider or data.get("provider") or data.get("name") + raw_provider = provider or data.get("provider") validated_provider = _validate_provider(raw_provider) state = data.get("install_state") if state not in _INSTALL_STATES: @@ -458,37 +569,28 @@ def _coerce_status( owner = data.get("owner") if owner is not None and not isinstance(owner, dict): raise InstallStatusMalformedError("owner must be an object or null") - return _with_legacy_name( - { - "schema_version": SCHEMA_VERSION, - "provider": validated_provider, - "revision": revision, - "install_state": cast(InstallState, state), - "attempt_id": attempt_id, - "target_fingerprint_json": fingerprint_json, - "target_fingerprint_sha256": fingerprint_sha, - "started_at": _optional_str(data.get("started_at")), - "last_transition_at": _optional_str(data.get("last_transition_at")), - "last_progress_at": _optional_str(data.get("last_progress_at")), - "completed_at": _optional_str(data.get("completed_at")), - "progress_bytes_received": _optional_int( - data.get("progress_bytes_received") - ), - "progress_bytes_total": _optional_int(data.get("progress_bytes_total")), - "install_error": _optional_str(data.get("install_error")), - "error_code": _optional_str(data.get("error_code")), - "owner": owner, - } - ) + return { + "schema_version": SCHEMA_VERSION, + "provider": validated_provider, + "revision": revision, + "install_state": cast(InstallState, state), + "attempt_id": attempt_id, + "target_fingerprint_json": fingerprint_json, + "target_fingerprint_sha256": fingerprint_sha, + "started_at": _optional_str(data.get("started_at")), + "last_transition_at": _optional_str(data.get("last_transition_at")), + "last_progress_at": _optional_str(data.get("last_progress_at")), + "completed_at": _optional_str(data.get("completed_at")), + "progress_bytes_received": _optional_int(data.get("progress_bytes_received")), + "progress_bytes_total": _optional_int(data.get("progress_bytes_total")), + "install_error": _optional_str(data.get("install_error")), + "error_code": _optional_str(data.get("error_code")), + "owner": owner, + } def _persistable_status(status: InstallStatus) -> dict[str, Any]: - return {key: value for key, value in status.items() if key != "name"} - - -def _with_legacy_name(status: dict[str, Any]) -> InstallStatus: - status["name"] = status["provider"] - return cast(InstallStatus, status) + return dict(status) def _optional_str(value: Any) -> str | None: @@ -520,11 +622,6 @@ def _validate_provider(value: object) -> ProviderName: return cast(ProviderName, value) -def _validate_scope(scope: str) -> None: - if scope != "bundled": - raise ValueError("install status scope must be 'bundled'") - - def _normalize_fingerprint_value(value: Any) -> Any: if isinstance(value, dict): return { @@ -559,13 +656,16 @@ __all__ = [ "ProviderName", "SCHEMA_VERSION", "TERMINAL_STATES", + "assert_install_attempt_current", "begin_install_attempt", + "begin_or_replace_install_attempt", "bump_progress", "canonical_fingerprint", "fingerprint_sha256", "make_idle_status", "migrate_legacy_provider_install_state", "now_iso", + "observe_install_attempt", "provider_status_path", "read_install_status", "record_interrupted_install", diff --git a/solstone/think/providers/local_install.py b/solstone/think/providers/local_install.py index b14634884..a851ea73f 100644 --- a/solstone/think/providers/local_install.py +++ b/solstone/think/providers/local_install.py @@ -3,34 +3,41 @@ """Install and inspect bundled local provider artifacts. -This module is the sole writer for ``providers.bundled.local`` install state. -It performs no network access at import time. +This module owns local provider artifact acquisition. It performs no network +access at import time. """ from __future__ import annotations import hashlib import logging -import os import platform import shutil import stat import sys -import time from dataclasses import dataclass from pathlib import Path from typing import Any, Callable -from solstone.think.journal_config import ( - JournalConfigMutation, - mutate_journal_config, - read_journal_config, -) +from solstone.think.journal_config import read_journal_config from solstone.think.models import LOCAL_MODEL +from solstone.think.providers.artifact_proof import ( + ReadinessOutcome, + artifact_manifest_path, + build_manifest, + prove_cuda_sidecar, + prove_manifest, + write_manifest, +) +from solstone.think.providers.install_lease import InstallLease, acquire_install_lease from solstone.think.providers.install_state import ( IN_FLIGHT_STATES, InstallStatus, + assert_install_attempt_current, + begin_or_replace_install_attempt, bump_progress, + canonical_fingerprint, + fingerprint_sha256, read_install_status, transition_state, write_install_status, @@ -51,20 +58,6 @@ from solstone.think.utils import get_journal LOG = logging.getLogger(__name__) LOCAL_PROVIDER_NAME = "local" _PROBE_TIMEOUT_SECONDS = 10 -_PROGRESS_MIN_INTERVAL_SECONDS = 1.0 # rate-limit durable install-progress writes to ~1/sec (download throughput fix) -_LOCAL_METADATA_KEYS = frozenset( - { - "binary_artifact", - "binary_sha256", - "binary_path", - "model_id", - "model_path", - "model_sha256", - "mmproj_path", - "mmproj_sha256", - "vulkan_device_index", - } -) @dataclass(frozen=True) @@ -280,36 +273,17 @@ def install_hint() -> str: def _read_local_status() -> InstallStatus: - return read_install_status(scope="bundled", name=LOCAL_PROVIDER_NAME) + return read_install_status(name=LOCAL_PROVIDER_NAME) def _write_local_status(status: InstallStatus) -> InstallStatus: - write_install_status(status, scope="bundled") + write_install_status(status) return status -def _write_local_metadata(updates: dict[str, str]) -> None: - unknown_keys = sorted(set(updates) - _LOCAL_METADATA_KEYS) - if unknown_keys: - raise ValueError(f"unknown local install metadata key: {unknown_keys[0]}") - - def apply(config: dict[str, Any]) -> JournalConfigMutation[None]: - slot = ( - config.setdefault("providers", {}) - .setdefault("bundled", {}) - .setdefault(LOCAL_PROVIDER_NAME, {}) - ) - changed = any(slot.get(key) != value for key, value in updates.items()) - for key, value in updates.items(): - slot[key] = value - return JournalConfigMutation(changed=changed, value=None) - - mutate_journal_config(apply) - - def gpu_device_override() -> int | None: config = read_journal_config() - record = config.get("providers", {}).get("bundled", {}).get(LOCAL_PROVIDER_NAME, {}) + record = config.get("providers", {}).get(LOCAL_PROVIDER_NAME, {}) if not isinstance(record, dict): return None value = record.get("vulkan_device_index") @@ -329,6 +303,148 @@ def _record_local_progress(received: int, total: int | None) -> None: _write_local_status(bump_progress(status, received=received, total=total)) +def _model_pin_identity(model_id: str) -> dict[str, Any]: + spec = LOCAL_MODEL_SPECS[normalize_model_id(model_id)] + return { + "unit": "local-model", + "model_id": spec.model_id, + "repo": spec.repo, + "revision": spec.revision, + "filename": spec.filename, + "sha256": spec.sha256, + "mmproj_filename": spec.mmproj_filename, + "mmproj_sha256": spec.mmproj_sha256, + } + + +def _vulkan_pin_identity( + artifact_key: str | None = None, + pin: dict[str, str] | None = None, +) -> dict[str, Any]: + artifact_key = artifact_key or llama_server_artifact_key() + pin = pin or pin_for_current_platform() + return { + "unit": "llama-server-vulkan", + "artifact_key": artifact_key, + "release_tag": pin["release_tag"], + "filename": pin["filename"], + "sha256": pin["sha256"], + "binary_name": pin["binary_name"], + } + + +def _cuda_pin_identity( + arch: str | None = None, + wanted_files: tuple[str, ...] | None = None, +) -> dict[str, Any]: + arch = arch or _oci_arch() + wanted_files = wanted_files or CUDA_SERVER_PIN.wanted_files_for_arch(arch) + return { + "unit": "llama-server-cuda", + "artifact_key": llama_server_artifact_key(), + "image_ref": CUDA_SERVER_PIN.image_ref, + "arch": arch, + "binary_name": CUDA_SERVER_PIN.binary_name, + "wanted_files": list(wanted_files), + } + + +def target_fingerprint(model_id: str = LOCAL_MODEL) -> dict[str, Any]: + from solstone.think.providers import local_cuda + + selected_model = normalize_model_id(model_id) + choice = local_cuda.resolve_local_backend(CUDA_SERVER_PIN) + runtime_pin = ( + _cuda_pin_identity() if choice.backend == "cuda" else _vulkan_pin_identity() + ) + return { + "provider": LOCAL_PROVIDER_NAME, + "runtime": "llama.cpp", + "backend": choice.backend, + "backend_reason": choice.reason, + "runtime_pin": runtime_pin, + "model_pin": _model_pin_identity(selected_model), + } + + +def _fingerprint_sha_for_target(fingerprint: dict[str, Any]) -> str: + return fingerprint_sha256(canonical_fingerprint(fingerprint)) + + +def _manifest_target_sha( + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any], +) -> str: + if attempt_status is not None and attempt_status["target_fingerprint_sha256"]: + return str(attempt_status["target_fingerprint_sha256"]) + return _fingerprint_sha_for_target(fingerprint) + + +def _manifest_entry(path: Path, root: Path, role: str) -> dict[str, Any]: + return { + "relative_path": path.relative_to(root).as_posix(), + "role": role, + "size": path.stat().st_size, + "sha256": _sha256_file(path), + } + + +def _runtime_inventory(root: Path, *, exclude_names: set[str]) -> list[dict[str, Any]]: + inventory: list[dict[str, Any]] = [] + for path in sorted(root.rglob("*")): + if not path.is_file() or path.name == artifact_manifest_path(root).name: + continue + if path.name in exclude_names: + continue + role = "runtime_binary" if path.name == "llama-server" else "runtime_support" + inventory.append(_manifest_entry(path, root, role)) + return inventory + + +def _write_vulkan_manifest( + *, + artifact_key: str, + pin: dict[str, str], + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any] | None = None, +) -> None: + install_dir = binary_install_dir(artifact_key, pin) + fingerprint = fingerprint or target_fingerprint() + manifest = build_manifest( + provider=LOCAL_PROVIDER_NAME, + unit="llama-server-vulkan", + target_fingerprint_sha256=_manifest_target_sha(attempt_status, fingerprint), + source={"pin_identity": _vulkan_pin_identity(artifact_key, pin)}, + inventory=_runtime_inventory(install_dir, exclude_names={pin["filename"]}), + attempt_id=attempt_status["attempt_id"] if attempt_status else None, + ) + write_manifest(artifact_manifest_path(install_dir), manifest) + + +def _write_model_manifest( + *, + model_id: str, + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any] | None = None, +) -> None: + spec = LOCAL_MODEL_SPECS[normalize_model_id(model_id)] + root = model_dir(spec.model_id) + inventory = [_manifest_entry(model_path(spec.model_id), root, "model")] + projector = mmproj_path(spec.model_id) + if projector is not None: + inventory.append(_manifest_entry(projector, root, "projector")) + fingerprint = fingerprint or target_fingerprint(spec.model_id) + manifest = build_manifest( + provider=LOCAL_PROVIDER_NAME, + unit="local-model", + target_fingerprint_sha256=_manifest_target_sha(attempt_status, fingerprint), + source={"pin_identity": _model_pin_identity(spec.model_id)}, + inventory=inventory, + attempt_id=attempt_status["attempt_id"] if attempt_status else None, + ) + write_manifest(artifact_manifest_path(root), manifest) + + def _sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: @@ -362,25 +478,14 @@ def _download_file( total_header = response.headers.get("content-length") total = int(total_header) if total_header and total_header.isdigit() else None received = 0 - last_emit = time.monotonic() - last_emitted_received = -1 - first_chunk = True with tmp.open("wb") as handle: for chunk in response.iter_bytes(): if not chunk: continue handle.write(chunk) received += len(chunk) - if on_progress is None: - continue - now = time.monotonic() - if first_chunk or now - last_emit >= _PROGRESS_MIN_INTERVAL_SECONDS: + if on_progress is not None: on_progress(received, total) - last_emit = now - last_emitted_received = received - first_chunk = False - if on_progress is not None and received != last_emitted_received: - on_progress(received, total) tmp.replace(dest) @@ -480,12 +585,16 @@ def probe_binary_runnable(binary_path: str | Path) -> tuple[bool, str | None]: return False, detail -def install_llama_server() -> dict[str, Any]: +def install_llama_server( + *, + attempt_status: InstallStatus | None = None, + fingerprint: dict[str, Any] | None = None, +) -> dict[str, Any]: from solstone.think.providers import local_cuda choice = local_cuda.resolve_local_backend(CUDA_SERVER_PIN) if choice.backend == "cuda": - return _install_cuda_llama_server() + return _install_cuda_llama_server(attempt_status=attempt_status) artifact_key = llama_server_artifact_key() pin = pin_for_current_platform() @@ -500,7 +609,6 @@ def install_llama_server() -> dict[str, Any]: _write_local_status( transition_state(_read_local_status(), new_state="downloading") ) - _write_local_metadata({"binary_artifact": pin["filename"]}) _download_file(url, tarball, on_progress=_record_local_progress) _write_local_status( transition_state(_read_local_status(), new_state="verifying") @@ -516,6 +624,8 @@ def install_llama_server() -> dict[str, Any]: _safe_extract_tarball(tarball, install_dir) extracted = _find_extracted_binary(install_dir, pin["binary_name"]) final_path = binary_path_for_pin(artifact_key, pin) + if attempt_status is not None: + assert_install_attempt_current(attempt_status) inner_dir = extracted.parent if inner_dir != install_dir: for item in inner_dir.iterdir(): @@ -523,16 +633,13 @@ def install_llama_server() -> dict[str, Any]: inner_dir.rmdir() _chmod_executable(final_path) _clear_macos_quarantine(install_dir) - _write_local_metadata( - { - "binary_artifact": pin["filename"], - "binary_sha256": pin["sha256"], - "binary_path": str(final_path), - } - ) - return _write_local_status( - transition_state(_read_local_status(), new_state="installed") + _write_vulkan_manifest( + artifact_key=artifact_key, + pin=pin, + attempt_status=attempt_status, + fingerprint=fingerprint, ) + return _read_local_status() except Exception as exc: _write_local_status( transition_state(_read_local_status(), new_state="failed", error=str(exc)) @@ -540,7 +647,10 @@ def install_llama_server() -> dict[str, Any]: raise -def _install_cuda_llama_server() -> dict[str, Any]: +def _install_cuda_llama_server( + *, + attempt_status: InstallStatus | None = None, +) -> dict[str, Any]: from solstone.think.providers import oci_image try: @@ -559,10 +669,10 @@ def _install_cuda_llama_server() -> dict[str, Any]: _write_local_status( transition_state(_read_local_status(), new_state="verifying") ) + if attempt_status is not None: + assert_install_attempt_current(attempt_status) _chmod_executable(cuda_binary_path()) - return _write_local_status( - transition_state(_read_local_status(), new_state="installed") - ) + return _read_local_status() except Exception as exc: _write_local_status( transition_state(_read_local_status(), new_state="failed", error=str(exc)) @@ -570,7 +680,12 @@ def _install_cuda_llama_server() -> dict[str, Any]: raise -def install_model(model_id: str = LOCAL_MODEL) -> dict[str, Any]: +def install_model( + model_id: str = LOCAL_MODEL, + *, + attempt_status: InstallStatus | None = None, + fingerprint: dict[str, Any] | None = None, +) -> dict[str, Any]: spec = LOCAL_MODEL_SPECS[normalize_model_id(model_id)] url = f"https://huggingface.co/{spec.repo}/resolve/{spec.revision}/{spec.filename}" dest = model_path(spec.model_id) @@ -580,7 +695,6 @@ def install_model(model_id: str = LOCAL_MODEL) -> dict[str, Any]: _write_local_status( transition_state(_read_local_status(), new_state="downloading") ) - _write_local_metadata({"model_id": spec.model_id}) _download_file(url, dest, on_progress=_record_local_progress) if spec.mmproj_filename and mmproj_dest is not None: mmproj_url = ( @@ -592,19 +706,16 @@ def install_model(model_id: str = LOCAL_MODEL) -> dict[str, Any]: transition_state(_read_local_status(), new_state="verifying") ) _verify_sha256(dest, spec.sha256) - metadata = { - "model_id": spec.model_id, - "model_path": str(dest), - "model_sha256": spec.sha256, - } if spec.mmproj_sha256 and mmproj_dest is not None: _verify_sha256(mmproj_dest, spec.mmproj_sha256) - metadata["mmproj_path"] = str(mmproj_dest) - metadata["mmproj_sha256"] = spec.mmproj_sha256 - _write_local_metadata(metadata) - return _write_local_status( - transition_state(_read_local_status(), new_state="installed") + if attempt_status is not None: + assert_install_attempt_current(attempt_status) + _write_model_manifest( + model_id=spec.model_id, + attempt_status=attempt_status, + fingerprint=fingerprint, ) + return _read_local_status() except Exception as exc: _write_local_status( transition_state(_read_local_status(), new_state="failed", error=str(exc)) @@ -612,94 +723,165 @@ def install_model(model_id: str = LOCAL_MODEL) -> dict[str, Any]: raise -def install_local(model_id: str = LOCAL_MODEL) -> dict[str, Any]: +def install_local( + model_id: str = LOCAL_MODEL, + *, + lease: InstallLease | None = None, + attempt_status: InstallStatus | None = None, +) -> dict[str, Any]: from solstone.think.providers import fit_report selected_model = normalize_model_id(model_id) - readiness = inspect_readiness(selected_model) - if readiness["binary_installed"] and readiness["model_installed"]: + fingerprint = target_fingerprint(selected_model) + owned_lease = lease is None + if lease is None: + lease = acquire_install_lease(LOCAL_PROVIDER_NAME) + if lease is None: + raise LocalProviderError( + "install_busy", "Local provider install is already running." + ) + + try: + if attempt_status is None: + attempt_status = begin_or_replace_install_attempt( + LOCAL_PROVIDER_NAME, + fingerprint, + initial_state="resolving", + owner={"entry": "install_local"}, + ) + readiness = inspect_readiness(selected_model) + if readiness.status in {"proof-unavailable", "host-ineligible"}: + current = assert_install_attempt_current(attempt_status) + return _write_local_status( + transition_state( + current, + new_state="failed", + error=readiness.reason_code, + error_code=readiness.reason_code, + ) + ) + + if not readiness.ready: + report = fit_report.build_local_fit_report(selected_model) + rendered = fit_report.render_fit_report(report) + if report.overall == "blocked": + raise LocalProviderError("host_unfit", rendered) + if report.overall == "warning": + LOG.warning("local provider host fit warning:\n%s", rendered) + + if readiness.proof["binary"]["status"] == "missing-or-mismatched": + install_llama_server( + attempt_status=attempt_status, + fingerprint=fingerprint, + ) + if readiness.proof["model"]["status"] == "missing-or-mismatched": + install_model( + selected_model, + attempt_status=attempt_status, + fingerprint=fingerprint, + ) + + final_readiness = inspect_readiness(selected_model) + current = assert_install_attempt_current(attempt_status) + if final_readiness.ready: + return _write_local_status(transition_state(current, new_state="installed")) return _write_local_status( - transition_state(_read_local_status(), new_state="installed") + transition_state( + current, + new_state="failed", + error=final_readiness.reason_code, + error_code=final_readiness.reason_code, + ) ) + except Exception as exc: + try: + current = assert_install_attempt_current(attempt_status) + _write_local_status( + transition_state( + current, + new_state="failed", + error=str(exc), + error_code=getattr(exc, "reason_code", None), + ) + ) + except Exception: + pass + raise + finally: + if owned_lease: + lease.release() + + +def _proof_payload( + status: str, reason_code: str, *, cache_hit: bool = False +) -> dict[str, Any]: + return {"status": status, "reason_code": reason_code, "cache_hit": cache_hit} + + +def _proof_result_payload(result: Any) -> dict[str, Any]: + return _proof_payload( + result.status, + result.reason_code, + cache_hit=bool(getattr(result, "cache_hit", False)), + ) - report = fit_report.build_local_fit_report(selected_model) - rendered = fit_report.render_fit_report(report) - if report.overall == "blocked": - raise LocalProviderError("host_unfit", rendered) - if report.overall == "warning": - LOG.warning("local provider host fit warning:\n%s", rendered) - install_llama_server() - return install_model(selected_model) +def _combined_artifact_status( + *proofs: dict[str, Any], +) -> tuple[str, str]: + for proof in proofs: + if proof["status"] == "proof-unavailable": + return "proof-unavailable", str(proof["reason_code"]) + for proof in proofs: + if proof["status"] == "missing-or-mismatched": + return "missing-or-mismatched", str(proof["reason_code"]) + return "ready", "ready" def inspect_artifacts(model_id: str | None = None) -> dict[str, Any]: from solstone.think.providers import oci_image - config = read_journal_config() - record = config.get("providers", {}).get("bundled", {}).get(LOCAL_PROVIDER_NAME, {}) - if not isinstance(record, dict): - record = {} - selected_model = normalize_model_id( - model_id or record.get("model_id") or LOCAL_MODEL - ) + selected_model = normalize_model_id(model_id or LOCAL_MODEL) spec = LOCAL_MODEL_SPECS[selected_model] - - # The persisted record is a cache keyed by model_id. Trust a recorded - # artifact path only when the record is for the selected model AND the path - # lives under that model's directory; otherwise it is stale (e.g. left by a - # prior model's install before a LOCAL_MODEL change) and must be ignored so - # we recompute from the spec. Never pair a recorded path from one model with - # a freshly-recomputed path from another — a mixed gguf/mmproj pair aborts - # llama-server at spawn with an n_embd text/projector mismatch. - expected_dir = model_dir(selected_model) - - def _trusted_record_path(value: str | None) -> Path | None: - if not value or record.get("model_id") != selected_model: - return None - candidate = Path(value) - return candidate if candidate.parent == expected_dir else None - - gguf_path = _trusted_record_path(record.get("model_path")) or model_path( - selected_model - ) - resolved_mmproj = _trusted_record_path(record.get("mmproj_path")) or mmproj_path( - selected_model - ) - mmproj_installed = resolved_mmproj is None or resolved_mmproj.exists() + gguf_path = model_path(selected_model) + resolved_mmproj = mmproj_path(selected_model) pin = pin_for_current_platform() vulkan_binary_path = binary_path_for_pin(pin=pin) - recorded_binary_path = record.get("binary_path") - vulkan_binary_installed = ( - record.get("binary_artifact") == pin["filename"] - and record.get("binary_sha256") == pin["sha256"] - and recorded_binary_path is not None - and Path(recorded_binary_path) == vulkan_binary_path - and vulkan_binary_path.exists() - and os.access(vulkan_binary_path, os.X_OK) + vulkan_proof = prove_manifest( + artifact_manifest_path(binary_install_dir(pin=pin)), + provider=LOCAL_PROVIDER_NAME, + pin_identity=_vulkan_pin_identity(pin=pin), ) + vulkan_payload = _proof_result_payload(vulkan_proof) arch = _oci_arch() cuda_binary = cuda_binary_path() - cuda_binary_installed = ( - oci_image.verify_sidecar_install( - CUDA_SERVER_PIN.image_ref, - arch, - CUDA_SERVER_PIN.wanted_files_for_arch(arch), - cuda_binary_dir(), - ) - and cuda_binary.exists() - and os.access(cuda_binary, os.X_OK) + cuda_wanted_files = CUDA_SERVER_PIN.wanted_files_for_arch(arch) + cuda_proof = prove_cuda_sidecar( + provider=LOCAL_PROVIDER_NAME, + image_ref=CUDA_SERVER_PIN.image_ref, + arch=arch, + wanted_files=cuda_wanted_files, + target_dir=cuda_binary_dir(), + pin_identity=_cuda_pin_identity(arch, cuda_wanted_files), + verifier=oci_image.verify_sidecar_install, ) + cuda_payload = _proof_result_payload(cuda_proof) + model_proof = prove_manifest( + artifact_manifest_path(model_dir(selected_model)), + provider=LOCAL_PROVIDER_NAME, + pin_identity=_model_pin_identity(selected_model), + ) + model_payload = _proof_result_payload(model_proof) return { - "binary_installed": vulkan_binary_installed or cuda_binary_installed, - "model_installed": gguf_path.exists() and mmproj_installed, - "gguf_installed": gguf_path.exists(), - "mmproj_installed": mmproj_installed, - "vulkan_binary_installed": vulkan_binary_installed, - "cuda_binary_installed": cuda_binary_installed, + "binary_installed": vulkan_proof.ready or cuda_proof.ready, + "model_installed": model_proof.ready, + "gguf_installed": model_proof.ready, + "mmproj_installed": model_proof.ready, + "vulkan_binary_installed": vulkan_proof.ready, + "cuda_binary_installed": cuda_proof.ready, "vulkan_binary_path": str(vulkan_binary_path), "cuda_binary_path": str(cuda_binary), "binary_path": str(vulkan_binary_path), @@ -707,10 +889,13 @@ def inspect_artifacts(model_id: str | None = None) -> dict[str, Any]: "mmproj_path": str(resolved_mmproj) if resolved_mmproj is not None else None, "model_id": selected_model, "min_ram_bytes": spec.min_ram_bytes, + "vulkan_proof": vulkan_payload, + "cuda_proof": cuda_payload, + "model_proof": model_payload, } -def inspect_readiness(model_id: str | None = None) -> dict[str, Any]: +def inspect_readiness(model_id: str | None = None) -> ReadinessOutcome: from solstone.think.providers import local_cuda choice = local_cuda.resolve_local_backend(CUDA_SERVER_PIN) @@ -736,41 +921,88 @@ def inspect_readiness(model_id: str | None = None) -> dict[str, Any]: gpu_available = selected_gpu is not None gpu_probe_ok = local_vulkan.gpu_probe_ok() - return { - "install_state": status["install_state"], - "binary_installed": binary_installed, - "model_installed": artifacts["model_installed"], - "gguf_installed": artifacts["gguf_installed"], - "mmproj_installed": artifacts["mmproj_installed"], - "ram_sufficient": memory_verdict.severity != "blocked", - "gpu_available": gpu_available, - "gpu_probe_ok": gpu_probe_ok, - "binary_path": str(binary_path), - "model_path": artifacts["model_path"], - "mmproj_path": artifacts["mmproj_path"], - "model_id": artifacts["model_id"], - "install_error": status["install_error"], - "backend": choice.backend, - "backend_reason": choice.reason, - } + binary_proof = ( + artifacts["cuda_proof"] + if choice.backend == "cuda" + else artifacts["vulkan_proof"] + ) + artifact_status, artifact_reason = _combined_artifact_status( + binary_proof, + artifacts["model_proof"], + ) + ram_sufficient = memory_verdict.severity != "blocked" + if artifact_status != "ready": + readiness_status = artifact_status + reason_code = artifact_reason + elif not ram_sufficient: + readiness_status = "host-ineligible" + reason_code = "ram_insufficient" + elif not gpu_probe_ok: + readiness_status = "host-ineligible" + reason_code = "gpu_probe_failed" + elif not gpu_available: + readiness_status = "host-ineligible" + reason_code = "gpu_unavailable" + else: + readiness_status = "ready" + reason_code = "ready" + + return ReadinessOutcome( + provider=LOCAL_PROVIDER_NAME, + status=readiness_status, # type: ignore[arg-type] + reason_code=reason_code, + target={ + "model_id": artifacts["model_id"], + "target_fingerprint_json": status["target_fingerprint_json"], + "target_fingerprint_sha256": status["target_fingerprint_sha256"], + }, + install={ + "install_state": status["install_state"], + "install_error": status["install_error"], + "error_code": status["error_code"], + "attempt_id": status["attempt_id"], + "progress_bytes_received": status["progress_bytes_received"], + "progress_bytes_total": status["progress_bytes_total"], + "last_transition_at": status["last_transition_at"], + "last_progress_at": status["last_progress_at"], + }, + host={ + "ram_sufficient": ram_sufficient, + "gpu_available": gpu_available, + "gpu_probe_ok": gpu_probe_ok, + "backend": choice.backend, + "backend_reason": choice.reason, + }, + artifacts={ + **artifacts, + "binary_installed": binary_installed, + "binary_path": str(binary_path), + }, + proof={ + "binary": binary_proof, + "model": artifacts["model_proof"], + "vulkan": artifacts["vulkan_proof"], + "cuda": artifacts["cuda_proof"], + }, + ) def ensure_artifacts_installed(model_id: str) -> LocalArtifacts: selected_model = normalize_model_id(model_id) readiness = inspect_readiness(selected_model) - if not readiness["binary_installed"]: + if not readiness.artifacts["binary_installed"]: raise LocalProviderError("binary_missing", "Local runtime is not installed.") - if not readiness["model_installed"]: + if not readiness.artifacts["model_installed"]: raise LocalProviderError( "model_missing", "Local model files are not installed." ) - mmproj = readiness.get("mmproj_path") + mmproj = readiness.artifacts.get("mmproj_path") return LocalArtifacts( - backend=str(readiness["backend"]), - backend_reason=str(readiness["backend_reason"]), - binary_path=Path(readiness["binary_path"]), - lib_dir=cuda_binary_dir() if readiness["backend"] == "cuda" else None, - gguf_path=Path(readiness["model_path"]), + backend=str(readiness.host["backend"]), + backend_reason=str(readiness.host["backend_reason"]), + binary_path=Path(readiness.artifacts["binary_path"]), + lib_dir=cuda_binary_dir() if readiness.host["backend"] == "cuda" else None, + gguf_path=Path(readiness.artifacts["model_path"]), mmproj_path=Path(mmproj) if mmproj else None, ) @@ -795,4 +1027,5 @@ __all__ = [ "gpu_device_override", "inspect_readiness", "ensure_artifacts_installed", + "target_fingerprint", ] diff --git a/solstone/think/providers/mlx_install.py b/solstone/think/providers/mlx_install.py index dcaa1a7f9..7c7ee43bb 100644 --- a/solstone/think/providers/mlx_install.py +++ b/solstone/think/providers/mlx_install.py @@ -16,14 +16,22 @@ from dataclasses import dataclass from pathlib import Path from typing import Any -from solstone.think.journal_config import ( - JournalConfigMutation, - mutate_journal_config, - read_journal_config, -) from solstone.think.models import GEMMA4_26B_A4B_4BIT, QWEN_35_9B +from solstone.think.providers.artifact_proof import ( + ReadinessOutcome, + build_manifest, + mlx_snapshot_manifest_path, + mlx_variant_manifest_path, + prove_manifest, + write_manifest, +) +from solstone.think.providers.install_lease import InstallLease, acquire_install_lease from solstone.think.providers.install_state import ( InstallStatus, + assert_install_attempt_current, + begin_or_replace_install_attempt, + canonical_fingerprint, + fingerprint_sha256, read_install_status, transition_state, write_install_status, @@ -39,14 +47,6 @@ _GEMMA4_MIN_POSITION_EMBEDDING_SIZE = 10240 _LOCAL_NAME = "local" _HASH_CHUNK_SIZE = 1024 * 1024 _REWRITTEN_VARIANT_FILES = frozenset({"config.json", "processor_config.json"}) -_MLX_METADATA_KEYS = frozenset( - { - "mlx_model_id", - "mlx_revision", - "mlx_snapshot_dir", - "mlx_variant_dir", - } -) @dataclass(frozen=True) @@ -84,11 +84,11 @@ class MLXVerificationError(RuntimeError): def _read_status() -> InstallStatus: - return read_install_status(scope="bundled", name=_LOCAL_NAME) + return read_install_status(name=_LOCAL_NAME) def _write_status(status: InstallStatus) -> InstallStatus: - write_install_status(status, scope="bundled") + write_install_status(status) return status @@ -128,6 +128,40 @@ def resolve_model_spec(model_id: str | None = None) -> MLXModelSpec: return spec +def _pin_identity(spec: MLXModelSpec) -> dict[str, Any]: + return { + "unit": "mlx-snapshot", + "model_id": spec.name, + "repo": spec.repo, + "revision": spec.revision, + "soft_token_budget": ( + MLX_SOFT_TOKEN_BUDGET if spec.name == GEMMA4_26B_A4B_4BIT else None + ), + } + + +def target_fingerprint(model_id: str | None = None) -> dict[str, Any]: + spec = resolve_model_spec(model_id) + return { + "provider": _LOCAL_NAME, + "runtime": "mlx", + "model_pin": _pin_identity(spec), + } + + +def _fingerprint_sha_for_target(fingerprint: dict[str, Any]) -> str: + return fingerprint_sha256(canonical_fingerprint(fingerprint)) + + +def _manifest_target_sha( + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any], +) -> str: + if attempt_status is not None and attempt_status["target_fingerprint_sha256"]: + return str(attempt_status["target_fingerprint_sha256"]) + return _fingerprint_sha_for_target(fingerprint) + + def snapshot_dir_for_spec(spec: MLXModelSpec) -> Path: from huggingface_hub import constants from huggingface_hub.file_download import repo_folder_name @@ -194,7 +228,9 @@ def _remote_safetensors_metadata( return found -def validate_snapshot_sha256(spec: MLXModelSpec, snapshot_dir: Path) -> None: +def validate_snapshot_sha256( + spec: MLXModelSpec, snapshot_dir: Path +) -> dict[str, tuple[str, int]]: safetensors_paths = _safetensors_paths(snapshot_dir) metadata = _remote_safetensors_metadata(spec, safetensors_paths) @@ -208,6 +244,7 @@ def validate_snapshot_sha256(spec: MLXModelSpec, snapshot_dir: Path) -> None: actual_sha = digest.hexdigest() if actual_sha != expected_sha: raise MLXVerificationError(f"sha256 mismatch for {rel_path}") + return metadata def _read_json(path: Path) -> dict[str, Any]: @@ -218,6 +255,90 @@ def _write_json(path: Path, data: dict[str, Any]) -> None: path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") +def _variant_pin_identity(spec: MLXModelSpec) -> dict[str, Any]: + identity = dict(_pin_identity(spec)) + identity["unit"] = "mlx-variant" + identity["variant"] = f"solstone-budget{MLX_SOFT_TOKEN_BUDGET}" + return identity + + +def _manifest_inventory_for_tree( + root: Path, + *, + role_prefix: str, + known_hashes: dict[str, tuple[str, int]] | None = None, +) -> list[dict[str, Any]]: + inventory: list[dict[str, Any]] = [] + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + rel_path = path.relative_to(root).as_posix() + known = known_hashes.get(rel_path) if known_hashes is not None else None + if known is None: + digest = _sha256_file(path) + size = path.stat().st_size + else: + digest, size = known + inventory.append( + { + "relative_path": rel_path, + "role": role_prefix, + "size": size, + "sha256": digest, + } + ) + return inventory + + +def _write_snapshot_manifest( + spec: MLXModelSpec, + snapshot_dir: Path, + *, + metadata: dict[str, tuple[str, int]], + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any], +) -> None: + manifest = build_manifest( + provider=_LOCAL_NAME, + unit="mlx-snapshot", + target_fingerprint_sha256=_manifest_target_sha(attempt_status, fingerprint), + source={"pin_identity": _pin_identity(spec)}, + inventory=_manifest_inventory_for_tree( + snapshot_dir, role_prefix="snapshot_file", known_hashes=metadata + ), + external_root=snapshot_dir, + attempt_id=attempt_status["attempt_id"] if attempt_status else None, + ) + write_manifest(mlx_snapshot_manifest_path(spec.repo, spec.revision), manifest) + + +def _write_variant_manifest( + spec: MLXModelSpec, + variant_dir: Path, + *, + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any], +) -> None: + manifest = build_manifest( + provider=_LOCAL_NAME, + unit="mlx-variant", + target_fingerprint_sha256=_manifest_target_sha(attempt_status, fingerprint), + source={"pin_identity": _variant_pin_identity(spec)}, + inventory=_manifest_inventory_for_tree(variant_dir, role_prefix="variant_file"), + external_root=variant_dir, + attempt_id=attempt_status["attempt_id"] if attempt_status else None, + ) + write_manifest(mlx_variant_manifest_path(spec.repo, spec.revision), manifest) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(_HASH_CHUNK_SIZE), b""): + digest.update(chunk) + return digest.hexdigest() + + def _validate_gemma4_position_embedding(config: dict[str, Any]) -> None: vision_config = config.get("vision_config") if not isinstance(vision_config, dict): @@ -347,43 +468,8 @@ def _artifact_presence(spec: MLXModelSpec) -> dict[str, Any]: } -def _write_mlx_metadata( - spec: MLXModelSpec, - *, - snapshot_dir: Path, - variant_dir: Path | None, -) -> None: - def apply(config: dict[str, Any]) -> JournalConfigMutation[None]: - slot = ( - config.setdefault("providers", {}) - .setdefault("bundled", {}) - .setdefault(_LOCAL_NAME, {}) - ) - next_values = { - "mlx_model_id": spec.name, - "mlx_revision": spec.revision, - "mlx_snapshot_dir": str(snapshot_dir), - } - if variant_dir is not None: - next_values["mlx_variant_dir"] = str(variant_dir) - changed = any( - slot.get(key) != next_values.get(key) for key in _MLX_METADATA_KEYS - ) - for key in _MLX_METADATA_KEYS: - slot.pop(key, None) - slot.update(next_values) - return JournalConfigMutation(changed=changed, value=None) - - mutate_journal_config(apply) - - def inspect_artifacts(model_id: str | None = None) -> dict[str, Any]: - config = read_journal_config() - record = config.get("providers", {}).get("bundled", {}).get(_LOCAL_NAME, {}) - if not isinstance(record, dict): - record = {} - selected_model = model_id or record.get("mlx_model_id") or QWEN_35_9B - spec = resolve_model_spec(str(selected_model)) + spec = resolve_model_spec(model_id) presence = _artifact_presence(spec) return { "model_installed": presence["model_installed"], @@ -400,75 +486,215 @@ def inspect_artifacts(model_id: str | None = None) -> dict[str, Any]: } -def inspect_readiness(model_id: str | None = None) -> dict[str, Any]: +def inspect_readiness(model_id: str | None = None) -> ReadinessOutcome: status = _read_status() artifacts = inspect_artifacts(model_id) + spec = resolve_model_spec(str(artifacts["model_id"])) verdict = assess_memory(MLX_AVAILABLE_FLOOR_BYTES, block_below_floor=True) - return { - "install_state": status["install_state"], - "model_installed": artifacts["model_installed"], - "snapshot_installed": artifacts["snapshot_installed"], - "variant_installed": artifacts["variant_installed"], - "ram_sufficient": verdict.severity != "blocked", - "platform_supported": is_mlx_platform_supported(), - "package_available": _check_platform_and_package()[0], - "model_id": artifacts["model_id"], - "snapshot_dir": artifacts["snapshot_dir"], - "variant_dir": artifacts["variant_dir"], - "runtime_dir": artifacts["runtime_dir"], - "install_error": status["install_error"], - } + snapshot_proof = prove_manifest( + mlx_snapshot_manifest_path(spec.repo, spec.revision), + provider=_LOCAL_NAME, + pin_identity=_pin_identity(spec), + ) + if spec.name == GEMMA4_26B_A4B_4BIT: + variant_proof = prove_manifest( + mlx_variant_manifest_path(spec.repo, spec.revision), + provider=_LOCAL_NAME, + pin_identity=_variant_pin_identity(spec), + ) + else: + variant_proof = None + platform_supported = is_mlx_platform_supported() + package_available = _check_platform_and_package()[0] + ram_sufficient = verdict.severity != "blocked" + proof_payloads = [ + { + "status": snapshot_proof.status, + "reason_code": snapshot_proof.reason_code, + "cache_hit": snapshot_proof.cache_hit, + } + ] + if variant_proof is not None: + proof_payloads.append( + { + "status": variant_proof.status, + "reason_code": variant_proof.reason_code, + "cache_hit": variant_proof.cache_hit, + } + ) + if any(proof["status"] == "proof-unavailable" for proof in proof_payloads): + readiness_status = "proof-unavailable" + reason_code = next( + str(proof["reason_code"]) + for proof in proof_payloads + if proof["status"] == "proof-unavailable" + ) + elif any(proof["status"] == "missing-or-mismatched" for proof in proof_payloads): + readiness_status = "missing-or-mismatched" + reason_code = next( + str(proof["reason_code"]) + for proof in proof_payloads + if proof["status"] == "missing-or-mismatched" + ) + elif not platform_supported: + readiness_status = "host-ineligible" + reason_code = "platform_unsupported" + elif not package_available: + readiness_status = "host-ineligible" + reason_code = "package_unavailable" + elif not ram_sufficient: + readiness_status = "host-ineligible" + reason_code = "ram_insufficient" + else: + readiness_status = "ready" + reason_code = "ready" + + model_installed = snapshot_proof.ready and ( + variant_proof is None or variant_proof.ready + ) + return ReadinessOutcome( + provider=_LOCAL_NAME, + status=readiness_status, # type: ignore[arg-type] + reason_code=reason_code, + target={ + "model_id": artifacts["model_id"], + "target_fingerprint_json": status["target_fingerprint_json"], + "target_fingerprint_sha256": status["target_fingerprint_sha256"], + }, + install={ + "install_state": status["install_state"], + "install_error": status["install_error"], + "error_code": status["error_code"], + "attempt_id": status["attempt_id"], + "progress_bytes_received": status["progress_bytes_received"], + "progress_bytes_total": status["progress_bytes_total"], + "last_transition_at": status["last_transition_at"], + "last_progress_at": status["last_progress_at"], + }, + host={ + "ram_sufficient": ram_sufficient, + "platform_supported": platform_supported, + "package_available": package_available, + }, + artifacts={ + **artifacts, + "model_installed": model_installed, + "snapshot_installed": snapshot_proof.ready, + "variant_installed": variant_proof is None or variant_proof.ready, + }, + proof={ + "snapshot": proof_payloads[0], + "variant": proof_payloads[1] if len(proof_payloads) > 1 else None, + }, + ) -def install_local_mlx(model_id: str = QWEN_35_9B) -> InstallStatus: +def install_local_mlx( + model_id: str = QWEN_35_9B, + *, + lease: InstallLease | None = None, + attempt_status: InstallStatus | None = None, +) -> InstallStatus: import huggingface_hub + fingerprint = target_fingerprint(model_id) + owned_lease = lease is None + if lease is None: + lease = acquire_install_lease(_LOCAL_NAME) + if lease is None: + raise MLXInstallUnavailableError( + "Local provider install is already running." + ) try: - _write_status(transition_state(_read_status(), new_state="resolving")) - spec = resolve_model_spec(model_id) - presence = _artifact_presence(spec) - if presence["model_installed"]: - _write_mlx_metadata( - spec, - snapshot_dir=presence["snapshot_dir"], - variant_dir=presence["variant_dir"], + if attempt_status is None: + attempt_status = begin_or_replace_install_attempt( + _LOCAL_NAME, + fingerprint, + initial_state="resolving", + owner={"entry": "install_local_mlx"}, ) + spec = resolve_model_spec(model_id) + readiness = inspect_readiness(model_id) + if readiness.status in {"proof-unavailable", "host-ineligible"}: + current = assert_install_attempt_current(attempt_status) return _write_status( - transition_state(_read_status(), new_state="installed") + transition_state( + current, + new_state="failed", + error=readiness.reason_code, + error_code=readiness.reason_code, + ) ) from solstone.think.providers import fit_report - report = fit_report.build_mlx_fit_report(spec.name) - rendered = fit_report.render_fit_report(report) - if report.overall == "blocked": - raise MLXInstallUnavailableError(rendered) - if report.overall == "warning": - LOG.warning("MLX provider host fit warning:\n%s", rendered) - - _write_status(transition_state(_read_status(), new_state="downloading")) - snapshot_dir = Path( - huggingface_hub.snapshot_download( - repo_id=spec.repo, - revision=spec.revision, + if not readiness.ready: + report = fit_report.build_mlx_fit_report(spec.name) + rendered = fit_report.render_fit_report(report) + if report.overall == "blocked": + raise MLXInstallUnavailableError(rendered) + if report.overall == "warning": + LOG.warning("MLX provider host fit warning:\n%s", rendered) + + _write_status(transition_state(_read_status(), new_state="downloading")) + snapshot_dir = Path( + huggingface_hub.snapshot_download( + repo_id=spec.repo, + revision=spec.revision, + ) ) - ) - - _write_status(transition_state(_read_status(), new_state="verifying")) - validate_snapshot_sha256(spec, snapshot_dir) - _write_status(transition_state(_read_status(), new_state="installing")) - variant_dir = None - if spec.name == GEMMA4_26B_A4B_4BIT: - variant_dir = create_gemma4_variant(snapshot_dir) + _write_status(transition_state(_read_status(), new_state="verifying")) + metadata = validate_snapshot_sha256(spec, snapshot_dir) + assert_install_attempt_current(attempt_status) + _write_snapshot_manifest( + spec, + snapshot_dir, + metadata=metadata, + attempt_status=attempt_status, + fingerprint=fingerprint, + ) - _write_mlx_metadata(spec, snapshot_dir=snapshot_dir, variant_dir=variant_dir) - return _write_status(transition_state(_read_status(), new_state="installed")) - except Exception as exc: - _write_status( - transition_state(_read_status(), new_state="failed", error=str(exc)) + _write_status(transition_state(_read_status(), new_state="installing")) + if spec.name == GEMMA4_26B_A4B_4BIT: + variant_dir = create_gemma4_variant(snapshot_dir) + assert_install_attempt_current(attempt_status) + _write_variant_manifest( + spec, + variant_dir, + attempt_status=attempt_status, + fingerprint=fingerprint, + ) + + final_readiness = inspect_readiness(model_id) + current = assert_install_attempt_current(attempt_status) + if final_readiness.ready: + return _write_status(transition_state(current, new_state="installed")) + return _write_status( + transition_state( + current, + new_state="failed", + error=final_readiness.reason_code, + error_code=final_readiness.reason_code, + ) ) + except Exception as exc: + try: + current = assert_install_attempt_current(attempt_status) + _write_status( + transition_state( + current, + new_state="failed", + error=str(exc), + error_code=getattr(exc, "reason_code", None), + ) + ) + except Exception: + pass raise + finally: + if owned_lease: + lease.release() __all__ = [ @@ -486,6 +712,7 @@ __all__ = [ "is_mlx_platform_supported", "resolve_model_spec", "snapshot_dir_for_spec", + "target_fingerprint", "validate_snapshot_sha256", "variant_dir_for_snapshot", ] diff --git a/solstone/think/providers/parakeet_install.py b/solstone/think/providers/parakeet_install.py index 93e448333..01e84b7d3 100644 --- a/solstone/think/providers/parakeet_install.py +++ b/solstone/think/providers/parakeet_install.py @@ -3,36 +3,37 @@ """Install and inspect bundled parakeet.cpp provider artifacts. -This module is the sole writer for ``providers.bundled.parakeet`` install state. -It performs no network access at import time. +This module owns parakeet.cpp provider artifact acquisition. It performs no +network access at import time. """ from __future__ import annotations -import errno -import fcntl import hashlib import logging -import os -import random import shutil import stat -import time -from collections.abc import Iterator -from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import Any, Callable from solstone.think import parakeet_readiness -from solstone.think.journal_config import ( - JournalConfigMutation, - mutate_journal_config, +from solstone.think.providers.artifact_proof import ( + ReadinessOutcome, + artifact_manifest_path, + build_manifest, + prove_manifest, + write_manifest, ) +from solstone.think.providers.install_lease import InstallLease, acquire_install_lease from solstone.think.providers.install_state import ( IN_FLIGHT_STATES, InstallStatus, + assert_install_attempt_current, + begin_or_replace_install_attempt, bump_progress, + canonical_fingerprint, + fingerprint_sha256, read_install_status, transition_state, write_install_status, @@ -42,23 +43,6 @@ from solstone.think.utils import get_journal LOG = logging.getLogger(__name__) PARAKEET_PROVIDER_NAME = "parakeet" _PROBE_TIMEOUT_SECONDS = 10 -_INSTALL_LOCK_TIMEOUT_SECONDS = 60.0 * 60.0 -_INSTALL_LOCK_POLL_INTERVAL_SECONDS = 1.0 -_PARAKEET_METADATA_KEYS = frozenset( - { - "binary_artifact_cpu", - "binary_sha256_cpu", - "binary_path_cpu", - "binary_artifact_vulkan", - "binary_sha256_vulkan", - "binary_path_vulkan", - "model_repo", - "model_filename", - "model_revision", - "model_path", - "model_sha256", - } -) PARAKEET_SERVER_PINS: dict[tuple[str, str], dict[str, str]] = { ("x86_64-unknown-linux-gnu", "vulkan"): { @@ -169,41 +153,17 @@ def install_hint() -> str: def _read_parakeet_status( journal_path: str | Path | None = None, ) -> InstallStatus: - return read_install_status( - scope="bundled", name=PARAKEET_PROVIDER_NAME, journal_path=journal_path - ) + return read_install_status(name=PARAKEET_PROVIDER_NAME, journal_path=journal_path) def _write_parakeet_status( status: InstallStatus, journal_path: str | Path | None = None, ) -> InstallStatus: - write_install_status(status, scope="bundled", journal_path=journal_path) + write_install_status(status, journal_path=journal_path) return status -def _write_parakeet_metadata( - updates: dict[str, str], - journal_path: str | Path | None = None, -) -> None: - unknown_keys = sorted(set(updates) - _PARAKEET_METADATA_KEYS) - if unknown_keys: - raise ValueError(f"unknown parakeet install metadata key: {unknown_keys[0]}") - - def apply(config: dict[str, Any]) -> JournalConfigMutation[None]: - slot = ( - config.setdefault("providers", {}) - .setdefault("bundled", {}) - .setdefault(PARAKEET_PROVIDER_NAME, {}) - ) - changed = any(slot.get(key) != value for key, value in updates.items()) - for key, value in updates.items(): - slot[key] = value - return JournalConfigMutation(changed=changed, value=None) - - mutate_journal_config(apply, journal_path=journal_path) - - def _record_parakeet_progress( received: int, total: int | None, @@ -215,56 +175,136 @@ def _record_parakeet_progress( return _write_parakeet_status( bump_progress(status, received=received, total=total), - journal_path, + journal_path=journal_path, ) -def _install_lock_path(journal_path: str | Path | None = None) -> Path: - return cache_root(journal_path) / "install" +def _binary_pin_identity(artifact_key: str, backend: str) -> dict[str, Any]: + pin = _pin_for_backend(artifact_key, backend) + return { + "unit": "parakeet-server", + "artifact_key": artifact_key, + "backend": backend, + "release_tag": parakeet_readiness.PARAKEET_CPP_RELEASE_TAG, + "filename": pin["filename"], + "sha256": pin["sha256"], + "binary_name": parakeet_readiness.PARAKEET_CPP_BINARY_NAME, + } -@contextmanager -def _hold_install_lock(journal_path: str | Path | None = None) -> Iterator[None]: - path = _install_lock_path(journal_path) - lock_path = path.parent / f"{path.name}.lock" - path.parent.mkdir(parents=True, exist_ok=True) - deadline = time.monotonic() + _INSTALL_LOCK_TIMEOUT_SECONDS - lock_file = open(lock_path, "w", encoding="utf-8") - try: - while True: - try: - fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) - break - except OSError as exc: - if exc.errno not in (errno.EACCES, errno.EAGAIN, errno.EWOULDBLOCK): - raise - if time.monotonic() >= deadline: - raise ParakeetProviderError( - "install_lock_timeout", - f"Timed out waiting for parakeet install lock at {lock_path}", - ) from exc - time.sleep(random.uniform(0.1, _INSTALL_LOCK_POLL_INTERVAL_SECONDS)) - try: - yield - finally: - fcntl.flock(lock_file, fcntl.LOCK_UN) - finally: - lock_file.close() +def _model_pin_identity() -> dict[str, Any]: + spec = PARAKEET_MODEL_SPEC + return { + "unit": "parakeet-model", + "repo": spec.repo, + "filename": spec.filename, + "revision": spec.revision, + "sha256": spec.sha256, + } -def _ready_status_if_installed( +def target_fingerprint( + *, journal_path: str | Path | None = None, -) -> InstallStatus | None: - try: - parakeet_readiness.check_parakeet_cpp_files( - cache_root(journal_path), parakeet_server_artifact_key() +) -> dict[str, Any]: + artifact_key = parakeet_server_artifact_key() + return { + "provider": PARAKEET_PROVIDER_NAME, + "runtime": "parakeet.cpp", + "artifact_key": artifact_key, + "binary_pins": [ + _binary_pin_identity(artifact_key, backend) + for backend in parakeet_readiness.PARAKEET_CPP_BINARY_BACKENDS + ], + "model_pin": _model_pin_identity(), + "cache_root": str(cache_root(journal_path)), + } + + +def _fingerprint_sha_for_target(fingerprint: dict[str, Any]) -> str: + return fingerprint_sha256(canonical_fingerprint(fingerprint)) + + +def _manifest_target_sha( + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any], +) -> str: + if attempt_status is not None and attempt_status["target_fingerprint_sha256"]: + return str(attempt_status["target_fingerprint_sha256"]) + return _fingerprint_sha_for_target(fingerprint) + + +def _manifest_entry(path: Path, root: Path, role: str) -> dict[str, Any]: + return { + "relative_path": path.relative_to(root).as_posix(), + "role": role, + "size": path.stat().st_size, + "sha256": _sha256_file(path), + } + + +def _runtime_inventory( + root: Path, + *, + backend: str, + exclude_names: set[str], +) -> list[dict[str, Any]]: + inventory: list[dict[str, Any]] = [] + for path in sorted(root.rglob("*")): + if not path.is_file() or path.name == artifact_manifest_path(root).name: + continue + if path.name in exclude_names: + continue + role = ( + f"runtime_binary_{backend}" + if path.name == parakeet_readiness.PARAKEET_CPP_BINARY_NAME + else f"runtime_support_{backend}" ) - except (ParakeetProviderError, RuntimeError): - return None - return _write_parakeet_status( - transition_state(_read_parakeet_status(journal_path), new_state="installed"), - journal_path, + inventory.append(_manifest_entry(path, root, role)) + return inventory + + +def _write_binary_manifest( + *, + backend: str, + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any] | None, + journal_path: str | Path | None, +) -> None: + artifact_key = parakeet_server_artifact_key() + pin = _pin_for_backend(artifact_key, backend) + root = binary_install_dir(backend, journal_path) + fingerprint = fingerprint or target_fingerprint(journal_path=journal_path) + manifest = build_manifest( + provider=PARAKEET_PROVIDER_NAME, + unit=f"parakeet-server-{backend}", + target_fingerprint_sha256=_manifest_target_sha(attempt_status, fingerprint), + source={"pin_identity": _binary_pin_identity(artifact_key, backend)}, + inventory=_runtime_inventory( + root, backend=backend, exclude_names={pin["filename"]} + ), + attempt_id=attempt_status["attempt_id"] if attempt_status else None, ) + write_manifest(artifact_manifest_path(root), manifest) + + +def _write_model_manifest( + *, + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any] | None, + journal_path: str | Path | None, +) -> None: + root = model_dir(journal_path) + fingerprint = fingerprint or target_fingerprint(journal_path=journal_path) + manifest = build_manifest( + provider=PARAKEET_PROVIDER_NAME, + unit="parakeet-model", + target_fingerprint_sha256=_manifest_target_sha(attempt_status, fingerprint), + source={"pin_identity": _model_pin_identity()}, + inventory=[_manifest_entry(model_path(journal_path), root, "model")], + attempt_id=attempt_status["attempt_id"] if attempt_status else None, + ) + write_manifest(artifact_manifest_path(root), manifest) def _sha256_file(path: Path) -> str: @@ -393,6 +433,9 @@ def probe_binary_runnable(binary_path: str | Path) -> tuple[bool, str | None]: def _install_parakeet_server_unlocked( backend: str, journal_path: str | Path | None = None, + *, + attempt_status: InstallStatus | None = None, + fingerprint: dict[str, Any] | None = None, ) -> dict[str, Any]: backend = _validate_backend(backend) artifact_key = parakeet_server_artifact_key() @@ -411,10 +454,6 @@ def _install_parakeet_server_unlocked( ), journal_path, ) - _write_parakeet_metadata( - {f"binary_artifact_{backend}": pin["filename"]}, - journal_path, - ) _download_file( url, tarball, @@ -441,26 +480,21 @@ def _install_parakeet_server_unlocked( install_dir, parakeet_readiness.PARAKEET_CPP_BINARY_NAME ) final_path = binary_path(backend, journal_path) + if attempt_status is not None: + assert_install_attempt_current(attempt_status, journal_path=journal_path) inner_dir = extracted.parent if inner_dir != install_dir: for item in inner_dir.iterdir(): shutil.move(str(item), str(install_dir / item.name)) inner_dir.rmdir() _chmod_executable(final_path) - _write_parakeet_metadata( - { - f"binary_artifact_{backend}": pin["filename"], - f"binary_sha256_{backend}": pin["sha256"], - f"binary_path_{backend}": str(final_path), - }, - journal_path, - ) - return _write_parakeet_status( - transition_state( - _read_parakeet_status(journal_path), new_state="installed" - ), - journal_path, + _write_binary_manifest( + backend=backend, + attempt_status=attempt_status, + fingerprint=fingerprint, + journal_path=journal_path, ) + return _read_parakeet_status(journal_path) except Exception as exc: _write_parakeet_status( transition_state( @@ -478,12 +512,14 @@ def install_parakeet_server( *, journal_path: str | Path | None = None, ) -> dict[str, Any]: - with _hold_install_lock(journal_path): - return _install_parakeet_server_unlocked(backend, journal_path) + return _install_parakeet_server_unlocked(backend, journal_path) def _install_model_unlocked( journal_path: str | Path | None = None, + *, + attempt_status: InstallStatus | None = None, + fingerprint: dict[str, Any] | None = None, ) -> dict[str, Any]: spec = PARAKEET_MODEL_SPEC url = f"https://huggingface.co/{spec.repo}/resolve/{spec.revision}/{spec.filename}" @@ -496,14 +532,6 @@ def _install_model_unlocked( ), journal_path, ) - _write_parakeet_metadata( - { - "model_repo": spec.repo, - "model_filename": spec.filename, - "model_revision": spec.revision, - }, - journal_path, - ) _download_file( url, dest, @@ -518,22 +546,14 @@ def _install_model_unlocked( journal_path, ) _verify_sha256(dest, spec.sha256) - _write_parakeet_metadata( - { - "model_repo": spec.repo, - "model_filename": spec.filename, - "model_revision": spec.revision, - "model_path": str(dest), - "model_sha256": spec.sha256, - }, - journal_path, - ) - return _write_parakeet_status( - transition_state( - _read_parakeet_status(journal_path), new_state="installed" - ), - journal_path, + if attempt_status is not None: + assert_install_attempt_current(attempt_status, journal_path=journal_path) + _write_model_manifest( + attempt_status=attempt_status, + fingerprint=fingerprint, + journal_path=journal_path, ) + return _read_parakeet_status(journal_path) except Exception as exc: _write_parakeet_status( transition_state( @@ -550,52 +570,207 @@ def install_model( *, journal_path: str | Path | None = None, ) -> dict[str, Any]: - with _hold_install_lock(journal_path): - return _install_model_unlocked(journal_path) + return _install_model_unlocked(journal_path) def install_parakeet( *, force: bool = False, journal_path: str | Path | None = None, + lease: InstallLease | None = None, + attempt_status: InstallStatus | None = None, ) -> dict[str, Any]: - with _hold_install_lock(journal_path): - if not force: - ready_status = _ready_status_if_installed(journal_path) - if ready_status is not None: - return ready_status + fingerprint = target_fingerprint(journal_path=journal_path) + owned_lease = lease is None + if lease is None: + lease = acquire_install_lease(PARAKEET_PROVIDER_NAME, journal_path=journal_path) + if lease is None: + raise ParakeetProviderError( + "install_busy", "Parakeet provider install is already running." + ) + try: + if attempt_status is None: + attempt_status = begin_or_replace_install_attempt( + PARAKEET_PROVIDER_NAME, + fingerprint, + initial_state="resolving", + owner={"entry": "install_parakeet"}, + journal_path=journal_path, + ) + readiness = inspect_readiness(journal_path) + if readiness.status in {"proof-unavailable", "host-ineligible"}: + current = assert_install_attempt_current( + attempt_status, journal_path=journal_path + ) + return _write_parakeet_status( + transition_state( + current, + new_state="failed", + error=readiness.reason_code, + error_code=readiness.reason_code, + ), + journal_path, + ) from solstone.think.providers import fit_report - report = fit_report.build_parakeet_fit_report(journal_path) - rendered = fit_report.render_fit_report(report) - if report.overall == "blocked": - raise ParakeetProviderError("host_unfit", rendered) - if report.overall == "warning": - LOG.warning("parakeet.cpp host fit warning:\n%s", rendered) - for backend in parakeet_readiness.PARAKEET_CPP_BINARY_BACKENDS: - _install_parakeet_server_unlocked(backend, journal_path) - return _install_model_unlocked(journal_path) + if force or not readiness.ready: + report = fit_report.build_parakeet_fit_report(journal_path) + rendered = fit_report.render_fit_report(report) + if report.overall == "blocked": + raise ParakeetProviderError("host_unfit", rendered) + if report.overall == "warning": + LOG.warning("parakeet.cpp host fit warning:\n%s", rendered) + for backend in parakeet_readiness.PARAKEET_CPP_BINARY_BACKENDS: + if ( + force + or readiness.proof[f"binary_{backend}"]["status"] + == "missing-or-mismatched" + ): + _install_parakeet_server_unlocked( + backend, + journal_path, + attempt_status=attempt_status, + fingerprint=fingerprint, + ) + if force or readiness.proof["model"]["status"] == "missing-or-mismatched": + _install_model_unlocked( + journal_path, + attempt_status=attempt_status, + fingerprint=fingerprint, + ) + + final_readiness = inspect_readiness(journal_path) + current = assert_install_attempt_current( + attempt_status, journal_path=journal_path + ) + if final_readiness.ready: + return _write_parakeet_status( + transition_state(current, new_state="installed"), + journal_path, + ) + return _write_parakeet_status( + transition_state( + current, + new_state="failed", + error=final_readiness.reason_code, + error_code=final_readiness.reason_code, + ), + journal_path, + ) + except Exception as exc: + try: + current = assert_install_attempt_current( + attempt_status, journal_path=journal_path + ) + _write_parakeet_status( + transition_state( + current, + new_state="failed", + error=str(exc), + error_code=getattr(exc, "reason_code", None), + ), + journal_path, + ) + except Exception: + pass + raise + finally: + if owned_lease: + lease.release() + + +def _proof_result_payload(result: Any) -> dict[str, Any]: + return { + "status": result.status, + "reason_code": result.reason_code, + "cache_hit": bool(getattr(result, "cache_hit", False)), + } + + +def _combined_artifact_status( + *proofs: dict[str, Any], +) -> tuple[str, str]: + for proof in proofs: + if proof["status"] == "proof-unavailable": + return "proof-unavailable", str(proof["reason_code"]) + for proof in proofs: + if proof["status"] == "missing-or-mismatched": + return "missing-or-mismatched", str(proof["reason_code"]) + return "ready", "ready" -def inspect_readiness(journal_path: str | Path | None = None) -> dict[str, Any]: +def inspect_readiness(journal_path: str | Path | None = None) -> ReadinessOutcome: status = _read_parakeet_status(journal_path) + artifact_key = parakeet_server_artifact_key() cpu_path = binary_path("cpu", journal_path) vulkan_path = binary_path("vulkan", journal_path) gguf_path = model_path(journal_path) - cpu_installed = cpu_path.exists() and os.access(cpu_path, os.X_OK) - vulkan_installed = vulkan_path.exists() and os.access(vulkan_path, os.X_OK) - model_installed = gguf_path.exists() - return { - "install_state": status["install_state"], - "binary_installed": cpu_installed and vulkan_installed, - "binary_cpu_installed": cpu_installed, - "binary_vulkan_installed": vulkan_installed, - "model_installed": model_installed, - "binary_path_cpu": str(cpu_path), - "binary_path_vulkan": str(vulkan_path), - "model_path": str(gguf_path), - "install_error": status["install_error"], - } + cpu_proof = prove_manifest( + artifact_manifest_path(binary_install_dir("cpu", journal_path)), + provider=PARAKEET_PROVIDER_NAME, + pin_identity=_binary_pin_identity(artifact_key, "cpu"), + journal_path=journal_path, + ) + vulkan_proof = prove_manifest( + artifact_manifest_path(binary_install_dir("vulkan", journal_path)), + provider=PARAKEET_PROVIDER_NAME, + pin_identity=_binary_pin_identity(artifact_key, "vulkan"), + journal_path=journal_path, + ) + model_proof = prove_manifest( + artifact_manifest_path(model_dir(journal_path)), + provider=PARAKEET_PROVIDER_NAME, + pin_identity=_model_pin_identity(), + journal_path=journal_path, + ) + cpu_payload = _proof_result_payload(cpu_proof) + vulkan_payload = _proof_result_payload(vulkan_proof) + model_payload = _proof_result_payload(model_proof) + readiness_status, reason_code = _combined_artifact_status( + cpu_payload, vulkan_payload, model_payload + ) + binary_status, binary_reason_code = _combined_artifact_status( + cpu_payload, vulkan_payload + ) + return ReadinessOutcome( + provider=PARAKEET_PROVIDER_NAME, + status=readiness_status, # type: ignore[arg-type] + reason_code=reason_code, + target={ + "target_fingerprint_json": status["target_fingerprint_json"], + "target_fingerprint_sha256": status["target_fingerprint_sha256"], + }, + install={ + "install_state": status["install_state"], + "install_error": status["install_error"], + "error_code": status["error_code"], + "attempt_id": status["attempt_id"], + "progress_bytes_received": status["progress_bytes_received"], + "progress_bytes_total": status["progress_bytes_total"], + "last_transition_at": status["last_transition_at"], + "last_progress_at": status["last_progress_at"], + }, + host={}, + artifacts={ + "binary_installed": cpu_proof.ready and vulkan_proof.ready, + "binary_cpu_installed": cpu_proof.ready, + "binary_vulkan_installed": vulkan_proof.ready, + "model_installed": model_proof.ready, + "binary_path_cpu": str(cpu_path), + "binary_path_vulkan": str(vulkan_path), + "model_path": str(gguf_path), + }, + proof={ + "binary": { + "status": binary_status, + "reason_code": binary_reason_code, + "cache_hit": cpu_payload["cache_hit"] and vulkan_payload["cache_hit"], + }, + "binary_cpu": cpu_payload, + "binary_vulkan": vulkan_payload, + "model": model_payload, + }, + ) def ensure_artifacts_installed( @@ -605,13 +780,15 @@ def ensure_artifacts_installed( ) -> tuple[Path, Path]: backend = _validate_backend(backend) readiness = inspect_readiness(journal_path) - if not readiness["binary_installed"]: + if not readiness.artifacts["binary_installed"]: raise ParakeetProviderError( "binary_missing", "Parakeet server binaries are not installed." ) - if not readiness["model_installed"]: + if not readiness.artifacts["model_installed"]: raise ParakeetProviderError("model_missing", "Parakeet model is not installed.") - return Path(readiness[f"binary_path_{backend}"]), Path(readiness["model_path"]) + return Path(readiness.artifacts[f"binary_path_{backend}"]), Path( + readiness.artifacts["model_path"] + ) __all__ = [ diff --git a/solstone/think/providers/state.py b/solstone/think/providers/state.py index 0798b9af3..7c51ccef0 100644 --- a/solstone/think/providers/state.py +++ b/solstone/think/providers/state.py @@ -308,8 +308,8 @@ def local_status_dict() -> dict: from solstone.think.providers import local_server, mlx_install readiness = mlx_install.inspect_readiness() - runtime_available = bool(readiness["package_available"]) - model_installed = bool(readiness["model_installed"]) + runtime_available = bool(readiness.host["package_available"]) + model_installed = bool(readiness.artifacts["model_installed"]) configured = runtime_available and model_installed if not selected: @@ -325,7 +325,9 @@ def local_status_dict() -> dict: server_healthy = local_server.is_healthy() if not runtime_available: issues.append("runtime_missing") - if not model_installed: + if readiness.status == "proof-unavailable": + issues.append(readiness.reason_code) + elif not model_installed: issues.append("model_missing") if configured and not server_healthy: issues.append("server_unhealthy") @@ -342,8 +344,8 @@ def local_status_dict() -> dict: from solstone.think.providers import local_install, local_server readiness = local_install.inspect_readiness() - binary_installed = bool(readiness["binary_installed"]) - model_installed = bool(readiness["model_installed"]) + binary_installed = bool(readiness.artifacts["binary_installed"]) + model_installed = bool(readiness.artifacts["model_installed"]) configured = binary_installed and model_installed if not selected: @@ -357,14 +359,18 @@ def local_status_dict() -> dict: issues: list[str] = [] server_healthy = local_server.is_healthy() - if not readiness.get("gpu_available", True): + if not readiness.host.get("gpu_available", True): issues.append("gpu_unavailable") - if not binary_installed: + if readiness.status == "proof-unavailable": + issues.append(readiness.reason_code) + elif readiness.proof["binary"]["status"] == "missing-or-mismatched": issues.append("binary_missing") - if not model_installed: + if readiness.proof["model"]["status"] == "missing-or-mismatched": issues.append("model_missing") if configured and not server_healthy: - runnable, detail = local_install.probe_binary_runnable(readiness["binary_path"]) + runnable, detail = local_install.probe_binary_runnable( + readiness.artifacts["binary_path"] + ) if runnable: issues.append("server_unhealthy") else: @@ -500,23 +506,23 @@ def _local_readiness_for_provider( from solstone.think.providers.install_state import IN_FLIGHT_STATES readiness = mlx_install.inspect_readiness() - model_id = str(readiness["model_id"]) + model_id = str(readiness.target["model_id"]) - if readiness["install_state"] in IN_FLIGHT_STATES: + if readiness.install["install_state"] in IN_FLIGHT_STATES: return _state( provider, interface, "blocked", "local_model_installing", model=model_id, - message=str(readiness["install_state"]), + message=str(readiness.install["install_state"]), source="local_install", ) if not ( - readiness["platform_supported"] - and readiness["package_available"] - and readiness["model_installed"] + readiness.host["platform_supported"] + and readiness.host["package_available"] + and readiness.artifacts["model_installed"] ): return _state( provider, @@ -524,7 +530,7 @@ def _local_readiness_for_provider( "blocked", "local_model_missing", model=model_id, - message=str(readiness.get("install_error") or "") or None, + message=str(readiness.install.get("install_error") or "") or None, source="local_install", ) @@ -562,20 +568,20 @@ def _local_readiness_for_provider( selected_model = model or LOCAL_MODEL readiness = local_install.inspect_readiness(selected_model) - model_id = str(readiness.get("model_id") or selected_model) + model_id = str(readiness.target.get("model_id") or selected_model) - if readiness["install_state"] in IN_FLIGHT_STATES: + if readiness.install["install_state"] in IN_FLIGHT_STATES: return _state( provider, interface, "blocked", "local_model_installing", model=model_id, - message=str(readiness["install_state"]), + message=str(readiness.install["install_state"]), source="local_install", ) - if not readiness.get("gpu_probe_ok", True): + if not readiness.host.get("gpu_probe_ok", True): return _state( provider, interface, @@ -585,7 +591,7 @@ def _local_readiness_for_provider( source="local_install", ) - if not readiness.get("gpu_available", True): + if not readiness.host.get("gpu_available", True): return _state( provider, interface, @@ -595,14 +601,17 @@ def _local_readiness_for_provider( source="local_install", ) - if not readiness["binary_installed"] or not readiness["model_installed"]: + if ( + not readiness.artifacts["binary_installed"] + or not readiness.artifacts["model_installed"] + ): return _state( provider, interface, "blocked", "local_model_missing", model=model_id, - message=str(readiness.get("install_error") or "") or None, + message=str(readiness.install.get("install_error") or "") or None, source="local_install", ) diff --git a/solstone/think/supervisor.py b/solstone/think/supervisor.py index d8d011609..92e17c941 100644 --- a/solstone/think/supervisor.py +++ b/solstone/think/supervisor.py @@ -251,8 +251,6 @@ _sync_conflict_shutdown: bool = False _supervisor_ref: str | None = None _supervisor_start: float | None = None _parent_death_sigterm_sent = threading.Event() -_parakeet_bootstrap_lock = threading.Lock() -_parakeet_bootstrap_thread: threading.Thread | None = None def app_supervised_graceful_budget_s() -> float: @@ -1780,21 +1778,21 @@ def _start_mlx_local_server() -> RunnerManagedProcess | None: from solstone.think.providers import local_server, mlx_install readiness = mlx_install.inspect_readiness() - readiness_keys = ( - "platform_supported", - "package_available", - "ram_sufficient", - "model_installed", - ) - if not all(readiness.get(key) for key in readiness_keys): + readiness_values = { + "platform_supported": readiness.host["platform_supported"], + "package_available": readiness.host["package_available"], + "ram_sufficient": readiness.host["ram_sufficient"], + "model_installed": readiness.artifacts["model_installed"], + } + if not all(readiness_values.values()): logging.info( "MLX local model not ready; skipping mlx-vlm server startup: %s", - {key: readiness.get(key) for key in readiness_keys}, + readiness_values, ) return None - runtime_dir = readiness["runtime_dir"] - model_id = readiness["model_id"] + runtime_dir = readiness.artifacts["runtime_dir"] + model_id = readiness.target["model_id"] port = find_available_port() write_service_port("local", port) script_path = str(Path(sys.executable).with_name(MLX_SERVER_PROCESS_NAME)) @@ -1938,33 +1936,38 @@ def _request_parakeet_server_start() -> None: logging.exception("could not request parakeet-server start") -def _run_parakeet_bootstrap_worker(journal_path: Path | None = None) -> None: +def _run_parakeet_bootstrap_worker( + journal_path: Path | None = None, + lease: Any | None = None, + attempt_status: Any | None = None, + ack: threading.Event | None = None, +) -> None: """Install parakeet.cpp artifacts in the background, then retry startup.""" - current_thread = threading.current_thread() + if ack is not None: + ack.set() try: from solstone.think.providers import parakeet_install - if journal_path is None: - parakeet_install.install_parakeet() - else: - parakeet_install.install_parakeet(journal_path=journal_path) + parakeet_install.install_parakeet( + journal_path=journal_path, + lease=lease, + attempt_status=attempt_status, + ) except Exception: logging.exception("parakeet.cpp provider bootstrap failed") else: logging.info("parakeet.cpp provider bootstrap complete; requesting startup") _request_parakeet_server_start() finally: - global _parakeet_bootstrap_thread - with _parakeet_bootstrap_lock: - if _parakeet_bootstrap_thread is current_thread: - _parakeet_bootstrap_thread = None + if lease is not None: + lease.release() def _start_parakeet_bootstrap_if_needed(reason: str) -> None: """Start one non-blocking parakeet.cpp install worker when artifacts are absent.""" - global _parakeet_bootstrap_thread from solstone.think.providers import parakeet_install - from solstone.think.providers.install_state import IN_FLIGHT_STATES + from solstone.think.providers.install_lease import acquire_install_lease + from solstone.think.providers.install_state import begin_or_replace_install_attempt try: readiness = parakeet_install.inspect_readiness() @@ -1972,31 +1975,40 @@ def _start_parakeet_bootstrap_if_needed(reason: str) -> None: logging.info( "could not inspect parakeet.cpp readiness before bootstrap: %s", exc ) - readiness = {} + readiness = None - if readiness.get("binary_installed") and readiness.get("model_installed"): - return - if readiness.get("install_state") in IN_FLIGHT_STATES: - logging.info( - "parakeet.cpp provider install already %s; not starting another worker", - readiness.get("install_state"), - ) + if readiness is not None and readiness.ready: return - with _parakeet_bootstrap_lock: - if ( - _parakeet_bootstrap_thread is not None - and _parakeet_bootstrap_thread.is_alive() - ): - logging.info("parakeet.cpp provider bootstrap already running") - return - journal_path = Path(get_journal()) + journal_path = Path(get_journal()) + lease = acquire_install_lease("parakeet", journal_path=journal_path) + if lease is None: + logging.info("parakeet.cpp provider bootstrap already running") + return + try: + fingerprint = parakeet_install.target_fingerprint(journal_path=journal_path) + attempt_status = begin_or_replace_install_attempt( + "parakeet", + fingerprint, + initial_state="downloading", + owner={"entry": "supervisor_parakeet_bootstrap"}, + journal_path=journal_path, + ) + ack = threading.Event() thread = threading.Thread( - target=lambda: _run_parakeet_bootstrap_worker(journal_path), + target=lambda: _run_parakeet_bootstrap_worker( + journal_path, + lease, + attempt_status, + ack, + ), name="parakeet-cpp-provider-bootstrap", daemon=True, ) - _parakeet_bootstrap_thread = thread + except Exception: + lease.release() + logging.exception("could not prepare parakeet.cpp provider bootstrap worker") + return logging.info( "Parakeet artifacts not ready; starting background provider install: %s", @@ -2005,10 +2017,12 @@ def _start_parakeet_bootstrap_if_needed(reason: str) -> None: try: thread.start() except Exception: - with _parakeet_bootstrap_lock: - if _parakeet_bootstrap_thread is thread: - _parakeet_bootstrap_thread = None + lease.release() logging.exception("could not start parakeet.cpp provider bootstrap worker") + return + if not ack.wait(timeout=5.0): + lease.release() + logging.error("parakeet.cpp provider bootstrap worker did not acknowledge") def _build_parakeet_cmd( diff --git a/tests/test_chat_reasons.py b/tests/test_chat_reasons.py index b0d169bc7..a11bc2d4b 100644 --- a/tests/test_chat_reasons.py +++ b/tests/test_chat_reasons.py @@ -19,6 +19,7 @@ EXPECTED_CODES = { "local_model_missing", "model_missing", "binary_missing", + "install_busy", "local_model_installing", "local_model_loading", "local_model_not_ready", diff --git a/tests/test_install_models.py b/tests/test_install_models.py index 4abd49578..8a055a152 100644 --- a/tests/test_install_models.py +++ b/tests/test_install_models.py @@ -559,9 +559,21 @@ def test_install_models_linux_routes_through_parakeet_provider( calls = [] paths = _ready_paths(tmp_path) + journal = tmp_path / "journal" + (journal / "config").mkdir(parents=True) + (journal / "config" / "journal.json").write_text( + '{"providers": {}}\n', + encoding="utf-8", + ) + monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) + import solstone.think.utils as think_utils + + think_utils._journal_path_cache = None monkeypatch.setattr( - parakeet_install, "install_parakeet", lambda: calls.append("install") + parakeet_install, + "install_parakeet", + lambda **_kwargs: calls.append("install"), ) monkeypatch.setattr(install_models, "_check_linux_cpp_ready", lambda: paths) diff --git a/tests/test_install_provider.py b/tests/test_install_provider.py index f78e4e4f1..f5a09bfae 100644 --- a/tests/test_install_provider.py +++ b/tests/test_install_provider.py @@ -10,6 +10,7 @@ import pytest from solstone.think import install_provider from solstone.think.providers import fit_report +from solstone.think.providers.artifact_proof import ReadinessOutcome def _fit(severity: fit_report.FitSeverity) -> fit_report.FitReport: @@ -19,24 +20,91 @@ def _fit(severity: fit_report.FitSeverity) -> fit_report.FitReport: ) +class _FakeLease: + def __init__(self) -> None: + self.released = False + + def release(self) -> None: + self.released = True + + +def _readiness(provider: str, *, ready: bool) -> ReadinessOutcome: + return ReadinessOutcome( + provider=provider, + status="ready" if ready else "missing-or-mismatched", + reason_code="ready" if ready else "manifest_missing", + target={"model_id": "model"}, + install={ + "install_state": "idle", + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={}, + artifacts={ + "binary_installed": ready, + "model_installed": ready, + }, + proof={ + "binary": { + "status": "ready" if ready else "missing-or-mismatched", + "reason_code": "ready" if ready else "manifest_missing", + "cache_hit": False, + }, + "model": { + "status": "ready" if ready else "missing-or-mismatched", + "reason_code": "ready" if ready else "manifest_missing", + "cache_hit": False, + }, + }, + ) + + +def _patch_lease_and_attempt( + monkeypatch: pytest.MonkeyPatch, provider: str +) -> _FakeLease: + lease = _FakeLease() + monkeypatch.setattr( + install_provider, + "acquire_install_lease", + lambda name: lease if name == provider else None, + ) + monkeypatch.setattr( + install_provider, + "begin_or_replace_install_attempt", + lambda name, fingerprint, **_kwargs: { + "provider": name, + "install_state": "resolving", + "attempt_id": "attempt", + "target_fingerprint_sha256": install_provider._target_sha(fingerprint), + }, + ) + return lease + + def test_install_provider_local_prints_install_status(monkeypatch, capsys): calls = [] parakeet_calls = [] - def install_local(): + def install_local(**_kwargs): calls.append(True) - return {"name": "local", "install_state": "installed"} + return {"provider": "local", "install_state": "installed"} - def install_parakeet(): + def install_parakeet(**_kwargs): parakeet_calls.append(True) - return {"name": "parakeet", "install_state": "installed"} + return {"provider": "parakeet", "install_state": "installed"} monkeypatch.setattr(sys, "argv", ["journal install-provider", "local"]) monkeypatch.setattr( install_provider.local_install, "inspect_readiness", - lambda: {"binary_installed": False, "model_installed": False}, + lambda: _readiness("local", ready=False), ) + lease = _patch_lease_and_attempt(monkeypatch, "local") monkeypatch.setattr(fit_report, "build_local_fit_report", lambda _model: _fit("ok")) monkeypatch.setattr(install_provider.local_install, "install_local", install_local) monkeypatch.setattr( @@ -49,8 +117,9 @@ def test_install_provider_local_prints_install_status(monkeypatch, capsys): assert calls == [True] assert parakeet_calls == [] + assert lease.released is True assert json.loads(capsys.readouterr().out) == { - "name": "local", + "provider": "local", "install_state": "installed", } @@ -58,16 +127,17 @@ def test_install_provider_local_prints_install_status(monkeypatch, capsys): def test_install_provider_parakeet_prints_disclosure_and_status(monkeypatch, capsys): calls = [] - def install_parakeet(): + def install_parakeet(**_kwargs): calls.append(True) - return {"name": "parakeet", "install_state": "installed"} + return {"provider": "parakeet", "install_state": "installed"} monkeypatch.setattr(sys, "argv", ["journal install-provider", "parakeet"]) monkeypatch.setattr( install_provider.parakeet_install, "inspect_readiness", - lambda: {"binary_installed": False, "model_installed": False}, + lambda: _readiness("parakeet", ready=False), ) + lease = _patch_lease_and_attempt(monkeypatch, "parakeet") monkeypatch.setattr(fit_report, "build_parakeet_fit_report", lambda: _fit("ok")) monkeypatch.setattr( install_provider.parakeet_install, @@ -79,8 +149,9 @@ def test_install_provider_parakeet_prints_disclosure_and_status(monkeypatch, cap captured = capsys.readouterr() assert calls == [True] + assert lease.released is True assert json.loads(captured.out) == { - "name": "parakeet", + "provider": "parakeet", "install_state": "installed", } assert install_provider.PARAKEET_DOWNLOAD_DISCLOSURE in captured.err @@ -93,15 +164,20 @@ def test_install_provider_parakeet_prints_disclosure_and_status(monkeypatch, cap def test_install_provider_local_skips_fit_report_when_ready(monkeypatch, capsys): calls = [] - def install_local(): + def install_local(**_kwargs): calls.append(True) - return {"name": "local", "install_state": "installed"} + return {"provider": "local", "install_state": "installed"} monkeypatch.setattr(sys, "argv", ["journal install-provider", "local"]) monkeypatch.setattr( install_provider.local_install, "inspect_readiness", - lambda: {"binary_installed": True, "model_installed": True}, + lambda: _readiness("local", ready=True), + ) + monkeypatch.setattr( + install_provider, + "read_install_status", + lambda name: {"provider": name, "install_state": "installed"}, ) monkeypatch.setattr( fit_report, @@ -113,16 +189,16 @@ def test_install_provider_local_skips_fit_report_when_ready(monkeypatch, capsys) assert install_provider.main() == 0 captured = capsys.readouterr() - assert calls == [True] + assert calls == [] assert "local already installed" in captured.err assert json.loads(captured.out) == { - "name": "local", + "provider": "local", "install_state": "installed", } def test_install_provider_local_expected_error_returns_nonzero(monkeypatch, capsys): - def install_local(): + def install_local(**_kwargs): raise install_provider.local_install.LocalProviderError( "host_unfit", "blocked detail" ) @@ -131,8 +207,9 @@ def test_install_provider_local_expected_error_returns_nonzero(monkeypatch, caps monkeypatch.setattr( install_provider.local_install, "inspect_readiness", - lambda: {"binary_installed": False, "model_installed": False}, + lambda: _readiness("local", ready=False), ) + lease = _patch_lease_and_attempt(monkeypatch, "local") monkeypatch.setattr( fit_report, "build_local_fit_report", lambda _model: _fit("blocked") ) @@ -141,6 +218,7 @@ def test_install_provider_local_expected_error_returns_nonzero(monkeypatch, caps assert install_provider.main() == 1 captured = capsys.readouterr() + assert lease.released is True assert "blocked detail" in captured.err assert captured.out == "" @@ -149,13 +227,13 @@ def test_install_provider_unsupported_rejects_without_install(monkeypatch, capsy local_calls = [] parakeet_calls = [] - def install_local(): + def install_local(**_kwargs): local_calls.append(True) - return {"name": "local", "install_state": "installed"} + return {"provider": "local", "install_state": "installed"} - def install_parakeet(): + def install_parakeet(**_kwargs): parakeet_calls.append(True) - return {"name": "parakeet", "install_state": "installed"} + return {"provider": "parakeet", "install_state": "installed"} monkeypatch.setattr(sys, "argv", ["journal install-provider", "foo"]) monkeypatch.setattr(install_provider.local_install, "install_local", install_local) diff --git a/tests/test_install_state.py b/tests/test_install_state.py index 9685bba4f..af30e0d86 100644 --- a/tests/test_install_state.py +++ b/tests/test_install_state.py @@ -236,6 +236,7 @@ def test_migration_api_removes_legacy_status_fields(tmp_path, monkeypatch) -> No "install_state": "failed", "install_error": "old", "model_id": "kept", + "vulkan_device_index": "1", } } } @@ -248,8 +249,9 @@ def test_migration_api_removes_legacy_status_fields(tmp_path, monkeypatch) -> No result = install_state.migrate_legacy_provider_install_state() data = json.loads(config_path.read_text(encoding="utf-8")) - assert result == {"removed": 2} - assert data["providers"]["bundled"]["local"] == {"model_id": "kept"} + assert result == {"removed": 4, "moved": 1} + assert data["providers"]["bundled"]["local"] == {} + assert data["providers"]["local"] == {"vulkan_device_index": "1"} def test_two_process_stale_transition_one_writer_wins(tmp_path, monkeypatch) -> None: diff --git a/tests/test_journal_config_changed_contract.py b/tests/test_journal_config_changed_contract.py index 8a0fb7c6d..b8d323eff 100644 --- a/tests/test_journal_config_changed_contract.py +++ b/tests/test_journal_config_changed_contract.py @@ -581,46 +581,33 @@ def test_backup_state_mutation_paths_report_changed_and_noop( assert changes[-1] is False -def test_provider_install_mutation_paths_report_changed_and_noop( +def test_provider_install_migration_reports_changed_and_noop( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) - cases: list[tuple[Any, Callable[[], None]]] = [] - parakeet_install = importlib.import_module( - "solstone.think.providers.parakeet_install" - ) - local_install = importlib.import_module("solstone.think.providers.local_install") - mlx_install = importlib.import_module("solstone.think.providers.mlx_install") - spec = mlx_install.MLXModelSpec("mlx-model", "repo", "rev", 1) - cases.extend( - [ - ( - parakeet_install, - lambda: parakeet_install._write_parakeet_metadata( - {"model_repo": "repo"} - ), - ), - ( - local_install, - lambda: local_install._write_local_metadata({"model_id": "model"}), - ), - ( - mlx_install, - lambda: mlx_install._write_mlx_metadata( - spec, - snapshot_dir=tmp_path / "snapshot", - variant_dir=tmp_path / "variant", - ), - ), - ] + install_state = importlib.import_module("solstone.think.providers.install_state") + changes = _spy_changed(monkeypatch, install_state) + seed_journal_config( + { + "providers": { + "bundled": { + "local": { + "install_state": "installed", + "model_id": "local/model", + "vulkan_device_index": "1", + }, + "parakeet": {"model_repo": "repo"}, + }, + "local": {}, + } + }, + tmp_path, ) - for module, invoke in cases: - seed_journal_config({"providers": {"bundled": {}}}, tmp_path) - changes = _spy_changed(monkeypatch, module) - invoke() - assert changes[-1] is True - invoke() - assert changes[-1] is False + + install_state.migrate_legacy_provider_install_state(journal_path=tmp_path) + assert changes[-1] is True + install_state.migrate_legacy_provider_install_state(journal_path=tmp_path) + assert changes[-1] is False def test_pairing_oura_and_sol_voice_paths_report_changed_and_noop( diff --git a/tests/test_journal_config_transactions.py b/tests/test_journal_config_transactions.py index bac8d8578..4d434db6e 100644 --- a/tests/test_journal_config_transactions.py +++ b/tests/test_journal_config_transactions.py @@ -82,7 +82,7 @@ def _provider_progress_worker(journal_path: str, barrier: Any, errors: Any) -> N ) status = transition_state(make_idle_status("parakeet"), new_state="downloading") - write_install_status(status, scope="bundled") + write_install_status(status) except BaseException: errors.put(traceback.format_exc()) raise diff --git a/tests/test_local.py b/tests/test_local.py index 915b026b8..fc3d53f56 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -22,6 +22,7 @@ from solstone.think.models import ( LOCAL_MODEL, get_model_provider, ) +from solstone.think.providers.artifact_proof import ReadinessOutcome from solstone.think.talents import TalentHookError @@ -2474,6 +2475,61 @@ def _select_local_provider(monkeypatch) -> None: ) +def _provider_local_readiness( + *, + binary_installed: bool = True, + model_installed: bool = True, + ram_sufficient: bool = True, + gpu_available: bool = True, + gpu_probe_ok: bool = True, + binary_path: str = "/fake/llama-server", +) -> ReadinessOutcome: + ready = binary_installed and model_installed + return ReadinessOutcome( + provider="local", + status="ready" if ready else "missing-or-mismatched", + reason_code="ready" if ready else "manifest_missing", + target={"model_id": LOCAL_MODEL}, + install={ + "install_state": "idle", + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": ram_sufficient, + "gpu_available": gpu_available, + "gpu_probe_ok": gpu_probe_ok, + "backend": "vulkan", + "backend_reason": "test vulkan", + }, + artifacts={ + "binary_installed": binary_installed, + "model_installed": model_installed, + "binary_path": binary_path, + "model_path": "/tmp/model.gguf", + "mmproj_path": None, + "model_id": LOCAL_MODEL, + }, + proof={ + "binary": { + "status": "ready" if binary_installed else "missing-or-mismatched", + "reason_code": "ready" if binary_installed else "manifest_missing", + "cache_hit": False, + }, + "model": { + "status": "ready" if model_installed else "missing-or-mismatched", + "reason_code": "ready" if model_installed else "manifest_missing", + "cache_hit": False, + }, + }, + ) + + def test_build_provider_status_local_not_selected_is_inert(monkeypatch): from solstone.think.providers import build_provider_status @@ -2488,13 +2544,7 @@ def test_build_provider_status_local_not_selected_is_inert(monkeypatch): ) monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": True, - "binary_path": "/fake/llama-server", - }, + lambda: _provider_local_readiness(), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", @@ -2519,12 +2569,7 @@ def test_build_provider_status_local_readiness(monkeypatch): _select_local_provider(monkeypatch) monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": True, - }, + lambda: _provider_local_readiness(), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: True @@ -2549,13 +2594,7 @@ def test_build_provider_status_local_launch_failure_adds_probe_detail_and_hint( detail = "dyld: Library not loaded: @rpath/libllama.dylib" monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": True, - "binary_path": "/fake/llama-server", - }, + lambda: _provider_local_readiness(), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: False @@ -2584,13 +2623,7 @@ def test_build_provider_status_local_server_unhealthy_when_probe_runnable( _select_local_provider(monkeypatch) monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": True, - "binary_path": "/fake/llama-server", - }, + lambda: _provider_local_readiness(), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: False @@ -2619,13 +2652,7 @@ def test_build_provider_status_local_healthy_skips_probe(monkeypatch): monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": True, - "binary_path": "/fake/llama-server", - }, + lambda: _provider_local_readiness(), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: True @@ -2648,12 +2675,11 @@ def test_local_provider_status_carries_install_hint_substring(monkeypatch): _select_local_provider(monkeypatch) monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": False, - "model_installed": False, - "ram_sufficient": False, - "gpu_available": True, - }, + lambda: _provider_local_readiness( + binary_installed=False, + model_installed=False, + ram_sufficient=False, + ), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: False @@ -2680,13 +2706,7 @@ def test_local_provider_status_reports_gpu_unavailable_issue(monkeypatch): _select_local_provider(monkeypatch) monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": False, - "binary_path": "/fake/llama-server", - }, + lambda: _provider_local_readiness(gpu_available=False), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: True @@ -2705,13 +2725,7 @@ def test_build_provider_status_local_configured_ignores_ram_flag(monkeypatch): _select_local_provider(monkeypatch) monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": False, - "gpu_available": True, - "binary_path": "/fake/llama-server", - }, + lambda: _provider_local_readiness(ram_sufficient=False), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: True diff --git a/tests/test_local_install.py b/tests/test_local_install.py index f6b4929b4..adfb48db9 100644 --- a/tests/test_local_install.py +++ b/tests/test_local_install.py @@ -13,7 +13,6 @@ from types import SimpleNamespace import pytest -from solstone.think.journal_config import read_journal_config from solstone.think.models import LOCAL_MODEL from solstone.think.providers import ( fit_report, @@ -23,8 +22,14 @@ from solstone.think.providers import ( memory, oci_image, ) +from solstone.think.providers.artifact_proof import ( + ReadinessOutcome, + artifact_manifest_path, + prove_manifest, +) from solstone.think.providers.install_state import read_install_status from solstone.think.providers.local import LOCAL_MODEL_SPECS +from solstone.think.providers.local_endpoint import resolve_local_endpoint def _init_journal(tmp_path, monkeypatch) -> None: @@ -38,11 +43,107 @@ def _init_journal(tmp_path, monkeypatch) -> None: def _local_status() -> dict: - return read_install_status(scope="bundled", name="local") + return read_install_status(name="local") + + +def _write_provider_local_config(tmp_path: Path, updates: dict[str, object]) -> None: + path = tmp_path / "config" / "journal.json" + config = json.loads(path.read_text(encoding="utf-8")) + provider_config = config.setdefault("providers", {}).setdefault("local", {}) + provider_config.update(updates) + path.write_text(json.dumps(config) + "\n", encoding="utf-8") + + +def _fake_local_readiness( + *, + binary_installed: bool, + model_installed: bool, + binary_path: Path, + model_path: Path, + mmproj_path: Path | None = None, + ram_sufficient: bool = True, + backend: str = "vulkan", + backend_reason: str = "test vulkan", +) -> ReadinessOutcome: + missing_binary = not binary_installed + missing_model = not model_installed + status = ( + "ready" if not missing_binary and not missing_model else "missing-or-mismatched" + ) + reason_code = ( + "ready" + if status == "ready" + else "binary_missing" + if missing_binary + else "model_missing" + ) + binary_status = "ready" if binary_installed else "missing-or-mismatched" + model_status = "ready" if model_installed else "missing-or-mismatched" + return ReadinessOutcome( + provider=local_install.LOCAL_PROVIDER_NAME, + status=status, + reason_code=reason_code, + target={"model_id": LOCAL_MODEL}, + install={ + "install_state": "idle", + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": ram_sufficient, + "gpu_available": True, + "gpu_probe_ok": True, + "backend": backend, + "backend_reason": backend_reason, + }, + artifacts={ + "binary_installed": binary_installed, + "model_installed": model_installed, + "binary_path": str(binary_path), + "model_path": str(model_path), + "mmproj_path": str(mmproj_path) if mmproj_path is not None else None, + }, + proof={ + "binary": { + "status": binary_status, + "reason_code": "ready" if binary_installed else "manifest_missing", + "cache_hit": False, + }, + "model": { + "status": model_status, + "reason_code": "ready" if model_installed else "manifest_missing", + "cache_hit": False, + }, + }, + ) -def _local_slot() -> dict: - return read_journal_config()["providers"]["bundled"]["local"] +def _write_ready_vulkan_manifest( + *, + artifact_key: str | None = None, + pin: dict[str, str] | None = None, +) -> None: + resolved_pin = pin or local_install.pin_for_current_platform() + resolved_key = artifact_key or local_install.llama_server_artifact_key() + local_install._write_vulkan_manifest( + artifact_key=resolved_key, + pin=resolved_pin, + attempt_status=None, + fingerprint=local_install.target_fingerprint(LOCAL_MODEL), + ) + + +def _write_ready_model_manifest(model_id: str = LOCAL_MODEL) -> None: + local_install._write_model_manifest( + model_id=model_id, + attempt_status=None, + fingerprint=local_install.target_fingerprint(model_id), + ) def _fit(severity: fit_report.FitSeverity) -> fit_report.FitReport: @@ -102,15 +203,13 @@ def _download_with_fake_stream( chunks: list[bytes], chunk_times: list[float], ) -> tuple[Path, list[tuple[int, int | None]]]: - clock = [0.0] total = sum(len(chunk) for chunk in chunks) calls: list[tuple[int, int | None]] = [] - monkeypatch.setattr(local_install.time, "monotonic", lambda: clock[0]) def fake_stream(method, url, **_kwargs): assert method == "GET" assert url == "https://example.test/artifact" - return _FakeStream(chunks, chunk_times, total, clock) + return _FakeStream(chunks, chunk_times, total, [0.0]) def record_progress(received: int, reported_total: int | None) -> None: calls.append((received, reported_total)) @@ -129,7 +228,7 @@ def test_install_hint_literal() -> None: assert local_install.install_hint() == "journal install-provider local" -def test_download_file_rate_limits_many_progress_chunks(tmp_path, monkeypatch): +def test_download_file_reports_each_progress_chunk(tmp_path, monkeypatch): chunks = [b"x"] * 20 chunk_times = [index * 0.01 for index in range(len(chunks))] @@ -138,8 +237,7 @@ def test_download_file_rate_limits_many_progress_chunks(tmp_path, monkeypatch): ) total = sum(len(chunk) for chunk in chunks) - assert calls == [(1, total), (total, total)] - assert len(calls) < len(chunks) + assert calls == [(index, total) for index in range(1, len(chunks) + 1)] def test_download_file_emits_first_progress_promptly(tmp_path, monkeypatch): @@ -162,11 +260,15 @@ def test_download_file_emits_interval_crossing_progress(tmp_path, monkeypatch): ) total = sum(len(chunk) for chunk in chunks) - boundary_received = sum(len(chunk) for chunk in chunks[:4]) - assert calls == [(1, total), (boundary_received, total), (total, total)] + expected = [] + received = 0 + for chunk in chunks: + received += len(chunk) + expected.append((received, total)) + assert calls == expected -def test_download_file_emits_final_progress_once_with_dedupe(tmp_path, monkeypatch): +def test_download_file_emits_each_final_chunk_once(tmp_path, monkeypatch): chunks = [b"aa", b"bbb"] _dest, inside_window_calls = _download_with_fake_stream( @@ -350,13 +452,18 @@ def test_install_llama_server_relocates_binary_and_libraries(tmp_path, monkeypat result = local_install.install_llama_server() - assert result["install_state"] == "installed" + assert result["install_state"] == "verifying" + assert prove_manifest( + artifact_manifest_path(install_dir), + provider="local", + pin_identity=local_install._vulkan_pin_identity(artifact_key, pin), + ).ready assert_flat_layout() assert quarantine_calls == [install_dir] result = local_install.install_llama_server() - assert result["install_state"] == "installed" + assert result["install_state"] == "verifying" assert_flat_layout() assert quarantine_calls == [install_dir, install_dir] @@ -408,10 +515,7 @@ def test_install_llama_server_sha256_mismatch_fails_closed_before_extract( assert status["install_state"] == "failed" assert status["install_error"] is not None assert "sha256 mismatch" in status["install_error"] - slot = _local_slot() - assert slot["binary_artifact"] == pin["filename"] - assert "binary_sha256" not in slot - assert "binary_path" not in slot + assert not artifact_manifest_path(install_dir).exists() assert not binary_path.exists() assert not (install_dir / inner_name).exists() assert sorted(child.name for child in install_dir.iterdir()) == [pin["filename"]] @@ -428,7 +532,7 @@ def test_install_llama_server_writes_canonical_sequence(tmp_path, monkeypatch): final_path = local_install.binary_path_for_pin("test-platform", pin) final_path.parent.mkdir(parents=True) final_path.write_text("binary", encoding="utf-8") - observed: list[tuple[str, str, dict]] = [] + observed: list[tuple[str, str]] = [] monkeypatch.setattr( local_install, "llama_server_artifact_key", lambda: "test-platform" @@ -436,14 +540,14 @@ def test_install_llama_server_writes_canonical_sequence(tmp_path, monkeypatch): monkeypatch.setattr(local_install, "pin_for_current_platform", lambda: pin) def fake_download(_url, _dest, **_kwargs): - observed.append( - ("download", _local_status()["install_state"], dict(_local_slot())) - ) + observed.append(("download", _local_status()["install_state"])) + _dest.parent.mkdir(parents=True, exist_ok=True) + _dest.write_bytes(b"artifact") + _dest.parent.mkdir(parents=True, exist_ok=True) + _dest.write_bytes(b"artifact") def fake_verify(_path, _expected): - observed.append( - ("verify", _local_status()["install_state"], dict(_local_slot())) - ) + observed.append(("verify", _local_status()["install_state"])) monkeypatch.setattr(local_install, "_download_file", fake_download) monkeypatch.setattr(local_install, "_verify_sha256", fake_verify) @@ -460,14 +564,13 @@ def test_install_llama_server_writes_canonical_sequence(tmp_path, monkeypatch): assert [entry[0] for entry in observed] == ["download", "verify"] assert observed[0][1] == "downloading" - assert observed[0][2]["binary_artifact"] == "llama.tar.gz" assert observed[1][1] == "verifying" - assert result["install_state"] == "installed" - slot = _local_slot() - assert slot["binary_artifact"] == "llama.tar.gz" - assert slot["binary_sha256"] == "abc123" - assert slot["binary_path"] == str(final_path) - assert "state" not in slot + assert result["install_state"] == "verifying" + assert prove_manifest( + artifact_manifest_path(final_path.parent), + provider=local_install.LOCAL_PROVIDER_NAME, + pin_identity=local_install._vulkan_pin_identity("test-platform", pin), + ).ready @pytest.mark.parametrize( @@ -492,7 +595,6 @@ def test_install_llama_server_cuda_uses_arch_specific_oci_wanted_files( "resolve_local_backend", lambda _pin: local_cuda.BackendChoice("cuda", "test cuda"), ) - metadata_calls: list[dict[str, str]] = [] pull_calls: list[tuple[str, str, tuple[str, ...], Path]] = [] def fake_pull_and_install( @@ -515,16 +617,10 @@ def test_install_llama_server_cuda_uses_arch_specific_oci_wanted_files( ) monkeypatch.setattr(oci_image, "pull_and_install", fake_pull_and_install) - monkeypatch.setattr( - local_install, - "_write_local_metadata", - lambda updates: metadata_calls.append(updates), - ) - result = local_install.install_llama_server() wanted_files = local_install.CUDA_SERVER_PIN.wanted_files_for_arch(arch) - assert result["install_state"] == "installed" + assert result["install_state"] == "verifying" assert pull_calls == [ ( local_install.CUDA_SERVER_PIN.image_ref, @@ -535,7 +631,6 @@ def test_install_llama_server_cuda_uses_arch_specific_oci_wanted_files( ] assert expected_cpu in pull_calls[0][2] assert unexpected_cpu not in pull_calls[0][2] - assert metadata_calls == [] assert local_install.cuda_binary_path().stat().st_mode & 0o111 @@ -575,8 +670,12 @@ def test_install_llama_server_vulkan_choice_does_not_pull_oci(tmp_path, monkeypa result = local_install.install_llama_server() - assert result["install_state"] == "installed" - assert _local_slot()["binary_path"] == str(final_path) + assert result["install_state"] == "verifying" + assert prove_manifest( + artifact_manifest_path(final_path.parent), + provider=local_install.LOCAL_PROVIDER_NAME, + pin_identity=local_install._vulkan_pin_identity("test-platform", pin), + ).ready def test_probe_binary_runnable_returns_true_for_zero_exit(tmp_path): @@ -638,17 +737,15 @@ def test_probe_binary_runnable_handles_missing_path(tmp_path): def test_install_model_writes_canonical_sequence(tmp_path, monkeypatch): _init_journal(tmp_path, monkeypatch) spec = LOCAL_MODEL_SPECS[LOCAL_MODEL] - observed: list[tuple[str, str, dict]] = [] + observed: list[tuple[str, str]] = [] def fake_download(_url, _dest, **_kwargs): - observed.append( - ("download", _local_status()["install_state"], dict(_local_slot())) - ) + observed.append(("download", _local_status()["install_state"])) + _dest.parent.mkdir(parents=True, exist_ok=True) + _dest.write_bytes(b"artifact") def fake_verify(_path, _expected): - observed.append( - ("verify", _local_status()["install_state"], dict(_local_slot())) - ) + observed.append(("verify", _local_status()["install_state"])) monkeypatch.setattr(local_install, "_download_file", fake_download) monkeypatch.setattr(local_install, "_verify_sha256", fake_verify) @@ -662,16 +759,13 @@ def test_install_model_writes_canonical_sequence(tmp_path, monkeypatch): "verify", ] assert observed[0][1] == "downloading" - assert observed[0][2]["model_id"] == LOCAL_MODEL assert observed[2][1] == "verifying" - assert result["install_state"] == "installed" - slot = _local_slot() - assert slot["model_id"] == LOCAL_MODEL - assert slot["model_path"] == str(local_install.model_path(spec.model_id)) - assert slot["model_sha256"] == spec.sha256 - assert slot["mmproj_path"] == str(local_install.mmproj_path(spec.model_id)) - assert slot["mmproj_sha256"] == spec.mmproj_sha256 - assert "state" not in slot + assert result["install_state"] == "verifying" + assert prove_manifest( + artifact_manifest_path(local_install.model_dir(spec.model_id)), + provider=local_install.LOCAL_PROVIDER_NAME, + pin_identity=local_install._model_pin_identity(spec.model_id), + ).ready def test_install_model_threads_optional_mmproj_artifact(tmp_path, monkeypatch): @@ -704,9 +798,11 @@ def test_install_model_threads_optional_mmproj_artifact(tmp_path, monkeypatch): assert mmproj_path is not None assert downloads == [gguf_path, mmproj_path] assert verifies == [(gguf_path, spec.sha256), (mmproj_path, "mmproj-sha")] - slot = _local_slot() - assert slot["mmproj_path"] == str(mmproj_path) - assert slot["mmproj_sha256"] == "mmproj-sha" + assert prove_manifest( + artifact_manifest_path(local_install.model_dir(LOCAL_MODEL)), + provider=local_install.LOCAL_PROVIDER_NAME, + pin_identity=local_install._model_pin_identity(LOCAL_MODEL), + ).ready def test_install_local_blocks_before_downloads(tmp_path, monkeypatch): @@ -714,7 +810,12 @@ def test_install_local_blocks_before_downloads(tmp_path, monkeypatch): monkeypatch.setattr( local_install, "inspect_readiness", - lambda model_id: {"binary_installed": False, "model_installed": False}, + lambda model_id: _fake_local_readiness( + binary_installed=False, + model_installed=False, + binary_path=tmp_path / "llama-server", + model_path=tmp_path / "model.gguf", + ), ) monkeypatch.setattr( fit_report, "build_local_fit_report", lambda model_id: _fit("blocked") @@ -748,12 +849,20 @@ def test_install_local_warning_continues_to_download(tmp_path, monkeypatch): final_path.parent.mkdir(parents=True) final_path.write_text("binary", encoding="utf-8") downloads: list[Path] = [] + readiness_calls = 0 + + def fake_readiness(model_id: str) -> ReadinessOutcome: + nonlocal readiness_calls + readiness_calls += 1 + return _fake_local_readiness( + binary_installed=readiness_calls > 1, + model_installed=readiness_calls > 1, + binary_path=final_path, + model_path=local_install.model_path(model_id), + mmproj_path=local_install.mmproj_path(model_id), + ) - monkeypatch.setattr( - local_install, - "inspect_readiness", - lambda model_id: {"binary_installed": False, "model_installed": False}, - ) + monkeypatch.setattr(local_install, "inspect_readiness", fake_readiness) monkeypatch.setattr( fit_report, "build_local_fit_report", lambda model_id: _fit("warning") ) @@ -793,7 +902,12 @@ def test_install_local_ready_short_circuits_before_fit_report(tmp_path, monkeypa monkeypatch.setattr( local_install, "inspect_readiness", - lambda model_id: {"binary_installed": True, "model_installed": True}, + lambda model_id: _fake_local_readiness( + binary_installed=True, + model_installed=True, + binary_path=tmp_path / "llama-server", + model_path=tmp_path / "model.gguf", + ), ) monkeypatch.setattr( fit_report, @@ -813,7 +927,7 @@ def test_install_local_ready_short_circuits_before_fit_report(tmp_path, monkeypa result = local_install.install_local(LOCAL_MODEL) - assert result["name"] == local_install.LOCAL_PROVIDER_NAME + assert result["provider"] == local_install.LOCAL_PROVIDER_NAME assert result["install_state"] == "installed" @@ -827,31 +941,33 @@ def test_install_local_reinstalls_runtime_when_binary_record_stale( mmproj = local_install.mmproj_path(LOCAL_MODEL) assert mmproj is not None mmproj.write_text("mmproj", encoding="utf-8") - canonical = local_install.binary_path_for_pin() - canonical.parent.mkdir(parents=True, exist_ok=True) - canonical.write_bytes(b"llama-server") - canonical.chmod(0o755) - local_install._write_local_metadata( - { - "binary_artifact": "llama-stale-bin-ubuntu-x64.tar.gz", - "binary_sha256": "deadbeef" * 8, - "binary_path": str(canonical), - } - ) monkeypatch.setattr(local_vulkan, "detect_gpus", lambda: []) monkeypatch.setattr( fit_report, "build_local_fit_report", lambda model_id: _fit("ok") ) calls: list[str] = [] + readiness_calls = 0 + + def fake_readiness(model_id: str) -> ReadinessOutcome: + nonlocal readiness_calls + readiness_calls += 1 + return _fake_local_readiness( + binary_installed=readiness_calls > 1, + model_installed=readiness_calls > 1, + binary_path=local_install.binary_path_for_pin(), + model_path=local_install.model_path(model_id), + mmproj_path=local_install.mmproj_path(model_id), + ) - def fake_install_llama_server(): + def fake_install_llama_server(**_kwargs): calls.append("llama_server") - return {"install_state": "installed"} + return {"install_state": "verifying"} - def fake_install_model(model_id: str): + def fake_install_model(model_id: str, **_kwargs): calls.append("model") - return {"install_state": "installed", "model_id": model_id} + return {"install_state": "verifying", "model_id": model_id} + monkeypatch.setattr(local_install, "inspect_readiness", fake_readiness) monkeypatch.setattr( local_install, "install_llama_server", fake_install_llama_server ) @@ -859,7 +975,7 @@ def test_install_local_reinstalls_runtime_when_binary_record_stale( result = local_install.install_local(LOCAL_MODEL) - assert result == {"install_state": "installed", "model_id": LOCAL_MODEL} + assert result["install_state"] == "installed" assert calls == ["llama_server", "model"] @@ -872,16 +988,15 @@ def test_ensure_artifacts_installed_returns_binary_gguf_and_optional_mmproj( monkeypatch.setattr( local_install, "inspect_readiness", - lambda model_id: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "binary_path": str(binary), - "model_path": str(gguf), - "mmproj_path": str(mmproj), - "backend": "vulkan", - "backend_reason": "test vulkan", - }, + lambda model_id: _fake_local_readiness( + binary_installed=True, + model_installed=True, + binary_path=binary, + model_path=gguf, + mmproj_path=mmproj, + backend="vulkan", + backend_reason="test vulkan", + ), ) assert local_install.ensure_artifacts_installed( @@ -904,16 +1019,15 @@ def test_ensure_artifacts_installed_ignores_low_memory_when_artifacts_exist( monkeypatch.setattr( local_install, "inspect_readiness", - lambda model_id: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": False, - "binary_path": str(binary), - "model_path": str(gguf), - "mmproj_path": None, - "backend": "vulkan", - "backend_reason": "test vulkan", - }, + lambda model_id: _fake_local_readiness( + binary_installed=True, + model_installed=True, + binary_path=binary, + model_path=gguf, + ram_sufficient=False, + backend="vulkan", + backend_reason="test vulkan", + ), ) assert local_install.ensure_artifacts_installed( @@ -935,16 +1049,14 @@ def test_ensure_artifacts_installed_returns_cuda_lib_dir(tmp_path, monkeypatch): monkeypatch.setattr( local_install, "inspect_readiness", - lambda model_id: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "binary_path": str(binary), - "model_path": str(gguf), - "mmproj_path": None, - "backend": "cuda", - "backend_reason": "test cuda", - }, + lambda model_id: _fake_local_readiness( + binary_installed=True, + model_installed=True, + binary_path=binary, + model_path=gguf, + backend="cuda", + backend_reason="test cuda", + ), ) assert local_install.ensure_artifacts_installed( @@ -975,16 +1087,12 @@ def test_ensure_artifacts_installed_raises_for_missing_artifacts( monkeypatch.setattr( local_install, "inspect_readiness", - lambda model_id: { - "binary_installed": binary_installed, - "model_installed": model_installed, - "ram_sufficient": True, - "binary_path": str(binary), - "model_path": str(gguf), - "mmproj_path": None, - "backend": "vulkan", - "backend_reason": "test vulkan", - }, + lambda model_id: _fake_local_readiness( + binary_installed=binary_installed, + model_installed=model_installed, + binary_path=binary, + model_path=gguf, + ), ) with pytest.raises(local_install.LocalProviderError) as exc_info: @@ -1005,7 +1113,7 @@ def test_inspect_readiness_reports_ram_sufficient_for_low_or_unknown_memory( readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["ram_sufficient"] is True + assert readiness.host["ram_sufficient"] is True @pytest.mark.parametrize("sidecar_ok", [True, False]) @@ -1024,8 +1132,24 @@ def test_inspect_readiness_cuda_uses_sidecar_full_set( ) binary = local_install.cuda_binary_path() binary.parent.mkdir(parents=True, exist_ok=True) - binary.write_text("binary", encoding="utf-8") - binary.chmod(0o755) + wanted_files = local_install.CUDA_SERVER_PIN.wanted_files_for_arch( + local_install._oci_arch() + ) + for name in wanted_files: + member = binary.parent / name + member.write_text(name, encoding="utf-8") + member.chmod(0o755) + (binary.parent / ".oci-install.json").write_text( + json.dumps( + { + "image_ref": local_install.CUDA_SERVER_PIN.image_ref, + "arch": local_install._oci_arch(), + "files": {name: "0" * 64 for name in wanted_files}, + } + ) + + "\n", + encoding="utf-8", + ) verify_calls: list[tuple[str, str, tuple[str, ...], Path]] = [] def fake_verify( @@ -1048,15 +1172,12 @@ def test_inspect_readiness_cuda_uses_sidecar_full_set( readiness = local_install.inspect_readiness(LOCAL_MODEL) - wanted_files = local_install.CUDA_SERVER_PIN.wanted_files_for_arch( - local_install._oci_arch() - ) - assert readiness["backend"] == "cuda" - assert readiness["backend_reason"] == "test cuda" - assert readiness["binary_path"] == str(binary) - assert readiness["binary_installed"] is sidecar_ok - assert readiness["gpu_available"] is True - assert readiness["gpu_probe_ok"] is True + assert readiness.host["backend"] == "cuda" + assert readiness.host["backend_reason"] == "test cuda" + assert readiness.artifacts["binary_path"] == str(binary) + assert readiness.artifacts["binary_installed"] is sidecar_ok + assert readiness.host["gpu_available"] is True + assert readiness.host["gpu_probe_ok"] is True assert verify_calls == [ ( local_install.CUDA_SERVER_PIN.image_ref, @@ -1084,9 +1205,9 @@ def test_inspect_readiness_reports_gpu_available_with_hardware(tmp_path, monkeyp readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["gpu_available"] is True - assert readiness["backend"] == "vulkan" - assert readiness["backend_reason"] == "test vulkan" + assert readiness.host["gpu_available"] is True + assert readiness.host["backend"] == "vulkan" + assert readiness.host["backend_reason"] == "test vulkan" def test_inspect_readiness_reports_gpu_unavailable_without_hardware( @@ -1097,7 +1218,7 @@ def test_inspect_readiness_reports_gpu_unavailable_without_hardware( readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["gpu_available"] is False + assert readiness.host["gpu_available"] is False def test_inspect_readiness_stale_non_cuda_binary_record_reports_not_installed( @@ -1109,18 +1230,12 @@ def test_inspect_readiness_stale_non_cuda_binary_record_reports_not_installed( canonical.parent.mkdir(parents=True, exist_ok=True) canonical.write_bytes(b"llama-server") canonical.chmod(0o755) - local_install._write_local_metadata( - { - "binary_artifact": "llama-stale-bin-ubuntu-x64.tar.gz", - "binary_sha256": "deadbeef" * 8, - "binary_path": str(canonical), - } - ) readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["binary_installed"] is False - assert readiness["binary_path"] == str(canonical) + assert readiness.artifacts["binary_installed"] is False + assert readiness.artifacts["binary_path"] == str(canonical) + assert readiness.proof["binary"]["status"] == "missing-or-mismatched" def test_inspect_readiness_matching_non_cuda_binary_record_reports_installed( @@ -1133,17 +1248,11 @@ def test_inspect_readiness_matching_non_cuda_binary_record_reports_installed( canonical.parent.mkdir(parents=True, exist_ok=True) canonical.write_bytes(b"llama-server") canonical.chmod(0o755) - local_install._write_local_metadata( - { - "binary_artifact": pin["filename"], - "binary_sha256": pin["sha256"], - "binary_path": str(canonical), - } - ) + _write_ready_vulkan_manifest(pin=pin) readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["binary_installed"] is True + assert readiness.artifacts["binary_installed"] is True def test_inspect_readiness_honors_vulkan_device_override(tmp_path, monkeypatch): @@ -1163,14 +1272,15 @@ def test_inspect_readiness_honors_vulkan_device_override(tmp_path, monkeypatch): ), ] monkeypatch.setattr(local_vulkan, "detect_gpus", lambda: devices) - local_install._write_local_metadata({"vulkan_device_index": "0"}) + _write_provider_local_config(tmp_path, {"vulkan_device_index": "0"}) assert local_install.gpu_device_override() == 0 - assert local_install.inspect_readiness(LOCAL_MODEL)["gpu_available"] is True + assert resolve_local_endpoint().is_bundled is True + assert local_install.inspect_readiness(LOCAL_MODEL).host["gpu_available"] is True - local_install._write_local_metadata({"vulkan_device_index": "1"}) + _write_provider_local_config(tmp_path, {"vulkan_device_index": "1"}) - assert local_install.inspect_readiness(LOCAL_MODEL)["gpu_available"] is False + assert local_install.inspect_readiness(LOCAL_MODEL).host["gpu_available"] is False def test_inspect_readiness_ignores_stale_model_path_after_model_change( @@ -1186,9 +1296,6 @@ def test_inspect_readiness_ignores_stale_model_path_after_model_change( stale_dir.mkdir(parents=True, exist_ok=True) stale_gguf = stale_dir / "coder-7b-Q4_K_M.gguf" stale_gguf.write_text("stale", encoding="utf-8") - local_install._write_local_metadata( - {"model_id": "local/old-coder-7b", "model_path": str(stale_gguf)} - ) # Stage the selected model's artifacts in its own directory. gguf = local_install.model_path(LOCAL_MODEL) @@ -1197,15 +1304,18 @@ def test_inspect_readiness_ignores_stale_model_path_after_model_change( mmproj = local_install.mmproj_path(LOCAL_MODEL) assert mmproj is not None mmproj.write_text("mmproj", encoding="utf-8") + _write_ready_model_manifest(LOCAL_MODEL) readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["model_id"] == LOCAL_MODEL - assert readiness["model_path"] == str(gguf) - assert readiness["mmproj_path"] == str(mmproj) - assert Path(readiness["model_path"]).parent == local_install.model_dir(LOCAL_MODEL) - assert readiness["model_path"] != str(stale_gguf) - assert readiness["model_installed"] is True + assert readiness.artifacts["model_id"] == LOCAL_MODEL + assert readiness.artifacts["model_path"] == str(gguf) + assert readiness.artifacts["mmproj_path"] == str(mmproj) + assert Path(readiness.artifacts["model_path"]).parent == local_install.model_dir( + LOCAL_MODEL + ) + assert readiness.artifacts["model_path"] != str(stale_gguf) + assert readiness.artifacts["model_installed"] is True def test_inspect_readiness_not_installed_off_stale_record(tmp_path, monkeypatch): @@ -1217,15 +1327,14 @@ def test_inspect_readiness_not_installed_off_stale_record(tmp_path, monkeypatch) stale_dir.mkdir(parents=True, exist_ok=True) stale_gguf = stale_dir / "coder-7b-Q4_K_M.gguf" stale_gguf.write_text("stale", encoding="utf-8") - local_install._write_local_metadata( - {"model_id": "local/old-coder-7b", "model_path": str(stale_gguf)} - ) readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["model_installed"] is False - assert readiness["gguf_installed"] is False - assert readiness["model_path"] == str(local_install.model_path(LOCAL_MODEL)) + assert readiness.artifacts["model_installed"] is False + assert readiness.artifacts["gguf_installed"] is False + assert readiness.artifacts["model_path"] == str( + local_install.model_path(LOCAL_MODEL) + ) def test_install_llama_server_failure_writes_canonical_failed(tmp_path, monkeypatch): @@ -1252,5 +1361,3 @@ def test_install_llama_server_failure_writes_canonical_failed(tmp_path, monkeypa status = _local_status() assert status["install_state"] == "failed" assert status["install_error"] == "network broke" - slot = _local_slot() - assert "state" not in slot diff --git a/tests/test_mlx_install.py b/tests/test_mlx_install.py index 40c5ca364..1d09ddbfc 100644 --- a/tests/test_mlx_install.py +++ b/tests/test_mlx_install.py @@ -18,6 +18,11 @@ from huggingface_hub import constants as hf_constants from solstone.think.journal_config import read_journal_config from solstone.think.models import GEMMA4_26B_A4B_4BIT, QWEN_35_9B from solstone.think.providers import fit_report, memory, mlx_install +from solstone.think.providers.artifact_proof import ( + mlx_snapshot_manifest_path, + mlx_variant_manifest_path, + prove_manifest, +) from solstone.think.providers.install_state import read_install_status @@ -33,15 +38,12 @@ def _init_journal(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def _local_status() -> dict: - return read_install_status(scope="bundled", name="local") - - -def _local_slot() -> dict: - return read_journal_config()["providers"]["bundled"]["local"] + return read_install_status(name="local") def _allow_install(monkeypatch: pytest.MonkeyPatch, *, ram_gb: int = 64) -> None: monkeypatch.setattr(mlx_install, "_check_platform_and_package", lambda: (True, "")) + monkeypatch.setattr(mlx_install, "is_mlx_platform_supported", lambda: True) monkeypatch.setattr( memory.psutil, "virtual_memory", @@ -104,6 +106,15 @@ def _write_snapshot( return snapshot_dir, hashlib.sha256(weight_bytes).hexdigest() +def _metadata_for_snapshot(snapshot_dir: Path) -> dict[str, tuple[str, int]]: + metadata: dict[str, tuple[str, int]] = {} + for path in snapshot_dir.rglob("*.safetensors"): + rel_path = path.relative_to(snapshot_dir).as_posix() + digest = hashlib.sha256(path.read_bytes()).hexdigest() + metadata[rel_path] = (digest, path.stat().st_size) + return metadata + + def test_module_import_is_mlx_vlm_free(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setitem(sys.modules, "mlx_vlm", None) @@ -143,7 +154,7 @@ def test_inspect_readiness_ram_sufficient_matches_available_floor( ), ) - assert mlx_install.inspect_readiness(QWEN_35_9B)["ram_sufficient"] is True + assert mlx_install.inspect_readiness(QWEN_35_9B).host["ram_sufficient"] is True monkeypatch.setattr( memory.psutil, @@ -155,7 +166,7 @@ def test_inspect_readiness_ram_sufficient_matches_available_floor( ) readiness = mlx_install.inspect_readiness(QWEN_35_9B) - assert readiness["ram_sufficient"] is False + assert readiness.host["ram_sufficient"] is False def test_install_local_mlx_writes_canonical_sequence( @@ -180,6 +191,7 @@ def test_install_local_mlx_writes_canonical_sequence( def fake_verify(_spec, _snapshot_dir): observed.append(("verify", _local_status()["install_state"], {})) + return _metadata_for_snapshot(_snapshot_dir) def fake_create(_snapshot_dir): observed.append(("install", _local_status()["install_state"], {})) @@ -206,11 +218,16 @@ def test_install_local_mlx_writes_canonical_sequence( "installing", ] assert result["install_state"] == "installed" - slot = _local_slot() - assert slot["mlx_model_id"] == GEMMA4_26B_A4B_4BIT - assert slot["mlx_revision"] == spec.revision - assert slot["mlx_snapshot_dir"] == str(snapshot_dir) - assert slot["mlx_variant_dir"] == str(variant_dir) + assert prove_manifest( + mlx_snapshot_manifest_path(spec.repo, spec.revision), + provider="local", + pin_identity=mlx_install._pin_identity(spec), + ).ready + assert prove_manifest( + mlx_variant_manifest_path(spec.repo, spec.revision), + provider="local", + pin_identity=mlx_install._variant_pin_identity(spec), + ).ready assert "mlx" not in read_journal_config()["providers"] @@ -277,7 +294,11 @@ def test_install_local_mlx_warning_continues_to_download( return str(snapshot_dir) monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download) - monkeypatch.setattr(mlx_install, "validate_snapshot_sha256", lambda *_args: None) + monkeypatch.setattr( + mlx_install, + "validate_snapshot_sha256", + lambda _spec, snapshot_dir: _metadata_for_snapshot(snapshot_dir), + ) assert mlx_install.install_local_mlx()["install_state"] == "installed" assert calls == {"download": 1} @@ -346,7 +367,11 @@ def test_installing_failure_transitions_failed( monkeypatch.setattr( huggingface_hub, "snapshot_download", lambda **_kwargs: str(snapshot_dir) ) - monkeypatch.setattr(mlx_install, "validate_snapshot_sha256", lambda *_args: None) + monkeypatch.setattr( + mlx_install, + "validate_snapshot_sha256", + lambda _spec, snapshot_dir: _metadata_for_snapshot(snapshot_dir), + ) with pytest.raises(FileNotFoundError): mlx_install.install_local_mlx(GEMMA4_26B_A4B_4BIT) @@ -374,6 +399,7 @@ def test_idempotent_rerun_skips_network( def fake_verify(_spec, _snapshot_dir): calls["verify"] += 1 + return _metadata_for_snapshot(_snapshot_dir) monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download) monkeypatch.setattr(mlx_install, "validate_snapshot_sha256", fake_verify) @@ -388,7 +414,11 @@ def test_idempotent_rerun_skips_network( ) assert mlx_install.install_local_mlx()["install_state"] == "installed" assert calls == {"download": 1, "verify": 1} - assert _local_slot()["mlx_model_id"] == QWEN_35_9B + assert prove_manifest( + mlx_snapshot_manifest_path(spec.repo, spec.revision), + provider="local", + pin_identity=mlx_install._pin_identity(spec), + ).ready def test_validate_snapshot_sha256_uses_lfs_metadata( diff --git a/tests/test_parakeet_install.py b/tests/test_parakeet_install.py index 53dfb0b83..dbef2e4bc 100644 --- a/tests/test_parakeet_install.py +++ b/tests/test_parakeet_install.py @@ -7,19 +7,17 @@ import io import os import shutil import tarfile -from contextlib import contextmanager from pathlib import Path from typing import Any import pytest from solstone.think import parakeet_readiness -from solstone.think.journal_config import ( - JournalConfigTransaction, - read_journal_config, -) -from solstone.think.journal_io.errors import LockTimeout from solstone.think.providers import fit_report, parakeet_install +from solstone.think.providers.artifact_proof import ( + artifact_manifest_path, + prove_manifest, +) from solstone.think.providers.install_state import read_install_status from tests.helpers.journal_config import seed_journal_config @@ -51,11 +49,7 @@ def _init_journal(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def _parakeet_status() -> dict: - return read_install_status(scope="bundled", name="parakeet") - - -def _parakeet_slot() -> dict: - return read_journal_config()["providers"]["bundled"]["parakeet"] + return read_install_status(name="parakeet") def _fit(severity: fit_report.FitSeverity) -> fit_report.FitReport: @@ -94,9 +88,33 @@ def _stage_ready_files() -> tuple[Path, Path, Path]: path.chmod(0o755) model.parent.mkdir(parents=True, exist_ok=True) model.write_text("model\n", encoding="utf-8") + fingerprint = parakeet_install.target_fingerprint() + for backend in parakeet_readiness.PARAKEET_CPP_BINARY_BACKENDS: + parakeet_install._write_binary_manifest( + backend=backend, + attempt_status=None, + fingerprint=fingerprint, + journal_path=None, + ) + parakeet_install._write_model_manifest( + attempt_status=None, + fingerprint=fingerprint, + journal_path=None, + ) return cpu, vulkan, model +def _write_ready_binary_manifests() -> None: + fingerprint = parakeet_install.target_fingerprint() + for backend in parakeet_readiness.PARAKEET_CPP_BINARY_BACKENDS: + parakeet_install._write_binary_manifest( + backend=backend, + attempt_status=None, + fingerprint=fingerprint, + journal_path=None, + ) + + def test_install_hint_literal() -> None: assert parakeet_install.install_hint() == "journal install-provider parakeet" @@ -169,7 +187,7 @@ def test_install_parakeet_server_relocates_and_chmods_binary( install_dir = parakeet_install.binary_install_dir("cpu") final_path = parakeet_install.binary_path("cpu") - assert result["install_state"] == "installed" + assert result["install_state"] == "verifying" assert final_path.exists() assert ( final_path.read_bytes() @@ -182,6 +200,13 @@ def test_install_parakeet_server_relocates_and_chmods_binary( install_dir / f"parakeet-{parakeet_readiness.PARAKEET_CPP_RELEASE_TAG}-bin-linux-cpu-x64" ).exists() + assert prove_manifest( + artifact_manifest_path(install_dir), + provider=parakeet_install.PARAKEET_PROVIDER_NAME, + pin_identity=parakeet_install._binary_pin_identity( + parakeet_install.parakeet_server_artifact_key(), "cpu" + ), + ).ready def test_sha256_mismatch_fails_closed_and_records_failed_state( @@ -205,7 +230,7 @@ def test_sha256_mismatch_fails_closed_and_records_failed_state( assert "sha256 mismatch" in status["install_error"] -def test_install_parakeet_writes_distinct_binary_and_model_metadata( +def test_install_parakeet_writes_distinct_binary_and_model_manifests( tmp_path, monkeypatch ) -> None: _init_journal(tmp_path, monkeypatch) @@ -230,107 +255,41 @@ def test_install_parakeet_writes_distinct_binary_and_model_metadata( result = parakeet_install.install_parakeet() assert result["install_state"] == "installed" - slot = _parakeet_slot() - assert ( - slot["binary_artifact_cpu"] - == parakeet_install.PARAKEET_SERVER_PINS[ - (parakeet_install.parakeet_server_artifact_key(), "cpu") - ]["filename"] + artifact_key = parakeet_install.parakeet_server_artifact_key() + cpu_manifest = artifact_manifest_path(parakeet_install.binary_install_dir("cpu")) + vulkan_manifest = artifact_manifest_path( + parakeet_install.binary_install_dir("vulkan") ) + assert parakeet_install.binary_path("cpu") != parakeet_install.binary_path("vulkan") + assert parakeet_install.binary_path("cpu").is_file() + assert parakeet_install.binary_path("vulkan").is_file() + assert parakeet_install.model_path().is_file() assert ( - slot["binary_artifact_vulkan"] - == parakeet_install.PARAKEET_SERVER_PINS[ - (parakeet_install.parakeet_server_artifact_key(), "vulkan") - ]["filename"] + prove_manifest( + cpu_manifest, + provider=parakeet_install.PARAKEET_PROVIDER_NAME, + pin_identity=parakeet_install._binary_pin_identity(artifact_key, "cpu"), + ).ready + is True ) - assert slot["binary_path_cpu"] != slot["binary_path_vulkan"] - assert Path(slot["binary_path_cpu"]).is_file() - assert Path(slot["binary_path_vulkan"]).is_file() - assert slot["model_repo"] == parakeet_readiness.PARAKEET_CPP_MODEL_REPO - assert slot["model_filename"] == parakeet_readiness.PARAKEET_CPP_MODEL_FILENAME - assert slot["model_revision"] == parakeet_readiness.PARAKEET_CPP_MODEL_REVISION - assert slot["model_path"] == str(parakeet_install.model_path()) - assert Path(slot["model_path"]).is_file() - - -def test_write_parakeet_metadata_waits_for_config_lock_and_preserves_commits( - tmp_path, - monkeypatch, -) -> None: - _init_journal(tmp_path, monkeypatch) - journal_path = tmp_path - config = { - "setup": { - "completed_at": "2026-07-01T00:00:00+00:00", - "completed_by": "setup-writer", - }, - "service": {"port": 5015, "host": "127.0.0.1"}, - "providers": {"bundled": {}}, - } - calls: list[tuple[str, object]] = [] - - def recording_mutate(mutator, *, journal_path=None): - calls.append(("mutate", journal_path)) - mutation = mutator(config) - return JournalConfigTransaction( - value=mutation.value, - changed=mutation.changed, - written=mutation.changed, - ) - - monkeypatch.setattr(parakeet_install, "mutate_journal_config", recording_mutate) - - parakeet_install._write_parakeet_metadata( - {"model_repo": "openai/parakeet-test"}, - journal_path=journal_path, + assert ( + prove_manifest( + vulkan_manifest, + provider=parakeet_install.PARAKEET_PROVIDER_NAME, + pin_identity=parakeet_install._binary_pin_identity(artifact_key, "vulkan"), + ).ready + is True ) - - assert calls == [("mutate", journal_path)] - persisted = config - assert persisted["setup"]["completed_at"] == "2026-07-01T00:00:00+00:00" - assert persisted["setup"]["completed_by"] == "setup-writer" - assert persisted["service"]["port"] == 5015 - assert persisted["service"]["host"] == "127.0.0.1" assert ( - persisted["providers"]["bundled"]["parakeet"]["model_repo"] - == "openai/parakeet-test" + prove_manifest( + artifact_manifest_path(parakeet_install.model_dir()), + provider=parakeet_install.PARAKEET_PROVIDER_NAME, + pin_identity=parakeet_install._model_pin_identity(), + ).ready + is True ) -def test_write_parakeet_metadata_rejects_unknown_key_without_lock( - monkeypatch, -) -> None: - mutate_calls: list[object] = [] - - def recording_mutate(mutator, *, journal_path=None): - mutate_calls.append(journal_path) - pytest.fail("metadata validation must happen before transaction entry") - - monkeypatch.setattr(parakeet_install, "mutate_journal_config", recording_mutate) - - with pytest.raises(ValueError) as exc_info: - parakeet_install._write_parakeet_metadata({"unexpected": "value"}) - - assert str(exc_info.value) == "unknown parakeet install metadata key: unexpected" - assert mutate_calls == [] - - -def test_write_parakeet_metadata_propagates_config_lock_timeout_without_config_io( - monkeypatch, -) -> None: - timeout = LockTimeout(path=Path("busy.lock"), timeout=0.01) - - def busy_mutate(mutator, *, journal_path=None): - raise timeout - - monkeypatch.setattr(parakeet_install, "mutate_journal_config", busy_mutate) - - with pytest.raises(LockTimeout) as exc_info: - parakeet_install._write_parakeet_metadata({"model_repo": "openai/test"}) - - assert exc_info.value is timeout - - def test_install_parakeet_blocks_before_downloads(tmp_path, monkeypatch) -> None: _init_journal(tmp_path, monkeypatch) monkeypatch.setattr( @@ -385,34 +344,25 @@ def test_install_parakeet_warning_continues_to_download( assert downloads -def test_install_parakeet_rechecks_readiness_under_provider_lock( +def test_install_parakeet_ready_short_circuits_before_component_installs( tmp_path, monkeypatch ) -> None: _init_journal(tmp_path, monkeypatch) _stage_ready_files() - lock_calls = [] - - @contextmanager - def fake_hold_install_lock(journal_path=None): - lock_calls.append(parakeet_install._install_lock_path(journal_path)) - yield - - monkeypatch.setattr(parakeet_install, "_hold_install_lock", fake_hold_install_lock) monkeypatch.setattr( parakeet_install, "_install_parakeet_server_unlocked", - lambda *_args: pytest.fail("ready artifacts should not reinstall"), + lambda *_args, **_kwargs: pytest.fail("ready artifacts should not reinstall"), ) monkeypatch.setattr( parakeet_install, "_install_model_unlocked", - lambda: pytest.fail("ready model should not reinstall"), + lambda *_args, **_kwargs: pytest.fail("ready model should not reinstall"), ) result = parakeet_install.install_parakeet() assert result["install_state"] == "installed" - assert lock_calls == [parakeet_install.cache_root() / "install"] def test_ensure_artifacts_installed_resolves_requested_backend( @@ -440,6 +390,7 @@ def test_ensure_artifacts_installed_reports_missing_binary_and_model( path.parent.mkdir(parents=True, exist_ok=True) path.write_text("server\n", encoding="utf-8") path.chmod(0o755) + _write_ready_binary_manifests() with pytest.raises(parakeet_install.ParakeetProviderError) as model_exc: parakeet_install.ensure_artifacts_installed("cpu") diff --git a/tests/test_provider_state.py b/tests/test_provider_state.py index 59fd83592..d6b5a4062 100644 --- a/tests/test_provider_state.py +++ b/tests/test_provider_state.py @@ -16,6 +16,7 @@ from solstone.think.providers import ( mlx_install, state, ) +from solstone.think.providers.artifact_proof import ReadinessOutcome from solstone.think.providers.shared import ( RUNTIME_REASON_CODES, classify_provider_error, @@ -45,23 +46,62 @@ def _readiness( gpu_probe_ok: bool | None = None, install_state: str = "installed", backend: str = "vulkan", -) -> dict: - payload = { - "install_state": install_state, - "binary_installed": binary, - "model_installed": model, - "ram_sufficient": ram, - "gpu_available": gpu, - "binary_path": "/tmp/llama-server", - "model_path": "/tmp/model.gguf", - "model_id": LOCAL_MODEL, - "install_error": None, - "backend": backend, - "backend_reason": f"test {backend}", - } - if gpu_probe_ok is not None: - payload["gpu_probe_ok"] = gpu_probe_ok - return payload +) -> ReadinessOutcome: + missing_binary = not binary + missing_model = not model + status = "ready" if binary and model and ram and gpu else "missing-or-mismatched" + reason_code = ( + "ready" + if status == "ready" + else "binary_missing" + if missing_binary + else "model_missing" + if missing_model + else "host_unfit" + ) + return ReadinessOutcome( + provider="local", + status=status, + reason_code=reason_code, + target={"model_id": LOCAL_MODEL}, + install={ + "install_state": install_state, + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": ram, + "gpu_available": gpu, + "gpu_probe_ok": True if gpu_probe_ok is None else gpu_probe_ok, + "backend": backend, + "backend_reason": f"test {backend}", + }, + artifacts={ + "binary_installed": binary, + "model_installed": model, + "binary_path": "/tmp/llama-server", + "model_path": "/tmp/model.gguf", + "model_id": LOCAL_MODEL, + "mmproj_path": None, + }, + proof={ + "binary": { + "status": "ready" if binary else "missing-or-mismatched", + "reason_code": "ready" if binary else "manifest_missing", + "cache_hit": False, + }, + "model": { + "status": "ready" if model else "missing-or-mismatched", + "reason_code": "ready" if model else "manifest_missing", + "cache_hit": False, + }, + }, + ) def _mlx_readiness( @@ -70,21 +110,49 @@ def _mlx_readiness( model_installed: bool = True, platform_supported: bool = True, package_available: bool = True, -) -> dict: - return { - "install_state": install_state, - "model_installed": model_installed, - "snapshot_installed": model_installed, - "variant_installed": True, - "ram_sufficient": True, - "platform_supported": platform_supported, - "package_available": package_available, - "model_id": QWEN_35_9B, - "snapshot_dir": "/tmp/snap", - "variant_dir": None, - "runtime_dir": "/tmp/snap", - "install_error": None, - } +) -> ReadinessOutcome: + return ReadinessOutcome( + provider="local", + status=( + "ready" + if model_installed and platform_supported and package_available + else "missing-or-mismatched" + ), + reason_code="ready" if model_installed else "manifest_missing", + target={"model_id": QWEN_35_9B}, + install={ + "install_state": install_state, + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": True, + "platform_supported": platform_supported, + "package_available": package_available, + }, + artifacts={ + "model_installed": model_installed, + "snapshot_installed": model_installed, + "variant_installed": True, + "model_id": QWEN_35_9B, + "snapshot_dir": "/tmp/snap", + "variant_dir": None, + "runtime_dir": "/tmp/snap", + }, + proof={ + "snapshot": { + "status": "ready" if model_installed else "missing-or-mismatched", + "reason_code": "ready" if model_installed else "manifest_missing", + "cache_hit": False, + }, + "variant": None, + }, + ) def _block_linux_local_path(monkeypatch) -> None: @@ -461,12 +529,17 @@ def test_local_readiness_gpu_unavailable_flows_from_inspect_without_launch( if mmproj is not None: mmproj.write_text("mmproj", encoding="utf-8") pin = local_install.pin_for_current_platform() - local_install._write_local_metadata( - { - "binary_artifact": pin["filename"], - "binary_sha256": pin["sha256"], - "binary_path": str(binary), - } + fingerprint = local_install.target_fingerprint(LOCAL_MODEL) + local_install._write_vulkan_manifest( + artifact_key=local_install.llama_server_artifact_key(), + pin=pin, + attempt_status=None, + fingerprint=fingerprint, + ) + local_install._write_model_manifest( + model_id=LOCAL_MODEL, + attempt_status=None, + fingerprint=fingerprint, ) monkeypatch.setattr(local_vulkan, "detect_gpus", lambda: []) @@ -479,9 +552,9 @@ def test_local_readiness_gpu_unavailable_flows_from_inspect_without_launch( readiness = local_install.inspect_readiness(LOCAL_MODEL) provider_state = state.readiness_for_provider("local", "generate") - assert readiness["binary_installed"] is True - assert readiness["model_installed"] is True - assert readiness["gpu_available"] is False + assert readiness.artifacts["binary_installed"] is True + assert readiness.artifacts["model_installed"] is True + assert readiness.host["gpu_available"] is False assert provider_state.status == "blocked" assert provider_state.reason_code == "gpu_unavailable" assert provider_state.source == "local_install" @@ -509,12 +582,17 @@ def test_local_readiness_gpu_probe_failed_flows_from_inspect_without_launch( if mmproj is not None: mmproj.write_text("mmproj", encoding="utf-8") pin = local_install.pin_for_current_platform() - local_install._write_local_metadata( - { - "binary_artifact": pin["filename"], - "binary_sha256": pin["sha256"], - "binary_path": str(binary), - } + fingerprint = local_install.target_fingerprint(LOCAL_MODEL) + local_install._write_vulkan_manifest( + artifact_key=local_install.llama_server_artifact_key(), + pin=pin, + attempt_status=None, + fingerprint=fingerprint, + ) + local_install._write_model_manifest( + model_id=LOCAL_MODEL, + attempt_status=None, + fingerprint=fingerprint, ) monkeypatch.setattr( @@ -539,10 +617,10 @@ def test_local_readiness_gpu_probe_failed_flows_from_inspect_without_launch( readiness = local_install.inspect_readiness(LOCAL_MODEL) provider_state = state.readiness_for_provider("local", "generate") - assert readiness["binary_installed"] is True - assert readiness["model_installed"] is True - assert readiness["gpu_available"] is True - assert readiness["gpu_probe_ok"] is False + assert readiness.artifacts["binary_installed"] is True + assert readiness.artifacts["model_installed"] is True + assert readiness.host["gpu_available"] is True + assert readiness.host["gpu_probe_ok"] is False assert provider_state.status == "blocked" assert provider_state.reason_code == "gpu_probe_failed" assert provider_state.source == "local_install" diff --git a/tests/test_supervisor.py b/tests/test_supervisor.py index 64295c4f9..c578a30fc 100644 --- a/tests/test_supervisor.py +++ b/tests/test_supervisor.py @@ -28,6 +28,47 @@ from solstone.think.processing import ( ProcessingSettings, TimeWindowSettings, ) +from solstone.think.providers.artifact_proof import ReadinessOutcome + + +def _mlx_readiness( + *, + model_installed: bool = True, + ram_sufficient: bool = True, + platform_supported: bool = True, + package_available: bool = True, + runtime_dir: str = "/tmp/snap", + model_id: str = "mlx-model", +) -> ReadinessOutcome: + ready = ( + model_installed and ram_sufficient and platform_supported and package_available + ) + return ReadinessOutcome( + provider="local", + status="ready" if ready else "missing-or-mismatched", + reason_code="ready" if ready else "manifest_missing", + target={"model_id": model_id}, + install={ + "install_state": "idle", + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "platform_supported": platform_supported, + "package_available": package_available, + "ram_sufficient": ram_sufficient, + }, + artifacts={ + "model_installed": model_installed, + "runtime_dir": runtime_dir, + }, + proof={}, + ) @pytest.fixture(autouse=True) @@ -3139,14 +3180,10 @@ def test_start_local_server_launches_mlx_server_on_darwin( monkeypatch.setattr( mlx_install, "inspect_readiness", - lambda: { - "platform_supported": True, - "package_available": True, - "ram_sufficient": True, - "model_installed": True, - "runtime_dir": str(runtime_dir), - "model_id": "gemma-4-26b-a4b-it-mlx-4bit", - }, + lambda: _mlx_readiness( + runtime_dir=str(runtime_dir), + model_id="gemma-4-26b-a4b-it-mlx-4bit", + ), ) monkeypatch.setattr(mod, "find_available_port", lambda: 2468) monkeypatch.setattr( @@ -3201,12 +3238,7 @@ def test_start_local_server_skips_when_mlx_not_installed_on_darwin(monkeypatch): monkeypatch.setattr( mlx_install, "inspect_readiness", - lambda: { - "platform_supported": True, - "package_available": True, - "ram_sufficient": True, - "model_installed": False, - }, + lambda: _mlx_readiness(model_installed=False), ) launch = MagicMock() monkeypatch.setattr(mod, "_launch_process", launch) @@ -3223,12 +3255,7 @@ def test_start_local_server_skips_when_mlx_memory_blocked_on_darwin(monkeypatch) monkeypatch.setattr( mlx_install, "inspect_readiness", - lambda: { - "platform_supported": True, - "package_available": True, - "ram_sufficient": False, - "model_installed": True, - }, + lambda: _mlx_readiness(ram_sufficient=False), ) launch = MagicMock() monkeypatch.setattr(mod, "_launch_process", launch) diff --git a/tests/test_supervisor_parakeet.py b/tests/test_supervisor_parakeet.py index 64fa48e6d..a9b39dea9 100644 --- a/tests/test_supervisor_parakeet.py +++ b/tests/test_supervisor_parakeet.py @@ -17,6 +17,7 @@ from solstone.think.providers import ( parakeet_install, parakeet_server, ) +from solstone.think.providers.artifact_proof import ReadinessOutcome from solstone.think.providers.parakeet_placement import ( PARAKEET_ATT_CONTEXT_ENV, PARAKEET_ATT_CONTEXT_FRAMES, @@ -25,6 +26,45 @@ from solstone.think.providers.parakeet_placement import ( _LaunchRecord = dict[str, Any] +class _FakeLease: + def __init__(self) -> None: + self.released = False + + def release(self) -> None: + self.released = True + + +def _parakeet_readiness( + *, + install_state: str = "idle", + binary_installed: bool = False, + model_installed: bool = False, +) -> ReadinessOutcome: + ready = binary_installed and model_installed + return ReadinessOutcome( + provider="parakeet", + status="ready" if ready else "missing-or-mismatched", + reason_code="ready" if ready else "manifest_missing", + target={}, + install={ + "install_state": install_state, + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={}, + artifacts={ + "binary_installed": binary_installed, + "model_installed": model_installed, + }, + proof={}, + ) + + def _assert_att_context(launch: _LaunchRecord) -> None: assert launch["env"][PARAKEET_ATT_CONTEXT_ENV] == str(PARAKEET_ATT_CONTEXT_FRAMES) @@ -627,7 +667,15 @@ def test_start_parakeet_server_starts_background_install_when_missing( def start(self): started.append(self.name) + class _Event: + def set(self): + return None + + def wait(self, timeout=None): + return True + monkeypatch.setattr(supervisor.threading, "Thread", _Thread) + monkeypatch.setattr(supervisor.threading, "Event", _Event) monkeypatch.setattr( parakeet_install, "ensure_artifacts_installed", @@ -640,13 +688,8 @@ def test_start_parakeet_server_starts_background_install_when_missing( monkeypatch.setattr( parakeet_install, "inspect_readiness", - lambda: { - "install_state": "idle", - "binary_installed": False, - "model_installed": False, - }, + lambda: _parakeet_readiness(), ) - monkeypatch.setattr(supervisor, "_parakeet_bootstrap_thread", None) assert supervisor.start_parakeet_server() is None assert started == ["parakeet-cpp-provider-bootstrap"] @@ -654,49 +697,67 @@ def test_start_parakeet_server_starts_background_install_when_missing( def test_parakeet_bootstrap_worker_requests_start_after_install(monkeypatch) -> None: - calls: list[str] = [] + calls: list[dict[str, Any]] = [] requests: list[str] = [] + lease = _FakeLease() + attempt_status = {"attempt_id": "attempt"} + ack = supervisor.threading.Event() monkeypatch.setattr( - parakeet_install, "install_parakeet", lambda: calls.append("install") + parakeet_install, + "install_parakeet", + lambda **kwargs: calls.append(kwargs), ) monkeypatch.setattr( supervisor, "_request_parakeet_server_start", lambda: requests.append("start") ) - monkeypatch.setattr( - supervisor, - "_parakeet_bootstrap_thread", - supervisor.threading.current_thread(), - ) - supervisor._run_parakeet_bootstrap_worker() + supervisor._run_parakeet_bootstrap_worker( + lease=lease, + attempt_status=attempt_status, + ack=ack, + ) - assert calls == ["install"] + assert calls == [ + {"journal_path": None, "lease": lease, "attempt_status": attempt_status} + ] assert requests == ["start"] - assert supervisor._parakeet_bootstrap_thread is None + assert ack.is_set() + assert lease.released is True def test_parakeet_bootstrap_worker_uses_captured_journal_path( monkeypatch, tmp_path ) -> None: - calls: list[Path] = [] + calls: list[dict[str, Any]] = [] requests: list[str] = [] journal_path = tmp_path / "original" + lease = _FakeLease() + attempt_status = {"attempt_id": "attempt"} + ack = supervisor.threading.Event() - def fake_install_parakeet(*, journal_path=None): - calls.append(journal_path) + def fake_install_parakeet(**kwargs): + calls.append(kwargs) monkeypatch.setattr(parakeet_install, "install_parakeet", fake_install_parakeet) monkeypatch.setattr( supervisor, "_request_parakeet_server_start", lambda: requests.append("start") ) - monkeypatch.setattr( - supervisor, - "_parakeet_bootstrap_thread", - supervisor.threading.current_thread(), - ) - supervisor._run_parakeet_bootstrap_worker(journal_path) + supervisor._run_parakeet_bootstrap_worker( + journal_path, + lease, + attempt_status, + ack, + ) - assert calls == [journal_path] + assert calls == [ + { + "journal_path": journal_path, + "lease": lease, + "attempt_status": attempt_status, + } + ] assert requests == ["start"] + assert ack.is_set() + assert lease.released is True -- 2.51.2