diff --git a/solstone/apps/thinking/install_copy.py b/solstone/apps/thinking/install_copy.py index f2afac701..b8d5fc126 100644 --- a/solstone/apps/thinking/install_copy.py +++ b/solstone/apps/thinking/install_copy.py @@ -12,7 +12,6 @@ INSTALL_PHASE_INSTALLED = "Installed" INSTALL_PHASE_FAILED_PREFIX = "Install failed — " INSTALL_FAILED_FALLBACK = "try again" -INSTALL_FAILED_NO_PROGRESS = "no progress for 60 seconds — try again" INSTALL_FAILED_UV_MISSING = ( "uv not found — install uv (https://github.com/astral-sh/uv) and retry" ) @@ -64,7 +63,6 @@ __all__ = [ "INSTALL_PHASE_INSTALLED", "INSTALL_PHASE_FAILED_PREFIX", "INSTALL_FAILED_FALLBACK", - "INSTALL_FAILED_NO_PROGRESS", "INSTALL_FAILED_UV_MISSING", "INSTALL_BUTTON_INSTALL", "INSTALL_BUTTON_INSTALLING", diff --git a/solstone/apps/thinking/local_bootstrap.py b/solstone/apps/thinking/local_bootstrap.py index 6579b30ed..e3dee82da 100644 --- a/solstone/apps/thinking/local_bootstrap.py +++ b/solstone/apps/thinking/local_bootstrap.py @@ -18,10 +18,15 @@ from solstone.think.callosum import callosum_send from solstone.think.models import LOCAL_MODEL, QWEN_35_9B from solstone.think.providers import local_install, mlx_install from solstone.think.providers.fit_report import FitReport +from solstone.think.providers.install_lease import ( + InstallLease, + acquire_install_lease, + probe_install_lease_free, +) from solstone.think.providers.install_state import ( IN_FLIGHT_STATES, InstallStatus, - make_idle_status, + begin_or_replace_install_attempt, read_install_status, transition_state, write_install_status, @@ -107,7 +112,9 @@ def list_local_models() -> list[dict[str, object]]: def check_binary_present() -> bool: """Return whether the pinned llama-server binary is installed.""" try: - return bool(local_install.inspect_readiness(LOCAL_MODEL)["binary_installed"]) + return bool( + local_install.inspect_readiness(LOCAL_MODEL).artifacts["binary_installed"] + ) except Exception: return False @@ -116,7 +123,9 @@ def check_model_present(model: str) -> bool: """Return whether the pinned GGUF model is installed.""" try: model_id = normalize_model_id(model) - return bool(local_install.inspect_readiness(model_id)["model_installed"]) + return bool( + local_install.inspect_readiness(model_id).artifacts["model_installed"] + ) except Exception: return False @@ -146,17 +155,17 @@ def get_availability_payload(model: str) -> dict[str, bool | float | int | str | min_ram_gb = MLX_AVAILABLE_FLOOR_BYTES // 1024**3 memory_blocked = memory_verdict.severity == "blocked" available = bool( - readiness["platform_supported"] - and readiness["package_available"] + readiness.host["platform_supported"] + and readiness.host["package_available"] and not memory_blocked - and readiness["model_installed"] + and readiness.artifacts["model_installed"] ) warning = ( LOCAL_MLX_MEMORY_WARNING_UNKNOWN if memory_verdict.severity == "warning" else "" ) - if not readiness["platform_supported"]: + if not readiness.host["platform_supported"]: reason = "requires Apple Silicon macOS" elif memory_blocked: assert memory_verdict.available_bytes is not None @@ -165,20 +174,20 @@ def get_availability_payload(model: str) -> dict[str, bool | float | int | str | f"(need {gb_label(memory_verdict.required_bytes)} GB available, " f"have {gb_label(memory_verdict.available_bytes)} GB available)" ) - elif not readiness["package_available"]: + elif not readiness.host["package_available"]: reason = "mlx-vlm runtime is not installed" - elif not readiness["model_installed"]: + elif not readiness.artifacts["model_installed"]: reason = "local model files are not installed" else: reason = "" return { - "model": readiness["model_id"], - "platform_supported": readiness["platform_supported"], + "model": readiness.target["model_id"], + "platform_supported": readiness.host["platform_supported"], "total_memory_gb": gb(total_memory_bytes), "available_memory_gb": gb(memory_verdict.available_bytes), "min_ram_gb": min_ram_gb, - "binary_present": readiness["package_available"], - "model_present": readiness["model_installed"], + "binary_present": readiness.host["package_available"], + "model_present": readiness.artifacts["model_installed"], "available": available, "reason": reason, "warning": warning, @@ -186,8 +195,9 @@ def get_availability_payload(model: str) -> dict[str, bool | float | int | str | } spec = LOCAL_MODEL_SPECS[model_id] - binary_present = check_binary_present() - model_present = check_model_present(model_id) + readiness = local_install.inspect_readiness(model_id) + binary_present = bool(readiness.artifacts["binary_installed"]) + model_present = bool(readiness.artifacts["model_installed"]) platform_supported, reason = _platform_supported() total_memory_gb = gb(read_total_bytes()) memory_verdict = assess_memory(spec.min_ram_bytes, block_below_floor=False) @@ -227,20 +237,14 @@ def get_availability_payload(model: str) -> dict[str, bool | float | int | str | def _read_status() -> InstallStatus: - return read_install_status(scope="bundled", name=local_install.LOCAL_PROVIDER_NAME) + return read_install_status(name=local_install.LOCAL_PROVIDER_NAME) def _write_status(status: InstallStatus) -> InstallStatus: - write_install_status(status, scope="bundled") + write_install_status(status) return status -def _has_live_thread(model: str) -> bool: - with _INSTALL_LOCK: - thread = _INSTALL_THREADS.get(model) - return thread is not None and thread.is_alive() - - def _payload_for_status( _model: str, status: InstallStatus ) -> dict[str, int | str | None]: @@ -263,15 +267,24 @@ def _payload_for_status( } -def _normalize_stalled_status(_model: str, status: InstallStatus) -> InstallStatus: - return status +def _payload_for_read_status( + model: str, + status: InstallStatus, +) -> dict[str, int | str | None]: + if status["install_state"] in IN_FLIGHT_STATES and probe_install_lease_free( + local_install.LOCAL_PROVIDER_NAME + ): + payload = _payload_for_status(model, status) + payload["install_state"] = "failed" + payload["install_error"] = "install_interrupted" + return payload + return _payload_for_status(model, status) def get_state(model: str) -> dict[str, int | str | None]: - """Return the serialized bootstrap state, applying stall detection.""" + """Return the serialized bootstrap state without mutating on-disk state.""" model_id = _resolve_model_id(model) - status = _normalize_stalled_status(model_id, _read_status()) - return _payload_for_status(model_id, status) + return _payload_for_read_status(model_id, _read_status()) def start_bootstrap(model: str) -> tuple[dict[str, str], int]: @@ -281,28 +294,57 @@ def start_bootstrap(model: str) -> tuple[dict[str, str], int]: raise LocalBootstrapUnavailableError("BYO local endpoint is active") model_id = _resolve_model_id(model) - get_state(model_id) - status = _read_status() - if status["install_state"] == "installed": + readiness = ( + mlx_install.inspect_readiness(model_id) + if _is_mlx_backend() + else local_install.inspect_readiness(model_id) + ) + if readiness.ready: return {"install_state": "installed"}, 200 + if readiness.status in {"proof-unavailable", "host-ineligible"}: + raise LocalBootstrapUnavailableError(readiness.reason_code) availability = get_availability_payload(model_id) installed = bool(availability["binary_present"] and availability["model_present"]) + fingerprint = ( + mlx_install.target_fingerprint(model_id) + if _is_mlx_backend() + else local_install.target_fingerprint(model_id) + ) + from solstone.think.providers.install_state import ( + canonical_fingerprint, + fingerprint_sha256, + ) + + target_sha = fingerprint_sha256(canonical_fingerprint(fingerprint)) + lease = acquire_install_lease(local_install.LOCAL_PROVIDER_NAME) + if lease is None: + status = _read_status() + if ( + status["install_state"] in IN_FLIGHT_STATES + and status["target_fingerprint_sha256"] == target_sha + ): + return {"install_state": status["install_state"]}, 200 + return { + "install_state": status["install_state"], + "reason_code": "install_busy", + }, 409 + with _INSTALL_LOCK: status = _read_status() - if status["install_state"] == "installed": + if readiness.ready: + lease.release() return {"install_state": "installed"}, 200 if status["install_state"] == "idle" and installed: - _write_status( - transition_state( - make_idle_status(local_install.LOCAL_PROVIDER_NAME), - new_state="installed", - ) - ) + lease.release() return {"install_state": "installed"}, 200 - if status["install_state"] in IN_FLIGHT_STATES: + if ( + status["install_state"] in IN_FLIGHT_STATES + and status["target_fingerprint_sha256"] == target_sha + ): + lease.release() return {"install_state": status["install_state"]}, 200 # Only genuinely-missing artifacts reach here: build the host-fit report @@ -311,27 +353,36 @@ def start_bootstrap(model: str) -> tuple[dict[str, str], int]: report = _fit_report_for_model(model_id) blocked_reason = _blocked_reason(report) if blocked_reason: + lease.release() raise LocalBootstrapUnavailableError(blocked_reason) disk_reason = _disk_blocked_reason(report) if disk_reason: + lease.release() raise LocalBootstrapUnavailableError(disk_reason) try: worker = ( _mlx_bootstrap_worker if _is_mlx_backend() else _run_bootstrap_worker ) + attempt_status = begin_or_replace_install_attempt( + local_install.LOCAL_PROVIDER_NAME, + fingerprint, + initial_state="downloading", + owner={"entry": "thinking_bootstrap"}, + ) + ack = threading.Event() thread = threading.Thread( target=worker, - args=(model_id,), + args=(model_id, lease, attempt_status, ack), name=f"local-provider-bootstrap-{model_id}", daemon=True, ) except Exception as exc: + lease.release() _write_status(transition_state(status, new_state="failed", error=str(exc))) raise LocalBootstrapStartError(str(exc)) from exc - _write_status(transition_state(status, new_state="downloading")) _INSTALL_THREADS[model_id] = thread try: @@ -340,10 +391,17 @@ def start_bootstrap(model: str) -> tuple[dict[str, str], int]: with _INSTALL_LOCK: if _INSTALL_THREADS.get(model_id) is thread: _INSTALL_THREADS.pop(model_id, None) + lease.release() _write_status( transition_state(_read_status(), new_state="failed", error=str(exc)) ) raise LocalBootstrapStartError(str(exc)) from exc + if not ack.wait(timeout=5.0): + with _INSTALL_LOCK: + if _INSTALL_THREADS.get(model_id) is thread: + _INSTALL_THREADS.pop(model_id, None) + lease.release() + raise LocalBootstrapStartError("local bootstrap worker did not acknowledge") return {"install_state": "downloading"}, 202 @@ -371,13 +429,24 @@ def _disk_blocked_reason(report: FitReport) -> str: return "" -def _mlx_bootstrap_worker(model: str) -> None: +def _mlx_bootstrap_worker( + model: str, + lease: InstallLease, + attempt_status: InstallStatus, + ack: threading.Event, +) -> None: current_thread = threading.current_thread() + ack.set() try: - mlx_install.install_local_mlx(model) + mlx_install.install_local_mlx( + model, + lease=lease, + attempt_status=attempt_status, + ) except Exception: logger.exception("local MLX provider bootstrap failed") finally: + lease.release() with _INSTALL_LOCK: if _INSTALL_THREADS.get(model) is current_thread: _INSTALL_THREADS.pop(model, None) @@ -392,12 +461,20 @@ def _request_local_server_start() -> None: logger.exception("could not request local server start") -def _run_bootstrap_worker(model: str) -> None: +def _run_bootstrap_worker( + model: str, + lease: InstallLease, + attempt_status: InstallStatus, + ack: threading.Event, +) -> None: current_thread = threading.current_thread() + ack.set() try: - local_install.install_llama_server() - _write_status(transition_state(_read_status(), new_state="downloading")) - local_install.install_model(model) + local_install.install_local( + model, + lease=lease, + attempt_status=attempt_status, + ) except Exception as exc: logger.exception("local provider bootstrap failed") _write_status( @@ -407,6 +484,7 @@ def _run_bootstrap_worker(model: str) -> None: logger.info("local provider bootstrap complete; requesting local server start") _request_local_server_start() finally: + lease.release() with _INSTALL_LOCK: if _INSTALL_THREADS.get(model) is current_thread: _INSTALL_THREADS.pop(model, None) diff --git a/solstone/apps/thinking/tests/test_local_bootstrap_routes.py b/solstone/apps/thinking/tests/test_local_bootstrap_routes.py index 49b2e2a37..59f344e50 100644 --- a/solstone/apps/thinking/tests/test_local_bootstrap_routes.py +++ b/solstone/apps/thinking/tests/test_local_bootstrap_routes.py @@ -15,7 +15,8 @@ import pytest from solstone.apps.thinking import local_bootstrap from solstone.convey import create_app from solstone.think.models import LOCAL_MODEL, QWEN_35_9B -from solstone.think.providers import fit_report, memory +from solstone.think.providers import fit_report, local_cuda, local_vulkan, memory +from solstone.think.providers.artifact_proof import ReadinessOutcome from solstone.think.providers.install_state import ( InstallState, InstallStatus, @@ -64,15 +65,26 @@ class _FakeThread: def __init__(self, *args, **kwargs): type(self).init_count += 1 type(self).targets.append(kwargs.get("target")) + self.args = kwargs.get("args", ()) self.alive = True def start(self): type(self).start_count += 1 + if self.args and hasattr(self.args[-1], "set"): + self.args[-1].set() def is_alive(self): return self.alive +class _FakeLease: + def __init__(self) -> None: + self.released = False + + def release(self) -> None: + self.released = True + + def _write_local_status( state: InstallState, *, @@ -86,7 +98,7 @@ def _write_local_status( ) status["last_transition_at"] = "2026-05-23T00:00:00+00:00" status["last_progress_at"] = last_progress_at - return write_install_status(status, scope="bundled") + return write_install_status(status) def _old_progress_iso() -> str: @@ -98,22 +110,108 @@ def _fresh_progress_iso() -> str: def _mlx_readiness(**overrides): - readiness = { - "install_state": "idle", - "model_installed": True, - "snapshot_installed": True, - "variant_installed": True, - "ram_sufficient": True, - "platform_supported": True, - "package_available": True, - "model_id": QWEN_35_9B, - "snapshot_dir": "/tmp/qwen-snapshot", - "variant_dir": None, - "runtime_dir": "/tmp/qwen-snapshot", - "install_error": None, - } - readiness.update(overrides) - return readiness + install_state = overrides.pop("install_state", "idle") + model_installed = overrides.pop("model_installed", True) + snapshot_installed = overrides.pop("snapshot_installed", model_installed) + variant_installed = overrides.pop("variant_installed", True) + platform_supported = overrides.pop("platform_supported", True) + package_available = overrides.pop("package_available", True) + status = ( + "ready" + if model_installed and platform_supported and package_available + else "missing-or-mismatched" + ) + return ReadinessOutcome( + provider="local", + status=status, + reason_code="ready" if status == "ready" else "manifest_missing", + target={"model_id": overrides.pop("model_id", QWEN_35_9B)}, + install={ + "install_state": install_state, + "install_error": overrides.pop("install_error", None), + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": overrides.pop("ram_sufficient", True), + "platform_supported": platform_supported, + "package_available": package_available, + }, + artifacts={ + "model_installed": model_installed, + "snapshot_installed": snapshot_installed, + "variant_installed": variant_installed, + "snapshot_dir": overrides.pop("snapshot_dir", "/tmp/qwen-snapshot"), + "variant_dir": overrides.pop("variant_dir", None), + "runtime_dir": overrides.pop("runtime_dir", "/tmp/qwen-snapshot"), + }, + proof={ + "snapshot": { + "status": "ready" if snapshot_installed else "missing-or-mismatched", + "reason_code": "ready" if snapshot_installed else "manifest_missing", + "cache_hit": False, + }, + "variant": { + "status": "ready" if variant_installed else "missing-or-mismatched", + "reason_code": "ready" if variant_installed else "manifest_missing", + "cache_hit": False, + }, + }, + ) + + +def _local_readiness(**overrides) -> ReadinessOutcome: + binary_installed = overrides.pop("binary_installed", True) + model_installed = overrides.pop("model_installed", True) + status = ( + "ready" if binary_installed and model_installed else "missing-or-mismatched" + ) + return ReadinessOutcome( + provider="local", + status=status, + reason_code="ready" if status == "ready" else "manifest_missing", + target={"model_id": overrides.pop("model_id", LOCAL_MODEL)}, + install={ + "install_state": overrides.pop("install_state", "idle"), + "install_error": overrides.pop("install_error", None), + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": overrides.pop("ram_sufficient", True), + "gpu_available": overrides.pop("gpu_available", True), + "gpu_probe_ok": overrides.pop("gpu_probe_ok", True), + "backend": "vulkan", + "backend_reason": "test vulkan", + }, + artifacts={ + "binary_installed": binary_installed, + "model_installed": model_installed, + "binary_path": overrides.pop("binary_path", "/tmp/llama-server"), + "model_path": overrides.pop("model_path", "/tmp/model.gguf"), + "mmproj_path": None, + }, + proof={ + "binary": { + "status": "ready" if binary_installed else "missing-or-mismatched", + "reason_code": "ready" if binary_installed else "manifest_missing", + "cache_hit": False, + }, + "model": { + "status": "ready" if model_installed else "missing-or-mismatched", + "reason_code": "ready" if model_installed else "manifest_missing", + "cache_hit": False, + }, + }, + ) def _fit( @@ -153,8 +251,11 @@ def test_local_bootstrap_linux_contract_for_model_helpers(): def test_local_availability_payload_exact_shape(settings_env, monkeypatch): journal_path, _config = settings_env(_settings_config()) - monkeypatch.setattr(local_bootstrap, "check_binary_present", lambda: True) - monkeypatch.setattr(local_bootstrap, "check_model_present", lambda _model: True) + monkeypatch.setattr( + local_bootstrap.local_install, + "inspect_readiness", + lambda _model: _local_readiness(), + ) monkeypatch.setattr(local_bootstrap, "_platform_supported", lambda: (True, "")) monkeypatch.setattr( memory.psutil, @@ -325,8 +426,11 @@ def test_local_availability_warns_but_does_not_block_on_low_memory( settings_env, monkeypatch ): journal_path, _config = settings_env(_settings_config()) - monkeypatch.setattr(local_bootstrap, "check_binary_present", lambda: True) - monkeypatch.setattr(local_bootstrap, "check_model_present", lambda _model: True) + monkeypatch.setattr( + local_bootstrap.local_install, + "inspect_readiness", + lambda _model: _local_readiness(), + ) monkeypatch.setattr(local_bootstrap, "_platform_supported", lambda: (True, "")) monkeypatch.setattr( memory.psutil, @@ -405,9 +509,9 @@ def test_local_bootstrap_post_rejects_byo_endpoint(settings_env): @pytest.mark.parametrize( ("state", "expected_payload", "expected_status"), [ - ("installed", {"install_state": "installed"}, 200), - ("downloading", {"install_state": "downloading"}, 200), - ("verifying", {"install_state": "verifying"}, 200), + ("installed", {"install_state": "downloading"}, 202), + ("downloading", {"install_state": "downloading"}, 202), + ("verifying", {"install_state": "downloading"}, 202), ("idle", {"install_state": "downloading"}, 202), ("failed", {"install_state": "downloading"}, 202), ], @@ -455,6 +559,59 @@ def test_start_bootstrap_payload_for_canonical_states( ) +def test_start_bootstrap_repairs_stale_installed_without_manifest( + settings_env, monkeypatch +): + settings_env(_settings_config()) + _write_local_status("installed") + binary = local_bootstrap.local_install.binary_path_for_pin() + binary.parent.mkdir(parents=True, exist_ok=True) + binary.write_bytes(b"stale binary") + binary.chmod(0o755) + spec = LOCAL_MODEL_SPECS[LOCAL_MODEL] + model = local_bootstrap.local_install.model_path(LOCAL_MODEL) + model.parent.mkdir(parents=True, exist_ok=True) + model.write_bytes(b"stale model") + if spec.mmproj_filename: + mmproj = local_bootstrap.local_install.mmproj_path(LOCAL_MODEL) + assert mmproj is not None + mmproj.write_bytes(b"stale projector") + monkeypatch.setattr( + local_cuda, + "resolve_local_backend", + lambda _pin: local_cuda.BackendChoice("vulkan", "test vulkan"), + ) + monkeypatch.setattr(local_vulkan, "detect_gpus", lambda: []) + monkeypatch.setattr(local_vulkan, "gpu_probe_ok", lambda: True) + monkeypatch.setattr( + memory.psutil, + "virtual_memory", + lambda: SimpleNamespace(available=32 * 1024**3, total=32 * 1024**3), + ) + monkeypatch.setattr( + memory.shutil, + "disk_usage", + lambda _path: SimpleNamespace(free=100 * 1024**3), + ) + monkeypatch.setattr( + local_bootstrap, "_fit_report_for_model", lambda _model: _fit("ok") + ) + _FakeThread.init_count = 0 + _FakeThread.start_count = 0 + monkeypatch.setattr(local_bootstrap.threading, "Thread", _FakeThread) + + assert local_bootstrap.start_bootstrap(LOCAL_MODEL) == ( + {"install_state": "downloading"}, + 202, + ) + + assert _FakeThread.start_count == 1 + status = read_install_status(name="local") + assert status["install_state"] == "downloading" + assert binary.read_bytes() == b"stale binary" + assert model.read_bytes() == b"stale model" + + def test_start_bootstrap_low_memory_warning_does_not_block(settings_env, monkeypatch): settings_env(_settings_config()) _write_local_status("idle") @@ -525,20 +682,25 @@ def test_start_bootstrap_insufficient_disk_blocks_before_worker( local_bootstrap.start_bootstrap(LOCAL_MODEL) assert _FakeThread.init_count == 0 - status = read_install_status(scope="bundled", name="local") + status = read_install_status(name="local") assert status["install_state"] == "idle" def test_local_bootstrap_status_returns_canonical_shape(settings_env): journal_path, _config = settings_env(_settings_config()) _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) - status = read_install_status(scope="bundled", name="local") + status = read_install_status(name="local") status["progress_bytes_received"] = 12 status["progress_bytes_total"] = 24 - write_install_status(status, scope="bundled") + write_install_status(status) client = _client(journal_path) + lease = local_bootstrap.acquire_install_lease("local") + assert lease is not None - response = client.get("/app/thinking/api/local/bootstrap/status") + try: + response = client.get("/app/thinking/api/local/bootstrap/status") + finally: + lease.release() assert response.status_code == 200 payload = response.get_json() @@ -562,20 +724,23 @@ def test_local_bootstrap_lazy_stall_without_live_thread_stays_read_only(settings payload = local_bootstrap.get_state(LOCAL_MODEL) - assert payload["install_state"] == "downloading" - assert payload["install_error"] is None - persisted = read_install_status(scope="bundled", name="local") + assert payload["install_state"] == "failed" + assert payload["install_error"] == "install_interrupted" + persisted = read_install_status(name="local") assert persisted["install_state"] == "downloading" assert persisted["install_error"] is None -def test_local_bootstrap_lazy_stall_with_live_thread_stays_in_flight(settings_env): +def test_local_bootstrap_in_flight_with_held_lease_stays_in_flight(settings_env): settings_env(_settings_config()) _write_local_status("verifying", last_progress_at=_old_progress_iso()) - with local_bootstrap._INSTALL_LOCK: - local_bootstrap._INSTALL_THREADS[LOCAL_MODEL] = _FakeThread() + lease = local_bootstrap.acquire_install_lease("local") + assert lease is not None - payload = local_bootstrap.get_state(LOCAL_MODEL) + try: + payload = local_bootstrap.get_state(LOCAL_MODEL) + finally: + lease.release() assert payload["install_state"] == "verifying" assert payload["install_error"] is None @@ -609,16 +774,19 @@ def test_mlx_availability_ignores_install_state_when_snapshot_missing( assert payload["reason"] == "local model files are not installed" -def test_mlx_bootstrap_lazy_stall_with_live_thread_stays_in_flight( +def test_mlx_bootstrap_in_flight_with_held_lease_stays_in_flight( settings_env, monkeypatch ): settings_env(_settings_config()) _write_local_status("downloading", last_progress_at=_old_progress_iso()) monkeypatch.setattr(local_bootstrap, "_is_mlx_backend", lambda: True) - with local_bootstrap._INSTALL_LOCK: - local_bootstrap._INSTALL_THREADS[QWEN_35_9B] = _FakeThread() + lease = local_bootstrap.acquire_install_lease("local") + assert lease is not None - payload = local_bootstrap.get_state(QWEN_35_9B) + try: + payload = local_bootstrap.get_state(QWEN_35_9B) + finally: + lease.release() assert payload["install_state"] == "downloading" assert payload["install_error"] is None @@ -633,8 +801,8 @@ def test_mlx_bootstrap_lazy_stall_without_live_thread_stays_read_only( payload = local_bootstrap.get_state(QWEN_35_9B) - assert payload["install_state"] == "downloading" - assert payload["install_error"] is None + assert payload["install_state"] == "failed" + assert payload["install_error"] == "install_interrupted" @pytest.mark.parametrize("state", ["installed", "failed"]) @@ -649,20 +817,14 @@ def test_local_bootstrap_restart_terminal_states_have_no_bytes(settings_env, sta assert payload["progress_bytes_total"] is None -def test_local_bootstrap_migrates_preexisting_install_without_worker( +def test_local_bootstrap_ready_short_circuit_does_not_publish_terminal( settings_env, monkeypatch ): settings_env(_settings_config()) monkeypatch.setattr( local_bootstrap.local_install, "inspect_readiness", - lambda _model=None: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "binary_path": "/tmp/llama-server", - "model_path": "/tmp/model.gguf", - }, + lambda _model=None: _local_readiness(), ) monkeypatch.setattr(local_bootstrap, "_platform_supported", lambda: (True, "")) monkeypatch.setattr( @@ -687,8 +849,8 @@ def test_local_bootstrap_migrates_preexisting_install_without_worker( {"install_state": "installed"}, 200, ) - status = read_install_status(scope="bundled", name="local") - assert status["install_state"] == "installed" + status = read_install_status(name="local") + assert status["install_state"] == "idle" def test_mlx_start_bootstrap_dispatches_to_mlx_worker(settings_env, monkeypatch): @@ -733,41 +895,45 @@ def test_mlx_start_bootstrap_dispatches_to_mlx_worker(settings_env, monkeypatch) assert _FakeThread.targets == [local_bootstrap._mlx_bootstrap_worker] -def test_local_worker_resets_progress_between_binary_and_model( +def test_local_worker_delegates_to_top_level_installer_with_lease_and_attempt( settings_env, monkeypatch ): settings_env(_settings_config()) + attempt_status = _write_local_status( + "downloading", last_progress_at=_fresh_progress_iso() + ) + lease = _FakeLease() + ack = threading.Event() observed = {} - _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) - def fake_llama_server(): - status = read_install_status(scope="bundled", name="local") - write_install_status( - transition_state(status, new_state="installed"), - scope="bundled", + def fake_install_local(model, *, lease, attempt_status): + observed.update( + { + "model": model, + "lease": lease, + "attempt_status": attempt_status, + "ack_set": ack.is_set(), + } ) - - def fake_install_model(model): - observed.update(local_bootstrap.get_state(model)) - status = read_install_status(scope="bundled", name="local") + status = read_install_status(name="local") write_install_status( transition_state(status, new_state="installed"), - scope="bundled", ) monkeypatch.setattr( - local_bootstrap.local_install, "install_llama_server", fake_llama_server - ) - monkeypatch.setattr( - local_bootstrap.local_install, "install_model", fake_install_model + local_bootstrap.local_install, "install_local", fake_install_local ) monkeypatch.setattr(local_bootstrap, "callosum_send", Mock(return_value=True)) - local_bootstrap._run_bootstrap_worker(LOCAL_MODEL) + local_bootstrap._run_bootstrap_worker(LOCAL_MODEL, lease, attempt_status, ack) - assert observed["install_state"] == "downloading" - assert observed["progress_bytes_total"] is None - assert observed["progress_bytes_received"] is None + assert observed == { + "model": LOCAL_MODEL, + "lease": lease, + "attempt_status": attempt_status, + "ack_set": True, + } + assert lease.released is True @pytest.mark.parametrize( @@ -778,14 +944,17 @@ def test_local_worker_success_requests_local_server_start( settings_env, monkeypatch, send_behavior ): settings_env(_settings_config()) - _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) + attempt_status = _write_local_status( + "downloading", last_progress_at=_fresh_progress_iso() + ) + lease = _FakeLease() + ack = threading.Event() - def fake_install_model(model): + def fake_install_local(model, *, lease, attempt_status): assert model == LOCAL_MODEL - status = read_install_status(scope="bundled", name="local") + status = read_install_status(name="local") write_install_status( transition_state(status, new_state="installed"), - scope="bundled", ) if send_behavior == "raise": @@ -794,17 +963,16 @@ def test_local_worker_success_requests_local_server_start( callosum_send = Mock(return_value=send_behavior == "success") monkeypatch.setattr( - local_bootstrap.local_install, "install_llama_server", lambda: None - ) - monkeypatch.setattr( - local_bootstrap.local_install, "install_model", fake_install_model + local_bootstrap.local_install, "install_local", fake_install_local ) monkeypatch.setattr(local_bootstrap, "callosum_send", callosum_send) - local_bootstrap._run_bootstrap_worker(LOCAL_MODEL) + local_bootstrap._run_bootstrap_worker(LOCAL_MODEL, lease, attempt_status, ack) callosum_send.assert_called_once_with("supervisor", "start_local") - status = read_install_status(scope="bundled", name="local") + assert ack.is_set() + assert lease.released is True + status = read_install_status(name="local") assert status["install_state"] == "installed" assert status["install_error"] is None @@ -813,23 +981,26 @@ def test_local_worker_install_model_failure_does_not_request_local_server_start( settings_env, monkeypatch ): settings_env(_settings_config()) - _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) + attempt_status = _write_local_status( + "downloading", last_progress_at=_fresh_progress_iso() + ) + lease = _FakeLease() + ack = threading.Event() callosum_send = Mock(return_value=True) - monkeypatch.setattr( - local_bootstrap.local_install, "install_llama_server", lambda: None - ) monkeypatch.setattr( local_bootstrap.local_install, - "install_model", + "install_local", Mock(side_effect=RuntimeError("model download broke")), ) monkeypatch.setattr(local_bootstrap, "callosum_send", callosum_send) - local_bootstrap._run_bootstrap_worker(LOCAL_MODEL) + local_bootstrap._run_bootstrap_worker(LOCAL_MODEL, lease, attempt_status, ack) callosum_send.assert_not_called() - status = read_install_status(scope="bundled", name="local") + assert ack.is_set() + assert lease.released is True + status = read_install_status(name="local") assert status["install_state"] == "failed" assert status["install_error"] == "model download broke" @@ -839,24 +1010,24 @@ def test_local_worker_cleans_registered_thread(settings_env, monkeypatch): current = threading.current_thread() with local_bootstrap._INSTALL_LOCK: local_bootstrap._INSTALL_THREADS[LOCAL_MODEL] = current - _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) + attempt_status = _write_local_status( + "downloading", last_progress_at=_fresh_progress_iso() + ) + lease = _FakeLease() + ack = threading.Event() - def fake_install_model(_model): - status = read_install_status(scope="bundled", name="local") + def fake_install_local(_model, *, lease, attempt_status): + status = read_install_status(name="local") write_install_status( transition_state(status, new_state="installed"), - scope="bundled", ) monkeypatch.setattr( - local_bootstrap.local_install, "install_llama_server", lambda: None - ) - monkeypatch.setattr( - local_bootstrap.local_install, "install_model", fake_install_model + local_bootstrap.local_install, "install_local", fake_install_local ) monkeypatch.setattr(local_bootstrap, "callosum_send", Mock(return_value=True)) - local_bootstrap._run_bootstrap_worker(LOCAL_MODEL) + local_bootstrap._run_bootstrap_worker(LOCAL_MODEL, lease, attempt_status, ack) with local_bootstrap._INSTALL_LOCK: assert LOCAL_MODEL not in local_bootstrap._INSTALL_THREADS @@ -867,20 +1038,25 @@ def test_local_worker_cleans_registered_thread_after_failure(settings_env, monke current = threading.current_thread() with local_bootstrap._INSTALL_LOCK: local_bootstrap._INSTALL_THREADS[LOCAL_MODEL] = current - _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) + attempt_status = _write_local_status( + "downloading", last_progress_at=_fresh_progress_iso() + ) + lease = _FakeLease() + ack = threading.Event() monkeypatch.setattr( local_bootstrap.local_install, - "install_llama_server", - lambda: (_ for _ in ()).throw(RuntimeError("binary download broke")), + "install_local", + Mock(side_effect=RuntimeError("binary download broke")), ) - local_bootstrap._run_bootstrap_worker(LOCAL_MODEL) + local_bootstrap._run_bootstrap_worker(LOCAL_MODEL, lease, attempt_status, ack) with local_bootstrap._INSTALL_LOCK: thread = local_bootstrap._INSTALL_THREADS.get(LOCAL_MODEL) assert thread is None or not thread.is_alive() - status = read_install_status(scope="bundled", name="local") + assert lease.released is True + status = read_install_status(name="local") assert status["install_state"] == "failed" assert status["install_error"] == "binary download broke" @@ -892,14 +1068,17 @@ def test_mlx_worker_preserves_install_error_and_cleans_thread( current = threading.current_thread() with local_bootstrap._INSTALL_LOCK: local_bootstrap._INSTALL_THREADS[QWEN_35_9B] = current - _write_local_status("downloading", last_progress_at=_fresh_progress_iso()) + attempt_status = _write_local_status( + "downloading", last_progress_at=_fresh_progress_iso() + ) + lease = _FakeLease() + ack = threading.Event() monkeypatch.setattr(local_bootstrap, "_is_mlx_backend", lambda: True) - def fake_install_mlx(_model): - status = read_install_status(scope="bundled", name="local") + def fake_install_mlx(_model, *, lease, attempt_status): + status = read_install_status(name="local") write_install_status( transition_state(status, new_state="failed", error="verify broke"), - scope="bundled", ) raise local_bootstrap.mlx_install.MLXVerificationError("verify broke") @@ -909,11 +1088,13 @@ def test_mlx_worker_preserves_install_error_and_cleans_thread( fake_install_mlx, ) - local_bootstrap._mlx_bootstrap_worker(QWEN_35_9B) + local_bootstrap._mlx_bootstrap_worker(QWEN_35_9B, lease, attempt_status, ack) with local_bootstrap._INSTALL_LOCK: assert QWEN_35_9B not in local_bootstrap._INSTALL_THREADS - status = read_install_status(scope="bundled", name="local") + assert ack.is_set() + assert lease.released is True + status = read_install_status(name="local") assert status["install_state"] == "failed" assert status["install_error"] == "verify broke" payload = local_bootstrap.get_state(QWEN_35_9B) diff --git a/solstone/apps/thinking/tests/test_providers_payload_extended.py b/solstone/apps/thinking/tests/test_providers_payload_extended.py index 3f8fdb24e..711122de9 100644 --- a/solstone/apps/thinking/tests/test_providers_payload_extended.py +++ b/solstone/apps/thinking/tests/test_providers_payload_extended.py @@ -13,6 +13,7 @@ from solstone.apps.thinking import routes from solstone.apps.thinking.local_bootstrap import LOCAL_MODEL_SPECS from solstone.apps.thinking.model_tiers import MODEL_TIERS from solstone.think.models import LOCAL_MODEL, NO_BRAIN_PROVIDER, resolve_provider +from solstone.think.providers.artifact_proof import ReadinessOutcome from solstone.think.providers.install_state import InstallState from solstone.think.providers.state import ProviderState @@ -65,6 +66,43 @@ def _assert_install_status(payload: dict) -> None: assert payload["install_state"] in CANONICAL_INSTALL_STATES +def _local_readiness_gpu_probe_failed() -> ReadinessOutcome: + return ReadinessOutcome( + provider="local", + status="ready", + reason_code="ready", + target={"model_id": LOCAL_MODEL}, + install={ + "install_state": "installed", + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": True, + "gpu_available": True, + "gpu_probe_ok": False, + "backend": "vulkan", + "backend_reason": "test vulkan", + }, + artifacts={ + "binary_installed": True, + "model_installed": True, + "binary_path": "/tmp/llama-server", + "model_path": "/tmp/model.gguf", + "model_id": LOCAL_MODEL, + }, + proof={ + "binary": {"status": "ready", "reason_code": "ready", "cache_hit": False}, + "model": {"status": "ready", "reason_code": "ready", "cache_hit": False}, + }, + ) + + def _patch_selected_providers(monkeypatch, *, provider: str = "google") -> None: monkeypatch.setattr( "solstone.think.models.resolve_provider", @@ -1158,18 +1196,7 @@ def test_get_providers_ai_readiness_surfaces_gpu_probe_failed_from_inspect( ): monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda _model=None: { - "install_state": "installed", - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": True, - "gpu_probe_ok": False, - "binary_path": "/tmp/llama-server", - "model_path": "/tmp/model.gguf", - "model_id": LOCAL_MODEL, - "install_error": None, - }, + lambda _model=None: _local_readiness_gpu_probe_failed(), ) monkeypatch.setattr( "solstone.think.providers.local_server.probe_state", diff --git a/solstone/apps/thinking/tests/test_thinking_install_poll_js.py b/solstone/apps/thinking/tests/test_thinking_install_poll_js.py index e9eb6f1d7..2af34e106 100644 --- a/solstone/apps/thinking/tests/test_thinking_install_poll_js.py +++ b/solstone/apps/thinking/tests/test_thinking_install_poll_js.py @@ -11,13 +11,13 @@ from pathlib import Path import pytest from solstone.apps.thinking import copy as thinking_copy -from solstone.apps.thinking.install_copy import INSTALL_FAILED_NO_PROGRESS from solstone.apps.thinking.tests.js_extract import ( extract_js_const, extract_js_function, ) STATIC = Path(__file__).resolve().parents[1] / "static" / "thinking.js" +SERVER_INSTALL_ERROR = "install_interrupted" def _node_script(body: str) -> str: @@ -35,7 +35,7 @@ def _node_script(body: str) -> str: extract_js_function(source, "startInstallPoll"), "function assert(condition, message) { if (!condition) throw new Error(message); }", f"const text = {json.dumps(thinking_copy.LOCAL_INSTALL)};", - f"const installFailedNoProgress = {json.dumps(INSTALL_FAILED_NO_PROGRESS)};", + f"const serverInstallError = {json.dumps(SERVER_INSTALL_ERROR)};", body, ] return "\n".join(parts) @@ -309,7 +309,7 @@ async function main() { install_state: 'failed', progress_bytes_received: null, progress_bytes_total: null, - install_error: installFailedNoProgress, + install_error: serverInstallError, }, ]; let fetchCalls = 0; @@ -342,7 +342,7 @@ async function main() { assert(result.install_state === 'failed', 'failed status should be terminal'); assert(fetchCalls === 2, 'poll should stop after failed'); assert(sleeps.length === 1, 'poll should sleep only before failed'); - assert(failed.message === installFailedNoProgress, 'server install_error should render verbatim'); + assert(failed.message === serverInstallError, 'server install_error should render verbatim'); assert(failed.bootstrap === true, 'failed install should offer retry'); assert(failed.bootstrapLabel === text.retry, 'retry label should come from copy'); assert(failed.tone === 'bad', 'failed install should render as an error'); diff --git a/solstone/convey/provider_readiness.py b/solstone/convey/provider_readiness.py index f532304e2..b9856a994 100644 --- a/solstone/convey/provider_readiness.py +++ b/solstone/convey/provider_readiness.py @@ -139,6 +139,12 @@ _ENTRIES: dict[str, _Entry] = { detail=_LOCAL_SETUP_DETAIL, recovery_action=_LOCAL_SETUP_ACTION, ), + "install_busy": _Entry( + klass="setup", + summary="local model setup is already running", + detail="A local provider install is already in progress. Try again shortly.", + recovery_action=None, + ), "local_model_installing": _Entry( klass="setup", summary="local model setup is finishing", diff --git a/solstone/convey/static/chat_reasons.js b/solstone/convey/static/chat_reasons.js index ffeaa97fd..2dabf10cf 100644 --- a/solstone/convey/static/chat_reasons.js +++ b/solstone/convey/static/chat_reasons.js @@ -42,6 +42,10 @@ "template": "local model setup is not finished", "action": {"label": "Open Local Model Setup", "href": "/app/thinking/#local-setup"} }, + "install_busy": { + "template": "local model setup is already running", + "action": null + }, "local_model_installing": { "template": "local model setup is finishing", "action": null diff --git a/solstone/think/install_models.py b/solstone/think/install_models.py index bafdfce4b..ddd175b61 100644 --- a/solstone/think/install_models.py +++ b/solstone/think/install_models.py @@ -386,12 +386,62 @@ def _install_rfdetr_model(*, check: bool, force: bool) -> int: def _install_linux_cpp(*, force: bool = False) -> int: from solstone.think.providers import parakeet_install + from solstone.think.providers.install_lease import acquire_install_lease + from solstone.think.providers.install_state import ( + IN_FLIGHT_STATES, + begin_or_replace_install_attempt, + canonical_fingerprint, + fingerprint_sha256, + observe_install_attempt, + read_install_status, + ) + + def progress_line(status: dict[str, Any]) -> None: + received = status.get("progress_bytes_received") + total = status.get("progress_bytes_total") + suffix = "" + if received is not None: + suffix = f" {received}" + if total is not None: + suffix += f"/{total}" + print(f"observing parakeet install: {status['install_state']}{suffix}") try: - if force: - parakeet_install.install_parakeet(force=True) + fingerprint = parakeet_install.target_fingerprint() + target_sha = fingerprint_sha256(canonical_fingerprint(fingerprint)) + lease = acquire_install_lease("parakeet") + if lease is None: + status = read_install_status(name="parakeet") + if ( + status["install_state"] not in IN_FLIGHT_STATES + or status["target_fingerprint_sha256"] != target_sha + ): + return _fail("parakeet install already running for a different target") + final = observe_install_attempt( + "parakeet", + target_fingerprint_sha256=target_sha, + timeout_s=60.0 * 60.0, + progress=progress_line, + ) + if final is None: + return _fail("timed out observing parakeet install") + if final["install_state"] != "installed": + return _fail(final.get("install_error") or "parakeet install failed") else: - parakeet_install.install_parakeet() + try: + attempt_status = begin_or_replace_install_attempt( + "parakeet", + fingerprint, + initial_state="resolving", + owner={"entry": "install_models"}, + ) + parakeet_install.install_parakeet( + force=force, + lease=lease, + attempt_status=attempt_status, + ) + finally: + lease.release() paths = _check_linux_cpp_ready() except Exception as exc: return _fail(f"parakeet install failed: {exc}") diff --git a/solstone/think/install_provider.py b/solstone/think/install_provider.py index eee38aa1d..ad381da0a 100644 --- a/solstone/think/install_provider.py +++ b/solstone/think/install_provider.py @@ -15,6 +15,15 @@ import sys from solstone.think.providers import local_install, parakeet_install from solstone.think.providers.fit_report import FitReport +from solstone.think.providers.install_lease import acquire_install_lease +from solstone.think.providers.install_state import ( + IN_FLIGHT_STATES, + begin_or_replace_install_attempt, + canonical_fingerprint, + fingerprint_sha256, + observe_install_attempt, + read_install_status, +) from solstone.think.utils import require_solstone PARAKEET_DOWNLOAD_DISCLOSURE = ( @@ -30,6 +39,48 @@ def _render_fit_report(report: FitReport) -> None: print(fit_report.render_fit_report(report), file=sys.stderr) +def _target_sha(fingerprint: dict) -> str: + return fingerprint_sha256(canonical_fingerprint(fingerprint)) + + +def _progress_line(status: dict) -> None: + received = status.get("progress_bytes_received") + total = status.get("progress_bytes_total") + progress = "" + if received is not None: + progress = f" {received}" + if total is not None: + progress += f"/{total}" + print( + f"observing {status['provider']} install: {status['install_state']}{progress}", + file=sys.stderr, + ) + + +def _observe_same_target(provider: str, target_sha: str) -> int: + status = read_install_status(name=provider) + if ( + status["install_state"] not in IN_FLIGHT_STATES + or status["target_fingerprint_sha256"] != target_sha + ): + print( + f"{provider} install already running for a different target", + file=sys.stderr, + ) + return 1 + final = observe_install_attempt( + provider, + target_fingerprint_sha256=target_sha, + timeout_s=60.0 * 60.0, + progress=_progress_line, + ) + if final is None: + print(f"timed out observing {provider} install", file=sys.stderr) + return 1 + print(json.dumps(final, indent=2)) + return 0 if final["install_state"] == "installed" else 1 + + def main() -> int: parser = argparse.ArgumentParser( prog="journal install-provider", @@ -50,34 +101,63 @@ def main() -> int: if args.name == "parakeet": print(PARAKEET_DOWNLOAD_DISCLOSURE, file=sys.stderr) readiness = parakeet_install.inspect_readiness() - installed = bool(readiness["binary_installed"] and readiness["model_installed"]) - if installed: + if readiness.ready: print("parakeet already installed", file=sys.stderr) - else: + print(json.dumps(read_install_status(name="parakeet"), indent=2)) + return 0 + fingerprint = parakeet_install.target_fingerprint() + target_sha = _target_sha(fingerprint) + lease = acquire_install_lease("parakeet") + if lease is None: + return _observe_same_target("parakeet", target_sha) + try: from solstone.think.providers import fit_report _render_fit_report(fit_report.build_parakeet_fit_report()) - try: - status = parakeet_install.install_parakeet() + attempt_status = begin_or_replace_install_attempt( + "parakeet", + fingerprint, + initial_state="resolving", + owner={"entry": "install_provider"}, + ) + status = parakeet_install.install_parakeet( + lease=lease, + attempt_status=attempt_status, + ) except parakeet_install.ParakeetProviderError as exc: print(str(exc), file=sys.stderr) return 1 + finally: + lease.release() print(json.dumps(status, indent=2)) return 0 readiness = local_install.inspect_readiness() - installed = bool(readiness["binary_installed"] and readiness["model_installed"]) - if installed: + if readiness.ready: print("local already installed", file=sys.stderr) - else: + print(json.dumps(read_install_status(name="local"), indent=2)) + return 0 + fingerprint = local_install.target_fingerprint() + target_sha = _target_sha(fingerprint) + lease = acquire_install_lease("local") + if lease is None: + return _observe_same_target("local", target_sha) + try: from solstone.think.providers import fit_report _render_fit_report(fit_report.build_local_fit_report(local_install.LOCAL_MODEL)) - try: - status = local_install.install_local() + attempt_status = begin_or_replace_install_attempt( + "local", + fingerprint, + initial_state="resolving", + owner={"entry": "install_provider"}, + ) + status = local_install.install_local(lease=lease, attempt_status=attempt_status) except local_install.LocalProviderError as exc: print(str(exc), file=sys.stderr) return 1 + finally: + lease.release() print(json.dumps(status, indent=2)) return 0 diff --git a/solstone/think/providers/artifact_proof.py b/solstone/think/providers/artifact_proof.py index 2ccf48cd9..8decc5cdb 100644 --- a/solstone/think/providers/artifact_proof.py +++ b/solstone/think/providers/artifact_proof.py @@ -51,6 +51,10 @@ class ReadinessOutcome: artifacts: dict[str, Any] proof: dict[str, Any] + @property + def ready(self) -> bool: + return self.status == "ready" + @dataclass(frozen=True) class ProofResult: diff --git a/solstone/think/providers/install_state.py b/solstone/think/providers/install_state.py index efe9d2c44..b89cc22b7 100644 --- a/solstone/think/providers/install_state.py +++ b/solstone/think/providers/install_state.py @@ -11,7 +11,7 @@ import time import uuid from datetime import datetime, timezone from pathlib import Path -from typing import Any, Literal, NotRequired, TypedDict, cast, get_args +from typing import Any, Callable, Literal, TypedDict, cast, get_args from solstone.think.journal_config import JournalConfigMutation, mutate_journal_config from solstone.think.journal_io.atomic import atomic_replace @@ -47,7 +47,6 @@ class InstallStatus(TypedDict): install_error: str | None error_code: str | None owner: dict[str, Any] | None - name: NotRequired[str] class InstallStateError(RuntimeError): @@ -71,7 +70,7 @@ IN_FLIGHT_STATES: frozenset[InstallState] = frozenset( TERMINAL_STATES: frozenset[InstallState] = frozenset({"idle", "installed", "failed"}) _INSTALL_STATES = frozenset(get_args(InstallState)) _STATUS_MODE = 0o600 -_LEGACY_STATUS_KEYS = frozenset( +_LEGACY_OPERATIONAL_KEYS = frozenset( { "install_state", "last_transition_at", @@ -79,6 +78,29 @@ _LEGACY_STATUS_KEYS = frozenset( "progress_bytes_received", "progress_bytes_total", "install_error", + "binary_artifact", + "binary_sha256", + "binary_path", + "model_id", + "model_path", + "model_sha256", + "mmproj_path", + "mmproj_sha256", + "mlx_model_id", + "mlx_revision", + "mlx_snapshot_dir", + "mlx_variant_dir", + "binary_artifact_cpu", + "binary_sha256_cpu", + "binary_path_cpu", + "binary_artifact_vulkan", + "binary_sha256_vulkan", + "binary_path_vulkan", + "model_repo", + "model_filename", + "model_revision", + "model_path", + "model_sha256", } ) _LAST_PROGRESS_WRITE_MONOTONIC: dict[tuple[str, str], float] = {} @@ -115,26 +137,24 @@ def provider_status_path( def make_idle_status(name: str) -> InstallStatus: provider = _validate_provider(name) - return _with_legacy_name( - { - "schema_version": SCHEMA_VERSION, - "provider": provider, - "revision": 0, - "install_state": "idle", - "attempt_id": None, - "target_fingerprint_json": None, - "target_fingerprint_sha256": None, - "started_at": None, - "last_transition_at": None, - "last_progress_at": None, - "completed_at": None, - "progress_bytes_received": None, - "progress_bytes_total": None, - "install_error": None, - "error_code": None, - "owner": None, - } - ) + return { + "schema_version": SCHEMA_VERSION, + "provider": provider, + "revision": 0, + "install_state": "idle", + "attempt_id": None, + "target_fingerprint_json": None, + "target_fingerprint_sha256": None, + "started_at": None, + "last_transition_at": None, + "last_progress_at": None, + "completed_at": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "install_error": None, + "error_code": None, + "owner": None, + } def begin_install_attempt( @@ -160,6 +180,91 @@ def begin_install_attempt( ) +def begin_or_replace_install_attempt( + provider: str, + fingerprint: dict[str, Any], + *, + initial_state: InstallState = "resolving", + owner: dict[str, Any] | None = None, + journal_path: str | Path | None = None, +) -> InstallStatus: + """Begin an attempt after the caller owns the provider lease.""" + current = read_install_status(name=provider, journal_path=journal_path) + if current["install_state"] in IN_FLIGHT_STATES: + record_interrupted_install( + provider, + attempt_id=str(current["attempt_id"]), + target_fingerprint_sha256=current["target_fingerprint_sha256"], + journal_path=journal_path, + ) + return begin_install_attempt( + provider, + fingerprint, + initial_state=initial_state, + owner=owner, + journal_path=journal_path, + ) + + +def assert_install_attempt_current( + status: InstallStatus, + *, + journal_path: str | Path | None = None, +) -> InstallStatus: + """Return current status if the same in-flight attempt still owns the slot.""" + attempt = _coerce_status(status) + current = read_install_status(name=attempt["provider"], journal_path=journal_path) + if current["install_state"] not in IN_FLIGHT_STATES: + raise InstallStatusConflictError("install attempt is no longer in-flight") + if current["attempt_id"] != attempt["attempt_id"]: + raise InstallStatusConflictError("install attempt id changed") + if current["target_fingerprint_sha256"] != attempt["target_fingerprint_sha256"]: + raise InstallStatusConflictError("install target fingerprint changed") + return current + + +def observe_install_attempt( + provider: str, + *, + target_fingerprint_sha256: str, + timeout_s: float, + poll_interval_s: float = 1.0, + progress_interval_s: float = 10.0, + progress: Callable[[InstallStatus], None] | None = None, + journal_path: str | Path | None = None, +) -> InstallStatus | None: + """Poll the status record for a same-target in-flight attempt to finish.""" + deadline = time.monotonic() + timeout_s + last_progress_emit = 0.0 + last_progress_key: tuple[Any, ...] | None = None + while True: + status = read_install_status(name=provider, journal_path=journal_path) + if status["target_fingerprint_sha256"] != target_fingerprint_sha256: + return status + if progress is not None: + progress_key = ( + status["install_state"], + status["progress_bytes_received"], + status["progress_bytes_total"], + status["install_error"], + status["error_code"], + ) + now = time.monotonic() + if ( + last_progress_key is None + or progress_key != last_progress_key + or now - last_progress_emit >= progress_interval_s + ): + progress(status) + last_progress_key = progress_key + last_progress_emit = now + if status["install_state"] in TERMINAL_STATES: + return status + if time.monotonic() >= deadline: + return None + time.sleep(poll_interval_s) + + def transition_state( status: InstallStatus, *, @@ -178,35 +283,33 @@ def transition_state( next_attempt_id = uuid.uuid4().hex is_terminal = new_state in TERMINAL_STATES - return _with_legacy_name( - { - "schema_version": SCHEMA_VERSION, - "provider": current["provider"], - "revision": current["revision"], - "install_state": new_state, - "attempt_id": None if new_state == "idle" else next_attempt_id, - "target_fingerprint_json": current["target_fingerprint_json"], - "target_fingerprint_sha256": current["target_fingerprint_sha256"], - "started_at": ( - timestamp - if current["install_state"] in TERMINAL_STATES - and new_state in IN_FLIGHT_STATES - else current["started_at"] - ), - "last_transition_at": timestamp, - "last_progress_at": timestamp if new_state in IN_FLIGHT_STATES else None, - "completed_at": timestamp if is_terminal and new_state != "idle" else None, - "progress_bytes_received": ( - None if is_terminal else current["progress_bytes_received"] - ), - "progress_bytes_total": ( - None if is_terminal else current["progress_bytes_total"] - ), - "install_error": error if new_state == "failed" else None, - "error_code": error_code if new_state == "failed" else None, - "owner": current["owner"], - } - ) + return { + "schema_version": SCHEMA_VERSION, + "provider": current["provider"], + "revision": current["revision"], + "install_state": new_state, + "attempt_id": None if new_state == "idle" else next_attempt_id, + "target_fingerprint_json": current["target_fingerprint_json"], + "target_fingerprint_sha256": current["target_fingerprint_sha256"], + "started_at": ( + timestamp + if current["install_state"] in TERMINAL_STATES + and new_state in IN_FLIGHT_STATES + else current["started_at"] + ), + "last_transition_at": timestamp, + "last_progress_at": timestamp if new_state in IN_FLIGHT_STATES else None, + "completed_at": timestamp if is_terminal and new_state != "idle" else None, + "progress_bytes_received": ( + None if is_terminal else current["progress_bytes_received"] + ), + "progress_bytes_total": None + if is_terminal + else current["progress_bytes_total"], + "install_error": error if new_state == "failed" else None, + "error_code": error_code if new_state == "failed" else None, + "owner": current["owner"], + } def bump_progress( @@ -218,32 +321,28 @@ def bump_progress( current = _coerce_status(status) if current["install_state"] not in IN_FLIGHT_STATES: raise ValueError("install progress can only be bumped for in-flight states") - return _with_legacy_name( - { - **current, - "last_progress_at": now_iso(), - "progress_bytes_received": ( - _nonnegative_int(received) - if received is not None - else current["progress_bytes_received"] - ), - "progress_bytes_total": ( - _nonnegative_int(total) - if total is not None - else current["progress_bytes_total"] - ), - } - ) + return { + **current, + "last_progress_at": now_iso(), + "progress_bytes_received": ( + _nonnegative_int(received) + if received is not None + else current["progress_bytes_received"] + ), + "progress_bytes_total": ( + _nonnegative_int(total) + if total is not None + else current["progress_bytes_total"] + ), + } def read_install_status( *, - scope: str = "bundled", name: str, journal_path: str | Path | None = None, ) -> InstallStatus: """Read provider install status; absent status is synthetic idle.""" - _validate_scope(scope) provider = _validate_provider(name) path = provider_status_path(provider, journal_path=journal_path) if not path.exists(): @@ -254,24 +353,22 @@ def read_install_status( raise InstallStatusMalformedError(f"malformed install status: {path}") from exc if not isinstance(data, dict): raise InstallStatusMalformedError(f"install status must be an object: {path}") - return _with_legacy_name(_coerce_status(data, provider=provider)) + return _coerce_status(data, provider=provider) def write_install_status( status: InstallStatus, *, - scope: str = "bundled", journal_path: str | Path | None = None, ) -> InstallStatus: """Write provider install status under a sidecar flock.""" - _validate_scope(scope) incoming = _coerce_status(status) path = provider_status_path(incoming["provider"], journal_path=journal_path) with hold_lock(path, mode=_STATUS_MODE): current = _read_current_unlocked(path, incoming["provider"]) accepted = _accept_transition(current, incoming) if accepted is current: - return _with_legacy_name(current) + return current stored = {**accepted, "revision": current["revision"] + 1} atomic_replace( path, @@ -279,7 +376,7 @@ def write_install_status( mode=_STATUS_MODE, ) _record_progress_write(stored) - return _with_legacy_name(stored) + return stored def record_interrupted_install( @@ -315,21 +412,35 @@ def migrate_legacy_provider_install_state( *, journal_path: str | Path | None = None, ) -> dict[str, int]: - """Remove legacy provider install status fields from journal config.""" + """Remove legacy provider install operational fields from journal config.""" def apply(config: dict[str, Any]) -> JournalConfigMutation[dict[str, int]]: removed = 0 + moved = 0 bundled = config.get("providers", {}).get("bundled") if isinstance(bundled, dict): + local_record = bundled.get("local") + if isinstance(local_record, dict) and "vulkan_device_index" in local_record: + providers = config.setdefault("providers", {}) + owner_config = providers.setdefault("local", {}) + if isinstance(owner_config, dict): + value = local_record.pop("vulkan_device_index") + if owner_config.get("vulkan_device_index") != value: + owner_config["vulkan_device_index"] = value + moved += 1 + removed += 1 for provider in PROVIDERS: record = bundled.get(provider) if not isinstance(record, dict): continue - for key in _LEGACY_STATUS_KEYS: + for key in _LEGACY_OPERATIONAL_KEYS: if key in record: record.pop(key, None) removed += 1 - return JournalConfigMutation(changed=removed > 0, value={"removed": removed}) + return JournalConfigMutation( + changed=removed > 0 or moved > 0, + value={"removed": removed, "moved": moved}, + ) return mutate_journal_config(apply, journal_path=journal_path).value @@ -343,7 +454,7 @@ def _read_current_unlocked(path: Path, provider: ProviderName) -> InstallStatus: raise InstallStatusMalformedError(f"malformed install status: {path}") from exc if not isinstance(data, dict): raise InstallStatusMalformedError(f"install status must be an object: {path}") - return _with_legacy_name(_coerce_status(data, provider=provider)) + return _coerce_status(data, provider=provider) def _accept_transition( @@ -429,7 +540,7 @@ def _coerce_status( *, provider: ProviderName | None = None, ) -> InstallStatus: - raw_provider = provider or data.get("provider") or data.get("name") + raw_provider = provider or data.get("provider") validated_provider = _validate_provider(raw_provider) state = data.get("install_state") if state not in _INSTALL_STATES: @@ -458,37 +569,28 @@ def _coerce_status( owner = data.get("owner") if owner is not None and not isinstance(owner, dict): raise InstallStatusMalformedError("owner must be an object or null") - return _with_legacy_name( - { - "schema_version": SCHEMA_VERSION, - "provider": validated_provider, - "revision": revision, - "install_state": cast(InstallState, state), - "attempt_id": attempt_id, - "target_fingerprint_json": fingerprint_json, - "target_fingerprint_sha256": fingerprint_sha, - "started_at": _optional_str(data.get("started_at")), - "last_transition_at": _optional_str(data.get("last_transition_at")), - "last_progress_at": _optional_str(data.get("last_progress_at")), - "completed_at": _optional_str(data.get("completed_at")), - "progress_bytes_received": _optional_int( - data.get("progress_bytes_received") - ), - "progress_bytes_total": _optional_int(data.get("progress_bytes_total")), - "install_error": _optional_str(data.get("install_error")), - "error_code": _optional_str(data.get("error_code")), - "owner": owner, - } - ) + return { + "schema_version": SCHEMA_VERSION, + "provider": validated_provider, + "revision": revision, + "install_state": cast(InstallState, state), + "attempt_id": attempt_id, + "target_fingerprint_json": fingerprint_json, + "target_fingerprint_sha256": fingerprint_sha, + "started_at": _optional_str(data.get("started_at")), + "last_transition_at": _optional_str(data.get("last_transition_at")), + "last_progress_at": _optional_str(data.get("last_progress_at")), + "completed_at": _optional_str(data.get("completed_at")), + "progress_bytes_received": _optional_int(data.get("progress_bytes_received")), + "progress_bytes_total": _optional_int(data.get("progress_bytes_total")), + "install_error": _optional_str(data.get("install_error")), + "error_code": _optional_str(data.get("error_code")), + "owner": owner, + } def _persistable_status(status: InstallStatus) -> dict[str, Any]: - return {key: value for key, value in status.items() if key != "name"} - - -def _with_legacy_name(status: dict[str, Any]) -> InstallStatus: - status["name"] = status["provider"] - return cast(InstallStatus, status) + return dict(status) def _optional_str(value: Any) -> str | None: @@ -520,11 +622,6 @@ def _validate_provider(value: object) -> ProviderName: return cast(ProviderName, value) -def _validate_scope(scope: str) -> None: - if scope != "bundled": - raise ValueError("install status scope must be 'bundled'") - - def _normalize_fingerprint_value(value: Any) -> Any: if isinstance(value, dict): return { @@ -559,13 +656,16 @@ __all__ = [ "ProviderName", "SCHEMA_VERSION", "TERMINAL_STATES", + "assert_install_attempt_current", "begin_install_attempt", + "begin_or_replace_install_attempt", "bump_progress", "canonical_fingerprint", "fingerprint_sha256", "make_idle_status", "migrate_legacy_provider_install_state", "now_iso", + "observe_install_attempt", "provider_status_path", "read_install_status", "record_interrupted_install", diff --git a/solstone/think/providers/local_install.py b/solstone/think/providers/local_install.py index b14634884..a851ea73f 100644 --- a/solstone/think/providers/local_install.py +++ b/solstone/think/providers/local_install.py @@ -3,34 +3,41 @@ """Install and inspect bundled local provider artifacts. -This module is the sole writer for ``providers.bundled.local`` install state. -It performs no network access at import time. +This module owns local provider artifact acquisition. It performs no network +access at import time. """ from __future__ import annotations import hashlib import logging -import os import platform import shutil import stat import sys -import time from dataclasses import dataclass from pathlib import Path from typing import Any, Callable -from solstone.think.journal_config import ( - JournalConfigMutation, - mutate_journal_config, - read_journal_config, -) +from solstone.think.journal_config import read_journal_config from solstone.think.models import LOCAL_MODEL +from solstone.think.providers.artifact_proof import ( + ReadinessOutcome, + artifact_manifest_path, + build_manifest, + prove_cuda_sidecar, + prove_manifest, + write_manifest, +) +from solstone.think.providers.install_lease import InstallLease, acquire_install_lease from solstone.think.providers.install_state import ( IN_FLIGHT_STATES, InstallStatus, + assert_install_attempt_current, + begin_or_replace_install_attempt, bump_progress, + canonical_fingerprint, + fingerprint_sha256, read_install_status, transition_state, write_install_status, @@ -51,20 +58,6 @@ from solstone.think.utils import get_journal LOG = logging.getLogger(__name__) LOCAL_PROVIDER_NAME = "local" _PROBE_TIMEOUT_SECONDS = 10 -_PROGRESS_MIN_INTERVAL_SECONDS = 1.0 # rate-limit durable install-progress writes to ~1/sec (download throughput fix) -_LOCAL_METADATA_KEYS = frozenset( - { - "binary_artifact", - "binary_sha256", - "binary_path", - "model_id", - "model_path", - "model_sha256", - "mmproj_path", - "mmproj_sha256", - "vulkan_device_index", - } -) @dataclass(frozen=True) @@ -280,36 +273,17 @@ def install_hint() -> str: def _read_local_status() -> InstallStatus: - return read_install_status(scope="bundled", name=LOCAL_PROVIDER_NAME) + return read_install_status(name=LOCAL_PROVIDER_NAME) def _write_local_status(status: InstallStatus) -> InstallStatus: - write_install_status(status, scope="bundled") + write_install_status(status) return status -def _write_local_metadata(updates: dict[str, str]) -> None: - unknown_keys = sorted(set(updates) - _LOCAL_METADATA_KEYS) - if unknown_keys: - raise ValueError(f"unknown local install metadata key: {unknown_keys[0]}") - - def apply(config: dict[str, Any]) -> JournalConfigMutation[None]: - slot = ( - config.setdefault("providers", {}) - .setdefault("bundled", {}) - .setdefault(LOCAL_PROVIDER_NAME, {}) - ) - changed = any(slot.get(key) != value for key, value in updates.items()) - for key, value in updates.items(): - slot[key] = value - return JournalConfigMutation(changed=changed, value=None) - - mutate_journal_config(apply) - - def gpu_device_override() -> int | None: config = read_journal_config() - record = config.get("providers", {}).get("bundled", {}).get(LOCAL_PROVIDER_NAME, {}) + record = config.get("providers", {}).get(LOCAL_PROVIDER_NAME, {}) if not isinstance(record, dict): return None value = record.get("vulkan_device_index") @@ -329,6 +303,148 @@ def _record_local_progress(received: int, total: int | None) -> None: _write_local_status(bump_progress(status, received=received, total=total)) +def _model_pin_identity(model_id: str) -> dict[str, Any]: + spec = LOCAL_MODEL_SPECS[normalize_model_id(model_id)] + return { + "unit": "local-model", + "model_id": spec.model_id, + "repo": spec.repo, + "revision": spec.revision, + "filename": spec.filename, + "sha256": spec.sha256, + "mmproj_filename": spec.mmproj_filename, + "mmproj_sha256": spec.mmproj_sha256, + } + + +def _vulkan_pin_identity( + artifact_key: str | None = None, + pin: dict[str, str] | None = None, +) -> dict[str, Any]: + artifact_key = artifact_key or llama_server_artifact_key() + pin = pin or pin_for_current_platform() + return { + "unit": "llama-server-vulkan", + "artifact_key": artifact_key, + "release_tag": pin["release_tag"], + "filename": pin["filename"], + "sha256": pin["sha256"], + "binary_name": pin["binary_name"], + } + + +def _cuda_pin_identity( + arch: str | None = None, + wanted_files: tuple[str, ...] | None = None, +) -> dict[str, Any]: + arch = arch or _oci_arch() + wanted_files = wanted_files or CUDA_SERVER_PIN.wanted_files_for_arch(arch) + return { + "unit": "llama-server-cuda", + "artifact_key": llama_server_artifact_key(), + "image_ref": CUDA_SERVER_PIN.image_ref, + "arch": arch, + "binary_name": CUDA_SERVER_PIN.binary_name, + "wanted_files": list(wanted_files), + } + + +def target_fingerprint(model_id: str = LOCAL_MODEL) -> dict[str, Any]: + from solstone.think.providers import local_cuda + + selected_model = normalize_model_id(model_id) + choice = local_cuda.resolve_local_backend(CUDA_SERVER_PIN) + runtime_pin = ( + _cuda_pin_identity() if choice.backend == "cuda" else _vulkan_pin_identity() + ) + return { + "provider": LOCAL_PROVIDER_NAME, + "runtime": "llama.cpp", + "backend": choice.backend, + "backend_reason": choice.reason, + "runtime_pin": runtime_pin, + "model_pin": _model_pin_identity(selected_model), + } + + +def _fingerprint_sha_for_target(fingerprint: dict[str, Any]) -> str: + return fingerprint_sha256(canonical_fingerprint(fingerprint)) + + +def _manifest_target_sha( + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any], +) -> str: + if attempt_status is not None and attempt_status["target_fingerprint_sha256"]: + return str(attempt_status["target_fingerprint_sha256"]) + return _fingerprint_sha_for_target(fingerprint) + + +def _manifest_entry(path: Path, root: Path, role: str) -> dict[str, Any]: + return { + "relative_path": path.relative_to(root).as_posix(), + "role": role, + "size": path.stat().st_size, + "sha256": _sha256_file(path), + } + + +def _runtime_inventory(root: Path, *, exclude_names: set[str]) -> list[dict[str, Any]]: + inventory: list[dict[str, Any]] = [] + for path in sorted(root.rglob("*")): + if not path.is_file() or path.name == artifact_manifest_path(root).name: + continue + if path.name in exclude_names: + continue + role = "runtime_binary" if path.name == "llama-server" else "runtime_support" + inventory.append(_manifest_entry(path, root, role)) + return inventory + + +def _write_vulkan_manifest( + *, + artifact_key: str, + pin: dict[str, str], + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any] | None = None, +) -> None: + install_dir = binary_install_dir(artifact_key, pin) + fingerprint = fingerprint or target_fingerprint() + manifest = build_manifest( + provider=LOCAL_PROVIDER_NAME, + unit="llama-server-vulkan", + target_fingerprint_sha256=_manifest_target_sha(attempt_status, fingerprint), + source={"pin_identity": _vulkan_pin_identity(artifact_key, pin)}, + inventory=_runtime_inventory(install_dir, exclude_names={pin["filename"]}), + attempt_id=attempt_status["attempt_id"] if attempt_status else None, + ) + write_manifest(artifact_manifest_path(install_dir), manifest) + + +def _write_model_manifest( + *, + model_id: str, + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any] | None = None, +) -> None: + spec = LOCAL_MODEL_SPECS[normalize_model_id(model_id)] + root = model_dir(spec.model_id) + inventory = [_manifest_entry(model_path(spec.model_id), root, "model")] + projector = mmproj_path(spec.model_id) + if projector is not None: + inventory.append(_manifest_entry(projector, root, "projector")) + fingerprint = fingerprint or target_fingerprint(spec.model_id) + manifest = build_manifest( + provider=LOCAL_PROVIDER_NAME, + unit="local-model", + target_fingerprint_sha256=_manifest_target_sha(attempt_status, fingerprint), + source={"pin_identity": _model_pin_identity(spec.model_id)}, + inventory=inventory, + attempt_id=attempt_status["attempt_id"] if attempt_status else None, + ) + write_manifest(artifact_manifest_path(root), manifest) + + def _sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: @@ -362,25 +478,14 @@ def _download_file( total_header = response.headers.get("content-length") total = int(total_header) if total_header and total_header.isdigit() else None received = 0 - last_emit = time.monotonic() - last_emitted_received = -1 - first_chunk = True with tmp.open("wb") as handle: for chunk in response.iter_bytes(): if not chunk: continue handle.write(chunk) received += len(chunk) - if on_progress is None: - continue - now = time.monotonic() - if first_chunk or now - last_emit >= _PROGRESS_MIN_INTERVAL_SECONDS: + if on_progress is not None: on_progress(received, total) - last_emit = now - last_emitted_received = received - first_chunk = False - if on_progress is not None and received != last_emitted_received: - on_progress(received, total) tmp.replace(dest) @@ -480,12 +585,16 @@ def probe_binary_runnable(binary_path: str | Path) -> tuple[bool, str | None]: return False, detail -def install_llama_server() -> dict[str, Any]: +def install_llama_server( + *, + attempt_status: InstallStatus | None = None, + fingerprint: dict[str, Any] | None = None, +) -> dict[str, Any]: from solstone.think.providers import local_cuda choice = local_cuda.resolve_local_backend(CUDA_SERVER_PIN) if choice.backend == "cuda": - return _install_cuda_llama_server() + return _install_cuda_llama_server(attempt_status=attempt_status) artifact_key = llama_server_artifact_key() pin = pin_for_current_platform() @@ -500,7 +609,6 @@ def install_llama_server() -> dict[str, Any]: _write_local_status( transition_state(_read_local_status(), new_state="downloading") ) - _write_local_metadata({"binary_artifact": pin["filename"]}) _download_file(url, tarball, on_progress=_record_local_progress) _write_local_status( transition_state(_read_local_status(), new_state="verifying") @@ -516,6 +624,8 @@ def install_llama_server() -> dict[str, Any]: _safe_extract_tarball(tarball, install_dir) extracted = _find_extracted_binary(install_dir, pin["binary_name"]) final_path = binary_path_for_pin(artifact_key, pin) + if attempt_status is not None: + assert_install_attempt_current(attempt_status) inner_dir = extracted.parent if inner_dir != install_dir: for item in inner_dir.iterdir(): @@ -523,16 +633,13 @@ def install_llama_server() -> dict[str, Any]: inner_dir.rmdir() _chmod_executable(final_path) _clear_macos_quarantine(install_dir) - _write_local_metadata( - { - "binary_artifact": pin["filename"], - "binary_sha256": pin["sha256"], - "binary_path": str(final_path), - } - ) - return _write_local_status( - transition_state(_read_local_status(), new_state="installed") + _write_vulkan_manifest( + artifact_key=artifact_key, + pin=pin, + attempt_status=attempt_status, + fingerprint=fingerprint, ) + return _read_local_status() except Exception as exc: _write_local_status( transition_state(_read_local_status(), new_state="failed", error=str(exc)) @@ -540,7 +647,10 @@ def install_llama_server() -> dict[str, Any]: raise -def _install_cuda_llama_server() -> dict[str, Any]: +def _install_cuda_llama_server( + *, + attempt_status: InstallStatus | None = None, +) -> dict[str, Any]: from solstone.think.providers import oci_image try: @@ -559,10 +669,10 @@ def _install_cuda_llama_server() -> dict[str, Any]: _write_local_status( transition_state(_read_local_status(), new_state="verifying") ) + if attempt_status is not None: + assert_install_attempt_current(attempt_status) _chmod_executable(cuda_binary_path()) - return _write_local_status( - transition_state(_read_local_status(), new_state="installed") - ) + return _read_local_status() except Exception as exc: _write_local_status( transition_state(_read_local_status(), new_state="failed", error=str(exc)) @@ -570,7 +680,12 @@ def _install_cuda_llama_server() -> dict[str, Any]: raise -def install_model(model_id: str = LOCAL_MODEL) -> dict[str, Any]: +def install_model( + model_id: str = LOCAL_MODEL, + *, + attempt_status: InstallStatus | None = None, + fingerprint: dict[str, Any] | None = None, +) -> dict[str, Any]: spec = LOCAL_MODEL_SPECS[normalize_model_id(model_id)] url = f"https://huggingface.co/{spec.repo}/resolve/{spec.revision}/{spec.filename}" dest = model_path(spec.model_id) @@ -580,7 +695,6 @@ def install_model(model_id: str = LOCAL_MODEL) -> dict[str, Any]: _write_local_status( transition_state(_read_local_status(), new_state="downloading") ) - _write_local_metadata({"model_id": spec.model_id}) _download_file(url, dest, on_progress=_record_local_progress) if spec.mmproj_filename and mmproj_dest is not None: mmproj_url = ( @@ -592,19 +706,16 @@ def install_model(model_id: str = LOCAL_MODEL) -> dict[str, Any]: transition_state(_read_local_status(), new_state="verifying") ) _verify_sha256(dest, spec.sha256) - metadata = { - "model_id": spec.model_id, - "model_path": str(dest), - "model_sha256": spec.sha256, - } if spec.mmproj_sha256 and mmproj_dest is not None: _verify_sha256(mmproj_dest, spec.mmproj_sha256) - metadata["mmproj_path"] = str(mmproj_dest) - metadata["mmproj_sha256"] = spec.mmproj_sha256 - _write_local_metadata(metadata) - return _write_local_status( - transition_state(_read_local_status(), new_state="installed") + if attempt_status is not None: + assert_install_attempt_current(attempt_status) + _write_model_manifest( + model_id=spec.model_id, + attempt_status=attempt_status, + fingerprint=fingerprint, ) + return _read_local_status() except Exception as exc: _write_local_status( transition_state(_read_local_status(), new_state="failed", error=str(exc)) @@ -612,94 +723,165 @@ def install_model(model_id: str = LOCAL_MODEL) -> dict[str, Any]: raise -def install_local(model_id: str = LOCAL_MODEL) -> dict[str, Any]: +def install_local( + model_id: str = LOCAL_MODEL, + *, + lease: InstallLease | None = None, + attempt_status: InstallStatus | None = None, +) -> dict[str, Any]: from solstone.think.providers import fit_report selected_model = normalize_model_id(model_id) - readiness = inspect_readiness(selected_model) - if readiness["binary_installed"] and readiness["model_installed"]: + fingerprint = target_fingerprint(selected_model) + owned_lease = lease is None + if lease is None: + lease = acquire_install_lease(LOCAL_PROVIDER_NAME) + if lease is None: + raise LocalProviderError( + "install_busy", "Local provider install is already running." + ) + + try: + if attempt_status is None: + attempt_status = begin_or_replace_install_attempt( + LOCAL_PROVIDER_NAME, + fingerprint, + initial_state="resolving", + owner={"entry": "install_local"}, + ) + readiness = inspect_readiness(selected_model) + if readiness.status in {"proof-unavailable", "host-ineligible"}: + current = assert_install_attempt_current(attempt_status) + return _write_local_status( + transition_state( + current, + new_state="failed", + error=readiness.reason_code, + error_code=readiness.reason_code, + ) + ) + + if not readiness.ready: + report = fit_report.build_local_fit_report(selected_model) + rendered = fit_report.render_fit_report(report) + if report.overall == "blocked": + raise LocalProviderError("host_unfit", rendered) + if report.overall == "warning": + LOG.warning("local provider host fit warning:\n%s", rendered) + + if readiness.proof["binary"]["status"] == "missing-or-mismatched": + install_llama_server( + attempt_status=attempt_status, + fingerprint=fingerprint, + ) + if readiness.proof["model"]["status"] == "missing-or-mismatched": + install_model( + selected_model, + attempt_status=attempt_status, + fingerprint=fingerprint, + ) + + final_readiness = inspect_readiness(selected_model) + current = assert_install_attempt_current(attempt_status) + if final_readiness.ready: + return _write_local_status(transition_state(current, new_state="installed")) return _write_local_status( - transition_state(_read_local_status(), new_state="installed") + transition_state( + current, + new_state="failed", + error=final_readiness.reason_code, + error_code=final_readiness.reason_code, + ) ) + except Exception as exc: + try: + current = assert_install_attempt_current(attempt_status) + _write_local_status( + transition_state( + current, + new_state="failed", + error=str(exc), + error_code=getattr(exc, "reason_code", None), + ) + ) + except Exception: + pass + raise + finally: + if owned_lease: + lease.release() + + +def _proof_payload( + status: str, reason_code: str, *, cache_hit: bool = False +) -> dict[str, Any]: + return {"status": status, "reason_code": reason_code, "cache_hit": cache_hit} + + +def _proof_result_payload(result: Any) -> dict[str, Any]: + return _proof_payload( + result.status, + result.reason_code, + cache_hit=bool(getattr(result, "cache_hit", False)), + ) - report = fit_report.build_local_fit_report(selected_model) - rendered = fit_report.render_fit_report(report) - if report.overall == "blocked": - raise LocalProviderError("host_unfit", rendered) - if report.overall == "warning": - LOG.warning("local provider host fit warning:\n%s", rendered) - install_llama_server() - return install_model(selected_model) +def _combined_artifact_status( + *proofs: dict[str, Any], +) -> tuple[str, str]: + for proof in proofs: + if proof["status"] == "proof-unavailable": + return "proof-unavailable", str(proof["reason_code"]) + for proof in proofs: + if proof["status"] == "missing-or-mismatched": + return "missing-or-mismatched", str(proof["reason_code"]) + return "ready", "ready" def inspect_artifacts(model_id: str | None = None) -> dict[str, Any]: from solstone.think.providers import oci_image - config = read_journal_config() - record = config.get("providers", {}).get("bundled", {}).get(LOCAL_PROVIDER_NAME, {}) - if not isinstance(record, dict): - record = {} - selected_model = normalize_model_id( - model_id or record.get("model_id") or LOCAL_MODEL - ) + selected_model = normalize_model_id(model_id or LOCAL_MODEL) spec = LOCAL_MODEL_SPECS[selected_model] - - # The persisted record is a cache keyed by model_id. Trust a recorded - # artifact path only when the record is for the selected model AND the path - # lives under that model's directory; otherwise it is stale (e.g. left by a - # prior model's install before a LOCAL_MODEL change) and must be ignored so - # we recompute from the spec. Never pair a recorded path from one model with - # a freshly-recomputed path from another — a mixed gguf/mmproj pair aborts - # llama-server at spawn with an n_embd text/projector mismatch. - expected_dir = model_dir(selected_model) - - def _trusted_record_path(value: str | None) -> Path | None: - if not value or record.get("model_id") != selected_model: - return None - candidate = Path(value) - return candidate if candidate.parent == expected_dir else None - - gguf_path = _trusted_record_path(record.get("model_path")) or model_path( - selected_model - ) - resolved_mmproj = _trusted_record_path(record.get("mmproj_path")) or mmproj_path( - selected_model - ) - mmproj_installed = resolved_mmproj is None or resolved_mmproj.exists() + gguf_path = model_path(selected_model) + resolved_mmproj = mmproj_path(selected_model) pin = pin_for_current_platform() vulkan_binary_path = binary_path_for_pin(pin=pin) - recorded_binary_path = record.get("binary_path") - vulkan_binary_installed = ( - record.get("binary_artifact") == pin["filename"] - and record.get("binary_sha256") == pin["sha256"] - and recorded_binary_path is not None - and Path(recorded_binary_path) == vulkan_binary_path - and vulkan_binary_path.exists() - and os.access(vulkan_binary_path, os.X_OK) + vulkan_proof = prove_manifest( + artifact_manifest_path(binary_install_dir(pin=pin)), + provider=LOCAL_PROVIDER_NAME, + pin_identity=_vulkan_pin_identity(pin=pin), ) + vulkan_payload = _proof_result_payload(vulkan_proof) arch = _oci_arch() cuda_binary = cuda_binary_path() - cuda_binary_installed = ( - oci_image.verify_sidecar_install( - CUDA_SERVER_PIN.image_ref, - arch, - CUDA_SERVER_PIN.wanted_files_for_arch(arch), - cuda_binary_dir(), - ) - and cuda_binary.exists() - and os.access(cuda_binary, os.X_OK) + cuda_wanted_files = CUDA_SERVER_PIN.wanted_files_for_arch(arch) + cuda_proof = prove_cuda_sidecar( + provider=LOCAL_PROVIDER_NAME, + image_ref=CUDA_SERVER_PIN.image_ref, + arch=arch, + wanted_files=cuda_wanted_files, + target_dir=cuda_binary_dir(), + pin_identity=_cuda_pin_identity(arch, cuda_wanted_files), + verifier=oci_image.verify_sidecar_install, ) + cuda_payload = _proof_result_payload(cuda_proof) + model_proof = prove_manifest( + artifact_manifest_path(model_dir(selected_model)), + provider=LOCAL_PROVIDER_NAME, + pin_identity=_model_pin_identity(selected_model), + ) + model_payload = _proof_result_payload(model_proof) return { - "binary_installed": vulkan_binary_installed or cuda_binary_installed, - "model_installed": gguf_path.exists() and mmproj_installed, - "gguf_installed": gguf_path.exists(), - "mmproj_installed": mmproj_installed, - "vulkan_binary_installed": vulkan_binary_installed, - "cuda_binary_installed": cuda_binary_installed, + "binary_installed": vulkan_proof.ready or cuda_proof.ready, + "model_installed": model_proof.ready, + "gguf_installed": model_proof.ready, + "mmproj_installed": model_proof.ready, + "vulkan_binary_installed": vulkan_proof.ready, + "cuda_binary_installed": cuda_proof.ready, "vulkan_binary_path": str(vulkan_binary_path), "cuda_binary_path": str(cuda_binary), "binary_path": str(vulkan_binary_path), @@ -707,10 +889,13 @@ def inspect_artifacts(model_id: str | None = None) -> dict[str, Any]: "mmproj_path": str(resolved_mmproj) if resolved_mmproj is not None else None, "model_id": selected_model, "min_ram_bytes": spec.min_ram_bytes, + "vulkan_proof": vulkan_payload, + "cuda_proof": cuda_payload, + "model_proof": model_payload, } -def inspect_readiness(model_id: str | None = None) -> dict[str, Any]: +def inspect_readiness(model_id: str | None = None) -> ReadinessOutcome: from solstone.think.providers import local_cuda choice = local_cuda.resolve_local_backend(CUDA_SERVER_PIN) @@ -736,41 +921,88 @@ def inspect_readiness(model_id: str | None = None) -> dict[str, Any]: gpu_available = selected_gpu is not None gpu_probe_ok = local_vulkan.gpu_probe_ok() - return { - "install_state": status["install_state"], - "binary_installed": binary_installed, - "model_installed": artifacts["model_installed"], - "gguf_installed": artifacts["gguf_installed"], - "mmproj_installed": artifacts["mmproj_installed"], - "ram_sufficient": memory_verdict.severity != "blocked", - "gpu_available": gpu_available, - "gpu_probe_ok": gpu_probe_ok, - "binary_path": str(binary_path), - "model_path": artifacts["model_path"], - "mmproj_path": artifacts["mmproj_path"], - "model_id": artifacts["model_id"], - "install_error": status["install_error"], - "backend": choice.backend, - "backend_reason": choice.reason, - } + binary_proof = ( + artifacts["cuda_proof"] + if choice.backend == "cuda" + else artifacts["vulkan_proof"] + ) + artifact_status, artifact_reason = _combined_artifact_status( + binary_proof, + artifacts["model_proof"], + ) + ram_sufficient = memory_verdict.severity != "blocked" + if artifact_status != "ready": + readiness_status = artifact_status + reason_code = artifact_reason + elif not ram_sufficient: + readiness_status = "host-ineligible" + reason_code = "ram_insufficient" + elif not gpu_probe_ok: + readiness_status = "host-ineligible" + reason_code = "gpu_probe_failed" + elif not gpu_available: + readiness_status = "host-ineligible" + reason_code = "gpu_unavailable" + else: + readiness_status = "ready" + reason_code = "ready" + + return ReadinessOutcome( + provider=LOCAL_PROVIDER_NAME, + status=readiness_status, # type: ignore[arg-type] + reason_code=reason_code, + target={ + "model_id": artifacts["model_id"], + "target_fingerprint_json": status["target_fingerprint_json"], + "target_fingerprint_sha256": status["target_fingerprint_sha256"], + }, + install={ + "install_state": status["install_state"], + "install_error": status["install_error"], + "error_code": status["error_code"], + "attempt_id": status["attempt_id"], + "progress_bytes_received": status["progress_bytes_received"], + "progress_bytes_total": status["progress_bytes_total"], + "last_transition_at": status["last_transition_at"], + "last_progress_at": status["last_progress_at"], + }, + host={ + "ram_sufficient": ram_sufficient, + "gpu_available": gpu_available, + "gpu_probe_ok": gpu_probe_ok, + "backend": choice.backend, + "backend_reason": choice.reason, + }, + artifacts={ + **artifacts, + "binary_installed": binary_installed, + "binary_path": str(binary_path), + }, + proof={ + "binary": binary_proof, + "model": artifacts["model_proof"], + "vulkan": artifacts["vulkan_proof"], + "cuda": artifacts["cuda_proof"], + }, + ) def ensure_artifacts_installed(model_id: str) -> LocalArtifacts: selected_model = normalize_model_id(model_id) readiness = inspect_readiness(selected_model) - if not readiness["binary_installed"]: + if not readiness.artifacts["binary_installed"]: raise LocalProviderError("binary_missing", "Local runtime is not installed.") - if not readiness["model_installed"]: + if not readiness.artifacts["model_installed"]: raise LocalProviderError( "model_missing", "Local model files are not installed." ) - mmproj = readiness.get("mmproj_path") + mmproj = readiness.artifacts.get("mmproj_path") return LocalArtifacts( - backend=str(readiness["backend"]), - backend_reason=str(readiness["backend_reason"]), - binary_path=Path(readiness["binary_path"]), - lib_dir=cuda_binary_dir() if readiness["backend"] == "cuda" else None, - gguf_path=Path(readiness["model_path"]), + backend=str(readiness.host["backend"]), + backend_reason=str(readiness.host["backend_reason"]), + binary_path=Path(readiness.artifacts["binary_path"]), + lib_dir=cuda_binary_dir() if readiness.host["backend"] == "cuda" else None, + gguf_path=Path(readiness.artifacts["model_path"]), mmproj_path=Path(mmproj) if mmproj else None, ) @@ -795,4 +1027,5 @@ __all__ = [ "gpu_device_override", "inspect_readiness", "ensure_artifacts_installed", + "target_fingerprint", ] diff --git a/solstone/think/providers/mlx_install.py b/solstone/think/providers/mlx_install.py index dcaa1a7f9..7c7ee43bb 100644 --- a/solstone/think/providers/mlx_install.py +++ b/solstone/think/providers/mlx_install.py @@ -16,14 +16,22 @@ from dataclasses import dataclass from pathlib import Path from typing import Any -from solstone.think.journal_config import ( - JournalConfigMutation, - mutate_journal_config, - read_journal_config, -) from solstone.think.models import GEMMA4_26B_A4B_4BIT, QWEN_35_9B +from solstone.think.providers.artifact_proof import ( + ReadinessOutcome, + build_manifest, + mlx_snapshot_manifest_path, + mlx_variant_manifest_path, + prove_manifest, + write_manifest, +) +from solstone.think.providers.install_lease import InstallLease, acquire_install_lease from solstone.think.providers.install_state import ( InstallStatus, + assert_install_attempt_current, + begin_or_replace_install_attempt, + canonical_fingerprint, + fingerprint_sha256, read_install_status, transition_state, write_install_status, @@ -39,14 +47,6 @@ _GEMMA4_MIN_POSITION_EMBEDDING_SIZE = 10240 _LOCAL_NAME = "local" _HASH_CHUNK_SIZE = 1024 * 1024 _REWRITTEN_VARIANT_FILES = frozenset({"config.json", "processor_config.json"}) -_MLX_METADATA_KEYS = frozenset( - { - "mlx_model_id", - "mlx_revision", - "mlx_snapshot_dir", - "mlx_variant_dir", - } -) @dataclass(frozen=True) @@ -84,11 +84,11 @@ class MLXVerificationError(RuntimeError): def _read_status() -> InstallStatus: - return read_install_status(scope="bundled", name=_LOCAL_NAME) + return read_install_status(name=_LOCAL_NAME) def _write_status(status: InstallStatus) -> InstallStatus: - write_install_status(status, scope="bundled") + write_install_status(status) return status @@ -128,6 +128,40 @@ def resolve_model_spec(model_id: str | None = None) -> MLXModelSpec: return spec +def _pin_identity(spec: MLXModelSpec) -> dict[str, Any]: + return { + "unit": "mlx-snapshot", + "model_id": spec.name, + "repo": spec.repo, + "revision": spec.revision, + "soft_token_budget": ( + MLX_SOFT_TOKEN_BUDGET if spec.name == GEMMA4_26B_A4B_4BIT else None + ), + } + + +def target_fingerprint(model_id: str | None = None) -> dict[str, Any]: + spec = resolve_model_spec(model_id) + return { + "provider": _LOCAL_NAME, + "runtime": "mlx", + "model_pin": _pin_identity(spec), + } + + +def _fingerprint_sha_for_target(fingerprint: dict[str, Any]) -> str: + return fingerprint_sha256(canonical_fingerprint(fingerprint)) + + +def _manifest_target_sha( + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any], +) -> str: + if attempt_status is not None and attempt_status["target_fingerprint_sha256"]: + return str(attempt_status["target_fingerprint_sha256"]) + return _fingerprint_sha_for_target(fingerprint) + + def snapshot_dir_for_spec(spec: MLXModelSpec) -> Path: from huggingface_hub import constants from huggingface_hub.file_download import repo_folder_name @@ -194,7 +228,9 @@ def _remote_safetensors_metadata( return found -def validate_snapshot_sha256(spec: MLXModelSpec, snapshot_dir: Path) -> None: +def validate_snapshot_sha256( + spec: MLXModelSpec, snapshot_dir: Path +) -> dict[str, tuple[str, int]]: safetensors_paths = _safetensors_paths(snapshot_dir) metadata = _remote_safetensors_metadata(spec, safetensors_paths) @@ -208,6 +244,7 @@ def validate_snapshot_sha256(spec: MLXModelSpec, snapshot_dir: Path) -> None: actual_sha = digest.hexdigest() if actual_sha != expected_sha: raise MLXVerificationError(f"sha256 mismatch for {rel_path}") + return metadata def _read_json(path: Path) -> dict[str, Any]: @@ -218,6 +255,90 @@ def _write_json(path: Path, data: dict[str, Any]) -> None: path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") +def _variant_pin_identity(spec: MLXModelSpec) -> dict[str, Any]: + identity = dict(_pin_identity(spec)) + identity["unit"] = "mlx-variant" + identity["variant"] = f"solstone-budget{MLX_SOFT_TOKEN_BUDGET}" + return identity + + +def _manifest_inventory_for_tree( + root: Path, + *, + role_prefix: str, + known_hashes: dict[str, tuple[str, int]] | None = None, +) -> list[dict[str, Any]]: + inventory: list[dict[str, Any]] = [] + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + rel_path = path.relative_to(root).as_posix() + known = known_hashes.get(rel_path) if known_hashes is not None else None + if known is None: + digest = _sha256_file(path) + size = path.stat().st_size + else: + digest, size = known + inventory.append( + { + "relative_path": rel_path, + "role": role_prefix, + "size": size, + "sha256": digest, + } + ) + return inventory + + +def _write_snapshot_manifest( + spec: MLXModelSpec, + snapshot_dir: Path, + *, + metadata: dict[str, tuple[str, int]], + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any], +) -> None: + manifest = build_manifest( + provider=_LOCAL_NAME, + unit="mlx-snapshot", + target_fingerprint_sha256=_manifest_target_sha(attempt_status, fingerprint), + source={"pin_identity": _pin_identity(spec)}, + inventory=_manifest_inventory_for_tree( + snapshot_dir, role_prefix="snapshot_file", known_hashes=metadata + ), + external_root=snapshot_dir, + attempt_id=attempt_status["attempt_id"] if attempt_status else None, + ) + write_manifest(mlx_snapshot_manifest_path(spec.repo, spec.revision), manifest) + + +def _write_variant_manifest( + spec: MLXModelSpec, + variant_dir: Path, + *, + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any], +) -> None: + manifest = build_manifest( + provider=_LOCAL_NAME, + unit="mlx-variant", + target_fingerprint_sha256=_manifest_target_sha(attempt_status, fingerprint), + source={"pin_identity": _variant_pin_identity(spec)}, + inventory=_manifest_inventory_for_tree(variant_dir, role_prefix="variant_file"), + external_root=variant_dir, + attempt_id=attempt_status["attempt_id"] if attempt_status else None, + ) + write_manifest(mlx_variant_manifest_path(spec.repo, spec.revision), manifest) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(_HASH_CHUNK_SIZE), b""): + digest.update(chunk) + return digest.hexdigest() + + def _validate_gemma4_position_embedding(config: dict[str, Any]) -> None: vision_config = config.get("vision_config") if not isinstance(vision_config, dict): @@ -347,43 +468,8 @@ def _artifact_presence(spec: MLXModelSpec) -> dict[str, Any]: } -def _write_mlx_metadata( - spec: MLXModelSpec, - *, - snapshot_dir: Path, - variant_dir: Path | None, -) -> None: - def apply(config: dict[str, Any]) -> JournalConfigMutation[None]: - slot = ( - config.setdefault("providers", {}) - .setdefault("bundled", {}) - .setdefault(_LOCAL_NAME, {}) - ) - next_values = { - "mlx_model_id": spec.name, - "mlx_revision": spec.revision, - "mlx_snapshot_dir": str(snapshot_dir), - } - if variant_dir is not None: - next_values["mlx_variant_dir"] = str(variant_dir) - changed = any( - slot.get(key) != next_values.get(key) for key in _MLX_METADATA_KEYS - ) - for key in _MLX_METADATA_KEYS: - slot.pop(key, None) - slot.update(next_values) - return JournalConfigMutation(changed=changed, value=None) - - mutate_journal_config(apply) - - def inspect_artifacts(model_id: str | None = None) -> dict[str, Any]: - config = read_journal_config() - record = config.get("providers", {}).get("bundled", {}).get(_LOCAL_NAME, {}) - if not isinstance(record, dict): - record = {} - selected_model = model_id or record.get("mlx_model_id") or QWEN_35_9B - spec = resolve_model_spec(str(selected_model)) + spec = resolve_model_spec(model_id) presence = _artifact_presence(spec) return { "model_installed": presence["model_installed"], @@ -400,75 +486,215 @@ def inspect_artifacts(model_id: str | None = None) -> dict[str, Any]: } -def inspect_readiness(model_id: str | None = None) -> dict[str, Any]: +def inspect_readiness(model_id: str | None = None) -> ReadinessOutcome: status = _read_status() artifacts = inspect_artifacts(model_id) + spec = resolve_model_spec(str(artifacts["model_id"])) verdict = assess_memory(MLX_AVAILABLE_FLOOR_BYTES, block_below_floor=True) - return { - "install_state": status["install_state"], - "model_installed": artifacts["model_installed"], - "snapshot_installed": artifacts["snapshot_installed"], - "variant_installed": artifacts["variant_installed"], - "ram_sufficient": verdict.severity != "blocked", - "platform_supported": is_mlx_platform_supported(), - "package_available": _check_platform_and_package()[0], - "model_id": artifacts["model_id"], - "snapshot_dir": artifacts["snapshot_dir"], - "variant_dir": artifacts["variant_dir"], - "runtime_dir": artifacts["runtime_dir"], - "install_error": status["install_error"], - } + snapshot_proof = prove_manifest( + mlx_snapshot_manifest_path(spec.repo, spec.revision), + provider=_LOCAL_NAME, + pin_identity=_pin_identity(spec), + ) + if spec.name == GEMMA4_26B_A4B_4BIT: + variant_proof = prove_manifest( + mlx_variant_manifest_path(spec.repo, spec.revision), + provider=_LOCAL_NAME, + pin_identity=_variant_pin_identity(spec), + ) + else: + variant_proof = None + platform_supported = is_mlx_platform_supported() + package_available = _check_platform_and_package()[0] + ram_sufficient = verdict.severity != "blocked" + proof_payloads = [ + { + "status": snapshot_proof.status, + "reason_code": snapshot_proof.reason_code, + "cache_hit": snapshot_proof.cache_hit, + } + ] + if variant_proof is not None: + proof_payloads.append( + { + "status": variant_proof.status, + "reason_code": variant_proof.reason_code, + "cache_hit": variant_proof.cache_hit, + } + ) + if any(proof["status"] == "proof-unavailable" for proof in proof_payloads): + readiness_status = "proof-unavailable" + reason_code = next( + str(proof["reason_code"]) + for proof in proof_payloads + if proof["status"] == "proof-unavailable" + ) + elif any(proof["status"] == "missing-or-mismatched" for proof in proof_payloads): + readiness_status = "missing-or-mismatched" + reason_code = next( + str(proof["reason_code"]) + for proof in proof_payloads + if proof["status"] == "missing-or-mismatched" + ) + elif not platform_supported: + readiness_status = "host-ineligible" + reason_code = "platform_unsupported" + elif not package_available: + readiness_status = "host-ineligible" + reason_code = "package_unavailable" + elif not ram_sufficient: + readiness_status = "host-ineligible" + reason_code = "ram_insufficient" + else: + readiness_status = "ready" + reason_code = "ready" + + model_installed = snapshot_proof.ready and ( + variant_proof is None or variant_proof.ready + ) + return ReadinessOutcome( + provider=_LOCAL_NAME, + status=readiness_status, # type: ignore[arg-type] + reason_code=reason_code, + target={ + "model_id": artifacts["model_id"], + "target_fingerprint_json": status["target_fingerprint_json"], + "target_fingerprint_sha256": status["target_fingerprint_sha256"], + }, + install={ + "install_state": status["install_state"], + "install_error": status["install_error"], + "error_code": status["error_code"], + "attempt_id": status["attempt_id"], + "progress_bytes_received": status["progress_bytes_received"], + "progress_bytes_total": status["progress_bytes_total"], + "last_transition_at": status["last_transition_at"], + "last_progress_at": status["last_progress_at"], + }, + host={ + "ram_sufficient": ram_sufficient, + "platform_supported": platform_supported, + "package_available": package_available, + }, + artifacts={ + **artifacts, + "model_installed": model_installed, + "snapshot_installed": snapshot_proof.ready, + "variant_installed": variant_proof is None or variant_proof.ready, + }, + proof={ + "snapshot": proof_payloads[0], + "variant": proof_payloads[1] if len(proof_payloads) > 1 else None, + }, + ) -def install_local_mlx(model_id: str = QWEN_35_9B) -> InstallStatus: +def install_local_mlx( + model_id: str = QWEN_35_9B, + *, + lease: InstallLease | None = None, + attempt_status: InstallStatus | None = None, +) -> InstallStatus: import huggingface_hub + fingerprint = target_fingerprint(model_id) + owned_lease = lease is None + if lease is None: + lease = acquire_install_lease(_LOCAL_NAME) + if lease is None: + raise MLXInstallUnavailableError( + "Local provider install is already running." + ) try: - _write_status(transition_state(_read_status(), new_state="resolving")) - spec = resolve_model_spec(model_id) - presence = _artifact_presence(spec) - if presence["model_installed"]: - _write_mlx_metadata( - spec, - snapshot_dir=presence["snapshot_dir"], - variant_dir=presence["variant_dir"], + if attempt_status is None: + attempt_status = begin_or_replace_install_attempt( + _LOCAL_NAME, + fingerprint, + initial_state="resolving", + owner={"entry": "install_local_mlx"}, ) + spec = resolve_model_spec(model_id) + readiness = inspect_readiness(model_id) + if readiness.status in {"proof-unavailable", "host-ineligible"}: + current = assert_install_attempt_current(attempt_status) return _write_status( - transition_state(_read_status(), new_state="installed") + transition_state( + current, + new_state="failed", + error=readiness.reason_code, + error_code=readiness.reason_code, + ) ) from solstone.think.providers import fit_report - report = fit_report.build_mlx_fit_report(spec.name) - rendered = fit_report.render_fit_report(report) - if report.overall == "blocked": - raise MLXInstallUnavailableError(rendered) - if report.overall == "warning": - LOG.warning("MLX provider host fit warning:\n%s", rendered) - - _write_status(transition_state(_read_status(), new_state="downloading")) - snapshot_dir = Path( - huggingface_hub.snapshot_download( - repo_id=spec.repo, - revision=spec.revision, + if not readiness.ready: + report = fit_report.build_mlx_fit_report(spec.name) + rendered = fit_report.render_fit_report(report) + if report.overall == "blocked": + raise MLXInstallUnavailableError(rendered) + if report.overall == "warning": + LOG.warning("MLX provider host fit warning:\n%s", rendered) + + _write_status(transition_state(_read_status(), new_state="downloading")) + snapshot_dir = Path( + huggingface_hub.snapshot_download( + repo_id=spec.repo, + revision=spec.revision, + ) ) - ) - - _write_status(transition_state(_read_status(), new_state="verifying")) - validate_snapshot_sha256(spec, snapshot_dir) - _write_status(transition_state(_read_status(), new_state="installing")) - variant_dir = None - if spec.name == GEMMA4_26B_A4B_4BIT: - variant_dir = create_gemma4_variant(snapshot_dir) + _write_status(transition_state(_read_status(), new_state="verifying")) + metadata = validate_snapshot_sha256(spec, snapshot_dir) + assert_install_attempt_current(attempt_status) + _write_snapshot_manifest( + spec, + snapshot_dir, + metadata=metadata, + attempt_status=attempt_status, + fingerprint=fingerprint, + ) - _write_mlx_metadata(spec, snapshot_dir=snapshot_dir, variant_dir=variant_dir) - return _write_status(transition_state(_read_status(), new_state="installed")) - except Exception as exc: - _write_status( - transition_state(_read_status(), new_state="failed", error=str(exc)) + _write_status(transition_state(_read_status(), new_state="installing")) + if spec.name == GEMMA4_26B_A4B_4BIT: + variant_dir = create_gemma4_variant(snapshot_dir) + assert_install_attempt_current(attempt_status) + _write_variant_manifest( + spec, + variant_dir, + attempt_status=attempt_status, + fingerprint=fingerprint, + ) + + final_readiness = inspect_readiness(model_id) + current = assert_install_attempt_current(attempt_status) + if final_readiness.ready: + return _write_status(transition_state(current, new_state="installed")) + return _write_status( + transition_state( + current, + new_state="failed", + error=final_readiness.reason_code, + error_code=final_readiness.reason_code, + ) ) + except Exception as exc: + try: + current = assert_install_attempt_current(attempt_status) + _write_status( + transition_state( + current, + new_state="failed", + error=str(exc), + error_code=getattr(exc, "reason_code", None), + ) + ) + except Exception: + pass raise + finally: + if owned_lease: + lease.release() __all__ = [ @@ -486,6 +712,7 @@ __all__ = [ "is_mlx_platform_supported", "resolve_model_spec", "snapshot_dir_for_spec", + "target_fingerprint", "validate_snapshot_sha256", "variant_dir_for_snapshot", ] diff --git a/solstone/think/providers/parakeet_install.py b/solstone/think/providers/parakeet_install.py index 93e448333..01e84b7d3 100644 --- a/solstone/think/providers/parakeet_install.py +++ b/solstone/think/providers/parakeet_install.py @@ -3,36 +3,37 @@ """Install and inspect bundled parakeet.cpp provider artifacts. -This module is the sole writer for ``providers.bundled.parakeet`` install state. -It performs no network access at import time. +This module owns parakeet.cpp provider artifact acquisition. It performs no +network access at import time. """ from __future__ import annotations -import errno -import fcntl import hashlib import logging -import os -import random import shutil import stat -import time -from collections.abc import Iterator -from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import Any, Callable from solstone.think import parakeet_readiness -from solstone.think.journal_config import ( - JournalConfigMutation, - mutate_journal_config, +from solstone.think.providers.artifact_proof import ( + ReadinessOutcome, + artifact_manifest_path, + build_manifest, + prove_manifest, + write_manifest, ) +from solstone.think.providers.install_lease import InstallLease, acquire_install_lease from solstone.think.providers.install_state import ( IN_FLIGHT_STATES, InstallStatus, + assert_install_attempt_current, + begin_or_replace_install_attempt, bump_progress, + canonical_fingerprint, + fingerprint_sha256, read_install_status, transition_state, write_install_status, @@ -42,23 +43,6 @@ from solstone.think.utils import get_journal LOG = logging.getLogger(__name__) PARAKEET_PROVIDER_NAME = "parakeet" _PROBE_TIMEOUT_SECONDS = 10 -_INSTALL_LOCK_TIMEOUT_SECONDS = 60.0 * 60.0 -_INSTALL_LOCK_POLL_INTERVAL_SECONDS = 1.0 -_PARAKEET_METADATA_KEYS = frozenset( - { - "binary_artifact_cpu", - "binary_sha256_cpu", - "binary_path_cpu", - "binary_artifact_vulkan", - "binary_sha256_vulkan", - "binary_path_vulkan", - "model_repo", - "model_filename", - "model_revision", - "model_path", - "model_sha256", - } -) PARAKEET_SERVER_PINS: dict[tuple[str, str], dict[str, str]] = { ("x86_64-unknown-linux-gnu", "vulkan"): { @@ -169,41 +153,17 @@ def install_hint() -> str: def _read_parakeet_status( journal_path: str | Path | None = None, ) -> InstallStatus: - return read_install_status( - scope="bundled", name=PARAKEET_PROVIDER_NAME, journal_path=journal_path - ) + return read_install_status(name=PARAKEET_PROVIDER_NAME, journal_path=journal_path) def _write_parakeet_status( status: InstallStatus, journal_path: str | Path | None = None, ) -> InstallStatus: - write_install_status(status, scope="bundled", journal_path=journal_path) + write_install_status(status, journal_path=journal_path) return status -def _write_parakeet_metadata( - updates: dict[str, str], - journal_path: str | Path | None = None, -) -> None: - unknown_keys = sorted(set(updates) - _PARAKEET_METADATA_KEYS) - if unknown_keys: - raise ValueError(f"unknown parakeet install metadata key: {unknown_keys[0]}") - - def apply(config: dict[str, Any]) -> JournalConfigMutation[None]: - slot = ( - config.setdefault("providers", {}) - .setdefault("bundled", {}) - .setdefault(PARAKEET_PROVIDER_NAME, {}) - ) - changed = any(slot.get(key) != value for key, value in updates.items()) - for key, value in updates.items(): - slot[key] = value - return JournalConfigMutation(changed=changed, value=None) - - mutate_journal_config(apply, journal_path=journal_path) - - def _record_parakeet_progress( received: int, total: int | None, @@ -215,56 +175,136 @@ def _record_parakeet_progress( return _write_parakeet_status( bump_progress(status, received=received, total=total), - journal_path, + journal_path=journal_path, ) -def _install_lock_path(journal_path: str | Path | None = None) -> Path: - return cache_root(journal_path) / "install" +def _binary_pin_identity(artifact_key: str, backend: str) -> dict[str, Any]: + pin = _pin_for_backend(artifact_key, backend) + return { + "unit": "parakeet-server", + "artifact_key": artifact_key, + "backend": backend, + "release_tag": parakeet_readiness.PARAKEET_CPP_RELEASE_TAG, + "filename": pin["filename"], + "sha256": pin["sha256"], + "binary_name": parakeet_readiness.PARAKEET_CPP_BINARY_NAME, + } -@contextmanager -def _hold_install_lock(journal_path: str | Path | None = None) -> Iterator[None]: - path = _install_lock_path(journal_path) - lock_path = path.parent / f"{path.name}.lock" - path.parent.mkdir(parents=True, exist_ok=True) - deadline = time.monotonic() + _INSTALL_LOCK_TIMEOUT_SECONDS - lock_file = open(lock_path, "w", encoding="utf-8") - try: - while True: - try: - fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) - break - except OSError as exc: - if exc.errno not in (errno.EACCES, errno.EAGAIN, errno.EWOULDBLOCK): - raise - if time.monotonic() >= deadline: - raise ParakeetProviderError( - "install_lock_timeout", - f"Timed out waiting for parakeet install lock at {lock_path}", - ) from exc - time.sleep(random.uniform(0.1, _INSTALL_LOCK_POLL_INTERVAL_SECONDS)) - try: - yield - finally: - fcntl.flock(lock_file, fcntl.LOCK_UN) - finally: - lock_file.close() +def _model_pin_identity() -> dict[str, Any]: + spec = PARAKEET_MODEL_SPEC + return { + "unit": "parakeet-model", + "repo": spec.repo, + "filename": spec.filename, + "revision": spec.revision, + "sha256": spec.sha256, + } -def _ready_status_if_installed( +def target_fingerprint( + *, journal_path: str | Path | None = None, -) -> InstallStatus | None: - try: - parakeet_readiness.check_parakeet_cpp_files( - cache_root(journal_path), parakeet_server_artifact_key() +) -> dict[str, Any]: + artifact_key = parakeet_server_artifact_key() + return { + "provider": PARAKEET_PROVIDER_NAME, + "runtime": "parakeet.cpp", + "artifact_key": artifact_key, + "binary_pins": [ + _binary_pin_identity(artifact_key, backend) + for backend in parakeet_readiness.PARAKEET_CPP_BINARY_BACKENDS + ], + "model_pin": _model_pin_identity(), + "cache_root": str(cache_root(journal_path)), + } + + +def _fingerprint_sha_for_target(fingerprint: dict[str, Any]) -> str: + return fingerprint_sha256(canonical_fingerprint(fingerprint)) + + +def _manifest_target_sha( + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any], +) -> str: + if attempt_status is not None and attempt_status["target_fingerprint_sha256"]: + return str(attempt_status["target_fingerprint_sha256"]) + return _fingerprint_sha_for_target(fingerprint) + + +def _manifest_entry(path: Path, root: Path, role: str) -> dict[str, Any]: + return { + "relative_path": path.relative_to(root).as_posix(), + "role": role, + "size": path.stat().st_size, + "sha256": _sha256_file(path), + } + + +def _runtime_inventory( + root: Path, + *, + backend: str, + exclude_names: set[str], +) -> list[dict[str, Any]]: + inventory: list[dict[str, Any]] = [] + for path in sorted(root.rglob("*")): + if not path.is_file() or path.name == artifact_manifest_path(root).name: + continue + if path.name in exclude_names: + continue + role = ( + f"runtime_binary_{backend}" + if path.name == parakeet_readiness.PARAKEET_CPP_BINARY_NAME + else f"runtime_support_{backend}" ) - except (ParakeetProviderError, RuntimeError): - return None - return _write_parakeet_status( - transition_state(_read_parakeet_status(journal_path), new_state="installed"), - journal_path, + inventory.append(_manifest_entry(path, root, role)) + return inventory + + +def _write_binary_manifest( + *, + backend: str, + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any] | None, + journal_path: str | Path | None, +) -> None: + artifact_key = parakeet_server_artifact_key() + pin = _pin_for_backend(artifact_key, backend) + root = binary_install_dir(backend, journal_path) + fingerprint = fingerprint or target_fingerprint(journal_path=journal_path) + manifest = build_manifest( + provider=PARAKEET_PROVIDER_NAME, + unit=f"parakeet-server-{backend}", + target_fingerprint_sha256=_manifest_target_sha(attempt_status, fingerprint), + source={"pin_identity": _binary_pin_identity(artifact_key, backend)}, + inventory=_runtime_inventory( + root, backend=backend, exclude_names={pin["filename"]} + ), + attempt_id=attempt_status["attempt_id"] if attempt_status else None, ) + write_manifest(artifact_manifest_path(root), manifest) + + +def _write_model_manifest( + *, + attempt_status: InstallStatus | None, + fingerprint: dict[str, Any] | None, + journal_path: str | Path | None, +) -> None: + root = model_dir(journal_path) + fingerprint = fingerprint or target_fingerprint(journal_path=journal_path) + manifest = build_manifest( + provider=PARAKEET_PROVIDER_NAME, + unit="parakeet-model", + target_fingerprint_sha256=_manifest_target_sha(attempt_status, fingerprint), + source={"pin_identity": _model_pin_identity()}, + inventory=[_manifest_entry(model_path(journal_path), root, "model")], + attempt_id=attempt_status["attempt_id"] if attempt_status else None, + ) + write_manifest(artifact_manifest_path(root), manifest) def _sha256_file(path: Path) -> str: @@ -393,6 +433,9 @@ def probe_binary_runnable(binary_path: str | Path) -> tuple[bool, str | None]: def _install_parakeet_server_unlocked( backend: str, journal_path: str | Path | None = None, + *, + attempt_status: InstallStatus | None = None, + fingerprint: dict[str, Any] | None = None, ) -> dict[str, Any]: backend = _validate_backend(backend) artifact_key = parakeet_server_artifact_key() @@ -411,10 +454,6 @@ def _install_parakeet_server_unlocked( ), journal_path, ) - _write_parakeet_metadata( - {f"binary_artifact_{backend}": pin["filename"]}, - journal_path, - ) _download_file( url, tarball, @@ -441,26 +480,21 @@ def _install_parakeet_server_unlocked( install_dir, parakeet_readiness.PARAKEET_CPP_BINARY_NAME ) final_path = binary_path(backend, journal_path) + if attempt_status is not None: + assert_install_attempt_current(attempt_status, journal_path=journal_path) inner_dir = extracted.parent if inner_dir != install_dir: for item in inner_dir.iterdir(): shutil.move(str(item), str(install_dir / item.name)) inner_dir.rmdir() _chmod_executable(final_path) - _write_parakeet_metadata( - { - f"binary_artifact_{backend}": pin["filename"], - f"binary_sha256_{backend}": pin["sha256"], - f"binary_path_{backend}": str(final_path), - }, - journal_path, - ) - return _write_parakeet_status( - transition_state( - _read_parakeet_status(journal_path), new_state="installed" - ), - journal_path, + _write_binary_manifest( + backend=backend, + attempt_status=attempt_status, + fingerprint=fingerprint, + journal_path=journal_path, ) + return _read_parakeet_status(journal_path) except Exception as exc: _write_parakeet_status( transition_state( @@ -478,12 +512,14 @@ def install_parakeet_server( *, journal_path: str | Path | None = None, ) -> dict[str, Any]: - with _hold_install_lock(journal_path): - return _install_parakeet_server_unlocked(backend, journal_path) + return _install_parakeet_server_unlocked(backend, journal_path) def _install_model_unlocked( journal_path: str | Path | None = None, + *, + attempt_status: InstallStatus | None = None, + fingerprint: dict[str, Any] | None = None, ) -> dict[str, Any]: spec = PARAKEET_MODEL_SPEC url = f"https://huggingface.co/{spec.repo}/resolve/{spec.revision}/{spec.filename}" @@ -496,14 +532,6 @@ def _install_model_unlocked( ), journal_path, ) - _write_parakeet_metadata( - { - "model_repo": spec.repo, - "model_filename": spec.filename, - "model_revision": spec.revision, - }, - journal_path, - ) _download_file( url, dest, @@ -518,22 +546,14 @@ def _install_model_unlocked( journal_path, ) _verify_sha256(dest, spec.sha256) - _write_parakeet_metadata( - { - "model_repo": spec.repo, - "model_filename": spec.filename, - "model_revision": spec.revision, - "model_path": str(dest), - "model_sha256": spec.sha256, - }, - journal_path, - ) - return _write_parakeet_status( - transition_state( - _read_parakeet_status(journal_path), new_state="installed" - ), - journal_path, + if attempt_status is not None: + assert_install_attempt_current(attempt_status, journal_path=journal_path) + _write_model_manifest( + attempt_status=attempt_status, + fingerprint=fingerprint, + journal_path=journal_path, ) + return _read_parakeet_status(journal_path) except Exception as exc: _write_parakeet_status( transition_state( @@ -550,52 +570,207 @@ def install_model( *, journal_path: str | Path | None = None, ) -> dict[str, Any]: - with _hold_install_lock(journal_path): - return _install_model_unlocked(journal_path) + return _install_model_unlocked(journal_path) def install_parakeet( *, force: bool = False, journal_path: str | Path | None = None, + lease: InstallLease | None = None, + attempt_status: InstallStatus | None = None, ) -> dict[str, Any]: - with _hold_install_lock(journal_path): - if not force: - ready_status = _ready_status_if_installed(journal_path) - if ready_status is not None: - return ready_status + fingerprint = target_fingerprint(journal_path=journal_path) + owned_lease = lease is None + if lease is None: + lease = acquire_install_lease(PARAKEET_PROVIDER_NAME, journal_path=journal_path) + if lease is None: + raise ParakeetProviderError( + "install_busy", "Parakeet provider install is already running." + ) + try: + if attempt_status is None: + attempt_status = begin_or_replace_install_attempt( + PARAKEET_PROVIDER_NAME, + fingerprint, + initial_state="resolving", + owner={"entry": "install_parakeet"}, + journal_path=journal_path, + ) + readiness = inspect_readiness(journal_path) + if readiness.status in {"proof-unavailable", "host-ineligible"}: + current = assert_install_attempt_current( + attempt_status, journal_path=journal_path + ) + return _write_parakeet_status( + transition_state( + current, + new_state="failed", + error=readiness.reason_code, + error_code=readiness.reason_code, + ), + journal_path, + ) from solstone.think.providers import fit_report - report = fit_report.build_parakeet_fit_report(journal_path) - rendered = fit_report.render_fit_report(report) - if report.overall == "blocked": - raise ParakeetProviderError("host_unfit", rendered) - if report.overall == "warning": - LOG.warning("parakeet.cpp host fit warning:\n%s", rendered) - for backend in parakeet_readiness.PARAKEET_CPP_BINARY_BACKENDS: - _install_parakeet_server_unlocked(backend, journal_path) - return _install_model_unlocked(journal_path) + if force or not readiness.ready: + report = fit_report.build_parakeet_fit_report(journal_path) + rendered = fit_report.render_fit_report(report) + if report.overall == "blocked": + raise ParakeetProviderError("host_unfit", rendered) + if report.overall == "warning": + LOG.warning("parakeet.cpp host fit warning:\n%s", rendered) + for backend in parakeet_readiness.PARAKEET_CPP_BINARY_BACKENDS: + if ( + force + or readiness.proof[f"binary_{backend}"]["status"] + == "missing-or-mismatched" + ): + _install_parakeet_server_unlocked( + backend, + journal_path, + attempt_status=attempt_status, + fingerprint=fingerprint, + ) + if force or readiness.proof["model"]["status"] == "missing-or-mismatched": + _install_model_unlocked( + journal_path, + attempt_status=attempt_status, + fingerprint=fingerprint, + ) + + final_readiness = inspect_readiness(journal_path) + current = assert_install_attempt_current( + attempt_status, journal_path=journal_path + ) + if final_readiness.ready: + return _write_parakeet_status( + transition_state(current, new_state="installed"), + journal_path, + ) + return _write_parakeet_status( + transition_state( + current, + new_state="failed", + error=final_readiness.reason_code, + error_code=final_readiness.reason_code, + ), + journal_path, + ) + except Exception as exc: + try: + current = assert_install_attempt_current( + attempt_status, journal_path=journal_path + ) + _write_parakeet_status( + transition_state( + current, + new_state="failed", + error=str(exc), + error_code=getattr(exc, "reason_code", None), + ), + journal_path, + ) + except Exception: + pass + raise + finally: + if owned_lease: + lease.release() + + +def _proof_result_payload(result: Any) -> dict[str, Any]: + return { + "status": result.status, + "reason_code": result.reason_code, + "cache_hit": bool(getattr(result, "cache_hit", False)), + } + + +def _combined_artifact_status( + *proofs: dict[str, Any], +) -> tuple[str, str]: + for proof in proofs: + if proof["status"] == "proof-unavailable": + return "proof-unavailable", str(proof["reason_code"]) + for proof in proofs: + if proof["status"] == "missing-or-mismatched": + return "missing-or-mismatched", str(proof["reason_code"]) + return "ready", "ready" -def inspect_readiness(journal_path: str | Path | None = None) -> dict[str, Any]: +def inspect_readiness(journal_path: str | Path | None = None) -> ReadinessOutcome: status = _read_parakeet_status(journal_path) + artifact_key = parakeet_server_artifact_key() cpu_path = binary_path("cpu", journal_path) vulkan_path = binary_path("vulkan", journal_path) gguf_path = model_path(journal_path) - cpu_installed = cpu_path.exists() and os.access(cpu_path, os.X_OK) - vulkan_installed = vulkan_path.exists() and os.access(vulkan_path, os.X_OK) - model_installed = gguf_path.exists() - return { - "install_state": status["install_state"], - "binary_installed": cpu_installed and vulkan_installed, - "binary_cpu_installed": cpu_installed, - "binary_vulkan_installed": vulkan_installed, - "model_installed": model_installed, - "binary_path_cpu": str(cpu_path), - "binary_path_vulkan": str(vulkan_path), - "model_path": str(gguf_path), - "install_error": status["install_error"], - } + cpu_proof = prove_manifest( + artifact_manifest_path(binary_install_dir("cpu", journal_path)), + provider=PARAKEET_PROVIDER_NAME, + pin_identity=_binary_pin_identity(artifact_key, "cpu"), + journal_path=journal_path, + ) + vulkan_proof = prove_manifest( + artifact_manifest_path(binary_install_dir("vulkan", journal_path)), + provider=PARAKEET_PROVIDER_NAME, + pin_identity=_binary_pin_identity(artifact_key, "vulkan"), + journal_path=journal_path, + ) + model_proof = prove_manifest( + artifact_manifest_path(model_dir(journal_path)), + provider=PARAKEET_PROVIDER_NAME, + pin_identity=_model_pin_identity(), + journal_path=journal_path, + ) + cpu_payload = _proof_result_payload(cpu_proof) + vulkan_payload = _proof_result_payload(vulkan_proof) + model_payload = _proof_result_payload(model_proof) + readiness_status, reason_code = _combined_artifact_status( + cpu_payload, vulkan_payload, model_payload + ) + binary_status, binary_reason_code = _combined_artifact_status( + cpu_payload, vulkan_payload + ) + return ReadinessOutcome( + provider=PARAKEET_PROVIDER_NAME, + status=readiness_status, # type: ignore[arg-type] + reason_code=reason_code, + target={ + "target_fingerprint_json": status["target_fingerprint_json"], + "target_fingerprint_sha256": status["target_fingerprint_sha256"], + }, + install={ + "install_state": status["install_state"], + "install_error": status["install_error"], + "error_code": status["error_code"], + "attempt_id": status["attempt_id"], + "progress_bytes_received": status["progress_bytes_received"], + "progress_bytes_total": status["progress_bytes_total"], + "last_transition_at": status["last_transition_at"], + "last_progress_at": status["last_progress_at"], + }, + host={}, + artifacts={ + "binary_installed": cpu_proof.ready and vulkan_proof.ready, + "binary_cpu_installed": cpu_proof.ready, + "binary_vulkan_installed": vulkan_proof.ready, + "model_installed": model_proof.ready, + "binary_path_cpu": str(cpu_path), + "binary_path_vulkan": str(vulkan_path), + "model_path": str(gguf_path), + }, + proof={ + "binary": { + "status": binary_status, + "reason_code": binary_reason_code, + "cache_hit": cpu_payload["cache_hit"] and vulkan_payload["cache_hit"], + }, + "binary_cpu": cpu_payload, + "binary_vulkan": vulkan_payload, + "model": model_payload, + }, + ) def ensure_artifacts_installed( @@ -605,13 +780,15 @@ def ensure_artifacts_installed( ) -> tuple[Path, Path]: backend = _validate_backend(backend) readiness = inspect_readiness(journal_path) - if not readiness["binary_installed"]: + if not readiness.artifacts["binary_installed"]: raise ParakeetProviderError( "binary_missing", "Parakeet server binaries are not installed." ) - if not readiness["model_installed"]: + if not readiness.artifacts["model_installed"]: raise ParakeetProviderError("model_missing", "Parakeet model is not installed.") - return Path(readiness[f"binary_path_{backend}"]), Path(readiness["model_path"]) + return Path(readiness.artifacts[f"binary_path_{backend}"]), Path( + readiness.artifacts["model_path"] + ) __all__ = [ diff --git a/solstone/think/providers/state.py b/solstone/think/providers/state.py index 0798b9af3..7c51ccef0 100644 --- a/solstone/think/providers/state.py +++ b/solstone/think/providers/state.py @@ -308,8 +308,8 @@ def local_status_dict() -> dict: from solstone.think.providers import local_server, mlx_install readiness = mlx_install.inspect_readiness() - runtime_available = bool(readiness["package_available"]) - model_installed = bool(readiness["model_installed"]) + runtime_available = bool(readiness.host["package_available"]) + model_installed = bool(readiness.artifacts["model_installed"]) configured = runtime_available and model_installed if not selected: @@ -325,7 +325,9 @@ def local_status_dict() -> dict: server_healthy = local_server.is_healthy() if not runtime_available: issues.append("runtime_missing") - if not model_installed: + if readiness.status == "proof-unavailable": + issues.append(readiness.reason_code) + elif not model_installed: issues.append("model_missing") if configured and not server_healthy: issues.append("server_unhealthy") @@ -342,8 +344,8 @@ def local_status_dict() -> dict: from solstone.think.providers import local_install, local_server readiness = local_install.inspect_readiness() - binary_installed = bool(readiness["binary_installed"]) - model_installed = bool(readiness["model_installed"]) + binary_installed = bool(readiness.artifacts["binary_installed"]) + model_installed = bool(readiness.artifacts["model_installed"]) configured = binary_installed and model_installed if not selected: @@ -357,14 +359,18 @@ def local_status_dict() -> dict: issues: list[str] = [] server_healthy = local_server.is_healthy() - if not readiness.get("gpu_available", True): + if not readiness.host.get("gpu_available", True): issues.append("gpu_unavailable") - if not binary_installed: + if readiness.status == "proof-unavailable": + issues.append(readiness.reason_code) + elif readiness.proof["binary"]["status"] == "missing-or-mismatched": issues.append("binary_missing") - if not model_installed: + if readiness.proof["model"]["status"] == "missing-or-mismatched": issues.append("model_missing") if configured and not server_healthy: - runnable, detail = local_install.probe_binary_runnable(readiness["binary_path"]) + runnable, detail = local_install.probe_binary_runnable( + readiness.artifacts["binary_path"] + ) if runnable: issues.append("server_unhealthy") else: @@ -500,23 +506,23 @@ def _local_readiness_for_provider( from solstone.think.providers.install_state import IN_FLIGHT_STATES readiness = mlx_install.inspect_readiness() - model_id = str(readiness["model_id"]) + model_id = str(readiness.target["model_id"]) - if readiness["install_state"] in IN_FLIGHT_STATES: + if readiness.install["install_state"] in IN_FLIGHT_STATES: return _state( provider, interface, "blocked", "local_model_installing", model=model_id, - message=str(readiness["install_state"]), + message=str(readiness.install["install_state"]), source="local_install", ) if not ( - readiness["platform_supported"] - and readiness["package_available"] - and readiness["model_installed"] + readiness.host["platform_supported"] + and readiness.host["package_available"] + and readiness.artifacts["model_installed"] ): return _state( provider, @@ -524,7 +530,7 @@ def _local_readiness_for_provider( "blocked", "local_model_missing", model=model_id, - message=str(readiness.get("install_error") or "") or None, + message=str(readiness.install.get("install_error") or "") or None, source="local_install", ) @@ -562,20 +568,20 @@ def _local_readiness_for_provider( selected_model = model or LOCAL_MODEL readiness = local_install.inspect_readiness(selected_model) - model_id = str(readiness.get("model_id") or selected_model) + model_id = str(readiness.target.get("model_id") or selected_model) - if readiness["install_state"] in IN_FLIGHT_STATES: + if readiness.install["install_state"] in IN_FLIGHT_STATES: return _state( provider, interface, "blocked", "local_model_installing", model=model_id, - message=str(readiness["install_state"]), + message=str(readiness.install["install_state"]), source="local_install", ) - if not readiness.get("gpu_probe_ok", True): + if not readiness.host.get("gpu_probe_ok", True): return _state( provider, interface, @@ -585,7 +591,7 @@ def _local_readiness_for_provider( source="local_install", ) - if not readiness.get("gpu_available", True): + if not readiness.host.get("gpu_available", True): return _state( provider, interface, @@ -595,14 +601,17 @@ def _local_readiness_for_provider( source="local_install", ) - if not readiness["binary_installed"] or not readiness["model_installed"]: + if ( + not readiness.artifacts["binary_installed"] + or not readiness.artifacts["model_installed"] + ): return _state( provider, interface, "blocked", "local_model_missing", model=model_id, - message=str(readiness.get("install_error") or "") or None, + message=str(readiness.install.get("install_error") or "") or None, source="local_install", ) diff --git a/solstone/think/supervisor.py b/solstone/think/supervisor.py index d8d011609..92e17c941 100644 --- a/solstone/think/supervisor.py +++ b/solstone/think/supervisor.py @@ -251,8 +251,6 @@ _sync_conflict_shutdown: bool = False _supervisor_ref: str | None = None _supervisor_start: float | None = None _parent_death_sigterm_sent = threading.Event() -_parakeet_bootstrap_lock = threading.Lock() -_parakeet_bootstrap_thread: threading.Thread | None = None def app_supervised_graceful_budget_s() -> float: @@ -1780,21 +1778,21 @@ def _start_mlx_local_server() -> RunnerManagedProcess | None: from solstone.think.providers import local_server, mlx_install readiness = mlx_install.inspect_readiness() - readiness_keys = ( - "platform_supported", - "package_available", - "ram_sufficient", - "model_installed", - ) - if not all(readiness.get(key) for key in readiness_keys): + readiness_values = { + "platform_supported": readiness.host["platform_supported"], + "package_available": readiness.host["package_available"], + "ram_sufficient": readiness.host["ram_sufficient"], + "model_installed": readiness.artifacts["model_installed"], + } + if not all(readiness_values.values()): logging.info( "MLX local model not ready; skipping mlx-vlm server startup: %s", - {key: readiness.get(key) for key in readiness_keys}, + readiness_values, ) return None - runtime_dir = readiness["runtime_dir"] - model_id = readiness["model_id"] + runtime_dir = readiness.artifacts["runtime_dir"] + model_id = readiness.target["model_id"] port = find_available_port() write_service_port("local", port) script_path = str(Path(sys.executable).with_name(MLX_SERVER_PROCESS_NAME)) @@ -1938,33 +1936,38 @@ def _request_parakeet_server_start() -> None: logging.exception("could not request parakeet-server start") -def _run_parakeet_bootstrap_worker(journal_path: Path | None = None) -> None: +def _run_parakeet_bootstrap_worker( + journal_path: Path | None = None, + lease: Any | None = None, + attempt_status: Any | None = None, + ack: threading.Event | None = None, +) -> None: """Install parakeet.cpp artifacts in the background, then retry startup.""" - current_thread = threading.current_thread() + if ack is not None: + ack.set() try: from solstone.think.providers import parakeet_install - if journal_path is None: - parakeet_install.install_parakeet() - else: - parakeet_install.install_parakeet(journal_path=journal_path) + parakeet_install.install_parakeet( + journal_path=journal_path, + lease=lease, + attempt_status=attempt_status, + ) except Exception: logging.exception("parakeet.cpp provider bootstrap failed") else: logging.info("parakeet.cpp provider bootstrap complete; requesting startup") _request_parakeet_server_start() finally: - global _parakeet_bootstrap_thread - with _parakeet_bootstrap_lock: - if _parakeet_bootstrap_thread is current_thread: - _parakeet_bootstrap_thread = None + if lease is not None: + lease.release() def _start_parakeet_bootstrap_if_needed(reason: str) -> None: """Start one non-blocking parakeet.cpp install worker when artifacts are absent.""" - global _parakeet_bootstrap_thread from solstone.think.providers import parakeet_install - from solstone.think.providers.install_state import IN_FLIGHT_STATES + from solstone.think.providers.install_lease import acquire_install_lease + from solstone.think.providers.install_state import begin_or_replace_install_attempt try: readiness = parakeet_install.inspect_readiness() @@ -1972,31 +1975,40 @@ def _start_parakeet_bootstrap_if_needed(reason: str) -> None: logging.info( "could not inspect parakeet.cpp readiness before bootstrap: %s", exc ) - readiness = {} + readiness = None - if readiness.get("binary_installed") and readiness.get("model_installed"): - return - if readiness.get("install_state") in IN_FLIGHT_STATES: - logging.info( - "parakeet.cpp provider install already %s; not starting another worker", - readiness.get("install_state"), - ) + if readiness is not None and readiness.ready: return - with _parakeet_bootstrap_lock: - if ( - _parakeet_bootstrap_thread is not None - and _parakeet_bootstrap_thread.is_alive() - ): - logging.info("parakeet.cpp provider bootstrap already running") - return - journal_path = Path(get_journal()) + journal_path = Path(get_journal()) + lease = acquire_install_lease("parakeet", journal_path=journal_path) + if lease is None: + logging.info("parakeet.cpp provider bootstrap already running") + return + try: + fingerprint = parakeet_install.target_fingerprint(journal_path=journal_path) + attempt_status = begin_or_replace_install_attempt( + "parakeet", + fingerprint, + initial_state="downloading", + owner={"entry": "supervisor_parakeet_bootstrap"}, + journal_path=journal_path, + ) + ack = threading.Event() thread = threading.Thread( - target=lambda: _run_parakeet_bootstrap_worker(journal_path), + target=lambda: _run_parakeet_bootstrap_worker( + journal_path, + lease, + attempt_status, + ack, + ), name="parakeet-cpp-provider-bootstrap", daemon=True, ) - _parakeet_bootstrap_thread = thread + except Exception: + lease.release() + logging.exception("could not prepare parakeet.cpp provider bootstrap worker") + return logging.info( "Parakeet artifacts not ready; starting background provider install: %s", @@ -2005,10 +2017,12 @@ def _start_parakeet_bootstrap_if_needed(reason: str) -> None: try: thread.start() except Exception: - with _parakeet_bootstrap_lock: - if _parakeet_bootstrap_thread is thread: - _parakeet_bootstrap_thread = None + lease.release() logging.exception("could not start parakeet.cpp provider bootstrap worker") + return + if not ack.wait(timeout=5.0): + lease.release() + logging.error("parakeet.cpp provider bootstrap worker did not acknowledge") def _build_parakeet_cmd( diff --git a/tests/test_chat_reasons.py b/tests/test_chat_reasons.py index b0d169bc7..a11bc2d4b 100644 --- a/tests/test_chat_reasons.py +++ b/tests/test_chat_reasons.py @@ -19,6 +19,7 @@ EXPECTED_CODES = { "local_model_missing", "model_missing", "binary_missing", + "install_busy", "local_model_installing", "local_model_loading", "local_model_not_ready", diff --git a/tests/test_install_models.py b/tests/test_install_models.py index 4abd49578..8a055a152 100644 --- a/tests/test_install_models.py +++ b/tests/test_install_models.py @@ -559,9 +559,21 @@ def test_install_models_linux_routes_through_parakeet_provider( calls = [] paths = _ready_paths(tmp_path) + journal = tmp_path / "journal" + (journal / "config").mkdir(parents=True) + (journal / "config" / "journal.json").write_text( + '{"providers": {}}\n', + encoding="utf-8", + ) + monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) + import solstone.think.utils as think_utils + + think_utils._journal_path_cache = None monkeypatch.setattr( - parakeet_install, "install_parakeet", lambda: calls.append("install") + parakeet_install, + "install_parakeet", + lambda **_kwargs: calls.append("install"), ) monkeypatch.setattr(install_models, "_check_linux_cpp_ready", lambda: paths) diff --git a/tests/test_install_provider.py b/tests/test_install_provider.py index f78e4e4f1..f5a09bfae 100644 --- a/tests/test_install_provider.py +++ b/tests/test_install_provider.py @@ -10,6 +10,7 @@ import pytest from solstone.think import install_provider from solstone.think.providers import fit_report +from solstone.think.providers.artifact_proof import ReadinessOutcome def _fit(severity: fit_report.FitSeverity) -> fit_report.FitReport: @@ -19,24 +20,91 @@ def _fit(severity: fit_report.FitSeverity) -> fit_report.FitReport: ) +class _FakeLease: + def __init__(self) -> None: + self.released = False + + def release(self) -> None: + self.released = True + + +def _readiness(provider: str, *, ready: bool) -> ReadinessOutcome: + return ReadinessOutcome( + provider=provider, + status="ready" if ready else "missing-or-mismatched", + reason_code="ready" if ready else "manifest_missing", + target={"model_id": "model"}, + install={ + "install_state": "idle", + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={}, + artifacts={ + "binary_installed": ready, + "model_installed": ready, + }, + proof={ + "binary": { + "status": "ready" if ready else "missing-or-mismatched", + "reason_code": "ready" if ready else "manifest_missing", + "cache_hit": False, + }, + "model": { + "status": "ready" if ready else "missing-or-mismatched", + "reason_code": "ready" if ready else "manifest_missing", + "cache_hit": False, + }, + }, + ) + + +def _patch_lease_and_attempt( + monkeypatch: pytest.MonkeyPatch, provider: str +) -> _FakeLease: + lease = _FakeLease() + monkeypatch.setattr( + install_provider, + "acquire_install_lease", + lambda name: lease if name == provider else None, + ) + monkeypatch.setattr( + install_provider, + "begin_or_replace_install_attempt", + lambda name, fingerprint, **_kwargs: { + "provider": name, + "install_state": "resolving", + "attempt_id": "attempt", + "target_fingerprint_sha256": install_provider._target_sha(fingerprint), + }, + ) + return lease + + def test_install_provider_local_prints_install_status(monkeypatch, capsys): calls = [] parakeet_calls = [] - def install_local(): + def install_local(**_kwargs): calls.append(True) - return {"name": "local", "install_state": "installed"} + return {"provider": "local", "install_state": "installed"} - def install_parakeet(): + def install_parakeet(**_kwargs): parakeet_calls.append(True) - return {"name": "parakeet", "install_state": "installed"} + return {"provider": "parakeet", "install_state": "installed"} monkeypatch.setattr(sys, "argv", ["journal install-provider", "local"]) monkeypatch.setattr( install_provider.local_install, "inspect_readiness", - lambda: {"binary_installed": False, "model_installed": False}, + lambda: _readiness("local", ready=False), ) + lease = _patch_lease_and_attempt(monkeypatch, "local") monkeypatch.setattr(fit_report, "build_local_fit_report", lambda _model: _fit("ok")) monkeypatch.setattr(install_provider.local_install, "install_local", install_local) monkeypatch.setattr( @@ -49,8 +117,9 @@ def test_install_provider_local_prints_install_status(monkeypatch, capsys): assert calls == [True] assert parakeet_calls == [] + assert lease.released is True assert json.loads(capsys.readouterr().out) == { - "name": "local", + "provider": "local", "install_state": "installed", } @@ -58,16 +127,17 @@ def test_install_provider_local_prints_install_status(monkeypatch, capsys): def test_install_provider_parakeet_prints_disclosure_and_status(monkeypatch, capsys): calls = [] - def install_parakeet(): + def install_parakeet(**_kwargs): calls.append(True) - return {"name": "parakeet", "install_state": "installed"} + return {"provider": "parakeet", "install_state": "installed"} monkeypatch.setattr(sys, "argv", ["journal install-provider", "parakeet"]) monkeypatch.setattr( install_provider.parakeet_install, "inspect_readiness", - lambda: {"binary_installed": False, "model_installed": False}, + lambda: _readiness("parakeet", ready=False), ) + lease = _patch_lease_and_attempt(monkeypatch, "parakeet") monkeypatch.setattr(fit_report, "build_parakeet_fit_report", lambda: _fit("ok")) monkeypatch.setattr( install_provider.parakeet_install, @@ -79,8 +149,9 @@ def test_install_provider_parakeet_prints_disclosure_and_status(monkeypatch, cap captured = capsys.readouterr() assert calls == [True] + assert lease.released is True assert json.loads(captured.out) == { - "name": "parakeet", + "provider": "parakeet", "install_state": "installed", } assert install_provider.PARAKEET_DOWNLOAD_DISCLOSURE in captured.err @@ -93,15 +164,20 @@ def test_install_provider_parakeet_prints_disclosure_and_status(monkeypatch, cap def test_install_provider_local_skips_fit_report_when_ready(monkeypatch, capsys): calls = [] - def install_local(): + def install_local(**_kwargs): calls.append(True) - return {"name": "local", "install_state": "installed"} + return {"provider": "local", "install_state": "installed"} monkeypatch.setattr(sys, "argv", ["journal install-provider", "local"]) monkeypatch.setattr( install_provider.local_install, "inspect_readiness", - lambda: {"binary_installed": True, "model_installed": True}, + lambda: _readiness("local", ready=True), + ) + monkeypatch.setattr( + install_provider, + "read_install_status", + lambda name: {"provider": name, "install_state": "installed"}, ) monkeypatch.setattr( fit_report, @@ -113,16 +189,16 @@ def test_install_provider_local_skips_fit_report_when_ready(monkeypatch, capsys) assert install_provider.main() == 0 captured = capsys.readouterr() - assert calls == [True] + assert calls == [] assert "local already installed" in captured.err assert json.loads(captured.out) == { - "name": "local", + "provider": "local", "install_state": "installed", } def test_install_provider_local_expected_error_returns_nonzero(monkeypatch, capsys): - def install_local(): + def install_local(**_kwargs): raise install_provider.local_install.LocalProviderError( "host_unfit", "blocked detail" ) @@ -131,8 +207,9 @@ def test_install_provider_local_expected_error_returns_nonzero(monkeypatch, caps monkeypatch.setattr( install_provider.local_install, "inspect_readiness", - lambda: {"binary_installed": False, "model_installed": False}, + lambda: _readiness("local", ready=False), ) + lease = _patch_lease_and_attempt(monkeypatch, "local") monkeypatch.setattr( fit_report, "build_local_fit_report", lambda _model: _fit("blocked") ) @@ -141,6 +218,7 @@ def test_install_provider_local_expected_error_returns_nonzero(monkeypatch, caps assert install_provider.main() == 1 captured = capsys.readouterr() + assert lease.released is True assert "blocked detail" in captured.err assert captured.out == "" @@ -149,13 +227,13 @@ def test_install_provider_unsupported_rejects_without_install(monkeypatch, capsy local_calls = [] parakeet_calls = [] - def install_local(): + def install_local(**_kwargs): local_calls.append(True) - return {"name": "local", "install_state": "installed"} + return {"provider": "local", "install_state": "installed"} - def install_parakeet(): + def install_parakeet(**_kwargs): parakeet_calls.append(True) - return {"name": "parakeet", "install_state": "installed"} + return {"provider": "parakeet", "install_state": "installed"} monkeypatch.setattr(sys, "argv", ["journal install-provider", "foo"]) monkeypatch.setattr(install_provider.local_install, "install_local", install_local) diff --git a/tests/test_install_state.py b/tests/test_install_state.py index 9685bba4f..af30e0d86 100644 --- a/tests/test_install_state.py +++ b/tests/test_install_state.py @@ -236,6 +236,7 @@ def test_migration_api_removes_legacy_status_fields(tmp_path, monkeypatch) -> No "install_state": "failed", "install_error": "old", "model_id": "kept", + "vulkan_device_index": "1", } } } @@ -248,8 +249,9 @@ def test_migration_api_removes_legacy_status_fields(tmp_path, monkeypatch) -> No result = install_state.migrate_legacy_provider_install_state() data = json.loads(config_path.read_text(encoding="utf-8")) - assert result == {"removed": 2} - assert data["providers"]["bundled"]["local"] == {"model_id": "kept"} + assert result == {"removed": 4, "moved": 1} + assert data["providers"]["bundled"]["local"] == {} + assert data["providers"]["local"] == {"vulkan_device_index": "1"} def test_two_process_stale_transition_one_writer_wins(tmp_path, monkeypatch) -> None: diff --git a/tests/test_journal_config_changed_contract.py b/tests/test_journal_config_changed_contract.py index 8a0fb7c6d..b8d323eff 100644 --- a/tests/test_journal_config_changed_contract.py +++ b/tests/test_journal_config_changed_contract.py @@ -581,46 +581,33 @@ def test_backup_state_mutation_paths_report_changed_and_noop( assert changes[-1] is False -def test_provider_install_mutation_paths_report_changed_and_noop( +def test_provider_install_migration_reports_changed_and_noop( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) - cases: list[tuple[Any, Callable[[], None]]] = [] - parakeet_install = importlib.import_module( - "solstone.think.providers.parakeet_install" - ) - local_install = importlib.import_module("solstone.think.providers.local_install") - mlx_install = importlib.import_module("solstone.think.providers.mlx_install") - spec = mlx_install.MLXModelSpec("mlx-model", "repo", "rev", 1) - cases.extend( - [ - ( - parakeet_install, - lambda: parakeet_install._write_parakeet_metadata( - {"model_repo": "repo"} - ), - ), - ( - local_install, - lambda: local_install._write_local_metadata({"model_id": "model"}), - ), - ( - mlx_install, - lambda: mlx_install._write_mlx_metadata( - spec, - snapshot_dir=tmp_path / "snapshot", - variant_dir=tmp_path / "variant", - ), - ), - ] + install_state = importlib.import_module("solstone.think.providers.install_state") + changes = _spy_changed(monkeypatch, install_state) + seed_journal_config( + { + "providers": { + "bundled": { + "local": { + "install_state": "installed", + "model_id": "local/model", + "vulkan_device_index": "1", + }, + "parakeet": {"model_repo": "repo"}, + }, + "local": {}, + } + }, + tmp_path, ) - for module, invoke in cases: - seed_journal_config({"providers": {"bundled": {}}}, tmp_path) - changes = _spy_changed(monkeypatch, module) - invoke() - assert changes[-1] is True - invoke() - assert changes[-1] is False + + install_state.migrate_legacy_provider_install_state(journal_path=tmp_path) + assert changes[-1] is True + install_state.migrate_legacy_provider_install_state(journal_path=tmp_path) + assert changes[-1] is False def test_pairing_oura_and_sol_voice_paths_report_changed_and_noop( diff --git a/tests/test_journal_config_transactions.py b/tests/test_journal_config_transactions.py index bac8d8578..4d434db6e 100644 --- a/tests/test_journal_config_transactions.py +++ b/tests/test_journal_config_transactions.py @@ -82,7 +82,7 @@ def _provider_progress_worker(journal_path: str, barrier: Any, errors: Any) -> N ) status = transition_state(make_idle_status("parakeet"), new_state="downloading") - write_install_status(status, scope="bundled") + write_install_status(status) except BaseException: errors.put(traceback.format_exc()) raise diff --git a/tests/test_local.py b/tests/test_local.py index 915b026b8..fc3d53f56 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -22,6 +22,7 @@ from solstone.think.models import ( LOCAL_MODEL, get_model_provider, ) +from solstone.think.providers.artifact_proof import ReadinessOutcome from solstone.think.talents import TalentHookError @@ -2474,6 +2475,61 @@ def _select_local_provider(monkeypatch) -> None: ) +def _provider_local_readiness( + *, + binary_installed: bool = True, + model_installed: bool = True, + ram_sufficient: bool = True, + gpu_available: bool = True, + gpu_probe_ok: bool = True, + binary_path: str = "/fake/llama-server", +) -> ReadinessOutcome: + ready = binary_installed and model_installed + return ReadinessOutcome( + provider="local", + status="ready" if ready else "missing-or-mismatched", + reason_code="ready" if ready else "manifest_missing", + target={"model_id": LOCAL_MODEL}, + install={ + "install_state": "idle", + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": ram_sufficient, + "gpu_available": gpu_available, + "gpu_probe_ok": gpu_probe_ok, + "backend": "vulkan", + "backend_reason": "test vulkan", + }, + artifacts={ + "binary_installed": binary_installed, + "model_installed": model_installed, + "binary_path": binary_path, + "model_path": "/tmp/model.gguf", + "mmproj_path": None, + "model_id": LOCAL_MODEL, + }, + proof={ + "binary": { + "status": "ready" if binary_installed else "missing-or-mismatched", + "reason_code": "ready" if binary_installed else "manifest_missing", + "cache_hit": False, + }, + "model": { + "status": "ready" if model_installed else "missing-or-mismatched", + "reason_code": "ready" if model_installed else "manifest_missing", + "cache_hit": False, + }, + }, + ) + + def test_build_provider_status_local_not_selected_is_inert(monkeypatch): from solstone.think.providers import build_provider_status @@ -2488,13 +2544,7 @@ def test_build_provider_status_local_not_selected_is_inert(monkeypatch): ) monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": True, - "binary_path": "/fake/llama-server", - }, + lambda: _provider_local_readiness(), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", @@ -2519,12 +2569,7 @@ def test_build_provider_status_local_readiness(monkeypatch): _select_local_provider(monkeypatch) monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": True, - }, + lambda: _provider_local_readiness(), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: True @@ -2549,13 +2594,7 @@ def test_build_provider_status_local_launch_failure_adds_probe_detail_and_hint( detail = "dyld: Library not loaded: @rpath/libllama.dylib" monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": True, - "binary_path": "/fake/llama-server", - }, + lambda: _provider_local_readiness(), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: False @@ -2584,13 +2623,7 @@ def test_build_provider_status_local_server_unhealthy_when_probe_runnable( _select_local_provider(monkeypatch) monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": True, - "binary_path": "/fake/llama-server", - }, + lambda: _provider_local_readiness(), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: False @@ -2619,13 +2652,7 @@ def test_build_provider_status_local_healthy_skips_probe(monkeypatch): monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": True, - "binary_path": "/fake/llama-server", - }, + lambda: _provider_local_readiness(), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: True @@ -2648,12 +2675,11 @@ def test_local_provider_status_carries_install_hint_substring(monkeypatch): _select_local_provider(monkeypatch) monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": False, - "model_installed": False, - "ram_sufficient": False, - "gpu_available": True, - }, + lambda: _provider_local_readiness( + binary_installed=False, + model_installed=False, + ram_sufficient=False, + ), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: False @@ -2680,13 +2706,7 @@ def test_local_provider_status_reports_gpu_unavailable_issue(monkeypatch): _select_local_provider(monkeypatch) monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "gpu_available": False, - "binary_path": "/fake/llama-server", - }, + lambda: _provider_local_readiness(gpu_available=False), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: True @@ -2705,13 +2725,7 @@ def test_build_provider_status_local_configured_ignores_ram_flag(monkeypatch): _select_local_provider(monkeypatch) monkeypatch.setattr( "solstone.think.providers.local_install.inspect_readiness", - lambda: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": False, - "gpu_available": True, - "binary_path": "/fake/llama-server", - }, + lambda: _provider_local_readiness(ram_sufficient=False), ) monkeypatch.setattr( "solstone.think.providers.local_server.is_healthy", lambda: True diff --git a/tests/test_local_install.py b/tests/test_local_install.py index f6b4929b4..adfb48db9 100644 --- a/tests/test_local_install.py +++ b/tests/test_local_install.py @@ -13,7 +13,6 @@ from types import SimpleNamespace import pytest -from solstone.think.journal_config import read_journal_config from solstone.think.models import LOCAL_MODEL from solstone.think.providers import ( fit_report, @@ -23,8 +22,14 @@ from solstone.think.providers import ( memory, oci_image, ) +from solstone.think.providers.artifact_proof import ( + ReadinessOutcome, + artifact_manifest_path, + prove_manifest, +) from solstone.think.providers.install_state import read_install_status from solstone.think.providers.local import LOCAL_MODEL_SPECS +from solstone.think.providers.local_endpoint import resolve_local_endpoint def _init_journal(tmp_path, monkeypatch) -> None: @@ -38,11 +43,107 @@ def _init_journal(tmp_path, monkeypatch) -> None: def _local_status() -> dict: - return read_install_status(scope="bundled", name="local") + return read_install_status(name="local") + + +def _write_provider_local_config(tmp_path: Path, updates: dict[str, object]) -> None: + path = tmp_path / "config" / "journal.json" + config = json.loads(path.read_text(encoding="utf-8")) + provider_config = config.setdefault("providers", {}).setdefault("local", {}) + provider_config.update(updates) + path.write_text(json.dumps(config) + "\n", encoding="utf-8") + + +def _fake_local_readiness( + *, + binary_installed: bool, + model_installed: bool, + binary_path: Path, + model_path: Path, + mmproj_path: Path | None = None, + ram_sufficient: bool = True, + backend: str = "vulkan", + backend_reason: str = "test vulkan", +) -> ReadinessOutcome: + missing_binary = not binary_installed + missing_model = not model_installed + status = ( + "ready" if not missing_binary and not missing_model else "missing-or-mismatched" + ) + reason_code = ( + "ready" + if status == "ready" + else "binary_missing" + if missing_binary + else "model_missing" + ) + binary_status = "ready" if binary_installed else "missing-or-mismatched" + model_status = "ready" if model_installed else "missing-or-mismatched" + return ReadinessOutcome( + provider=local_install.LOCAL_PROVIDER_NAME, + status=status, + reason_code=reason_code, + target={"model_id": LOCAL_MODEL}, + install={ + "install_state": "idle", + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": ram_sufficient, + "gpu_available": True, + "gpu_probe_ok": True, + "backend": backend, + "backend_reason": backend_reason, + }, + artifacts={ + "binary_installed": binary_installed, + "model_installed": model_installed, + "binary_path": str(binary_path), + "model_path": str(model_path), + "mmproj_path": str(mmproj_path) if mmproj_path is not None else None, + }, + proof={ + "binary": { + "status": binary_status, + "reason_code": "ready" if binary_installed else "manifest_missing", + "cache_hit": False, + }, + "model": { + "status": model_status, + "reason_code": "ready" if model_installed else "manifest_missing", + "cache_hit": False, + }, + }, + ) -def _local_slot() -> dict: - return read_journal_config()["providers"]["bundled"]["local"] +def _write_ready_vulkan_manifest( + *, + artifact_key: str | None = None, + pin: dict[str, str] | None = None, +) -> None: + resolved_pin = pin or local_install.pin_for_current_platform() + resolved_key = artifact_key or local_install.llama_server_artifact_key() + local_install._write_vulkan_manifest( + artifact_key=resolved_key, + pin=resolved_pin, + attempt_status=None, + fingerprint=local_install.target_fingerprint(LOCAL_MODEL), + ) + + +def _write_ready_model_manifest(model_id: str = LOCAL_MODEL) -> None: + local_install._write_model_manifest( + model_id=model_id, + attempt_status=None, + fingerprint=local_install.target_fingerprint(model_id), + ) def _fit(severity: fit_report.FitSeverity) -> fit_report.FitReport: @@ -102,15 +203,13 @@ def _download_with_fake_stream( chunks: list[bytes], chunk_times: list[float], ) -> tuple[Path, list[tuple[int, int | None]]]: - clock = [0.0] total = sum(len(chunk) for chunk in chunks) calls: list[tuple[int, int | None]] = [] - monkeypatch.setattr(local_install.time, "monotonic", lambda: clock[0]) def fake_stream(method, url, **_kwargs): assert method == "GET" assert url == "https://example.test/artifact" - return _FakeStream(chunks, chunk_times, total, clock) + return _FakeStream(chunks, chunk_times, total, [0.0]) def record_progress(received: int, reported_total: int | None) -> None: calls.append((received, reported_total)) @@ -129,7 +228,7 @@ def test_install_hint_literal() -> None: assert local_install.install_hint() == "journal install-provider local" -def test_download_file_rate_limits_many_progress_chunks(tmp_path, monkeypatch): +def test_download_file_reports_each_progress_chunk(tmp_path, monkeypatch): chunks = [b"x"] * 20 chunk_times = [index * 0.01 for index in range(len(chunks))] @@ -138,8 +237,7 @@ def test_download_file_rate_limits_many_progress_chunks(tmp_path, monkeypatch): ) total = sum(len(chunk) for chunk in chunks) - assert calls == [(1, total), (total, total)] - assert len(calls) < len(chunks) + assert calls == [(index, total) for index in range(1, len(chunks) + 1)] def test_download_file_emits_first_progress_promptly(tmp_path, monkeypatch): @@ -162,11 +260,15 @@ def test_download_file_emits_interval_crossing_progress(tmp_path, monkeypatch): ) total = sum(len(chunk) for chunk in chunks) - boundary_received = sum(len(chunk) for chunk in chunks[:4]) - assert calls == [(1, total), (boundary_received, total), (total, total)] + expected = [] + received = 0 + for chunk in chunks: + received += len(chunk) + expected.append((received, total)) + assert calls == expected -def test_download_file_emits_final_progress_once_with_dedupe(tmp_path, monkeypatch): +def test_download_file_emits_each_final_chunk_once(tmp_path, monkeypatch): chunks = [b"aa", b"bbb"] _dest, inside_window_calls = _download_with_fake_stream( @@ -350,13 +452,18 @@ def test_install_llama_server_relocates_binary_and_libraries(tmp_path, monkeypat result = local_install.install_llama_server() - assert result["install_state"] == "installed" + assert result["install_state"] == "verifying" + assert prove_manifest( + artifact_manifest_path(install_dir), + provider="local", + pin_identity=local_install._vulkan_pin_identity(artifact_key, pin), + ).ready assert_flat_layout() assert quarantine_calls == [install_dir] result = local_install.install_llama_server() - assert result["install_state"] == "installed" + assert result["install_state"] == "verifying" assert_flat_layout() assert quarantine_calls == [install_dir, install_dir] @@ -408,10 +515,7 @@ def test_install_llama_server_sha256_mismatch_fails_closed_before_extract( assert status["install_state"] == "failed" assert status["install_error"] is not None assert "sha256 mismatch" in status["install_error"] - slot = _local_slot() - assert slot["binary_artifact"] == pin["filename"] - assert "binary_sha256" not in slot - assert "binary_path" not in slot + assert not artifact_manifest_path(install_dir).exists() assert not binary_path.exists() assert not (install_dir / inner_name).exists() assert sorted(child.name for child in install_dir.iterdir()) == [pin["filename"]] @@ -428,7 +532,7 @@ def test_install_llama_server_writes_canonical_sequence(tmp_path, monkeypatch): final_path = local_install.binary_path_for_pin("test-platform", pin) final_path.parent.mkdir(parents=True) final_path.write_text("binary", encoding="utf-8") - observed: list[tuple[str, str, dict]] = [] + observed: list[tuple[str, str]] = [] monkeypatch.setattr( local_install, "llama_server_artifact_key", lambda: "test-platform" @@ -436,14 +540,14 @@ def test_install_llama_server_writes_canonical_sequence(tmp_path, monkeypatch): monkeypatch.setattr(local_install, "pin_for_current_platform", lambda: pin) def fake_download(_url, _dest, **_kwargs): - observed.append( - ("download", _local_status()["install_state"], dict(_local_slot())) - ) + observed.append(("download", _local_status()["install_state"])) + _dest.parent.mkdir(parents=True, exist_ok=True) + _dest.write_bytes(b"artifact") + _dest.parent.mkdir(parents=True, exist_ok=True) + _dest.write_bytes(b"artifact") def fake_verify(_path, _expected): - observed.append( - ("verify", _local_status()["install_state"], dict(_local_slot())) - ) + observed.append(("verify", _local_status()["install_state"])) monkeypatch.setattr(local_install, "_download_file", fake_download) monkeypatch.setattr(local_install, "_verify_sha256", fake_verify) @@ -460,14 +564,13 @@ def test_install_llama_server_writes_canonical_sequence(tmp_path, monkeypatch): assert [entry[0] for entry in observed] == ["download", "verify"] assert observed[0][1] == "downloading" - assert observed[0][2]["binary_artifact"] == "llama.tar.gz" assert observed[1][1] == "verifying" - assert result["install_state"] == "installed" - slot = _local_slot() - assert slot["binary_artifact"] == "llama.tar.gz" - assert slot["binary_sha256"] == "abc123" - assert slot["binary_path"] == str(final_path) - assert "state" not in slot + assert result["install_state"] == "verifying" + assert prove_manifest( + artifact_manifest_path(final_path.parent), + provider=local_install.LOCAL_PROVIDER_NAME, + pin_identity=local_install._vulkan_pin_identity("test-platform", pin), + ).ready @pytest.mark.parametrize( @@ -492,7 +595,6 @@ def test_install_llama_server_cuda_uses_arch_specific_oci_wanted_files( "resolve_local_backend", lambda _pin: local_cuda.BackendChoice("cuda", "test cuda"), ) - metadata_calls: list[dict[str, str]] = [] pull_calls: list[tuple[str, str, tuple[str, ...], Path]] = [] def fake_pull_and_install( @@ -515,16 +617,10 @@ def test_install_llama_server_cuda_uses_arch_specific_oci_wanted_files( ) monkeypatch.setattr(oci_image, "pull_and_install", fake_pull_and_install) - monkeypatch.setattr( - local_install, - "_write_local_metadata", - lambda updates: metadata_calls.append(updates), - ) - result = local_install.install_llama_server() wanted_files = local_install.CUDA_SERVER_PIN.wanted_files_for_arch(arch) - assert result["install_state"] == "installed" + assert result["install_state"] == "verifying" assert pull_calls == [ ( local_install.CUDA_SERVER_PIN.image_ref, @@ -535,7 +631,6 @@ def test_install_llama_server_cuda_uses_arch_specific_oci_wanted_files( ] assert expected_cpu in pull_calls[0][2] assert unexpected_cpu not in pull_calls[0][2] - assert metadata_calls == [] assert local_install.cuda_binary_path().stat().st_mode & 0o111 @@ -575,8 +670,12 @@ def test_install_llama_server_vulkan_choice_does_not_pull_oci(tmp_path, monkeypa result = local_install.install_llama_server() - assert result["install_state"] == "installed" - assert _local_slot()["binary_path"] == str(final_path) + assert result["install_state"] == "verifying" + assert prove_manifest( + artifact_manifest_path(final_path.parent), + provider=local_install.LOCAL_PROVIDER_NAME, + pin_identity=local_install._vulkan_pin_identity("test-platform", pin), + ).ready def test_probe_binary_runnable_returns_true_for_zero_exit(tmp_path): @@ -638,17 +737,15 @@ def test_probe_binary_runnable_handles_missing_path(tmp_path): def test_install_model_writes_canonical_sequence(tmp_path, monkeypatch): _init_journal(tmp_path, monkeypatch) spec = LOCAL_MODEL_SPECS[LOCAL_MODEL] - observed: list[tuple[str, str, dict]] = [] + observed: list[tuple[str, str]] = [] def fake_download(_url, _dest, **_kwargs): - observed.append( - ("download", _local_status()["install_state"], dict(_local_slot())) - ) + observed.append(("download", _local_status()["install_state"])) + _dest.parent.mkdir(parents=True, exist_ok=True) + _dest.write_bytes(b"artifact") def fake_verify(_path, _expected): - observed.append( - ("verify", _local_status()["install_state"], dict(_local_slot())) - ) + observed.append(("verify", _local_status()["install_state"])) monkeypatch.setattr(local_install, "_download_file", fake_download) monkeypatch.setattr(local_install, "_verify_sha256", fake_verify) @@ -662,16 +759,13 @@ def test_install_model_writes_canonical_sequence(tmp_path, monkeypatch): "verify", ] assert observed[0][1] == "downloading" - assert observed[0][2]["model_id"] == LOCAL_MODEL assert observed[2][1] == "verifying" - assert result["install_state"] == "installed" - slot = _local_slot() - assert slot["model_id"] == LOCAL_MODEL - assert slot["model_path"] == str(local_install.model_path(spec.model_id)) - assert slot["model_sha256"] == spec.sha256 - assert slot["mmproj_path"] == str(local_install.mmproj_path(spec.model_id)) - assert slot["mmproj_sha256"] == spec.mmproj_sha256 - assert "state" not in slot + assert result["install_state"] == "verifying" + assert prove_manifest( + artifact_manifest_path(local_install.model_dir(spec.model_id)), + provider=local_install.LOCAL_PROVIDER_NAME, + pin_identity=local_install._model_pin_identity(spec.model_id), + ).ready def test_install_model_threads_optional_mmproj_artifact(tmp_path, monkeypatch): @@ -704,9 +798,11 @@ def test_install_model_threads_optional_mmproj_artifact(tmp_path, monkeypatch): assert mmproj_path is not None assert downloads == [gguf_path, mmproj_path] assert verifies == [(gguf_path, spec.sha256), (mmproj_path, "mmproj-sha")] - slot = _local_slot() - assert slot["mmproj_path"] == str(mmproj_path) - assert slot["mmproj_sha256"] == "mmproj-sha" + assert prove_manifest( + artifact_manifest_path(local_install.model_dir(LOCAL_MODEL)), + provider=local_install.LOCAL_PROVIDER_NAME, + pin_identity=local_install._model_pin_identity(LOCAL_MODEL), + ).ready def test_install_local_blocks_before_downloads(tmp_path, monkeypatch): @@ -714,7 +810,12 @@ def test_install_local_blocks_before_downloads(tmp_path, monkeypatch): monkeypatch.setattr( local_install, "inspect_readiness", - lambda model_id: {"binary_installed": False, "model_installed": False}, + lambda model_id: _fake_local_readiness( + binary_installed=False, + model_installed=False, + binary_path=tmp_path / "llama-server", + model_path=tmp_path / "model.gguf", + ), ) monkeypatch.setattr( fit_report, "build_local_fit_report", lambda model_id: _fit("blocked") @@ -748,12 +849,20 @@ def test_install_local_warning_continues_to_download(tmp_path, monkeypatch): final_path.parent.mkdir(parents=True) final_path.write_text("binary", encoding="utf-8") downloads: list[Path] = [] + readiness_calls = 0 + + def fake_readiness(model_id: str) -> ReadinessOutcome: + nonlocal readiness_calls + readiness_calls += 1 + return _fake_local_readiness( + binary_installed=readiness_calls > 1, + model_installed=readiness_calls > 1, + binary_path=final_path, + model_path=local_install.model_path(model_id), + mmproj_path=local_install.mmproj_path(model_id), + ) - monkeypatch.setattr( - local_install, - "inspect_readiness", - lambda model_id: {"binary_installed": False, "model_installed": False}, - ) + monkeypatch.setattr(local_install, "inspect_readiness", fake_readiness) monkeypatch.setattr( fit_report, "build_local_fit_report", lambda model_id: _fit("warning") ) @@ -793,7 +902,12 @@ def test_install_local_ready_short_circuits_before_fit_report(tmp_path, monkeypa monkeypatch.setattr( local_install, "inspect_readiness", - lambda model_id: {"binary_installed": True, "model_installed": True}, + lambda model_id: _fake_local_readiness( + binary_installed=True, + model_installed=True, + binary_path=tmp_path / "llama-server", + model_path=tmp_path / "model.gguf", + ), ) monkeypatch.setattr( fit_report, @@ -813,7 +927,7 @@ def test_install_local_ready_short_circuits_before_fit_report(tmp_path, monkeypa result = local_install.install_local(LOCAL_MODEL) - assert result["name"] == local_install.LOCAL_PROVIDER_NAME + assert result["provider"] == local_install.LOCAL_PROVIDER_NAME assert result["install_state"] == "installed" @@ -827,31 +941,33 @@ def test_install_local_reinstalls_runtime_when_binary_record_stale( mmproj = local_install.mmproj_path(LOCAL_MODEL) assert mmproj is not None mmproj.write_text("mmproj", encoding="utf-8") - canonical = local_install.binary_path_for_pin() - canonical.parent.mkdir(parents=True, exist_ok=True) - canonical.write_bytes(b"llama-server") - canonical.chmod(0o755) - local_install._write_local_metadata( - { - "binary_artifact": "llama-stale-bin-ubuntu-x64.tar.gz", - "binary_sha256": "deadbeef" * 8, - "binary_path": str(canonical), - } - ) monkeypatch.setattr(local_vulkan, "detect_gpus", lambda: []) monkeypatch.setattr( fit_report, "build_local_fit_report", lambda model_id: _fit("ok") ) calls: list[str] = [] + readiness_calls = 0 + + def fake_readiness(model_id: str) -> ReadinessOutcome: + nonlocal readiness_calls + readiness_calls += 1 + return _fake_local_readiness( + binary_installed=readiness_calls > 1, + model_installed=readiness_calls > 1, + binary_path=local_install.binary_path_for_pin(), + model_path=local_install.model_path(model_id), + mmproj_path=local_install.mmproj_path(model_id), + ) - def fake_install_llama_server(): + def fake_install_llama_server(**_kwargs): calls.append("llama_server") - return {"install_state": "installed"} + return {"install_state": "verifying"} - def fake_install_model(model_id: str): + def fake_install_model(model_id: str, **_kwargs): calls.append("model") - return {"install_state": "installed", "model_id": model_id} + return {"install_state": "verifying", "model_id": model_id} + monkeypatch.setattr(local_install, "inspect_readiness", fake_readiness) monkeypatch.setattr( local_install, "install_llama_server", fake_install_llama_server ) @@ -859,7 +975,7 @@ def test_install_local_reinstalls_runtime_when_binary_record_stale( result = local_install.install_local(LOCAL_MODEL) - assert result == {"install_state": "installed", "model_id": LOCAL_MODEL} + assert result["install_state"] == "installed" assert calls == ["llama_server", "model"] @@ -872,16 +988,15 @@ def test_ensure_artifacts_installed_returns_binary_gguf_and_optional_mmproj( monkeypatch.setattr( local_install, "inspect_readiness", - lambda model_id: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "binary_path": str(binary), - "model_path": str(gguf), - "mmproj_path": str(mmproj), - "backend": "vulkan", - "backend_reason": "test vulkan", - }, + lambda model_id: _fake_local_readiness( + binary_installed=True, + model_installed=True, + binary_path=binary, + model_path=gguf, + mmproj_path=mmproj, + backend="vulkan", + backend_reason="test vulkan", + ), ) assert local_install.ensure_artifacts_installed( @@ -904,16 +1019,15 @@ def test_ensure_artifacts_installed_ignores_low_memory_when_artifacts_exist( monkeypatch.setattr( local_install, "inspect_readiness", - lambda model_id: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": False, - "binary_path": str(binary), - "model_path": str(gguf), - "mmproj_path": None, - "backend": "vulkan", - "backend_reason": "test vulkan", - }, + lambda model_id: _fake_local_readiness( + binary_installed=True, + model_installed=True, + binary_path=binary, + model_path=gguf, + ram_sufficient=False, + backend="vulkan", + backend_reason="test vulkan", + ), ) assert local_install.ensure_artifacts_installed( @@ -935,16 +1049,14 @@ def test_ensure_artifacts_installed_returns_cuda_lib_dir(tmp_path, monkeypatch): monkeypatch.setattr( local_install, "inspect_readiness", - lambda model_id: { - "binary_installed": True, - "model_installed": True, - "ram_sufficient": True, - "binary_path": str(binary), - "model_path": str(gguf), - "mmproj_path": None, - "backend": "cuda", - "backend_reason": "test cuda", - }, + lambda model_id: _fake_local_readiness( + binary_installed=True, + model_installed=True, + binary_path=binary, + model_path=gguf, + backend="cuda", + backend_reason="test cuda", + ), ) assert local_install.ensure_artifacts_installed( @@ -975,16 +1087,12 @@ def test_ensure_artifacts_installed_raises_for_missing_artifacts( monkeypatch.setattr( local_install, "inspect_readiness", - lambda model_id: { - "binary_installed": binary_installed, - "model_installed": model_installed, - "ram_sufficient": True, - "binary_path": str(binary), - "model_path": str(gguf), - "mmproj_path": None, - "backend": "vulkan", - "backend_reason": "test vulkan", - }, + lambda model_id: _fake_local_readiness( + binary_installed=binary_installed, + model_installed=model_installed, + binary_path=binary, + model_path=gguf, + ), ) with pytest.raises(local_install.LocalProviderError) as exc_info: @@ -1005,7 +1113,7 @@ def test_inspect_readiness_reports_ram_sufficient_for_low_or_unknown_memory( readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["ram_sufficient"] is True + assert readiness.host["ram_sufficient"] is True @pytest.mark.parametrize("sidecar_ok", [True, False]) @@ -1024,8 +1132,24 @@ def test_inspect_readiness_cuda_uses_sidecar_full_set( ) binary = local_install.cuda_binary_path() binary.parent.mkdir(parents=True, exist_ok=True) - binary.write_text("binary", encoding="utf-8") - binary.chmod(0o755) + wanted_files = local_install.CUDA_SERVER_PIN.wanted_files_for_arch( + local_install._oci_arch() + ) + for name in wanted_files: + member = binary.parent / name + member.write_text(name, encoding="utf-8") + member.chmod(0o755) + (binary.parent / ".oci-install.json").write_text( + json.dumps( + { + "image_ref": local_install.CUDA_SERVER_PIN.image_ref, + "arch": local_install._oci_arch(), + "files": {name: "0" * 64 for name in wanted_files}, + } + ) + + "\n", + encoding="utf-8", + ) verify_calls: list[tuple[str, str, tuple[str, ...], Path]] = [] def fake_verify( @@ -1048,15 +1172,12 @@ def test_inspect_readiness_cuda_uses_sidecar_full_set( readiness = local_install.inspect_readiness(LOCAL_MODEL) - wanted_files = local_install.CUDA_SERVER_PIN.wanted_files_for_arch( - local_install._oci_arch() - ) - assert readiness["backend"] == "cuda" - assert readiness["backend_reason"] == "test cuda" - assert readiness["binary_path"] == str(binary) - assert readiness["binary_installed"] is sidecar_ok - assert readiness["gpu_available"] is True - assert readiness["gpu_probe_ok"] is True + assert readiness.host["backend"] == "cuda" + assert readiness.host["backend_reason"] == "test cuda" + assert readiness.artifacts["binary_path"] == str(binary) + assert readiness.artifacts["binary_installed"] is sidecar_ok + assert readiness.host["gpu_available"] is True + assert readiness.host["gpu_probe_ok"] is True assert verify_calls == [ ( local_install.CUDA_SERVER_PIN.image_ref, @@ -1084,9 +1205,9 @@ def test_inspect_readiness_reports_gpu_available_with_hardware(tmp_path, monkeyp readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["gpu_available"] is True - assert readiness["backend"] == "vulkan" - assert readiness["backend_reason"] == "test vulkan" + assert readiness.host["gpu_available"] is True + assert readiness.host["backend"] == "vulkan" + assert readiness.host["backend_reason"] == "test vulkan" def test_inspect_readiness_reports_gpu_unavailable_without_hardware( @@ -1097,7 +1218,7 @@ def test_inspect_readiness_reports_gpu_unavailable_without_hardware( readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["gpu_available"] is False + assert readiness.host["gpu_available"] is False def test_inspect_readiness_stale_non_cuda_binary_record_reports_not_installed( @@ -1109,18 +1230,12 @@ def test_inspect_readiness_stale_non_cuda_binary_record_reports_not_installed( canonical.parent.mkdir(parents=True, exist_ok=True) canonical.write_bytes(b"llama-server") canonical.chmod(0o755) - local_install._write_local_metadata( - { - "binary_artifact": "llama-stale-bin-ubuntu-x64.tar.gz", - "binary_sha256": "deadbeef" * 8, - "binary_path": str(canonical), - } - ) readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["binary_installed"] is False - assert readiness["binary_path"] == str(canonical) + assert readiness.artifacts["binary_installed"] is False + assert readiness.artifacts["binary_path"] == str(canonical) + assert readiness.proof["binary"]["status"] == "missing-or-mismatched" def test_inspect_readiness_matching_non_cuda_binary_record_reports_installed( @@ -1133,17 +1248,11 @@ def test_inspect_readiness_matching_non_cuda_binary_record_reports_installed( canonical.parent.mkdir(parents=True, exist_ok=True) canonical.write_bytes(b"llama-server") canonical.chmod(0o755) - local_install._write_local_metadata( - { - "binary_artifact": pin["filename"], - "binary_sha256": pin["sha256"], - "binary_path": str(canonical), - } - ) + _write_ready_vulkan_manifest(pin=pin) readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["binary_installed"] is True + assert readiness.artifacts["binary_installed"] is True def test_inspect_readiness_honors_vulkan_device_override(tmp_path, monkeypatch): @@ -1163,14 +1272,15 @@ def test_inspect_readiness_honors_vulkan_device_override(tmp_path, monkeypatch): ), ] monkeypatch.setattr(local_vulkan, "detect_gpus", lambda: devices) - local_install._write_local_metadata({"vulkan_device_index": "0"}) + _write_provider_local_config(tmp_path, {"vulkan_device_index": "0"}) assert local_install.gpu_device_override() == 0 - assert local_install.inspect_readiness(LOCAL_MODEL)["gpu_available"] is True + assert resolve_local_endpoint().is_bundled is True + assert local_install.inspect_readiness(LOCAL_MODEL).host["gpu_available"] is True - local_install._write_local_metadata({"vulkan_device_index": "1"}) + _write_provider_local_config(tmp_path, {"vulkan_device_index": "1"}) - assert local_install.inspect_readiness(LOCAL_MODEL)["gpu_available"] is False + assert local_install.inspect_readiness(LOCAL_MODEL).host["gpu_available"] is False def test_inspect_readiness_ignores_stale_model_path_after_model_change( @@ -1186,9 +1296,6 @@ def test_inspect_readiness_ignores_stale_model_path_after_model_change( stale_dir.mkdir(parents=True, exist_ok=True) stale_gguf = stale_dir / "coder-7b-Q4_K_M.gguf" stale_gguf.write_text("stale", encoding="utf-8") - local_install._write_local_metadata( - {"model_id": "local/old-coder-7b", "model_path": str(stale_gguf)} - ) # Stage the selected model's artifacts in its own directory. gguf = local_install.model_path(LOCAL_MODEL) @@ -1197,15 +1304,18 @@ def test_inspect_readiness_ignores_stale_model_path_after_model_change( mmproj = local_install.mmproj_path(LOCAL_MODEL) assert mmproj is not None mmproj.write_text("mmproj", encoding="utf-8") + _write_ready_model_manifest(LOCAL_MODEL) readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["model_id"] == LOCAL_MODEL - assert readiness["model_path"] == str(gguf) - assert readiness["mmproj_path"] == str(mmproj) - assert Path(readiness["model_path"]).parent == local_install.model_dir(LOCAL_MODEL) - assert readiness["model_path"] != str(stale_gguf) - assert readiness["model_installed"] is True + assert readiness.artifacts["model_id"] == LOCAL_MODEL + assert readiness.artifacts["model_path"] == str(gguf) + assert readiness.artifacts["mmproj_path"] == str(mmproj) + assert Path(readiness.artifacts["model_path"]).parent == local_install.model_dir( + LOCAL_MODEL + ) + assert readiness.artifacts["model_path"] != str(stale_gguf) + assert readiness.artifacts["model_installed"] is True def test_inspect_readiness_not_installed_off_stale_record(tmp_path, monkeypatch): @@ -1217,15 +1327,14 @@ def test_inspect_readiness_not_installed_off_stale_record(tmp_path, monkeypatch) stale_dir.mkdir(parents=True, exist_ok=True) stale_gguf = stale_dir / "coder-7b-Q4_K_M.gguf" stale_gguf.write_text("stale", encoding="utf-8") - local_install._write_local_metadata( - {"model_id": "local/old-coder-7b", "model_path": str(stale_gguf)} - ) readiness = local_install.inspect_readiness(LOCAL_MODEL) - assert readiness["model_installed"] is False - assert readiness["gguf_installed"] is False - assert readiness["model_path"] == str(local_install.model_path(LOCAL_MODEL)) + assert readiness.artifacts["model_installed"] is False + assert readiness.artifacts["gguf_installed"] is False + assert readiness.artifacts["model_path"] == str( + local_install.model_path(LOCAL_MODEL) + ) def test_install_llama_server_failure_writes_canonical_failed(tmp_path, monkeypatch): @@ -1252,5 +1361,3 @@ def test_install_llama_server_failure_writes_canonical_failed(tmp_path, monkeypa status = _local_status() assert status["install_state"] == "failed" assert status["install_error"] == "network broke" - slot = _local_slot() - assert "state" not in slot diff --git a/tests/test_mlx_install.py b/tests/test_mlx_install.py index 40c5ca364..1d09ddbfc 100644 --- a/tests/test_mlx_install.py +++ b/tests/test_mlx_install.py @@ -18,6 +18,11 @@ from huggingface_hub import constants as hf_constants from solstone.think.journal_config import read_journal_config from solstone.think.models import GEMMA4_26B_A4B_4BIT, QWEN_35_9B from solstone.think.providers import fit_report, memory, mlx_install +from solstone.think.providers.artifact_proof import ( + mlx_snapshot_manifest_path, + mlx_variant_manifest_path, + prove_manifest, +) from solstone.think.providers.install_state import read_install_status @@ -33,15 +38,12 @@ def _init_journal(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def _local_status() -> dict: - return read_install_status(scope="bundled", name="local") - - -def _local_slot() -> dict: - return read_journal_config()["providers"]["bundled"]["local"] + return read_install_status(name="local") def _allow_install(monkeypatch: pytest.MonkeyPatch, *, ram_gb: int = 64) -> None: monkeypatch.setattr(mlx_install, "_check_platform_and_package", lambda: (True, "")) + monkeypatch.setattr(mlx_install, "is_mlx_platform_supported", lambda: True) monkeypatch.setattr( memory.psutil, "virtual_memory", @@ -104,6 +106,15 @@ def _write_snapshot( return snapshot_dir, hashlib.sha256(weight_bytes).hexdigest() +def _metadata_for_snapshot(snapshot_dir: Path) -> dict[str, tuple[str, int]]: + metadata: dict[str, tuple[str, int]] = {} + for path in snapshot_dir.rglob("*.safetensors"): + rel_path = path.relative_to(snapshot_dir).as_posix() + digest = hashlib.sha256(path.read_bytes()).hexdigest() + metadata[rel_path] = (digest, path.stat().st_size) + return metadata + + def test_module_import_is_mlx_vlm_free(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setitem(sys.modules, "mlx_vlm", None) @@ -143,7 +154,7 @@ def test_inspect_readiness_ram_sufficient_matches_available_floor( ), ) - assert mlx_install.inspect_readiness(QWEN_35_9B)["ram_sufficient"] is True + assert mlx_install.inspect_readiness(QWEN_35_9B).host["ram_sufficient"] is True monkeypatch.setattr( memory.psutil, @@ -155,7 +166,7 @@ def test_inspect_readiness_ram_sufficient_matches_available_floor( ) readiness = mlx_install.inspect_readiness(QWEN_35_9B) - assert readiness["ram_sufficient"] is False + assert readiness.host["ram_sufficient"] is False def test_install_local_mlx_writes_canonical_sequence( @@ -180,6 +191,7 @@ def test_install_local_mlx_writes_canonical_sequence( def fake_verify(_spec, _snapshot_dir): observed.append(("verify", _local_status()["install_state"], {})) + return _metadata_for_snapshot(_snapshot_dir) def fake_create(_snapshot_dir): observed.append(("install", _local_status()["install_state"], {})) @@ -206,11 +218,16 @@ def test_install_local_mlx_writes_canonical_sequence( "installing", ] assert result["install_state"] == "installed" - slot = _local_slot() - assert slot["mlx_model_id"] == GEMMA4_26B_A4B_4BIT - assert slot["mlx_revision"] == spec.revision - assert slot["mlx_snapshot_dir"] == str(snapshot_dir) - assert slot["mlx_variant_dir"] == str(variant_dir) + assert prove_manifest( + mlx_snapshot_manifest_path(spec.repo, spec.revision), + provider="local", + pin_identity=mlx_install._pin_identity(spec), + ).ready + assert prove_manifest( + mlx_variant_manifest_path(spec.repo, spec.revision), + provider="local", + pin_identity=mlx_install._variant_pin_identity(spec), + ).ready assert "mlx" not in read_journal_config()["providers"] @@ -277,7 +294,11 @@ def test_install_local_mlx_warning_continues_to_download( return str(snapshot_dir) monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download) - monkeypatch.setattr(mlx_install, "validate_snapshot_sha256", lambda *_args: None) + monkeypatch.setattr( + mlx_install, + "validate_snapshot_sha256", + lambda _spec, snapshot_dir: _metadata_for_snapshot(snapshot_dir), + ) assert mlx_install.install_local_mlx()["install_state"] == "installed" assert calls == {"download": 1} @@ -346,7 +367,11 @@ def test_installing_failure_transitions_failed( monkeypatch.setattr( huggingface_hub, "snapshot_download", lambda **_kwargs: str(snapshot_dir) ) - monkeypatch.setattr(mlx_install, "validate_snapshot_sha256", lambda *_args: None) + monkeypatch.setattr( + mlx_install, + "validate_snapshot_sha256", + lambda _spec, snapshot_dir: _metadata_for_snapshot(snapshot_dir), + ) with pytest.raises(FileNotFoundError): mlx_install.install_local_mlx(GEMMA4_26B_A4B_4BIT) @@ -374,6 +399,7 @@ def test_idempotent_rerun_skips_network( def fake_verify(_spec, _snapshot_dir): calls["verify"] += 1 + return _metadata_for_snapshot(_snapshot_dir) monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download) monkeypatch.setattr(mlx_install, "validate_snapshot_sha256", fake_verify) @@ -388,7 +414,11 @@ def test_idempotent_rerun_skips_network( ) assert mlx_install.install_local_mlx()["install_state"] == "installed" assert calls == {"download": 1, "verify": 1} - assert _local_slot()["mlx_model_id"] == QWEN_35_9B + assert prove_manifest( + mlx_snapshot_manifest_path(spec.repo, spec.revision), + provider="local", + pin_identity=mlx_install._pin_identity(spec), + ).ready def test_validate_snapshot_sha256_uses_lfs_metadata( diff --git a/tests/test_parakeet_install.py b/tests/test_parakeet_install.py index 53dfb0b83..dbef2e4bc 100644 --- a/tests/test_parakeet_install.py +++ b/tests/test_parakeet_install.py @@ -7,19 +7,17 @@ import io import os import shutil import tarfile -from contextlib import contextmanager from pathlib import Path from typing import Any import pytest from solstone.think import parakeet_readiness -from solstone.think.journal_config import ( - JournalConfigTransaction, - read_journal_config, -) -from solstone.think.journal_io.errors import LockTimeout from solstone.think.providers import fit_report, parakeet_install +from solstone.think.providers.artifact_proof import ( + artifact_manifest_path, + prove_manifest, +) from solstone.think.providers.install_state import read_install_status from tests.helpers.journal_config import seed_journal_config @@ -51,11 +49,7 @@ def _init_journal(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def _parakeet_status() -> dict: - return read_install_status(scope="bundled", name="parakeet") - - -def _parakeet_slot() -> dict: - return read_journal_config()["providers"]["bundled"]["parakeet"] + return read_install_status(name="parakeet") def _fit(severity: fit_report.FitSeverity) -> fit_report.FitReport: @@ -94,9 +88,33 @@ def _stage_ready_files() -> tuple[Path, Path, Path]: path.chmod(0o755) model.parent.mkdir(parents=True, exist_ok=True) model.write_text("model\n", encoding="utf-8") + fingerprint = parakeet_install.target_fingerprint() + for backend in parakeet_readiness.PARAKEET_CPP_BINARY_BACKENDS: + parakeet_install._write_binary_manifest( + backend=backend, + attempt_status=None, + fingerprint=fingerprint, + journal_path=None, + ) + parakeet_install._write_model_manifest( + attempt_status=None, + fingerprint=fingerprint, + journal_path=None, + ) return cpu, vulkan, model +def _write_ready_binary_manifests() -> None: + fingerprint = parakeet_install.target_fingerprint() + for backend in parakeet_readiness.PARAKEET_CPP_BINARY_BACKENDS: + parakeet_install._write_binary_manifest( + backend=backend, + attempt_status=None, + fingerprint=fingerprint, + journal_path=None, + ) + + def test_install_hint_literal() -> None: assert parakeet_install.install_hint() == "journal install-provider parakeet" @@ -169,7 +187,7 @@ def test_install_parakeet_server_relocates_and_chmods_binary( install_dir = parakeet_install.binary_install_dir("cpu") final_path = parakeet_install.binary_path("cpu") - assert result["install_state"] == "installed" + assert result["install_state"] == "verifying" assert final_path.exists() assert ( final_path.read_bytes() @@ -182,6 +200,13 @@ def test_install_parakeet_server_relocates_and_chmods_binary( install_dir / f"parakeet-{parakeet_readiness.PARAKEET_CPP_RELEASE_TAG}-bin-linux-cpu-x64" ).exists() + assert prove_manifest( + artifact_manifest_path(install_dir), + provider=parakeet_install.PARAKEET_PROVIDER_NAME, + pin_identity=parakeet_install._binary_pin_identity( + parakeet_install.parakeet_server_artifact_key(), "cpu" + ), + ).ready def test_sha256_mismatch_fails_closed_and_records_failed_state( @@ -205,7 +230,7 @@ def test_sha256_mismatch_fails_closed_and_records_failed_state( assert "sha256 mismatch" in status["install_error"] -def test_install_parakeet_writes_distinct_binary_and_model_metadata( +def test_install_parakeet_writes_distinct_binary_and_model_manifests( tmp_path, monkeypatch ) -> None: _init_journal(tmp_path, monkeypatch) @@ -230,107 +255,41 @@ def test_install_parakeet_writes_distinct_binary_and_model_metadata( result = parakeet_install.install_parakeet() assert result["install_state"] == "installed" - slot = _parakeet_slot() - assert ( - slot["binary_artifact_cpu"] - == parakeet_install.PARAKEET_SERVER_PINS[ - (parakeet_install.parakeet_server_artifact_key(), "cpu") - ]["filename"] + artifact_key = parakeet_install.parakeet_server_artifact_key() + cpu_manifest = artifact_manifest_path(parakeet_install.binary_install_dir("cpu")) + vulkan_manifest = artifact_manifest_path( + parakeet_install.binary_install_dir("vulkan") ) + assert parakeet_install.binary_path("cpu") != parakeet_install.binary_path("vulkan") + assert parakeet_install.binary_path("cpu").is_file() + assert parakeet_install.binary_path("vulkan").is_file() + assert parakeet_install.model_path().is_file() assert ( - slot["binary_artifact_vulkan"] - == parakeet_install.PARAKEET_SERVER_PINS[ - (parakeet_install.parakeet_server_artifact_key(), "vulkan") - ]["filename"] + prove_manifest( + cpu_manifest, + provider=parakeet_install.PARAKEET_PROVIDER_NAME, + pin_identity=parakeet_install._binary_pin_identity(artifact_key, "cpu"), + ).ready + is True ) - assert slot["binary_path_cpu"] != slot["binary_path_vulkan"] - assert Path(slot["binary_path_cpu"]).is_file() - assert Path(slot["binary_path_vulkan"]).is_file() - assert slot["model_repo"] == parakeet_readiness.PARAKEET_CPP_MODEL_REPO - assert slot["model_filename"] == parakeet_readiness.PARAKEET_CPP_MODEL_FILENAME - assert slot["model_revision"] == parakeet_readiness.PARAKEET_CPP_MODEL_REVISION - assert slot["model_path"] == str(parakeet_install.model_path()) - assert Path(slot["model_path"]).is_file() - - -def test_write_parakeet_metadata_waits_for_config_lock_and_preserves_commits( - tmp_path, - monkeypatch, -) -> None: - _init_journal(tmp_path, monkeypatch) - journal_path = tmp_path - config = { - "setup": { - "completed_at": "2026-07-01T00:00:00+00:00", - "completed_by": "setup-writer", - }, - "service": {"port": 5015, "host": "127.0.0.1"}, - "providers": {"bundled": {}}, - } - calls: list[tuple[str, object]] = [] - - def recording_mutate(mutator, *, journal_path=None): - calls.append(("mutate", journal_path)) - mutation = mutator(config) - return JournalConfigTransaction( - value=mutation.value, - changed=mutation.changed, - written=mutation.changed, - ) - - monkeypatch.setattr(parakeet_install, "mutate_journal_config", recording_mutate) - - parakeet_install._write_parakeet_metadata( - {"model_repo": "openai/parakeet-test"}, - journal_path=journal_path, + assert ( + prove_manifest( + vulkan_manifest, + provider=parakeet_install.PARAKEET_PROVIDER_NAME, + pin_identity=parakeet_install._binary_pin_identity(artifact_key, "vulkan"), + ).ready + is True ) - - assert calls == [("mutate", journal_path)] - persisted = config - assert persisted["setup"]["completed_at"] == "2026-07-01T00:00:00+00:00" - assert persisted["setup"]["completed_by"] == "setup-writer" - assert persisted["service"]["port"] == 5015 - assert persisted["service"]["host"] == "127.0.0.1" assert ( - persisted["providers"]["bundled"]["parakeet"]["model_repo"] - == "openai/parakeet-test" + prove_manifest( + artifact_manifest_path(parakeet_install.model_dir()), + provider=parakeet_install.PARAKEET_PROVIDER_NAME, + pin_identity=parakeet_install._model_pin_identity(), + ).ready + is True ) -def test_write_parakeet_metadata_rejects_unknown_key_without_lock( - monkeypatch, -) -> None: - mutate_calls: list[object] = [] - - def recording_mutate(mutator, *, journal_path=None): - mutate_calls.append(journal_path) - pytest.fail("metadata validation must happen before transaction entry") - - monkeypatch.setattr(parakeet_install, "mutate_journal_config", recording_mutate) - - with pytest.raises(ValueError) as exc_info: - parakeet_install._write_parakeet_metadata({"unexpected": "value"}) - - assert str(exc_info.value) == "unknown parakeet install metadata key: unexpected" - assert mutate_calls == [] - - -def test_write_parakeet_metadata_propagates_config_lock_timeout_without_config_io( - monkeypatch, -) -> None: - timeout = LockTimeout(path=Path("busy.lock"), timeout=0.01) - - def busy_mutate(mutator, *, journal_path=None): - raise timeout - - monkeypatch.setattr(parakeet_install, "mutate_journal_config", busy_mutate) - - with pytest.raises(LockTimeout) as exc_info: - parakeet_install._write_parakeet_metadata({"model_repo": "openai/test"}) - - assert exc_info.value is timeout - - def test_install_parakeet_blocks_before_downloads(tmp_path, monkeypatch) -> None: _init_journal(tmp_path, monkeypatch) monkeypatch.setattr( @@ -385,34 +344,25 @@ def test_install_parakeet_warning_continues_to_download( assert downloads -def test_install_parakeet_rechecks_readiness_under_provider_lock( +def test_install_parakeet_ready_short_circuits_before_component_installs( tmp_path, monkeypatch ) -> None: _init_journal(tmp_path, monkeypatch) _stage_ready_files() - lock_calls = [] - - @contextmanager - def fake_hold_install_lock(journal_path=None): - lock_calls.append(parakeet_install._install_lock_path(journal_path)) - yield - - monkeypatch.setattr(parakeet_install, "_hold_install_lock", fake_hold_install_lock) monkeypatch.setattr( parakeet_install, "_install_parakeet_server_unlocked", - lambda *_args: pytest.fail("ready artifacts should not reinstall"), + lambda *_args, **_kwargs: pytest.fail("ready artifacts should not reinstall"), ) monkeypatch.setattr( parakeet_install, "_install_model_unlocked", - lambda: pytest.fail("ready model should not reinstall"), + lambda *_args, **_kwargs: pytest.fail("ready model should not reinstall"), ) result = parakeet_install.install_parakeet() assert result["install_state"] == "installed" - assert lock_calls == [parakeet_install.cache_root() / "install"] def test_ensure_artifacts_installed_resolves_requested_backend( @@ -440,6 +390,7 @@ def test_ensure_artifacts_installed_reports_missing_binary_and_model( path.parent.mkdir(parents=True, exist_ok=True) path.write_text("server\n", encoding="utf-8") path.chmod(0o755) + _write_ready_binary_manifests() with pytest.raises(parakeet_install.ParakeetProviderError) as model_exc: parakeet_install.ensure_artifacts_installed("cpu") diff --git a/tests/test_provider_state.py b/tests/test_provider_state.py index 59fd83592..d6b5a4062 100644 --- a/tests/test_provider_state.py +++ b/tests/test_provider_state.py @@ -16,6 +16,7 @@ from solstone.think.providers import ( mlx_install, state, ) +from solstone.think.providers.artifact_proof import ReadinessOutcome from solstone.think.providers.shared import ( RUNTIME_REASON_CODES, classify_provider_error, @@ -45,23 +46,62 @@ def _readiness( gpu_probe_ok: bool | None = None, install_state: str = "installed", backend: str = "vulkan", -) -> dict: - payload = { - "install_state": install_state, - "binary_installed": binary, - "model_installed": model, - "ram_sufficient": ram, - "gpu_available": gpu, - "binary_path": "/tmp/llama-server", - "model_path": "/tmp/model.gguf", - "model_id": LOCAL_MODEL, - "install_error": None, - "backend": backend, - "backend_reason": f"test {backend}", - } - if gpu_probe_ok is not None: - payload["gpu_probe_ok"] = gpu_probe_ok - return payload +) -> ReadinessOutcome: + missing_binary = not binary + missing_model = not model + status = "ready" if binary and model and ram and gpu else "missing-or-mismatched" + reason_code = ( + "ready" + if status == "ready" + else "binary_missing" + if missing_binary + else "model_missing" + if missing_model + else "host_unfit" + ) + return ReadinessOutcome( + provider="local", + status=status, + reason_code=reason_code, + target={"model_id": LOCAL_MODEL}, + install={ + "install_state": install_state, + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": ram, + "gpu_available": gpu, + "gpu_probe_ok": True if gpu_probe_ok is None else gpu_probe_ok, + "backend": backend, + "backend_reason": f"test {backend}", + }, + artifacts={ + "binary_installed": binary, + "model_installed": model, + "binary_path": "/tmp/llama-server", + "model_path": "/tmp/model.gguf", + "model_id": LOCAL_MODEL, + "mmproj_path": None, + }, + proof={ + "binary": { + "status": "ready" if binary else "missing-or-mismatched", + "reason_code": "ready" if binary else "manifest_missing", + "cache_hit": False, + }, + "model": { + "status": "ready" if model else "missing-or-mismatched", + "reason_code": "ready" if model else "manifest_missing", + "cache_hit": False, + }, + }, + ) def _mlx_readiness( @@ -70,21 +110,49 @@ def _mlx_readiness( model_installed: bool = True, platform_supported: bool = True, package_available: bool = True, -) -> dict: - return { - "install_state": install_state, - "model_installed": model_installed, - "snapshot_installed": model_installed, - "variant_installed": True, - "ram_sufficient": True, - "platform_supported": platform_supported, - "package_available": package_available, - "model_id": QWEN_35_9B, - "snapshot_dir": "/tmp/snap", - "variant_dir": None, - "runtime_dir": "/tmp/snap", - "install_error": None, - } +) -> ReadinessOutcome: + return ReadinessOutcome( + provider="local", + status=( + "ready" + if model_installed and platform_supported and package_available + else "missing-or-mismatched" + ), + reason_code="ready" if model_installed else "manifest_missing", + target={"model_id": QWEN_35_9B}, + install={ + "install_state": install_state, + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "ram_sufficient": True, + "platform_supported": platform_supported, + "package_available": package_available, + }, + artifacts={ + "model_installed": model_installed, + "snapshot_installed": model_installed, + "variant_installed": True, + "model_id": QWEN_35_9B, + "snapshot_dir": "/tmp/snap", + "variant_dir": None, + "runtime_dir": "/tmp/snap", + }, + proof={ + "snapshot": { + "status": "ready" if model_installed else "missing-or-mismatched", + "reason_code": "ready" if model_installed else "manifest_missing", + "cache_hit": False, + }, + "variant": None, + }, + ) def _block_linux_local_path(monkeypatch) -> None: @@ -461,12 +529,17 @@ def test_local_readiness_gpu_unavailable_flows_from_inspect_without_launch( if mmproj is not None: mmproj.write_text("mmproj", encoding="utf-8") pin = local_install.pin_for_current_platform() - local_install._write_local_metadata( - { - "binary_artifact": pin["filename"], - "binary_sha256": pin["sha256"], - "binary_path": str(binary), - } + fingerprint = local_install.target_fingerprint(LOCAL_MODEL) + local_install._write_vulkan_manifest( + artifact_key=local_install.llama_server_artifact_key(), + pin=pin, + attempt_status=None, + fingerprint=fingerprint, + ) + local_install._write_model_manifest( + model_id=LOCAL_MODEL, + attempt_status=None, + fingerprint=fingerprint, ) monkeypatch.setattr(local_vulkan, "detect_gpus", lambda: []) @@ -479,9 +552,9 @@ def test_local_readiness_gpu_unavailable_flows_from_inspect_without_launch( readiness = local_install.inspect_readiness(LOCAL_MODEL) provider_state = state.readiness_for_provider("local", "generate") - assert readiness["binary_installed"] is True - assert readiness["model_installed"] is True - assert readiness["gpu_available"] is False + assert readiness.artifacts["binary_installed"] is True + assert readiness.artifacts["model_installed"] is True + assert readiness.host["gpu_available"] is False assert provider_state.status == "blocked" assert provider_state.reason_code == "gpu_unavailable" assert provider_state.source == "local_install" @@ -509,12 +582,17 @@ def test_local_readiness_gpu_probe_failed_flows_from_inspect_without_launch( if mmproj is not None: mmproj.write_text("mmproj", encoding="utf-8") pin = local_install.pin_for_current_platform() - local_install._write_local_metadata( - { - "binary_artifact": pin["filename"], - "binary_sha256": pin["sha256"], - "binary_path": str(binary), - } + fingerprint = local_install.target_fingerprint(LOCAL_MODEL) + local_install._write_vulkan_manifest( + artifact_key=local_install.llama_server_artifact_key(), + pin=pin, + attempt_status=None, + fingerprint=fingerprint, + ) + local_install._write_model_manifest( + model_id=LOCAL_MODEL, + attempt_status=None, + fingerprint=fingerprint, ) monkeypatch.setattr( @@ -539,10 +617,10 @@ def test_local_readiness_gpu_probe_failed_flows_from_inspect_without_launch( readiness = local_install.inspect_readiness(LOCAL_MODEL) provider_state = state.readiness_for_provider("local", "generate") - assert readiness["binary_installed"] is True - assert readiness["model_installed"] is True - assert readiness["gpu_available"] is True - assert readiness["gpu_probe_ok"] is False + assert readiness.artifacts["binary_installed"] is True + assert readiness.artifacts["model_installed"] is True + assert readiness.host["gpu_available"] is True + assert readiness.host["gpu_probe_ok"] is False assert provider_state.status == "blocked" assert provider_state.reason_code == "gpu_probe_failed" assert provider_state.source == "local_install" diff --git a/tests/test_supervisor.py b/tests/test_supervisor.py index 64295c4f9..c578a30fc 100644 --- a/tests/test_supervisor.py +++ b/tests/test_supervisor.py @@ -28,6 +28,47 @@ from solstone.think.processing import ( ProcessingSettings, TimeWindowSettings, ) +from solstone.think.providers.artifact_proof import ReadinessOutcome + + +def _mlx_readiness( + *, + model_installed: bool = True, + ram_sufficient: bool = True, + platform_supported: bool = True, + package_available: bool = True, + runtime_dir: str = "/tmp/snap", + model_id: str = "mlx-model", +) -> ReadinessOutcome: + ready = ( + model_installed and ram_sufficient and platform_supported and package_available + ) + return ReadinessOutcome( + provider="local", + status="ready" if ready else "missing-or-mismatched", + reason_code="ready" if ready else "manifest_missing", + target={"model_id": model_id}, + install={ + "install_state": "idle", + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={ + "platform_supported": platform_supported, + "package_available": package_available, + "ram_sufficient": ram_sufficient, + }, + artifacts={ + "model_installed": model_installed, + "runtime_dir": runtime_dir, + }, + proof={}, + ) @pytest.fixture(autouse=True) @@ -3139,14 +3180,10 @@ def test_start_local_server_launches_mlx_server_on_darwin( monkeypatch.setattr( mlx_install, "inspect_readiness", - lambda: { - "platform_supported": True, - "package_available": True, - "ram_sufficient": True, - "model_installed": True, - "runtime_dir": str(runtime_dir), - "model_id": "gemma-4-26b-a4b-it-mlx-4bit", - }, + lambda: _mlx_readiness( + runtime_dir=str(runtime_dir), + model_id="gemma-4-26b-a4b-it-mlx-4bit", + ), ) monkeypatch.setattr(mod, "find_available_port", lambda: 2468) monkeypatch.setattr( @@ -3201,12 +3238,7 @@ def test_start_local_server_skips_when_mlx_not_installed_on_darwin(monkeypatch): monkeypatch.setattr( mlx_install, "inspect_readiness", - lambda: { - "platform_supported": True, - "package_available": True, - "ram_sufficient": True, - "model_installed": False, - }, + lambda: _mlx_readiness(model_installed=False), ) launch = MagicMock() monkeypatch.setattr(mod, "_launch_process", launch) @@ -3223,12 +3255,7 @@ def test_start_local_server_skips_when_mlx_memory_blocked_on_darwin(monkeypatch) monkeypatch.setattr( mlx_install, "inspect_readiness", - lambda: { - "platform_supported": True, - "package_available": True, - "ram_sufficient": False, - "model_installed": True, - }, + lambda: _mlx_readiness(ram_sufficient=False), ) launch = MagicMock() monkeypatch.setattr(mod, "_launch_process", launch) diff --git a/tests/test_supervisor_parakeet.py b/tests/test_supervisor_parakeet.py index 64fa48e6d..a9b39dea9 100644 --- a/tests/test_supervisor_parakeet.py +++ b/tests/test_supervisor_parakeet.py @@ -17,6 +17,7 @@ from solstone.think.providers import ( parakeet_install, parakeet_server, ) +from solstone.think.providers.artifact_proof import ReadinessOutcome from solstone.think.providers.parakeet_placement import ( PARAKEET_ATT_CONTEXT_ENV, PARAKEET_ATT_CONTEXT_FRAMES, @@ -25,6 +26,45 @@ from solstone.think.providers.parakeet_placement import ( _LaunchRecord = dict[str, Any] +class _FakeLease: + def __init__(self) -> None: + self.released = False + + def release(self) -> None: + self.released = True + + +def _parakeet_readiness( + *, + install_state: str = "idle", + binary_installed: bool = False, + model_installed: bool = False, +) -> ReadinessOutcome: + ready = binary_installed and model_installed + return ReadinessOutcome( + provider="parakeet", + status="ready" if ready else "missing-or-mismatched", + reason_code="ready" if ready else "manifest_missing", + target={}, + install={ + "install_state": install_state, + "install_error": None, + "error_code": None, + "attempt_id": None, + "progress_bytes_received": None, + "progress_bytes_total": None, + "last_transition_at": None, + "last_progress_at": None, + }, + host={}, + artifacts={ + "binary_installed": binary_installed, + "model_installed": model_installed, + }, + proof={}, + ) + + def _assert_att_context(launch: _LaunchRecord) -> None: assert launch["env"][PARAKEET_ATT_CONTEXT_ENV] == str(PARAKEET_ATT_CONTEXT_FRAMES) @@ -627,7 +667,15 @@ def test_start_parakeet_server_starts_background_install_when_missing( def start(self): started.append(self.name) + class _Event: + def set(self): + return None + + def wait(self, timeout=None): + return True + monkeypatch.setattr(supervisor.threading, "Thread", _Thread) + monkeypatch.setattr(supervisor.threading, "Event", _Event) monkeypatch.setattr( parakeet_install, "ensure_artifacts_installed", @@ -640,13 +688,8 @@ def test_start_parakeet_server_starts_background_install_when_missing( monkeypatch.setattr( parakeet_install, "inspect_readiness", - lambda: { - "install_state": "idle", - "binary_installed": False, - "model_installed": False, - }, + lambda: _parakeet_readiness(), ) - monkeypatch.setattr(supervisor, "_parakeet_bootstrap_thread", None) assert supervisor.start_parakeet_server() is None assert started == ["parakeet-cpp-provider-bootstrap"] @@ -654,49 +697,67 @@ def test_start_parakeet_server_starts_background_install_when_missing( def test_parakeet_bootstrap_worker_requests_start_after_install(monkeypatch) -> None: - calls: list[str] = [] + calls: list[dict[str, Any]] = [] requests: list[str] = [] + lease = _FakeLease() + attempt_status = {"attempt_id": "attempt"} + ack = supervisor.threading.Event() monkeypatch.setattr( - parakeet_install, "install_parakeet", lambda: calls.append("install") + parakeet_install, + "install_parakeet", + lambda **kwargs: calls.append(kwargs), ) monkeypatch.setattr( supervisor, "_request_parakeet_server_start", lambda: requests.append("start") ) - monkeypatch.setattr( - supervisor, - "_parakeet_bootstrap_thread", - supervisor.threading.current_thread(), - ) - supervisor._run_parakeet_bootstrap_worker() + supervisor._run_parakeet_bootstrap_worker( + lease=lease, + attempt_status=attempt_status, + ack=ack, + ) - assert calls == ["install"] + assert calls == [ + {"journal_path": None, "lease": lease, "attempt_status": attempt_status} + ] assert requests == ["start"] - assert supervisor._parakeet_bootstrap_thread is None + assert ack.is_set() + assert lease.released is True def test_parakeet_bootstrap_worker_uses_captured_journal_path( monkeypatch, tmp_path ) -> None: - calls: list[Path] = [] + calls: list[dict[str, Any]] = [] requests: list[str] = [] journal_path = tmp_path / "original" + lease = _FakeLease() + attempt_status = {"attempt_id": "attempt"} + ack = supervisor.threading.Event() - def fake_install_parakeet(*, journal_path=None): - calls.append(journal_path) + def fake_install_parakeet(**kwargs): + calls.append(kwargs) monkeypatch.setattr(parakeet_install, "install_parakeet", fake_install_parakeet) monkeypatch.setattr( supervisor, "_request_parakeet_server_start", lambda: requests.append("start") ) - monkeypatch.setattr( - supervisor, - "_parakeet_bootstrap_thread", - supervisor.threading.current_thread(), - ) - supervisor._run_parakeet_bootstrap_worker(journal_path) + supervisor._run_parakeet_bootstrap_worker( + journal_path, + lease, + attempt_status, + ack, + ) - assert calls == [journal_path] + assert calls == [ + { + "journal_path": journal_path, + "lease": lease, + "attempt_status": attempt_status, + } + ] assert requests == ["start"] + assert ack.is_set() + assert lease.released is True