diff --git a/scripts/check_journal_io_mechanic.py b/scripts/check_journal_io_mechanic.py index cefd7230f..a0e2d7019 100644 --- a/scripts/check_journal_io_mechanic.py +++ b/scripts/check_journal_io_mechanic.py @@ -98,6 +98,7 @@ EXCLUDED_FILES: frozenset[str] = frozenset( "solstone/think/providers/mlx_install.py", "solstone/think/providers/ced_install.py", "solstone/think/providers/rerank_install.py", + "solstone/think/providers/rfdetr_install.py", "solstone/think/services/scout.py", "solstone/think/services/spl.py", "solstone/think/steward.py", diff --git a/solstone/THIRD_PARTY_NOTICES.md b/solstone/THIRD_PARTY_NOTICES.md index 1f11419f6..ba726d46d 100644 --- a/solstone/THIRD_PARTY_NOTICES.md +++ b/solstone/THIRD_PARTY_NOTICES.md @@ -101,6 +101,37 @@ Source: License notice: Apache License 2.0 (Apache-2.0). +## runtime-downloaded provider artifacts (rf-detr.cpp) + +These artifacts are fetched on demand into the journal provider cache for local +object detection. They are not bundled in this repository. + +### rf-detr.cpp engine binary + +Attribution: rf-detr.cpp (Ettore Di Giacinto / mudler); binary built and +released by sol pbc. + +Source: + +- Release binary: https://github.com/solpbc/rf-detr.cpp/releases/download/bin-65c0ffcc-1/rfdetr-cli-65c0ffcc-linux-cpu-x64.tar.gz +- Project: https://github.com/mudler/rf-detr.cpp +- Pinned engine ref: 65c0ffcc +- Downloaded file: `rfdetr-cli` (extracted from the tarball) + +License notice: Apache License 2.0 (Apache-2.0). + +### RF-DETR nano GGUF model weights + +Attribution: RF-DETR (Roboflow); GGUF conversion mudler/rfdetr-cpp-nano. + +Source: + +- Model repository: https://huggingface.co/mudler/rfdetr-cpp-nano +- Pinned revision: c3dc0c037df499f5503545247df6618415fca643 +- Downloaded file: `rfdetr-nano-f16.gguf` + +License notice: Apache License 2.0 (Apache-2.0). + ## WeSpeaker ResNet34 / VoxCeleb Attribution: WeSpeaker project, ResNet34 speaker embedding model trained on diff --git a/solstone/think/install_models.py b/solstone/think/install_models.py index 5b2de691c..9742df4e9 100644 --- a/solstone/think/install_models.py +++ b/solstone/think/install_models.py @@ -370,6 +370,19 @@ def _install_ced_assets(*, check: bool, force: bool) -> int: return 0 +def _install_rfdetr_model(*, check: bool, force: bool) -> int: + from solstone.think.providers import rfdetr_install + + try: + if check: + rfdetr_install.check_rfdetr_model() + else: + rfdetr_install.install_rfdetr(force=force) + except rfdetr_install.RfdetrInstallError as exc: + return _fail(str(exc)) + return 0 + + def _install_linux_cpp(*, force: bool = False) -> int: from solstone.think.providers import parakeet_install @@ -500,6 +513,10 @@ def main() -> int: if result != 0: return result + result = _install_rfdetr_model(check=args.check, force=args.force) + if result != 0: + return result + if variant is None: print( "parakeet install: unsupported platform " diff --git a/solstone/think/providers/fit_report.py b/solstone/think/providers/fit_report.py index a7316ce35..380ea9979 100644 --- a/solstone/think/providers/fit_report.py +++ b/solstone/think/providers/fit_report.py @@ -20,6 +20,9 @@ from solstone.think.providers.memory import ( FitSeverity = Literal["ok", "warning", "blocked", "unknown"] +# The rf-detr.cpp binary has no pinned size; the 60 MiB GGUF dominates the gate. +_RFDETR_ENGINE_BINARY_DISK_BYTES = 1024 * 1024 + @dataclass(frozen=True) class FitCheck: @@ -123,6 +126,29 @@ def build_parakeet_fit_report( return FitReport(artifact="parakeet.cpp artifacts", checks=checks) +def build_rfdetr_fit_report( + journal_path: str | Path | None = None, +) -> FitReport: + from solstone.think.providers import rfdetr_install + + checks = ( + _rfdetr_platform_check(), + _disk_check( + "disk", + rfdetr_install.cache_root(journal_path), + ( + ( + "rf-detr GGUF model", + rfdetr_install.RFDETR_SPEC.model.size_bytes, + ), + ("rf-detr CLI binary", _RFDETR_ENGINE_BINARY_DISK_BYTES), + ), + (), + ), + ) + return FitReport(artifact="rf-detr.cpp artifacts", checks=checks) + + def build_coreml_parakeet_fit_report( os_name: str, arch: str, @@ -199,6 +225,23 @@ def _parakeet_platform_check() -> FitCheck: ) +def _rfdetr_platform_check() -> FitCheck: + from solstone.think.providers import rfdetr_install + + if rfdetr_install._rfdetr_platform_supported(): + return FitCheck( + "platform", + "ok", + "pinned rf-detr.cpp artifacts are available for x86_64-linux", + ) + os_name, arch = rfdetr_install._platform_info() + return FitCheck( + "platform", + "blocked", + f"rf-detr.cpp requires x86_64 Linux, got {os_name}/{arch}", + ) + + def _coreml_platform_check(os_name: str, arch: str) -> FitCheck: if os_name == "darwin" and arch == "arm64": return FitCheck("platform", "ok", "CoreML parakeet supports darwin/arm64") @@ -400,5 +443,6 @@ __all__ = [ "build_local_fit_report", "build_mlx_fit_report", "build_parakeet_fit_report", + "build_rfdetr_fit_report", "render_fit_report", ] diff --git a/solstone/think/providers/rfdetr_install.py b/solstone/think/providers/rfdetr_install.py new file mode 100644 index 000000000..d393c1e51 --- /dev/null +++ b/solstone/think/providers/rfdetr_install.py @@ -0,0 +1,567 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Install and verify local rf-detr.cpp provider artifacts. + +This module performs no network access at import time. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import platform +import shutil +import stat +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from solstone.think.utils import get_journal + +LOG = logging.getLogger(__name__) + +ENGINE_REF = "65c0ffcc" +MODEL_NAME = "rfdetr-nano-f16" +ENGINE_BINARY_NAME = "rfdetr-cli" +SIDECAR_NAME = ".rfdetr-install.json" + + +class RfdetrInstallError(RuntimeError): + """rf-detr.cpp artifact acquisition failure with a recovery reason code.""" + + def __init__(self, reason_code: str, message: str) -> None: + super().__init__(message) + self.reason_code = reason_code + + +@dataclass(frozen=True) +class RfdetrEngineSpec: + ref: str + release_tag: str + tarball_name: str + tarball_sha256: str + binary_name: str + binary_sha256: str + + +@dataclass(frozen=True) +class RfdetrModelSpec: + repo: str + revision: str + filename: str + sha256: str + size_bytes: int + + +@dataclass(frozen=True) +class RfdetrSpec: + engine: RfdetrEngineSpec + model: RfdetrModelSpec + + +@dataclass(frozen=True) +class RfdetrFileSpec: + path: str + sha256: str + size_bytes: int | None = None + + +RFDETR_SPEC = RfdetrSpec( + engine=RfdetrEngineSpec( + ref=ENGINE_REF, + release_tag=f"bin-{ENGINE_REF}-1", + tarball_name=f"rfdetr-cli-{ENGINE_REF}-linux-cpu-x64.tar.gz", + tarball_sha256=( + "74f3258a94c975444923be0cc451d90c1e8d9e2595d3cab6876a11086d8357dd" + ), + binary_name=ENGINE_BINARY_NAME, + binary_sha256=( + "7c4fb4d499d53509d5099e768510a164c6647b84480c72170b865233504f367c" + ), + ), + model=RfdetrModelSpec( + repo="mudler/rfdetr-cpp-nano", + revision="c3dc0c037df499f5503545247df6618415fca643", + filename=f"{MODEL_NAME}.gguf", + sha256="d798cc448faa53209b88fc905c91beb1dd104634b95f6948cc4877540a8fd3ee", + size_bytes=63439488, + ), +) + +RfdetrInstallStatus = Literal["installed", "platform_unavailable"] + + +@dataclass(frozen=True) +class RfdetrInstallRecord: + status: RfdetrInstallStatus + engine_ref: str | None = None + engine_sha256: str | None = None + model_repo: str | None = None + model_revision: str | None = None + model_file: str | None = None + model_sha256: str | None = None + + def to_json(self) -> str: + data = { + "status": self.status, + "engine_ref": self.engine_ref, + "engine_sha256": self.engine_sha256, + "model_repo": self.model_repo, + "model_revision": self.model_revision, + "model_file": self.model_file, + "model_sha256": self.model_sha256, + } + return ( + json.dumps( + {key: value for key, value in data.items() if value is not None}, + indent=2, + sort_keys=True, + ) + + "\n" + ) + + @classmethod + def from_json(cls, text: str) -> RfdetrInstallRecord: + data = json.loads(text) + if not isinstance(data, dict): + raise ValueError("rf-detr install record must be an object") + status = data.get("status") + if status not in {"installed", "platform_unavailable"}: + raise ValueError( + "rf-detr install record status must be installed or " + "platform_unavailable" + ) + if status == "platform_unavailable": + return cls(status="platform_unavailable") + + engine_ref = data.get("engine_ref") + engine_sha256 = data.get("engine_sha256") + model_repo = data.get("model_repo") + model_revision = data.get("model_revision") + model_file = data.get("model_file") + model_sha256 = data.get("model_sha256") + for name, value in ( + ("engine_ref", engine_ref), + ("engine_sha256", engine_sha256), + ("model_repo", model_repo), + ("model_revision", model_revision), + ("model_file", model_file), + ("model_sha256", model_sha256), + ): + if not isinstance(value, str) or not value: + raise ValueError( + f"rf-detr install record {name} must be a non-empty string" + ) + + return cls( + status="installed", + engine_ref=engine_ref, + engine_sha256=engine_sha256, + model_repo=model_repo, + model_revision=model_revision, + model_file=model_file, + model_sha256=model_sha256, + ) + + +@dataclass(frozen=True) +class RfdetrPaths: + status: Literal["installed", "not_installed", "platform_unavailable"] + binary_path: Path | None = None + model_path: Path | None = None + + +def _platform_info() -> tuple[str, str]: + os_name = "linux" if sys.platform.startswith("linux") else sys.platform + return os_name, platform.machine().lower() + + +def _rfdetr_platform_supported( + os_name: str | None = None, arch: str | None = None +) -> bool: + if os_name is None or arch is None: + os_name, arch = _platform_info() + return os_name == "linux" and arch.lower() in {"amd64", "x64", "x86_64"} + + +def cache_root(journal_path: str | Path | None = None) -> Path: + root = Path(journal_path) if journal_path is not None else Path(get_journal()) + return root / "cache" / "providers" / "rfdetr" + + +def binary_path( + *, spec: RfdetrSpec = RFDETR_SPEC, journal_path: str | Path | None = None +) -> Path: + return ( + cache_root(journal_path) / "engine" / spec.engine.ref / spec.engine.binary_name + ) + + +def model_path( + *, spec: RfdetrSpec = RFDETR_SPEC, journal_path: str | Path | None = None +) -> Path: + return ( + cache_root(journal_path) / "model" / spec.model.revision / spec.model.filename + ) + + +def sidecar_path(*, journal_path: str | Path | None = None) -> Path: + return cache_root(journal_path) / SIDECAR_NAME + + +def _engine_extract_dir( + spec: RfdetrSpec, journal_path: str | Path | None = None +) -> Path: + return binary_path(spec=spec, journal_path=journal_path).parent / ".extract" + + +def _engine_tarball_path( + spec: RfdetrSpec, journal_path: str | Path | None = None +) -> Path: + return ( + binary_path(spec=spec, journal_path=journal_path).parent + / spec.engine.tarball_name + ) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _verify_file(path: Path, file_spec: RfdetrFileSpec) -> None: + if not path.is_file(): + raise RfdetrInstallError( + "file_missing", f"rf-detr asset missing: {file_spec.path}" + ) + if file_spec.size_bytes is not None: + actual_size = path.stat().st_size + if actual_size != file_spec.size_bytes: + raise RfdetrInstallError( + "size_mismatch", + ( + f"size mismatch for {file_spec.path}: " + f"expected {file_spec.size_bytes}, got {actual_size}" + ), + ) + actual_sha256 = _sha256_file(path) + if actual_sha256 != file_spec.sha256: + raise RfdetrInstallError( + "sha256_mismatch", + ( + f"sha256 mismatch for {file_spec.path}: " + f"expected {file_spec.sha256}, got {actual_sha256}" + ), + ) + + +def _tmp_path(dest: Path) -> Path: + return dest.with_name(f"{dest.name}.tmp") + + +def _download_file( + url: str, + dest: Path, + file_spec: RfdetrFileSpec, + *, + timeout_s: float = 600.0, +) -> None: + import httpx + + dest.parent.mkdir(parents=True, exist_ok=True) + tmp = _tmp_path(dest) + dest.unlink(missing_ok=True) + tmp.unlink(missing_ok=True) + try: + with httpx.stream( + "GET", url, timeout=timeout_s, follow_redirects=True + ) as response: + response.raise_for_status() + with tmp.open("wb") as handle: + for chunk in response.iter_bytes(): + if chunk: + handle.write(chunk) + _verify_file(tmp, file_spec) + tmp.replace(dest) + except RfdetrInstallError: + tmp.unlink(missing_ok=True) + raise + except Exception as exc: + tmp.unlink(missing_ok=True) + raise RfdetrInstallError( + "download_failed", f"failed to download {file_spec.path}: {exc}" + ) from exc + + +def _write_sidecar(path: Path, record: RfdetrInstallRecord) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + "w", dir=path.parent, delete=False, encoding="utf-8" + ) as handle: + handle.write(record.to_json()) + tmp_path = Path(handle.name) + tmp_path.replace(path) + + +def _safe_extract_tarball(tarball: Path, dest: Path) -> None: + import tarfile + + dest.mkdir(parents=True, exist_ok=True) + dest_resolved = dest.resolve() + with tarfile.open(tarball, "r:*") as archive: + for member in archive.getmembers(): + target = (dest / member.name).resolve() + if target != dest_resolved and dest_resolved not in target.parents: + raise RfdetrInstallError( + "archive_path_traversal", + f"Unsafe tar member path: {member.name}", + ) + archive.extractall(dest) + + +def _find_extracted_binary(dest: Path, binary_name: str) -> Path: + direct = dest / binary_name + if direct.exists(): + return direct + matches = [path for path in dest.rglob(binary_name) if path.is_file()] + if not matches: + raise RfdetrInstallError( + "binary_missing", + f"Extracted archive did not contain {binary_name}", + ) + if len(matches) > 1: + matches.sort(key=lambda path: len(path.parts)) + return matches[0] + + +def _chmod_executable(path: Path) -> None: + mode = path.stat().st_mode + path.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def _engine_url(spec: RfdetrSpec) -> str: + return ( + "https://github.com/solpbc/rf-detr.cpp/releases/download/" + f"{spec.engine.release_tag}/{spec.engine.tarball_name}" + ) + + +def _model_url(spec: RfdetrSpec) -> str: + return ( + f"https://huggingface.co/{spec.model.repo}/resolve/" + f"{spec.model.revision}/{spec.model.filename}" + ) + + +def _install_engine(spec: RfdetrSpec, journal_path: str | Path | None = None) -> None: + tarball_dest = _engine_tarball_path(spec, journal_path) + extract_dir = _engine_extract_dir(spec, journal_path) + final_path = binary_path(spec=spec, journal_path=journal_path) + _download_file( + _engine_url(spec), + tarball_dest, + RfdetrFileSpec( + spec.engine.tarball_name, + spec.engine.tarball_sha256, + None, + ), + ) + shutil.rmtree(extract_dir, ignore_errors=True) + _safe_extract_tarball(tarball_dest, extract_dir) + found = _find_extracted_binary(extract_dir, spec.engine.binary_name) + _verify_file( + found, + RfdetrFileSpec( + spec.engine.binary_name, + spec.engine.binary_sha256, + None, + ), + ) + _chmod_executable(found) + final_path.parent.mkdir(parents=True, exist_ok=True) + found.replace(final_path) + shutil.rmtree(extract_dir, ignore_errors=True) + tarball_dest.unlink(missing_ok=True) + + +def _install_model(spec: RfdetrSpec, journal_path: str | Path | None = None) -> None: + _download_file( + _model_url(spec), + model_path(spec=spec, journal_path=journal_path), + RfdetrFileSpec( + spec.model.filename, + spec.model.sha256, + spec.model.size_bytes, + ), + ) + + +def _record_for_spec(spec: RfdetrSpec) -> RfdetrInstallRecord: + return RfdetrInstallRecord( + status="installed", + engine_ref=spec.engine.ref, + engine_sha256=spec.engine.binary_sha256, + model_repo=spec.model.repo, + model_revision=spec.model.revision, + model_file=spec.model.filename, + model_sha256=spec.model.sha256, + ) + + +def _cleanup_partial_install( + *, spec: RfdetrSpec = RFDETR_SPEC, journal_path: str | Path | None = None +) -> None: + binary = binary_path(spec=spec, journal_path=journal_path) + binary.unlink(missing_ok=True) + _tmp_path(binary).unlink(missing_ok=True) + tarball = _engine_tarball_path(spec, journal_path) + tarball.unlink(missing_ok=True) + _tmp_path(tarball).unlink(missing_ok=True) + shutil.rmtree(_engine_extract_dir(spec, journal_path), ignore_errors=True) + model = model_path(spec=spec, journal_path=journal_path) + model.unlink(missing_ok=True) + _tmp_path(model).unlink(missing_ok=True) + sidecar_path(journal_path=journal_path).unlink(missing_ok=True) + + +def check_rfdetr_model( + *, spec: RfdetrSpec = RFDETR_SPEC, journal_path: str | Path | None = None +) -> RfdetrInstallRecord: + # Deliberately diverges from rerank: unsupported hosts are a clean check pass. + if not _rfdetr_platform_supported(): + return RfdetrInstallRecord(status="platform_unavailable") + + sidecar = sidecar_path(journal_path=journal_path) + if not sidecar.is_file(): + raise RfdetrInstallError( + "sidecar_missing", f"rf-detr sidecar missing: {sidecar}" + ) + try: + record = RfdetrInstallRecord.from_json(sidecar.read_text(encoding="utf-8")) + except Exception as exc: + raise RfdetrInstallError( + "sidecar_invalid", f"rf-detr sidecar invalid: {sidecar}: {exc}" + ) from exc + if record.status == "platform_unavailable": + return record + if ( + record.engine_ref != spec.engine.ref + or record.engine_sha256 != spec.engine.binary_sha256 + or record.model_repo != spec.model.repo + or record.model_revision != spec.model.revision + or record.model_file != spec.model.filename + or record.model_sha256 != spec.model.sha256 + ): + raise RfdetrInstallError( + "sidecar_mismatch", "rf-detr sidecar does not match pinned artifacts" + ) + _verify_file( + binary_path(spec=spec, journal_path=journal_path), + RfdetrFileSpec( + spec.engine.binary_name, + spec.engine.binary_sha256, + None, + ), + ) + _verify_file( + model_path(spec=spec, journal_path=journal_path), + RfdetrFileSpec( + spec.model.filename, + spec.model.sha256, + spec.model.size_bytes, + ), + ) + return record + + +def install_rfdetr( + *, + force: bool = False, + spec: RfdetrSpec = RFDETR_SPEC, + journal_path: str | Path | None = None, +) -> RfdetrInstallRecord: + os_name, arch = _platform_info() + if not _rfdetr_platform_supported(os_name, arch): + record = RfdetrInstallRecord(status="platform_unavailable") + _write_sidecar(sidecar_path(journal_path=journal_path), record) + LOG.info("rf-detr.cpp platform unavailable on %s/%s", os_name, arch) + return record + + if not force: + try: + return check_rfdetr_model(spec=spec, journal_path=journal_path) + except RfdetrInstallError: + pass + + from solstone.think.providers.fit_report import ( + build_rfdetr_fit_report, + render_fit_report, + ) + + report = build_rfdetr_fit_report(journal_path) + rendered = render_fit_report(report) + if report.overall == "blocked": + raise RfdetrInstallError("host_unfit", rendered) + if report.overall == "warning": + LOG.warning("rf-detr.cpp host fit warning:\n%s", rendered) + + try: + _cleanup_partial_install(spec=spec, journal_path=journal_path) + _install_engine(spec, journal_path) + _install_model(spec, journal_path) + record = _record_for_spec(spec) + _write_sidecar(sidecar_path(journal_path=journal_path), record) + return record + except RfdetrInstallError: + _cleanup_partial_install(spec=spec, journal_path=journal_path) + raise + except Exception as exc: + _cleanup_partial_install(spec=spec, journal_path=journal_path) + raise RfdetrInstallError( + "install_failed", f"rf-detr install failed: {exc}" + ) from exc + + +def rfdetr_paths( + *, spec: RfdetrSpec = RFDETR_SPEC, journal_path: str | Path | None = None +) -> RfdetrPaths: + if not _rfdetr_platform_supported(): + return RfdetrPaths(status="platform_unavailable") + try: + record = check_rfdetr_model(spec=spec, journal_path=journal_path) + except RfdetrInstallError: + return RfdetrPaths(status="not_installed") + if record.status == "platform_unavailable": + return RfdetrPaths(status="not_installed") + return RfdetrPaths( + status="installed", + binary_path=binary_path(spec=spec, journal_path=journal_path), + model_path=model_path(spec=spec, journal_path=journal_path), + ) + + +__all__ = [ + "ENGINE_REF", + "MODEL_NAME", + "RFDETR_SPEC", + "RfdetrEngineSpec", + "RfdetrInstallError", + "RfdetrInstallRecord", + "RfdetrModelSpec", + "RfdetrPaths", + "RfdetrSpec", + "binary_path", + "cache_root", + "check_rfdetr_model", + "install_rfdetr", + "model_path", + "rfdetr_paths", + "sidecar_path", +] diff --git a/tests/test_install_models.py b/tests/test_install_models.py index d8b90b715..7ed797718 100644 --- a/tests/test_install_models.py +++ b/tests/test_install_models.py @@ -222,6 +222,9 @@ def _prepare_check_main(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( install_models, "_install_ced_assets", lambda *, check, force: 0 ) + monkeypatch.setattr( + install_models, "_install_rfdetr_model", lambda *, check, force: 0 + ) def test_main_check_missing_cpp_artifacts_returns_nonzero( @@ -271,6 +274,11 @@ def test_main_rerank_failure_short_circuits_before_parakeet( "_install_ced_assets", lambda *, check, force: pytest.fail("ced install should not start"), ) + monkeypatch.setattr( + install_models, + "_install_rfdetr_model", + lambda *, check, force: pytest.fail("rf-detr check should not start"), + ) monkeypatch.setattr( install_models, "_check_linux_cpp_ready", @@ -312,6 +320,9 @@ def test_main_runs_ced_after_rerank_before_parakeet( "_check_linux_cpp_ready", lambda: calls.append(("parakeet",)) or paths, ) + monkeypatch.setattr( + install_models, "_install_rfdetr_model", lambda *, check, force: 0 + ) assert install_models.main() == 0 assert calls == [ @@ -436,6 +447,80 @@ def test_install_ced_assets_downloads_when_missing( assert calls == [("check",), ("install", False)] +def test_main_rfdetr_failure_short_circuits_before_parakeet( + monkeypatch: pytest.MonkeyPatch, +): + calls = [] + monkeypatch.setattr(sys, "argv", ["sol install-models", "--check"]) + monkeypatch.delenv(install_models.JOURNAL_VARIANT_ENV, raising=False) + monkeypatch.setattr(install_models, "_platform_info", lambda: ("linux", "x86_64")) + monkeypatch.setattr(install_models, "_detect_linux_variant", lambda: "cpu") + monkeypatch.setattr(install_models, "_verify_bundled_assets", lambda: None) + monkeypatch.setattr( + install_models, "_install_rerank_model", lambda *, check, force: 0 + ) + monkeypatch.setattr( + install_models, "_install_ced_assets", lambda *, check, force: 0 + ) + monkeypatch.setattr( + install_models, + "_install_rfdetr_model", + lambda *, check, force: calls.append((check, force)) or 7, + ) + monkeypatch.setattr( + install_models, + "_check_linux_cpp_ready", + lambda: pytest.fail("parakeet check should not start"), + ) + monkeypatch.setattr( + install_models, + "_install_models", + lambda *_args, **_kwargs: pytest.fail("parakeet install should not start"), + ) + + assert install_models.main() == 7 + assert calls == [(True, False)] + + +def test_main_rfdetr_success_continues_to_parakeet( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +): + calls = [] + paths = _ready_paths(tmp_path) + monkeypatch.setattr(sys, "argv", ["sol install-models", "--check"]) + monkeypatch.delenv(install_models.JOURNAL_VARIANT_ENV, raising=False) + monkeypatch.setattr(install_models, "_platform_info", lambda: ("linux", "x86_64")) + monkeypatch.setattr(install_models, "_detect_linux_variant", lambda: "cpu") + monkeypatch.setattr(install_models, "_verify_bundled_assets", lambda: None) + monkeypatch.setattr( + install_models, + "_install_rerank_model", + lambda *, check, force: calls.append(("rerank", check, force)) or 0, + ) + monkeypatch.setattr( + install_models, "_install_ced_assets", lambda *, check, force: 0 + ) + monkeypatch.setattr( + install_models, + "_install_rfdetr_model", + lambda *, check, force: calls.append(("rfdetr", check, force)) or 0, + ) + + def ready_paths() -> dict[str, Path]: + calls.append(("parakeet", True, False)) + return paths + + monkeypatch.setattr(install_models, "_check_linux_cpp_ready", ready_paths) + + assert install_models.main() == 0 + assert calls == [ + ("rerank", True, False), + ("rfdetr", True, False), + ("parakeet", True, False), + ] + + def test_run_mac_helper_soft_fails_on_packaged_install( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -495,6 +580,9 @@ def test_main_force_reinstalls_linux_cpp( monkeypatch.setattr( install_models, "_install_ced_assets", lambda *, check, force: 0 ) + monkeypatch.setattr( + install_models, "_install_rfdetr_model", lambda *, check, force: 0 + ) monkeypatch.setattr( install_models, "_check_linux_cpp_ready", lambda: _ready_paths(tmp_path) ) @@ -525,6 +613,9 @@ def test_main_linux_blocks_before_install_models( monkeypatch.setattr( install_models, "_install_ced_assets", lambda *, check, force: 0 ) + monkeypatch.setattr( + install_models, "_install_rfdetr_model", lambda *, check, force: 0 + ) monkeypatch.setattr( fit_report, "build_parakeet_fit_report", lambda: _fit("blocked") ) @@ -553,6 +644,9 @@ def test_main_linux_warning_continues_to_install_models( monkeypatch.setattr( install_models, "_install_ced_assets", lambda *, check, force: 0 ) + monkeypatch.setattr( + install_models, "_install_rfdetr_model", lambda *, check, force: 0 + ) monkeypatch.setattr( fit_report, "build_parakeet_fit_report", lambda: _fit("warning") ) @@ -582,6 +676,9 @@ def test_main_coreml_blocks_before_install_models( monkeypatch.setattr( install_models, "_install_ced_assets", lambda *, check, force: 0 ) + monkeypatch.setattr( + install_models, "_install_rfdetr_model", lambda *, check, force: 0 + ) monkeypatch.setattr( fit_report, "build_coreml_parakeet_fit_report", @@ -611,6 +708,9 @@ def test_main_skips_install_when_linux_cpp_ready( monkeypatch.setattr( install_models, "_install_ced_assets", lambda *, check, force: 0 ) + monkeypatch.setattr( + install_models, "_install_rfdetr_model", lambda *, check, force: 0 + ) monkeypatch.setattr( install_models, "_check_linux_cpp_ready", lambda: _ready_paths(tmp_path) ) diff --git a/tests/test_rfdetr_install.py b/tests/test_rfdetr_install.py new file mode 100644 index 000000000..d7146ad42 --- /dev/null +++ b/tests/test_rfdetr_install.py @@ -0,0 +1,464 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +import hashlib +import io +import os +import tarfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import httpx +import pytest + +from solstone.think.providers import rfdetr_install + +ENGINE_BYTES = b"rf-detr cli fixture bytes" +MODEL_BYTES = b"rf-detr model fixture bytes" + + +@dataclass(frozen=True) +class _EngineFixture: + tarball_bytes: bytes + tarball_sha256: str + binary_sha256: str + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +@pytest.fixture +def engine_tarball(tmp_path: Path) -> _EngineFixture: + src = ( + tmp_path + / "engine-src" + / f"rfdetr-cli-{rfdetr_install.ENGINE_REF}-linux-cpu-x64" + ) + src.mkdir(parents=True) + (src / rfdetr_install.ENGINE_BINARY_NAME).write_bytes(ENGINE_BYTES) + (src / "LICENSE").write_text("fixture license\n", encoding="utf-8") + (src / "PROVENANCE.txt").write_text("fixture provenance\n", encoding="utf-8") + tarball = tmp_path / rfdetr_install.RFDETR_SPEC.engine.tarball_name + with tarfile.open(tarball, "w:gz") as archive: + archive.add(src, arcname=src.name) + tarball_bytes = tarball.read_bytes() + return _EngineFixture( + tarball_bytes=tarball_bytes, + tarball_sha256=_sha256(tarball_bytes), + binary_sha256=_sha256(ENGINE_BYTES), + ) + + +def _fixture_spec( + engine: _EngineFixture, + *, + model_bytes: bytes = MODEL_BYTES, + tarball_sha256: str | None = None, + binary_sha256: str | None = None, + model_sha256: str | None = None, + model_size_bytes: int | None = None, +) -> rfdetr_install.RfdetrSpec: + pinned = rfdetr_install.RFDETR_SPEC + return rfdetr_install.RfdetrSpec( + engine=rfdetr_install.RfdetrEngineSpec( + ref=pinned.engine.ref, + release_tag=pinned.engine.release_tag, + tarball_name=pinned.engine.tarball_name, + tarball_sha256=tarball_sha256 or engine.tarball_sha256, + binary_name=pinned.engine.binary_name, + binary_sha256=binary_sha256 or engine.binary_sha256, + ), + model=rfdetr_install.RfdetrModelSpec( + repo=pinned.model.repo, + revision=pinned.model.revision, + filename=pinned.model.filename, + sha256=model_sha256 or _sha256(model_bytes), + size_bytes=( + len(model_bytes) if model_size_bytes is None else model_size_bytes + ), + ), + ) + + +def _payloads( + spec: rfdetr_install.RfdetrSpec, + engine: _EngineFixture, + *, + model_bytes: bytes = MODEL_BYTES, +) -> dict[str, bytes]: + return { + spec.engine.tarball_name: engine.tarball_bytes, + spec.model.filename: model_bytes, + } + + +def _fake_download_factory( + payloads: dict[str, bytes], + calls: list[str] | None = None, +): + def fake_download( + _url: str, + dest: Path, + file_spec: rfdetr_install.RfdetrFileSpec, + **_kwargs: Any, + ) -> None: + if calls is not None: + calls.append(file_spec.path) + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(payloads[file_spec.path]) + + return fake_download + + +class _FakeResponse: + def __init__(self, payload: bytes) -> None: + self._payload = payload + + def raise_for_status(self) -> None: + return + + def iter_bytes(self): + midpoint = max(1, len(self._payload) // 2) + yield self._payload[:midpoint] + yield self._payload[midpoint:] + + +class _FakeStream: + def __init__(self, response: _FakeResponse) -> None: + self.response = response + + def __enter__(self) -> _FakeResponse: + return self.response + + def __exit__(self, *_args: object) -> bool: + return False + + +def _patch_stream_map( + monkeypatch: pytest.MonkeyPatch, payloads: dict[str, bytes] +) -> None: + def fake_stream(_method: str, url: str, **_kwargs: Any) -> _FakeStream: + for needle, payload in payloads.items(): + if needle in url: + return _FakeStream(_FakeResponse(payload)) + raise AssertionError(f"unexpected download URL: {url}") + + monkeypatch.setattr(httpx, "stream", fake_stream) + + +def _tmp_path(path: Path) -> Path: + return path.with_name(f"{path.name}.tmp") + + +def _assert_cleaned(spec: rfdetr_install.RfdetrSpec, journal_path: Path) -> None: + binary = rfdetr_install.binary_path(spec=spec, journal_path=journal_path) + model = rfdetr_install.model_path(spec=spec, journal_path=journal_path) + tarball = rfdetr_install._engine_tarball_path(spec, journal_path) + assert not binary.exists() + assert not _tmp_path(binary).exists() + assert not model.exists() + assert not _tmp_path(model).exists() + assert not tarball.exists() + assert not _tmp_path(tarball).exists() + assert not rfdetr_install._engine_extract_dir(spec, journal_path).exists() + assert not rfdetr_install.sidecar_path(journal_path=journal_path).exists() + + +def _mark_supported(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(rfdetr_install, "_platform_info", lambda: ("linux", "x86_64")) + + +def test_sidecar_round_trip() -> None: + record = rfdetr_install.RfdetrInstallRecord( + status="installed", + engine_ref=rfdetr_install.RFDETR_SPEC.engine.ref, + engine_sha256=rfdetr_install.RFDETR_SPEC.engine.binary_sha256, + model_repo=rfdetr_install.RFDETR_SPEC.model.repo, + model_revision=rfdetr_install.RFDETR_SPEC.model.revision, + model_file=rfdetr_install.RFDETR_SPEC.model.filename, + model_sha256=rfdetr_install.RFDETR_SPEC.model.sha256, + ) + assert rfdetr_install.RfdetrInstallRecord.from_json(record.to_json()) == record + + unavailable = rfdetr_install.RfdetrInstallRecord(status="platform_unavailable") + assert ( + rfdetr_install.RfdetrInstallRecord.from_json(unavailable.to_json()) + == unavailable + ) + + +def test_install_writes_files_and_sidecar( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, engine_tarball: _EngineFixture +) -> None: + _mark_supported(monkeypatch) + spec = _fixture_spec(engine_tarball) + monkeypatch.setattr( + rfdetr_install, + "_download_file", + _fake_download_factory(_payloads(spec, engine_tarball)), + ) + + record = rfdetr_install.install_rfdetr(spec=spec, journal_path=tmp_path) + + binary = rfdetr_install.binary_path(spec=spec, journal_path=tmp_path) + model = rfdetr_install.model_path(spec=spec, journal_path=tmp_path) + sidecar = rfdetr_install.sidecar_path(journal_path=tmp_path) + assert binary.read_bytes() == ENGINE_BYTES + assert os.access(binary, os.X_OK) + assert model.read_bytes() == MODEL_BYTES + assert sidecar.is_file() + assert record.status == "installed" + assert record == rfdetr_install.RfdetrInstallRecord.from_json( + sidecar.read_text(encoding="utf-8") + ) + + +def test_present_valid_install_is_noop( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, engine_tarball: _EngineFixture +) -> None: + _mark_supported(monkeypatch) + spec = _fixture_spec(engine_tarball) + monkeypatch.setattr( + rfdetr_install, + "_download_file", + _fake_download_factory(_payloads(spec, engine_tarball)), + ) + rfdetr_install.install_rfdetr(spec=spec, journal_path=tmp_path) + monkeypatch.setattr( + rfdetr_install, + "_download_file", + lambda *_args, **_kwargs: pytest.fail("download should not start"), + ) + + record = rfdetr_install.install_rfdetr(spec=spec, journal_path=tmp_path) + + assert record.engine_ref == spec.engine.ref + assert record.model_file == spec.model.filename + + +def test_check_valid_uses_no_network( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, engine_tarball: _EngineFixture +) -> None: + _mark_supported(monkeypatch) + spec = _fixture_spec(engine_tarball) + monkeypatch.setattr( + rfdetr_install, + "_download_file", + _fake_download_factory(_payloads(spec, engine_tarball)), + ) + rfdetr_install.install_rfdetr(spec=spec, journal_path=tmp_path) + monkeypatch.setattr( + rfdetr_install, + "_download_file", + lambda *_args, **_kwargs: pytest.fail("download should not start"), + ) + + assert ( + rfdetr_install.check_rfdetr_model(spec=spec, journal_path=tmp_path).engine_ref + == spec.engine.ref + ) + + +def test_check_missing_raises_without_download( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, engine_tarball: _EngineFixture +) -> None: + _mark_supported(monkeypatch) + spec = _fixture_spec(engine_tarball) + monkeypatch.setattr( + rfdetr_install, + "_download_file", + lambda *_args, **_kwargs: pytest.fail("download should not start"), + ) + + with pytest.raises(rfdetr_install.RfdetrInstallError) as exc_info: + rfdetr_install.check_rfdetr_model(spec=spec, journal_path=tmp_path) + + assert exc_info.value.reason_code == "sidecar_missing" + + +def test_force_refetches_when_present( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, engine_tarball: _EngineFixture +) -> None: + _mark_supported(monkeypatch) + spec = _fixture_spec(engine_tarball) + monkeypatch.setattr( + rfdetr_install, + "_download_file", + _fake_download_factory(_payloads(spec, engine_tarball)), + ) + rfdetr_install.install_rfdetr(spec=spec, journal_path=tmp_path) + calls: list[str] = [] + monkeypatch.setattr( + rfdetr_install, + "_download_file", + _fake_download_factory(_payloads(spec, engine_tarball), calls), + ) + + rfdetr_install.install_rfdetr(force=True, spec=spec, journal_path=tmp_path) + + assert calls == [spec.engine.tarball_name, spec.model.filename] + + +def test_model_sha256_mismatch_cleans_partial_install( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, engine_tarball: _EngineFixture +) -> None: + _mark_supported(monkeypatch) + spec = _fixture_spec(engine_tarball) + bad_model = bytes([MODEL_BYTES[0] ^ 1]) + MODEL_BYTES[1:] + _patch_stream_map( + monkeypatch, + _payloads(spec, engine_tarball, model_bytes=bad_model), + ) + + with pytest.raises(rfdetr_install.RfdetrInstallError) as exc_info: + rfdetr_install.install_rfdetr(spec=spec, journal_path=tmp_path) + + assert exc_info.value.reason_code == "sha256_mismatch" + _assert_cleaned(spec, tmp_path) + + +def test_tarball_sha256_mismatch_cleans_partial_install( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, engine_tarball: _EngineFixture +) -> None: + _mark_supported(monkeypatch) + spec = _fixture_spec(engine_tarball, tarball_sha256="0" * 64) + _patch_stream_map(monkeypatch, _payloads(spec, engine_tarball)) + + with pytest.raises(rfdetr_install.RfdetrInstallError) as exc_info: + rfdetr_install.install_rfdetr(spec=spec, journal_path=tmp_path) + + assert exc_info.value.reason_code == "sha256_mismatch" + _assert_cleaned(spec, tmp_path) + + +def test_inner_binary_sha256_mismatch_cleans_partial_install( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, engine_tarball: _EngineFixture +) -> None: + _mark_supported(monkeypatch) + spec = _fixture_spec(engine_tarball, binary_sha256="0" * 64) + _patch_stream_map(monkeypatch, _payloads(spec, engine_tarball)) + + with pytest.raises(rfdetr_install.RfdetrInstallError) as exc_info: + rfdetr_install.install_rfdetr(spec=spec, journal_path=tmp_path) + + assert exc_info.value.reason_code == "sha256_mismatch" + _assert_cleaned(spec, tmp_path) + + +def test_model_download_verifies_before_rename( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, engine_tarball: _EngineFixture +) -> None: + spec = _fixture_spec(engine_tarball) + file_spec = rfdetr_install.RfdetrFileSpec( + spec.model.filename, + spec.model.sha256, + spec.model.size_bytes, + ) + bad_model = bytes([MODEL_BYTES[0] ^ 1]) + MODEL_BYTES[1:] + _patch_stream_map(monkeypatch, {spec.model.filename: bad_model}) + dest = tmp_path / spec.model.filename + + with pytest.raises(rfdetr_install.RfdetrInstallError) as exc_info: + rfdetr_install._download_file( + f"https://example.test/{spec.model.filename}", dest, file_spec + ) + + assert exc_info.value.reason_code == "sha256_mismatch" + assert not dest.exists() + assert not _tmp_path(dest).exists() + + +def test_extract_rejects_path_traversal(tmp_path: Path) -> None: + tarball = tmp_path / "bad.tar.gz" + with tarfile.open(tarball, "w:gz") as archive: + member = tarfile.TarInfo("../evil") + payload = b"bad" + member.size = len(payload) + archive.addfile(member, io.BytesIO(payload)) + + with pytest.raises(rfdetr_install.RfdetrInstallError) as exc_info: + rfdetr_install._safe_extract_tarball(tarball, tmp_path / "extract") + + assert exc_info.value.reason_code == "archive_path_traversal" + + +def test_platform_unavailable_writes_marker_no_download( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, engine_tarball: _EngineFixture +) -> None: + monkeypatch.setattr(rfdetr_install, "_platform_info", lambda: ("linux", "aarch64")) + spec = _fixture_spec(engine_tarball) + monkeypatch.setattr( + rfdetr_install, + "_download_file", + lambda *_args, **_kwargs: pytest.fail("download should not start"), + ) + + record = rfdetr_install.install_rfdetr(spec=spec, journal_path=tmp_path) + + assert record.status == "platform_unavailable" + sidecar = rfdetr_install.sidecar_path(journal_path=tmp_path) + assert sidecar.is_file() + assert ( + rfdetr_install.RfdetrInstallRecord.from_json( + sidecar.read_text(encoding="utf-8") + ).status + == "platform_unavailable" + ) + assert not rfdetr_install.binary_path(spec=spec, journal_path=tmp_path).exists() + assert not rfdetr_install.model_path(spec=spec, journal_path=tmp_path).exists() + assert ( + rfdetr_install.check_rfdetr_model(spec=spec, journal_path=tmp_path).status + == "platform_unavailable" + ) + + +def test_rfdetr_paths_three_states( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, engine_tarball: _EngineFixture +) -> None: + _mark_supported(monkeypatch) + spec = _fixture_spec(engine_tarball) + empty = tmp_path / "empty" + assert ( + rfdetr_install.rfdetr_paths(spec=spec, journal_path=empty).status + == "not_installed" + ) + monkeypatch.setattr( + rfdetr_install, + "_download_file", + _fake_download_factory(_payloads(spec, engine_tarball)), + ) + rfdetr_install.install_rfdetr(spec=spec, journal_path=tmp_path) + + installed = rfdetr_install.rfdetr_paths(spec=spec, journal_path=tmp_path) + assert installed.status == "installed" + assert installed.binary_path == rfdetr_install.binary_path( + spec=spec, journal_path=tmp_path + ) + assert installed.model_path == rfdetr_install.model_path( + spec=spec, journal_path=tmp_path + ) + + monkeypatch.setattr(rfdetr_install, "_platform_info", lambda: ("linux", "aarch64")) + unavailable = rfdetr_install.rfdetr_paths(spec=spec, journal_path=tmp_path) + assert unavailable.status == "platform_unavailable" + assert unavailable.binary_path is None + assert unavailable.model_path is None + + +def test_invalid_sidecar_shape_fails_check( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, engine_tarball: _EngineFixture +) -> None: + _mark_supported(monkeypatch) + spec = _fixture_spec(engine_tarball) + sidecar = rfdetr_install.sidecar_path(journal_path=tmp_path) + sidecar.parent.mkdir(parents=True, exist_ok=True) + sidecar.write_text("{}\n", encoding="utf-8") + + with pytest.raises(rfdetr_install.RfdetrInstallError) as exc_info: + rfdetr_install.check_rfdetr_model(spec=spec, journal_path=tmp_path) + + assert exc_info.value.reason_code == "sidecar_invalid"