diff --git a/AGENTS.md b/AGENTS.md index 7505f517c..2eeb0c643 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -250,6 +250,7 @@ Each domain has exactly **one** write-owning module (or one tightly-scoped famil | Provider install leases (`health/providers/{local,parakeet}.lease`) | `solstone/think/providers/install_lease.py` | | Provider runtime health and retry-token records (`health/providers/runtime/{local,parakeet}.json`, `health/providers/runtime/{local,parakeet}.retry-token.json`, `health/providers/runtime/{local,parakeet}.operation.lock`) | `solstone/think/providers/runtime_health.py` | | Provider artifact manifests (`cache/providers/**/.solstone-provider-manifest.json`, `cache/providers/local/mlx/**/*.manifest.json`) | `solstone/think/providers/artifact_proof.py` | +| nvattest appraiser cache (`cache/providers/nvattest/**`) | `solstone/think/providers/nvattest_install.py` | | Media offload ledger (`health/offload/.jsonl`) | `solstone/think/offload_ledger.py` | | Parakeet server placement record (`health/parakeet-cpp.placement`) | `solstone/think/providers/parakeet_server.py` | | Hosted backup binding (`backup/hosted/binding.json`) | `solstone/think/backup/hosted.py` | diff --git a/CHANGELOG.md b/CHANGELOG.md index 9750b5c01..168dacc8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Format adapted from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), al ### Fixed - a model reply that comes back empty is now a failure everywhere: key and model checks, thinking-engine health, and daily analysis all agree, so one path no longer accepts a blank answer another path rejects. +- confidential processing now gets its nvattest appraiser on first use when the machine supports it, and Thinking names appraiser setup or integrity problems instead of showing a generic attestation rejection. - choosing an exact Gemini model from Thinking's advisory now remembers it for when confidential processing is off, while sol keeps thinking with confidential processing now. ## [0.9.0] - 2026-07-19 diff --git a/scripts/check_journal_io_access.py b/scripts/check_journal_io_access.py index 0bb43b7d7..a1091cc09 100644 --- a/scripts/check_journal_io_access.py +++ b/scripts/check_journal_io_access.py @@ -149,6 +149,8 @@ OWNER_FILES: frozenset[str] = frozenset( # Provider install status, proof cache, and artifact manifests. "solstone/think/providers/artifact_proof.py", "solstone/think/providers/install_state.py", + # Provider cache-local nvattest artifacts and install single-flight lock. + "solstone/think/providers/nvattest_install.py", "solstone/think/providers/runtime_health.py", "solstone/think/schedule_config.py", "solstone/think/push/devices.py", diff --git a/scripts/spp_ratls_loopback_e2e.py b/scripts/spp_ratls_loopback_e2e.py index b9ed883bf..00b70638a 100644 --- a/scripts/spp_ratls_loopback_e2e.py +++ b/scripts/spp_ratls_loopback_e2e.py @@ -92,7 +92,10 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--nvattest-dir", type=Path, - help="nvattest install root containing bin/nvattest and lib/ (requires --real)", + help=( + "nvattest install root containing bin/nvattest, lib/, and " + "share/ca/ca-bundle.pem (requires --real)" + ), ) parser.add_argument( "--upstream-port", @@ -145,7 +148,9 @@ def validate_runtime_args( try: locate_nvattest(args.nvattest_dir) except GpuAppraisalError: - parser.error("--nvattest-dir must contain bin/nvattest and lib/") + parser.error( + "--nvattest-dir must contain bin/nvattest, lib/, and share/ca/ca-bundle.pem" + ) return RealModeConfig( nvattest_dir=args.nvattest_dir.resolve(), diff --git a/solstone/apps/thinking/tests/test_confidential_attestation_payload.py b/solstone/apps/thinking/tests/test_confidential_attestation_payload.py index 5572b3dbe..40a3727de 100644 --- a/solstone/apps/thinking/tests/test_confidential_attestation_payload.py +++ b/solstone/apps/thinking/tests/test_confidential_attestation_payload.py @@ -475,6 +475,43 @@ def test_build_brain_presentation_maps_confidential_attestation( assert presentation["confidential_attestation"] == expected +@pytest.mark.parametrize( + ("reason", "aggregate", "component_status", "expected_state"), + [ + ("nvattest_install_failed", "unhealthy", "failed", "failed"), + ("nvattest_platform_unsupported", "blocked", "blocked", "failed"), + ("nvattest_unavailable", "blocked", "blocked", "failed"), + ("nvattest_integrity_failed", "unhealthy", "failed", "failed"), + ("nvattest_install_in_progress", "blocked", "blocked", "verifying"), + ], +) +def test_build_brain_presentation_preserves_nvattest_reason( + monkeypatch, + reason: str, + aggregate: str, + component_status: str, + expected_state: str, +): + inspection = _inspection( + aggregate=aggregate, + reason=reason, + record=_record( + lane_prerequisites=_component(component_status, reason), + generate=_component("not_attempted", reason), + cogitate=_component("not_attempted", reason), + ), + ) + + presentation = _build_presentation(monkeypatch, inspection, configured=True) + + assert presentation["confidential_attestation"] == { + "state": expected_state, + "reason": reason, + "observed_at": NOW_ISO, + "expires_at": None, + } + + @pytest.mark.parametrize( "inspection", [ diff --git a/solstone/think/brain_cli.py b/solstone/think/brain_cli.py index 9bf229245..3fc1154a6 100644 --- a/solstone/think/brain_cli.py +++ b/solstone/think/brain_cli.py @@ -67,6 +67,33 @@ from solstone.think.utils import require_solstone, setup_cli LOG = logging.getLogger("solstone.think.brain_cli") +_SPP_ATTESTATION_FAILURE_REASON_TO_BRAIN_REASON = { + "gateway_unreachable": "attestation_not_verified", + "nvattest_install_in_progress": "nvattest_install_in_progress", + "nvattest_platform_unsupported": "nvattest_platform_unsupported", + "nvattest_unavailable": "nvattest_unavailable", + "nvattest_install_failed": "nvattest_install_failed", + "nvattest_integrity_failed": "nvattest_integrity_failed", + "tls_handshake_failed": "attestation_rejected", + "proof_http_failed": "attestation_rejected", + "certificate_invalid": "attestation_rejected", + "certificate_extension_missing": "attestation_rejected", + "certificate_extension_not_critical": "attestation_rejected", + "certificate_extension_invalid": "attestation_rejected", + "certificate_evidence_invalid": "attestation_rejected", + "nonce_mismatch": "attestation_rejected", + "spki_mismatch": "attestation_rejected", + "cpu_verification_failed": "attestation_rejected", + "gpu_nonce_mismatch": "attestation_rejected", + "gpu_appraisal_failed": "attestation_rejected", + "composite_appraisal_failed": "attestation_rejected", + "exporter_proof_invalid": "attestation_rejected", + "exporter_mismatch": "attestation_rejected", + "exporter_quote_failed": "attestation_rejected", + "endpoint_invalid": "attestation_rejected", + "unexpected_error": "attestation_rejected", +} + RefreshOutcome = Literal["busy", "stale_expected_fingerprint", "lost_fence"] _REFRESH_EXIT_3: frozenset[str] = frozenset( {"busy", "stale_expected_fingerprint", "lost_fence"} @@ -405,11 +432,15 @@ def _spp_prerequisite(now: datetime) -> tuple[BrainEvidenceComponent, str | None state = spp.get_attestation_state() if state.failure is not None: - reason = ( - "attestation_not_verified" - if state.failure.kind == "unreachable" - else "attestation_rejected" + reason = _SPP_ATTESTATION_FAILURE_REASON_TO_BRAIN_REASON.get( + state.failure.reason_code ) + if reason is None: + LOG.warning( + "event=spp_attestation_reason_unmapped raw_reason=%s", + state.failure.reason_code, + ) + reason = "attestation_rejected" return _failed_component(now, reason), reason if state.session is None: return _failed_component( diff --git a/solstone/think/brain_health.py b/solstone/think/brain_health.py index dfac94d5e..2676ee140 100644 --- a/solstone/think/brain_health.py +++ b/solstone/think/brain_health.py @@ -450,6 +450,25 @@ def _confidential_attestation_from_inspection( "observed_at": observed_at, "expires_at": expires_at, } + if reason == "nvattest_install_in_progress": + return { + "state": "verifying", + "reason": reason, + "observed_at": observed_at, + "expires_at": expires_at, + } + if reason in { + "nvattest_platform_unsupported", + "nvattest_unavailable", + "nvattest_install_failed", + "nvattest_integrity_failed", + }: + return { + "state": "failed", + "reason": reason, + "observed_at": observed_at, + "expires_at": expires_at, + } if reason == "attestation_not_verified": return { "state": "unreachable", diff --git a/solstone/think/providers/brain_state.py b/solstone/think/providers/brain_state.py index 98f4ee2d6..8b096c3ca 100644 --- a/solstone/think/providers/brain_state.py +++ b/solstone/think/providers/brain_state.py @@ -106,6 +106,9 @@ BrainReasonCode = Literal[ "local_runtime_not_ready", "local_artifact_not_ready", "attestation_not_verified", + "nvattest_install_in_progress", + "nvattest_platform_unsupported", + "nvattest_unavailable", "provider_key_invalid", "model_not_found", "provider_quota_exceeded", @@ -118,6 +121,8 @@ BrainReasonCode = Literal[ "cogitate_terminal_error", "attestation_rejected", "attestation_expired", + "nvattest_install_failed", + "nvattest_integrity_failed", "local_server_unhealthy", "configuration_invalid", "fingerprint_key_unavailable", @@ -152,6 +157,9 @@ BRAIN_REASON_TO_AGGREGATE: dict[str, BrainAggregateState] = { "local_runtime_not_ready": "blocked", "local_artifact_not_ready": "blocked", "attestation_not_verified": "blocked", + "nvattest_install_in_progress": "blocked", + "nvattest_platform_unsupported": "blocked", + "nvattest_unavailable": "blocked", "provider_key_invalid": "unhealthy", "model_not_found": "unhealthy", "provider_quota_exceeded": "unhealthy", @@ -164,6 +172,8 @@ BRAIN_REASON_TO_AGGREGATE: dict[str, BrainAggregateState] = { "cogitate_terminal_error": "unhealthy", "attestation_rejected": "unhealthy", "attestation_expired": "unhealthy", + "nvattest_install_failed": "unhealthy", + "nvattest_integrity_failed": "unhealthy", "local_server_unhealthy": "unhealthy", "configuration_invalid": "unknown", "fingerprint_key_unavailable": "unknown", @@ -201,6 +211,11 @@ BRAIN_EVIDENCE_REASON_CODES: dict[str, frozenset[str]] = { "attestation_not_verified", "attestation_rejected", "attestation_expired", + "nvattest_install_in_progress", + "nvattest_platform_unsupported", + "nvattest_unavailable", + "nvattest_install_failed", + "nvattest_integrity_failed", "local_server_unhealthy", "local_runtime_state_invalid", "local_runtime_state_unavailable", @@ -266,8 +281,8 @@ if set(BRAIN_REASON_TO_AGGREGATE) != BRAIN_REASON_CODES: _EVIDENCE_ALLOWED_REASON_CODES = frozenset().union( *BRAIN_EVIDENCE_REASON_CODES.values() ) -if len(_EVIDENCE_ALLOWED_REASON_CODES) != 26: - raise RuntimeError("brain evidence reason partition must contain 26 reasons") +if len(_EVIDENCE_ALLOWED_REASON_CODES) != 31: + raise RuntimeError("brain evidence reason partition must contain 31 reasons") if len(BRAIN_PROJECTION_ONLY_REASON_CODES) != 10: raise RuntimeError("brain projection-only reason partition must contain 10 reasons") if _EVIDENCE_ALLOWED_REASON_CODES & BRAIN_PROJECTION_ONLY_REASON_CODES: diff --git a/solstone/think/providers/nvattest_install.py b/solstone/think/providers/nvattest_install.py index d37205392..e003da266 100644 --- a/solstone/think/providers/nvattest_install.py +++ b/solstone/think/providers/nvattest_install.py @@ -11,12 +11,17 @@ from __future__ import annotations import hashlib import json import os +import platform import shutil import stat +import sys import tempfile from dataclasses import dataclass from pathlib import Path +from typing import Literal +from solstone.think.journal_io import LockTimeout +from solstone.think.journal_io.locking import hold_lock from solstone.think.providers.rfdetr_install import ( RfdetrInstallError, ) @@ -26,16 +31,19 @@ from solstone.think.providers.rfdetr_install import ( from solstone.think.utils import get_journal SPP_NVATTEST_DIR_ENV = "SPP_NVATTEST_DIR" -NVATTEST_VERSION = "1.2.2" -NVATTEST_ARCHIVE_NAME = "libnvat-linux-x86_64-1.2.2.1780962352-archive.tar.xz" -NVATTEST_ARCHIVE_URL = ( - "https://developer.download.nvidia.com/compute/nvat/redist/libnvat/" - f"linux-x86_64/{NVATTEST_ARCHIVE_NAME}" -) -NVATTEST_ARCHIVE_SHA256 = ( - "3f10da6fca794b7e3025c6645447947ec8bc45bcfde5b5b1d23241c7115630db" -) SIDECAR_NAME = ".nvattest-install.json" +CA_BUNDLE_RELATIVE_PATH = Path("share") / "ca" / "ca-bundle.pem" +ENSURE_LOCK_TIMEOUT_S = 0.1 +ENSURE_LOCK_POLL_INTERVAL_S = 0.02 + +NvattestArchiveKey = Literal["linux-x86_64"] +NvattestEnsureStatus = Literal[ + "already_installed", + "installed", + "install_in_flight", + "install_failed", + "platform_unsupported", +] class NvattestInstallError(RuntimeError): @@ -75,12 +83,51 @@ class NvattestInstallRecord: ) -NVATTEST_ARCHIVE_SPEC = NvattestArchiveSpec( - version=NVATTEST_VERSION, - url=NVATTEST_ARCHIVE_URL, - archive_name=NVATTEST_ARCHIVE_NAME, - sha256=NVATTEST_ARCHIVE_SHA256, -) +@dataclass(frozen=True, slots=True) +class NvattestEnsureResult: + status: NvattestEnsureStatus + nvattest_dir: Path | None = None + reason_code: str | None = None + detail: str | None = None + + +NVATTEST_ARCHIVES: dict[NvattestArchiveKey, NvattestArchiveSpec] = { + "linux-x86_64": NvattestArchiveSpec( + version="1.2.2-sol.1", + url=( + "https://updates.solstone.app/providers/nvattest/" + "libnvat-linux-x86_64-1.2.2-sol.1-archive.tar.xz" + ), + archive_name="libnvat-linux-x86_64-1.2.2-sol.1-archive.tar.xz", + sha256="60ef75d1873e7129f03ea80d107d92b2ef216d2a8815958617b30d9c721d474a", + ), +} + + +def nvattest_archive_key( + os_name: str | None = None, + arch: str | None = None, +) -> NvattestArchiveKey | None: + if os_name is None: + os_name = "linux" if sys.platform.startswith("linux") else sys.platform + if arch is None: + arch = platform.machine() + normalized_arch = arch.lower() + if os_name == "linux" and normalized_arch in {"amd64", "x64", "x86_64"}: + return "linux-x86_64" + return None + + +def resolve_nvattest_archive_spec( + archive_key: NvattestArchiveKey | None = None, +) -> NvattestArchiveSpec: + resolved = archive_key or nvattest_archive_key() + if resolved is None: + raise NvattestInstallError( + "platform_unsupported", + "nvattest archive unsupported on this platform", + ) + return NVATTEST_ARCHIVES[resolved] def cache_root(journal_path: str | Path | None = None) -> Path: @@ -103,14 +150,77 @@ def resolve_nvattest_dir( return cache_root(journal_path) +def ensure_nvattest_installed( + *, + explicit_override: str | Path | None = None, + journal_path: str | Path | None = None, + spec: NvattestArchiveSpec | None = None, + lock_timeout: float = ENSURE_LOCK_TIMEOUT_S, +) -> NvattestEnsureResult: + """Ensure the journal-cache nvattest install is ready without blocking peers.""" + + nvattest_dir = resolve_nvattest_dir( + explicit_override, + journal_path=journal_path, + ) + if explicit_override is not None or os.environ.get(SPP_NVATTEST_DIR_ENV): + # Override layout validation stays in nvgpu.binary so appraiser reasons + # still traverse binary -> composite -> ratls instead of install plumbing. + return NvattestEnsureResult( + status="already_installed", + nvattest_dir=nvattest_dir, + ) + + try: + resolved_spec = spec or resolve_nvattest_archive_spec() + except NvattestInstallError as exc: + return NvattestEnsureResult( + status="platform_unsupported", + reason_code=exc.reason_code, + detail=str(exc), + ) + + try: + with hold_lock( + _install_lock_path(journal_path), + timeout=lock_timeout, + poll_interval=ENSURE_LOCK_POLL_INTERVAL_S, + ): + if _installed(nvattest_dir, resolved_spec): + return NvattestEnsureResult( + status="already_installed", + nvattest_dir=nvattest_dir, + ) + try: + installed = install_nvattest( + spec=resolved_spec, + journal_path=journal_path, + ) + except NvattestInstallError as exc: + return NvattestEnsureResult( + status="install_failed", + nvattest_dir=nvattest_dir, + reason_code=exc.reason_code, + detail=str(exc), + ) + return NvattestEnsureResult(status="installed", nvattest_dir=installed) + except LockTimeout: + return NvattestEnsureResult( + status="install_in_flight", + nvattest_dir=nvattest_dir, + reason_code="install-in-progress", + ) + + def install_nvattest( *, force: bool = False, - spec: NvattestArchiveSpec = NVATTEST_ARCHIVE_SPEC, + spec: NvattestArchiveSpec | None = None, journal_path: str | Path | None = None, ) -> Path: """Download, verify, and install nvattest into the journal provider cache.""" + spec = spec or resolve_nvattest_archive_spec() root = cache_root(journal_path) if not force and _installed(root, spec): return root @@ -138,7 +248,11 @@ def install_nvattest( def _has_runtime_layout(root: Path) -> bool: - return (root / "bin" / "nvattest").is_file() and (root / "lib").is_dir() + return ( + (root / "bin" / "nvattest").is_file() + and (root / "lib").is_dir() + and (root / CA_BUNDLE_RELATIVE_PATH).is_file() + ) def _installed(root: Path, spec: NvattestArchiveSpec) -> bool: @@ -163,6 +277,10 @@ def _archive_path( return cache_root(journal_path) / ".downloads" / spec.archive_name +def _install_lock_path(journal_path: str | Path | None = None) -> Path: + return cache_root(journal_path) / ".install" + + def _sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: @@ -234,7 +352,7 @@ def _find_extracted_root(extract_dir: Path) -> Path: for path in extract_dir.rglob("nvattest") if path.is_file() and path.parent.name == "bin" - and (path.parent.parent / "lib").is_dir() + and _has_runtime_layout(path.parent.parent) ] if len(matches) != 1: raise NvattestInstallError( @@ -247,10 +365,14 @@ def _find_extracted_root(extract_dir: Path) -> Path: def _install_extracted_tree(source: Path, root: Path) -> None: binary = source / "bin" / "nvattest" lib_dir = source / "lib" - if not binary.is_file() or not lib_dir.is_dir(): + ca_bundle = source / CA_BUNDLE_RELATIVE_PATH + if not binary.is_file() or not lib_dir.is_dir() or not ca_bundle.is_file(): raise NvattestInstallError( "archive_layout_invalid", - "extracted archive must contain bin/nvattest and lib/", + ( + "extracted archive must contain bin/nvattest, lib/, " + "and share/ca/ca-bundle.pem" + ), ) root.mkdir(parents=True, exist_ok=True) diff --git a/solstone/think/services/spp_attest/composite.py b/solstone/think/services/spp_attest/composite.py index 871b8a373..b992dbe6b 100644 --- a/solstone/think/services/spp_attest/composite.py +++ b/solstone/think/services/spp_attest/composite.py @@ -29,7 +29,12 @@ from solstone.think.services.spp_attest.tlv import decode_gpu_envelope log = logging.getLogger(__name__) _GPU_REASONS = frozenset( - {"nvattest_unavailable", "gpu_nonce_mismatch", "gpu_appraisal_failed"} + { + "nvattest_unavailable", + "nvattest_integrity_failed", + "gpu_nonce_mismatch", + "gpu_appraisal_failed", + } ) diff --git a/solstone/think/services/spp_attest/nvgpu/appraise.py b/solstone/think/services/spp_attest/nvgpu/appraise.py index 7553b61c0..5e5e35861 100644 --- a/solstone/think/services/spp_attest/nvgpu/appraise.py +++ b/solstone/think/services/spp_attest/nvgpu/appraise.py @@ -5,7 +5,9 @@ from __future__ import annotations +import hashlib import json +import logging import subprocess import tempfile from pathlib import Path @@ -25,6 +27,8 @@ from solstone.think.services.spp_attest.nvgpu.evidence import to_nvattest_eviden from solstone.think.services.spp_attest.snp import AppraisalStep from solstone.think.services.spp_attest.tlv import GpuEnvelope +log = logging.getLogger(__name__) + def appraise_gpu_leg( envelope: GpuEnvelope, @@ -59,13 +63,21 @@ def appraise_gpu_leg( handle.write(json.dumps(evidence, sort_keys=True)) handle.write("\n") - command = build_nvattest_attest_command( - nvattest_dir=nvattest_dir, - evidence_file=evidence_path, - owner_nonce=owner_nonce, - rim_store=rim_store, - rim_dir=rim_dir, - ) + try: + command = build_nvattest_attest_command( + nvattest_dir=nvattest_dir, + evidence_file=evidence_path, + owner_nonce=owner_nonce, + rim_store=rim_store, + rim_dir=rim_dir, + ) + except GpuAppraisalError as exc: + _log_gpu_appraisal_failure( + exc.reason, + exception_class=type(exc).__name__, + stderr=None, + ) + raise try: completed = subprocess.run( command.argv, @@ -75,11 +87,21 @@ def appraise_gpu_leg( check=False, ) except OSError as exc: + _log_gpu_appraisal_failure( + "nvattest_unavailable", + exception_class=type(exc).__name__, + stderr=None, + ) raise GpuAppraisalError("nvattest_unavailable") from exc try: stdout_obj = parse_nvattest_stdout(completed.stdout) except ValueError as exc: + _log_gpu_appraisal_failure( + "gpu_appraisal_failed", + returncode=completed.returncode, + stderr=completed.stderr, + ) raise GpuAppraisalError("gpu_appraisal_failed") from exc decision = classify_nvattest_result( @@ -88,6 +110,11 @@ def appraise_gpu_leg( owner_nonce=owner_nonce, ) if not isinstance(decision, NvattestAcceptance): + _log_gpu_appraisal_failure( + decision.reason, + returncode=completed.returncode, + stderr=completed.stderr, + ) raise GpuAppraisalError(decision.reason) steps = [ @@ -111,6 +138,11 @@ def appraise_gpu_leg( steps=steps, ) except ValueError as exc: + _log_gpu_appraisal_failure( + "gpu_appraisal_failed", + returncode=completed.returncode, + stderr=completed.stderr, + ) raise GpuAppraisalError("gpu_appraisal_failed") from exc finally: if evidence_path is not None: @@ -119,3 +151,40 @@ def appraise_gpu_leg( def _ok(name: str, detail: str) -> AppraisalStep: return AppraisalStep(name=name, status="ok", detail=detail) + + +def _log_gpu_appraisal_failure( + reason_code: str, + *, + stderr: str | bytes | None, + returncode: object | None = None, + exception_class: str | None = None, +) -> None: + stderr_bytes = _stderr_bytes(stderr) + digest = hashlib.sha256(stderr_bytes).hexdigest()[:16] + if exception_class is not None: + log.warning( + "event=nvattest_gpu_appraisal_failed reason=%s exception=%s " + "stderr_len=%d stderr_sha256=%s", + reason_code, + exception_class, + len(stderr_bytes), + digest, + ) + return + log.warning( + "event=nvattest_gpu_appraisal_failed reason=%s returncode=%s " + "stderr_len=%d stderr_sha256=%s", + reason_code, + returncode, + len(stderr_bytes), + digest, + ) + + +def _stderr_bytes(stderr: str | bytes | None) -> bytes: + if stderr is None: + return b"" + if isinstance(stderr, bytes): + return stderr + return stderr.encode("utf-8", "surrogateescape") diff --git a/solstone/think/services/spp_attest/nvgpu/binary.py b/solstone/think/services/spp_attest/nvgpu/binary.py index 94d5ded97..5000ca7fb 100644 --- a/solstone/think/services/spp_attest/nvgpu/binary.py +++ b/solstone/think/services/spp_attest/nvgpu/binary.py @@ -12,6 +12,8 @@ from pathlib import Path from solstone.think.services.spp_attest.nvgpu.errors import GpuAppraisalError from solstone.think.services.spp_attest.tlv import SPDM_NONCE_SIZE +CA_BUNDLE_RELATIVE_PATH = Path("share") / "ca" / "ca-bundle.pem" + @dataclass(frozen=True, slots=True) class NvattestCommand: @@ -19,19 +21,22 @@ class NvattestCommand: env: dict[str, str] -def locate_nvattest(nvattest_dir: Path) -> tuple[Path, Path]: - """Return the nvattest binary and lib directory under an injected install dir.""" +def locate_nvattest(nvattest_dir: Path) -> tuple[Path, Path, Path]: + """Return the nvattest binary, lib directory, and CA bundle.""" root = nvattest_dir.resolve() binary = root / "bin" / "nvattest" lib_dir = root / "lib" + ca_bundle = root / CA_BUNDLE_RELATIVE_PATH if not root.is_dir(): raise GpuAppraisalError("nvattest_unavailable") if not binary.is_file(): raise GpuAppraisalError("nvattest_unavailable") if not lib_dir.is_dir(): raise GpuAppraisalError("nvattest_unavailable") - return binary, lib_dir + if not ca_bundle.is_file(): + raise GpuAppraisalError("nvattest_integrity_failed") + return binary, lib_dir, ca_bundle def build_nvattest_attest_command( @@ -53,7 +58,7 @@ def build_nvattest_attest_command( if rim_store == "remote" and rim_dir is not None: raise ValueError("rim_dir is only valid when rim_store == 'dir'") - binary, lib_dir = locate_nvattest(nvattest_dir) + binary, lib_dir, ca_bundle = locate_nvattest(nvattest_dir) argv = [ str(binary), "--format", @@ -69,6 +74,8 @@ def build_nvattest_attest_command( "local", "--rim-store", rim_store, + "--ca-bundle", + str(ca_bundle), ] if rim_dir is not None: argv.extend(["--rim-dir", str(rim_dir)]) diff --git a/solstone/think/services/spp_attest/nvgpu/errors.py b/solstone/think/services/spp_attest/nvgpu/errors.py index e78d3e45f..a27c0fbd6 100644 --- a/solstone/think/services/spp_attest/nvgpu/errors.py +++ b/solstone/think/services/spp_attest/nvgpu/errors.py @@ -11,6 +11,7 @@ from solstone.think.services.spp_attest.errors import VerificationError GpuAppraisalReason = Literal[ "nvattest_unavailable", + "nvattest_integrity_failed", "gpu_nonce_mismatch", "gpu_appraisal_failed", ] diff --git a/solstone/think/services/spp_attest/ratls/verify.py b/solstone/think/services/spp_attest/ratls/verify.py index f578bbb5a..a6dc2be31 100644 --- a/solstone/think/services/spp_attest/ratls/verify.py +++ b/solstone/think/services/spp_attest/ratls/verify.py @@ -126,6 +126,8 @@ def verify_certificate_evidence( code = "gpu_nonce_mismatch" elif "nvattest_unavailable" in reason: code = "nvattest_unavailable" + elif "nvattest_integrity_failed" in reason: + code = "nvattest_integrity_failed" elif "gpu_appraisal_failed" in reason: code = "gpu_appraisal_failed" else: diff --git a/solstone/think/services/spp_transport.py b/solstone/think/services/spp_transport.py index 907b8dca9..23e26f107 100644 --- a/solstone/think/services/spp_transport.py +++ b/solstone/think/services/spp_transport.py @@ -12,13 +12,17 @@ import socket import threading import time from datetime import datetime, timezone +from pathlib import Path from typing import Any, Literal from urllib.parse import urlsplit from OpenSSL import SSL from solstone.think.models import AttestationFailedError, AttestationStaleError -from solstone.think.providers.nvattest_install import resolve_nvattest_dir +from solstone.think.providers.nvattest_install import ( + ensure_nvattest_installed, + resolve_nvattest_dir, +) from solstone.think.services import spp from solstone.think.services.spp_attest.cadence import AttestationSession from solstone.think.services.spp_attest.composite import verify_composite @@ -75,6 +79,33 @@ def _attestation_failed( ) +def _nvattest_prerequisite_failed( + kind: Literal["failed", "unreachable"], + reason_code: str, +) -> None: + with _LOCK: + _teardown_locked() + _attestation_failed(kind, reason_code) + + +def _ensure_nvattest_for_attestation(block: dict[str, Any]) -> Path | None: + if spp.confidential_provenance() is None: + return None + result = ensure_nvattest_installed(explicit_override=block.get("nvattest_dir")) + if result.status in {"already_installed", "installed"}: + return result.nvattest_dir + if result.status == "install_in_flight": + # Another process owns appraiser acquisition; evidence has not been rejected. + _nvattest_prerequisite_failed("unreachable", "nvattest_install_in_progress") + if result.status == "platform_unsupported": + # This host cannot acquire the appraiser archive, so attestation cannot pass. + _nvattest_prerequisite_failed("failed", "nvattest_platform_unsupported") + if result.status == "install_failed": + # Local appraiser acquisition failed before evidence verification could run. + _nvattest_prerequisite_failed("failed", "nvattest_install_failed") + _nvattest_prerequisite_failed("failed", "unexpected_error") + + def _endpoint_from_block(block: dict[str, Any]) -> RatlsEndpoint: endpoint_url = str(block.get("endpoint_url") or "") parsed = urlsplit(endpoint_url) @@ -174,13 +205,20 @@ def _start_listener_locked() -> None: thread.start() -def _establish_channel_locked(block: dict[str, Any], now: datetime) -> AttestedChannel: +def _establish_channel_locked( + block: dict[str, Any], + now: datetime, + *, + nvattest_dir: Path | None = None, +) -> AttestedChannel: try: endpoint = _endpoint_from_block(block) return establish_attested_channel( endpoint, owner_nonce=secrets.token_bytes(OWNER_NONCE_BYTES), - nvattest_dir=resolve_nvattest_dir(block.get("nvattest_dir")), + nvattest_dir=nvattest_dir + if nvattest_dir is not None + else resolve_nvattest_dir(block.get("nvattest_dir")), now=now, composite_verifier=verify_composite, monotonic_now=time.monotonic, @@ -202,8 +240,13 @@ def _establish_channel_locked(block: dict[str, Any], now: datetime) -> AttestedC _attestation_failed("failed", "unexpected_error") -def _establish_and_record_locked(block: dict[str, Any], now: datetime) -> None: - channel = _establish_channel_locked(block, now) +def _establish_and_record_locked( + block: dict[str, Any], + now: datetime, + *, + nvattest_dir: Path | None = None, +) -> None: + channel = _establish_channel_locked(block, now, nvattest_dir=nvattest_dir) _start_listener_locked() _POOL.append(channel) spp.record_attestation_verified( @@ -216,30 +259,41 @@ def _establish_and_record_locked(block: dict[str, Any], now: datetime) -> None: ) +def _reuse_or_raise_stale_locked(now: datetime) -> bool: + state = spp.get_attestation_state() + if ( + state.session is not None + and state.session.status(now) == "verified" + and _transport_live_locked() + ): + return True + if ( + state.session is not None + and state.session.status(now) != "verified" + and _transport_live_locked() + ): + _teardown_locked() + raise AttestationStaleError( + "the confidential attestation cadence lapsed (attestation_stale)" + ) + return False + + def verify_confidential_attestation(block: dict[str, Any]) -> None: global _CONFIDENTIAL_BLOCK now = datetime.now(timezone.utc) with _LOCK: _CONFIDENTIAL_BLOCK = dict(block) - state = spp.get_attestation_state() - if ( - state.session is not None - and state.session.status(now) == "verified" - and _transport_live_locked() - ): + if _reuse_or_raise_stale_locked(now): return - if ( - state.session is not None - and state.session.status(now) != "verified" - and _transport_live_locked() - ): - _teardown_locked() - raise AttestationStaleError( - "the confidential attestation cadence lapsed (attestation_stale)" - ) - _establish_and_record_locked(block, now) + nvattest_dir = _ensure_nvattest_for_attestation(block) + with _LOCK: + _CONFIDENTIAL_BLOCK = dict(block) + if _reuse_or_raise_stale_locked(now): + return + _establish_and_record_locked(block, now, nvattest_dir=nvattest_dir) def confidential_egress_base_url(endpoint_base_url: str) -> str: @@ -295,12 +349,16 @@ def recheck_confidential_attestation() -> None: if block is None: return now = datetime.now(timezone.utc) + try: + nvattest_dir = _ensure_nvattest_for_attestation(block) + except AttestationFailedError: + return with _LOCK: _teardown_locked() spp.clear_attestation_state() _CONFIDENTIAL_BLOCK = dict(block) try: - _establish_and_record_locked(block, now) + _establish_and_record_locked(block, now, nvattest_dir=nvattest_dir) except AttestationFailedError: return @@ -320,6 +378,8 @@ def _borrow_or_establish_channel_locked(now: datetime) -> AttestedChannel | None if _CONFIDENTIAL_BLOCK is None: return _activate_channel_locked(channel) if channel is None: + # Public entry points run ensure-install before the first verified session; + # this locked refill only opens an extra channel and must not download. channel = _establish_channel_locked(_CONFIDENTIAL_BLOCK, now) return _activate_channel_locked(channel) diff --git a/tests/services/test_spp_attest_composite.py b/tests/services/test_spp_attest_composite.py index 047dd672a..be023cb64 100644 --- a/tests/services/test_spp_attest_composite.py +++ b/tests/services/test_spp_attest_composite.py @@ -69,6 +69,8 @@ def _fake_nvattest_dir(tmp_path: Path) -> Path: (root / "bin").mkdir(parents=True) (root / "bin" / "nvattest").write_text("#!/bin/sh\n", encoding="utf-8") (root / "lib").mkdir() + (root / "share" / "ca").mkdir(parents=True) + (root / "share" / "ca" / "ca-bundle.pem").write_text("ca\n", encoding="utf-8") return root @@ -345,8 +347,16 @@ def test_verify_composite_bounds_out_of_band_gpu_reason_without_leak( _assert_owner_message_safe(exc_info.value) -def test_verify_composite_rejects_gpu_unavailable_without_cpu_only_pass( +@pytest.mark.parametrize( + "reason_code", + [ + "nvattest_unavailable", + "nvattest_integrity_failed", + ], +) +def test_verify_composite_rejects_gpu_appraiser_prerequisite_without_cpu_only_pass( tmp_path: Path, + reason_code: str, ) -> None: def unavailable_gpu_appraiser( _envelope: GpuEnvelope, @@ -355,7 +365,7 @@ def test_verify_composite_rejects_gpu_unavailable_without_cpu_only_pass( nvattest_dir: Path, ) -> GpuAppraisal: assert nvattest_dir - raise GpuAppraisalError("nvattest_unavailable") + raise GpuAppraisalError(reason_code) with pytest.raises(AttestationFailedError) as exc_info: verify_composite( @@ -368,7 +378,5 @@ def test_verify_composite_rejects_gpu_unavailable_without_cpu_only_pass( gpu_appraiser=unavailable_gpu_appraiser, ) - assert exc_info.value.detail == ( - "the GPU leg rejected the evidence (nvattest_unavailable)" - ) + assert exc_info.value.detail == f"the GPU leg rejected the evidence ({reason_code})" _assert_owner_message_safe(exc_info.value) diff --git a/tests/services/test_spp_attest_nvgpu.py b/tests/services/test_spp_attest_nvgpu.py index 80ecf05ee..f9e2dc979 100644 --- a/tests/services/test_spp_attest_nvgpu.py +++ b/tests/services/test_spp_attest_nvgpu.py @@ -4,7 +4,9 @@ from __future__ import annotations import base64 +import hashlib import json +import logging import subprocess from copy import deepcopy from pathlib import Path @@ -51,6 +53,8 @@ def _fake_nvattest_dir(tmp_path: Path) -> Path: (root / "bin").mkdir(parents=True, exist_ok=True) (root / "bin" / "nvattest").write_text("#!/bin/sh\n", encoding="utf-8") (root / "lib").mkdir(exist_ok=True) + (root / "share" / "ca").mkdir(parents=True, exist_ok=True) + (root / "share" / "ca" / "ca-bundle.pem").write_text("ca\n", encoding="utf-8") return root @@ -360,6 +364,38 @@ def test_gpu_appraisal_error_message_omits_vendor_stderr_marker( assert marker not in str(exc_info.value) +def test_gpu_appraisal_failure_log_uses_bounded_stderr_digest( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + stderr = "collector detail\n" * 5000 + caplog.set_level(logging.WARNING, logger=appraise_module.log.name) + + with pytest.raises(GpuAppraisalError): + _run_appraisal_with_stdout( + monkeypatch, + tmp_path, + "not json", + stderr=stderr, + ) + + messages = [ + record.getMessage() + for record in caplog.records + if record.name == appraise_module.log.name + and "event=nvattest_gpu_appraisal_failed" in record.getMessage() + ] + assert len(messages) == 1 + message = messages[0] + fingerprint = message.rsplit("stderr_sha256=", 1)[1].split()[0] + assert len(fingerprint) == 16 + assert fingerprint == hashlib.sha256(stderr.encode("utf-8")).hexdigest()[:16] + assert f"stderr_len={len(stderr.encode('utf-8'))}" in message + assert len(message) < 180 + assert "collector detail" not in message + + def test_bool_false_returncode_rejects( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -409,6 +445,23 @@ def test_nvattest_command_env_inherits_parent_and_sets_library_path( assert command.env["SPP_NVATTEST_PARENT_SENTINEL"] == "kept" assert command.env["LD_LIBRARY_PATH"] == str(nvattest_dir / "lib") + assert command.argv[command.argv.index("--ca-bundle") + 1] == str( + nvattest_dir / "share" / "ca" / "ca-bundle.pem" + ) + + +def test_nvattest_command_rejects_missing_ca_bundle(tmp_path: Path) -> None: + nvattest_dir = _fake_nvattest_dir(tmp_path) + (nvattest_dir / "share" / "ca" / "ca-bundle.pem").unlink() + + with pytest.raises(GpuAppraisalError) as exc_info: + build_nvattest_attest_command( + nvattest_dir=nvattest_dir, + evidence_file=tmp_path / "evidence.json", + owner_nonce=_owner_nonce(), + ) + + assert exc_info.value.reason == "nvattest_integrity_failed" def test_nvattest_command_uses_absolute_install_paths_not_path_or_python_namespace( diff --git a/tests/services/test_spp_attest_purity.py b/tests/services/test_spp_attest_purity.py index 1cb315da6..28595bd9c 100644 --- a/tests/services/test_spp_attest_purity.py +++ b/tests/services/test_spp_attest_purity.py @@ -165,6 +165,11 @@ def test_nvgpu_appraise_removes_temp_evidence_file_on_return_and_raise( (nvattest_dir / "bin").mkdir(parents=True) (nvattest_dir / "bin" / "nvattest").write_text("#!/bin/sh\n", encoding="utf-8") (nvattest_dir / "lib").mkdir() + (nvattest_dir / "share" / "ca").mkdir(parents=True) + (nvattest_dir / "share" / "ca" / "ca-bundle.pem").write_text( + "ca\n", + encoding="utf-8", + ) envelope = decode_gpu_envelope((FIXTURE_DIR / "gpu-envelope.tlv").read_bytes()) owner_nonce = bytes.fromhex((FIXTURE_DIR / "nonce.hex").read_text().strip()) observed: list[Path] = [] diff --git a/tests/services/test_spp_attest_ratls_verify.py b/tests/services/test_spp_attest_ratls_verify.py index ad06c539b..1532187cc 100644 --- a/tests/services/test_spp_attest_ratls_verify.py +++ b/tests/services/test_spp_attest_ratls_verify.py @@ -220,15 +220,30 @@ def test_verify_certificate_evidence_rejects_spki_mismatch(tmp_path: Path) -> No assert exc_info.value.reason_code == "spki_mismatch" -def test_verify_certificate_evidence_maps_composite_failure(tmp_path: Path) -> None: +@pytest.mark.parametrize( + ("detail", "expected_reason"), + [ + ( + "the CPU leg rejected evidence (cpu_verification_failed)", + "cpu_verification_failed", + ), + ( + "the GPU leg rejected evidence (nvattest_integrity_failed)", + "nvattest_integrity_failed", + ), + ], +) +def test_verify_certificate_evidence_maps_composite_failure( + tmp_path: Path, + detail: str, + expected_reason: str, +) -> None: nonce = b"n" * 32 key, spki = _key_and_spki() evidence = _evidence(nonce, spki) def composite_verifier(_bundle, **_kwargs): - raise AttestationFailedError( - "the CPU leg rejected evidence (cpu_verification_failed)" - ) + raise AttestationFailedError(detail) with pytest.raises(RatlsVerificationError) as exc_info: verify_certificate_evidence( @@ -239,7 +254,7 @@ def test_verify_certificate_evidence_maps_composite_failure(tmp_path: Path) -> N composite_verifier=composite_verifier, ) - assert exc_info.value.reason_code == "cpu_verification_failed" + assert exc_info.value.reason_code == expected_reason def test_verify_exporter_proof_binds_quote_to_exporter(monkeypatch) -> None: diff --git a/tests/services/test_spp_ratls_loopback_e2e.py b/tests/services/test_spp_ratls_loopback_e2e.py index ca6e6473b..baee0a0a5 100644 --- a/tests/services/test_spp_ratls_loopback_e2e.py +++ b/tests/services/test_spp_ratls_loopback_e2e.py @@ -21,6 +21,8 @@ def _nvattest_root(tmp_path: Path) -> Path: (root / "bin").mkdir(parents=True) (root / "bin" / "nvattest").write_text("#!/bin/sh\n", encoding="utf-8") (root / "lib").mkdir() + (root / "share" / "ca").mkdir(parents=True) + (root / "share" / "ca" / "ca-bundle.pem").write_text("ca\n", encoding="utf-8") return root diff --git a/tests/services/test_spp_transport.py b/tests/services/test_spp_transport.py index 046dc666c..0f752b08f 100644 --- a/tests/services/test_spp_transport.py +++ b/tests/services/test_spp_transport.py @@ -7,6 +7,7 @@ import json import time from datetime import datetime, timedelta, timezone from pathlib import Path +from types import SimpleNamespace from unittest.mock import Mock import pytest @@ -59,7 +60,17 @@ class _AliveThread: @pytest.fixture(autouse=True) -def _clear_transport_state(): +def _clear_transport_state(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + spp_transport, + "ensure_nvattest_installed", + lambda **_kwargs: SimpleNamespace( + status="already_installed", + nvattest_dir=Path("/tmp/solstone-nvattest-test"), + reason_code=None, + detail=None, + ), + ) spp.delete_attestation_state() spp_transport.teardown_confidential_transport() yield @@ -331,6 +342,7 @@ RATLS_VERIFICATION_REASON_CODES = ( "cpu_verification_failed", "gpu_nonce_mismatch", "nvattest_unavailable", + "nvattest_integrity_failed", "gpu_appraisal_failed", "composite_appraisal_failed", "exporter_proof_invalid", @@ -393,6 +405,45 @@ def test_attestation_failure_buckets_real_reason_codes_at_transport_catch_site( assert failure.reason_code == reason_code +@pytest.mark.parametrize( + ("status", "kind", "reason_code"), + [ + ("install_in_flight", "unreachable", "nvattest_install_in_progress"), + ("platform_unsupported", "failed", "nvattest_platform_unsupported"), + ("install_failed", "failed", "nvattest_install_failed"), + ], +) +def test_verify_confidential_attestation_records_nvattest_install_prerequisites( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + status: str, + kind: str, + reason_code: str, +) -> None: + block = _write_confidential_config(tmp_path, monkeypatch) + establish = Mock(side_effect=AssertionError("verify should not run")) + monkeypatch.setattr(spp_transport, "establish_attested_channel", establish) + monkeypatch.setattr( + spp_transport, + "ensure_nvattest_installed", + lambda **_kwargs: SimpleNamespace( + status=status, + nvattest_dir=None, + reason_code=None, + detail=None, + ), + ) + + with pytest.raises(AttestationFailedError): + spp_transport.verify_confidential_attestation(block) + + establish.assert_not_called() + failure = spp.get_attestation_state().failure + assert failure is not None + assert failure.kind == kind + assert failure.reason_code == reason_code + + def test_recheck_confidential_attestation_records_success( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -417,6 +468,41 @@ def test_recheck_confidential_attestation_records_success( assert block["endpoint_url"] == "https://spp.example.test:9443" +def test_recheck_confidential_attestation_ensures_nvattest_before_verify( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + block = _write_confidential_config(tmp_path, monkeypatch) + _patch_listener(monkeypatch) + nvattest_dir = tmp_path / "cache" / "providers" / "nvattest" + ensured: list[dict[str, object]] = [] + + def fake_ensure_nvattest_installed(**kwargs): + ensured.append(kwargs) + return SimpleNamespace( + status="already_installed", + nvattest_dir=nvattest_dir, + reason_code=None, + detail=None, + ) + + def fake_establish(_endpoint, **kwargs): + assert ensured + assert kwargs["nvattest_dir"] == nvattest_dir + return _FakeChannel(object()) + + monkeypatch.setattr( + spp_transport, "ensure_nvattest_installed", fake_ensure_nvattest_installed + ) + monkeypatch.setattr(spp_transport, "establish_attested_channel", fake_establish) + + spp_transport.recheck_confidential_attestation() + + assert ensured == [{"explicit_override": None}] + assert spp.get_attestation_state().failure is None + assert block["endpoint_url"] == "https://spp.example.test:9443" + + def test_recheck_confidential_attestation_fails_closed_and_preserves_last_verified( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -441,11 +527,22 @@ def test_recheck_confidential_attestation_fails_closed_and_preserves_last_verifi def test_recheck_confidential_attestation_off_is_noop( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + config_dir = tmp_path / "config" + config_dir.mkdir(parents=True) + (config_dir / "journal.json").write_text("{}", encoding="utf-8") establish = Mock(side_effect=AssertionError("attestation attempted")) monkeypatch.setattr(spp_transport, "establish_attested_channel", establish) + monkeypatch.setattr( + spp_transport, + "ensure_nvattest_installed", + Mock(side_effect=AssertionError("nvattest ensure attempted")), + ) spp_transport.recheck_confidential_attestation() establish.assert_not_called() + assert not (tmp_path / "cache" / "providers" / "nvattest").exists() diff --git a/tests/test_brain_cli.py b/tests/test_brain_cli.py index bb5f48568..01544eba2 100644 --- a/tests/test_brain_cli.py +++ b/tests/test_brain_cli.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse import hashlib import json +import logging import sys from datetime import datetime, timezone from pathlib import Path @@ -408,6 +409,64 @@ def _write_unhealthy_record(journal: Path) -> None: ) +@pytest.mark.parametrize( + ("raw_reason", "expected_reason", "expected_status"), + [ + ("nvattest_install_in_progress", "nvattest_install_in_progress", "blocked"), + ("nvattest_platform_unsupported", "nvattest_platform_unsupported", "blocked"), + ("nvattest_unavailable", "nvattest_unavailable", "blocked"), + ("nvattest_install_failed", "nvattest_install_failed", "failed"), + ("nvattest_integrity_failed", "nvattest_integrity_failed", "failed"), + ("gateway_unreachable", "attestation_not_verified", "blocked"), + ], +) +def test_spp_prerequisite_maps_failure_reason_code( + monkeypatch: pytest.MonkeyPatch, + raw_reason: str, + expected_reason: str, + expected_status: str, +) -> None: + from solstone.think.services import spp + + spp.delete_attestation_state() + monkeypatch.setattr( + "solstone.think.services.spp_transport.recheck_confidential_attestation", + lambda: spp.record_attestation_failed("failed", raw_reason), + ) + try: + component, reason = brain_cli._spp_prerequisite(NOW) + finally: + spp.delete_attestation_state() + + assert reason == expected_reason + assert component["reason_code"] == expected_reason + assert component["status"] == expected_status + + +def test_spp_prerequisite_warns_and_fails_closed_on_unmapped_reason( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + from solstone.think.services import spp + + spp.delete_attestation_state() + monkeypatch.setattr( + "solstone.think.services.spp_transport.recheck_confidential_attestation", + lambda: spp.record_attestation_failed("failed", "new_raw_reason"), + ) + caplog.set_level(logging.WARNING, logger=brain_cli.LOG.name) + try: + component, reason = brain_cli._spp_prerequisite(NOW) + finally: + spp.delete_attestation_state() + + assert reason == "attestation_rejected" + assert component["reason_code"] == "attestation_rejected" + assert ( + "event=spp_attestation_reason_unmapped raw_reason=new_raw_reason" in caplog.text + ) + + def _fake_runtime_inspection( *, phase: str = "ready", desired: str = RUNTIME_FP ) -> dict[str, Any]: diff --git a/tests/test_brain_state.py b/tests/test_brain_state.py index 70fe40c8d..c3a5a29c8 100644 --- a/tests/test_brain_state.py +++ b/tests/test_brain_state.py @@ -257,6 +257,9 @@ def test_vocabularies_and_reason_mapping_are_closed() -> None: "local_runtime_not_ready", "local_artifact_not_ready", "attestation_not_verified", + "nvattest_install_in_progress", + "nvattest_platform_unsupported", + "nvattest_unavailable", "provider_key_invalid", "model_not_found", "provider_quota_exceeded", @@ -269,6 +272,8 @@ def test_vocabularies_and_reason_mapping_are_closed() -> None: "cogitate_terminal_error", "attestation_rejected", "attestation_expired", + "nvattest_install_failed", + "nvattest_integrity_failed", "local_server_unhealthy", "configuration_invalid", "fingerprint_key_unavailable", @@ -307,7 +312,7 @@ def test_vocabularies_and_reason_mapping_are_closed() -> None: assert set(BRAIN_REASON_TO_AGGREGATE) == BRAIN_REASON_CODES assert set(BRAIN_REASON_TO_AGGREGATE.values()) <= BRAIN_AGGREGATE_STATES evidence_reasons = frozenset().union(*BRAIN_EVIDENCE_REASON_CODES.values()) - assert len(evidence_reasons) == 26 + assert len(evidence_reasons) == 31 assert len(BRAIN_PROJECTION_ONLY_REASON_CODES) == 10 assert evidence_reasons | BRAIN_PROJECTION_ONLY_REASON_CODES == BRAIN_REASON_CODES assert not (evidence_reasons & BRAIN_PROJECTION_ONLY_REASON_CODES) diff --git a/tests/test_nvattest_install.py b/tests/test_nvattest_install.py index 107673618..556da19fe 100644 --- a/tests/test_nvattest_install.py +++ b/tests/test_nvattest_install.py @@ -7,13 +7,35 @@ import hashlib import json import shutil import tarfile +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path import pytest +from solstone.think.journal_io import LockTimeout from solstone.think.providers import nvattest_install +def test_linux_x86_64_archive_pin_is_exact() -> None: + spec = nvattest_install.NVATTEST_ARCHIVES["linux-x86_64"] + expected_url = ( + "https://updates.solstone.app/providers/nvattest/" + "libnvat-linux-x86_64-1.2.2-sol.1-archive.tar.xz" + ) + legacy_sha = "3f10da6fca794b7e3025c6645447947ec8bc45bcfde5b5b1d23241c7115630db" + + assert spec.version == "1.2.2-sol.1" + assert spec.url == expected_url + assert ( + spec.sha256 + == "60ef75d1873e7129f03ea80d107d92b2ef216d2a8815958617b30d9c721d474a" + ) + source = Path(nvattest_install.__file__).read_text(encoding="utf-8") + assert "developer.download.nvidia.com" not in source + assert legacy_sha not in source + + def test_install_nvattest_reinstalls_partial_cache_without_sidecar( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -84,6 +106,8 @@ def test_install_nvattest_valid_sidecar_is_noop( (root / "bin").mkdir(parents=True) (root / "bin" / "nvattest").write_text("installed\n", encoding="utf-8") (root / "lib").mkdir() + (root / "share" / "ca").mkdir(parents=True) + (root / "share" / "ca" / "ca-bundle.pem").write_text("ca\n", encoding="utf-8") spec = nvattest_install.NvattestArchiveSpec( version="1.0.0", url="https://example.invalid/nvattest.tar.gz", @@ -104,7 +128,160 @@ def test_install_nvattest_valid_sidecar_is_noop( assert nvattest_install.install_nvattest(spec=spec, journal_path=tmp_path) == root -def _fixture_spec(tmp_path: Path) -> nvattest_install.NvattestArchiveSpec: +def test_install_nvattest_upgrades_old_nvidia_sidecar( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = nvattest_install.cache_root(tmp_path) + (root / "bin").mkdir(parents=True) + (root / "bin" / "nvattest").write_text("old\n", encoding="utf-8") + (root / "lib").mkdir() + (root / "share" / "ca").mkdir(parents=True) + (root / "share" / "ca" / "ca-bundle.pem").write_text("old-ca\n", encoding="utf-8") + (root / nvattest_install.SIDECAR_NAME).write_text( + json.dumps( + { + "archive_sha256": ( + "3f10da6fca794b7e3025c6645447947ec8bc45bcfde5b5b1d23241c7115630db" + ), + "version": "1.2.2", + } + ) + + "\n", + encoding="utf-8", + ) + spec = _fixture_spec(tmp_path, version="1.2.2-sol.1") + calls: list[Path] = [] + + def fake_download(_url: str, dest: Path, _expected_sha256: str) -> None: + calls.append(dest) + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(tmp_path / spec.archive_name, dest) + + monkeypatch.setattr(nvattest_install, "_download_file", fake_download) + + nvattest_install.install_nvattest(spec=spec, journal_path=tmp_path) + + assert len(calls) == 1 + assert (root / "bin" / "nvattest").read_text(encoding="utf-8") == "new\n" + sidecar = json.loads( + (root / nvattest_install.SIDECAR_NAME).read_text(encoding="utf-8") + ) + assert sidecar["version"] == "1.2.2-sol.1" + + +def test_install_nvattest_rejects_hash_mismatch_without_partial_tree( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeResponse: + def __enter__(self): + return self + + def __exit__(self, *_exc_info) -> bool: + return False + + def raise_for_status(self) -> None: + return None + + def iter_bytes(self): + yield b"not the pinned archive" + + def fake_stream(*_args, **_kwargs): + return FakeResponse() + + monkeypatch.setattr("httpx.stream", fake_stream) + spec = nvattest_install.NvattestArchiveSpec( + version="1.2.2-sol.1", + url="https://example.invalid/nvattest.tar.xz", + archive_name="nvattest.tar.xz", + sha256="0" * 64, + ) + + with pytest.raises(nvattest_install.NvattestInstallError) as exc_info: + nvattest_install.install_nvattest(spec=spec, journal_path=tmp_path) + + assert exc_info.value.reason_code == "sha256_mismatch" + root = nvattest_install.cache_root(tmp_path) + assert not (root / "bin").exists() + assert not (root / "lib").exists() + assert not (root / "share").exists() + assert not (root / ".downloads" / "nvattest.tar.xz").exists() + assert not (root / ".downloads" / "nvattest.tar.xz.tmp").exists() + + +def test_ensure_nvattest_unsupported_platform_does_not_touch_cache( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(nvattest_install, "nvattest_archive_key", lambda: None) + + result = nvattest_install.ensure_nvattest_installed(journal_path=tmp_path) + + assert result.status == "platform_unsupported" + assert result.reason_code == "platform_unsupported" + assert not nvattest_install.cache_root(tmp_path).exists() + + +def test_ensure_nvattest_lock_timeout_is_in_flight( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + @contextmanager + def fake_hold_lock(path: Path, *, timeout: float, **_kwargs) -> Iterator[None]: + raise LockTimeout(path, timeout) + yield + + monkeypatch.setattr(nvattest_install, "hold_lock", fake_hold_lock) + + result = nvattest_install.ensure_nvattest_installed(journal_path=tmp_path) + + assert result.status == "install_in_flight" + assert result.reason_code == "install-in-progress" + + +def test_ensure_nvattest_override_skips_cache_download( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + override = tmp_path / "override" + monkeypatch.setenv(nvattest_install.SPP_NVATTEST_DIR_ENV, str(override)) + monkeypatch.setattr( + nvattest_install, + "_download_file", + lambda *_args, **_kwargs: pytest.fail("download should not run"), + ) + + result = nvattest_install.ensure_nvattest_installed(journal_path=tmp_path) + + assert result.status == "already_installed" + assert result.nvattest_dir == override + assert not nvattest_install.cache_root(tmp_path).exists() + + +def test_install_nvattest_accepts_wrapped_archive( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + spec = _fixture_spec(tmp_path, wrapped=True) + + def fake_download(_url: str, dest: Path, _expected_sha256: str) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(tmp_path / spec.archive_name, dest) + + monkeypatch.setattr(nvattest_install, "_download_file", fake_download) + + installed = nvattest_install.install_nvattest(spec=spec, journal_path=tmp_path) + + assert (installed / "bin" / "nvattest").read_text(encoding="utf-8") == "new\n" + + +def _fixture_spec( + tmp_path: Path, + *, + version: str = "9.9.9", + wrapped: bool = False, +) -> nvattest_install.NvattestArchiveSpec: archive_name = "nvattest-fixture.tar.gz" source = tmp_path / "source" / "nvattest-fixture" (source / "bin").mkdir(parents=True) @@ -113,12 +290,22 @@ def _fixture_spec(tmp_path: Path) -> nvattest_install.NvattestArchiveSpec: (source / "lib" / "libnvat.so.1").write_text("library\n", encoding="utf-8") (source / "lib" / "libnvat.so").symlink_to("libnvat.so.1") (source / "LICENSE").write_text("license\n", encoding="utf-8") + (source / "share" / "ca").mkdir(parents=True) + (source / "share" / "ca" / "ca-bundle.pem").write_text("ca\n", encoding="utf-8") + (source / "share" / "THIRD_PARTY_NOTICES.md").write_text( + "notices\n", + encoding="utf-8", + ) archive_path = tmp_path / archive_name with tarfile.open(archive_path, "w:gz") as archive: - archive.add(source, arcname=source.name) + if wrapped: + archive.add(source, arcname=source.name) + else: + for child in source.iterdir(): + archive.add(child, arcname=child.name) return nvattest_install.NvattestArchiveSpec( - version="9.9.9", + version=version, url="https://example.invalid/nvattest-fixture.tar.gz", archive_name=archive_name, sha256=hashlib.sha256(archive_path.read_bytes()).hexdigest(),