From 83789dad2f8eed1e00d3e129a855eed79be8adb7 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Tue, 28 Jul 2026 05:43:42 -0600 Subject: [PATCH] test(providers): pin nvattest executable bits to the real archives Ground truth is derived from the published archives through the repo's own extraction path, never from the authority. Member set, kind, and link target are already covered by the digest-pinned companion manifests, so this fixture covers only the executable bit. The offline check runs in make ci via make test, and the network-using generator is deliberately excluded from install-checks. Co-Authored-By: Codex --- Makefile | 3 + scripts/build_nvattest_payload_facts.py | 205 ++++++++++++++++++ ...4-1.2.2-sol.2-archive.executable-bits.json | 14 ++ ...4-1.2.2-sol.2-archive.executable-bits.json | 14 ++ ...4-1.2.2-sol.2-archive.executable-bits.json | 14 ++ tests/test_nvattest_authority.py | 77 +++++++ 6 files changed, 327 insertions(+) create mode 100644 scripts/build_nvattest_payload_facts.py create mode 100644 tests/fixtures/nvattest/libnvat-linux-aarch64-1.2.2-sol.2-archive.executable-bits.json create mode 100644 tests/fixtures/nvattest/libnvat-linux-x86_64-1.2.2-sol.2-archive.executable-bits.json create mode 100644 tests/fixtures/nvattest/libnvat-macos-arm64-1.2.2-sol.2-archive.executable-bits.json diff --git a/Makefile b/Makefile index a8c139bce..7ddf14f6c 100644 --- a/Makefile +++ b/Makefile @@ -833,6 +833,9 @@ check-journal-resolution-vectors: .installed nvattest-authority: $(VENV_BIN)/python scripts/build_nvattest_authority.py +nvattest-payload-facts: + $(VENV_BIN)/python scripts/build_nvattest_payload_facts.py + check-nvattest-authority: .installed $(VENV_BIN)/python scripts/build_nvattest_authority.py --check diff --git a/scripts/build_nvattest_payload_facts.py b/scripts/build_nvattest_payload_facts.py new file mode 100644 index 000000000..0100260e4 --- /dev/null +++ b/scripts/build_nvattest_payload_facts.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Build committed nvattest payload executable-bit fixtures.""" + +from __future__ import annotations + +import argparse +import difflib +import hashlib +import json +import os +import sys +import tempfile +import urllib.request +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from solstone.think.providers import nvattest_install +from solstone.think.providers.nvattest_authority import authority_entry + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE_DIR = ROOT / "tests" / "fixtures" / "nvattest" +TARGET_KEYS = ("linux-aarch64", "linux-x86_64", "macos-arm64") +SCHEMA_VERSION = 1 +USER_AGENT = "solstone-nvattest-payload-facts/1.0" + + +class NvattestPayloadFactsError(RuntimeError): + pass + + +def render_payload_facts_json(payload: dict[str, Any]) -> str: + return json.dumps(payload, indent=2, sort_keys=True) + "\n" + + +def _archive_name(url: str) -> str: + name = Path(urlparse(url).path).name + if not name.endswith(".tar.xz"): + raise NvattestPayloadFactsError(f"nvattest archive url is invalid: {url}") + return name + + +def _fixture_path(url: str) -> Path: + archive_name = _archive_name(url) + return FIXTURE_DIR / f"{archive_name.removesuffix('.tar.xz')}.executable-bits.json" + + +def _download_verified_archive(url: str, expected_sha256: str, dest: Path) -> str: + dest.parent.mkdir(parents=True, exist_ok=True) + tmp = dest.with_name(f".{dest.name}.tmp") + digest = hashlib.sha256() + try: + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=120) as response: + with tmp.open("wb") as handle: + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + handle.write(chunk) + handle.flush() + os.fsync(handle.fileno()) + actual_sha256 = digest.hexdigest() + if actual_sha256 != expected_sha256: + raise NvattestPayloadFactsError( + f"sha256 mismatch for {_archive_name(url)}: " + f"expected {expected_sha256}, got {actual_sha256}" + ) + os.replace(tmp, dest) + return actual_sha256 + except NvattestPayloadFactsError: + tmp.unlink(missing_ok=True) + raise + except Exception as exc: + tmp.unlink(missing_ok=True) + raise NvattestPayloadFactsError( + f"failed to download nvattest archive {_archive_name(url)}: {exc}" + ) from exc + + +def _find_payload_root(extract_dir: Path) -> Path: + entries = sorted(extract_dir.iterdir(), key=lambda item: item.name) + if not entries: + raise NvattestPayloadFactsError("nvattest archive extracted no payload members") + if len(entries) == 1: + only = entries[0] + if only.is_dir() and not only.is_symlink(): + return only + raise NvattestPayloadFactsError( + f"nvattest archive single extracted member is not a directory: {only.name}" + ) + return extract_dir + + +def _derive_executable_bits(payload_root: Path) -> dict[str, bool]: + observed: dict[str, dict[str, Any]] = {} + observed_dirs: set[str] = set() + for child in sorted(payload_root.iterdir(), key=lambda item: item.name): + nvattest_install._scan_payload_path( + child, + payload_root, + observed, + observed_dirs, + ) + if not observed: + raise NvattestPayloadFactsError("nvattest archive contains no payload members") + executable: dict[str, bool] = {} + for relpath, fact in sorted(observed.items()): + kind = fact.get("kind") + if kind not in {"regular", "symlink"}: + raise NvattestPayloadFactsError( + f"nvattest archive member {relpath} has unsupported kind {kind!r}" + ) + value = fact.get("executable") + if not isinstance(value, bool): + raise NvattestPayloadFactsError( + f"nvattest archive member {relpath} executable bit is invalid" + ) + executable[relpath] = value + return executable + + +def build_payload_facts(target_key: str, work_dir: Path) -> tuple[Path, str]: + entry = authority_entry(target_key) + url = entry.artifact.url + expected_sha256 = entry.artifact.sha256 + target_dir = work_dir / target_key + archive = target_dir / _archive_name(url) + extract_dir = target_dir / "extract" + actual_sha256 = _download_verified_archive(url, expected_sha256, archive) + nvattest_install._safe_extract_nvattest_tarball(archive, extract_dir) + payload = { + "archive_sha256": actual_sha256, + "executable": _derive_executable_bits(_find_payload_root(extract_dir)), + "schema_version": SCHEMA_VERSION, + "target": target_key, + } + return _fixture_path(url), render_payload_facts_json(payload) + + +def _render_all(work_dir: Path) -> dict[Path, str]: + return { + path: text + for target_key in TARGET_KEYS + for path, text in (build_payload_facts(target_key, work_dir),) + } + + +def write_outputs() -> None: + with tempfile.TemporaryDirectory(prefix="solstone-nvattest-payload-facts-") as tmp: + outputs = _render_all(Path(tmp)) + FIXTURE_DIR.mkdir(parents=True, exist_ok=True) + for path, text in sorted(outputs.items()): + path.write_text(text, encoding="utf-8") + print(f"wrote {path.relative_to(ROOT)}") + + +def check_outputs() -> int: + with tempfile.TemporaryDirectory(prefix="solstone-nvattest-payload-facts-") as tmp: + outputs = _render_all(Path(tmp)) + status = 0 + for path, expected in sorted(outputs.items()): + try: + actual = path.read_text(encoding="utf-8") + except FileNotFoundError: + actual = "" + if actual == expected: + continue + status = 1 + relpath = path.relative_to(ROOT) + print( + f"nvattest executable-bit fixture is stale: {relpath}. " + "Run: make nvattest-payload-facts", + file=sys.stderr, + ) + diff = difflib.unified_diff( + actual.splitlines(keepends=True), + expected.splitlines(keepends=True), + fromfile=f"{relpath} (actual)", + tofile=f"{relpath} (expected)", + ) + print("".join(diff), file=sys.stderr, end="") + return status + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + try: + if args.check: + return check_outputs() + write_outputs() + return 0 + except NvattestPayloadFactsError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/nvattest/libnvat-linux-aarch64-1.2.2-sol.2-archive.executable-bits.json b/tests/fixtures/nvattest/libnvat-linux-aarch64-1.2.2-sol.2-archive.executable-bits.json new file mode 100644 index 000000000..b65a5b7d4 --- /dev/null +++ b/tests/fixtures/nvattest/libnvat-linux-aarch64-1.2.2-sol.2-archive.executable-bits.json @@ -0,0 +1,14 @@ +{ + "archive_sha256": "7a13a15192f5005a700dbe154da28cbc2a2b5f3f387113c8cd89a36083a2bd80", + "executable": { + "LICENSE": false, + "bin/nvattest": true, + "lib/libnvat.so": false, + "lib/libnvat.so.1": false, + "lib/libnvat.so.1.2.2": true, + "share/THIRD_PARTY_NOTICES.md": false, + "share/ca/ca-bundle.pem": false + }, + "schema_version": 1, + "target": "linux-aarch64" +} diff --git a/tests/fixtures/nvattest/libnvat-linux-x86_64-1.2.2-sol.2-archive.executable-bits.json b/tests/fixtures/nvattest/libnvat-linux-x86_64-1.2.2-sol.2-archive.executable-bits.json new file mode 100644 index 000000000..05e763cbb --- /dev/null +++ b/tests/fixtures/nvattest/libnvat-linux-x86_64-1.2.2-sol.2-archive.executable-bits.json @@ -0,0 +1,14 @@ +{ + "archive_sha256": "3e2d207a3bb6eab9c47fc9cf65d7990f0d11541ff8691dce18e786e2ce9b26c7", + "executable": { + "LICENSE": false, + "bin/nvattest": true, + "lib/libnvat.so": false, + "lib/libnvat.so.1": false, + "lib/libnvat.so.1.2.2": true, + "share/THIRD_PARTY_NOTICES.md": false, + "share/ca/ca-bundle.pem": false + }, + "schema_version": 1, + "target": "linux-x86_64" +} diff --git a/tests/fixtures/nvattest/libnvat-macos-arm64-1.2.2-sol.2-archive.executable-bits.json b/tests/fixtures/nvattest/libnvat-macos-arm64-1.2.2-sol.2-archive.executable-bits.json new file mode 100644 index 000000000..ef3f4ea01 --- /dev/null +++ b/tests/fixtures/nvattest/libnvat-macos-arm64-1.2.2-sol.2-archive.executable-bits.json @@ -0,0 +1,14 @@ +{ + "archive_sha256": "f2b01f60ee52f9c38c9e5e0f3cea6e8078a1efa165fad1f5c59994563120e365", + "executable": { + "LICENSE": false, + "bin/nvattest": true, + "lib/libnvat.1.2.2.dylib": true, + "lib/libnvat.1.dylib": false, + "lib/libnvat.dylib": false, + "share/THIRD_PARTY_NOTICES.md": false, + "share/ca/ca-bundle.pem": false + }, + "schema_version": 1, + "target": "macos-arm64" +} diff --git a/tests/test_nvattest_authority.py b/tests/test_nvattest_authority.py index 62f1bd66b..1e9454ceb 100644 --- a/tests/test_nvattest_authority.py +++ b/tests/test_nvattest_authority.py @@ -23,6 +23,13 @@ MIRROR_PATH = ( REPO_ROOT / "solstone" / "think" / "providers" / "nvattest_authority_v1.json" ) INSTALL_PATH = REPO_ROOT / "solstone" / "think" / "providers" / "nvattest_install.py" +PAYLOAD_FACTS_FIXTURE_DIR = REPO_ROOT / "tests" / "fixtures" / "nvattest" +PAYLOAD_FACTS_SCHEMA_VERSION = 1 +PAYLOAD_FACTS_REPAIR = ( + "correct solstone/think/providers/nvattest_authority.py and run " + "make nvattest-authority if the authority is wrong; otherwise run " + "make nvattest-payload-facts if the derived fixture is stale" +) LEGACY_NVIDIA_SHA256 = ( "3f10da6fca794b7e3025c6645447947ec8bc45bcfde5b5b1d23241c7115630db" ) @@ -86,6 +93,22 @@ def _assert_rejected(payload: dict) -> None: nvattest_authority.validate_authority_payload(payload) +def _payload_facts_fixture_path(artifact_name: str) -> Path: + return ( + PAYLOAD_FACTS_FIXTURE_DIR + / f"{artifact_name.removesuffix('.tar.xz')}.executable-bits.json" + ) + + +def _drift_failure(error: str, *, expected: str, actual: str) -> str: + return ( + f"ERROR: {error}\n" + f" expected: {expected}\n" + f" actual: {actual}\n" + f" repair command: {PAYLOAD_FACTS_REPAIR}" + ) + + def test_authority_payload_contains_exact_operator_literals() -> None: payload = nvattest_authority.authority_payload() assert payload["schema_version"] == 1 @@ -177,6 +200,60 @@ def test_authority_payload_contains_exact_inventories() -> None: assert payload["targets"]["macos-arm64"]["inventory"] == macos_inventory +def test_payload_executable_bit_fixtures_match_authority() -> None: + """Executable-bit fixtures cover only mode drift. + + Member set, kind, and link targets for the production archives are covered by + tests/test_release_nvattest_proof.py::test_production_companion_manifests_match_authority_and_validate. + """ + + payload = nvattest_authority.authority_payload() + for target_key, target in payload["targets"].items(): + artifact = target["artifact"] + fixture_path = _payload_facts_fixture_path(str(artifact["name"])) + fixture = json.loads(fixture_path.read_text(encoding="utf-8")) + inventory = target["inventory"] + expected_executable = { + str(member["relpath"]): member["executable"] for member in inventory + } + + assert fixture.get("schema_version") == PAYLOAD_FACTS_SCHEMA_VERSION, ( + _drift_failure( + f"nvattest executable-bit fixture schema drift for {target_key}", + expected=str(PAYLOAD_FACTS_SCHEMA_VERSION), + actual=repr(fixture.get("schema_version")), + ) + ) + assert fixture.get("target") == target_key, _drift_failure( + f"nvattest executable-bit fixture target drift for {target_key}", + expected=target_key, + actual=repr(fixture.get("target")), + ) + assert fixture.get("archive_sha256") == artifact["sha256"], _drift_failure( + f"nvattest executable-bit fixture archive drift for {target_key}", + expected=str(artifact["sha256"]), + actual=repr(fixture.get("archive_sha256")), + ) + executable = fixture.get("executable") + assert isinstance(executable, dict), _drift_failure( + f"nvattest executable-bit fixture payload is invalid for {target_key}", + expected="executable object", + actual=type(executable).__name__, + ) + assert set(executable) == set(expected_executable), _drift_failure( + f"nvattest executable-bit fixture member set drift for {target_key}", + expected=", ".join(sorted(expected_executable)), + actual=", ".join(sorted(executable)) or "", + ) + for relpath, expected in sorted(expected_executable.items()): + actual = executable[relpath] + assert actual == expected, _drift_failure( + f"nvattest executable bit drift for {target_key} {relpath}", + expected=repr(expected), + actual=repr(actual), + ) + + def test_authority_json_matches_constants() -> None: actual = json.loads(MIRROR_PATH.read_text(encoding="utf-8")) assert actual == nvattest_authority.authority_payload() -- 2.51.2