diff --git a/scripts/check_wheel_contents.py b/scripts/check_wheel_contents.py index a623829b0..9adb6c4a1 100644 --- a/scripts/check_wheel_contents.py +++ b/scripts/check_wheel_contents.py @@ -15,10 +15,15 @@ import tarfile import zipfile from pathlib import Path +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + from solstone.think.probe import ( SOLSTONE_CORE_PLATFORM_TAGS, + CorePlatform, current_solstone_core_platform, is_solstone_core_covered_platform, + normalize_solstone_core_machine, ) EXPECTED_MODEL_SHA256 = { @@ -67,6 +72,27 @@ def _core_wheel_tag(path: Path) -> str: return stem.split("-")[-1] +def _parse_core_platform(value: str) -> CorePlatform: + try: + system, machine = value.split("/", 1) + except ValueError as exc: + raise argparse.ArgumentTypeError( + f"expected platform as system/machine, got {value!r}" + ) from exc + platform_tuple: CorePlatform = ( + "darwin" if system == "darwin" else "linux", + normalize_solstone_core_machine(system, machine), + ) + if platform_tuple not in SOLSTONE_CORE_PLATFORM_TAGS: + supported = ", ".join( + f"{system}/{machine}" for system, machine in SOLSTONE_CORE_PLATFORM_TAGS + ) + raise argparse.ArgumentTypeError( + f"unsupported solstone-core platform {value!r}; supported: {supported}" + ) + return platform_tuple + + def _onnx_members(path: Path) -> list[str]: with zipfile.ZipFile(path) as wheel: return [name for name in wheel.namelist() if name.endswith(".onnx")] @@ -206,7 +232,13 @@ def check_core_sdist(path: Path) -> list[str]: return errors -def check_dist(dist_dir: Path, expected: dict[str, str], max_bytes: int) -> list[str]: +def check_dist( + dist_dir: Path, + expected: dict[str, str], + max_bytes: int, + *, + required_core_platforms: tuple[CorePlatform, ...] = (), +) -> list[str]: errors: list[str] = [] wheels = sorted(dist_dir.glob("*.whl")) base_wheels = [path for path in wheels if _is_base_wheel(path)] @@ -221,10 +253,21 @@ def check_dist(dist_dir: Path, expected: dict[str, str], max_bytes: int) -> list if not models_wheels: errors.append(f"{dist_dir}: no solstone_journal_models wheel found") system, machine = current_solstone_core_platform() - if is_solstone_core_covered_platform(system, machine) and not core_wheels: - errors.append( - f"{dist_dir}: no solstone_core wheel found for {system}/{machine}" + required_tags: dict[str, str] = {} + if is_solstone_core_covered_platform(system, machine): + required_tags[SOLSTONE_CORE_PLATFORM_TAGS[(system, machine)]] = ( + f"{system}/{machine}" + ) + for platform_tuple in required_core_platforms: + required_tags[SOLSTONE_CORE_PLATFORM_TAGS[platform_tuple]] = ( + f"{platform_tuple[0]}/{platform_tuple[1]}" ) + found_core_tags = {_core_wheel_tag(path) for path in core_wheels} + for tag, platform_name in sorted(required_tags.items()): + if tag not in found_core_tags: + errors.append( + f"{dist_dir}: no solstone_core wheel found for {platform_name} ({tag})" + ) if not core_sdists: errors.append(f"{dist_dir}: no solstone_core sdist found") @@ -242,10 +285,22 @@ def check_dist(dist_dir: Path, expected: dict[str, str], max_bytes: int) -> list def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() + parser.add_argument( + "--require-core-platform", + action="append", + default=[], + type=_parse_core_platform, + help="require a solstone-core wheel for system/machine, e.g. darwin/arm64", + ) parser.add_argument("dist_dir", type=Path) args = parser.parse_args(argv) - errors = check_dist(args.dist_dir, EXPECTED_MODEL_SHA256, MAX_BASE_WHEEL_BYTES) + errors = check_dist( + args.dist_dir, + EXPECTED_MODEL_SHA256, + MAX_BASE_WHEEL_BYTES, + required_core_platforms=tuple(args.require_core_platform), + ) if errors: print("ERROR: wheel content check failed", file=sys.stderr) for error in errors: diff --git a/scripts/release.sh b/scripts/release.sh index 02e90f95f..f16a5ec39 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -215,7 +215,7 @@ ssh "$PRO5E_HOST" "tmux-run hopper ~/projects/solstone 'set -e; \ echo "==> [3/5] rsyncing macOS wheel back" rsync -av --include='*macosx_14_0_arm64.whl' --exclude='*' \ "$PRO5E_HOST:projects/solstone/dist/" ./dist/ -python3 scripts/check_wheel_contents.py dist/ +python3 scripts/check_wheel_contents.py --require-core-platform darwin/arm64 dist/ echo echo "release artifacts:" diff --git a/scripts/repack_wheel_record.py b/scripts/repack_wheel_record.py index 8b81791ae..dcd1500af 100755 --- a/scripts/repack_wheel_record.py +++ b/scripts/repack_wheel_record.py @@ -26,6 +26,15 @@ def _relative_files(root: Path) -> list[Path]: return sorted(path for path in root.rglob("*") if path.is_file()) +def _original_file_attrs(wheel_path: Path) -> dict[str, tuple[int, int]]: + with zipfile.ZipFile(wheel_path) as wheel: + return { + info.filename: (info.external_attr, info.create_system) + for info in wheel.infolist() + if not info.is_dir() + } + + def _rewrite_record(root: Path) -> None: record_paths = list(root.glob("*.dist-info/RECORD")) if len(record_paths) != 1: @@ -54,6 +63,7 @@ def repack(unpacked_dir: Path, wheel_path: Path) -> None: if not wheel_path.name.endswith(".whl"): raise SystemExit(f"wheel path must end in .whl: {wheel_path}") + original_attrs = _original_file_attrs(wheel_path) _rewrite_record(unpacked_dir) fd, tmp_name = tempfile.mkstemp( prefix=f".{wheel_path.name}.", suffix=".tmp", dir=str(wheel_path.parent) @@ -65,7 +75,15 @@ def repack(unpacked_dir: Path, wheel_path: Path) -> None: tmp_path, "w", compression=zipfile.ZIP_DEFLATED ) as archive: for path in _relative_files(unpacked_dir): - archive.write(path, path.relative_to(unpacked_dir).as_posix()) + arcname = path.relative_to(unpacked_dir).as_posix() + info = zipfile.ZipInfo(arcname) + info.compress_type = zipfile.ZIP_DEFLATED + if arcname in original_attrs: + info.external_attr, info.create_system = original_attrs[arcname] + else: + info.create_system = 3 + info.external_attr = (path.stat().st_mode & 0o777) << 16 + archive.writestr(info, path.read_bytes()) os.replace(tmp_path, wheel_path) finally: tmp_path.unlink(missing_ok=True) diff --git a/solstone/think/probe.py b/solstone/think/probe.py index 93a1aed1c..451d593ae 100644 --- a/solstone/think/probe.py +++ b/solstone/think/probe.py @@ -80,15 +80,27 @@ SOLSTONE_CORE_COVERED_PLATFORMS: tuple[CorePlatform, ...] = ( ("linux", "aarch64"), ("darwin", "arm64"), ) -SOLSTONE_CORE_PLATFORM_MARKERS: tuple[str, ...] = ( - "sys_platform == 'linux' and platform_machine == 'x86_64'", - "sys_platform == 'linux' and platform_machine == 'aarch64'", - "sys_platform == 'darwin' and platform_machine == 'arm64'", + + +def _solstone_core_platform_marker(platform_tuple: CorePlatform) -> str: + system, machine = platform_tuple + return f"sys_platform == '{system}' and platform_machine == '{machine}'" + + +def _solstone_core_platform_tag(platform_tuple: CorePlatform) -> str: + system, machine = platform_tuple + if system == "darwin": + return f"macosx_14_0_{machine}" + return f"manylinux_2_17_{machine}.manylinux2014_{machine}" + + +SOLSTONE_CORE_PLATFORM_MARKERS: tuple[str, ...] = tuple( + _solstone_core_platform_marker(platform_tuple) + for platform_tuple in SOLSTONE_CORE_COVERED_PLATFORMS ) SOLSTONE_CORE_PLATFORM_TAGS: dict[CorePlatform, str] = { - ("linux", "x86_64"): "manylinux_2_17_x86_64.manylinux2014_x86_64", - ("linux", "aarch64"): "manylinux_2_17_aarch64.manylinux2014_aarch64", - ("darwin", "arm64"): "macosx_14_0_arm64", + platform_tuple: _solstone_core_platform_tag(platform_tuple) + for platform_tuple in SOLSTONE_CORE_COVERED_PLATFORMS } diff --git a/tests/test_check_wheel_contents.py b/tests/test_check_wheel_contents.py index d6fdac841..37a227d27 100644 --- a/tests/test_check_wheel_contents.py +++ b/tests/test_check_wheel_contents.py @@ -5,6 +5,9 @@ from __future__ import annotations import base64 import hashlib +import os +import subprocess +import sys import tarfile import zipfile from io import BytesIO @@ -12,6 +15,8 @@ from pathlib import Path import scripts.check_wheel_contents as checker +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "check_wheel_contents.py" + def _record_hash(content: bytes) -> str: digest = hashlib.sha256(content).digest() @@ -60,6 +65,25 @@ def _write_core_wheel( return wheel_path +def test_script_runs_without_site_packages_from_outside_repo(tmp_path: Path) -> None: + env = os.environ.copy() + env.pop("PYTHONPATH", None) + env.pop("VIRTUAL_ENV", None) + + result = subprocess.run( + [sys.executable, "-S", "-E", str(SCRIPT), "--help"], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + timeout=15, + ) + + assert result.returncode == 0, result.stderr + assert "usage:" in result.stdout + + def test_core_wheel_validator_accepts_static_manylinux_wheel(tmp_path: Path) -> None: wheel = _write_core_wheel(tmp_path) @@ -110,6 +134,53 @@ def _write_core_sdist(path: Path, *, missing: str | None = None) -> Path: return sdist +def _write_minimal_wheel(path: Path, name: str) -> Path: + wheel_path = path / f"{name}-1.2.3-py3-none-any.whl" + with zipfile.ZipFile(wheel_path, "w") as wheel: + _write_member( + wheel, + f"{name}-1.2.3.dist-info/METADATA", + f"Name: {name}\nVersion: 1.2.3\n".encode(), + ) + return wheel_path + + +def _write_minimal_dist(path: Path) -> None: + _write_minimal_wheel(path, "solstone") + _write_minimal_wheel(path, "solstone_journal_models") + _write_core_sdist(path) + + +def test_dist_check_requires_requested_core_platform(tmp_path: Path) -> None: + _write_minimal_dist(tmp_path) + _write_core_wheel(tmp_path) + + errors = checker.check_dist( + tmp_path, + {}, + checker.MAX_BASE_WHEEL_BYTES, + required_core_platforms=(("darwin", "arm64"),), + ) + + assert any("darwin/arm64" in error for error in errors) + assert any("macosx_14_0_arm64" in error for error in errors) + + +def test_dist_check_accepts_requested_core_platform(tmp_path: Path) -> None: + _write_minimal_dist(tmp_path) + _write_core_wheel(tmp_path) + _write_core_wheel(tmp_path, tag="macosx_14_0_arm64") + + errors = checker.check_dist( + tmp_path, + {}, + checker.MAX_BASE_WHEEL_BYTES, + required_core_platforms=(("darwin", "arm64"),), + ) + + assert not [error for error in errors if "darwin/arm64" in error] + + def test_core_sdist_validator_requires_rust_workspace_sources( tmp_path: Path, ) -> None: diff --git a/tests/test_render_packaging.py b/tests/test_render_packaging.py index 74806050f..d31afb431 100644 --- a/tests/test_render_packaging.py +++ b/tests/test_render_packaging.py @@ -12,6 +12,8 @@ from textwrap import dedent import pytest +from solstone.think.probe import SOLSTONE_CORE_PLATFORM_MARKERS + SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "render_packaging.py" SPEC = importlib.util.spec_from_file_location("render_packaging", SCRIPT) assert SPEC is not None @@ -45,6 +47,10 @@ def _write(path: Path, text: str) -> None: def _fixture_root(tmp_path: Path, *, root_version: str = "1.2.3") -> Path: + core_pins = "\n".join( + f' "solstone-core==0.0.1; {marker}",' + for marker in SOLSTONE_CORE_PLATFORM_MARKERS + ) _write( tmp_path / "pyproject.toml", f""" @@ -55,9 +61,7 @@ def _fixture_root(tmp_path: Path, *, root_version: str = "1.2.3") -> Path: [project.optional-dependencies] journal-host = [ "solstone-journal-models==1.0.0", - "solstone-core==0.0.1; sys_platform == 'linux' and platform_machine == 'x86_64'", - "solstone-core==0.0.1; sys_platform == 'linux' and platform_machine == 'aarch64'", - "solstone-core==0.0.1; sys_platform == 'darwin' and platform_machine == 'arm64'", + {core_pins} ] journal = ["solstone-journal-host==0.7.0"] journal-cuda = ["solstone-journal-host==0.7.0"] @@ -214,18 +218,8 @@ def test_render_updates_python_leaves_and_cargo_lockstep(tmp_path: Path) -> None assert 'version = "2.3.4"' in core_leaf assert "solstone[journal-host]==" not in core_leaf root_pyproject = rendered[root / "pyproject.toml"] - assert ( - "\"solstone-core==2.3.4; sys_platform == 'linux' and platform_machine == 'x86_64'\"" - in root_pyproject - ) - assert ( - "\"solstone-core==2.3.4; sys_platform == 'linux' and platform_machine == 'aarch64'\"" - in root_pyproject - ) - assert ( - "\"solstone-core==2.3.4; sys_platform == 'darwin' and platform_machine == 'arm64'\"" - in root_pyproject - ) + for marker in SOLSTONE_CORE_PLATFORM_MARKERS: + assert f'"solstone-core==2.3.4; {marker}"' in root_pyproject def test_check_reports_synthetic_packaging_drift( diff --git a/tests/test_repack_wheel_record.py b/tests/test_repack_wheel_record.py new file mode 100644 index 000000000..d8aee74e2 --- /dev/null +++ b/tests/test_repack_wheel_record.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +import base64 +import hashlib +import zipfile +from pathlib import Path + +import scripts.repack_wheel_record as repacker + + +def _record_hash(content: bytes) -> str: + digest = hashlib.sha256(content).digest() + encoded = base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") + return f"sha256={encoded}" + + +def _write_member( + wheel: zipfile.ZipFile, + name: str, + content: bytes, + *, + mode: int = 0o644, +) -> None: + info = zipfile.ZipInfo(name) + info.external_attr = mode << 16 + wheel.writestr(info, content) + + +def _write_core_wheel(path: Path) -> Path: + wheel_path = path / "solstone_core-1.2.3-py3-none-macosx_14_0_arm64.whl" + members = { + "solstone_core-1.2.3.dist-info/METADATA": b"Name: solstone-core\nVersion: 1.2.3\n", + "solstone_core-1.2.3.dist-info/WHEEL": b"Wheel-Version: 1.0\n", + "solstone_core-1.2.3.data/scripts/solstone-core": b"#!/bin/sh\necho before\n", + } + rows = [ + f"{name},{_record_hash(content)},{len(content)}" + for name, content in members.items() + ] + rows.append("solstone_core-1.2.3.dist-info/RECORD,,") + + with zipfile.ZipFile(wheel_path, "w") as wheel: + for name, content in members.items(): + mode = 0o755 if name.endswith("/solstone-core") else 0o644 + _write_member(wheel, name, content, mode=mode) + _write_member( + wheel, + "solstone_core-1.2.3.dist-info/RECORD", + "\n".join(rows).encode(), + ) + return wheel_path + + +def _record_rows(wheel: zipfile.ZipFile) -> dict[str, tuple[str, str]]: + record_name = next(name for name in wheel.namelist() if name.endswith("/RECORD")) + rows: dict[str, tuple[str, str]] = {} + for row in wheel.read(record_name).decode("utf-8").splitlines(): + member, hash_value, size = row.split(",") + rows[member] = (hash_value, size) + return rows + + +def test_repack_preserves_original_executable_mode_and_rewrites_record( + tmp_path: Path, +) -> None: + wheel_path = _write_core_wheel(tmp_path) + unpacked = tmp_path / "unpacked" + with zipfile.ZipFile(wheel_path) as wheel: + wheel.extractall(unpacked) + + binary = unpacked / "solstone_core-1.2.3.data" / "scripts" / "solstone-core" + assert (binary.stat().st_mode & 0o777) == 0o644 + signed_content = b"#!/bin/sh\necho signed\n" + binary.write_bytes(signed_content) + + repacker.repack(unpacked, wheel_path) + + with zipfile.ZipFile(wheel_path) as wheel: + info = wheel.getinfo( + "solstone_core-1.2.3.data/scripts/solstone-core", + ) + assert ((info.external_attr >> 16) & 0o777) == 0o755 + rows = _record_rows(wheel) + record_hash, record_size = rows[ + "solstone_core-1.2.3.data/scripts/solstone-core" + ] + assert record_hash == _record_hash(signed_content) + assert record_size == str(len(signed_content)) diff --git a/tests/test_solstone_core_platforms.py b/tests/test_solstone_core_platforms.py index 3c631353e..c41346d5f 100644 --- a/tests/test_solstone_core_platforms.py +++ b/tests/test_solstone_core_platforms.py @@ -3,6 +3,7 @@ from __future__ import annotations +import re import tomllib from pathlib import Path @@ -15,6 +16,9 @@ from solstone.think.probe import ( ) ROOT = Path(__file__).resolve().parents[1] +MARKER_PLATFORM_RE = re.compile( + r"sys_platform == '(?P[^']+)' and platform_machine == '(?P[^']+)'" +) def _core_pin_markers() -> list[str]: @@ -27,15 +31,24 @@ def _core_pin_markers() -> list[str]: ] +def _marker_platform_tuple(marker_text: str) -> tuple[str, str]: + match = MARKER_PLATFORM_RE.fullmatch(marker_text) + assert match is not None + return (match.group("system"), match.group("machine")) + + def test_core_pin_markers_match_probe_covered_platforms() -> None: marker_texts = _core_pin_markers() assert sorted(marker_texts) == sorted(SOLSTONE_CORE_PLATFORM_MARKERS) markers = [Marker(text) for text in marker_texts] - platform_tuples = [ - *SOLSTONE_CORE_COVERED_PLATFORMS, - ("linux", "riscv64"), - ("darwin", "x86_64"), - ] + platform_tuples = sorted( + { + *SOLSTONE_CORE_COVERED_PLATFORMS, + *(_marker_platform_tuple(text) for text in marker_texts), + ("linux", "riscv64"), + ("darwin", "x86_64"), + } + ) for system, machine in platform_tuples: marker_matches = [ diff --git a/tests/test_solstone_core_wheel_install.py b/tests/test_solstone_core_wheel_install.py index 2a0028a3f..da7c7ce1e 100644 --- a/tests/test_solstone_core_wheel_install.py +++ b/tests/test_solstone_core_wheel_install.py @@ -4,7 +4,6 @@ from __future__ import annotations import os -import platform import shutil import subprocess import sys @@ -16,31 +15,21 @@ ROOT = Path(__file__).resolve().parents[1] @pytest.mark.skipif( - sys.platform != "linux" or platform.machine() != "x86_64", - reason="local Linux wheel install test runs on linux/x86_64 only", + sys.platform != "linux", + reason="local Linux wheel install test runs on Linux only", ) def test_locally_built_linux_core_wheel_installs_and_runs( tmp_path: Path, ) -> None: if shutil.which("uv") is None: pytest.skip("uv is not installed") - if shutil.which("rustup") is None: - pytest.skip("rustup is not installed") - installed_targets = subprocess.run( - ["rustup", "target", "list", "--installed"], - cwd=ROOT, - capture_output=True, - text=True, - check=True, - ).stdout.splitlines() - if "x86_64-unknown-linux-musl" not in installed_targets: - pytest.skip("x86_64-unknown-linux-musl target is not installed") + if shutil.which("cargo") is None: + pytest.skip("cargo is not installed") dist_dir = tmp_path / "dist" env = os.environ.copy() - env["MATURIN_PEP517_ARGS"] = ( - "--compatibility manylinux2014 --target x86_64-unknown-linux-musl" - ) + env.pop("MATURIN_PEP517_ARGS", None) + env.pop("CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER", None) subprocess.run( [ "uv", @@ -56,7 +45,7 @@ def test_locally_built_linux_core_wheel_installs_and_runs( env=env, check=True, ) - wheels = sorted(dist_dir.glob("solstone_core-*manylinux2014_x86_64.whl")) + wheels = sorted(dist_dir.glob("solstone_core-*.whl")) assert len(wheels) == 1 venv = tmp_path / "venv" diff --git a/tests/test_supervisor_sync_gate.py b/tests/test_supervisor_sync_gate.py index 9fbefc400..bf7f1a9fd 100644 --- a/tests/test_supervisor_sync_gate.py +++ b/tests/test_supervisor_sync_gate.py @@ -168,6 +168,40 @@ def test_core_handshake_skew_exits_before_pending_tasks(tmp_path, monkeypatch, c assert "9.9.9" in log_text +def test_core_handshake_version_command_error_exits_78(tmp_path, monkeypatch, capsys): + _set_identity(monkeypatch) + venv_bin = tmp_path / "venv" / "bin" + venv_bin.mkdir(parents=True) + python = venv_bin / "python" + python.write_text("", encoding="utf-8") + helper = venv_bin / "solstone-core" + helper.write_text("#!/bin/sh\necho helper failed >&2\nexit 7\n", encoding="utf-8") + helper.chmod(0o755) + + mod = _load_supervisor(tmp_path, monkeypatch) + monkeypatch.setattr(mod.core_handshake.sys, "executable", str(python)) + monkeypatch.setattr(mod.core_handshake, "is_source_checkout", lambda: False) + monkeypatch.setattr( + mod.core_handshake, + "distribution_version", + lambda _name: "1.2.3", + ) + monkeypatch.setattr( + mod.core_handshake, + "current_solstone_core_platform", + lambda: ("linux", "x86_64"), + ) + + with pytest.raises(SystemExit) as exc: + mod.main() + + assert exc.value.code == mod.core_handshake.EX_CONFIG + captured = capsys.readouterr() + assert "--version exited 7" in captured.err + assert "1.2.3" in captured.err + assert "helper failed" in captured.err + + def test_mid_run_foreign_heartbeat_sets_shutdown_emits_event_returns( tmp_path, monkeypatch ):