diff --git a/Makefile b/Makefile index ff514a278..2246d4c6d 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ export TMPDIR := /var/tmp PYTEST_BASETEMP_INIT := BASETEMP=$$(mktemp -d /var/tmp/solstone-pytest-XXXXXX); trap 'rm -rf "$$BASETEMP"' EXIT INT TERM; PYTEST_BASETEMP_FLAG := --basetemp "$$BASETEMP" -.PHONY: install hopper-install uninstall test test-cov test-integration test-performance test-app test-only format format-check install-checks ci clean clean-install coverage watch versions update update-prices preflight pre-commit skills render-packaging check-rust-fmt check-rust-msrv check-rust-clippy check-rust-test check-rust-ios check-rust-deny check-release-advisory-liveness check-rust-release-manifest audit openapi check-openapi check-openapi-observer-client-contract contract check-contract journal-resolution-vectors check-journal-resolution-vectors sandbox-probe-contract check-sandbox-probe-contract build-native-sol-grammar-oracle check-native-sol-grammar-oracle build-native-sol-root-contract check-native-sol-root-contract check-core-sdist-compile-inputs build-native-sol-journal-host-commands check-native-sol-journal-host-commands build-journal-access-rejection-inventory check-journal-access-rejection-inventory check-native-sol-python-manifest build-native-sol-inventory check-native-sol-inventory check-native-sol-architecture check-native-sol-contract-routes check-native-sol-conformance check-native-sol-coverage check-native-sol-no-python-spawn check-native-sol-compat check-native-sol-docs-links check-removed-time-parser-ready dev all sandbox sandbox-stop install-models parakeet-helper parakeet-helper-clean wheel-macos wheel-macos-clean verify verify-api verify-schemathesis update-api-baselines eval-schemas service-logs check-layer-hygiene check-api-conventions check-journal-io-access check-journal-io-mechanic check-journal-config-owner check-call-http-only check-no-legacy-chat check-channel-adapter-scrub check-brain-health-cutover check-tools-http-only check-access-imports-clean check-convey-bind-imports-clean check-schema-bounds check-thin-base-install check-extras-consistency check-cogitate-prompts smoke-cogitate release release-test publish-release publish-release-test FORCE +.PHONY: install hopper-install uninstall test test-cov test-integration test-performance test-app test-only format format-check install-checks ci clean clean-install coverage watch versions update update-prices preflight pre-commit skills render-packaging check-rust-fmt check-rust-msrv check-rust-clippy check-rust-test check-rust-ios check-rust-deny check-release-advisory-liveness check-rust-release-manifest audit openapi check-openapi check-openapi-observer-client-contract contract check-contract journal-resolution-vectors check-journal-resolution-vectors sandbox-probe-contract check-sandbox-probe-contract build-native-sol-grammar-oracle check-native-sol-grammar-oracle build-native-sol-root-contract check-native-sol-root-contract check-core-sdist-compile-inputs build-native-sol-journal-host-commands check-native-sol-journal-host-commands build-journal-access-rejection-inventory check-journal-access-rejection-inventory check-native-sol-python-manifest build-native-sol-inventory check-native-sol-inventory check-native-sol-architecture check-native-sol-contract-routes check-native-sol-conformance check-native-sol-coverage check-native-sol-no-python-spawn check-native-sol-compat check-native-sol-docs-links check-removed-time-parser-ready dev all sandbox sandbox-stop install-models parakeet-helper parakeet-helper-clean wheel-speakers-analyze-linux wheel-speakers-analyze-linux-x86_64 wheel-speakers-analyze-linux-aarch64 wheel-macos wheel-macos-clean verify verify-api verify-schemathesis update-api-baselines eval-schemas service-logs check-layer-hygiene check-api-conventions check-journal-io-access check-journal-io-mechanic check-journal-config-owner check-call-http-only check-no-legacy-chat check-channel-adapter-scrub check-brain-health-cutover check-tools-http-only check-access-imports-clean check-convey-bind-imports-clean check-schema-bounds check-thin-base-install check-extras-consistency check-cogitate-prompts smoke-cogitate release release-test publish-release publish-release-test FORCE # Default target - install package in editable mode all: install @@ -28,6 +28,11 @@ RUST_MANIFEST := core/Cargo.toml IOS_TARGET := aarch64-apple-ios REQUIRE_CARGO := command -v cargo >/dev/null 2>&1 || { echo "cargo is required for Rust checks; install cargo and retry" >&2; exit 1; } REQUIRE_RUSTUP := command -v rustup >/dev/null 2>&1 || { echo "rustup is required for the iOS gate; install rustup and retry" >&2; exit 1; } +# Prep measured, rather than merely anticipated, that a host GNU cargo build of +# solstone-core-speakers-analyze reaches GLIBC_2.34. These zig-GNU maturin args +# are therefore the checked-in developer path for the helper's GLIBC_2.27 floor. +SPEAKERS_ANALYZE_LINUX_X86_64_MATURIN_ARGS := --locked --zig --compatibility manylinux_2_27 --auditwheel skip --target x86_64-unknown-linux-gnu +SPEAKERS_ANALYZE_LINUX_AARCH64_MATURIN_ARGS := --locked --zig --compatibility manylinux_2_27 --auditwheel skip --target aarch64-unknown-linux-gnu # Pick the GPU (CUDA) journal runtime only on x86_64 NVIDIA hosts. The # CUDA bundle resolves onnxruntime-gpu, which ships NO aarch64 wheel on PyPI, so # an aarch64 NVIDIA host (e.g. DGX Spark / GB10) that auto-selected `cuda` would @@ -160,6 +165,18 @@ preflight: render-packaging: python3 scripts/render_packaging.py +wheel-speakers-analyze-linux: wheel-speakers-analyze-linux-x86_64 + +wheel-speakers-analyze-linux-x86_64: + python3 scripts/stage_speakers_analyze_runtime.py --target linux-x86_64 + rm -f dist/solstone_core_speakers_analyze-*.whl + ORT_PREFER_DYNAMIC_LINK=true ORT_LIB_PATH="$(CURDIR)/target/speakers-analyze-runtime-link/linux-x86_64" MATURIN_PEP517_ARGS="$(SPEAKERS_ANALYZE_LINUX_X86_64_MATURIN_ARGS)" $(UV) build --package solstone-core-speakers-analyze --wheel + +wheel-speakers-analyze-linux-aarch64: + python3 scripts/stage_speakers_analyze_runtime.py --target linux-aarch64 + rm -f dist/solstone_core_speakers_analyze-*.whl + ORT_PREFER_DYNAMIC_LINK=true ORT_LIB_PATH="$(CURDIR)/target/speakers-analyze-runtime-link/linux-aarch64" MATURIN_PEP517_ARGS="$(SPEAKERS_ANALYZE_LINUX_AARCH64_MATURIN_ARGS)" $(UV) build --package solstone-core-speakers-analyze --wheel + check-rust-fmt: @$(REQUIRE_CARGO) cargo fmt --manifest-path $(RUST_MANIFEST) --all -- --check @@ -379,6 +396,30 @@ wheel-macos: parakeet-helper SOURCE_COMMIT=$$(git rev-parse HEAD); \ CORE_LOCK_SHA256=$$(shasum -a 256 core/Cargo.lock | awk '{print $$1}'); \ python3 -m scripts.record_macos_native_wheel --role core --wheel "$$CORE_MAC_WHEEL" --signing-facts "$$CORE_FACTS" --source-commit "$$SOURCE_COMMIT" --core-lock-sha256 "$$CORE_LOCK_SHA256" --out dist/macos-native-core.json + @echo "==> staging macosx_14_0_arm64 solstone-core-speakers-analyze runtime" + python3 scripts/stage_speakers_analyze_runtime.py --target macos-arm64 + @echo "==> building macosx_14_0_arm64 solstone-core-speakers-analyze wheel" + MACOSX_DEPLOYMENT_TARGET=14.0 MATURIN_PEP517_ARGS="--locked --target aarch64-apple-darwin" $(UV) build --package solstone-core-speakers-analyze --wheel + @echo "==> signing and notarizing solstone-core-speakers-analyze and bundled ONNX Runtime dylib" + @SPEAKERS_MAC_WHEEL=$$(ls dist/solstone_core_speakers_analyze-*-macosx_14_0_arm64.whl); \ + SPEAKERS_FACTS=$$(mktemp); \ + SPEAKERS_TMP=$$(mktemp -d); \ + trap 'rm -rf "$$SPEAKERS_TMP" "$$SPEAKERS_FACTS"' EXIT; \ + python3 -m zipfile -e "$$SPEAKERS_MAC_WHEEL" "$$SPEAKERS_TMP"; \ + SPEAKERS_BINARY=$$(find "$$SPEAKERS_TMP" -path "*.data/scripts/solstone-core-speakers-analyze" -type f -print -quit); \ + test -n "$$SPEAKERS_BINARY" || { echo "missing solstone-core-speakers-analyze binary in $$SPEAKERS_MAC_WHEEL" >&2; exit 1; }; \ + echo "==> signing and notarizing solstone-core-speakers-analyze"; \ + ./scripts/sign-and-notarize-helper.sh "$$SPEAKERS_BINARY" > "$$SPEAKERS_TMP/solstone-core-speakers-analyze.signing-facts.json"; \ + ONNXRUNTIME_DYLIB=$$(find "$$SPEAKERS_TMP" -path "*.data/data/lib/solstone-core-speakers-analyze/libonnxruntime.1.25.0.dylib" -type f -print -quit); \ + test -n "$$ONNXRUNTIME_DYLIB" || { echo "missing libonnxruntime.1.25.0.dylib in $$SPEAKERS_MAC_WHEEL" >&2; exit 1; }; \ + echo "==> signing and notarizing libonnxruntime.1.25.0.dylib"; \ + ./scripts/sign-and-notarize-helper.sh "$$ONNXRUNTIME_DYLIB" > "$$SPEAKERS_TMP/libonnxruntime.1.25.0.dylib.signing-facts.json"; \ + python3 -c 'import json, sys; from pathlib import Path; root = Path(sys.argv[1]); out = Path(sys.argv[2]); names = ("solstone-core-speakers-analyze", "libonnxruntime.1.25.0.dylib"); payload = {"members": {name: json.loads((root / f"{name}.signing-facts.json").read_text()) for name in names}}; out.write_text(json.dumps(payload, sort_keys=True) + "\n")' "$$SPEAKERS_TMP" "$$SPEAKERS_FACTS"; \ + python3 scripts/repack_wheel_record.py "$$SPEAKERS_TMP" "$$SPEAKERS_MAC_WHEEL"; \ + SOURCE_COMMIT=$$(git rev-parse HEAD); \ + CORE_LOCK_SHA256=$$(shasum -a 256 core/Cargo.lock | awk '{print $$1}'); \ + python3 -m scripts.record_macos_native_wheel --role speakers-analyze --wheel "$$SPEAKERS_MAC_WHEEL" --signing-facts "$$SPEAKERS_FACTS" --source-commit "$$SOURCE_COMMIT" --core-lock-sha256 "$$CORE_LOCK_SHA256" --out dist/macos-native-speakers-analyze.json; \ + rm -rf packages/solstone-core-speakers-analyze/wheel-data else wheel-macos: @echo "wheel-macos: only supported on Darwin/arm64 (got $(shell uname -s)/$(shell uname -m))" >&2 diff --git a/core/crates/solstone-core-speakers-analyze/build.rs b/core/crates/solstone-core-speakers-analyze/build.rs new file mode 100644 index 000000000..b4c7a8686 --- /dev/null +++ b/core/crates/solstone-core-speakers-analyze/build.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 sol pbc + +fn main() { + // GNU ld defaults to new dtags, so this emits DT_RUNPATH rather than old + // DT_RPATH. The dynamic loader searches RUNPATH after LD_LIBRARY_PATH, + // keeping scripts/resolve_onnxruntime_capi.py's dev resolver in control. + println!( + "cargo:rustc-link-arg-bin=solstone-core-speakers-analyze=-Wl,-rpath,$ORIGIN/../lib/solstone-core-speakers-analyze" + ); +} diff --git a/core/deny.toml b/core/deny.toml index 807b2c9a6..e85122ba6 100644 --- a/core/deny.toml +++ b/core/deny.toml @@ -25,6 +25,7 @@ deny = [{ name = "pyo3" }, { name = "pyo3-ffi" }, { name = "cpython" }] targets = [ "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-gnu", "aarch64-unknown-linux-musl", "aarch64-apple-darwin", "aarch64-apple-ios", diff --git a/packages/solstone-core-speakers-analyze/.gitignore b/packages/solstone-core-speakers-analyze/.gitignore new file mode 100644 index 000000000..115a6b621 --- /dev/null +++ b/packages/solstone-core-speakers-analyze/.gitignore @@ -0,0 +1,2 @@ +wheel-data/ +*.egg-info/ diff --git a/packages/solstone-core-speakers-analyze/pyproject.toml b/packages/solstone-core-speakers-analyze/pyproject.toml new file mode 100644 index 000000000..3d6b29baa --- /dev/null +++ b/packages/solstone-core-speakers-analyze/pyproject.toml @@ -0,0 +1,47 @@ +[build-system] +requires = ["maturin==1.14.1"] +build-backend = "maturin" + +[project] +name = "solstone-core-speakers-analyze" +version = "1.0.17" +description = "solstone speaker analysis helper binary with bundled CPU ONNX Runtime" +requires-python = ">=3.12" +license = {text = "AGPL-3.0-only"} + +[tool.maturin] +bindings = "bin" +manifest-path = "../../core/crates/solstone-core-speakers-analyze/Cargo.toml" +profile = "release" +strip = true +data = "wheel-data" + +# This package's build inputs live outside its directory (see manifest-path), +# and its wheel payload includes staged ONNX Runtime shared-library bytes. +# uv's default cache key for a path dependency is this pyproject.toml alone, so +# a Rust-only or runtime-staging change could leave `uv sync` reporting "no +# changes" and the environment kept serving the previously built binary. +# Declaring the real inputs makes uv rebuild when they move. `[tool.uv]` is +# never published, so wheel metadata is unaffected. Keep core/target/ out of +# the list: the build writes it, and keying on it would leave every sync +# permanently dirty. The `.rust-core-hash` stamp in the Makefile is the other +# half — it is what makes `make install` run a `uv sync` at all after a +# Rust-only change. +[tool.uv] +cache-keys = [ + { file = "pyproject.toml" }, + { file = "wheel-data/**" }, + { file = "../../scripts/stage_speakers_analyze_runtime.py" }, + { file = "../../core/Cargo.toml" }, + { file = "../../core/Cargo.lock" }, + { file = "../../core/crates/solstone-core-speakers/Cargo.toml" }, + { file = "../../core/crates/solstone-core-speakers/**/*.rs" }, + { file = "../../core/crates/solstone-core-speakers-onnx/Cargo.toml" }, + { file = "../../core/crates/solstone-core-speakers-onnx/**/*.rs" }, + { file = "../../core/crates/solstone-core-speakers-analyze/Cargo.toml" }, + # Link-time input: keep this explicit because the neighbouring `**/*.rs` + # glob is not guaranteed to match a root-level build.rs, and a stale build + # script can silently drop the wheel binary's $ORIGIN RUNPATH. + { file = "../../core/crates/solstone-core-speakers-analyze/build.rs" }, + { file = "../../core/crates/solstone-core-speakers-analyze/**/*.rs" }, +] diff --git a/pyproject.toml b/pyproject.toml index 9b66fc857..ca6a8f277 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -217,13 +217,14 @@ override-dependencies = [ ] [tool.uv.workspace] -members = ["packages/solstone-journal", "packages/solstone-journal-cuda", "packages/solstone-journal-models", "packages/solstone-core"] +members = ["packages/solstone-journal", "packages/solstone-journal-cuda", "packages/solstone-journal-models", "packages/solstone-core", "packages/solstone-core-speakers-analyze"] [tool.uv.sources] solstone-journal = {workspace = true} solstone-journal-cuda = {workspace = true} solstone-journal-models = {workspace = true} solstone-core = {workspace = true} +solstone-core-speakers-analyze = {workspace = true} # Development tools configuration [tool.ruff] diff --git a/scripts/check_extras_consistency.py b/scripts/check_extras_consistency.py index a435fd02a..3382fed32 100755 --- a/scripts/check_extras_consistency.py +++ b/scripts/check_extras_consistency.py @@ -33,9 +33,12 @@ The invariants are: 10. Each leaf has metadata-only setuptools config, a workspace source for `solstone`, the expected package name, and the root version. 11. uv workspace members/sources are exactly the two journal leaves plus - models plus core; `solstone-journal-host` is absent. + models plus core plus the speakers analyze helper; `solstone-journal-host` + is absent. 12. `[tool.uv].override-dependencies` contains both tombstone pins. 13. The Makefile no longer uses root journal extra spellings. + 14. The speakers analyze helper is a workspace-only maturin bin leaf with + staged wheel data and no Python dependency surface. """ import sys @@ -101,13 +104,29 @@ WORKSPACE_MEMBERS = [ "packages/solstone-journal-cuda", "packages/solstone-journal-models", "packages/solstone-core", + "packages/solstone-core-speakers-analyze", ] WORKSPACE_SOURCES = { "solstone-journal", "solstone-journal-cuda", "solstone-journal-models", "solstone-core", + "solstone-core-speakers-analyze", } +SPEAKERS_ANALYZE_CACHE_KEYS = [ + {"file": "pyproject.toml"}, + {"file": "wheel-data/**"}, + {"file": "../../scripts/stage_speakers_analyze_runtime.py"}, + {"file": "../../core/Cargo.toml"}, + {"file": "../../core/Cargo.lock"}, + {"file": "../../core/crates/solstone-core-speakers/Cargo.toml"}, + {"file": "../../core/crates/solstone-core-speakers/**/*.rs"}, + {"file": "../../core/crates/solstone-core-speakers-onnx/Cargo.toml"}, + {"file": "../../core/crates/solstone-core-speakers-onnx/**/*.rs"}, + {"file": "../../core/crates/solstone-core-speakers-analyze/Cargo.toml"}, + {"file": "../../core/crates/solstone-core-speakers-analyze/build.rs"}, + {"file": "../../core/crates/solstone-core-speakers-analyze/**/*.rs"}, +] def _names(reqs: list[str]) -> set[str]: @@ -281,6 +300,70 @@ def _check_core_leaf( errors.append("core leaf [tool.maturin].strip must be true") +def _check_speakers_analyze_leaf( + *, + data: dict, + root_version: str | None, + errors: list[str], +) -> None: + project = data.get("project", {}) + build_system = data.get("build-system", {}) + tool = data.get("tool", {}) + maturin = tool.get("maturin", {}) + uv = tool.get("uv", {}) + + if project.get("name") != "solstone-core-speakers-analyze": + errors.append( + "speakers analyze leaf [project].name must be " + "'solstone-core-speakers-analyze'" + ) + if project.get("version") != root_version: + errors.append( + "speakers analyze leaf [project].version must match root version " + f"{root_version}; found {project.get('version')!r}" + ) + if project.get("dependencies", []) != []: + errors.append("speakers analyze leaf must not define [project].dependencies") + if project.get("scripts", {}) != {}: + errors.append("speakers analyze leaf must not define [project.scripts]") + if build_system.get("build-backend") != "maturin": + errors.append( + "speakers analyze leaf [build-system].build-backend must be 'maturin'" + ) + requires = build_system.get("requires", []) + expected_requires = ["maturin==1.14.1"] + if requires != expected_requires: + errors.append( + "speakers analyze leaf [build-system].requires mismatch\n" + f" expected: {expected_requires!r}\n" + f" actual: {requires!r}\n" + " repair command: edit " + "packages/solstone-core-speakers-analyze/pyproject.toml to " + 'requires = ["maturin==1.14.1"]' + ) + if maturin.get("bindings") != "bin": + errors.append("speakers analyze leaf [tool.maturin].bindings must be 'bin'") + if ( + maturin.get("manifest-path") + != "../../core/crates/solstone-core-speakers-analyze/Cargo.toml" + ): + errors.append( + "speakers analyze leaf [tool.maturin].manifest-path must be " + "'../../core/crates/solstone-core-speakers-analyze/Cargo.toml'" + ) + if maturin.get("profile") != "release": + errors.append("speakers analyze leaf [tool.maturin].profile must be 'release'") + if maturin.get("strip") is not True: + errors.append("speakers analyze leaf [tool.maturin].strip must be true") + if maturin.get("data") != "wheel-data": + errors.append("speakers analyze leaf [tool.maturin].data must be 'wheel-data'") + if uv.get("cache-keys") != SPEAKERS_ANALYZE_CACHE_KEYS: + errors.append( + "speakers analyze leaf [tool.uv].cache-keys must match the " + "declared native build inputs" + ) + + def main(root: Path | None = None) -> int: root = Path(root) if root is not None else Path(__file__).resolve().parent.parent pyproject = root / "pyproject.toml" @@ -288,6 +371,9 @@ def main(root: Path | None = None) -> int: cuda_pyproject = root / "packages" / "solstone-journal-cuda" / "pyproject.toml" models_pyproject = root / "packages" / "solstone-journal-models" / "pyproject.toml" core_pyproject = root / "packages" / "solstone-core" / "pyproject.toml" + speakers_analyze_pyproject = ( + root / "packages" / "solstone-core-speakers-analyze" / "pyproject.toml" + ) makefile = root / "Makefile" errors: list[str] = [] @@ -296,6 +382,7 @@ def main(root: Path | None = None) -> int: cuda_data = _read_toml(cuda_pyproject, root, errors) models_data = _read_toml(models_pyproject, root, errors) core_data = _read_toml(core_pyproject, root, errors) + speakers_analyze_data = _read_toml(speakers_analyze_pyproject, root, errors) project = data.get("project", {}) root_version = project.get("version") @@ -428,6 +515,11 @@ def main(root: Path | None = None) -> int: errors=errors, ) _check_core_leaf(data=core_data, root_version=root_version, errors=errors) + _check_speakers_analyze_leaf( + data=speakers_analyze_data, + root_version=root_version, + errors=errors, + ) # 5. CPU leaf runtime split. missing_cpu_runtime = sorted(CPU_ONNXRUNTIME_DEPS - set(cpu_deps)) diff --git a/scripts/render_packaging.py b/scripts/render_packaging.py index 00665f5fd..c1fc18492 100644 --- a/scripts/render_packaging.py +++ b/scripts/render_packaging.py @@ -77,6 +77,10 @@ def _core_leaf_path(root: Path) -> Path: return root / "packages" / "solstone-core" / "pyproject.toml" +def _speakers_analyze_leaf_path(root: Path) -> Path: + return root / "packages" / "solstone-core-speakers-analyze" / "pyproject.toml" + + def _core_unsupported_tombstone_path(root: Path) -> Path: return ( root / "scripts" / "solstone-core-unsupported-platform-tombstone" / "setup.py" @@ -111,6 +115,21 @@ def _rewrite_core_leaf(text: str, version: str) -> str: return text +def _rewrite_speakers_analyze_leaf(text: str, version: str) -> str: + text, version_count = VERSION_RE.subn(f'version = "{version}"', text) + if version_count != 1: + raise PackagingRenderError( + "speakers analyze leaf pyproject must contain exactly one " + f"[project].version line; found {version_count}" + ) + if "solstone[journal-host]==" in text: + raise PackagingRenderError( + "speakers analyze leaf pyproject must not contain a " + "solstone[journal-host]== pin" + ) + return text + + def _rewrite_root_core_pins(text: str, version: str) -> str: expected = set(solstone_core_marker_pins(version)) seen_markers: list[str] = [] @@ -352,6 +371,9 @@ def render(root: Path = ROOT) -> dict[Path, str]: _core_leaf_path(root): _rewrite_core_leaf( _core_leaf_path(root).read_text(encoding="utf-8"), version ), + _speakers_analyze_leaf_path(root): _rewrite_speakers_analyze_leaf( + _speakers_analyze_leaf_path(root).read_text(encoding="utf-8"), version + ), _core_unsupported_tombstone_path(root): _rewrite_tombstone_setup( _core_unsupported_tombstone_path(root).read_text(encoding="utf-8"), version, diff --git a/scripts/stage_speakers_analyze_runtime.py b/scripts/stage_speakers_analyze_runtime.py new file mode 100644 index 000000000..9024d1d73 --- /dev/null +++ b/scripts/stage_speakers_analyze_runtime.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Stage CPU ONNX Runtime bytes for solstone-core-speakers-analyze wheels.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import shutil +import sys +import tempfile +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from zipfile import ZipFile + +ROOT = Path(__file__).resolve().parent.parent +PACKAGE_DIR = ROOT / "packages" / "solstone-core-speakers-analyze" +DEFAULT_CACHE_DIR = ROOT / "target" / "speakers-analyze-runtime-cache" +DEFAULT_LINK_ROOT = ROOT / "target" / "speakers-analyze-runtime-link" +DEFAULT_RECEIPT_ROOT = ROOT / "target" / "speakers-analyze-runtime-provenance" +RUNTIME_INSTALL_DIR = Path("data/lib/solstone-core-speakers-analyze") +NOTICE_INSTALL_DIR = Path("data/share/solstone-core-speakers-analyze/licenses") +LIB_MODE = 0o755 +NOTICE_MODE = 0o644 +FORBIDDEN_GPU_MEMBERS = ( + "onnxruntime/capi/libonnxruntime_providers_cuda.so", + "onnxruntime/capi/libonnxruntime_providers_tensorrt.so", +) + + +class StageError(RuntimeError): + """Raised when runtime staging cannot proceed safely.""" + + +@dataclass(frozen=True) +class NoticeSpec: + source_member: str + staged_name: str + sha256: str + + +@dataclass(frozen=True) +class TargetSpec: + key: str + wheel_url: str + wheel_sha256: str + runtime_member: str + runtime_sha256: str + runtime_staged_name: str + link_names: tuple[str, ...] + notices: tuple[NoticeSpec, ...] + + @property + def wheel_name(self) -> str: + return self.wheel_url.rsplit("/", 1)[-1] + + +COMMON_NOTICES = ( + NoticeSpec( + source_member="onnxruntime/LICENSE", + staged_name="onnxruntime-LICENSE.txt", + sha256="2f07c72751aed99790b8a4869cf2311df85a860b22ded05fa22803587a48922c", + ), + NoticeSpec( + source_member="onnxruntime/ThirdPartyNotices.txt", + staged_name="onnxruntime-ThirdPartyNotices.txt", + sha256="0e07b95f3a8d6230037707c5c4a2b554d12c4cb67369669ac255635528ffcee2", + ), +) + +TARGETS = { + "linux-x86_64": TargetSpec( + key="linux-x86_64", + wheel_url=( + "https://files.pythonhosted.org/packages/a9/1b/" + "d681878f227513917d8620e4ea504af5eb3313fc01f8aea7b19a976c65db/" + "onnxruntime-1.25.0-cp312-cp312-manylinux_2_27_x86_64." + "manylinux_2_28_x86_64.whl" + ), + wheel_sha256="be93baa694ef8e5831fcb7b542da21f502b122918b5b9612d9f02972e043ee01", + runtime_member="onnxruntime/capi/libonnxruntime.so.1.25.0", + runtime_sha256="6976c9c6b2db120e835a7091e2f4bd2308a76d3856a7181beb7e7a9b1e08f9e5", + runtime_staged_name="libonnxruntime.so.1", + link_names=( + "libonnxruntime.so.1.25.0", + "libonnxruntime.so.1", + "libonnxruntime.so", + ), + notices=COMMON_NOTICES, + ), + "linux-aarch64": TargetSpec( + key="linux-aarch64", + wheel_url=( + "https://files.pythonhosted.org/packages/5a/c6/" + "19c5bfbc60396791e975652f982bcff9ff4b27947c8e2bf0064ac5d5727b/" + "onnxruntime-1.25.0-cp312-cp312-manylinux_2_27_aarch64." + "manylinux_2_28_aarch64.whl" + ), + wheel_sha256="9c99238d20bfa80ac68c7b03c2c936d389189ae40997f78a30d151570d7e18bf", + runtime_member="onnxruntime/capi/libonnxruntime.so.1.25.0", + runtime_sha256="d47425026b2474e1deb0b8cf22f74cd943539af85873aa3fb8052862445beef3", + runtime_staged_name="libonnxruntime.so.1", + link_names=( + "libonnxruntime.so.1.25.0", + "libonnxruntime.so.1", + "libonnxruntime.so", + ), + notices=COMMON_NOTICES, + ), + "macos-arm64": TargetSpec( + key="macos-arm64", + wheel_url=( + "https://files.pythonhosted.org/packages/7a/69/" + "f98c6bda4c34ac382b70c36033a989ceffd1caf5afba47bd2ef26535850f/" + "onnxruntime-1.25.0-cp312-cp312-macosx_14_0_arm64.whl" + ), + wheel_sha256="8ecd3362de3fb496fb3e2d055a95d5acab611cf759a27609c6d99704c9d8f184", + runtime_member="onnxruntime/capi/libonnxruntime.1.25.0.dylib", + runtime_sha256="bafe7d3f3fa8e31195501e5694e73ef240708d5df039feb272b8d506d2783a74", + runtime_staged_name="libonnxruntime.1.25.0.dylib", + link_names=("libonnxruntime.1.25.0.dylib", "libonnxruntime.dylib"), + notices=COMMON_NOTICES, + ), +} + + +def _sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _fail(label: str, *, expected: str, actual: str, repair: str) -> None: + raise StageError( + f"{label}\n expected: {expected}\n actual: {actual}\n repair: {repair}" + ) + + +def _relative(path: Path) -> str: + try: + return str(path.relative_to(ROOT)) + except ValueError: + return str(path) + + +def _default_target() -> str: + system = platform.system() + machine = platform.machine().lower() + if system == "Darwin" and machine in {"arm64", "aarch64"}: + return "macos-arm64" + if system == "Linux" and machine in {"x86_64", "amd64"}: + return "linux-x86_64" + if system == "Linux" and machine in {"aarch64", "arm64"}: + return "linux-aarch64" + _fail( + "unsupported host for default speakers-analyze runtime target", + expected=", ".join(sorted(TARGETS)), + actual=f"{system}/{machine}", + repair="pass --target explicitly for a supported runtime target", + ) + raise AssertionError("unreachable") + + +def _assert_lock_contains(spec: TargetSpec) -> None: + lock_path = ROOT / "uv.lock" + try: + lock_text = lock_path.read_text(encoding="utf-8") + except FileNotFoundError as exc: + _fail( + "uv.lock is missing", + expected="uv.lock containing pinned onnxruntime wheel URLs", + actual="missing", + repair="restore uv.lock before staging runtime bytes", + ) + raise AssertionError("unreachable") from exc + + url_text = f'url = "{spec.wheel_url}"' + hash_text = f'hash = "sha256:{spec.wheel_sha256}"' + if url_text not in lock_text or hash_text not in lock_text: + _fail( + "pinned onnxruntime wheel is not present in uv.lock", + expected=f"{url_text} with {hash_text}", + actual="missing URL or hash", + repair=( + "restore the accepted uv.lock entry or update " + "scripts/stage_speakers_analyze_runtime.py through design review" + ), + ) + + +def _download(url: str, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + prefix=f"{dest.name}.", + suffix=".part", + dir=dest.parent, + delete=False, + ) as tmp: + tmp_path = Path(tmp.name) + try: + with urllib.request.urlopen(url) as response, tmp_path.open("wb") as handle: + shutil.copyfileobj(response, handle) + tmp_path.replace(dest) + except Exception: + tmp_path.unlink(missing_ok=True) + raise + + +def _ensure_wheel(spec: TargetSpec, cache_dir: Path, *, offline: bool) -> Path: + cache_path = cache_dir / spec.wheel_name + if cache_path.exists(): + actual = _sha256_file(cache_path) + if actual != spec.wheel_sha256: + _fail( + "cached onnxruntime wheel digest mismatch", + expected=spec.wheel_sha256, + actual=actual, + repair=f"delete {_relative(cache_path)} and rerun staging", + ) + return cache_path + + if offline: + _fail( + "onnxruntime wheel cache miss in offline mode", + expected=_relative(cache_path), + actual="missing", + repair="run staging once with network access or preseed the verified cache", + ) + + _download(spec.wheel_url, cache_path) + actual = _sha256_file(cache_path) + if actual != spec.wheel_sha256: + cache_path.unlink(missing_ok=True) + _fail( + "downloaded onnxruntime wheel digest mismatch", + expected=spec.wheel_sha256, + actual=actual, + repair="retry the download; if it persists, stop because the pinned bytes changed", + ) + return cache_path + + +def _zip_member(zip_file: ZipFile, member: str) -> bytes: + try: + return zip_file.read(member) + except KeyError: + _fail( + "onnxruntime wheel missing expected member", + expected=member, + actual="missing", + repair="restore the pinned onnxruntime wheel or update the target table through design review", + ) + raise AssertionError("unreachable") + + +def _write_file(path: Path, data: bytes, mode: int) -> dict[str, object]: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + path.chmod(mode) + return { + "path": _relative(path), + "sha256": _sha256_bytes(data), + "size": len(data), + } + + +def _stage_link_dir(spec: TargetSpec, link_dir: Path, runtime_data: bytes) -> None: + shutil.rmtree(link_dir, ignore_errors=True) + link_dir.mkdir(parents=True) + primary_name = spec.link_names[0] + primary = link_dir / primary_name + primary.write_bytes(runtime_data) + primary.chmod(LIB_MODE) + for name in spec.link_names[1:]: + link_path = link_dir / name + try: + link_path.symlink_to(primary_name) + except OSError: + shutil.copy2(primary, link_path) + + +def stage_runtime( + *, + spec: TargetSpec, + package_dir: Path, + cache_dir: Path, + link_root: Path, + receipt_path: Path, + offline: bool, +) -> dict[str, object]: + _assert_lock_contains(spec) + wheel_path = _ensure_wheel(spec, cache_dir, offline=offline) + wheel_digest = _sha256_file(wheel_path) + if wheel_digest != spec.wheel_sha256: + _fail( + "onnxruntime wheel digest mismatch before extraction", + expected=spec.wheel_sha256, + actual=wheel_digest, + repair=f"delete {_relative(wheel_path)} and rerun staging", + ) + + wheel_data_root = package_dir / "wheel-data" + shutil.rmtree(wheel_data_root, ignore_errors=True) + link_dir = link_root / spec.key + + with ZipFile(wheel_path) as zip_file: + names = set(zip_file.namelist()) + forbidden = sorted(name for name in names if name in FORBIDDEN_GPU_MEMBERS) + if forbidden: + _fail( + "onnxruntime CPU wheel contains forbidden GPU provider members", + expected="no CUDA or TensorRT provider libraries", + actual=", ".join(forbidden), + repair="use the pinned CPU onnxruntime wheel, never onnxruntime-gpu", + ) + + runtime_data = _zip_member(zip_file, spec.runtime_member) + runtime_sha = _sha256_bytes(runtime_data) + if runtime_sha != spec.runtime_sha256: + _fail( + "extracted onnxruntime library digest mismatch", + expected=spec.runtime_sha256, + actual=runtime_sha, + repair="restore the pinned wheel bytes or update the digest table through design review", + ) + + runtime_path = wheel_data_root / RUNTIME_INSTALL_DIR / spec.runtime_staged_name + runtime_record = _write_file(runtime_path, runtime_data, LIB_MODE) + _stage_link_dir(spec, link_dir, runtime_data) + + notice_records = [] + for notice in spec.notices: + notice_data = _zip_member(zip_file, notice.source_member) + notice_sha = _sha256_bytes(notice_data) + if notice_sha != notice.sha256: + _fail( + "extracted onnxruntime notice digest mismatch", + expected=f"{notice.source_member} sha256 {notice.sha256}", + actual=notice_sha, + repair="restore the pinned wheel bytes or update the notice digest through design review", + ) + notice_path = wheel_data_root / NOTICE_INSTALL_DIR / notice.staged_name + notice_record = _write_file(notice_path, notice_data, NOTICE_MODE) + notice_record["source_member"] = notice.source_member + notice_records.append(notice_record) + + receipt = { + "schema": "solstone.speakers-analyze-runtime-provenance.v1", + "target": spec.key, + "wheel": { + "url": spec.wheel_url, + "sha256": spec.wheel_sha256, + "path": _relative(wheel_path), + "size": wheel_path.stat().st_size, + }, + "runtime_library": { + "source_member": spec.runtime_member, + "sha256": spec.runtime_sha256, + **runtime_record, + }, + "notices": notice_records, + "forbidden_gpu_members_checked": list(FORBIDDEN_GPU_MEMBERS), + "wheel_data_root": _relative(wheel_data_root), + "link_dir": _relative(link_dir), + } + + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n") + return receipt + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Stage pinned CPU ONNX Runtime bytes for the speakers analyzer wheel." + ) + parser.add_argument( + "--target", + choices=sorted(TARGETS), + default=None, + help="runtime target to stage; defaults to the current supported host", + ) + parser.add_argument( + "--package-dir", + type=Path, + default=PACKAGE_DIR, + help="solstone-core-speakers-analyze package directory", + ) + parser.add_argument( + "--cache-dir", + type=Path, + default=DEFAULT_CACHE_DIR, + help="verified wheel cache directory", + ) + parser.add_argument( + "--link-root", + type=Path, + default=DEFAULT_LINK_ROOT, + help="root directory for target-specific build-time link helpers", + ) + parser.add_argument( + "--receipt", + type=Path, + default=None, + help="JSON provenance receipt path", + ) + parser.add_argument( + "--offline", + action="store_true", + help="require the pinned wheel to already be present in the cache", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + target = args.target or _default_target() + spec = TARGETS[target] + receipt_path = args.receipt or (DEFAULT_RECEIPT_ROOT / f"{target}.json") + try: + receipt = stage_runtime( + spec=spec, + package_dir=args.package_dir, + cache_dir=args.cache_dir, + link_root=args.link_root, + receipt_path=receipt_path, + offline=args.offline, + ) + except StageError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + except OSError as exc: + print(f"ERROR: staging filesystem operation failed: {exc}", file=sys.stderr) + return 1 + + print(json.dumps(receipt, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/solstone/think/probe.py b/solstone/think/probe.py index 5484f7ba0..e2f18905b 100644 --- a/solstone/think/probe.py +++ b/solstone/think/probe.py @@ -109,6 +109,29 @@ SOLSTONE_CORE_PLATFORM_TAGS: dict[CorePlatform, str] = { for platform_tuple in SOLSTONE_CORE_COVERED_PLATFORMS } +SOLSTONE_CORE_SPEAKERS_ANALYZE_COVERED_PLATFORMS: tuple[CorePlatform, ...] = ( + ("linux", "x86_64"), + ("linux", "aarch64"), + ("darwin", "arm64"), +) + + +def _solstone_core_speakers_analyze_platform_tag( + platform_tuple: CorePlatform, +) -> str: + system, machine = platform_tuple + if system == "darwin": + # Derived from the measured ONNX Runtime 1.25.0 dylib minos=14.0.0. + # Its match with solstone-core's literal tag is coincidental. + return f"macosx_14_0_{machine}" + return f"manylinux_2_27_{machine}" + + +SOLSTONE_CORE_SPEAKERS_ANALYZE_PLATFORM_TAGS: dict[CorePlatform, str] = { + platform_tuple: _solstone_core_speakers_analyze_platform_tag(platform_tuple) + for platform_tuple in SOLSTONE_CORE_SPEAKERS_ANALYZE_COVERED_PLATFORMS +} + def platform_tag() -> Platform: if sys.platform == "darwin": diff --git a/uv.lock b/uv.lock index 13456f911..8ff8bc5d3 100644 --- a/uv.lock +++ b/uv.lock @@ -26,6 +26,7 @@ resolution-markers = [ members = [ "solstone", "solstone-core", + "solstone-core-speakers-analyze", "solstone-journal", "solstone-journal-cuda", "solstone-journal-models", @@ -4423,6 +4424,11 @@ name = "solstone-core" version = "1.0.17" source = { editable = "packages/solstone-core" } +[[package]] +name = "solstone-core-speakers-analyze" +version = "1.0.17" +source = { editable = "packages/solstone-core-speakers-analyze" } + [[package]] name = "solstone-journal" version = "1.0.17"