From b2566b18d9263fcf57f70fa19004af8edd476317 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 29 Jul 2026 16:05:41 -0600 Subject: [PATCH] test(speakers): prove the sklearn collapse by scan, differential, and install Extract the production wheel-content AST import scan into a reusable checker and put sklearn in forbidden_prefixes because ImportFrom records from sklearn.cluster as sklearn.cluster rather than the exact sklearn module. Add a standing unit test that proves the scan fires for both import sklearn and from sklearn.cluster import HDBSCAN, while keeping tests.speaker_oracle as a test-only prefix. Add production-path mode to the discovery differential so the helper labels can flow through discovery.py's request, payload, temp-dir, and response-validation path while direct-binary remains available. Record the selected mode in provenance and update the docs. Add the clean-venv install leg as integration coverage: it installs the local root and leaf together by direct URI so the published 1.0.19 root cannot be resolved, then drives the real speakers-analyze helper binary for discovery-cluster. Co-Authored-By: GPT-5 Codex --- docs/PORTING.md | 7 +- scripts/check_wheel_contents.py | 35 +++++ tests/test_check_wheel_contents.py | 44 +++---- ...eaker_discovery_clustering_differential.py | 16 +++ ...eakers_analyze_leaf_install_integration.py | 121 ++++++++++++++++++ ...eaker_discovery_clustering_differential.py | 60 +++++++-- 6 files changed, 247 insertions(+), 36 deletions(-) diff --git a/docs/PORTING.md b/docs/PORTING.md index 65ae31660..be5f8c15b 100644 --- a/docs/PORTING.md +++ b/docs/PORTING.md @@ -247,8 +247,11 @@ Python-to-port parity checks. `tests/verify_speaker_discovery_clustering_differential.py` runs the unknown-speaker discovery clustering differential, feeding an `.npz` embedding -matrix to sklearn and the native analyzer `discovery-cluster` subcommand while -separating noise-boundary flips from cluster-to-cluster structural moves. +matrix to sklearn and to the native analyzer. Its default `production-path` mode +drives the `solstone.apps.speakers.discovery` kernel invocation path with the +operator-supplied helper binary, while `direct-binary` keeps the lower-level +`discovery-cluster` request mode available. The report separates noise-boundary +flips from cluster-to-cluster structural moves. `tests/verify_speaker_verdict.py` consumes those recorded bundles without rerunning speaker models, adding decision-flip replay for clustering, diff --git a/scripts/check_wheel_contents.py b/scripts/check_wheel_contents.py index d752853f7..d6242391e 100644 --- a/scripts/check_wheel_contents.py +++ b/scripts/check_wheel_contents.py @@ -7,6 +7,7 @@ from __future__ import annotations import argparse +import ast import base64 import hashlib import re @@ -210,12 +211,46 @@ SPEAKERS_ANALYZE_RUNTIME_LINK_CONTRACTS: dict[ SPEAKERS_ANALYZE_FORBIDDEN_PROVIDER_RE = re.compile( r"providers_(?:cuda|tensorrt|shared)", re.IGNORECASE ) +FORBIDDEN_PRODUCTION_IMPORT_EXACT = { + "kaldi_native_fbank", + "solstone.observe.model_assets", + "solstone.observe.transcribe.diarize", + "solstone.observe.transcribe.overlap", + "solstone.observe.transcribe.speakers_analyze_seam", + "solstone.think.speakers_analyze_handshake", + "solstone.think.speakers_analyze_runtime", +} +FORBIDDEN_PRODUCTION_IMPORT_PREFIXES = ("tests.speaker_oracle", "sklearn") def _is_base_wheel(path: Path) -> bool: return bool(re.match(r"solstone-\d", path.name)) +def forbidden_production_imports( + root: Path, + *, + forbidden_exact: set[str] = FORBIDDEN_PRODUCTION_IMPORT_EXACT, + forbidden_prefixes: tuple[str, ...] = FORBIDDEN_PRODUCTION_IMPORT_PREFIXES, +) -> list[str]: + source_root = root / "solstone" if (root / "solstone").is_dir() else root + violations: list[str] = [] + for path in sorted(source_root.rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + module_names: list[str] = [] + if isinstance(node, ast.Import): + module_names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.module: + module_names = [node.module] + for module_name in module_names: + if module_name in forbidden_exact or module_name.startswith( + forbidden_prefixes + ): + violations.append(f"{path.relative_to(root)} imports {module_name}") + return violations + + def _is_models_wheel(path: Path) -> bool: return path.name.startswith("solstone_journal_models-") diff --git a/tests/test_check_wheel_contents.py b/tests/test_check_wheel_contents.py index c04a2c26e..a1edea51a 100644 --- a/tests/test_check_wheel_contents.py +++ b/tests/test_check_wheel_contents.py @@ -3,7 +3,6 @@ from __future__ import annotations -import ast import os import re import subprocess @@ -96,31 +95,24 @@ def test_script_runs_without_site_packages_from_outside_repo(tmp_path: Path) -> def test_production_imports_do_not_reach_deleted_speaker_plane_or_oracle() -> None: - forbidden_exact = { - "kaldi_native_fbank", - "solstone.observe.model_assets", - "solstone.observe.transcribe.diarize", - "solstone.observe.transcribe.overlap", - "solstone.observe.transcribe.speakers_analyze_seam", - "solstone.think.speakers_analyze_handshake", - "solstone.think.speakers_analyze_runtime", - } - forbidden_prefixes = ("tests.speaker_oracle",) - violations: list[str] = [] - for path in sorted((ROOT / "solstone").rglob("*.py")): - tree = ast.parse(path.read_text(encoding="utf-8")) - for node in ast.walk(tree): - module_names: list[str] = [] - if isinstance(node, ast.Import): - module_names = [alias.name for alias in node.names] - elif isinstance(node, ast.ImportFrom) and node.module: - module_names = [node.module] - for module_name in module_names: - if module_name in forbidden_exact or module_name.startswith( - forbidden_prefixes - ): - violations.append(f"{path.relative_to(ROOT)} imports {module_name}") - assert violations == [] + assert checker.forbidden_production_imports(ROOT) == [] + + +def test_forbidden_production_imports_catches_sklearn_prefixes(tmp_path: Path) -> None: + source_dir = tmp_path / "solstone" / "apps" / "speakers" + source_dir.mkdir(parents=True) + (source_dir / "direct.py").write_text("import sklearn\n", encoding="utf-8") + (source_dir / "from_import.py").write_text( + "from sklearn.cluster import HDBSCAN\n", + encoding="utf-8", + ) + + violations = checker.forbidden_production_imports(tmp_path) + + assert violations == [ + "solstone/apps/speakers/direct.py imports sklearn", + "solstone/apps/speakers/from_import.py imports sklearn.cluster", + ] def test_core_wheel_validator_accepts_static_manylinux_wheel(tmp_path: Path) -> None: diff --git a/tests/test_speaker_discovery_clustering_differential.py b/tests/test_speaker_discovery_clustering_differential.py index 9a5f33445..4f181e5ce 100644 --- a/tests/test_speaker_discovery_clustering_differential.py +++ b/tests/test_speaker_discovery_clustering_differential.py @@ -103,3 +103,19 @@ def test_rust_bin_is_required() -> None: harness.parse_args(["matrix.npz"]) assert exc.value.code == 2 + + +def test_mode_defaults_to_production_path() -> None: + args = harness.parse_args(["matrix.npz", "--rust-bin", "/tmp/helper"]) + + assert args.mode == harness.MODE_PRODUCTION_PATH + + +def test_provenance_records_native_mode() -> None: + report = harness._base_report( + matrix_path=Path("matrix.npz"), + rust_bin=Path("/tmp/helper"), + mode=harness.MODE_DIRECT_BINARY, + ) + + assert report["provenance"]["inputs"]["mode"] == harness.MODE_DIRECT_BINARY diff --git a/tests/test_speakers_analyze_leaf_install_integration.py b/tests/test_speakers_analyze_leaf_install_integration.py index 9ee015e80..b9943f22b 100644 --- a/tests/test_speakers_analyze_leaf_install_integration.py +++ b/tests/test_speakers_analyze_leaf_install_integration.py @@ -3,10 +3,12 @@ from __future__ import annotations +import json import subprocess import venv from pathlib import Path +import numpy as np import pytest from solstone.think.speakers_analyze_installation import ( @@ -75,3 +77,122 @@ def test_cpu_leaf_install_reaches_speakers_analyze_helper(tmp_path: Path) -> Non ) print(invariant.stdout, end="") assert invariant.returncode == 0, invariant.stderr or invariant.stdout + + +@pytest.mark.integration +@pytest.mark.timeout(900) +def test_clean_leaf_install_without_sklearn_runs_discovery_cluster( + tmp_path: Path, +) -> None: + if not runtime_has_speakers_analyze_wheel_coverage(): + pytest.skip("host is not covered by solstone-core-speakers-analyze wheels") + + dist_dir = tmp_path / "dist" + for package in ("solstone", "solstone-journal"): + build = subprocess.run( + [ + "uv", + "build", + "--package", + package, + "--wheel", + "--out-dir", + str(dist_dir), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + timeout=180, + ) + assert build.returncode == 0, build.stderr or build.stdout + + root_wheels = sorted( + path + for path in dist_dir.glob("solstone-*.whl") + if not path.name.startswith("solstone_journal") + ) + leaf_wheels = sorted(dist_dir.glob("solstone_journal-*.whl")) + assert len(root_wheels) == 1 + assert len(leaf_wheels) == 1 + root_requirement = f"solstone[journal-host] @ {root_wheels[0].resolve().as_uri()}" + leaf_requirement = f"solstone-journal @ {leaf_wheels[0].resolve().as_uri()}" + + env_root = tmp_path / "venv" + venv.EnvBuilder(with_pip=True, symlinks=False).create(env_root) + python = env_root / "bin" / "python" + install = subprocess.run( + [ + str(python), + "-m", + "pip", + "install", + "--find-links", + str(dist_dir), + root_requirement, + leaf_requirement, + ], + capture_output=True, + text=True, + check=False, + timeout=600, + ) + assert install.returncode == 0, install.stderr or install.stdout + + payload_path = tmp_path / "embeddings.f32le" + matrix = np.eye(5, 256, dtype=np.float32) + matrix.astype(" Path: return resolved -def _provenance(*, matrix_path: Path | None, rust_bin: Path | None) -> dict[str, Any]: +def _provenance( + *, + matrix_path: Path | None, + rust_bin: Path | None, + mode: str | None, +) -> dict[str, Any]: return { "generated_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), "harness": { @@ -85,19 +94,27 @@ def _provenance(*, matrix_path: Path | None, rust_bin: Path | None) -> dict[str, "inputs": { "matrix_path": str(matrix_path) if matrix_path is not None else None, "rust_bin": str(rust_bin) if rust_bin is not None else None, + "mode": mode, }, } def _base_report( - *, matrix_path: Path | None = None, rust_bin: Path | None = None + *, + matrix_path: Path | None = None, + rust_bin: Path | None = None, + mode: str | None = None, ) -> dict[str, Any]: return { "schema": REPORT_SCHEMA, "schema_version": SCHEMA_VERSION, "classification": HARNESS_ERROR, "failure": None, - "provenance": _provenance(matrix_path=matrix_path, rust_bin=rust_bin), + "provenance": _provenance( + matrix_path=matrix_path, + rust_bin=rust_bin, + mode=mode, + ), "parameters": { "min_cluster_size": MIN_CLUSTER_SIZE, "min_samples": MIN_SAMPLES, @@ -188,6 +205,15 @@ def run_rust( return labels +def run_rust_production_path(matrix: np.ndarray, rust_bin: Path) -> np.ndarray: + return discovery._cluster_discovery_embeddings_native( + matrix, + helper_locator=lambda: rust_bin, + helper_invoker=discovery._invoke_discovery_helper, + temp_dir_factory=discovery.create_discovery_cluster_temp_dir, + ) + + def _cluster_count(labels: Sequence[int]) -> int: return len({int(label) for label in labels if int(label) != NOISE}) @@ -349,21 +375,33 @@ def _classify( def compare_matrix( - matrix: np.ndarray, rust_bin: Path, *, temp_parent: Path | None = None + matrix: np.ndarray, + rust_bin: Path, + *, + mode: str, + temp_parent: Path | None = None, ) -> dict[str, Any]: sklearn_labels = run_sklearn(matrix) - rust_labels = run_rust(matrix, rust_bin, temp_parent=temp_parent) + if mode == MODE_DIRECT_BINARY: + rust_labels = run_rust(matrix, rust_bin, temp_parent=temp_parent) + elif mode == MODE_PRODUCTION_PATH: + rust_labels = run_rust_production_path(matrix, rust_bin) + else: + raise HarnessError(f"unsupported native mode: {mode}") return _compare_clustering(sklearn_labels, rust_labels, cols=int(matrix.shape[1])) -def compare_matrix_file(matrix_path: Path, rust_bin: Path) -> dict[str, Any]: - report = _base_report(matrix_path=matrix_path, rust_bin=rust_bin) +def compare_matrix_file( + matrix_path: Path, rust_bin: Path, *, mode: str +) -> dict[str, Any]: + report = _base_report(matrix_path=matrix_path, rust_bin=rust_bin, mode=mode) try: matrix = load_matrix(matrix_path) report.update( compare_matrix( matrix, rust_bin, + mode=mode, temp_parent=_temp_parent_for_matrix(matrix_path), ) ) @@ -386,6 +424,12 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument( "--rust-bin", required=True, help="Path to Rust analyzer binary" ) + parser.add_argument( + "--mode", + choices=MODES, + default=MODE_PRODUCTION_PATH, + help="Native comparison path", + ) parser.add_argument( "--report", help="JSON report destination outside the repository" ) @@ -401,7 +445,7 @@ def main(argv: list[str] | None = None) -> int: report_path = _refuse_repo_destination(requested_report_path) matrix_path = Path(args.matrix_path) rust_bin = Path(args.rust_bin) - report = compare_matrix_file(matrix_path, rust_bin) + report = compare_matrix_file(matrix_path, rust_bin, mode=args.mode) except Exception as exc: report = _base_report() report["failure"] = {"class": HARNESS_ERROR, "message": str(exc)} -- 2.51.2