diff --git a/scripts/release_archive_manifest.py b/scripts/release_archive_manifest.py index 837e6bc78..655ff9f2b 100644 --- a/scripts/release_archive_manifest.py +++ b/scripts/release_archive_manifest.py @@ -10,6 +10,7 @@ import hashlib import stat import tarfile import zipfile +import zlib from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path @@ -67,25 +68,25 @@ def archive_manifest( def assert_archives_semantically_identical( retained: Path, - candidate: Path, + published: Path, *, retained_label: str = "retained archive", - candidate_label: str = "candidate archive", + published_label: str = "published archive", ) -> ArchiveManifest: """Raise unless two on-disk archives have identical canonical manifests.""" retained_manifest = archive_manifest(retained, label=retained_label) - candidate_manifest = archive_manifest( - candidate, + published_manifest = archive_manifest( + published, limits=retained_manifest, - label=candidate_label, + label=published_label, ) - if candidate_manifest.rows != retained_manifest.rows: + if published_manifest.rows != retained_manifest.rows: raise DriverError( [ failure( "release archive canonical manifest mismatch", expected=_manifest_summary(retained_manifest, retained_label), - actual=_manifest_diff(retained_manifest, candidate_manifest), + actual=_manifest_diff(retained_manifest, published_manifest), repair="restore matching model archives or cut the next version", ) ] @@ -122,8 +123,15 @@ def _zip_manifest( ) sha256 = "" if kind == "file": - with archive.open(info) as member: - sha256 = _sha256_stream(member) + try: + with archive.open(info) as member: + sha256 = _sha256_stream(member) + except ( + NotImplementedError, + RuntimeError, + zlib.error, + ) as exc: + _raise_member_unreadable(info.filename, label=label, exc=exc) rows.append( ArchiveManifestRow( path=info.filename, @@ -156,6 +164,7 @@ def _tar_manifest( if info is None: break _check_member_count(len(rows) + 1, limits=limits, label=label) + _validate_member_path(info.name, seen=seen, label=label) kind = _tar_kind(info, label=label) row_size = int(info.size) total_uncompressed = _check_member_limits( @@ -166,16 +175,18 @@ def _tar_manifest( limits=limits, label=label, ) - _validate_member_path(info.name, seen=seen, label=label) sha256 = "" if kind == "file": - member = archive.extractfile(info) - if member is None: - _raise_unsupported_kind( - info.name, "missing regular-file data", label - ) - with member: - sha256 = _sha256_stream(member) + try: + member = archive.extractfile(info) + if member is None: + _raise_unsupported_kind( + info.name, "missing regular-file data", label + ) + with member: + sha256 = _sha256_stream(member) + except (tarfile.TarError, EOFError, OSError, ValueError) as exc: + _raise_member_unreadable(info.name, label=label, exc=exc) rows.append( ArchiveManifestRow( path=info.name, @@ -322,7 +333,7 @@ def _check_member_limits( [ failure( "release archive member is not in retained manifest", - expected="candidate archive members from the retained manifest", + expected="published archive members from the retained manifest", actual=f"{label} unexpected member {member_path!r}", repair="restore matching model archives or cut the next version", ) @@ -387,6 +398,21 @@ def _raise_unsupported_kind(path: str, kind: str, label: str) -> None: ) +def _raise_member_unreadable(path: str, *, label: str, exc: Exception) -> None: + raise DriverError( + [ + failure( + "release archive member could not be read", + expected=( + "readable, unencrypted archive member using standard compression" + ), + actual=f"{label} {path!r}: {type(exc).__name__}", + repair="rebuild the archive without encryption or unsupported compression", + ) + ] + ) from None + + def _raise_malformed(path: Path, *, label: str, exc: Exception) -> None: raise DriverError( [ @@ -420,7 +446,7 @@ def _manifest_diff(expected: ArchiveManifest, actual: ArchiveManifest) -> str: ) return ( f"missing={missing} extra={extra} changed={changed} " - f"actual={_manifest_summary(actual, 'candidate archive')}" + f"actual={_manifest_summary(actual, 'published archive')}" ) diff --git a/scripts/release_publish.py b/scripts/release_publish.py index 081adec86..a4c9bf476 100644 --- a/scripts/release_publish.py +++ b/scripts/release_publish.py @@ -1020,7 +1020,7 @@ def _verify_uploaded_index( [ failure( "release publish package index verification timed out", - expected="all publish-owned retained files visible with matching SHA-256", + expected="all train retained files visible with matching SHA-256", actual=_describe_index_matches(last_matches), repair=config.resume_target, ) @@ -1086,15 +1086,15 @@ def _resolve_publish_set( if (match.project, match.version) != model_key ) train_state = _assert_pre_upload_index_clean_or_published(train_matches) - publish_uploads = tuple( + train_uploads = tuple( entry for entry in classified.uploads if not (entry.project == MODEL_PROJECT and entry.version == model_key[1]) ) return ( ResolvedPublishSet( - uploads=publish_uploads, - verify_expectations=_project_expectations_for(publish_uploads), + uploads=train_uploads, + verify_expectations=_project_expectations_for(train_uploads), reused_project=f"{model_match.project}=={model_match.version}", reused_files=model_names, reused_model_pin=ProjectExpectation( @@ -1167,7 +1167,7 @@ def _download_and_compare_reused_models( retained.path, destination, retained_label=f"retained {name}", - candidate_label=f"published {name}", + published_label=f"published {name}", ) diff --git a/tests/test_release_archive_manifest.py b/tests/test_release_archive_manifest.py index b7549be6c..460f023ca 100644 --- a/tests/test_release_archive_manifest.py +++ b/tests/test_release_archive_manifest.py @@ -67,6 +67,8 @@ def _patch_zip_central_directory( *, external_attr: int | None = None, create_system: int | None = None, + flag_bits: int | None = None, + compress_type: int | None = None, ) -> None: data = bytearray(path.read_bytes()) pos = 0 @@ -81,8 +83,18 @@ def _patch_zip_central_directory( if name == member: if create_system is not None: data[idx + 5] = create_system + if flag_bits is not None: + struct.pack_into(" None: + retained = tmp_path / "retained.whl" + published = tmp_path / "published.whl" + entries = [ + (_zip_dir_info("pkg", mode=0o755), b""), + (_zip_file_info("pkg/module.py", mode=0o664), b"VALUE = 1\n"), + ] + + _write_zip(retained, entries) + _write_zip(published, entries, reverse=True) + + manifest = assert_archives_semantically_identical(retained, published) + assert manifest.rows[0] == ArchiveManifestRow( + path="pkg/", + kind="dir", + mode=0o755, + size=0, + sha256="", + ) + + def test_duplicate_path_refuses(tmp_path: Path) -> None: archive = tmp_path / "duplicate.whl" info = _zip_file_info("pkg/file.txt") @@ -264,6 +297,34 @@ def test_zip_undecidable_permission_metadata_refuses( ) +def test_zip_encrypted_member_refuses_as_driver_error(tmp_path: Path) -> None: + archive = tmp_path / "encrypted.whl" + _write_zip(archive, [(_zip_file_info("pkg/file.txt"), b"content")]) + _patch_zip_central_directory(archive, "pkg/file.txt", flag_bits=1) + + with pytest.raises(DriverError) as excinfo: + archive_manifest(archive) + + assert _first_error(excinfo) == "release archive member could not be read" + assert "RuntimeError" in excinfo.value.failures[0].actual + assert "pkg/file.txt" in excinfo.value.failures[0].actual + + +def test_zip_unsupported_compression_refuses_as_driver_error( + tmp_path: Path, +) -> None: + archive = tmp_path / "unsupported-compression.whl" + _write_zip(archive, [(_zip_file_info("pkg/file.txt"), b"content")]) + _patch_zip_central_directory(archive, "pkg/file.txt", compress_type=99) + + with pytest.raises(DriverError) as excinfo: + archive_manifest(archive) + + assert _first_error(excinfo) == "release archive member could not be read" + assert "NotImplementedError" in excinfo.value.failures[0].actual + assert "pkg/file.txt" in excinfo.value.failures[0].actual + + def test_zip_symlink_refuses(tmp_path: Path) -> None: archive = tmp_path / "symlink.whl" _write_zip( @@ -297,6 +358,18 @@ def test_unsafe_member_path_refuses(tmp_path: Path, member_path: str) -> None: assert _first_error(excinfo) == "release archive member path is unsafe" +def test_tar_unsafe_path_refuses_before_limits(tmp_path: Path) -> None: + retained = tmp_path / "retained.tar.gz" + published = tmp_path / "published.tar.gz" + _write_tar(retained, [_tar_file_info("pkg/file.txt")], {"pkg/file.txt": b"a"}) + _write_tar(published, [_tar_file_info("/abs.txt")], {"/abs.txt": b"abcd"}) + + with pytest.raises(DriverError) as excinfo: + assert_archives_semantically_identical(retained, published) + + assert _first_error(excinfo) == "release archive member path is unsafe" + + @pytest.mark.parametrize( ("tar_type", "expected_kind"), (