diff --git a/solstone/observe/transfer.py b/solstone/observe/transfer.py index e4ea517b9..52b5e68de 100644 --- a/solstone/observe/transfer.py +++ b/solstone/observe/transfer.py @@ -339,13 +339,18 @@ def import_archive( imported = [] with tarfile.open(archive_path, "r:gz") as tar: members = tar.getmembers() - plan: list[tuple[str, str, Path, list[tuple[tarfile.TarInfo, Path]]]] = [] + plan: list[tuple[str, str, Path, list[tuple[tarfile.TarInfo, Path, str]]]] = [] for original_arc_key, target_arc_key in validation["import_as"].items(): _reject_if_unsafe(target_arc_key, "segment key") target_dir = contained_path(journal, f"{day}/{target_arc_key}") + manifest_by_name = { + f["name"]: f["sha256"] + for f in manifest["segments"][original_arc_key].get("files", []) + } planned_files = [] + planned_names = set() prefix = f"{original_arc_key}/" for member in members: if member.name.startswith(prefix) and member.isfile(): @@ -357,18 +362,34 @@ def import_archive( f"{target_arc_key!r}: {member.name!r}" ) _reject_if_unsafe(filename, "member filename") + if filename not in manifest_by_name: + raise ValueError( + f"Archive member for segment {original_arc_key!r} " + f"has no manifest hash entry: {filename!r}" + ) target_path = contained_path( journal, f"{day}/{target_arc_key}/{filename}", ) - planned_files.append((member, target_path)) + planned_files.append( + (member, target_path, manifest_by_name[filename]) + ) + planned_names.add(filename) + + missing = set(manifest_by_name) - planned_names + if missing: + missing_list = ", ".join(sorted(missing)) + raise ValueError( + f"Archive missing manifest-listed file(s) for segment " + f"{original_arc_key!r}: {missing_list}" + ) plan.append((original_arc_key, target_arc_key, target_dir, planned_files)) for original_arc_key, target_arc_key, target_dir, planned_files in plan: target_dir.mkdir(parents=True, exist_ok=True) - for member, target_path in planned_files: + for member, target_path, expected_sha256 in planned_files: source = tar.extractfile(member) if source: temp_path = None @@ -385,6 +406,13 @@ def import_archive( temp_path = Path(temp_handle.name) shutil.copyfileobj(source, temp_handle) temp_handle.close() + actual_sha256 = compute_file_sha256(temp_path) + if actual_sha256 != expected_sha256: + raise ValueError( + f"Archive content mismatch for {target_path}: " + f"manifest sha256 {expected_sha256} != extracted " + f"{actual_sha256}" + ) install_file(temp_path, target_path) promoted = True # Preserve modification time (install_file does not) diff --git a/tests/test_transfer.py b/tests/test_transfer.py index 84a0231eb..ba1a9b1d2 100644 --- a/tests/test_transfer.py +++ b/tests/test_transfer.py @@ -504,6 +504,98 @@ class TestTransferImport: assert first_path.read_bytes() == b"first content" assert list(day_dir.rglob("*.tmp")) == [] + def test_import_archive_rejects_member_sha256_mismatch(self, tmp_path, monkeypatch): + """Test manifest/member hash mismatch aborts before promoting that file.""" + import io + + from solstone.observe.transfer import import_archive + from solstone.observe.utils import compute_bytes_sha256 + + archive_path = tmp_path / "sha-mismatch.tgz" + synced_content = b"already synced" + good_content = b"good import" + bad_manifest_content = b"manifest bytes" + bad_member_content = b"tampered bytes" + manifest = { + "version": 1, + "day": "20250101", + "created_at": 1704067200000, + "host": "test-host", + "segments": { + "120000_300": { + "files": [ + { + "name": "audio.flac", + "sha256": compute_bytes_sha256(synced_content), + "size": len(synced_content), + } + ] + }, + "125000_300": { + "files": [ + { + "name": "audio.flac", + "sha256": compute_bytes_sha256(good_content), + "size": len(good_content), + } + ] + }, + "130000_300": { + "files": [ + { + "name": "audio.flac", + "sha256": compute_bytes_sha256(bad_manifest_content), + "size": len(bad_manifest_content), + } + ] + }, + }, + } + + with tarfile.open(archive_path, "w:gz") as tar: + for segment, content in ( + ("120000_300", synced_content), + ("125000_300", good_content), + ("130000_300", bad_member_content), + ): + info = tarfile.TarInfo(name=f"{segment}/audio.flac") + info.size = len(content) + tar.addfile(info, io.BytesIO(content)) + + manifest_json = json.dumps(manifest).encode() + manifest_info = tarfile.TarInfo(name="manifest.json") + manifest_info.size = len(manifest_json) + tar.addfile(manifest_info, io.BytesIO(manifest_json)) + + journal_path = tmp_path / "journal" + synced_path = ( + journal_path / "chronicle" / "20250101" / "120000_300" / "audio.flac" + ) + synced_path.parent.mkdir(parents=True) + synced_path.write_bytes(synced_content) + + monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal_path)) + + import solstone.think.utils as think_utils + + think_utils._journal_path_cache = None + + bad_target = ( + journal_path / "chronicle" / "20250101" / "130000_300" / "audio.flac" + ) + with pytest.raises(ValueError) as excinfo: + import_archive(archive_path) + + message = str(excinfo.value) + assert "Archive content mismatch" in message + assert str(bad_target) in message + assert not bad_target.exists() + assert ( + journal_path / "chronicle" / "20250101" / "125000_300" / "audio.flac" + ).read_bytes() == good_content + assert synced_path.read_bytes() == synced_content + assert list((journal_path / "chronicle").rglob("*.tmp")) == [] + def test_import_archive_routes_member_writes_through_install_file(self): """Test import_archive has no raw durable member write path.""" from solstone.observe import transfer