From eeec3ff6d1c5a39277fc3c09b3e39e9b9a43e667 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Mon, 13 Jul 2026 03:07:48 -0600 Subject: [PATCH] fix(sync): accept processed as terminal reconcile proof When your journal finishes processing a segment, it intentionally consumes the raw file and no longer keeps it on disk. The reconcile predicate read that as missing, so the segment re-uploaded on every pass and its local copy was never released. Treat processed as terminal proof only after the same submitted_name/name match and exact SHA-256 equality that make present honest; a processed entry that does not match the local byte still needs upload and is never deletable. This is additive within protocol v2; OBSERVER_PROTOCOL_VERSION stays 2. The needs-upload gate and cleanup delete gate continue to share _segment_proven_held. Bump to 0.4.5. --- CHANGELOG.md | 5 ++ pyproject.toml | 2 +- src/solstone_linux/__init__.py | 2 +- src/solstone_linux/sync.py | 17 +++- tests/test_sync.py | 152 +++++++++++++++++++++++++++++++++ 5 files changed, 175 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a258750..df463b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to solstone-linux are documented here. The format is based on Keep a Changelog (https://keepachangelog.com/), and this project adheres to Semantic Versioning. +## [0.4.5] - 2026-07-13 + +### Fixed +- local cleanup now recognizes segments your journal has already finished processing. when your journal takes a segment in and finishes with it, it no longer keeps the original file on hand. sol used to read that as the file having gone missing, so it re-sent the segment and never released its local copy. sol now takes it as done — after confirming the file name and fingerprint still match, so nothing leaves your machine unconfirmed. + ## [0.4.4] - 2026-07-04 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 5833bcd..73ff0e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "solstone-linux" -version = "0.4.4" +version = "0.4.5" description = "Standalone Linux desktop observer for solstone" readme = "README.md" license = "AGPL-3.0-only" diff --git a/src/solstone_linux/__init__.py b/src/solstone_linux/__init__.py index cd54a9b..5f34035 100644 --- a/src/solstone_linux/__init__.py +++ b/src/solstone_linux/__init__.py @@ -3,4 +3,4 @@ """Standalone Linux desktop observer for solstone.""" -__version__ = "0.4.4" +__version__ = "0.4.5" diff --git a/src/solstone_linux/sync.py b/src/solstone_linux/sync.py index f664180..b5c2eaa 100644 --- a/src/solstone_linux/sync.py +++ b/src/solstone_linux/sync.py @@ -62,6 +62,13 @@ CONTACT_FLUSH_INTERVAL = 30 SERVER_KEY_FILENAME = ".server_key" +# Per-file statuses that prove reconcile convergence after name and SHA match. +# "processed" means the journal intentionally consumed the raw byte after +# verified processing and deliberately does not keep that raw file on journal +# disk; it makes the segment eligible for configured local cache cleanup, but +# does not mean the raw byte is still stored. +TERMINAL_HELD_STATUSES = ("present", "relocated", "processed") + def _eligible_files(segment_dir: Path) -> list[Path]: """Files eligible for upload and reconcile.""" @@ -82,6 +89,14 @@ def _sha256_file(path: Path) -> str | None: def _segment_proven_held(segment_dir: Path, entry: dict) -> bool: + """Return True only when every local file has terminal journal proof. + + Each local file must match a listing entry by submitted_name (else name), + carry a status in TERMINAL_HELD_STATUSES, and match its SHA-256 exactly. + All three are required: a terminal status alone is never proof for this byte. + Anything else -- absent entry, unknown status, hash mismatch, unreadable file + -- needs upload and is never deletable. + """ local_files = _eligible_files(segment_dir) if not local_files: return False @@ -96,7 +111,7 @@ def _segment_proven_held(segment_dir: Path, entry: dict) -> bool: remote_file = remote_by_name.get(local_file.name) if remote_file is None: return False - if remote_file.get("status") not in ("present", "relocated"): + if remote_file.get("status") not in TERMINAL_HELD_STATUSES: return False local_sha = _sha256_file(local_file) if local_sha is None or local_sha != remote_file.get("sha256"): diff --git a/tests/test_sync.py b/tests/test_sync.py index d806fa0..64c127a 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1049,6 +1049,100 @@ class TestReconcilePredicate: assert _segment_proven_held(segment, item) + @pytest.mark.asyncio + async def test_processed_status_sha_match_skips_and_cleanup_deletes( + self, tmp_path: Path + ): + sync = self._make_sync(tmp_path) + day = "20260101" + segment = self._create_segment( + sync._config.captures_dir, day, "archon", "120000_300" + ) + item = _server_item("120000_300", segment, status="processed") + sync._client.get_server_segments = MagicMock( + side_effect=self._query_for(day, [item]) + ) + + assert _segment_proven_held(segment, item) + + with patch.object(sync, "_upload_segment", new_callable=AsyncMock) as upload: + await sync._sync() + + upload.assert_not_called() + assert not segment.exists() + + @pytest.mark.asyncio + async def test_processed_status_with_mismatched_sha_uploads_and_cleanup_keeps( + self, tmp_path: Path + ): + sync = self._make_sync(tmp_path) + day = "20260101" + segment = self._create_segment( + sync._config.captures_dir, day, "archon", "120000_300" + ) + item = _server_item( + "120000_300", + segment, + files=[ + _server_file( + segment / "screen.webm", status="processed", sha256="0" * 64 + ) + ], + ) + sync._client.get_server_segments = MagicMock( + side_effect=self._query_for(day, [item]) + ) + + assert not _segment_proven_held(segment, item) + + with patch.object( + sync, "_upload_segment", new_callable=AsyncMock, return_value=True + ) as upload: + await sync._sync() + + upload.assert_called_once() + sync._synced_days.add(day) + await sync._cleanup_synced_segments() + + assert segment.exists() + + @pytest.mark.asyncio + async def test_processed_status_with_mismatched_name_uploads_and_cleanup_keeps( + self, tmp_path: Path + ): + sync = self._make_sync(tmp_path) + day = "20260101" + segment = self._create_segment( + sync._config.captures_dir, day, "archon", "120000_300" + ) + item = _server_item( + "120000_300", + segment, + files=[ + _server_file( + segment / "screen.webm", + name="something-else.webm", + status="processed", + ) + ], + ) + sync._client.get_server_segments = MagicMock( + side_effect=self._query_for(day, [item]) + ) + + assert not _segment_proven_held(segment, item) + + with patch.object( + sync, "_upload_segment", new_callable=AsyncMock, return_value=True + ) as upload: + await sync._sync() + + upload.assert_called_once() + sync._synced_days.add(day) + await sync._cleanup_synced_segments() + + assert segment.exists() + @pytest.mark.asyncio async def test_missing_status_uploads_and_cleanup_keeps(self, tmp_path: Path): sync = self._make_sync(tmp_path) @@ -1061,6 +1155,33 @@ class TestReconcilePredicate: side_effect=self._query_for(day, [item]) ) + assert not _segment_proven_held(segment, item) + + with patch.object( + sync, "_upload_segment", new_callable=AsyncMock, return_value=True + ) as upload: + await sync._sync() + + upload.assert_called_once() + sync._synced_days.add(day) + await sync._cleanup_synced_segments() + + assert segment.exists() + + @pytest.mark.asyncio + async def test_unknown_status_uploads_and_cleanup_keeps(self, tmp_path: Path): + sync = self._make_sync(tmp_path) + day = "20260101" + segment = self._create_segment( + sync._config.captures_dir, day, "archon", "120000_300" + ) + item = _server_item("120000_300", segment, status="unknown") + sync._client.get_server_segments = MagicMock( + side_effect=self._query_for(day, [item]) + ) + + assert not _segment_proven_held(segment, item) + with patch.object( sync, "_upload_segment", new_callable=AsyncMock, return_value=True ) as upload: @@ -1092,6 +1213,37 @@ class TestReconcilePredicate: upload.assert_not_called() assert not segment.exists() + @pytest.mark.asyncio + async def test_mixed_present_and_processed_files_skip_and_cleanup_deletes( + self, tmp_path: Path + ): + sync = self._make_sync(tmp_path) + day = "20260101" + segment = self._create_segment( + sync._config.captures_dir, day, "archon", "120000_300" + ) + audio = segment / "audio.flac" + audio.write_bytes(b"audio") + item = _server_item( + "120000_300", + segment, + files=[ + _server_file(segment / "screen.webm", status="present"), + _server_file(audio, status="processed"), + ], + ) + sync._client.get_server_segments = MagicMock( + side_effect=self._query_for(day, [item]) + ) + + assert _segment_proven_held(segment, item) + + with patch.object(sync, "_upload_segment", new_callable=AsyncMock) as upload: + await sync._sync() + + upload.assert_not_called() + assert not segment.exists() + @pytest.mark.asyncio async def test_unreadable_sha_cleanup_keeps(self, tmp_path: Path): sync = self._make_sync(tmp_path) -- 2.51.2