From 42e675d5062bf86636b753d294907db00acb78b3 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Thu, 16 Jul 2026 16:33:21 -0600 Subject: [PATCH] test(backup): cover media offload restore Cover restore engine behavior for daily-key restic calls, _default stream layout, operated append-only sessions, BYO hosted seams, free-space refusal, no-op folding, partial restore sweeps, rollback, and callosum staying untouched. Pin the real restic contract that drove the design: restic restore : --target --include / may exit 0 while restoring zero files for a missing include, so tests assert missing files are caught by existence, size, and sha256 verification. Also cover the exact five offload routes, the three journal backup offload CLI verbs, last_restore state, device_total_bytes(), and resolve_segment_dir() default-stream behavior. --- solstone/apps/backup/tests/conftest.py | 2 + solstone/apps/backup/tests/test_routes.py | 409 +++++++++++++++++ tests/test_backup_cli.py | 131 ++++++ tests/test_backup_offload_state.py | 67 +++ tests/test_offload_measurement.py | 15 + tests/test_offload_restore.py | 534 ++++++++++++++++++++++ tests/test_think_utils.py | 38 ++ 7 files changed, 1196 insertions(+) create mode 100644 tests/test_offload_restore.py diff --git a/solstone/apps/backup/tests/conftest.py b/solstone/apps/backup/tests/conftest.py index af87ad577..26582771c 100644 --- a/solstone/apps/backup/tests/conftest.py +++ b/solstone/apps/backup/tests/conftest.py @@ -27,8 +27,10 @@ class Env: @pytest.fixture(autouse=True) def clear_backup_registry(): backup_routes._clear_registry() + backup_routes._clear_measurement_cache() yield backup_routes._clear_registry() + backup_routes._clear_measurement_cache() @pytest.fixture diff --git a/solstone/apps/backup/tests/test_routes.py b/solstone/apps/backup/tests/test_routes.py index fe3d841ac..69aeccf0d 100644 --- a/solstone/apps/backup/tests/test_routes.py +++ b/solstone/apps/backup/tests/test_routes.py @@ -12,6 +12,8 @@ from pathlib import Path from types import SimpleNamespace from unittest.mock import Mock +import pytest + from solstone.apps.backup import routes as backup_routes from solstone.apps.backup.copy import backup_copy_values from solstone.convey.config import DEFAULT_APP_ORDER @@ -170,6 +172,11 @@ def test_backup_spa_shell_workspace_and_route_resolution(backup_env) -> None: ("/app/backup/enable-hosted", "POST"), ("/app/backup/destination", "POST"), ("/app/backup/backup-now", "POST"), + ("/app/backup/offload/status", "GET"), + ("/app/backup/offload/config", "POST"), + ("/app/backup/offload/enable", "POST"), + ("/app/backup/offload/disable", "POST"), + ("/app/backup/offload/restore", "POST"), ("/app/backup/recovery-key/rotate", "POST"), ("/app/backup/retention", "POST"), ("/app/backup/teardown", "POST"), @@ -239,6 +246,408 @@ def test_retention_validation_errors_return_invalid_config_value(backup_env) -> assert response.get_json()["reason_code"] == "invalid_config_value" +def _offload_status_payload(enabled: bool = False) -> dict: + return { + "offload": { + "enabled": enabled, + "budget_bytes": 100, + "floor_bytes": 50, + }, + "last_offload": {"status": "ok"}, + "last_verification": {"status": "ok"}, + "last_restore": {"status": "no_op", "reason": "nothing_to_restore"}, + "device": {"free_bytes": 900, "total_bytes": 1000}, + "suggested_defaults": {"budget_bytes": 500, "floor_bytes": 100}, + "raw_media": {"total_bytes": 300, "total_files": 3}, + "backup_only": { + "total_bytes": 200, + "total_files": 2, + "total_segments": 1, + "total_days": 1, + "degraded": True, + "skipped_records": 4, + "unreadable_ledgers": ["health/offload/20260101.jsonl"], + }, + "days": [ + { + "day": "20260101", + "raw_media_bytes": 300, + "raw_media_files": 3, + "backup_only_bytes": 200, + "backup_only_files": 2, + "backup_only_segments": 1, + "degraded": True, + "skipped_records": 4, + "unreadable_ledgers": ["health/offload/20260101.jsonl"], + } + ], + } + + +def test_offload_status_route_returns_measurements_and_degraded_signal( + backup_env, + monkeypatch, +) -> None: + env = backup_env() + build_offload_status = Mock(return_value=_offload_status_payload(enabled=True)) + monkeypatch.setattr(backup_routes, "build_offload_status", build_offload_status) + + response = env.client.get("/app/backup/offload/status") + + assert response.status_code == 200 + data = response.get_json() + assert data["offload"] == { + "enabled": True, + "budget_bytes": 100, + "floor_bytes": 50, + } + assert data["last_offload"] == {"status": "ok"} + assert data["last_verification"] == {"status": "ok"} + assert data["last_restore"] == { + "status": "no_op", + "reason": "nothing_to_restore", + } + assert data["device"] == {"free_bytes": 900, "total_bytes": 1000} + assert data["suggested_defaults"] == {"budget_bytes": 500, "floor_bytes": 100} + assert data["backup_only"]["degraded"] is True + assert data["backup_only"]["skipped_records"] == 4 + assert data["days"][0]["degraded"] is True + assert data["operation"] is None + + +def test_offload_config_preserves_enabled_and_rejects_bad_values( + backup_env, + monkeypatch, +) -> None: + env = backup_env() + _write_config( + env, + { + "backup": { + "offload": { + "enabled": True, + "budget_bytes": 10, + "floor_bytes": 5, + } + } + }, + ) + monkeypatch.setattr( + backup_routes, + "build_offload_status", + Mock(return_value=_offload_status_payload(enabled=True)), + ) + + response = env.client.post( + "/app/backup/offload/config", + json={"budget_bytes": 100, "floor_bytes": 50}, + ) + invalid = env.client.post( + "/app/backup/offload/config", + json={"budget_bytes": 0, "floor_bytes": 50}, + ) + + assert response.status_code == 200 + assert invalid.status_code == 400 + assert invalid.get_json()["reason_code"] == "invalid_config_value" + assert json.loads(_config_path(env).read_text(encoding="utf-8"))["backup"][ + "offload" + ] == { + "enabled": True, + "budget_bytes": 100, + "floor_bytes": 50, + } + + +def test_offload_enable_gates_and_requests_first_verification( + backup_env, + monkeypatch, +) -> None: + env = backup_env() + request_verification_now = Mock(return_value=True) + monkeypatch.setattr( + backup_routes, + "build_offload_status", + Mock(return_value=_offload_status_payload(enabled=True)), + ) + monkeypatch.setattr( + backup_routes, + "request_verification_now", + request_verification_now, + ) + + _write_config( + env, + { + "backup": { + "enabled": False, + "confirmed_recovery_key": True, + "daily_key": "daily-secret", + "recovery_key": "A" * 64, + "offload": { + "enabled": False, + "budget_bytes": 10, + "floor_bytes": 5, + }, + } + }, + ) + disabled = env.client.post("/app/backup/offload/enable") + + _write_config( + env, + { + "backup": { + "enabled": True, + "confirmed_recovery_key": False, + "daily_key": "daily-secret", + "recovery_key": "A" * 64, + "offload": { + "enabled": False, + "budget_bytes": 10, + "floor_bytes": 5, + }, + } + }, + ) + unconfirmed = env.client.post("/app/backup/offload/enable") + + _write_config( + env, + { + "backup": { + "enabled": True, + "confirmed_recovery_key": True, + "daily_key": "daily-secret", + "recovery_key": "A" * 64, + "offload": { + "enabled": False, + "budget_bytes": 10, + "floor_bytes": 5, + }, + } + }, + ) + enabled = env.client.post("/app/backup/offload/enable") + + assert disabled.status_code == 400 + assert disabled.get_json()["reason_code"] == "invalid_operation_for_state" + assert unconfirmed.status_code == 400 + assert unconfirmed.get_json()["reason_code"] == "backup_not_confirmed" + assert enabled.status_code == 200 + assert json.loads(_config_path(env).read_text(encoding="utf-8"))["backup"][ + "offload" + ] == { + "enabled": True, + "budget_bytes": 10, + "floor_bytes": 5, + } + request_verification_now.assert_called_once_with() + + +def test_offload_enable_does_not_request_verification_after_first_run( + backup_env, + monkeypatch, +) -> None: + env = backup_env() + request_verification_now = Mock(return_value=True) + monkeypatch.setattr( + backup_routes, + "build_offload_status", + Mock(return_value=_offload_status_payload(enabled=True)), + ) + monkeypatch.setattr( + backup_routes, + "request_verification_now", + request_verification_now, + ) + _write_config( + env, + { + "backup": { + "enabled": True, + "confirmed_recovery_key": True, + "daily_key": "daily-secret", + "recovery_key": "A" * 64, + "last_verification": {"status": "ok"}, + "offload": { + "enabled": False, + "budget_bytes": 10, + "floor_bytes": 5, + }, + } + }, + ) + + response = env.client.post("/app/backup/offload/enable") + + assert response.status_code == 200 + request_verification_now.assert_not_called() + + +def test_offload_disable_preserves_config_and_does_not_touch_ledger_or_media( + backup_env, + monkeypatch, +) -> None: + env = backup_env() + ledger = env.journal / "health" / "offload" / "20260101.jsonl" + ledger.parent.mkdir(parents=True) + ledger.write_text('{"event_kind":"offload"}\n', encoding="utf-8") + media = env.journal / "chronicle" / "20260101" / "090000_300" / "audio.wav" + media.parent.mkdir(parents=True) + media.write_bytes(b"raw") + _write_config( + env, + { + "backup": { + "enabled": True, + "mode": "operated", + "daily_key": "daily-secret", + "recovery_key": "A" * 64, + "confirmed_recovery_key": True, + "custom_marker": {"keep": True}, + "offload": { + "enabled": True, + "budget_bytes": 10, + "floor_bytes": 5, + }, + } + }, + ) + monkeypatch.setattr( + backup_routes, + "build_offload_status", + Mock(return_value=_offload_status_payload(enabled=False)), + ) + + response = env.client.post("/app/backup/offload/disable") + + assert response.status_code == 200 + backup = json.loads(_config_path(env).read_text(encoding="utf-8"))["backup"] + assert backup["mode"] == "operated" + assert backup["daily_key"] == "daily-secret" + assert backup["custom_marker"] == {"keep": True} + assert backup["offload"] == { + "enabled": False, + "budget_bytes": 10, + "floor_bytes": 5, + } + assert ledger.read_text(encoding="utf-8") == '{"event_kind":"offload"}\n' + assert media.read_bytes() == b"raw" + + +def test_offload_restore_rejects_invalid_day_before_side_effects( + backup_env, + monkeypatch, +) -> None: + env = backup_env() + restore_day = Mock() + restore_all = Mock() + monkeypatch.setattr(backup_routes, "restore_offload_day", restore_day) + monkeypatch.setattr(backup_routes, "restore_offload_all", restore_all) + + response = env.client.post( + "/app/backup/offload/restore", + json={"day": "20260230"}, + ) + + assert response.status_code == 400 + assert response.get_json()["reason_code"] == "invalid_day" + restore_day.assert_not_called() + restore_all.assert_not_called() + assert not (env.journal / "chronicle" / "20260230").exists() + + +@pytest.mark.parametrize("offload_enabled", [True, False]) +def test_offload_restore_route_accepts_day_with_offload_enabled_or_disabled( + backup_env, + monkeypatch, + wait_until_helper, + offload_enabled: bool, +) -> None: + env = backup_env() + _write_config( + env, + { + "backup": { + "enabled": True, + "offload": { + "enabled": offload_enabled, + "budget_bytes": 10, + "floor_bytes": 5, + }, + } + }, + ) + restore_day = Mock(return_value=SimpleNamespace(status="ok", reason=None)) + monkeypatch.setattr(backup_routes, "restore_offload_day", restore_day) + + response = env.client.post( + "/app/backup/offload/restore", + json={"day": "20260228"}, + ) + final = _wait_for_phase(env, wait_until_helper, "done") + + assert response.status_code == 202 + assert final["reason_code"] is None + restore_day.assert_called_once_with("20260228") + + +def test_offload_restore_route_accepts_all_and_rejects_mixed_scope( + backup_env, + monkeypatch, + wait_until_helper, +) -> None: + env = backup_env() + restore_all = Mock( + return_value=SimpleNamespace(status="no_op", reason="nothing_to_restore") + ) + restore_day = Mock() + monkeypatch.setattr(backup_routes, "restore_offload_all", restore_all) + monkeypatch.setattr(backup_routes, "restore_offload_day", restore_day) + + response = env.client.post("/app/backup/offload/restore", json={"all": True}) + final = _wait_for_phase(env, wait_until_helper, "done") + backup_routes._clear_registry() + mixed = env.client.post( + "/app/backup/offload/restore", + json={"all": True, "day": "20260228"}, + ) + + assert response.status_code == 202 + assert final["reason_code"] == "nothing_to_restore" + restore_all.assert_called_once_with() + restore_day.assert_not_called() + assert mixed.status_code == 400 + assert mixed.get_json()["reason_code"] == "invalid_request_value" + + +def test_offload_restore_busy_path_uses_single_long_op_slot( + backup_env, + monkeypatch, + wait_until_helper, +) -> None: + env = backup_env() + started = threading.Event() + release = threading.Event() + + def slow_restore(_day: str) -> SimpleNamespace: + started.set() + release.wait(2) + return SimpleNamespace(status="ok", reason=None) + + monkeypatch.setattr(backup_routes, "restore_offload_day", slow_restore) + + first = env.client.post("/app/backup/offload/restore", json={"day": "20260228"}) + wait_until_helper(started.is_set) + second = env.client.post("/app/backup/offload/restore", json={"day": "20260228"}) + release.set() + + assert first.status_code == 202 + assert second.status_code == 503 + assert second.get_json()["reason_code"] == "backup_busy" + + def test_rotate_restore_and_teardown_routes_call_engine_hooks( backup_env, monkeypatch, diff --git a/tests/test_backup_cli.py b/tests/test_backup_cli.py index 20ccfd52e..597329ab7 100644 --- a/tests/test_backup_cli.py +++ b/tests/test_backup_cli.py @@ -21,6 +21,8 @@ from solstone.think.backup.restore import RestoreResult from solstone.think.backup.rotation import RotationResult from solstone.think.backup.state import BackupKeys from solstone.think.backup.teardown import TeardownResult +from solstone.think.offload import OffloadResult, OffloadSegmentDetail +from solstone.think.offload_restore import OffloadRestoreResult def _config_path(journal: Path) -> Path: @@ -110,6 +112,7 @@ def test_registry_and_command_tree(monkeypatch: pytest.MonkeyPatch) -> None: "prune", "status", "recovery-key", + "offload", "restore", "off", ): @@ -125,6 +128,12 @@ def test_registry_and_command_tree(monkeypatch: pytest.MonkeyPatch) -> None: assert "show" in recovery_help.output assert "rotate" in recovery_help.output + offload_help = runner.invoke(backup_cli.app, ["offload", "--help"]) + assert offload_help.exit_code == 0 + assert "status" in offload_help.output + assert "run" in offload_help.output + assert "restore" in offload_help.output + @pytest.mark.parametrize( ("reason_code", "expected_exit"), @@ -457,6 +466,128 @@ def test_prune_maps_engine_status( assert expected_text in invoke_result.output +def test_offload_status_json_delegates_to_status_builder( + monkeypatch: pytest.MonkeyPatch, +) -> None: + payload = { + "offload": {"enabled": True, "budget_bytes": 100, "floor_bytes": 50}, + "raw_media": {"total_bytes": 30, "total_files": 3}, + "backup_only": {"total_bytes": 20, "degraded": False}, + } + build_offload_status = Mock(return_value=payload) + monkeypatch.setattr(backup_cli, "build_offload_status", build_offload_status) + + result = CliRunner().invoke(backup_cli.app, ["offload", "status", "--json"]) + + assert result.exit_code == 0 + assert json.loads(result.output) == payload + build_offload_status.assert_called_once_with() + + +def test_offload_run_delegates_to_existing_pass_and_formats_like_maintenance( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, Any] = {} + + def fake_run_offload(*, dry_run: bool) -> OffloadResult: + captured["dry_run"] = dry_run + return OffloadResult( + status="ok", + reason=None, + files_offloaded=0, + bytes_offloaded=0, + ran_out_of_media=False, + dry_run=True, + details=( + OffloadSegmentDetail( + day="20260101", + stream="_default", + segment="090000_300", + files=2, + bytes=50, + ), + ), + ) + + monkeypatch.setattr(backup_cli, "run_offload", fake_run_offload) + + result = CliRunner().invoke(backup_cli.app, ["offload", "run", "--dry-run"]) + + assert result.exit_code == 0 + assert captured == {"dry_run": True} + assert ( + "backup offload: ok dry_run=true selected_files=2 selected_bytes=50 " + "ran_out_of_media=False segments=20260101/_default/090000_300:50" + in result.output + ) + + +def test_offload_restore_day_json_delegates_to_restore_engine( + monkeypatch: pytest.MonkeyPatch, +) -> None: + restore_day = Mock( + return_value=OffloadRestoreResult( + status="ok", + reason=None, + scope="day", + day="20260228", + segments_selected=1, + segments_restored=1, + files_expected=2, + files_restored=2, + bytes_expected=50, + bytes_restored=50, + details=(), + ) + ) + monkeypatch.setattr(backup_cli, "restore_offload_day", restore_day) + + result = CliRunner().invoke( + backup_cli.app, + ["offload", "restore", "20260228", "--json"], + ) + + assert result.exit_code == 0 + assert json.loads(result.output)["day"] == "20260228" + restore_day.assert_called_once_with("20260228") + + +def test_offload_restore_all_and_invalid_scope( + monkeypatch: pytest.MonkeyPatch, +) -> None: + restore_all = Mock( + return_value=OffloadRestoreResult( + status="no_op", + reason="nothing_to_restore", + scope="all", + day=None, + segments_selected=0, + segments_restored=0, + files_expected=0, + files_restored=0, + bytes_expected=0, + bytes_restored=0, + details=(), + ) + ) + monkeypatch.setattr(backup_cli, "restore_offload_all", restore_all) + + all_result = CliRunner().invoke(backup_cli.app, ["offload", "restore", "--all"]) + mixed_result = CliRunner().invoke( + backup_cli.app, + ["offload", "restore", "20260228", "--all"], + ) + missing_result = CliRunner().invoke(backup_cli.app, ["offload", "restore"]) + + assert all_result.exit_code == 0 + assert "status=no_op reason=nothing_to_restore" in all_result.output + restore_all.assert_called_once_with() + assert mixed_result.exit_code == 1 + assert "Use either a day or --all" in mixed_result.output + assert missing_result.exit_code == 1 + assert "Provide a day or --all" in missing_result.output + + def test_recovery_key_show_prints_display_only( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_backup_offload_state.py b/tests/test_backup_offload_state.py index 335e2466e..4b38b1961 100644 --- a/tests/test_backup_offload_state.py +++ b/tests/test_backup_offload_state.py @@ -167,6 +167,19 @@ def test_offload_state_records_and_status_view( "last_ok_time": None, "checked_subset": None, } + assert config["last_restore"] == { + "time": None, + "status": None, + "reason": None, + "scope": None, + "day": None, + "segments_selected": 0, + "segments_restored": 0, + "files_expected": 0, + "files_restored": 0, + "bytes_expected": 0, + "bytes_restored": 0, + } state.record_offload_result( status="ok", @@ -185,6 +198,32 @@ def test_offload_state_records_and_status_view( ran_out_of_media=False, ) state.record_verification_result(status="skipped", time=None, reason="disabled") + state.record_restore_result( + status="ok", + time=200, + reason=None, + scope="day", + day="20260101", + segments_selected=2, + segments_restored=2, + files_expected=5, + files_restored=5, + bytes_expected=500, + bytes_restored=500, + ) + state.record_restore_result( + status="error", + time=210, + reason="verification_failed", + scope="day", + day="20260102", + segments_selected=1, + segments_restored=0, + files_expected=2, + files_restored=0, + bytes_expected=200, + bytes_restored=0, + ) backup = _read_config(tmp_path)["backup"] assert backup["last_offload"] == { @@ -203,6 +242,19 @@ def test_offload_state_records_and_status_view( "last_ok_time": None, "checked_subset": None, } + assert backup["last_restore"] == { + "time": 210, + "status": "error", + "reason": "verification_failed", + "scope": "day", + "day": "20260102", + "segments_selected": 1, + "segments_restored": 0, + "files_expected": 2, + "files_restored": 0, + "bytes_expected": 200, + "bytes_restored": 0, + } with pytest.raises(ValueError): state.record_offload_result( status="degraded", @@ -213,6 +265,20 @@ def test_offload_state_records_and_status_view( ) with pytest.raises(ValueError): state.record_verification_result(status="stalled", time=1) + with pytest.raises(ValueError): + state.record_restore_result( + status="skipped", + time=1, + reason=None, + scope="day", + day="20260101", + segments_selected=0, + segments_restored=0, + files_expected=0, + files_restored=0, + bytes_expected=0, + bytes_restored=0, + ) view = state.status_view() assert view["offload"] == { @@ -222,6 +288,7 @@ def test_offload_state_records_and_status_view( } assert view["last_offload"] == backup["last_offload"] assert view["last_verification"] == backup["last_verification"] + assert view["last_restore"] == backup["last_restore"] def test_verification_record_preserves_last_ok_and_clears_failed_subset( diff --git a/tests/test_offload_measurement.py b/tests/test_offload_measurement.py index c242330bf..9362e341c 100644 --- a/tests/test_offload_measurement.py +++ b/tests/test_offload_measurement.py @@ -11,6 +11,7 @@ import pytest from solstone.think import offload_measurement from solstone.think.offload_measurement import ( device_free_bytes, + device_total_bytes, measure_raw_media_usage, suggest_offload_defaults, ) @@ -106,6 +107,20 @@ def test_device_free_bytes_uses_disk_usage_free( assert device_free_bytes() == 850 * GB +def test_device_total_bytes_uses_disk_usage_total( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + + def fake_disk_usage(path: Path) -> SimpleNamespace: + assert path == tmp_path + return SimpleNamespace(total=1000 * GB, used=100 * GB, free=850 * GB) + + monkeypatch.setattr(offload_measurement.shutil, "disk_usage", fake_disk_usage) + + assert device_total_bytes() == 1000 * GB + + @pytest.mark.parametrize( ("total", "budget", "floor"), [ diff --git a/tests/test_offload_restore.py b/tests/test_offload_restore.py new file mode 100644 index 000000000..095af7c46 --- /dev/null +++ b/tests/test_offload_restore.py @@ -0,0 +1,534 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any +from unittest.mock import Mock + +import pytest + +from solstone.think import offload_restore +from solstone.think.backup import engine +from solstone.think.backup.hosted import ( + HostedBinding, + HostedCredentials, + save_hosted_binding, +) +from solstone.think.backup.runner import ResticResult +from solstone.think.offload_ledger import ( + OffloadFile, + append_offload_event, + append_restore_event, +) +from solstone.think.utils import DEFAULT_STREAM + +DAY = "20260101" +SEGMENT = "090000_300" +CONTENT = b"audio-v1" +SHA = hashlib.sha256(CONTENT).hexdigest() + + +def _config_path(journal: Path) -> Path: + return journal / "config" / "journal.json" + + +def _write_config(journal: Path, backup: dict[str, Any]) -> None: + path = _config_path(journal) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"backup": backup}), encoding="utf-8") + + +def _backup_config(*, mode: str = "byo", enabled: bool = True) -> dict[str, Any]: + backup: dict[str, Any] = { + "enabled": enabled, + "mode": mode, + "daily_key": "daily-secret", + "recovery_key": "R" * 64, + "confirmed_recovery_key": True, + "offload": { + "enabled": True, + "budget_bytes": 10_000_000_000, + "floor_bytes": 1, + }, + } + if mode == "byo": + backup["destination"] = { + "repository": "s3:safe-bucket/path", + "backend": "s3", + "credentials": { + "access_key_id": "access-key", + "secret_access_key": "secret-key", + }, + } + return backup + + +def _segment_dir(journal: Path, *, stream: str = DEFAULT_STREAM) -> Path: + if stream == DEFAULT_STREAM: + path = journal / "chronicle" / DAY / SEGMENT + else: + path = journal / "chronicle" / DAY / stream / SEGMENT + path.mkdir(parents=True, exist_ok=True) + return path + + +def _segment_dir_for( + journal: Path, + day: str, + segment: str, + *, + stream: str = DEFAULT_STREAM, +) -> Path: + if stream == DEFAULT_STREAM: + path = journal / "chronicle" / day / segment + else: + path = journal / "chronicle" / day / stream / segment + path.mkdir(parents=True, exist_ok=True) + return path + + +def _seed_ledger(stream: str = DEFAULT_STREAM, *, snapshot_id: str = "snap1") -> None: + append_offload_event( + day=DAY, + stream=stream, + segment=SEGMENT, + snapshot_id=snapshot_id, + files=[OffloadFile(name="audio.wav", bytes=len(CONTENT), sha256=SHA)], + time=100, + ) + + +def _restic_result(returncode: int, args: list[str]) -> ResticResult: + return ResticResult( + returncode=returncode, + stdout="", + stderr="", + json=None, + argv=tuple(args), + ) + + +def _binding() -> HostedBinding: + return HostedBinding( + broker_endpoint="https://broker.example", + account_id="acct", + instance_id="inst", + bucket="bkt", + prefix="users/acct/inst", + broker_token="BTOKEN", + ) + + +def _creds() -> HostedCredentials: + return HostedCredentials( + access_key_id="AKID", + secret_access_key="SAK", + session_token="SESS", + endpoint="https://acct.r2.cloudflarestorage.com", + expires_at="2026-07-13T12:00:00Z", + ) + + +def test_restore_day_uses_daily_key_default_layout_and_no_pipeline( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + _write_config(tmp_path, _backup_config()) + segment_dir = _segment_dir(tmp_path) + _seed_ledger() + calls: list[tuple[list[str], dict[str, Any]]] = [] + fetch_hosted_credentials = Mock() + callosum_send = Mock() + + def fake_run_restic(args: list[str], **kwargs: Any) -> ResticResult: + calls.append((args, kwargs)) + (segment_dir / "audio.wav").write_bytes(CONTENT) + return _restic_result(0, args) + + monkeypatch.setattr(engine, "ensure_restic", Mock(return_value=Path("/restic"))) + monkeypatch.setattr(engine, "fetch_hosted_credentials", fetch_hosted_credentials) + monkeypatch.setattr(engine, "callosum_send", callosum_send) + monkeypatch.setattr(offload_restore, "run_restic", fake_run_restic) + monkeypatch.setattr(offload_restore, "device_free_bytes", lambda: 5_000_000_000) + monkeypatch.setattr(offload_restore.time, "time", lambda: 200) + + result = offload_restore.restore_day(DAY) + + assert result.status == "ok" + assert calls == [ + ( + [ + "restore", + f"snap1:{segment_dir}", + "--target", + str(segment_dir), + "--include", + "/audio.wav", + ], + { + "repository": "s3:safe-bucket/path", + "password": "daily-secret", + "restic_path": Path("/restic"), + "backend_env": { + "AWS_ACCESS_KEY_ID": "access-key", + "AWS_SECRET_ACCESS_KEY": "secret-key", + }, + "json": True, + "timeout": offload_restore.OFFLOAD_RESTORE_TIMEOUT_SECONDS, + }, + ) + ] + assert "R" * 64 not in json.dumps(calls, default=str) + fetch_hosted_credentials.assert_not_called() + callosum_send.assert_not_called() + assert json.loads(_config_path(tmp_path).read_text(encoding="utf-8"))["backup"][ + "last_restore" + ] == { + "time": 200, + "status": "ok", + "reason": None, + "scope": "day", + "day": DAY, + "segments_selected": 1, + "segments_restored": 1, + "files_expected": 1, + "files_restored": 1, + "bytes_expected": len(CONTENT), + "bytes_restored": len(CONTENT), + } + + +def test_restore_day_operated_uses_backup_scope_and_append_only_session( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + _write_config(tmp_path, _backup_config(mode="operated")) + save_hosted_binding(_binding()) + segment_dir = _segment_dir(tmp_path, stream="camera") + _seed_ledger("camera") + captured_scopes: list[str] = [] + calls: list[list[str]] = [] + + def fake_fetch( + _binding: HostedBinding, + *, + scope: str, + ) -> HostedCredentials: + captured_scopes.append(scope) + return _creds() + + def fake_run_restic(args: list[str], **_kwargs: Any) -> ResticResult: + calls.append(args) + (segment_dir / "audio.wav").write_bytes(CONTENT) + return _restic_result(0, args) + + monkeypatch.setattr(engine, "fetch_hosted_credentials", fake_fetch) + monkeypatch.setattr(engine, "ensure_restic", Mock(return_value=Path("/restic"))) + monkeypatch.setattr(engine, "ensure_rclone", Mock(return_value=Path("/rclone"))) + monkeypatch.setattr(offload_restore, "run_restic", fake_run_restic) + monkeypatch.setattr(offload_restore, "device_free_bytes", lambda: 5_000_000_000) + + result = offload_restore.restore_day(DAY) + + assert result.status == "ok" + assert captured_scopes == ["operated"] + assert calls[0][:4] == [ + "-o", + "rclone.program=/rclone", + "-o", + "rclone.args=serve restic --stdio --append-only --config /dev/null", + ] + assert calls[0][4:7] == ["restore", f"snap1:{segment_dir}", "--target"] + + +@pytest.mark.parametrize( + ("returncode", "reason"), + [ + (10, "repo_missing"), + (12, "auth_failed"), + (11, "locked"), + (124, "timeout"), + (77, "failed"), + ], +) +def test_restore_day_maps_restic_returncode_reasons( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + returncode: int, + reason: str, +) -> None: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + _write_config(tmp_path, _backup_config()) + _segment_dir(tmp_path) + _seed_ledger() + monkeypatch.setattr(engine, "ensure_restic", Mock(return_value=Path("/restic"))) + monkeypatch.setattr(offload_restore, "device_free_bytes", lambda: 5_000_000_000) + monkeypatch.setattr( + offload_restore, + "run_restic", + lambda args, **_kwargs: _restic_result(returncode, args), + ) + + result = offload_restore.restore_day(DAY) + + assert result.status == "error" + assert result.reason == reason + + +def test_missing_include_exit_zero_is_verified_as_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + _write_config(tmp_path, _backup_config()) + _segment_dir(tmp_path) + _seed_ledger() + monkeypatch.setattr(engine, "ensure_restic", Mock(return_value=Path("/restic"))) + monkeypatch.setattr(offload_restore, "device_free_bytes", lambda: 5_000_000_000) + monkeypatch.setattr( + offload_restore, + "run_restic", + lambda args, **_kwargs: _restic_result(0, args), + ) + + result = offload_restore.restore_day(DAY) + + assert result.status == "error" + assert result.reason == "missing_file_after_restore" + + +def test_verification_failure_rolls_back_recorded_attempted_files( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + _write_config(tmp_path, _backup_config()) + segment_dir = _segment_dir(tmp_path) + _seed_ledger() + + def fake_run_restic(args: list[str], **_kwargs: Any) -> ResticResult: + (segment_dir / "audio.wav").write_bytes(b"wrong") + return _restic_result(0, args) + + monkeypatch.setattr(engine, "ensure_restic", Mock(return_value=Path("/restic"))) + monkeypatch.setattr(offload_restore, "device_free_bytes", lambda: 5_000_000_000) + monkeypatch.setattr(offload_restore, "run_restic", fake_run_restic) + + result = offload_restore.restore_day(DAY) + + assert result.status == "error" + assert result.reason == "verification_failed" + assert not (segment_dir / "audio.wav").exists() + + +def test_restore_all_degraded_after_partial_success_continues( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + _write_config(tmp_path, _backup_config()) + first = _segment_dir(tmp_path) + second_segment = "091000_300" + second = tmp_path / "chronicle" / DAY / second_segment + second.mkdir(parents=True) + append_offload_event( + day=DAY, + stream=DEFAULT_STREAM, + segment=SEGMENT, + snapshot_id="snap1", + files=[OffloadFile(name="audio.wav", bytes=len(CONTENT), sha256=SHA)], + time=100, + ) + append_offload_event( + day=DAY, + stream=DEFAULT_STREAM, + segment=second_segment, + snapshot_id="snap2", + files=[OffloadFile(name="audio.wav", bytes=len(CONTENT), sha256=SHA)], + time=101, + ) + call_count = 0 + + def fake_run_restic(args: list[str], **_kwargs: Any) -> ResticResult: + nonlocal call_count + call_count += 1 + if call_count == 1: + (first / "audio.wav").write_bytes(CONTENT) + else: + (second / "audio.wav").write_bytes(b"wrong") + return _restic_result(0, args) + + monkeypatch.setattr(engine, "ensure_restic", Mock(return_value=Path("/restic"))) + monkeypatch.setattr(offload_restore, "device_free_bytes", lambda: 5_000_000_000) + monkeypatch.setattr(offload_restore, "run_restic", fake_run_restic) + + result = offload_restore.restore_all() + + assert result.status == "degraded" + assert result.reason == "verification_failed" + assert result.segments_restored == 1 + assert call_count == 2 + assert (first / "audio.wav").exists() + assert not (second / "audio.wav").exists() + + +def test_restore_all_runs_oldest_first( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + _write_config(tmp_path, _backup_config()) + newer_day = "20260102" + older_day = "20260101" + newer_dir = _segment_dir_for(tmp_path, newer_day, SEGMENT) + older_dir = _segment_dir_for(tmp_path, older_day, SEGMENT) + append_offload_event( + day=newer_day, + stream=DEFAULT_STREAM, + segment=SEGMENT, + snapshot_id="snap-new", + files=[OffloadFile(name="audio.wav", bytes=len(CONTENT), sha256=SHA)], + time=100, + ) + append_offload_event( + day=older_day, + stream=DEFAULT_STREAM, + segment=SEGMENT, + snapshot_id="snap-old", + files=[OffloadFile(name="audio.wav", bytes=len(CONTENT), sha256=SHA)], + time=101, + ) + restored_targets: list[Path] = [] + + def fake_run_restic(args: list[str], **_kwargs: Any) -> ResticResult: + target = Path(args[args.index("--target") + 1]) + restored_targets.append(target) + (target / "audio.wav").write_bytes(CONTENT) + return _restic_result(0, args) + + monkeypatch.setattr(engine, "ensure_restic", Mock(return_value=Path("/restic"))) + monkeypatch.setattr(offload_restore, "device_free_bytes", lambda: 5_000_000_000) + monkeypatch.setattr(offload_restore, "run_restic", fake_run_restic) + + result = offload_restore.restore_all() + + assert result.status == "ok" + assert restored_targets == [older_dir, newer_dir] + + +def test_restore_refuses_when_free_space_guard_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + _write_config(tmp_path, _backup_config()) + _segment_dir(tmp_path) + append_offload_event( + day=DAY, + stream=DEFAULT_STREAM, + segment=SEGMENT, + snapshot_id="snap1", + files=[OffloadFile(name="huge.wav", bytes=2_000_000_000, sha256=SHA)], + time=100, + ) + run_restic = Mock() + monkeypatch.setattr(offload_restore, "device_free_bytes", lambda: 2_999_999_999) + monkeypatch.setattr(offload_restore, "run_restic", run_restic) + + result = offload_restore.restore_day(DAY) + + assert result.status == "refused" + assert result.reason == "insufficient_free_space" + run_restic.assert_not_called() + + +def test_restore_no_op_and_backup_not_ready_reasons( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + _write_config(tmp_path, _backup_config()) + run_restic = Mock() + monkeypatch.setattr(offload_restore, "run_restic", run_restic) + + no_op = offload_restore.restore_day(DAY) + + assert no_op.status == "no_op" + assert no_op.reason == "nothing_to_restore" + run_restic.assert_not_called() + + _segment_dir(tmp_path) + _seed_ledger() + append_restore_event(day=DAY, stream=DEFAULT_STREAM, segment=SEGMENT, time=101) + + already_restored = offload_restore.restore_day(DAY) + + assert already_restored.status == "no_op" + assert already_restored.reason == "nothing_to_restore" + run_restic.assert_not_called() + + _write_config(tmp_path, _backup_config(enabled=False)) + _seed_ledger() + + not_ready = offload_restore.restore_day(DAY) + + assert not_ready.status == "error" + assert not_ready.reason == "backup_not_ready" + + +def test_restore_tool_unavailable_and_ledger_degraded_reasons( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + _write_config(tmp_path, _backup_config()) + _segment_dir(tmp_path) + _seed_ledger() + monkeypatch.setattr(offload_restore, "device_free_bytes", lambda: 5_000_000_000) + monkeypatch.setattr( + engine, + "ensure_restic", + Mock(side_effect=RuntimeError("missing")), + ) + + restic_unavailable = offload_restore.restore_day(DAY) + + assert restic_unavailable.reason == "restic_unavailable" + + ledger_path = tmp_path / "health" / "offload" / f"{DAY}.jsonl" + ledger_path.write_bytes(b"\xff") + + degraded = offload_restore.restore_day(DAY) + + assert degraded.status == "error" + assert degraded.reason == "ledger_degraded" + + +def test_operated_restore_reports_rclone_unavailable( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + _write_config(tmp_path, _backup_config(mode="operated")) + save_hosted_binding(_binding()) + _segment_dir(tmp_path) + _seed_ledger() + monkeypatch.setattr(offload_restore, "device_free_bytes", lambda: 5_000_000_000) + monkeypatch.setattr(engine, "fetch_hosted_credentials", Mock(return_value=_creds())) + monkeypatch.setattr(engine, "ensure_restic", Mock(return_value=Path("/restic"))) + monkeypatch.setattr( + engine, + "ensure_rclone", + Mock(side_effect=RuntimeError("missing")), + ) + + result = offload_restore.restore_day(DAY) + + assert result.status == "error" + assert result.reason == "rclone_unavailable" diff --git a/tests/test_think_utils.py b/tests/test_think_utils.py index 9186d3273..1af47bb0c 100644 --- a/tests/test_think_utils.py +++ b/tests/test_think_utils.py @@ -25,6 +25,7 @@ from solstone.think.utils import ( get_journal_info, get_project_root, iter_segments, + resolve_segment_dir, segment_key, segment_parse, segment_path, @@ -77,6 +78,43 @@ def test_segment_path_create_rejects_escaping_components(monkeypatch, tmp_path): assert not (tmp_path / "outside").exists() +def test_resolve_segment_dir_preserves_iter_segments_default_layout( + monkeypatch, + tmp_path, +): + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + day_dir = day_path("20240101") + default_segment = day_dir / "143022_300" + named_segment = day_dir / "camera" / "143022_300" + default_segment.mkdir() + named_segment.mkdir(parents=True) + + assert ( + resolve_segment_dir( + "20240101", + stream=DEFAULT_STREAM, + segment="143022_300", + ) + == default_segment + ) + assert ( + resolve_segment_dir( + "20240101", + stream="camera", + segment="143022_300", + ) + == named_segment + ) + + missing = resolve_segment_dir( + "20240102", + stream=DEFAULT_STREAM, + segment="090000_300", + ) + assert missing == tmp_path / "chronicle" / "20240102" / "090000_300" + assert not missing.exists() + + def setup_entities_new_structure( journal_path: Path, facet: str, -- 2.51.2