diff --git a/solstone/think/backup/state.py b/solstone/think/backup/state.py index ca5b5a54b..b8c5a101a 100644 --- a/solstone/think/backup/state.py +++ b/solstone/think/backup/state.py @@ -137,9 +137,13 @@ def _build_backup_keys(daily_key: Any, recovery_key: Any) -> BackupKeys | None: ) +def merge_backup_config(config: dict[str, Any]) -> dict[str, Any]: + return _merge_defaults(BACKUP_DEFAULTS, config.get("backup", {})) + + def get_backup_config() -> dict[str, Any]: config = read_journal_config() - return _merge_defaults(BACKUP_DEFAULTS, config.get("backup", {})) + return merge_backup_config(config) def get_destination() -> Destination | None: @@ -489,6 +493,7 @@ __all__ = [ "get_backup_config", "get_destination", "get_keys", + "merge_backup_config", "record_backup_result", "record_offload_result", "record_prune_result", diff --git a/solstone/think/retention.py b/solstone/think/retention.py index e83e3af38..c289e337d 100644 --- a/solstone/think/retention.py +++ b/solstone/think/retention.py @@ -29,6 +29,8 @@ from datetime import datetime from pathlib import Path from typing import Any +from solstone.apps.backup.copy import OFFLOAD_STALL_REASON_LABELS, OFFLOAD_STALLED_LEAD +from solstone.think.backup.state import merge_backup_config from solstone.think.data_state import DataState, derive_modality_state from solstone.think.media import AUDIO_EXTENSIONS as RAW_AUDIO_EXTENSIONS from solstone.think.media import MEDIA_EXTENSIONS as RAW_MEDIA_EXTENSIONS @@ -455,6 +457,26 @@ def check_storage_health( } ) + backup = merge_backup_config(config) + offload = backup["offload"] + last_offload = backup["last_offload"] + if offload.get("enabled") is True and last_offload.get("status") == "stalled": + reason = last_offload.get("reason") + reason_label = ( + OFFLOAD_STALL_REASON_LABELS.get(reason) if isinstance(reason, str) else None + ) + warnings.append( + { + "level": "warning", + "type": "offload_stalled", + "message": " ".join( + part for part in (OFFLOAD_STALLED_LEAD, reason_label) if part + ), + "current": None, + "threshold": None, + } + ) + return warnings diff --git a/tests/test_backup_state.py b/tests/test_backup_state.py index 50b0e9fab..9e3949938 100644 --- a/tests/test_backup_state.py +++ b/tests/test_backup_state.py @@ -60,6 +60,35 @@ def test_partial_backup_section_gets_per_field_defaults( assert config["last_backup"] == state.BACKUP_DEFAULTS["last_backup"] +def test_merge_backup_config_applies_defaults_to_raw_config() -> None: + config = state.merge_backup_config( + { + "backup": { + "offload": {"enabled": True}, + "last_offload": { + "status": "stalled", + "reason": "backup_failing", + }, + } + } + ) + + assert config["offload"] == { + "enabled": True, + "budget_bytes": None, + "floor_bytes": None, + } + assert config["last_offload"] == { + "time": None, + "status": "stalled", + "reason": "backup_failing", + "last_ok_time": None, + "files_offloaded": 0, + "bytes_offloaded": 0, + "ran_out_of_media": False, + } + + def test_generate_and_store_keys_get_or_create( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_retention.py b/tests/test_retention.py index cc671ce3a..7f9ee324f 100644 --- a/tests/test_retention.py +++ b/tests/test_retention.py @@ -13,6 +13,10 @@ from pathlib import Path import pytest from typer.testing import CliRunner +from solstone.apps.backup.copy import ( + OFFLOAD_STALL_REASON_LABELS, + OFFLOAD_STALLED_LEAD, +) from solstone.observe.processing_record import ( FAILED_ATTEMPT_BOUND, STATE_EMPTY, @@ -1167,6 +1171,86 @@ class TestCheckStorageHealth: assert len(warnings) == 1 assert warnings[0]["type"] == "raw_media_gb" + @pytest.mark.parametrize( + ("backup", "expected_message"), + [ + ({}, None), + ( + { + "offload": {"enabled": False}, + "last_offload": { + "status": "stalled", + "reason": "backup_failing", + }, + }, + None, + ), + ( + { + "offload": {"enabled": True}, + "last_offload": {"status": "ok", "reason": None}, + }, + None, + ), + ( + { + "offload": {"enabled": True}, + "last_offload": { + "status": "stalled", + "reason": "backup_failing", + }, + }, + f"{OFFLOAD_STALLED_LEAD} {OFFLOAD_STALL_REASON_LABELS['backup_failing']}", + ), + ( + { + "offload": {"enabled": True}, + "last_offload": {"status": "stalled", "reason": None}, + }, + OFFLOAD_STALLED_LEAD, + ), + ( + { + "offload": {"enabled": True}, + "last_offload": { + "status": "stalled", + "reason": "unknown_reason", + }, + }, + OFFLOAD_STALLED_LEAD, + ), + ], + ) + def test_offload_stalled_warning_from_backup_config( + self, + tmp_path, + backup, + expected_message, + ): + config = { + "retention": { + "storage_warning_disk_percent": None, + "storage_warning_raw_media_gb": None, + }, + "backup": backup, + } + summary = self._make_summary() + + warnings = check_storage_health(summary, tmp_path, config=config) + + if expected_message is None: + assert warnings == [] + return + assert warnings == [ + { + "level": "warning", + "type": "offload_stalled", + "message": expected_message, + "current": None, + "threshold": None, + } + ] + def test_missing_retention_section_uses_defaults(self, tmp_path, monkeypatch): """Missing retention section falls back to defaults (80% disk, null raw media).""" config = {} diff --git a/tests/test_think_segment_prephase.py b/tests/test_think_segment_prephase.py index bfc483df4..3764ef352 100644 --- a/tests/test_think_segment_prephase.py +++ b/tests/test_think_segment_prephase.py @@ -474,6 +474,92 @@ def test_daily_journal_stats_postphase_timeout_records_disposition( assert daily_called +def test_daily_postphase_emits_mixed_storage_warnings(journal_copy, monkeypatch): + mod = importlib.import_module("solstone.think.thinking") + callosum = importlib.import_module("solstone.think.callosum") + retention = importlib.import_module("solstone.think.retention") + sent = [] + warnings = [ + { + "level": "warning", + "type": "disk_percent", + "message": "disk warning", + "current": 95.0, + "threshold": 80, + }, + { + "level": "warning", + "type": "offload_stalled", + "message": "offload warning", + "current": None, + "threshold": None, + }, + ] + + def fake_daily(day, verbose, **kwargs): + return (5, 0, [], set()) + + def fake_send(tract, event, **fields): + sent.append((tract, event, fields)) + return True + + _patch_main_runtime(monkeypatch) + monkeypatch.setattr( + mod, "run_bounded_phase", lambda cmd, day, timeout=None: (True, False) + ) + monkeypatch.setattr(mod, "run_command", lambda cmd, day: True) + monkeypatch.setattr(mod, "run_queued_command", lambda cmd, day, timeout=600: True) + monkeypatch.setattr(mod, "run_daily_prompts", fake_daily) + monkeypatch.setattr(retention, "compute_storage_summary", lambda: object()) + monkeypatch.setattr( + retention, + "check_storage_health", + lambda summary, journal_path: warnings, + ) + monkeypatch.setattr(callosum, "callosum_send", fake_send) + monkeypatch.setattr("sys.argv", ["sol think", "--day", "20240101"]) + + mod.main() + + storage = [item for item in sent if item[:2] == ("storage", "warning")] + notification = [item for item in sent if item[:2] == ("notification", "show")] + assert storage == [ + ( + "storage", + "warning", + { + "level": "warning", + "type": "disk_percent", + "message": "disk warning", + "current": 95.0, + "threshold": 80, + }, + ), + ( + "storage", + "warning", + { + "level": "warning", + "type": "offload_stalled", + "message": "offload warning", + "current": None, + "threshold": None, + }, + ), + ] + assert notification == [ + ( + "notification", + "show", + { + "title": "Storage Warning", + "message": "disk warning", + "action": "/app/settings#storage", + }, + ) + ] + + def test_daily_segment_prephase_failure_has_no_timeout_reason( journal_copy, monkeypatch ):