From 0cd7baeac824d501cf0e8999344b58fd8558dcd4 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Sat, 25 Jul 2026 14:03:32 -0600 Subject: [PATCH] fix(home): surface backlog health in pulse Add a shared read-only convey backlog loader and a stuck_day_rows renderer helper so home and health read the same backlog source without adding IO to backlog_view. Require an explicit backlog signal for the home health glance and report stuck, unreadable, degraded, or stale backlog state before returning green. --- solstone/apps/health/routes.py | 15 +- solstone/apps/home/health_glance.py | 127 +++++ solstone/apps/home/routes.py | 4 + .../home/tests/test_health_glance_brain.py | 57 +- solstone/convey/backlog_source.py | 53 ++ solstone/convey/backlog_view.py | 37 +- tests/helpers/health_glance.py | 22 + tests/test_backlog_view.py | 50 +- tests/test_brain_health_cutover_parity.py | 2 + tests/test_home_connections.py | 6 + tests/test_home_health_glance.py | 513 ++++++++++++++++-- tests/test_home_routes.py | 50 +- tests/test_home_yesterdays_processing.py | 9 + 13 files changed, 877 insertions(+), 68 deletions(-) create mode 100644 solstone/convey/backlog_source.py create mode 100644 tests/helpers/health_glance.py diff --git a/solstone/apps/health/routes.py b/solstone/apps/health/routes.py index 774cfa409..d9f912079 100644 --- a/solstone/apps/health/routes.py +++ b/solstone/apps/health/routes.py @@ -1,9 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright (c) 2026 sol pbc -import json import logging -import os import re import socket from datetime import datetime, timezone @@ -12,6 +10,7 @@ from pathlib import Path from flask import Blueprint, Response, current_app, jsonify, request from solstone.convey import backlog_copy, state +from solstone.convey.backlog_source import load_backlog_source from solstone.convey.backlog_view import stuck_rows, verdict from solstone.convey.reasons import ( FILE_NOT_FOUND, @@ -59,17 +58,7 @@ OBSERVER_RESTART_SERVICES = {"sense"} def _load_backlog() -> dict | None: - stats_path = os.path.join(state.journal_root, "stats.json") - if not os.path.isfile(stats_path): - return None - try: - with open(stats_path, "r", encoding="utf-8") as f: - data = json.load(f) - except (OSError, ValueError): - logger.exception("Failed to read backlog from stats.json") - return None - backlog = data.get("backlog") - return backlog if isinstance(backlog, dict) else None + return load_backlog_source(state.journal_root).backlog def _safe_brain_snapshot() -> dict: diff --git a/solstone/apps/home/health_glance.py b/solstone/apps/home/health_glance.py index d4bd44716..33cb330a1 100644 --- a/solstone/apps/home/health_glance.py +++ b/solstone/apps/home/health_glance.py @@ -4,23 +4,33 @@ from __future__ import annotations import logging +from datetime import datetime, timezone from typing import Any from solstone.apps.home.needs_you import format_degraded_capture_line +from solstone.convey.backlog_source import BacklogSource +from solstone.convey.backlog_view import stuck_day_rows +from solstone.convey.provider_readiness import DISPLAY_NAMES logger = logging.getLogger(__name__) _HEALTH_DETAIL_HREF = "/app/health#focus=recent-errors&day=today" +BACKLOG_FRESHNESS_MAX_AGE_HOURS = 36 +_BACKLOG_HREF = "/app/health" def build_health_glance( capture_health: Any, pipeline_status: Any, last_observe_relative: str | None, + *, + backlog: BacklogSource, brain: dict[str, Any] | None = None, ) -> dict: issues = [] + issues.extend(_build_backlog_issues(backlog)) + capture_issue = _issue_safely( "capture health", capture_health, _build_capture_issue ) @@ -106,6 +116,123 @@ def _issue_safely(label: str, value: Any, builder: Any) -> dict | None: return None +def _build_backlog_issues(source: BacklogSource) -> list[dict]: + issues: list[dict] = [] + if source.validity != "valid": + issues.append(_backlog_unknown_issue()) + return issues + + backlog = source.backlog + if not isinstance(backlog, dict): + issues.append(_backlog_unknown_issue()) + return issues + + if backlog.get("degraded") is True: + issues.append(_backlog_unknown_issue()) + + freshness_issue = _build_backlog_freshness_issue(source.generated_at) + if freshness_issue is not None: + issues.append(freshness_issue) + + if _backlog_count(backlog.get("stuck_days")) > 0: + issues.append(_build_backlog_stuck_issue(backlog)) + + return issues + + +def _backlog_unknown_issue() -> dict: + return { + "text": "i can't tell if your journal is caught up right now.", + "severity": "amber", + "href": _BACKLOG_HREF, + } + + +def _build_backlog_freshness_issue(generated_at: str | None) -> dict | None: + generated_at_dt = _parse_generated_at(generated_at) + if generated_at_dt is None: + return { + "text": ( + "i can't tell if your journal is caught up; " + "the last update age is unknown." + ), + "severity": "amber", + "href": _BACKLOG_HREF, + } + + age = _now_utc() - generated_at_dt + if age.total_seconds() <= BACKLOG_FRESHNESS_MAX_AGE_HOURS * 3600: + return None + return { + "text": ( + "i can't tell if your journal is caught up; " + f"the last update was {_format_age(age)} ago." + ), + "severity": "amber", + "href": _BACKLOG_HREF, + } + + +def _build_backlog_stuck_issue(backlog: dict) -> dict: + rows = stuck_day_rows(backlog) + if rows: + text = _backlog_stuck_issue_text(rows[0]) + else: + text = "a journal day needs a hand." + return {"text": text, "severity": "red", "href": _BACKLOG_HREF} + + +def _backlog_stuck_issue_text(row: dict) -> str: + reason = row.get("reason") + text = reason.strip() if isinstance(reason, str) and reason.strip() else "" + if not text: + text = "a journal day needs a hand." + + provider = row.get("provider") + display_name = DISPLAY_NAMES.get(provider) if isinstance(provider, str) else None + if display_name: + return f"{display_name}: {text}" + return text + + +def _backlog_count(value: Any) -> int: + try: + count = int(value) + except (TypeError, ValueError): + return 0 + return max(0, count) + + +def _parse_generated_at(value: str | None) -> datetime | None: + if not isinstance(value, str) or not value.strip(): + return None + text = value.strip() + if text.endswith("Z"): + text = f"{text[:-1]}+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _format_age(delta: Any) -> str: + seconds = max(0, int(delta.total_seconds())) + hours = seconds // 3600 + if hours >= 1: + unit = "hour" if hours == 1 else "hours" + return f"{hours} {unit}" + minutes = max(1, seconds // 60) + unit = "minute" if minutes == 1 else "minutes" + return f"{minutes} {unit}" + + +def _now_utc() -> datetime: + return datetime.now(timezone.utc) + + def _build_capture_issue(capture_health: Any) -> dict | None: if not isinstance(capture_health, dict): return None diff --git a/solstone/apps/home/routes.py b/solstone/apps/home/routes.py index 324c1fcbc..996b98924 100644 --- a/solstone/apps/home/routes.py +++ b/solstone/apps/home/routes.py @@ -20,6 +20,8 @@ from solstone.apps.home.connections import build_connections_card from solstone.apps.home.health_glance import build_health_glance from solstone.apps.home.needs_you import classify_needs_you, needs_dedup_key from solstone.apps.home.owner_voice import build_owner_voice_needs +from solstone.convey import state +from solstone.convey.backlog_source import load_backlog_source from solstone.convey.bridge import get_cached_state from solstone.convey.shell_data import _resolve_attention from solstone.convey.utils import DATE_RE, format_date, relative_time @@ -921,10 +923,12 @@ def _build_pulse_context() -> dict[str, Any]: if summary: pipeline_status = {**pipeline_status, **summary} brain = build_brain_snapshot(datetime.now(timezone.utc), surface="home") + backlog_source = load_backlog_source(state.journal_root) health_glance = build_health_glance( capture_health, pipeline_status, last_observe_relative, + backlog=backlog_source, brain=brain, ) diff --git a/solstone/apps/home/tests/test_health_glance_brain.py b/solstone/apps/home/tests/test_health_glance_brain.py index f83975e11..dc793b416 100644 --- a/solstone/apps/home/tests/test_health_glance_brain.py +++ b/solstone/apps/home/tests/test_health_glance_brain.py @@ -21,6 +21,7 @@ from solstone.think.providers.brain_state import ( finish_brain_refresh, ) from solstone.think.providers.runtime_health import runtime_health_path +from tests.helpers.health_glance import healthy_backlog_source NOW = datetime(2026, 7, 21, 12, 0, 0, tzinfo=timezone.utc) BUNDLED_RUNTIME_FINGERPRINT = "b" * 64 @@ -174,7 +175,13 @@ def test_held_permit_checking_projects_home_status_row(tmp_path: Path) -> None: assert permit is not None try: brain = build_brain_snapshot(NOW, surface="home", journal_path=journal) - result = build_health_glance(_active_capture(), None, "5m ago", brain=brain) + result = build_health_glance( + _active_capture(), + None, + "5m ago", + brain=brain, + backlog=healthy_backlog_source(), + ) assert brain["state"] == "checking" assert brain["headline"] == HEADLINES["checking"] @@ -196,7 +203,11 @@ def test_checking_freshness_boundary_uses_explicit_now_seam( fresh_now, surface="home", journal_path=journal ) fresh_result = build_health_glance( - _active_capture(), None, "5m ago", brain=fresh_brain + _active_capture(), + None, + "5m ago", + brain=fresh_brain, + backlog=healthy_backlog_source(), ) assert fresh_brain["state"] == "checking" _assert_checking_row(fresh_result) @@ -209,7 +220,11 @@ def test_checking_freshness_boundary_uses_explicit_now_seam( expired_now, surface="home", journal_path=journal ) expired_result = build_health_glance( - _active_capture(), None, "5m ago", brain=expired_brain + _active_capture(), + None, + "5m ago", + brain=expired_brain, + backlog=healthy_backlog_source(), ) assert expired_brain["state"] == "unknown" @@ -232,7 +247,9 @@ def test_released_permit_projects_interrupted_unknown_issue( permit.release() brain = build_brain_snapshot(NOW, surface="home", journal_path=journal) - result = build_health_glance(_active_capture(), None, "5m ago", brain=brain) + result = build_health_glance( + _active_capture(), None, "5m ago", brain=brain, backlog=healthy_backlog_source() + ) assert brain["state"] == "unknown" assert brain["reason_code"] == "brain_check_interrupted" @@ -254,7 +271,11 @@ def test_bundled_runtime_transition_projects_home_rows( _write_runtime_health_record(journal, phase="starting") progressing_brain = build_brain_snapshot(NOW, surface="home", journal_path=journal) progressing_result = build_health_glance( - _active_capture(), None, "5m ago", brain=progressing_brain + _active_capture(), + None, + "5m ago", + brain=progressing_brain, + backlog=healthy_backlog_source(), ) assert progressing_brain["state"] == "blocked" @@ -266,7 +287,11 @@ def test_bundled_runtime_transition_projects_home_rows( _write_runtime_health_record(journal, phase="backoff") blocked_brain = build_brain_snapshot(NOW, surface="home", journal_path=journal) blocked_result = build_health_glance( - _active_capture(), None, "5m ago", brain=blocked_brain + _active_capture(), + None, + "5m ago", + brain=blocked_brain, + backlog=healthy_backlog_source(), ) assert blocked_brain["state"] == "blocked" @@ -287,12 +312,26 @@ def test_capture_variants_against_real_checking_projection(tmp_path: Path) -> No try: brain = build_brain_snapshot(NOW, surface="home", journal_path=journal) - active = build_health_glance(_active_capture(), None, "5m ago", brain=brain) + active = build_health_glance( + _active_capture(), + None, + "5m ago", + brain=brain, + backlog=healthy_backlog_source(), + ) no_observers = build_health_glance( - _no_observers_capture(), None, None, brain=brain + _no_observers_capture(), + None, + None, + brain=brain, + backlog=healthy_backlog_source(), ) unavailable = build_health_glance( - _unavailable_capture(), None, None, brain=brain + _unavailable_capture(), + None, + None, + brain=brain, + backlog=healthy_backlog_source(), ) assert brain["state"] == "checking" diff --git a/solstone/convey/backlog_source.py b/solstone/convey/backlog_source.py new file mode 100644 index 000000000..add71a084 --- /dev/null +++ b/solstone/convey/backlog_source.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Read-only loader for the serialized journal backlog source.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class BacklogSource: + backlog: dict | None + validity: str + generated_at: str | None + + +def load_backlog_source(journal_root: str) -> BacklogSource: + path = Path(journal_root) / "stats.json" + try: + with path.open("r", encoding="utf-8") as handle: + data: Any = json.load(handle) + except FileNotFoundError: + return BacklogSource(backlog=None, validity="missing", generated_at=None) + except (OSError, ValueError): + return BacklogSource(backlog=None, validity="unparseable", generated_at=None) + + if not isinstance(data, dict): + return BacklogSource(backlog=None, validity="malformed", generated_at=None) + + generated_at = data.get("generated_at") + if not isinstance(generated_at, str): + generated_at = None + + if "backlog" not in data: + return BacklogSource( + backlog=None, + validity="no_backlog_key", + generated_at=generated_at, + ) + + backlog = data.get("backlog") + if not isinstance(backlog, dict): + return BacklogSource( + backlog=None, + validity="malformed", + generated_at=generated_at, + ) + + return BacklogSource(backlog=backlog, validity="valid", generated_at=generated_at) diff --git a/solstone/convey/backlog_view.py b/solstone/convey/backlog_view.py index 212b29a00..b7963f24b 100644 --- a/solstone/convey/backlog_view.py +++ b/solstone/convey/backlog_view.py @@ -11,6 +11,7 @@ from typing import Any from solstone.convey import backlog_copy, provider_readiness __all__ = [ + "stuck_day_rows", "stuck_rows", "verdict", ] @@ -110,6 +111,19 @@ def _reason_copy(day: dict) -> str: return backlog_copy.BACKLOG_REASON_FAILING_STEP +def _stuck_row(day: dict) -> dict: + depth = _count(day.get("segments")) + _count(day.get("units")) + row = { + "day": day.get("day"), + "reason": _reason_copy(day), + "depth": depth if depth > 0 else None, + } + for field in ("reason_code", "provider", "model"): + if day.get(field): + row[field] = day.get(field) + return row + + def stuck_rows(backlog: dict | None) -> list[dict]: if backlog is None or backlog.get("degraded") is True: return [] @@ -118,14 +132,17 @@ def stuck_rows(backlog: dict | None) -> list[dict]: for day in backlog.get("days") or []: if not _needs_hand(day, backlog): continue - depth = _count(day.get("segments")) + _count(day.get("units")) - row = { - "day": day.get("day"), - "reason": _reason_copy(day), - "depth": depth if depth > 0 else None, - } - for field in ("reason_code", "provider", "model"): - if day.get(field): - row[field] = day.get(field) - rows.append(row) + rows.append(_stuck_row(day)) + return rows + + +def stuck_day_rows(backlog: dict | None) -> list[dict]: + if backlog is None or backlog.get("degraded") is True: + return [] + + rows = [] + for day in backlog.get("days") or []: + if day.get("state") != "stuck": + continue + rows.append(_stuck_row(day)) return rows diff --git a/tests/helpers/health_glance.py b/tests/helpers/health_glance.py new file mode 100644 index 000000000..a60157e0d --- /dev/null +++ b/tests/helpers/health_glance.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +from datetime import datetime, timezone + +from solstone.convey.backlog_source import BacklogSource + + +def healthy_backlog_source() -> BacklogSource: + return BacklogSource( + backlog={ + "pending_days": 0, + "stuck_days": 0, + "days": [], + "errors": [], + "degraded": False, + }, + validity="valid", + generated_at=datetime.now(timezone.utc).isoformat(), + ) diff --git a/tests/test_backlog_view.py b/tests/test_backlog_view.py index 110f8e1f8..689078984 100644 --- a/tests/test_backlog_view.py +++ b/tests/test_backlog_view.py @@ -6,7 +6,7 @@ from __future__ import annotations import pytest from solstone.convey import backlog_copy -from solstone.convey.backlog_view import stuck_rows, verdict +from solstone.convey.backlog_view import stuck_day_rows, stuck_rows, verdict def _assert_single_stuck_reason(reason_code: str, expected: str) -> None: @@ -169,6 +169,54 @@ def test_stuck_rows_maps_readiness_reasons_and_carries_operator_fields(): ] +def test_stuck_day_rows_excludes_pending_error_rows_without_changing_row_shape(): + backlog = { + "days": [ + { + "day": "20260601", + "state": "pending", + "error": {"reason_code": "provider_request_rejected"}, + "reason_code": "provider_request_rejected", + "provider": "google", + }, + { + "day": "20260602", + "state": "stuck", + "segments": 1, + "reason_code": "provider_request_rejected", + "provider": "google", + }, + ], + "errors": [], + } + + assert stuck_rows(backlog) == [ + { + "day": "20260601", + "reason": backlog_copy.BACKLOG_REASON_PROVIDER_REFUSED, + "depth": None, + "reason_code": "provider_request_rejected", + "provider": "google", + }, + { + "day": "20260602", + "reason": backlog_copy.BACKLOG_REASON_PROVIDER_REFUSED, + "depth": 1, + "reason_code": "provider_request_rejected", + "provider": "google", + }, + ] + assert stuck_day_rows(backlog) == [ + { + "day": "20260602", + "reason": backlog_copy.BACKLOG_REASON_PROVIDER_REFUSED, + "depth": 1, + "reason_code": "provider_request_rejected", + "provider": "google", + } + ] + + def test_verdict_pending_only_copy_does_not_claim_caught_up(): assert ( verdict({"pending_days": 1, "stuck_days": 0}) == "1 day is still catching up." diff --git a/tests/test_brain_health_cutover_parity.py b/tests/test_brain_health_cutover_parity.py index a3960419a..34021d978 100644 --- a/tests/test_brain_health_cutover_parity.py +++ b/tests/test_brain_health_cutover_parity.py @@ -9,6 +9,7 @@ from typing import Any from flask import Flask from solstone.think.brain_health import HEADLINES +from tests.helpers.health_glance import healthy_backlog_source @dataclass(frozen=True) @@ -158,6 +159,7 @@ def test_state_parity_matrix(monkeypatch): None, "5m ago", brain=home_brain, + backlog=healthy_backlog_source(), ) home_issue = home["issues"][0] if home["issues"] else None assert home_brain["headline"] == HEADLINES[case.state] diff --git a/tests/test_home_connections.py b/tests/test_home_connections.py index 038285890..5e3655f74 100644 --- a/tests/test_home_connections.py +++ b/tests/test_home_connections.py @@ -16,6 +16,7 @@ from solstone.apps.home.connections import _kind_words, build_connections_card from solstone.think.indexer import edges as edge_index from solstone.think.indexer.edges import insert_edges from solstone.think.indexer.journal import get_journal_index +from tests.helpers.health_glance import healthy_backlog_source EDGE_FIXTURE = Path(__file__).resolve().parent / "fixtures" / "edges_journal" CONTRACT_KIND_WORDS = { @@ -265,6 +266,11 @@ def test_build_pulse_context_survives_missing_connections_index( monkeypatch.setattr( home_routes, "read_steward_summary", lambda *args, **kwargs: None ) + monkeypatch.setattr( + home_routes, + "load_backlog_source", + lambda _journal_root: healthy_backlog_source(), + ) monkeypatch.setattr( home_routes, "build_brain_snapshot", diff --git a/tests/test_home_health_glance.py b/tests/test_home_health_glance.py index b63be5d5f..f797f6818 100644 --- a/tests/test_home_health_glance.py +++ b/tests/test_home_health_glance.py @@ -3,13 +3,17 @@ from __future__ import annotations +import json import re -from datetime import datetime +from datetime import datetime, timedelta, timezone import pytest +from solstone.apps.home import health_glance as health_glance_module from solstone.apps.home.health_glance import build_health_glance +from solstone.convey.backlog_source import BacklogSource, load_backlog_source from solstone.think.brain_health import HEADLINES +from tests.helpers.health_glance import healthy_backlog_source BANNED_RE = re.compile( r"\b(watch|capture|record|monitor|track|collect|observer|observation)\b", @@ -23,6 +27,7 @@ EXTENDED_BANNED_RE = re.compile( re.IGNORECASE, ) HEALTH_DETAIL_HREF = "/app/health#focus=recent-errors&day=today" +BACKLOG_NOW = datetime(2026, 7, 21, 12, 0, 0, tzinfo=timezone.utc) def _brain( @@ -114,6 +119,55 @@ def _no_observers_capture() -> dict: return {"status": "no_observers", "observers": []} +def _backlog_source( + backlog: dict | None, + *, + validity: str = "valid", + generated_at: str | None = None, +) -> BacklogSource: + return BacklogSource( + backlog=backlog, + validity=validity, + generated_at=generated_at + if generated_at is not None + else BACKLOG_NOW.isoformat(), + ) + + +def _clear_backlog() -> dict: + return { + "pending_days": 0, + "stuck_days": 0, + "days": [], + "errors": [], + "degraded": False, + } + + +def _stuck_backlog(day: dict | None = None) -> dict: + return { + "pending_days": 0, + "stuck_days": 1, + "days": [ + day + or { + "day": "20260720", + "state": "stuck", + "reason_code": "provider_request_rejected", + "provider": "google", + "segments": 1, + } + ], + "errors": [], + "degraded": False, + } + + +@pytest.fixture +def fixed_backlog_now(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(health_glance_module, "_now_utc", lambda: BACKLOG_NOW) + + def _assert_status_row(result: dict, *, verdict: str, headline: str) -> None: assert result["verdict"] == verdict assert result["severity"] == "amber" @@ -154,8 +208,270 @@ def _assert_single_brain_issue(result: dict, *, text: str, href: str) -> None: assert result["issues"][0] == {"text": text, "severity": "amber", "href": href} +def test_stuck_backlog_returns_red_attention_with_display_provider( + fixed_backlog_now: None, +) -> None: + result = build_health_glance( + _active_capture(), + None, + "5m ago", + backlog=_backlog_source(_stuck_backlog()), + ) + + assert result["verdict"] == "attention" + assert result["severity"] == "red" + assert any( + "the AI provider refused a request sol sent" in issue["text"] + and "Gemini" in issue["text"] + and "google" not in issue["text"] + for issue in result["issues"] + ) + + +def test_pending_only_backlog_with_error_row_does_not_raise_backlog_issue( + fixed_backlog_now: None, +) -> None: + backlog = { + "pending_days": 1, + "stuck_days": 0, + "days": [ + { + "day": "20260720", + "state": "pending", + "error": {"reason_code": "provider_request_rejected"}, + "reason_code": "provider_request_rejected", + "provider": "google", + } + ], + "errors": [], + "degraded": False, + } + + result = build_health_glance( + _active_capture(), None, "5m ago", backlog=_backlog_source(backlog) + ) + + assert result["verdict"] == "ok" + assert result["severity"] == "green" + assert result["issues"] == [] + + +def test_zero_backlog_active_observer_uses_existing_everything_working_green_return( + fixed_backlog_now: None, +) -> None: + result = build_health_glance( + _active_capture(), None, "5m ago", backlog=_backlog_source(_clear_backlog()) + ) + + assert result["verdict"] == "ok" + assert result["severity"] == "green" + assert result["headline"] == "everything's working" + assert result["last_observation"] == "5m ago" + assert result["issues"] == [] + + +def test_stuck_backlog_with_absent_pipeline_status_still_returns_attention( + fixed_backlog_now: None, +) -> None: + result = build_health_glance( + _active_capture(), None, None, backlog=_backlog_source(_stuck_backlog()) + ) + + assert result["verdict"] == "attention" + assert result["severity"] == "red" + + +@pytest.mark.parametrize( + "source", + [ + BacklogSource(backlog=None, validity="missing", generated_at=None), + BacklogSource(backlog=None, validity="unparseable", generated_at=None), + BacklogSource( + backlog=None, + validity="no_backlog_key", + generated_at=BACKLOG_NOW.isoformat(), + ), + BacklogSource( + backlog=None, + validity="malformed", + generated_at=BACKLOG_NOW.isoformat(), + ), + _backlog_source({**_clear_backlog(), "degraded": True}), + ], +) +def test_unreadable_or_degraded_backlog_source_is_not_green( + source: BacklogSource, + fixed_backlog_now: None, +) -> None: + result = build_health_glance(_active_capture(), None, "5m ago", backlog=source) + + assert result["verdict"] == "attention" + assert result["severity"] == "amber" + assert any("i can't tell" in issue["text"] for issue in result["issues"]) + + +def test_backlog_loader_outcomes_feed_non_green_glance( + tmp_path, + fixed_backlog_now: None, +) -> None: + cases = [ + ("missing", None, "missing"), + ("unparseable", "{not-json", "unparseable"), + ( + "no_backlog_key", + {"generated_at": BACKLOG_NOW.isoformat(), "schema_version": 1}, + "no_backlog_key", + ), + ( + "malformed", + {"generated_at": BACKLOG_NOW.isoformat(), "backlog": []}, + "malformed", + ), + ( + "degraded", + { + "generated_at": BACKLOG_NOW.isoformat(), + "backlog": {**_clear_backlog(), "degraded": True}, + }, + "valid", + ), + ] + + for name, payload, expected_validity in cases: + journal = tmp_path / name + journal.mkdir() + if isinstance(payload, str): + (journal / "stats.json").write_text(payload, encoding="utf-8") + elif payload is not None: + (journal / "stats.json").write_text(json.dumps(payload), encoding="utf-8") + + source = load_backlog_source(str(journal)) + result = build_health_glance(_active_capture(), None, "5m ago", backlog=source) + + assert source.validity == expected_validity + assert result["verdict"] == "attention" + assert result["severity"] == "amber" + assert any("i can't tell" in issue["text"] for issue in result["issues"]) + + +def test_backlog_issue_builder_exceptions_are_not_safely_omitted( + monkeypatch: pytest.MonkeyPatch, + fixed_backlog_now: None, +) -> None: + def raise_from_backlog(_backlog: dict) -> list[dict]: + raise RuntimeError("backlog boom") + + monkeypatch.setattr(health_glance_module, "stuck_day_rows", raise_from_backlog) + + with pytest.raises(RuntimeError, match="backlog boom"): + build_health_glance( + _active_capture(), None, "5m ago", backlog=_backlog_source(_stuck_backlog()) + ) + + +def test_backlog_freshness_controls_green_vs_stale_issue( + fixed_backlog_now: None, +) -> None: + fresh = build_health_glance( + _active_capture(), + None, + "5m ago", + backlog=_backlog_source( + _clear_backlog(), + generated_at=(BACKLOG_NOW - timedelta(hours=35)).isoformat(), + ), + ) + stale = build_health_glance( + _active_capture(), + None, + "5m ago", + backlog=_backlog_source( + _clear_backlog(), + generated_at=(BACKLOG_NOW - timedelta(hours=37)).isoformat(), + ), + ) + + assert fresh["verdict"] == "ok" + assert stale["verdict"] == "attention" + assert stale["severity"] == "amber" + assert any( + "i can't tell" in issue["text"] and "37 hours ago" in issue["text"] + for issue in stale["issues"] + ) + + +@pytest.mark.parametrize("generated_at", [None, "not-a-timestamp"]) +def test_backlog_missing_or_unparseable_generated_at_is_not_fresh( + generated_at: str | None, + fixed_backlog_now: None, +) -> None: + result = build_health_glance( + _active_capture(), + None, + "5m ago", + backlog=BacklogSource( + backlog=_clear_backlog(), + validity="valid", + generated_at=generated_at, + ), + ) + + assert result["verdict"] == "attention" + assert result["severity"] == "amber" + assert any( + "i can't tell" in issue["text"] and "age is unknown" in issue["text"] + for issue in result["issues"] + ) + + +@pytest.mark.parametrize( + "day", + [ + { + "day": "20260720", + "state": "stuck", + "reason_code": "provider_request_rejected", + }, + { + "day": "20260720", + "state": "stuck", + "reason_code": "provider_request_rejected", + "provider": "weirdslug", + }, + ], +) +def test_stuck_backlog_without_display_provider_degrades_cleanly( + day: dict, + fixed_backlog_now: None, +) -> None: + result = build_health_glance( + _active_capture(), None, "5m ago", backlog=_backlog_source(_stuck_backlog(day)) + ) + + assert result["verdict"] == "attention" + assert result["severity"] == "red" + text = result["issues"][0]["text"] + assert "None" not in text + assert "weirdslug" not in text + + +def test_backlog_argument_is_required_and_keyword_only() -> None: + with pytest.raises(TypeError): + build_health_glance(_active_capture(), None, "5m ago") + + with pytest.raises(TypeError): + build_health_glance( + _active_capture(), + None, + "5m ago", + healthy_backlog_source(), + ) + + def test_degraded_capture_returns_red_attention_issue(): - result = build_health_glance(_degraded_capture("fedora"), None, None) + result = build_health_glance( + _degraded_capture("fedora"), None, None, backlog=healthy_backlog_source() + ) assert result["verdict"] == "attention" assert result["severity"] == "red" @@ -171,7 +487,10 @@ def test_degraded_capture_returns_red_attention_issue(): def test_degraded_capture_collapses_multiple_observers_to_one_issue(): result = build_health_glance( - _degraded_capture("fedora", "phone", "tablet"), None, None + _degraded_capture("fedora", "phone", "tablet"), + None, + None, + backlog=healthy_backlog_source(), ) assert len(result["issues"]) == 1 @@ -186,6 +505,7 @@ def test_degraded_capture_and_pipeline_warning_returns_two_issues_red_verdict(): _degraded_capture("fedora"), {"status": "warning", "headline": "processing needs attention"}, None, + backlog=healthy_backlog_source(), ) assert result["verdict"] == "attention" @@ -199,6 +519,7 @@ def test_pipeline_warning_alone_returns_amber_attention_issue(): _active_capture(), {"status": "warning", "headline": "processing needs attention"}, None, + backlog=healthy_backlog_source(), ) assert result["verdict"] == "attention" @@ -213,6 +534,7 @@ def test_pipeline_warning_without_headline_uses_fallback_text(): _active_capture(), {"status": "warning", "message": "some bullet"}, None, + backlog=healthy_backlog_source(), ) assert result["verdict"] != "ok" @@ -234,7 +556,9 @@ def test_pipeline_warning_without_headline_uses_fallback_text(): ], ) def test_offline_and_stale_capture_do_not_return_ok(capture_health, issue_text): - result = build_health_glance(capture_health, None, None) + result = build_health_glance( + capture_health, None, None, backlog=healthy_backlog_source() + ) assert result["verdict"] != "ok" assert result["headline"] != "everything's working" @@ -242,7 +566,9 @@ def test_offline_and_stale_capture_do_not_return_ok(capture_health, issue_text): def test_active_capture_without_pipeline_returns_ok_with_last_observation(): - result = build_health_glance(_active_capture(), None, "5m ago") + result = build_health_glance( + _active_capture(), None, "5m ago", backlog=healthy_backlog_source() + ) assert result["verdict"] == "ok" assert result["severity"] == "green" @@ -253,7 +579,10 @@ def test_active_capture_without_pipeline_returns_ok_with_last_observation(): def test_no_observers_returns_ok_with_setup_cta(): result = build_health_glance( - {"status": "no_observers", "observers": []}, None, None + {"status": "no_observers", "observers": []}, + None, + None, + backlog=healthy_backlog_source(), ) assert result["verdict"] == "ok" @@ -267,7 +596,12 @@ def test_no_observers_returns_ok_with_setup_cta(): def test_unknown_observer_state_returns_unavailable(): - result = build_health_glance({"status": "unknown", "observers": []}, None, None) + result = build_health_glance( + {"status": "unknown", "observers": []}, + None, + None, + backlog=healthy_backlog_source(), + ) assert result["verdict"] == "unavailable" assert result["severity"] == "amber" @@ -285,6 +619,7 @@ def test_pipeline_open_support_action_routes_to_support(): "suggested_action": "open_support", }, None, + backlog=healthy_backlog_source(), ) assert result["issues"][0]["href"] == "/app/support" @@ -307,13 +642,21 @@ def test_pipeline_open_support_action_routes_to_support(): ], ) def test_pipeline_health_actions_route_to_recent_errors(pipeline_status): - result = build_health_glance(_active_capture(), pipeline_status, None) + result = build_health_glance( + _active_capture(), pipeline_status, None, backlog=healthy_backlog_source() + ) assert result["issues"][0]["href"] == HEALTH_DETAIL_HREF def test_brain_blocked_alone_returns_amber_attention_issue(): - result = build_health_glance(_active_capture(), None, None, brain=_blocked_brain()) + result = build_health_glance( + _active_capture(), + None, + None, + brain=_blocked_brain(), + backlog=healthy_backlog_source(), + ) assert result["issues"] == [ { @@ -329,7 +672,11 @@ def test_brain_blocked_alone_returns_amber_attention_issue(): def test_brain_blocked_combines_with_red_capture_issue(): result = build_health_glance( - _degraded_capture("fedora"), None, None, brain=_blocked_brain() + _degraded_capture("fedora"), + None, + None, + brain=_blocked_brain(), + backlog=healthy_backlog_source(), ) assert len(result["issues"]) == 2 @@ -346,7 +693,11 @@ def test_brain_blocked_combines_with_red_capture_issue(): def test_brain_ready_keeps_ok_glance(): result = build_health_glance( - _active_capture(), None, "5m ago", brain=_ready_brain() + _active_capture(), + None, + "5m ago", + brain=_ready_brain(), + backlog=healthy_backlog_source(), ) assert result["verdict"] == "ok" @@ -356,7 +707,11 @@ def test_brain_ready_keeps_ok_glance(): def test_checking_brain_returns_status_only_amber_row(): result = build_health_glance( - _active_capture(), None, "5m ago", brain=_checking_brain() + _active_capture(), + None, + "5m ago", + brain=_checking_brain(), + backlog=healthy_backlog_source(), ) _assert_status_row( @@ -372,6 +727,7 @@ def test_progressing_blocked_brain_returns_status_only_amber_row(): None, "5m ago", brain=_bundled_runtime_brain(progressing=True), + backlog=healthy_backlog_source(), ) _assert_status_row( @@ -395,7 +751,13 @@ def test_progressing_blocked_brain_returns_status_only_amber_row(): def test_no_observers_with_inflight_brain_returns_brain_status_row( brain, verdict, headline ): - result = build_health_glance(_no_observers_capture(), None, None, brain=brain) + result = build_health_glance( + _no_observers_capture(), + None, + None, + brain=brain, + backlog=healthy_backlog_source(), + ) _assert_status_row(result, verdict=verdict, headline=headline) assert result["verdict"] != "ok" @@ -404,7 +766,13 @@ def test_no_observers_with_inflight_brain_returns_brain_status_row( @pytest.mark.parametrize("brain", [_ready_brain(), None]) def test_no_observers_ready_or_absent_brain_keeps_setup_cta(brain): - result = build_health_glance(_no_observers_capture(), None, None, brain=brain) + result = build_health_glance( + _no_observers_capture(), + None, + None, + brain=brain, + backlog=healthy_backlog_source(), + ) _assert_no_observers_row(result) @@ -423,7 +791,9 @@ def test_no_observers_ready_or_absent_brain_keeps_setup_cta(brain): def test_unavailable_capture_with_inflight_brain_returns_unavailable_row( capture_health, brain ): - result = build_health_glance(capture_health, None, None, brain=brain) + result = build_health_glance( + capture_health, None, None, brain=brain, backlog=healthy_backlog_source() + ) _assert_unavailable_row(result) @@ -434,6 +804,7 @@ def test_bundled_runtime_progressing_suppresses_brain_action(): None, None, brain=_bundled_runtime_brain(progressing=True), + backlog=healthy_backlog_source(), ) _assert_status_row( @@ -449,6 +820,7 @@ def test_bundled_runtime_not_progressing_returns_local_setup_issue(): None, None, brain=_bundled_runtime_brain(progressing=False), + backlog=healthy_backlog_source(), ) _assert_single_brain_issue( @@ -471,7 +843,9 @@ def test_bundled_runtime_not_progressing_returns_local_setup_issue(): ], ) def test_actionable_brain_states_return_single_amber_issue(brain, text, href): - result = build_health_glance(_active_capture(), None, None, brain=brain) + result = build_health_glance( + _active_capture(), None, None, brain=brain, backlog=healthy_backlog_source() + ) _assert_single_brain_issue(result, text=text, href=href) @@ -481,16 +855,26 @@ def test_actionable_brain_states_return_single_amber_issue(brain, text, href): [_checking_brain(), _bundled_runtime_brain(progressing=True)], ) def test_capture_and_pipeline_attention_precede_inflight_brain_status(brain): - red_expected = build_health_glance(_degraded_capture("fedora"), None, None) + red_expected = build_health_glance( + _degraded_capture("fedora"), None, None, backlog=healthy_backlog_source() + ) red_actual = build_health_glance( - _degraded_capture("fedora"), None, None, brain=brain + _degraded_capture("fedora"), + None, + None, + brain=brain, + backlog=healthy_backlog_source(), ) assert red_actual == red_expected pipeline = {"status": "warning", "headline": "processing needs attention"} - amber_expected = build_health_glance(_active_capture(), pipeline, None) - amber_actual = build_health_glance(_active_capture(), pipeline, None, brain=brain) + amber_expected = build_health_glance( + _active_capture(), pipeline, None, backlog=healthy_backlog_source() + ) + amber_actual = build_health_glance( + _active_capture(), pipeline, None, brain=brain, backlog=healthy_backlog_source() + ) assert amber_actual == amber_expected @@ -507,7 +891,9 @@ def test_capture_and_pipeline_attention_precede_inflight_brain_status(brain): ], ) def test_canonical_non_ready_brain_states_do_not_return_ok_green(brain, may_be_green): - result = build_health_glance(_active_capture(), None, "5m ago", brain=brain) + result = build_health_glance( + _active_capture(), None, "5m ago", brain=brain, backlog=healthy_backlog_source() + ) if may_be_green: assert result["verdict"] == "ok" @@ -523,7 +909,9 @@ def test_canonical_non_ready_brain_states_do_not_return_ok_green(brain, may_be_g @pytest.mark.parametrize("brain", [None, "checking"]) def test_absent_or_non_dict_brain_keeps_active_capture_green(brain): - result = build_health_glance(_active_capture(), None, "5m ago", brain=brain) + result = build_health_glance( + _active_capture(), None, "5m ago", brain=brain, backlog=healthy_backlog_source() + ) assert result["verdict"] == "ok" assert result["severity"] == "green" @@ -535,12 +923,20 @@ def test_absent_or_non_dict_brain_keeps_active_capture_green(brain): def test_all_issue_and_cta_hrefs_are_local_paths(): states = [ - build_health_glance(_degraded_capture("fedora"), None, None), - build_health_glance({"status": "offline", "observers": []}, None, None), + build_health_glance( + _degraded_capture("fedora"), None, None, backlog=healthy_backlog_source() + ), + build_health_glance( + {"status": "offline", "observers": []}, + None, + None, + backlog=healthy_backlog_source(), + ), build_health_glance( {"status": "stale", "observers": [{"name": "fedora", "status": "stale"}]}, None, None, + backlog=healthy_backlog_source(), ), build_health_glance( _active_capture(), @@ -550,14 +946,21 @@ def test_all_issue_and_cta_hrefs_are_local_paths(): "suggested_action": "open_support", }, None, + backlog=healthy_backlog_source(), ), build_health_glance( _active_capture(), None, None, brain=_bundled_runtime_brain(progressing=False), + backlog=healthy_backlog_source(), + ), + build_health_glance( + {"status": "no_observers", "observers": []}, + None, + None, + backlog=healthy_backlog_source(), ), - build_health_glance({"status": "no_observers", "observers": []}, None, None), ] for state in states: @@ -570,7 +973,9 @@ def test_all_issue_and_cta_hrefs_are_local_paths(): def test_malformed_pipeline_drops_only_pipeline_issue(): - result = build_health_glance(_degraded_capture("fedora"), "warning", None) + result = build_health_glance( + _degraded_capture("fedora"), "warning", None, backlog=healthy_backlog_source() + ) assert result["verdict"] == "attention" assert result["severity"] == "red" @@ -580,28 +985,62 @@ def test_malformed_pipeline_drops_only_pipeline_issue(): def test_owner_facing_strings_use_allowed_terms(): states = [ - build_health_glance(_degraded_capture("fedora"), None, None), - build_health_glance({"status": "offline", "observers": []}, None, None), + build_health_glance( + _degraded_capture("fedora"), None, None, backlog=healthy_backlog_source() + ), + build_health_glance( + {"status": "offline", "observers": []}, + None, + None, + backlog=healthy_backlog_source(), + ), build_health_glance( {"status": "stale", "observers": [{"name": "fedora", "status": "stale"}]}, None, None, + backlog=healthy_backlog_source(), + ), + build_health_glance( + _active_capture(), None, "5m ago", backlog=healthy_backlog_source() + ), + build_health_glance( + _active_capture(), + None, + None, + brain=_blocked_brain(), + backlog=healthy_backlog_source(), + ), + build_health_glance( + _active_capture(), + None, + None, + brain=_checking_brain(), + backlog=healthy_backlog_source(), ), - build_health_glance(_active_capture(), None, "5m ago"), - build_health_glance(_active_capture(), None, None, brain=_blocked_brain()), - build_health_glance(_active_capture(), None, None, brain=_checking_brain()), build_health_glance( _active_capture(), None, None, brain=_bundled_runtime_brain(progressing=True), + backlog=healthy_backlog_source(), + ), + build_health_glance( + {"status": "no_observers", "observers": []}, + None, + None, + backlog=healthy_backlog_source(), + ), + build_health_glance( + {"status": "unknown", "observers": []}, + None, + None, + backlog=healthy_backlog_source(), ), - build_health_glance({"status": "no_observers", "observers": []}, None, None), - build_health_glance({"status": "unknown", "observers": []}, None, None), build_health_glance( _active_capture(), {"status": "warning", "headline": "processing needs attention"}, None, + backlog=healthy_backlog_source(), ), ] @@ -617,7 +1056,13 @@ def test_owner_facing_strings_use_allowed_terms(): def test_brain_blocked_chip_uses_owner_copy() -> None: - result = build_health_glance(_active_capture(), None, None, brain=_blocked_brain()) + result = build_health_glance( + _active_capture(), + None, + None, + brain=_blocked_brain(), + backlog=healthy_backlog_source(), + ) chip = next( issue for issue in result["issues"] if issue["text"] == HEADLINES["blocked"] ) diff --git a/tests/test_home_routes.py b/tests/test_home_routes.py index 9334f75ea..2d59781e8 100644 --- a/tests/test_home_routes.py +++ b/tests/test_home_routes.py @@ -3,14 +3,16 @@ from __future__ import annotations -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Any import pytest from solstone.convey import create_app +from solstone.convey.backlog_source import BacklogSource from solstone.think.day_accumulator import append_record +from tests.helpers.health_glance import healthy_backlog_source def _patch_minimal_pulse_context( @@ -66,6 +68,11 @@ def _patch_minimal_pulse_context( monkeypatch.setattr(home_routes, "_load_latest_weekly_reflection", lambda: None) monkeypatch.setattr(home_routes, "read_steward_health", lambda: None) monkeypatch.setattr(home_routes, "read_steward_summary", lambda *a, **k: None) + monkeypatch.setattr( + home_routes, + "load_backlog_source", + lambda _journal_root: healthy_backlog_source(), + ) monkeypatch.setattr( home_routes, "build_brain_snapshot", @@ -240,6 +247,47 @@ def test_pulse_and_briefing_needs_dedup_by_shared_source(monkeypatch): assert ctx["briefing_needs_deduped"] == [] +def test_build_pulse_context_routes_loaded_backlog_signal_to_glance(monkeypatch): + home_routes = _patch_minimal_pulse_context( + monkeypatch, + pulse_needs=[], + briefing_needs=[], + ) + source = BacklogSource( + backlog={ + "pending_days": 0, + "stuck_days": 1, + "days": [ + { + "day": "20260720", + "state": "stuck", + "reason_code": "provider_request_rejected", + "provider": "google", + } + ], + "errors": [], + "degraded": False, + }, + validity="valid", + generated_at=datetime.now(timezone.utc).isoformat(), + ) + monkeypatch.setattr( + home_routes, + "load_backlog_source", + lambda _journal_root: source, + ) + + ctx = home_routes._build_pulse_context() + + assert ctx["health_glance"]["verdict"] == "attention" + assert ctx["health_glance"]["severity"] == "red" + assert any( + "the AI provider refused a request sol sent" in issue["text"] + and "Gemini" in issue["text"] + for issue in ctx["health_glance"]["issues"] + ) + + def test_owner_voice_needs_exception_omits_owner_pair_only(monkeypatch): home_routes = _patch_minimal_pulse_context( monkeypatch, diff --git a/tests/test_home_yesterdays_processing.py b/tests/test_home_yesterdays_processing.py index b03bcce63..3068c6350 100644 --- a/tests/test_home_yesterdays_processing.py +++ b/tests/test_home_yesterdays_processing.py @@ -30,6 +30,7 @@ from solstone.apps.home.routes import ( _summarize_yesterday_processing, ) from solstone.think.brain_health import HEADLINES +from tests.helpers.health_glance import healthy_backlog_source FIXTURES = Path(__file__).parent / "fixtures" / "journal" @@ -229,6 +230,10 @@ def _patch_minimal_pulse_context(monkeypatch, pipeline_status): "solstone.apps.home.routes.read_steward_summary", lambda *a, **k: None, ) + monkeypatch.setattr( + "solstone.apps.home.routes.load_backlog_source", + lambda _journal_root: healthy_backlog_source(), + ) monkeypatch.setattr( "solstone.apps.home.routes.build_brain_snapshot", lambda *_a, **_k: _brain_snapshot(), @@ -845,6 +850,10 @@ def test_build_pulse_context_includes_yesterday_processing(monkeypatch): "solstone.apps.home.routes._collect_activities", lambda today: [] ) monkeypatch.setattr("solstone.apps.home.routes.read_steward_health", lambda: None) + monkeypatch.setattr( + "solstone.apps.home.routes.load_backlog_source", + lambda _journal_root: healthy_backlog_source(), + ) monkeypatch.setattr( "solstone.apps.home.routes.build_brain_snapshot", lambda *_a, **_k: _brain_snapshot(), -- 2.51.2