diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md index a58e403de..178f742a0 100644 --- a/docs/PROVIDERS.md +++ b/docs/PROVIDERS.md @@ -152,6 +152,10 @@ recovery action for the active profile. - Missing local runtime, model files, RAM, endpoint readiness, or confidential attestation fails closed rather than falling back to cloud. +Owner-facing brain health and Thinking readiness read canonical evidence from +`health/brain.json`. Confidential SPP egress remains authorized only by the +current process-local attestation state in `spp_transport`. + ## Migration Boundary The Thinking maintenance task collapses legacy `providers.generate` and diff --git a/scripts/check_brain_health_cutover.py b/scripts/check_brain_health_cutover.py index 3e60d39d4..7199ed566 100644 --- a/scripts/check_brain_health_cutover.py +++ b/scripts/check_brain_health_cutover.py @@ -50,6 +50,35 @@ BRAIN_READER_ALLOWLIST = { "solstone/think/surfaces/health.py", "solstone/think/top.py", } +BRAIN_READER_NAMES = { + "inspect_brain_state", + "build_brain_snapshot", + "build_brain_presentation", +} +PROCESS_LOCAL_ATTESTATION_ALLOWLIST = { + "solstone/think/brain_cli.py", + "solstone/think/services/spp_transport.py", + "solstone/think/services/spp.py", + "solstone/observe/transcribe/confidential.py", + "solstone/think/providers/local_endpoint.py", + "solstone/think/providers/state.py", +} +PROCESS_LOCAL_ATTESTATION_MODULES = { + "solstone.think.services.spp": {"get_attestation_state"}, + "solstone.think.providers.local_endpoint": {"probe_local_endpoint"}, + "solstone.think.services.spp_transport": { + "recheck_confidential_attestation", + }, +} +PROCESS_LOCAL_ATTESTATION_IMPORT_MODULES = { + "solstone.think.services": { + "spp": "solstone.think.services.spp", + "spp_transport": "solstone.think.services.spp_transport", + }, + "solstone.think.providers": { + "local_endpoint": "solstone.think.providers.local_endpoint", + }, +} @dataclass(frozen=True) @@ -129,6 +158,72 @@ def _imported_names(path: Path) -> set[str]: return names +def _dotted_name(node: ast.AST) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + parent = _dotted_name(node.value) + if parent: + return f"{parent}.{node.attr}" + return None + + +def _process_local_attestation_calls(path: Path, text: str) -> set[str]: + if path.suffix != ".py": + return set() + try: + tree = ast.parse(text, filename=str(path)) + except SyntaxError: + return set() + + module_aliases: dict[str, str] = {} + function_aliases: dict[str, str] = {} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name in PROCESS_LOCAL_ATTESTATION_MODULES: + local = alias.asname or alias.name.rsplit(".", 1)[-1] + module_aliases[local] = alias.name + elif isinstance(node, ast.ImportFrom): + imported_modules = PROCESS_LOCAL_ATTESTATION_IMPORT_MODULES.get( + node.module or "", + {}, + ) + target_functions = PROCESS_LOCAL_ATTESTATION_MODULES.get( + node.module or "", + set(), + ) + for alias in node.names: + local = alias.asname or alias.name + if alias.name in imported_modules: + module_aliases[local] = imported_modules[alias.name] + if alias.name in target_functions: + function_aliases[local] = f"{node.module}.{alias.name}" + + findings: set[str] = set() + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + dotted = _dotted_name(node.func) + if dotted is None: + continue + if dotted in function_aliases: + findings.add(function_aliases[dotted]) + continue + for module, functions in PROCESS_LOCAL_ATTESTATION_MODULES.items(): + for function in functions: + target = f"{module}.{function}" + if dotted == target: + findings.add(target) + continue + base, _, attr = dotted.rpartition(".") + if attr != function: + continue + if module_aliases.get(base) == module: + findings.add(target) + return findings + + def scan(root: Path, *, all_files: bool = False) -> list[Finding]: findings: list[Finding] = [] for path in _tracked_files(root, all_files=all_files): @@ -161,20 +256,24 @@ def scan(root: Path, *, all_files: bool = False) -> list[Finding]: if rel.startswith(("solstone/", "scripts/")) and not _is_test_path(rel): imported = _imported_names(path) - if {"inspect_brain_state", "build_brain_snapshot"} & imported: + if BRAIN_READER_NAMES & imported: if rel not in BRAIN_READER_ALLOWLIST: findings.append( Finding( rel, "unauthorized-brain-health-reader", - ", ".join( - sorted( - {"inspect_brain_state", "build_brain_snapshot"} - & imported - ) - ), + ", ".join(sorted(BRAIN_READER_NAMES & imported)), ) ) + attestation_calls = _process_local_attestation_calls(path, text) + if attestation_calls and rel not in PROCESS_LOCAL_ATTESTATION_ALLOWLIST: + findings.append( + Finding( + rel, + "unauthorized-process-local-attestation", + ", ".join(sorted(attestation_calls)), + ) + ) return findings diff --git a/solstone/apps/thinking/tests/test_confidential_attestation_payload.py b/solstone/apps/thinking/tests/test_confidential_attestation_payload.py index d92482540..faa1319fd 100644 --- a/solstone/apps/thinking/tests/test_confidential_attestation_payload.py +++ b/solstone/apps/thinking/tests/test_confidential_attestation_payload.py @@ -3,39 +3,55 @@ from __future__ import annotations +import hashlib import json -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone +from typing import Any +import pytest + +from solstone.apps.thinking import routes from solstone.convey import create_app -from solstone.think.services import spp -from solstone.think.services.spp_attest.cadence import AttestationSession -from solstone.think.services.spp_attest.composite import CompositeVerdict -from solstone.think.services.spp_attest.nvgpu.claims import GpuAppraisal -from solstone.think.services.spp_attest.snp import AppraisalStep, CpuAppraisal +from solstone.think import brain_health -ATTESTED_SUBSTRATE = "AMD SEV-SNP + NVIDIA GH100 A01 GSP BROM" -HOSTILE_ARCH = "" +NOW = datetime(2026, 4, 10, 12, 0, tzinfo=timezone.utc) +NOW_ISO = NOW.isoformat() +EXPIRES_ISO = datetime(2026, 4, 10, 13, 0, tzinfo=timezone.utc).isoformat() +_RECORD_DEFAULT = object() -def _client(settings_env, *, confidential: bool = False): +def _client( + settings_env, + *, + confidential: bool = False, + active_provider: str = "google", +): journal_path, config = settings_env() config["setup"] = {"completed_at": 1700000000000} + config.setdefault("providers", {})["active"] = { + "provider": active_provider, + "model": "local/qwen3.5-4b" + if active_provider == "local" + else "gemini-3.5-flash", + } if confidential: + credential = "credential-secret" config.setdefault("services", {})["confidential"] = { "enabled_at": "2026-05-24T00:00:00Z", "account_id": "acct-secret", "endpoint_url": "https://spp.example.test/v1", "served_model_id": "confidential-model", "credential_created_at": "2026-05-24T00:00:00Z", - "credential_fingerprint_sha256": "fingerprint-secret", - "prior_generate_provider": "google", - "prior_cogitate_provider": "openai", + "credential_fingerprint_sha256": hashlib.sha256( + credential.encode("utf-8") + ).hexdigest(), + "prior_active": {"provider": "google", "model": "gemini-3.5-flash"}, "prior_local_endpoint": None, } config.setdefault("providers", {})["local"] = { "endpoint_url": "https://spp.example.test", "served_model_id": "confidential-model", - "credential": "credential-secret", + "credential": credential, } (journal_path / "config" / "journal.json").write_text( json.dumps(config, indent=2) + "\n", @@ -46,54 +62,6 @@ def _client(settings_env, *, confidential: bool = False): return app.test_client() -def _session( - *, - now: datetime, - started_at: datetime, - tpm_heartbeat_at: datetime, - gpu_reattest_at: datetime, - substrate: str = ATTESTED_SUBSTRATE, - arch: str = "HOPPER", -) -> AttestationSession: - cpu = CpuAppraisal( - steps=[AppraisalStep("cpu", "ok", "test")], - hcla_version=2, - report_version=5, - cpuid={"family": 25}, - tcb={}, - pcr_sha256="pcr-machine-secret", - host_data="host-data-secret", - measurement="measurement-secret", - chip_id="chip-machine-secret", - ) - gpu = GpuAppraisal( - steps=[AppraisalStep("gpu", "ok", "test")], - driver_version="595.71.05", - vbios_version="96.00.88.00.11", - hwmodel="GH100 A01 GSP BROM", - ueid="ueid-machine-secret", - oemid="5703", - eat_nonce="eat-nonce-secret", - claims_version="3.0", - arch=arch, - envelope_gpu_uuid="GPU-machine-secret", - ) - verdict = CompositeVerdict( - verified=True, - legs=("cpu", "gpu"), - substrate=substrate, - checked_at=now, - cpu_provenance=cpu, - gpu_provenance=gpu, - ) - return AttestationSession( - verdict=verdict, - started_at=started_at, - tpm_heartbeat_at=tpm_heartbeat_at, - gpu_reattest_at=gpu_reattest_at, - ) - - def _providers(client) -> dict: response = client.get("/app/thinking/api/providers") assert response.status_code == 200 @@ -102,188 +70,377 @@ def _providers(client) -> dict: return payload -def test_active_lane_confidential_attestation_defaults_to_off(settings_env): - client = _client(settings_env) +def _brain_snapshot() -> dict[str, Any]: + return { + "state": "ready", + "headline": "sol can think", + "reason_code": None, + "reason_text": "ok", + "failing_component": None, + "action": None, + "identity": {"lane": "spp", "provider": "local", "model": "local/qwen3.5-4b"}, + "evidence": { + "observed_at": NOW_ISO, + "age_seconds": 0, + "age_text": "0s", + }, + "components": { + "generate": { + "status": "ok", + "reason_code": None, + "reason_text": "ok", + "observed_at": NOW_ISO, + }, + "cogitate": { + "status": "ok", + "reason_code": None, + "reason_text": "ok", + "observed_at": NOW_ISO, + }, + }, + "progressing": False, + } - payload = _providers(client) - assert payload["active_lane"]["confidential_attestation"] == { - "state": "off", - "provenance": None, - "last_verified": None, - "reason": "confidential_not_configured", +def _presentation(attestation: dict[str, Any]) -> dict[str, Any]: + return { + "brain": _brain_snapshot(), + "spp_active": True, + "spp_readiness": { + "generate_ready": True, + "cogitate_ready": True, + "issues": [], + }, + "confidential_attestation": attestation, } -def test_active_lane_confidential_attestation_configured_without_session_verifies( - settings_env, -): - client = _client(settings_env, confidential=True) +def _component(status: str = "ok", reason: str | None = None) -> dict[str, Any]: + component: dict[str, Any] = { + "status": status, + "observed_at": NOW_ISO, + } + if status == "ok": + component["expires_at"] = EXPIRES_ISO + if reason is not None: + component["reason_code"] = reason + return component - payload = _providers(client) - assert payload["active_lane"]["confidential_attestation"] == { - "state": "verifying", - "provenance": None, - "last_verified": None, - "reason": "attestation_not_yet_verified", +def _record( + *, + lane_prerequisites: dict[str, Any] | None = None, + generate: dict[str, Any] | None = None, + cogitate: dict[str, Any] | None = None, +) -> dict[str, Any]: + return { + "schema_version": 1, + "revision": 1, + "aggregate_state": "ready", + "reason_code": None, + "active_lane": "spp", + "active_provider": "local", + "active_model": "local/qwen3.5-4b", + "fingerprint_sha256": "a" * 64, + "checking": None, + "evidence": { + "configuration": _component(), + "lane_prerequisites": lane_prerequisites + if lane_prerequisites is not None + else _component(), + "generate": generate if generate is not None else _component(), + "cogitate": cogitate if cogitate is not None else _component(), + }, + "runtime_failure_marker": None, + "diagnostic": {}, + "updated_at": NOW_ISO, } -def test_active_lane_confidential_attestation_verified_session_serializes_safe_provenance( - settings_env, -): - client = _client(settings_env, confidential=True) - now = datetime.now(timezone.utc) - session = _session( - now=now, - started_at=now - timedelta(minutes=1), - tpm_heartbeat_at=now - timedelta(minutes=1), - gpu_reattest_at=now - timedelta(minutes=1), - ) - spp.record_attestation_verified(session) - - response = client.get("/app/thinking/api/providers") - assert response.status_code == 200 - payload = response.get_json() - attestation = payload["active_lane"]["confidential_attestation"] - - assert attestation == { - "state": "verified", - "provenance": { - "legs": ["cpu", "gpu"], - "substrate": ATTESTED_SUBSTRATE, - "checked_at": now.isoformat(), - }, - "last_verified": { - "legs": ["cpu", "gpu"], - "substrate": ATTESTED_SUBSTRATE, - "checked_at": now.isoformat(), +def _inspection( + *, + aggregate: str = "ready", + reason: str | None = None, + lane: str | None = "spp", + record: dict[str, Any] | None | object = _RECORD_DEFAULT, +) -> dict[str, Any]: + return { + "status": "ok", + "path": "/tmp/brain.json", + "record": _record() if record is _RECORD_DEFAULT else record, + "projection": { + "aggregate_state": aggregate, + "reason_code": reason, + "active_lane": lane, + "active_provider": "local" if lane == "spp" else "google", + "active_model": "local/qwen3.5-4b" if lane == "spp" else "gemini", + "fingerprint_sha256": "a" * 64, + "runtime_transition_in_progress": False, }, - "reason": None, + "reason_code": reason, + "error": None, } - serialized = response.get_data(as_text=True) - for forbidden in { - "chip-machine-secret", - "ueid-machine-secret", - "GPU-machine-secret", - "host-data-secret", - "measurement-secret", - "pcr-machine-secret", - "eat-nonce-secret", - }: - assert forbidden not in serialized - - -def test_active_lane_confidential_attestation_does_not_serialize_unverified_arch( - settings_env, -): - client = _client(settings_env, confidential=True) - now = datetime.now(timezone.utc) - session = _session( - now=now, - started_at=now - timedelta(minutes=1), - tpm_heartbeat_at=now - timedelta(minutes=1), - gpu_reattest_at=now - timedelta(minutes=1), - arch=HOSTILE_ARCH, - ) - spp.record_attestation_verified(session) - response = client.get("/app/thinking/api/providers") - assert response.status_code == 200 - payload = response.get_json() - attestation = payload["active_lane"]["confidential_attestation"] - assert attestation["provenance"]["substrate"] == ATTESTED_SUBSTRATE - assert HOSTILE_ARCH not in response.get_data(as_text=True) +def _build_presentation(monkeypatch, inspection: dict[str, Any], *, configured: bool): + monkeypatch.setattr( + brain_health, "inspect_brain_state", lambda *_a, **_k: inspection + ) + return brain_health.build_brain_presentation( + NOW, + surface="thinking", + spp_configured=configured, + ) -def test_active_lane_confidential_attestation_stale_session(settings_env): - client = _client(settings_env, confidential=True) - now = datetime.now(timezone.utc) - session = _session( - now=now, - started_at=now - timedelta(hours=2), - tpm_heartbeat_at=now, - gpu_reattest_at=now, - ) - spp.record_attestation_verified(session) +def test_active_lane_confidential_attestation_defaults_to_off(settings_env): + client = _client(settings_env) payload = _providers(client) assert payload["active_lane"]["confidential_attestation"] == { - "state": "stale", - "provenance": None, - "last_verified": { - "legs": ["cpu", "gpu"], - "substrate": ATTESTED_SUBSTRATE, - "checked_at": now.isoformat(), - }, - "reason": "attestation_stale", + "state": "off", + "reason": "confidential_not_configured", + "observed_at": None, } -def test_active_lane_confidential_attestation_failed_state(settings_env): +def test_active_lane_confidential_attestation_configured_but_inactive(settings_env): client = _client(settings_env, confidential=True) - spp.record_attestation_failed("failed", "gpu_nonce_mismatch") payload = _providers(client) assert payload["active_lane"]["confidential_attestation"] == { - "state": "failed", - "provenance": None, - "last_verified": None, - "reason": "attestation_failed", + "state": "inactive", + "reason": "confidential_not_active", + "observed_at": None, } -def test_active_lane_confidential_attestation_unreachable_state_preserves_last_verified( +def test_active_lane_confidential_attestation_uses_canonical_presentation( settings_env, + monkeypatch, ): - client = _client(settings_env, confidential=True) - now = datetime.now(timezone.utc) - session = _session( - now=now, - started_at=now - timedelta(minutes=1), - tpm_heartbeat_at=now - timedelta(minutes=1), - gpu_reattest_at=now - timedelta(minutes=1), + client = _client(settings_env, confidential=True, active_provider="local") + attestation = {"state": "verified", "reason": None, "observed_at": NOW_ISO} + monkeypatch.setattr( + routes, + "build_brain_presentation", + lambda *_args, **_kwargs: _presentation(attestation), ) - spp.record_attestation_verified(session) - spp.record_attestation_failed("unreachable", "gateway_unreachable") - payload = _providers(client) + response = client.get("/app/thinking/api/providers") + assert response.status_code == 200 + payload = response.get_json() - assert payload["active_lane"]["confidential_attestation"] == { - "state": "unreachable", - "provenance": None, - "last_verified": { - "legs": ["cpu", "gpu"], - "substrate": ATTESTED_SUBSTRATE, - "checked_at": now.isoformat(), - }, - "reason": "attestation_unreachable", + assert payload["active_lane"]["confidential_attestation"] == attestation + assert set(payload["active_lane"]["confidential_attestation"]) == { + "state", + "reason", + "observed_at", } - - -def test_active_lane_confidential_attestation_missing_provenance_degrades_to_failed( + serialized = response.get_data(as_text=True) + assert "last_verified" not in serialized + + +@pytest.mark.parametrize( + ("attestation", "expected"), + [ + ( + { + "state": "verifying", + "reason": "brain_check_in_progress", + "observed_at": None, + }, + { + "state": "verifying", + "reason": "brain_check_in_progress", + "observed_at": None, + }, + ), + ( + { + "state": "unreachable", + "reason": "attestation_not_verified", + "observed_at": NOW_ISO, + }, + { + "state": "unreachable", + "reason": "attestation_not_verified", + "observed_at": NOW_ISO, + }, + ), + ( + { + "state": "failed", + "reason": "attestation_rejected", + "observed_at": NOW_ISO, + }, + { + "state": "failed", + "reason": "attestation_rejected", + "observed_at": NOW_ISO, + }, + ), + ( + {"state": "stale", "reason": "attestation_expired", "observed_at": NOW_ISO}, + {"state": "stale", "reason": "attestation_expired", "observed_at": NOW_ISO}, + ), + ( + {"state": "stale", "reason": "brain_record_stale", "observed_at": None}, + {"state": "stale", "reason": "brain_record_stale", "observed_at": None}, + ), + ], +) +def test_route_serializes_closed_attestation_view( settings_env, + monkeypatch, + attestation, + expected, ): - client = _client(settings_env, confidential=True) - now = datetime.now(timezone.utc) - session = _session( - now=now, - started_at=now - timedelta(minutes=1), - tpm_heartbeat_at=now - timedelta(minutes=1), - gpu_reattest_at=now - timedelta(minutes=1), - substrate="", + client = _client(settings_env, confidential=True, active_provider="local") + monkeypatch.setattr( + routes, + "build_brain_presentation", + lambda *_args, **_kwargs: _presentation(attestation), ) - spp.record_attestation_verified(session) payload = _providers(client) - assert payload["active_lane"]["confidential_attestation"] == { - "state": "failed", - "provenance": None, - "last_verified": None, - "reason": "attestation_failed", - } + assert payload["active_lane"]["confidential_attestation"] == expected + + +@pytest.mark.parametrize( + ("inspection", "configured", "expected"), + [ + ( + _inspection(lane="byo-cloud"), + False, + { + "state": "off", + "reason": "confidential_not_configured", + "observed_at": None, + }, + ), + ( + _inspection(lane="byo-cloud"), + True, + { + "state": "inactive", + "reason": "confidential_not_active", + "observed_at": None, + }, + ), + ( + _inspection(aggregate="checking", reason="brain_check_in_progress"), + True, + { + "state": "verifying", + "reason": "brain_check_in_progress", + "observed_at": None, + }, + ), + ( + _inspection( + aggregate="unhealthy", + reason="provider_unavailable", + record=_record(generate=_component("failed", "provider_unavailable")), + ), + True, + {"state": "verified", "reason": None, "observed_at": NOW_ISO}, + ), + ( + _inspection( + aggregate="blocked", + reason="attestation_not_verified", + record=_record( + lane_prerequisites=_component( + "blocked", + "attestation_not_verified", + ), + generate=_component("not_attempted", "attestation_not_verified"), + cogitate=_component("not_attempted", "attestation_not_verified"), + ), + ), + True, + { + "state": "unreachable", + "reason": "attestation_not_verified", + "observed_at": NOW_ISO, + }, + ), + ( + _inspection( + aggregate="unhealthy", + reason="attestation_rejected", + record=_record( + lane_prerequisites=_component("failed", "attestation_rejected"), + generate=_component("not_attempted", "attestation_rejected"), + cogitate=_component("not_attempted", "attestation_rejected"), + ), + ), + True, + { + "state": "failed", + "reason": "attestation_rejected", + "observed_at": NOW_ISO, + }, + ), + ( + _inspection( + aggregate="unhealthy", + reason="attestation_expired", + record=_record( + lane_prerequisites=_component("failed", "attestation_expired"), + generate=_component("not_attempted", "attestation_expired"), + cogitate=_component("not_attempted", "attestation_expired"), + ), + ), + True, + {"state": "stale", "reason": "attestation_expired", "observed_at": NOW_ISO}, + ), + ( + _inspection( + aggregate="unknown", + reason="brain_record_missing", + record=None, + ), + True, + {"state": "stale", "reason": "brain_record_missing", "observed_at": None}, + ), + ], +) +def test_build_brain_presentation_maps_confidential_attestation( + monkeypatch, + inspection, + configured, + expected, +): + presentation = _build_presentation(monkeypatch, inspection, configured=configured) + + assert presentation["confidential_attestation"] == expected + + +@pytest.mark.parametrize( + "inspection", + [ + _inspection(aggregate="checking", reason="brain_check_in_progress"), + _inspection(aggregate="unknown", reason="brain_record_missing", record=None), + _inspection( + aggregate="ready", + reason=None, + record=_record(generate=_component("failed")), + ), + _inspection( + aggregate="unknown", + reason=None, + record=None, + ), + ], +) +def test_spp_readiness_not_ready_always_has_issue(monkeypatch, inspection): + presentation = _build_presentation(monkeypatch, inspection, configured=True) + readiness = presentation["spp_readiness"] + + if not (readiness["generate_ready"] and readiness["cogitate_ready"]): + assert readiness["issues"] diff --git a/solstone/apps/thinking/tests/test_confidential_routes.py b/solstone/apps/thinking/tests/test_confidential_routes.py index 364086fe6..45e7043ac 100644 --- a/solstone/apps/thinking/tests/test_confidential_routes.py +++ b/solstone/apps/thinking/tests/test_confidential_routes.py @@ -12,6 +12,7 @@ from unittest.mock import Mock import pytest +from solstone.apps.thinking import routes from solstone.apps.thinking.google_model_pins import ( GOOGLE_MODEL_RESOLUTION_TARGETS_FIELD, GOOGLE_PRO_ALIAS, @@ -200,10 +201,9 @@ def test_enable_confidential_returns_operation_and_lands_not_verified( assert payload["active_lane"]["confidential_provenance_configured"] is True assert payload["active_lane"]["confidential_operation"]["phase"] == "not_verified" assert payload["active_lane"]["confidential_attestation"] == { - "state": "verifying", - "provenance": None, - "last_verified": None, - "reason": "attestation_not_yet_verified", + "state": "stale", + "reason": "brain_record_missing", + "observed_at": None, } @@ -232,9 +232,8 @@ def test_enable_confidential_early_access_stays_off( assert payload["active_lane"]["confidential_provenance_configured"] is False assert payload["active_lane"]["confidential_attestation"] == { "state": "off", - "provenance": None, - "last_verified": None, "reason": "confidential_not_configured", + "observed_at": None, } @@ -669,35 +668,107 @@ def test_disable_confidential_restores_synchronously( assert spp_transport._CONFIDENTIAL_BLOCK == provenance -def test_recheck_confidential_rejects_when_off(thinking_client) -> None: +def test_recheck_confidential_rejects_when_off( + thinking_client, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + routes, + "request_brain_refresh", + Mock(side_effect=AssertionError("brain refresh attempted")), + ) response = thinking_client.post("/app/thinking/api/confidential/recheck") assert response.status_code == 400 assert response.get_json()["reason_code"] == "invalid_operation_for_state" -def test_recheck_confidential_returns_refreshed_provider_state( +def test_recheck_confidential_rejects_when_inactive_before_refresh( thinking_client, monkeypatch: pytest.MonkeyPatch, ) -> None: - spp.provision_confidential_handoff(_payload("recheck")) + monkeypatch.setattr( + routes, + "build_brain_presentation", + lambda *_a, **_k: { + "brain": {}, + "spp_active": False, + "spp_readiness": { + "generate_ready": False, + "cogitate_ready": False, + "issues": ["brain_record_missing"], + }, + "confidential_attestation": { + "state": "inactive", + "reason": "confidential_not_active", + "observed_at": None, + }, + }, + ) + monkeypatch.setattr( + routes, + "request_brain_refresh", + Mock(side_effect=AssertionError("brain refresh attempted")), + ) - def recheck() -> None: - spp.record_attestation_failed("failed", "gpu_nonce_mismatch") + response = thinking_client.post("/app/thinking/api/confidential/recheck") - monkeypatch.setattr(spp_transport, "recheck_confidential_attestation", recheck) + assert response.status_code == 400 + assert response.get_json()["reason_code"] == "invalid_operation_for_state" + + +def test_recheck_confidential_returns_brain_check_response( + thinking_client, + monkeypatch: pytest.MonkeyPatch, +) -> None: + spp.provision_confidential_handoff(_payload("recheck")) + brain = { + "state": "checking", + "headline": "checking how sol thinks", + "reason_code": "brain_check_in_progress", + "reason_text": "brain check in progress", + "failing_component": "configuration", + "action": None, + "identity": { + "lane": "spp", + "provider": "local", + "model": "local/qwen3.5-4b", + }, + "evidence": {"observed_at": None, "age_seconds": None, "age_text": None}, + "components": { + "generate": { + "status": None, + "reason_code": None, + "reason_text": "unknown", + "observed_at": None, + }, + "cogitate": { + "status": None, + "reason_code": None, + "reason_text": "unknown", + "observed_at": None, + }, + }, + "progressing": True, + } + refresh_surfaces: list[str] = [] + monkeypatch.setattr( + routes, + "request_brain_refresh", + lambda *, surface: refresh_surfaces.append(surface) or True, + ) + monkeypatch.setattr(routes, "build_brain_snapshot", lambda *_a, **_k: brain) + monkeypatch.setattr( + spp_transport, + "recheck_confidential_attestation", + Mock(side_effect=AssertionError("legacy recheck attempted")), + ) response = thinking_client.post("/app/thinking/api/confidential/recheck") assert response.status_code == 200 - payload = response.get_json() - assert payload["active_lane"]["confidential_enabled"] is True - assert payload["active_lane"]["confidential_attestation"] == { - "state": "failed", - "provenance": None, - "last_verified": None, - "reason": "attestation_failed", - } + assert response.get_json() == {"ok": True, "brain": brain} + assert refresh_surfaces == ["thinking"] def test_confidential_routes_and_provider_payload_are_secret_free( diff --git a/solstone/apps/thinking/tests/test_providers_payload_extended.py b/solstone/apps/thinking/tests/test_providers_payload_extended.py index 5115779f0..bec816f84 100644 --- a/solstone/apps/thinking/tests/test_providers_payload_extended.py +++ b/solstone/apps/thinking/tests/test_providers_payload_extended.py @@ -155,11 +155,36 @@ def _brain_payload( } +def _presentation( + snapshot: dict[str, object] | None = None, + *, + spp_active: bool = False, + spp_readiness: dict[str, object] | None = None, + confidential_attestation: dict[str, object] | None = None, +) -> dict[str, object]: + return { + "brain": snapshot or _brain_payload(), + "spp_active": spp_active, + "spp_readiness": spp_readiness + or { + "generate_ready": False, + "cogitate_ready": False, + "issues": ["brain_record_missing"], + }, + "confidential_attestation": confidential_attestation + or { + "state": "off", + "reason": "confidential_not_configured", + "observed_at": None, + }, + } + + def _patch_brain(monkeypatch, snapshot: dict[str, object] | None = None) -> None: monkeypatch.setattr( routes, - "build_brain_snapshot", - lambda *_args, **_kwargs: snapshot or _brain_payload(), + "build_brain_presentation", + lambda *_args, **_kwargs: _presentation(snapshot), ) @@ -501,10 +526,9 @@ def test_thinking_status_payloads_are_secret_free_with_scout_provenance( "confidential_provenance_configured": True, "confidential_operation": None, "confidential_attestation": { - "state": "verifying", - "provenance": None, - "last_verified": None, - "reason": "attestation_not_yet_verified", + "state": "inactive", + "reason": "confidential_not_active", + "observed_at": None, }, } @@ -1246,6 +1270,59 @@ def test_get_providers_uses_state_local_status(settings_client, monkeypatch): assert payload["provider_status"]["local"] == sentinel +def test_get_providers_uses_canonical_spp_local_status_without_probe( + settings_client, + monkeypatch, +): + spp_readiness = { + "generate_ready": True, + "cogitate_ready": False, + "issues": ["cogitate_terminal_error"], + } + monkeypatch.setattr( + routes, + "build_brain_presentation", + lambda *_args, **_kwargs: _presentation( + spp_active=True, + spp_readiness=spp_readiness, + confidential_attestation={ + "state": "verified", + "reason": None, + "observed_at": "2026-04-10T12:00:00Z", + }, + ), + ) + + def fail(*_args, **_kwargs): + raise AssertionError("process-local readiness path called") + + monkeypatch.setattr("solstone.think.providers.state.local_status_dict", fail) + monkeypatch.setattr( + "solstone.think.providers.local_endpoint.probe_local_endpoint", + fail, + ) + monkeypatch.setattr("solstone.think.services.spp.get_attestation_state", fail) + monkeypatch.setattr( + "solstone.think.services.spp_transport.recheck_confidential_attestation", + fail, + ) + + providers_response = settings_client.get("/app/thinking/api/providers") + local_response = settings_client.get("/app/thinking/api/providers/local/status") + + assert providers_response.status_code == 200 + assert local_response.status_code == 200 + expected = { + "configured": True, + "selected": True, + "generate_ready": True, + "cogitate_ready": False, + "issues": ["cogitate_terminal_error"], + } + assert providers_response.get_json()["provider_status"]["local"] == expected + assert local_response.get_json() == expected + + def test_get_providers_brain_shape(settings_client, monkeypatch): _patch_brain(monkeypatch) diff --git a/solstone/apps/thinking/tests/test_thinking_confidential_audio_js.py b/solstone/apps/thinking/tests/test_thinking_confidential_audio_js.py index 34428ff01..fedde1fc3 100644 --- a/solstone/apps/thinking/tests/test_thinking_confidential_audio_js.py +++ b/solstone/apps/thinking/tests/test_thinking_confidential_audio_js.py @@ -50,7 +50,7 @@ def test_confidential_audio_helpers_cover_state_matrix() -> None: """ const confidentialCopy = copy.confidential; const beats = confidentialCopy.setup.trust_beats; -const states = ['off', 'verifying', 'verified', 'failed', 'stale', 'unreachable']; +const states = ['off', 'inactive', 'verifying', 'verified', 'failed', 'stale', 'unreachable']; const deferralStates = new Set(['verifying', 'failed', 'stale', 'unreachable']); assert(confidentialAudioSetting({}) === true, 'absent setting defaults on'); diff --git a/solstone/apps/thinking/tests/test_thinking_confidential_poll_js.py b/solstone/apps/thinking/tests/test_thinking_confidential_poll_js.py index 7e4056a34..ac70012e8 100644 --- a/solstone/apps/thinking/tests/test_thinking_confidential_poll_js.py +++ b/solstone/apps/thinking/tests/test_thinking_confidential_poll_js.py @@ -26,8 +26,6 @@ def _node_script(body: str) -> str: extract_js_function(source, "formatCopy"), extract_js_function(source, "confidentialOperationIsTerminal"), extract_js_function(source, "confidentialOperationRender"), - extract_js_function(source, "confidentialLegsLabel"), - extract_js_function(source, "confidentialVerifiedValues"), extract_js_function(source, "confidentialSetupMetaLine"), extract_js_function(source, "confidentialNoticeLine"), extract_js_function(source, "confidentialSetupOperationLines"), @@ -36,6 +34,8 @@ def _node_script(body: str) -> str: extract_js_function(source, "confidentialGlanceForAttestation"), extract_js_function(source, "pollConfidentialUntilTerminal"), extract_js_function(source, "handleConfidentialPollError"), + extract_js_function(source, "requestBrainCheck"), + extract_js_function(source, "recheckConfidential"), "function assert(condition, message) { if (!condition) throw new Error(message); }", f"const copy = {json.dumps(thinking_copy.thinking_copy_payload())};", body, @@ -61,19 +61,21 @@ def test_confidential_render_mappings_are_pure() -> None: _node_script( """ const confidentialCopy = copy.confidential; -const completeProvenance = { - legs: ['cpu', 'gpu'], - substrate: 'AMD SEV-SNP + NVIDIA GH100 A01 GSP BROM', - checked_at: '2026-07-12T12:00:00+00:00', -}; -const verifiedLine = 'CPU + GPU · AMD SEV-SNP + NVIDIA GH100 A01 GSP BROM · checked just now'; +const verifiedLine = 'confidential hardware verified · checked just now'; const verified = confidentialAttestationRender({ state: 'verified', - provenance: completeProvenance, + observed_at: '2026-07-12T12:00:00+00:00', }, confidentialCopy, 'just now'); assert(verified.pill === 'active', 'verified pill'); assert(verified.tone === 'hot', 'verified tone'); assert(verified.message === verifiedLine, 'verified message'); +assert(verified.recheck === false, 'verified no recheck'); + +const inactive = confidentialAttestationRender({state: 'inactive'}, confidentialCopy); +assert(inactive.pill === 'available', 'inactive pill'); +assert(inactive.tone === '', 'inactive tone'); +assert(inactive.recheck === false, 'inactive no recheck'); +assert(inactive.message === 'confidential processing is available.', 'inactive message'); const failed = confidentialAttestationRender({state: 'failed'}, confidentialCopy); assert(failed.tone === 'bad', 'failed tone'); @@ -87,12 +89,15 @@ assert(early.message === 'confidential processing is coming — scouts get it fi assert(confidentialOperationIsTerminal({phase: 'early_access'}), 'early access terminal'); const glance = confidentialGlanceForAttestation( - {state: 'verified', provenance: completeProvenance}, + {state: 'verified', observed_at: '2026-07-12T12:00:00+00:00'}, copy, 'just now', ); assert(glance.label === 'sol is thinking with', 'verified glance label'); assert(glance.detail === verifiedLine, 'verified glance detail'); +const available = confidentialGlanceForAttestation({state: 'inactive'}, copy); +assert(available.label === 'available', 'inactive glance label'); +assert(available.detail === 'confidential processing is available.', 'inactive glance detail'); const blocked = confidentialGlanceForAttestation({state: 'unreachable'}, copy); assert(blocked.label === 'sol is holding', 'blocked glance label'); assert(blocked.detail === "can't reach confidential processing right now — sol isn't sending.", 'blocked glance detail'); @@ -107,14 +112,8 @@ def test_confidential_verified_render_and_setup_lines() -> None: _node_script( """ const confidentialCopy = copy.confidential; -const substrate = 'AMD SEV-SNP + NVIDIA GH100 A01 GSP BROM'; -const completeProvenance = { - legs: ['cpu', 'gpu'], - substrate, - checked_at: '2026-07-12T12:00:00+00:00', -}; -const expectedLine = `CPU + GPU · ${substrate} · checked 1 min ago`; -const verifiedAttestation = {state: 'verified', provenance: completeProvenance}; +const expectedLine = 'confidential hardware verified · checked 1 min ago'; +const verifiedAttestation = {state: 'verified', observed_at: '2026-07-12T12:00:00+00:00'}; const verified = confidentialAttestationRender(verifiedAttestation, confidentialCopy, '1 min ago'); const verifiedGlance = confidentialGlanceForAttestation(verifiedAttestation, copy, '1 min ago'); assert(verified.message === expectedLine, 'verified setup and lane-card line'); @@ -122,7 +121,7 @@ assert(verifiedGlance.detail === expectedLine, 'verified glance line'); assert(verified.message === verifiedGlance.detail, 'verified surfaces share formatter'); assert(confidentialSetupMetaLine(verifiedAttestation, 'just now') === '', 'verified meta hidden'); -for (const state of ['verifying', 'off', '']) { +for (const state of ['inactive', 'verifying', 'off', '']) { assert(confidentialSetupMetaLine({state}, 'just now') === '', `${state || 'empty'} meta hidden`); } @@ -177,20 +176,6 @@ for (const [phase, message] of [ assert(lines.notice.hidden === true, `${phase} notice hidden`); } -for (const provenance of [ - {substrate, checked_at: '2026-07-12T12:00:00+00:00'}, - {legs: ['cpu', 'gpu'], substrate: ' ', checked_at: '2026-07-12T12:00:00+00:00'}, - completeProvenance, -]) { - const checkedLabel = provenance === completeProvenance ? '' : 'just now'; - const incomplete = {state: 'verified', provenance}; - const rendered = confidentialAttestationRender(incomplete, confidentialCopy, checkedLabel); - const glance = confidentialGlanceForAttestation(incomplete, copy, checkedLabel); - assert(rendered.pill === 'off', 'incomplete verified pill'); - assert(rendered.tone === '', 'incomplete verified tone'); - assert(rendered.message === '', 'incomplete verified message'); - assert(glance.detail === '', 'incomplete verified glance detail'); -} console.log('PASS'); """ ) @@ -237,6 +222,50 @@ main().catch((error) => { console.error(error.stack || error); process.exit(1); ) +def test_confidential_recheck_posts_brain_check_then_rereads_providers() -> None: + _run_node( + _node_script( + """ +const calls = []; +const messages = []; +const state = {providers: {brain: {state: 'unhealthy'}, sentinel: true}}; +async function api(path, options) { + calls.push({path, options}); + assert(path === 'api/brain/check', 'posts brain check'); + return {ok: true, brain: {state: 'checking'}}; +} +function renderGlance() { + calls.push({path: 'renderGlance'}); +} +async function refreshProviders() { + assert(state.providers.sentinel === true, 'post did not replace providers'); + assert(state.providers.brain.state === 'checking', 'brain response merged first'); + calls.push({path: 'refreshProviders'}); + state.providers = {refreshed: true}; +} +function setMessage(id, message, tone = '') { + messages.push({id, message, tone}); +} + +async function main() { + await recheckConfidential(); + + assert(messages.length === 1, 'message cleared once'); + assert(messages[0].message === '', 'no optimistic verifying paint'); + assert(JSON.stringify(calls.map((call) => call.path)) === JSON.stringify([ + 'api/brain/check', + 'renderGlance', + 'refreshProviders', + ]), 'call order'); + assert(state.providers.refreshed === true, 'providers reread updates card state'); + console.log('PASS'); +} +main().catch((error) => { console.error(error.stack || error); process.exit(1); }); +""" + ) + ) + + def test_confidential_poll_timeout_and_error_handler_clear_current_generation() -> None: _run_node( _node_script( diff --git a/solstone/apps/thinking/tests/test_workspace_html.py b/solstone/apps/thinking/tests/test_workspace_html.py index 6fb9f1273..fcbc1af3c 100644 --- a/solstone/apps/thinking/tests/test_workspace_html.py +++ b/solstone/apps/thinking/tests/test_workspace_html.py @@ -188,7 +188,7 @@ def test_confidential_live_static_behavior_is_wired() -> None: assert 'role="button"' in card.group(1) assert 'tabindex="0"' in card.group(1) assert "api/confidential/enable" in js - assert "api/confidential/recheck" in js + assert "api/brain/check" in js assert "api/confidential/disable" in js assert "function openConsentTab(operation)" in js assert "confidentialProvenancePresent" in js @@ -263,7 +263,12 @@ def test_thinking_deck_copy_constants() -> None: "confidential_verified": { "label": "sol is thinking with", "value": "confidential processing", - "detail": "{legs} · {substrate} · checked {checked}", + "detail": "confidential hardware verified · checked {checked}", + }, + "confidential_available": { + "label": "available", + "value": "confidential processing", + "detail": "confidential processing is available.", }, "confidential_blocked": { "label": "sol is holding", @@ -460,8 +465,9 @@ def test_thinking_deck_copy_constants() -> None: } assert thinking_copy.CONFIDENTIAL_ATTESTATION_STATES == { "off": "", + "inactive": "confidential processing is available.", "verifying": "checking the hardware…", - "verified": "{legs} · {substrate} · checked {checked}", + "verified": "confidential hardware verified · checked {checked}", "failed": "couldn't verify the service — sol isn't sending.", "stale": "your journal needs to re-check the service before sending.", "unreachable": "can't reach confidential processing right now — sol isn't sending.", diff --git a/tests/baselines/api/thinking/providers.json b/tests/baselines/api/thinking/providers.json index 0c38b0624..9839d9746 100644 --- a/tests/baselines/api/thinking/providers.json +++ b/tests/baselines/api/thinking/providers.json @@ -5,8 +5,7 @@ }, "active_lane": { "confidential_attestation": { - "last_verified": null, - "provenance": null, + "observed_at": null, "reason": "confidential_not_configured", "state": "off" }, diff --git a/tests/test_brain_health_cutover_parity.py b/tests/test_brain_health_cutover_parity.py index bb9e1b77b..18a521bcd 100644 --- a/tests/test_brain_health_cutover_parity.py +++ b/tests/test_brain_health_cutover_parity.py @@ -245,8 +245,21 @@ def test_state_parity_matrix(monkeypatch, capsys): monkeypatch.setattr(thinking_routes.local_bootstrap, "get_state", lambda _m: {}) monkeypatch.setattr( thinking_routes, - "build_brain_snapshot", - lambda *_a, **_k: thinking_brain, + "build_brain_presentation", + lambda *_a, **_k: { + "brain": thinking_brain, + "spp_active": False, + "spp_readiness": { + "generate_ready": False, + "cogitate_ready": False, + "issues": ["brain_record_missing"], + }, + "confidential_attestation": { + "state": "off", + "reason": "confidential_not_configured", + "observed_at": None, + }, + }, ) thinking_payload = thinking_routes._provider_payload({}, "local/model") assert thinking_payload["brain"]["headline"] == HEADLINES[case.state] diff --git a/tests/test_check_brain_health_cutover.py b/tests/test_check_brain_health_cutover.py index 7664e4c71..cfea7ceab 100644 --- a/tests/test_check_brain_health_cutover.py +++ b/tests/test_check_brain_health_cutover.py @@ -67,7 +67,7 @@ def test_guard_flags_unauthorized_brain_reader_import(tmp_path, capsys) -> None: _write( tmp_path, "solstone/think/work.py", - "from solstone.think.brain_health import build_brain_snapshot\n", + "from solstone.think.brain_health import build_brain_presentation\n", ) assert _run(tmp_path) == 1 @@ -82,3 +82,68 @@ def test_guard_allows_declared_brain_reader_import(tmp_path) -> None: ) assert _run(tmp_path) == 0 + + +def test_guard_flags_process_local_attestation_calls_with_aliases( + tmp_path, + capsys, +) -> None: + _write( + tmp_path, + "solstone/apps/thinking/routes.py", + "\n".join( + [ + "from solstone.think.services import spp as service_spp", + "from solstone.think.providers.local_endpoint import probe_local_endpoint as probe", + "from solstone.think.services.spp_transport import recheck_confidential_attestation as recheck", + "def bad(endpoint):", + " service_spp.get_attestation_state()", + " probe(endpoint)", + " recheck()", + ] + ), + ) + + assert _run(tmp_path) == 1 + output = capsys.readouterr().out + assert "unauthorized-process-local-attestation" in output + assert "solstone.think.services.spp.get_attestation_state" in output + assert "solstone.think.providers.local_endpoint.probe_local_endpoint" in output + assert ( + "solstone.think.services.spp_transport.recheck_confidential_attestation" + in output + ) + + +def test_guard_allows_declared_process_local_attestation_callers(tmp_path) -> None: + _write( + tmp_path, + "solstone/think/brain_cli.py", + "\n".join( + [ + "from solstone.think.services import spp", + "from solstone.think.services.spp_transport import recheck_confidential_attestation", + "def refresh():", + " recheck_confidential_attestation()", + " spp.get_attestation_state()", + ] + ), + ) + _write( + tmp_path, + "solstone/think/providers/state.py", + "\n".join( + [ + "from solstone.think.providers.local_endpoint import probe_local_endpoint", + "def status(endpoint):", + " return probe_local_endpoint(endpoint)", + ] + ), + ) + _write( + tmp_path, + "tests/test_bad.py", + "from solstone.think.services import spp\nspp.get_attestation_state()\n", + ) + + assert _run(tmp_path) == 0 diff --git a/tests/test_no_implicit_cloud.py b/tests/test_no_implicit_cloud.py index 5204e982f..e574fba8b 100644 --- a/tests/test_no_implicit_cloud.py +++ b/tests/test_no_implicit_cloud.py @@ -5,6 +5,7 @@ from __future__ import annotations import ast import asyncio +import hashlib import importlib import time from datetime import datetime, timedelta, timezone @@ -409,6 +410,63 @@ def test_confidential_generate_stops_before_any_provider_dispatch( assert establish.call_count == 3 +def test_persisted_spp_brain_evidence_never_authorizes_generate_egress( + tmp_path, + monkeypatch, +): + from solstone.think.providers.brain_state import ( + begin_brain_refresh, + finish_brain_refresh, + ) + from solstone.think.services import spp + + _empty_journal(tmp_path, monkeypatch) + config = _confidential_config() + config["providers"] = { + "active": {"provider": "local", "model": LOCAL_MODEL}, + } + _add_local_endpoint(config) + config["services"]["confidential"]["credential_fingerprint_sha256"] = ( + hashlib.sha256(b"confidential-credential").hexdigest() + ) + _seed_journal_config(tmp_path, config) + + now = datetime.now(timezone.utc) + permit = begin_brain_refresh(now, journal_path=tmp_path) + assert permit is not None + component = { + "status": "ok", + "observed_at": now.isoformat(), + "expires_at": (now + timedelta(hours=1)).isoformat(), + } + finish_brain_refresh( + permit, + { + "configuration": component, + "lane_prerequisites": component, + "generate": component, + "cogitate": component, + }, + now, + journal_path=tmp_path, + ) + spp.delete_attestation_state() + + establish = _install_failing_confidential_transport(monkeypatch) + httpx_post = Mock(side_effect=AssertionError("local endpoint call attempted")) + httpx_get = Mock(side_effect=AssertionError("endpoint probe attempted")) + monkeypatch.setattr("httpx.post", httpx_post) + monkeypatch.setattr("httpx.get", httpx_get) + + with pytest.raises(AttestationFailedError) as generate_exc: + models.generate("hello", "any.context") + + _assert_attestation_failed(generate_exc.value) + httpx_post.assert_not_called() + httpx_get.assert_not_called() + establish.assert_called_once() + + def test_confidential_cogitate_stops_before_any_provider_dispatch( tmp_path, monkeypatch, diff --git a/tests/test_providers.py b/tests/test_providers.py index 7fe41e4a9..f6734ae24 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -69,6 +69,27 @@ def test_cloud_provider_status_never_carries_install_gate( ) +def test_local_provider_status_accepts_pre_resolved_row(monkeypatch) -> None: + sentinel = { + "configured": True, + "selected": True, + "generate_ready": False, + "cogitate_ready": True, + "issues": ["provider_unavailable"], + } + monkeypatch.setattr( + "solstone.think.providers.state.local_status_dict", + lambda: (_ for _ in ()).throw(AssertionError("local status probed")), + ) + + status = build_provider_status( + [{"name": "local", "label": "Local", "env_key": ""}], + local_status=sentinel, + ) + + assert status["local"] == sentinel + + def test_inert_upgrade_path_for_stale_bundled_config( tmp_path, monkeypatch, caplog ) -> None: diff --git a/tests/test_thinking_call_parity.py b/tests/test_thinking_call_parity.py index a9de2a2ab..d1e46f06d 100644 --- a/tests/test_thinking_call_parity.py +++ b/tests/test_thinking_call_parity.py @@ -136,9 +136,8 @@ def test_show_verbs_select_http_fields() -> None: assert providers_payload["active_lane"]["lane"] == "byo" assert providers_payload["active_lane"]["confidential_attestation"] == { "state": "off", - "provenance": None, - "last_verified": None, "reason": "confidential_not_configured", + "observed_at": None, }