diff --git a/docs/openapi/observer-client-contract/consumer-audit.json b/docs/openapi/observer-client-contract/consumer-audit.json new file mode 100644 index 000000000..5ca05a3ea --- /dev/null +++ b/docs/openapi/observer-client-contract/consumer-audit.json @@ -0,0 +1,276 @@ +{ + "audited_commits": [ + { + "commit": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + "consumer": "solstone-windows" + }, + { + "commit": "1c679db1ce6f9a65db70c5aae0ca2fad677416ef", + "consumer": "solstone-linux" + }, + { + "commit": "998c1095cd8f766dd188bece5ad6527444f8dfac", + "consumer": "solstone-browser" + } + ], + "direct_paths": [ + { + "classification": "bundled", + "consumer": "solstone-browser", + "path": "/app/observer/register", + "rationale": "Projected as observer.register for browser observer enrollment.", + "revision": "998c1095cd8f766dd188bece5ad6527444f8dfac", + "source_files": [ + "extension/journal.js" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-browser", + "path": "/app/observer/ingest", + "rationale": "Projected as observer.ingestUpload for browser upload.", + "revision": "998c1095cd8f766dd188bece5ad6527444f8dfac", + "source_files": [ + "extension/journal.js" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-browser", + "path": "/app/observer/ingest/event", + "rationale": "Projected as observer.ingestEvent for browser event relay.", + "revision": "998c1095cd8f766dd188bece5ad6527444f8dfac", + "source_files": [ + "extension/journal.js" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-browser", + "path": "/app/observer/ingest/segments/{day}", + "rationale": "Projected as observer.ingestSegments; browser reconcile consumes the legacy v1 array variant.", + "revision": "998c1095cd8f766dd188bece5ad6527444f8dfac", + "source_files": [ + "extension/journal.js" + ] + }, + { + "classification": "relay_session_control_excluded_from_journal_projection", + "consumer": "solstone-browser", + "path": "/enroll/device", + "rationale": "Relay enrollment/session-control path, not journal-owned API.", + "revision": "998c1095cd8f766dd188bece5ad6527444f8dfac", + "source_files": [ + "extension/journal.js" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-linux", + "path": "/app/observer/register", + "rationale": "Projected as observer.register for Linux observer enrollment.", + "revision": "1c679db1ce6f9a65db70c5aae0ca2fad677416ef", + "source_files": [ + "crates/solstone-linux/src/upload.rs" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-linux", + "path": "/app/observer/ingest", + "rationale": "Projected as observer.ingestUpload for Linux segment upload.", + "revision": "1c679db1ce6f9a65db70c5aae0ca2fad677416ef", + "source_files": [ + "crates/solstone-linux/src/upload.rs" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-linux", + "path": "/app/observer/ingest/event", + "rationale": "Projected as observer.ingestEvent for Linux event relay.", + "revision": "1c679db1ce6f9a65db70c5aae0ca2fad677416ef", + "source_files": [ + "crates/solstone-linux/src/upload.rs" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-linux", + "path": "/app/observer/ingest/segments/{day}", + "rationale": "Projected as observer.ingestSegments for Linux reconciliation.", + "revision": "1c679db1ce6f9a65db70c5aae0ca2fad677416ef", + "source_files": [ + "crates/solstone-linux/src/upload.rs" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-linux", + "path": "/app/observer/callosum", + "rationale": "Projected as observer.callosumStream for Linux chat bridge SSE.", + "revision": "1c679db1ce6f9a65db70c5aae0ca2fad677416ef", + "source_files": [ + "crates/solstone-linux/src/chat_bridge.rs" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-linux", + "path": "/api/chat/sol_chat_request/open", + "rationale": "Projected as chat.openSolChatRequest for Linux chat bridge.", + "revision": "1c679db1ce6f9a65db70c5aae0ca2fad677416ef", + "source_files": [ + "crates/solstone-linux/src/chat_bridge.rs" + ] + }, + { + "classification": "browser_navigation_excluded_from_api_projection", + "consumer": "solstone-linux", + "path": "/app/chat/{day}#event-{index}", + "rationale": "Browser navigation anchor, not an HTTP API contract path.", + "revision": "1c679db1ce6f9a65db70c5aae0ca2fad677416ef", + "source_files": [ + "crates/solstone-linux/src/chat_bridge.rs" + ] + }, + { + "classification": "consumer_drift_adoption_blocker", + "consumer": "solstone-linux", + "path": "/api/sol_voice", + "rationale": "Consumer uses the old settings path and reads linux_notify_send; journal serves /app/settings/api/sol_voice and nests the Linux boolean under system_notifications.linux.", + "revision": "1c679db1ce6f9a65db70c5aae0ca2fad677416ef", + "source_files": [ + "crates/solstone-linux/src/chat_bridge.rs" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-windows", + "path": "/app/network/pair", + "rationale": "Projected as link.pair for Windows pairing.", + "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + "source_files": [ + "crates/observer-pl/src/lib.rs", + "crates/observer-pl/src/wire.rs" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-windows", + "path": "/app/observer/register", + "rationale": "Projected as observer.register for Windows observer enrollment.", + "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + "source_files": [ + "crates/observer-pl/src/lib.rs", + "crates/observer-pl/src/wire.rs" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-windows", + "path": "/app/observer/ingest", + "rationale": "Projected as observer.ingestUpload for Windows segment upload.", + "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + "source_files": [ + "crates/observer-pl/src/lib.rs", + "crates/observer-pl/src/wire.rs" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-windows", + "path": "/app/observer/ingest/event", + "rationale": "Projected as observer.ingestEvent for Windows event relay.", + "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + "source_files": [ + "crates/observer-pl/src/lib.rs", + "crates/observer-pl/src/wire.rs" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-windows", + "path": "/app/observer/ingest/segments/{day}", + "rationale": "Projected as observer.ingestSegments for Windows reconciliation.", + "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + "source_files": [ + "crates/observer-pl/src/lib.rs", + "crates/observer-pl/src/wire.rs" + ] + }, + { + "classification": "bundled", + "consumer": "solstone-windows", + "path": "/sse/events", + "rationale": "Projected as callosum.rootEvents for Windows journal bridge SSE.", + "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + "source_files": [ + "crates/pl-transport-win/src/journal_bridge.rs" + ] + } + ], + "schema": "solstone.observer-client-consumer-audit.v1", + "searched_files": [ + { + "consumer": "solstone-browser", + "path": "extension/journal.js", + "revision": "998c1095cd8f766dd188bece5ad6527444f8dfac", + "role": "production" + }, + { + "consumer": "solstone-linux", + "path": "crates/solstone-linux/src/upload.rs", + "revision": "1c679db1ce6f9a65db70c5aae0ca2fad677416ef", + "role": "production" + }, + { + "consumer": "solstone-linux", + "path": "crates/solstone-linux/src/chat_bridge.rs", + "revision": "1c679db1ce6f9a65db70c5aae0ca2fad677416ef", + "role": "production" + }, + { + "consumer": "solstone-windows", + "path": "crates/observer-pl/src/lib.rs", + "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + "role": "production" + }, + { + "consumer": "solstone-windows", + "path": "crates/observer-pl/src/wire.rs", + "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + "role": "production" + }, + { + "consumer": "solstone-windows", + "path": "crates/pl-transport-win/src/journal_bridge.rs", + "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + "role": "production" + } + ], + "settings_drift_findings": [ + { + "consumer": "solstone-linux", + "id": "linux.sol_voice.path", + "rationale": "Linux reads /api/sol_voice, but the journal route is /app/settings/api/sol_voice.", + "status": "adoption_blocker", + "verified_citations": [ + "solstone/apps/settings/routes.py:86-89", + "solstone/apps/settings/routes.py:640-641" + ] + }, + { + "consumer": "solstone-linux", + "id": "linux.sol_voice.linux_notify_send", + "rationale": "Linux reads top-level linux_notify_send, but the journal response exposes system_notifications.linux.", + "status": "adoption_blocker", + "verified_citations": [ + "solstone/convey/sol_initiated/settings.py:41-42", + "solstone/convey/sol_initiated/settings.py:61-63", + "solstone/convey/sol_initiated/settings.py:211-213", + "solstone/apps/settings/tests/test_sol_voice_routes.py:58" + ] + } + ] +} diff --git a/docs/openapi/observer-client-contract/fixtures/wire-behavior.json b/docs/openapi/observer-client-contract/fixtures/wire-behavior.json new file mode 100644 index 000000000..7ae3d5fae --- /dev/null +++ b/docs/openapi/observer-client-contract/fixtures/wire-behavior.json @@ -0,0 +1,807 @@ +{ + "fixtures": [ + { + "id": "declared.observer.ingestSegments.custody_unknown_rejected", + "kind": "declared-negative", + "payload": { + "items": [ + { + "files": [ + { + "name": "audio.flac", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "size": 1, + "status": "unknown" + } + ], + "key": "120000_300", + "observed": false + } + ], + "protocol_version": 2, + "total": 1 + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "custody_unknown", + "operation_id": "observer.ingestSegments", + "status": 200 + }, + "schema_validation": { + "validates": false + } + }, + { + "id": "declared.observer.ingestSegments.envelope_total_mismatch", + "kind": "declared-negative", + "payload": { + "items": [], + "protocol_version": 2, + "total": 1 + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "envelope_total_mismatch", + "operation_id": "observer.ingestSegments", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "declared.observer.ingestUpload.status_unknown_rejected", + "kind": "declared-negative", + "payload": { + "status": "unknown" + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "status_unknown", + "operation_id": "observer.ingestUpload", + "status": 200 + }, + "schema_validation": { + "validates": false + } + }, + { + "id": "example.callosum.rootEvents.response.200.text-event-stream.default", + "kind": "openapi-example", + "payload": { + "event": "owner_message", + "message": "What changed?", + "tract": "chat", + "ts": 1781803200000 + }, + "provenance": { + "direction": "response", + "media_type": "text/event-stream", + "named_variant": "default", + "operation_id": "callosum.rootEvents", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.chat.openSolChatRequest.request.body.application-json.default", + "kind": "openapi-example", + "payload": { + "request_id": "sol-chat-1781803200000" + }, + "provenance": { + "direction": "request", + "media_type": "application/json", + "named_variant": "default", + "operation_id": "chat.openSolChatRequest", + "status": null + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.chat.openSolChatRequest.response.200.application-json.default", + "kind": "openapi-example", + "payload": { + "ok": true + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "default", + "operation_id": "chat.openSolChatRequest", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.link.pair.request.body.application-json.default", + "kind": "openapi-example", + "payload": { + "csr": "-----BEGIN CERTIFICATE REQUEST-----\n...\n-----END CERTIFICATE REQUEST-----\n", + "device_label": "Jer iPhone", + "nonce": "5f0d8c8b9f1e48b0a5f80b98f3d5e9b0", + "sender_instance_id": "ios-01" + }, + "provenance": { + "direction": "request", + "media_type": "application/json", + "named_variant": "default", + "operation_id": "link.pair", + "status": null + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.link.pair.response.200.application-json.default", + "kind": "openapi-example", + "payload": { + "ca_chain": [ + "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----\n" + ], + "client_cert": "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----\n", + "fingerprint": "sha256:abc123", + "home_attestation": "eyJhbGciOi...", + "home_label": "home", + "instance_id": "4d1f3d57-4f39-4930-b8f8-5e6f2a84d51a", + "local_endpoints": [ + { + "ip": "192.168.1.10", + "port": 7657, + "scope": "lan" + } + ] + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "default", + "operation_id": "link.pair", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.observer.callosumStream.response.200.text-event-stream.default", + "kind": "openapi-example", + "payload": { + "day": "20260618", + "event": "observing", + "segment": "143022_300", + "tract": "observe", + "ts": 1781803200000 + }, + "provenance": { + "direction": "response", + "media_type": "text/event-stream", + "named_variant": "default", + "operation_id": "observer.callosumStream", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.observer.ingestEvent.request.body.application-json.default", + "kind": "openapi-example", + "payload": { + "event": "status", + "state": "recording", + "tract": "observe" + }, + "provenance": { + "direction": "request", + "media_type": "application/json", + "named_variant": "default", + "operation_id": "observer.ingestEvent", + "status": null + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.observer.ingestEvent.response.200.application-json.default", + "kind": "openapi-example", + "payload": { + "status": "ok" + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "default", + "operation_id": "observer.ingestEvent", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.observer.ingestSegments.response.200.application-json.legacy", + "kind": "openapi-example", + "payload": [ + { + "files": [ + { + "name": "screen.png", + "sha256": "5f70bf18a086007016bb522ec180fd0b", + "size": 2048, + "status": "present" + } + ], + "key": "143022_300", + "observed": true + } + ], + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "legacy", + "operation_id": "observer.ingestSegments", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.observer.ingestSegments.response.200.application-json.v2", + "kind": "openapi-example", + "payload": { + "items": [ + { + "files": [ + { + "name": "screen.png", + "sha256": "5f70bf18a086007016bb522ec180fd0b", + "size": 2048, + "status": "present" + } + ], + "key": "143022_300", + "observed": true + } + ], + "protocol_version": 2, + "total": 1 + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "v2", + "operation_id": "observer.ingestSegments", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.observer.ingestUpload.request.body.multipart-form-data.default", + "kind": "openapi-example", + "payload": { + "day": "20260618", + "files": [ + "screen.png", + "audio.flac" + ], + "host": "archon", + "meta": "{\"facet\":\"work\"}", + "platform": "linux", + "segment": "143022_300" + }, + "provenance": { + "direction": "request", + "media_type": "multipart/form-data", + "named_variant": "default", + "operation_id": "observer.ingestUpload", + "status": null + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.observer.ingestUpload.response.200.application-json.duplicate", + "kind": "openapi-example", + "payload": { + "existing_segment": "143022_300", + "message": "All files already received", + "status": "duplicate" + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "duplicate", + "operation_id": "observer.ingestUpload", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.observer.ingestUpload.response.200.application-json.normal", + "kind": "openapi-example", + "payload": { + "bytes": 524288, + "files": [ + "screen.png", + "audio.flac" + ], + "segment": "143022_300", + "status": "ok" + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "normal", + "operation_id": "observer.ingestUpload", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.observer.register.request.body.application-json.default", + "kind": "openapi-example", + "payload": { + "hostname": "archon", + "label": "Archon desktop", + "platform": "linux", + "stream_type": "desktop", + "version": "1.4.0" + }, + "provenance": { + "direction": "request", + "media_type": "application/json", + "named_variant": "default", + "operation_id": "observer.register", + "status": null + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "example.observer.register.response.200.application-json.default", + "kind": "openapi-example", + "payload": { + "ingest_url": "/app/observer/ingest", + "key": "x7J7k2observerHandle", + "name": "archon", + "prefix": "x7J7k2ob", + "protocol_version": 2 + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "default", + "operation_id": "observer.register", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.auth.bearer.segments", + "kind": "recorded-response", + "payload": { + "items": [], + "protocol_version": 2, + "total": 0 + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "bearer", + "operation_id": "observer.ingestSegments", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.auth.handle.segments", + "kind": "recorded-response", + "payload": { + "items": [], + "protocol_version": 2, + "total": 0 + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "observer_handle", + "operation_id": "observer.ingestSegments", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.chat.openSolChatRequest.missing", + "kind": "recorded-response", + "payload": { + "detail": "request_id required", + "error": "I couldn't find a required field.", + "reason_code": "missing_required_field" + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "missing_required_field", + "operation_id": "chat.openSolChatRequest", + "status": 400 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.chat.openSolChatRequest.ok", + "kind": "recorded-response", + "payload": { + "ok": true + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "ok", + "operation_id": "chat.openSolChatRequest", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.ingestUpload.collision", + "kind": "recorded-response", + "payload": { + "bytes": 24, + "files": [ + "audio.flac", + "screen.mp4" + ], + "segment": "120000_301", + "segment_original": "120000_300", + "status": "collision" + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "collision", + "operation_id": "observer.ingestUpload", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.ingestUpload.conflict", + "kind": "recorded-response", + "payload": { + "conflicting_files": [ + "notes.txt" + ], + "detail": "Conflicting sidecar metadata for existing segment", + "error": "I couldn't bring in those observer sidecars because they conflict with files already held.", + "existing_segment": "120000_300", + "reason_code": "ingest_sidecar_conflict", + "status": "conflict" + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "conflict", + "operation_id": "observer.ingestUpload", + "status": 409 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.ingestUpload.duplicate", + "kind": "recorded-response", + "payload": { + "existing_segment": "120000_300", + "message": "All files already received", + "status": "duplicate" + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "duplicate", + "operation_id": "observer.ingestUpload", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.ingestUpload.failed", + "kind": "recorded-response", + "payload": { + "detail": "Uploaded file did not match the journal contract", + "error": "I couldn't use those observer files because they don't match the journal contract.", + "failed_path": "20250107/observer/failed/120000_300/1700000000000", + "invalid_files": [ + "audio.jsonl:2: 'text' is a required property" + ], + "reason_code": "ingest_contract_invalid", + "status": "failed" + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "failed", + "operation_id": "observer.ingestUpload", + "status": 422 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.ingestUpload.ok", + "kind": "recorded-response", + "payload": { + "bytes": 8, + "files": [ + "audio.flac" + ], + "segment": "120000_300", + "status": "ok" + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "ok", + "operation_id": "observer.ingestUpload", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.segments.custody_statuses", + "kind": "recorded-response", + "payload": { + "items": [ + { + "files": [ + { + "name": "audio.flac", + "sha256": "93a6beeb378a929f07f54d587e0d5fb2f33a121269aa31bf6ab87818b701d255", + "size": 13, + "status": "processed" + }, + { + "name": "screen.mp4", + "sha256": "9e7a7350d9be4ef540e1cf03da8db61f4da22f31fc27f56bbc5bd3f88bd00e8a", + "size": 14, + "status": "missing" + }, + { + "name": "notes.txt", + "sha256": "459e9e209d70522fb19bd418ba31d79562181e7bca54e85186b4c89f8d90c789", + "size": 13, + "status": "present" + } + ], + "key": "120000_300", + "observed": false + } + ], + "protocol_version": 2, + "total": 1 + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "custody_statuses", + "operation_id": "observer.ingestSegments", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.segments.legacy.absent_header", + "kind": "recorded-response", + "payload": [], + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "legacy_array_absent_header", + "operation_id": "observer.ingestSegments", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.segments.legacy.unparseable_header", + "kind": "recorded-response", + "payload": [], + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "legacy_array_unparseable_header", + "operation_id": "observer.ingestSegments", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.segments.submitted_name_omitted", + "kind": "recorded-response", + "payload": { + "items": [ + { + "files": [ + { + "name": "audio.flac", + "sha256": "b4fae6d15a821dfcf135b6b44c6ee85f8250521a83ab757a36613a4327cf3075", + "size": 15, + "status": "present" + } + ], + "key": "120000_300", + "observed": false + } + ], + "protocol_version": 2, + "total": 1 + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "submitted_name_omitted", + "operation_id": "observer.ingestSegments", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.segments.v2.envelope", + "kind": "recorded-response", + "payload": { + "items": [], + "protocol_version": 2, + "total": 0 + }, + "provenance": { + "direction": "response", + "media_type": "application/json", + "named_variant": "v2_envelope", + "operation_id": "observer.ingestSegments", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.sse.observer.data", + "kind": "recorded-sse-frame", + "payload": { + "event": "status", + "extra": "value", + "tract": "observe", + "ts": 0 + }, + "provenance": { + "direction": "response", + "media_type": "text/event-stream", + "named_variant": "data", + "operation_id": "observer.callosumStream", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.sse.observer.error", + "kind": "recorded-sse-frame", + "payload": { + "detail": "Observer revoked", + "error": "I couldn't use that paired device because it was revoked.", + "reason_code": "pl_revoked" + }, + "provenance": { + "direction": "response", + "media_type": "text/event-stream", + "named_variant": "error", + "operation_id": "observer.callosumStream", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.sse.observer.heartbeat", + "kind": "recorded-sse-frame", + "payload": ": heartbeat\n\n", + "provenance": { + "direction": "response", + "media_type": "text/event-stream", + "named_variant": "heartbeat", + "operation_id": "observer.callosumStream", + "status": 200 + }, + "schema_validation": { + "reason": "SSE heartbeat comments are not JSON schema payloads", + "validates": null + } + }, + { + "id": "recorded.sse.root.data_unknown_event", + "kind": "recorded-sse-frame", + "payload": { + "event": "unknown", + "extra": "value", + "tract": "future", + "ts": 0 + }, + "provenance": { + "direction": "response", + "media_type": "text/event-stream", + "named_variant": "data_unknown_event", + "operation_id": "callosum.rootEvents", + "status": 200 + }, + "schema_validation": { + "validates": true + } + }, + { + "id": "recorded.sse.root.heartbeat", + "kind": "recorded-sse-frame", + "payload": ": heartbeat\n\n", + "provenance": { + "direction": "response", + "media_type": "text/event-stream", + "named_variant": "heartbeat", + "operation_id": "callosum.rootEvents", + "status": 200 + }, + "schema_validation": { + "reason": "SSE heartbeat comments are not JSON schema payloads", + "validates": null + } + } + ], + "schema": "solstone.observer-client-contract-fixtures.v1" +} diff --git a/docs/openapi/observer-client-contract/manifest.json b/docs/openapi/observer-client-contract/manifest.json new file mode 100644 index 000000000..bb58cce09 --- /dev/null +++ b/docs/openapi/observer-client-contract/manifest.json @@ -0,0 +1,657 @@ +{ + "audited_consumer_revisions": [ + { + "consumer_identifier": "solstone-windows", + "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42" + }, + { + "consumer_identifier": "solstone-linux", + "revision": "1c679db1ce6f9a65db70c5aae0ca2fad677416ef" + }, + { + "consumer_identifier": "solstone-browser", + "revision": "998c1095cd8f766dd188bece5ad6527444f8dfac" + } + ], + "bundle_schema_identity": "solstone.observer-client-contract-bundle.schema.v1", + "bundle_semver": "1.0.0", + "component_closure": [ + "CallosumEvent", + "Error", + "SegmentFile", + "SegmentItem", + "SegmentsEnvelope" + ], + "consumer_identifiers": [ + "solstone-android", + "solstone-browser", + "solstone-linux", + "solstone-macos", + "solstone-swift", + "solstone-tmux", + "solstone-windows" + ], + "files": [ + { + "path": "consumer-audit.json", + "sha256": "f3562062aeb971c9dc95ae5d14333566b28431758bcd232c33c093757df7bc18" + }, + { + "path": "fixtures/wire-behavior.json", + "sha256": "9749a50daba9b4a270da045d350bc5edb7a42c9723fa0bf420c8fb8a4a0415f8" + }, + { + "path": "projection.openapi.json", + "sha256": "8a2b7037552edf710597f2ffa6fdc5aa715311df4ea8cf168e70abe4231c64ca" + }, + { + "path": "vectors.json", + "sha256": "7a5132c57b61e2a615a22719abc77e40b708d4a6636c45690cc522dc26c36dec" + } + ], + "generator_identity": "solstone.convey.contract.observer_bundle.v1", + "generator_inputs": [ + { + "id": "bundle.projection_builder", + "path": "solstone/convey/contract/observer_bundle.py", + "role": "projection_builder", + "sha256": "b4492b813745381f2df6a9f802c2f64460528842643c91453c5555e315acb791" + }, + { + "id": "bundle.recording", + "path": "solstone/convey/contract/observer_bundle_recording.py", + "role": "fixture_vector_builder", + "sha256": "414413b82297917c6f625d882adf17a662fe6d317edbcac90591cdf6285e1970" + }, + { + "id": "bundle.recording_fixture_journal", + "path": "tests/fixtures/journal", + "role": "recording_fixture_tree", + "sha256": "2f885b22fb1a7d48044a4b9527028618cbccf038a75334b253d79db1260bf0e5" + }, + { + "id": "extension.observer_sse_error_frame", + "path": "solstone/apps/observer/contract.py", + "role": "code_adjacent_extension", + "sha256": "bf4813fbb2623a9dcb12e06534481b5c8fbeb5ba9f0e63ca089ac7a991c4e5a9" + }, + { + "id": "extension.root_chat_native_subset", + "path": "solstone/convey/root_contract.py", + "role": "code_adjacent_extension", + "sha256": "26e7fe79cf78aa0a3c09e5074ce51136ccfb86b7aa0e33a16c18c69c746e99aa" + }, + { + "id": "fragment.chat", + "path": "solstone/convey/chat_contract.py", + "role": "code_adjacent_fragment", + "sha256": "435106016cdd3060ed65944f7383b7e4407dcf35b4d4dc256c38576a6cbfa754" + }, + { + "id": "fragment.link", + "path": "solstone/apps/network/contract.py", + "role": "code_adjacent_fragment", + "sha256": "eda57ec44dc3363adf721424f5622d3046630e81dd15ad65b52a57632ce90eeb" + }, + { + "id": "fragment.observer", + "path": "solstone/apps/observer/contract.py", + "role": "code_adjacent_fragment", + "sha256": "bf4813fbb2623a9dcb12e06534481b5c8fbeb5ba9f0e63ca089ac7a991c4e5a9" + }, + { + "id": "fragment.root", + "path": "solstone/convey/root_contract.py", + "role": "code_adjacent_fragment", + "sha256": "26e7fe79cf78aa0a3c09e5074ce51136ccfb86b7aa0e33a16c18c69c746e99aa" + }, + { + "id": "openapi.assembler", + "path": "solstone/convey/contract/assemble.py", + "role": "openapi_source", + "sha256": "73f281611a07c6fd7bdb96c26feea29e838888a991d23c8ef143506c3db182ce" + }, + { + "id": "producer.chat_routes", + "path": "solstone/convey/chat.py", + "role": "producer", + "sha256": "0a0ffd2952fddd1ef22abf8a1d2effd91ee7b4ccd7124b78f4632b3d6a21f785" + }, + { + "id": "producer.chat_sol_initiated_copy", + "path": "solstone/convey/sol_initiated/copy.py", + "role": "producer", + "sha256": "ca08d309b3582343159b5342dbb6603b21345fd0093034a90daec53eef5cb4cb" + }, + { + "id": "producer.chat_sol_initiated_events", + "path": "solstone/convey/sol_initiated/events.py", + "role": "producer", + "sha256": "9ec36a36baa53ec631700cdb9a6989d41ce57e9d092e49f4e85c965b24bfc6e8" + }, + { + "id": "producer.chat_stream", + "path": "solstone/convey/chat_stream.py", + "role": "producer", + "sha256": "a94a810a1a64f080e7632165fe864a084a3e2d725f90e870c3ed277b4ccdf45b" + }, + { + "id": "producer.observer_routes", + "path": "solstone/apps/observer/routes.py", + "role": "producer", + "sha256": "1337eeedf8853b31a479711ed8d076eabe5149e87bf2e73a113e5b5abfe92f06" + }, + { + "id": "producer.observer_utils", + "path": "solstone/apps/observer/utils.py", + "role": "producer", + "sha256": "940a30cf76693d9a2c76beb88d4505951de6e7f30c992f79342fbc656a1ad875" + }, + { + "id": "producer.protocol", + "path": "solstone/observe/protocol.py", + "role": "producer", + "sha256": "21feb3b5e23eb9df37544bdecec8e96ba0ec56ae74f73a8f0962d2dcc522af9e" + }, + { + "id": "producer.root_sse", + "path": "solstone/convey/root.py", + "role": "producer", + "sha256": "5d3f18c7fd43354412c3402cf19d348beb1437cfea5e00f3bc53f28a908246cd" + }, + { + "id": "reason_codes", + "path": "solstone/convey/reasons.py", + "role": "vocabulary_source", + "sha256": "ea857c2c4eb2308c05ad02e33a0ead9f1c68c5e7e08c4c0f7aebf52f8a7e8750" + } + ], + "observer_protocol_version": 2, + "openapi_document_version": "1.0.0", + "openapi_spec_version": "3.1.0", + "operation_ids": [ + "callosum.rootEvents", + "chat.openSolChatRequest", + "link.pair", + "observer.callosumStream", + "observer.ingestEvent", + "observer.ingestSegments", + "observer.ingestUpload", + "observer.register" + ], + "projection_path": "projection.openapi.json", + "schema_dialect_uri": "https://json-schema.org/draft/2020-12/schema", + "supported_response_variants": [ + 1, + 2 + ], + "vocabularies": [ + { + "classification": "closed", + "id": "SegmentFile.status", + "source_pointer": "/components/schemas/SegmentFile/properties/status", + "unknown_value_behavior": "reject", + "values": [ + "present", + "missing", + "processed" + ] + }, + { + "absent_or_unparseable": 1, + "classification": "extensible_integer", + "current": 2, + "id": "X-Solstone-Protocol-Version", + "source_pointer": "/paths/~1app~1observer~1ingest~1segments~1{day}/get/responses/200", + "supported_response_variants": [ + 1, + 2 + ], + "unknown_value_behavior": "version_greater_or_equal_current_uses_v2" + }, + { + "classification": "closed", + "id": "callosum.rootEvents.responses.403.reason_code", + "method": "GET", + "path": "/sse/events", + "source_pointer": "/paths/~1sse~1events/get/responses/403", + "unknown_value_behavior": "reject", + "values": [ + "pl_revoked" + ] + }, + { + "classification": "closed", + "id": "callosum.rootEvents.sse_frames", + "source_pointer": "/paths/~1sse~1events/get/responses/200", + "unknown_value_behavior": "reject", + "values": [ + "data", + "heartbeat" + ] + }, + { + "classification": "extensible", + "id": "callosum.tract_event", + "known_registry": { + "activity": [ + "live", + "recorded" + ], + "chat": [ + "owner_message", + "sol_message", + "talent_queued", + "talent_spawned", + "talent_finished", + "talent_errored", + "reflection_ready", + "chat_queue_depth", + "chat_error", + "sol_chat_request", + "sol_chat_request_superseded", + "owner_chat_open", + "owner_chat_dismissed", + "support_draft", + "result", + "support_submit_claim" + ], + "cortex": [ + "request", + "start", + "thinking", + "tool_start", + "tool_end", + "finish", + "error", + "talent_updated", + "info", + "status" + ], + "importer": [ + "started", + "status", + "completed", + "error" + ], + "logs": [ + "exec", + "line", + "exit" + ], + "navigate": [ + "request" + ], + "notification": [ + "*" + ], + "observe": [ + "status", + "observing", + "detected", + "described", + "transcribed", + "observed" + ], + "supervisor": [ + "started", + "stopped", + "restarting", + "status", + "queue" + ], + "sync": [ + "status" + ], + "think": [ + "started", + "status", + "group_started", + "group_completed", + "talent_started", + "talent_completed", + "completed", + "segments_started", + "segments_completed" + ] + }, + "source_pointer": "/", + "unknown_value_behavior": "preserve" + }, + { + "classification": "closed", + "id": "chat.openSolChatRequest.responses.400.reason_code", + "method": "POST", + "path": "/api/chat/sol_chat_request/open", + "source_pointer": "/paths/~1api~1chat~1sol_chat_request~1open/post/responses/400", + "unknown_value_behavior": "reject", + "values": [ + "missing_required_field" + ] + }, + { + "classification": "closed", + "id": "chat.openSolChatRequest.responses.403.reason_code", + "method": "POST", + "path": "/api/chat/sol_chat_request/open", + "source_pointer": "/paths/~1api~1chat~1sol_chat_request~1open/post/responses/403", + "unknown_value_behavior": "reject", + "values": [ + "pl_revoked" + ] + }, + { + "classification": "closed", + "id": "link.pair.responses.400.reason_code", + "method": "POST", + "path": "/app/network/pair", + "source_pointer": "/paths/~1app~1network~1pair/post/responses/400", + "unknown_value_behavior": "reject", + "values": [ + "missing_required_field", + "pairing_key_invalid", + "pairing_request_invalid" + ] + }, + { + "classification": "closed", + "id": "link.pair.responses.403.reason_code", + "method": "POST", + "path": "/app/network/pair", + "source_pointer": "/paths/~1app~1network~1pair/post/responses/403", + "unknown_value_behavior": "reject", + "values": [ + "pl_revoked" + ] + }, + { + "classification": "closed", + "id": "link.pair.responses.410.reason_code", + "method": "POST", + "path": "/app/network/pair", + "source_pointer": "/paths/~1app~1network~1pair/post/responses/410", + "unknown_value_behavior": "reject", + "values": [ + "operation_no_longer_available" + ] + }, + { + "classification": "closed", + "id": "observer.callosumStream.responses.200.sse_error.reason_code", + "method": "GET", + "path": "/app/observer/callosum", + "source_pointer": "/paths/~1app~1observer~1callosum/get/responses/200/x-sse-error-frame", + "unknown_value_behavior": "reject", + "values": [ + "auth_required", + "feature_unavailable", + "pl_revoked" + ] + }, + { + "classification": "closed", + "id": "observer.callosumStream.responses.401.reason_code", + "method": "GET", + "path": "/app/observer/callosum", + "source_pointer": "/paths/~1app~1observer~1callosum/get/responses/401", + "unknown_value_behavior": "reject", + "values": [ + "auth_key_invalid", + "auth_required", + "feature_unavailable", + "pl_revoked" + ] + }, + { + "classification": "closed", + "id": "observer.callosumStream.sse_frames", + "source_pointer": "/paths/~1app~1observer~1callosum/get/responses/200", + "unknown_value_behavior": "reject", + "values": [ + "data", + "error", + "heartbeat" + ] + }, + { + "classification": "closed", + "id": "observer.ingestEvent.responses.400.reason_code", + "method": "POST", + "path": "/app/observer/ingest/event", + "source_pointer": "/paths/~1app~1observer~1ingest~1event/post/responses/400", + "unknown_value_behavior": "reject", + "values": [ + "missing_required_field" + ] + }, + { + "classification": "closed", + "id": "observer.ingestEvent.responses.401.reason_code", + "method": "POST", + "path": "/app/observer/ingest/event", + "source_pointer": "/paths/~1app~1observer~1ingest~1event/post/responses/401", + "unknown_value_behavior": "reject", + "values": [ + "auth_key_invalid", + "auth_required" + ] + }, + { + "classification": "closed", + "id": "observer.ingestEvent.responses.403.reason_code", + "method": "POST", + "path": "/app/observer/ingest/event", + "source_pointer": "/paths/~1app~1observer~1ingest~1event/post/responses/403", + "unknown_value_behavior": "reject", + "values": [ + "feature_unavailable", + "pl_revoked" + ] + }, + { + "classification": "closed", + "id": "observer.ingestSegments.responses.400.reason_code", + "method": "GET", + "path": "/app/observer/ingest/segments/{day}", + "source_pointer": "/paths/~1app~1observer~1ingest~1segments~1{day}/get/responses/400", + "unknown_value_behavior": "reject", + "values": [ + "invalid_day" + ] + }, + { + "classification": "closed", + "id": "observer.ingestSegments.responses.401.reason_code", + "method": "GET", + "path": "/app/observer/ingest/segments/{day}", + "source_pointer": "/paths/~1app~1observer~1ingest~1segments~1{day}/get/responses/401", + "unknown_value_behavior": "reject", + "values": [ + "auth_key_invalid", + "auth_required" + ] + }, + { + "classification": "closed", + "id": "observer.ingestSegments.responses.403.reason_code", + "method": "GET", + "path": "/app/observer/ingest/segments/{day}", + "source_pointer": "/paths/~1app~1observer~1ingest~1segments~1{day}/get/responses/403", + "unknown_value_behavior": "reject", + "values": [ + "feature_unavailable", + "pl_revoked" + ] + }, + { + "classification": "closed", + "id": "observer.ingestUpload.responses.400.reason_code", + "method": "POST", + "path": "/app/observer/ingest", + "source_pointer": "/paths/~1app~1observer~1ingest/post/responses/400", + "unknown_value_behavior": "reject", + "values": [ + "ingest_no_files", + "invalid_day", + "invalid_segment_or_stream", + "missing_required_field" + ] + }, + { + "classification": "closed", + "id": "observer.ingestUpload.responses.401.reason_code", + "method": "POST", + "path": "/app/observer/ingest", + "source_pointer": "/paths/~1app~1observer~1ingest/post/responses/401", + "unknown_value_behavior": "reject", + "values": [ + "auth_key_invalid", + "auth_required" + ] + }, + { + "classification": "closed", + "id": "observer.ingestUpload.responses.403.reason_code", + "method": "POST", + "path": "/app/observer/ingest", + "source_pointer": "/paths/~1app~1observer~1ingest/post/responses/403", + "unknown_value_behavior": "reject", + "values": [ + "feature_unavailable", + "pl_revoked" + ] + }, + { + "classification": "closed", + "id": "observer.ingestUpload.responses.409.reason_code", + "method": "POST", + "path": "/app/observer/ingest", + "source_pointer": "/paths/~1app~1observer~1ingest/post/responses/409", + "unknown_value_behavior": "reject", + "values": [ + "ingest_sidecar_conflict" + ] + }, + { + "classification": "closed", + "id": "observer.ingestUpload.responses.422.reason_code", + "method": "POST", + "path": "/app/observer/ingest", + "source_pointer": "/paths/~1app~1observer~1ingest/post/responses/422", + "unknown_value_behavior": "reject", + "values": [ + "ingest_contract_invalid" + ] + }, + { + "classification": "closed", + "id": "observer.ingestUpload.responses.500.reason_code", + "method": "POST", + "path": "/app/observer/ingest", + "source_pointer": "/paths/~1app~1observer~1ingest/post/responses/500", + "unknown_value_behavior": "reject", + "values": [ + "ingest_storage_failed" + ] + }, + { + "classification": "closed", + "id": "observer.ingestUpload.responses.507.reason_code", + "method": "POST", + "path": "/app/observer/ingest", + "source_pointer": "/paths/~1app~1observer~1ingest/post/responses/507", + "unknown_value_behavior": "reject", + "values": [ + "ingest_storage_failed" + ] + }, + { + "classification": "closed", + "id": "observer.ingestUpload.status", + "source_pointer": "/paths/~1app~1observer~1ingest/post/responses/200/content/application~1json/schema/properties/status", + "unknown_value_behavior": "reject", + "values": [ + "ok", + "duplicate", + "collision", + "conflict", + "failed" + ] + }, + { + "classification": "closed", + "id": "observer.register.responses.400.reason_code", + "method": "POST", + "path": "/app/observer/register", + "source_pointer": "/paths/~1app~1observer~1register/post/responses/400", + "unknown_value_behavior": "reject", + "values": [ + "invalid_segment_or_stream", + "missing_required_field" + ] + }, + { + "classification": "closed", + "id": "observer.register.responses.403.reason_code", + "method": "POST", + "path": "/app/observer/register", + "source_pointer": "/paths/~1app~1observer~1register/post/responses/403", + "unknown_value_behavior": "reject", + "values": [ + "local_request_only" + ] + }, + { + "classification": "closed", + "id": "observer.register.responses.500.reason_code", + "method": "POST", + "path": "/app/observer/register", + "source_pointer": "/paths/~1app~1observer~1register/post/responses/500", + "unknown_value_behavior": "reject", + "values": [ + "settings_operation_failed" + ] + }, + { + "classification": "closed", + "id": "observer.status.ok", + "source_pointer": "/paths/~1app~1observer~1ingest~1event/post/responses/200/content/application~1json/schema/properties/status", + "unknown_value_behavior": "reject", + "values": [ + "ok" + ] + }, + { + "classification": "extensible", + "description": "Native-client-interest subset of CallosumEvent kinds on tract 'chat'. This is not an exhaustive stream vocabulary; root SSE can carry other tracts and events. Payloads remain open (CallosumEvent.additionalProperties).", + "id": "root.chat.native_interest_kinds", + "native_client_interest_subset": [ + "owner_message", + "sol_message", + "talent_queued", + "talent_spawned", + "talent_finished", + "talent_errored", + "chat_queue_depth", + "result", + "chat_error" + ], + "source_pointer": "/paths/~1sse~1events/get/responses/200", + "stream_exhaustive": false, + "unknown_value_behavior": "preserve" + } + ], + "windows_linux_rollout_targets": [ + { + "adoption_blocker_ids": [ + "linux.sol_voice.path", + "linux.sol_voice.linux_notify_send" + ], + "consumer_identifier": "solstone-linux" + }, + { + "adoption_blocker_ids": [], + "consumer_identifier": "solstone-windows" + } + ] +} diff --git a/docs/openapi/observer-client-contract/projection.openapi.json b/docs/openapi/observer-client-contract/projection.openapi.json new file mode 100644 index 000000000..b7dbb44c6 --- /dev/null +++ b/docs/openapi/observer-client-contract/projection.openapi.json @@ -0,0 +1,1518 @@ +{ + "components": { + "schemas": { + "CallosumEvent": { + "additionalProperties": true, + "properties": { + "event": { + "type": "string" + }, + "tract": { + "type": "string" + }, + "ts": { + "type": "integer" + } + }, + "required": [ + "tract", + "event", + "ts" + ], + "type": "object" + }, + "Error": { + "additionalProperties": true, + "properties": { + "detail": { + "type": "string" + }, + "error": { + "type": "string" + }, + "reason_code": { + "enum": [ + "activities_busy", + "activity_already_exists", + "activity_invalid", + "activity_not_found", + "activity_protected", + "agent_unavailable", + "auth_key_invalid", + "auth_required", + "awareness_busy", + "awareness_section_not_found", + "backup_busy", + "backup_not_confirmed", + "backup_operation_failed", + "backup_unavailable", + "chat_queue_full", + "config_busy", + "convey_busy", + "convey_operation_failed", + "corrupt_config", + "cross_origin_blocked", + "edge_index_unavailable", + "entity_alias_conflict", + "entity_already_exists", + "entity_blocked", + "entity_busy", + "entity_not_found", + "entity_operation_failed", + "facet_already_exists", + "facet_not_found", + "feature_unavailable", + "file_not_found", + "file_read_failed", + "health_report_failed", + "host_not_allowed", + "identity_busy", + "identity_not_locked", + "import_client_id_conflict", + "import_conflict", + "import_metadata_failed", + "import_not_found", + "ingest_contract_invalid", + "ingest_no_files", + "ingest_sidecar_conflict", + "ingest_storage_failed", + "invalid_config_value", + "invalid_day", + "invalid_entity_type", + "invalid_json_request", + "invalid_month", + "invalid_operation_for_state", + "invalid_path", + "invalid_request_value", + "invalid_segment_or_stream", + "journal_source_problem", + "ledger_item_not_found", + "local_request_only", + "missing_request_body", + "missing_required_field", + "observer_restart_failed", + "operation_no_longer_available", + "paired_device_not_found", + "pairing_key_invalid", + "pairing_relay_unavailable", + "pairing_request_invalid", + "pl_revoked", + "principal_entity_protected", + "provider_key_missing", + "push_request_invalid", + "raw_media_not_available", + "recovery_key_mismatch", + "reprocess_already_complete", + "reprocess_past_only", + "reprocess_unreachable", + "search_failed", + "service_busy", + "service_operation_failed", + "settings_operation_failed", + "speaker_attribution_state_invalid", + "speaker_command_failed", + "speaker_labels_busy", + "speaker_not_found", + "speaker_owner_centroid_required", + "speaker_owner_identity_required", + "speaker_owner_voice_too_close", + "speaker_review_unavailable", + "speaker_sentence_missing", + "speaker_voiceprint_busy", + "support_portal_failed", + "talent_not_found", + "talent_operation_failed", + "talent_run_malformed", + "talent_run_pending", + "timeline_month_not_found", + "unknown_service", + "voice_unavailable" + ], + "type": "string" + } + }, + "required": [ + "error", + "reason_code", + "detail" + ], + "type": "object" + }, + "SegmentFile": { + "additionalProperties": true, + "properties": { + "name": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "status": { + "enum": [ + "present", + "missing", + "processed" + ], + "type": "string", + "x-vocabulary": { + "classification": "closed", + "id": "SegmentFile.status", + "unknown_value_behavior": "reject" + } + }, + "submitted_name": { + "type": "string" + } + }, + "required": [ + "name", + "size", + "sha256", + "status" + ], + "type": "object" + }, + "SegmentItem": { + "additionalProperties": true, + "properties": { + "files": { + "items": { + "$ref": "#/components/schemas/SegmentFile" + }, + "type": "array" + }, + "key": { + "type": "string" + }, + "observed": { + "type": "boolean" + }, + "original_key": { + "type": "string" + } + }, + "required": [ + "key", + "observed", + "files" + ], + "type": "object" + }, + "SegmentsEnvelope": { + "additionalProperties": true, + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/SegmentItem" + }, + "type": "array" + }, + "protocol_version": { + "type": "integer" + }, + "total": { + "type": "integer" + } + }, + "required": [ + "items", + "total", + "protocol_version" + ], + "type": "object" + } + } + }, + "info": { + "description": "Generated OpenAPI projection for observer-owned client contract surfaces. Regenerate with make openapi.", + "title": "Solstone Observer Client Contract Projection", + "version": "1.0.0", + "x-generated": true, + "x-generated-by": "solstone.convey.contract.observer_bundle.v1" + }, + "openapi": "3.1.0", + "paths": { + "/api/chat/sol_chat_request/open": { + "post": { + "description": "Record that the owner opened a sol-initiated chat request. The request identifier must be a non-empty value after trimming.", + "operationId": "chat.openSolChatRequest", + "requestBody": { + "content": { + "application/json": { + "example": { + "request_id": "sol-chat-1781803200000" + }, + "schema": { + "additionalProperties": true, + "properties": { + "request_id": { + "minLength": 1, + "pattern": "\\S", + "type": "string" + } + }, + "required": [ + "request_id" + ], + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "example": { + "ok": true + }, + "schema": { + "additionalProperties": true, + "properties": { + "ok": { + "type": "boolean" + } + }, + "required": [ + "ok" + ], + "type": "object" + } + } + }, + "description": "Sol-initiated chat request opened." + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Request identifier was missing, malformed, empty, or blank.", + "x-reason-codes": [ + "missing_required_field" + ] + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Access gate rejected a revoked paired-link identity.", + "x-reason-codes": [ + "pl_revoked" + ] + } + }, + "summary": "Open sol chat request", + "tags": [ + "chat" + ] + } + }, + "/app/network/pair": { + "post": { + "description": "Accept a client CSR plus nonce, then return a signed certificate and home attestation.", + "operationId": "link.pair", + "parameters": [ + { + "description": "Pairing nonce; the body nonce can be used instead.", + "in": "query", + "name": "token", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "example": { + "csr": "-----BEGIN CERTIFICATE REQUEST-----\n...\n-----END CERTIFICATE REQUEST-----\n", + "device_label": "Jer iPhone", + "nonce": "5f0d8c8b9f1e48b0a5f80b98f3d5e9b0", + "sender_instance_id": "ios-01" + }, + "schema": { + "additionalProperties": true, + "properties": { + "csr": { + "type": "string" + }, + "device_label": { + "type": "string" + }, + "nonce": { + "type": "string" + }, + "sender_instance_id": { + "type": "string" + } + }, + "required": [ + "csr" + ], + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "example": { + "ca_chain": [ + "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----\n" + ], + "client_cert": "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----\n", + "fingerprint": "sha256:abc123", + "home_attestation": "eyJhbGciOi...", + "home_label": "home", + "instance_id": "4d1f3d57-4f39-4930-b8f8-5e6f2a84d51a", + "local_endpoints": [ + { + "ip": "192.168.1.10", + "port": 7657, + "scope": "lan" + } + ] + }, + "schema": { + "additionalProperties": true, + "properties": { + "ca_chain": { + "items": { + "type": "string" + }, + "type": "array" + }, + "client_cert": { + "type": "string" + }, + "fingerprint": { + "type": "string" + }, + "home_attestation": { + "type": "string" + }, + "home_label": { + "type": "string" + }, + "instance_id": { + "type": "string" + }, + "local_endpoints": { + "items": { + "additionalProperties": true, + "properties": { + "ip": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "scope": { + "type": "string" + } + }, + "required": [ + "ip", + "port", + "scope" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "client_cert", + "ca_chain", + "instance_id", + "home_label", + "home_attestation", + "fingerprint" + ], + "type": "object" + } + } + }, + "description": "Signed link material for the paired client." + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Pair request rejected by handler validation.", + "x-reason-codes": [ + "missing_required_field", + "pairing_key_invalid", + "pairing_request_invalid" + ] + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Access gate rejected a revoked paired-link identity.", + "x-reason-codes": [ + "pl_revoked" + ] + }, + "410": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Nonce expired or was already used.", + "x-reason-codes": [ + "operation_no_longer_available" + ] + } + }, + "summary": "Complete link pairing", + "tags": [ + "link" + ] + } + }, + "/app/observer/callosum": { + "get": { + "description": "Open an observer-authenticated Server-Sent Events feed. Frames: data `data: {json}\\n\\n`; heartbeat `: heartbeat\\n\\n`; error `event: error\\ndata: {Error}\\n\\n`.", + "operationId": "observer.callosumStream", + "parameters": [ + { + "description": "Observer handle. Preferred over Authorization when present.", + "in": "header", + "name": "X-Solstone-Observer", + "required": false, + "schema": { + "type": "string" + } + }, + { + "description": "Bearer observer handle.", + "in": "header", + "name": "Authorization", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "text/event-stream": { + "example": { + "day": "20260618", + "event": "observing", + "segment": "143022_300", + "tract": "observe", + "ts": 1781803200000 + }, + "schema": { + "$ref": "#/components/schemas/CallosumEvent" + } + } + }, + "description": "Callosum event stream. Data frames carry CallosumEvent JSON; heartbeat frames are comments; error frames carry Error JSON.", + "x-sse-error-frame": { + "description": "In-stream `event: error` frames emit only these. auth_key_invalid is excluded here because key validity is established at stream open (routes.py:259-261); mid-stream re-checks only cover observer missing/revoked/disabled (routes.py:281-292).", + "schema": { + "$ref": "#/components/schemas/Error" + }, + "x-reason-codes": [ + "auth_required", + "feature_unavailable", + "pl_revoked" + ] + }, + "x-sse-frame-kinds": { + "classification": "closed", + "id": "observer.callosumStream.sse_frames", + "unknown_value_behavior": "reject", + "values": [ + "data", + "error", + "heartbeat" + ] + } + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Stream-open observer auth failure; codes include 401 and 403 outcomes.", + "x-reason-codes": [ + "auth_key_invalid", + "auth_required", + "feature_unavailable", + "pl_revoked" + ] + } + }, + "summary": "Stream Callosum events", + "tags": [ + "observer" + ] + } + }, + "/app/observer/ingest": { + "post": { + "description": "Upload one capture segment as multipart form data and trigger local observe processing. Media parts must use a registry container format \u2014 audio: flac (audio/flac), opus (audio/opus), ogg (audio/ogg), m4a (audio/mp4), mp3 (audio/mpeg), wav (audio/wav); video: webm (video/webm), mp4 (video/mp4), mov (video/quicktime); image: png (image/png), jpg (image/jpeg), jpeg (image/jpeg), heic (image/heic), heif (image/heif), gif (image/gif), webp (image/webp), tiff (image/tiff). Headerless raw streams are not accepted as media containers; raw PCM must be WAV-wrapped before upload. Video segment timestamps are boundary-relative real capture offsets (seconds from the segment start), never synthetic frame indices. Duplicate identity is derived from the segment directory, with terminal processing proof holding absent legacy raw media and a journal-authored ingest manifest adding hash-strength when present, not from the observer history log. `collision` means conflicting content at the requested key (same stored filename, different bytes); an occupied but non-conflicting key joins that segment. Reserved names such as `stream.json` and `ingest.json` are never written from client bytes and never appear in segment listings as held. Every `duplicate` appends a corroborating history record so `/ingest/segments/` can prove the held files by filename and sha.", + "operationId": "observer.ingestUpload", + "parameters": [ + { + "description": "Observer handle. Preferred over Authorization when present.", + "in": "header", + "name": "X-Solstone-Observer", + "required": false, + "schema": { + "type": "string" + } + }, + { + "description": "Bearer observer handle.", + "in": "header", + "name": "Authorization", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "example": { + "day": "20260618", + "files": [ + "screen.png", + "audio.flac" + ], + "host": "archon", + "meta": "{\"facet\":\"work\"}", + "platform": "linux", + "segment": "143022_300" + }, + "schema": { + "additionalProperties": true, + "properties": { + "day": { + "type": "string" + }, + "files": { + "items": { + "format": "binary", + "type": "string" + }, + "type": "array" + }, + "host": { + "type": "string" + }, + "meta": { + "type": "string" + }, + "platform": { + "type": "string" + }, + "segment": { + "type": "string" + } + }, + "required": [ + "segment", + "day", + "files" + ], + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "examples": { + "duplicate": { + "summary": "Duplicate segment", + "value": { + "existing_segment": "143022_300", + "message": "All files already received", + "status": "duplicate" + } + }, + "normal": { + "summary": "New segment accepted", + "value": { + "bytes": 524288, + "files": [ + "screen.png", + "audio.flac" + ], + "segment": "143022_300", + "status": "ok" + } + } + }, + "schema": { + "additionalProperties": true, + "properties": { + "bytes": { + "type": "integer" + }, + "existing_segment": { + "type": "string" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array" + }, + "message": { + "type": "string" + }, + "segment": { + "type": "string" + }, + "segment_original": { + "type": "string" + }, + "status": { + "enum": [ + "ok", + "duplicate", + "collision", + "conflict", + "failed" + ], + "type": "string", + "x-vocabulary": { + "classification": "closed", + "id": "observer.ingestUpload.status", + "unknown_value_behavior": "reject" + } + } + }, + "required": [ + "status" + ], + "type": "object" + } + } + }, + "description": "Upload accepted, collision-adjusted, or duplicate. On `duplicate`, `existing_segment` is the authoritative stored segment key the client must adopt (do not re-upload); the server also appends a history record so the segment listing corroborates the duplicate. On `collision`, the requested key held conflicting content (same stored filename, different bytes), and the returned `segment` is the authoritative remapped key the client must adopt for subsequent references. An occupied key without conflicting content is not a collision; the upload joins that segment." + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Upload request failed validation.", + "x-reason-codes": [ + "ingest_no_files", + "invalid_day", + "invalid_segment_or_stream", + "missing_required_field" + ] + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Observer handle missing or invalid.", + "x-reason-codes": [ + "auth_key_invalid", + "auth_required" + ] + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Observer is disabled or revoked.", + "x-reason-codes": [ + "feature_unavailable", + "pl_revoked" + ] + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Uploaded sidecar metadata conflicts with an existing segment. The journal already holds different bytes under the named filename; strip the conflicting file and re-send. The response names `conflicting_files` and the held `existing_segment`.", + "x-reason-codes": [ + "ingest_sidecar_conflict" + ] + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Uploaded contract-covered file failed journal contract validation.", + "x-reason-codes": [ + "ingest_contract_invalid" + ] + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Uploaded file could not be stored.", + "x-reason-codes": [ + "ingest_storage_failed" + ] + }, + "507": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No segment slot was available after collision handling.", + "x-reason-codes": [ + "ingest_storage_failed" + ] + } + }, + "summary": "Upload observer segment files", + "tags": [ + "observer" + ] + } + }, + "/app/observer/ingest/event": { + "post": { + "description": "Relay an observer-originated event onto the local Callosum bus.", + "operationId": "observer.ingestEvent", + "parameters": [ + { + "description": "Observer handle. Preferred over Authorization when present.", + "in": "header", + "name": "X-Solstone-Observer", + "required": false, + "schema": { + "type": "string" + } + }, + { + "description": "Bearer observer handle.", + "in": "header", + "name": "Authorization", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "example": { + "event": "status", + "state": "recording", + "tract": "observe" + }, + "schema": { + "additionalProperties": true, + "properties": { + "event": { + "type": "string" + }, + "tract": { + "type": "string" + } + }, + "required": [ + "tract", + "event" + ], + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "example": { + "status": "ok" + }, + "schema": { + "additionalProperties": true, + "properties": { + "status": { + "enum": [ + "ok" + ], + "type": "string", + "x-vocabulary": { + "classification": "closed", + "id": "observer.status.ok", + "unknown_value_behavior": "reject" + } + } + }, + "required": [ + "status" + ], + "type": "object" + } + } + }, + "description": "Event relayed." + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Tract or event was missing.", + "x-reason-codes": [ + "missing_required_field" + ] + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Observer handle missing or invalid.", + "x-reason-codes": [ + "auth_key_invalid", + "auth_required" + ] + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Observer is disabled or revoked.", + "x-reason-codes": [ + "feature_unavailable", + "pl_revoked" + ] + } + }, + "summary": "Relay observer event", + "tags": [ + "observer" + ] + } + }, + "/app/observer/ingest/segments/{day}": { + "get": { + "description": "Return segment upload history for one day. Protocol version 2 and newer receive a collection envelope; older or absent protocol headers receive a legacy bare array. Per-file `status` is `present` (file at its recorded path), `processed` (the recorded raw media is absent because journal processing terminally consumed it, proven by a validated `solstone.processing.v1` record in the same-stem sidecar at the recorded segment path), or `missing` (not found). A local file is proven held by the journal only when its listing entry is `present` or `processed` AND the entry `sha256` matches the local file; a `missing` status \u2014 or any local file the listing does not positively account for \u2014 is needs-upload, and the client must never delete a local copy on that basis. Confirm-before-delete matches every upload-eligible local file to a listing entry per (filename, sha256) \u2014 the listing filename is `submitted_name` when present, else `name` \u2014 and deletes only on a proven-held match. Re-uploading the same bytes for a `processed` entry returns `duplicate` and creates no new segment directory. A peer that does not recognize `processed` degrades safely: it treats the file as not-held, re-uploads, and the server answers `duplicate`. `submitted_name` appears only when the stored filename differs from the submitted one; segment-level `original_key` carries the client's originally requested segment key when a collision remapped it.", + "operationId": "observer.ingestSegments", + "parameters": [ + { + "description": "Observer handle. Preferred over Authorization when present.", + "in": "header", + "name": "X-Solstone-Observer", + "required": false, + "schema": { + "type": "string" + } + }, + { + "description": "Bearer observer handle.", + "in": "header", + "name": "Authorization", + "required": false, + "schema": { + "type": "string" + } + }, + { + "description": "YYYYMMDD day.", + "in": "path", + "name": "day", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Fallback stream for legacy observer history rows.", + "in": "query", + "name": "stream", + "required": false, + "schema": { + "type": "string" + } + }, + { + "description": ">= current protocol version returns the collection envelope; lower or absent returns the legacy bare array.", + "in": "header", + "name": "X-Solstone-Protocol-Version", + "required": false, + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "examples": { + "legacy": { + "summary": "Legacy bare array", + "value": [ + { + "files": [ + { + "name": "screen.png", + "sha256": "5f70bf18a086007016bb522ec180fd0b", + "size": 2048, + "status": "present" + } + ], + "key": "143022_300", + "observed": true + } + ] + }, + "v2": { + "summary": "Protocol v2 envelope", + "value": { + "items": [ + { + "files": [ + { + "name": "screen.png", + "sha256": "5f70bf18a086007016bb522ec180fd0b", + "size": 2048, + "status": "present" + } + ], + "key": "143022_300", + "observed": true + } + ], + "protocol_version": 2, + "total": 1 + } + } + }, + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/SegmentsEnvelope" + }, + { + "items": { + "$ref": "#/components/schemas/SegmentItem" + }, + "type": "array" + } + ] + } + } + }, + "description": "Segment verification response.", + "x-vocabularies": { + "X-Solstone-Protocol-Version": { + "absent_or_unparseable": 1, + "classification": "extensible_integer", + "current": 2, + "supported_response_variants": [ + 1, + 2 + ], + "unknown_value_behavior": "version_greater_or_equal_current_uses_v2" + } + } + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Day path parameter was invalid.", + "x-reason-codes": [ + "invalid_day" + ] + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Observer handle missing or invalid.", + "x-reason-codes": [ + "auth_key_invalid", + "auth_required" + ] + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Observer is disabled or revoked.", + "x-reason-codes": [ + "feature_unavailable", + "pl_revoked" + ] + } + }, + "summary": "List uploaded observer segments", + "tags": [ + "observer" + ] + } + }, + "/app/observer/register": { + "post": { + "description": "Register a trusted local or paired-link observer and lock its stream identity.", + "operationId": "observer.register", + "requestBody": { + "content": { + "application/json": { + "example": { + "hostname": "archon", + "label": "Archon desktop", + "platform": "linux", + "stream_type": "desktop", + "version": "1.4.0" + }, + "schema": { + "additionalProperties": true, + "properties": { + "hostname": { + "type": "string" + }, + "label": { + "type": "string" + }, + "platform": { + "type": "string" + }, + "stream_type": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "platform", + "hostname", + "stream_type", + "version" + ], + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "example": { + "ingest_url": "/app/observer/ingest", + "key": "x7J7k2observerHandle", + "name": "archon", + "prefix": "x7J7k2ob", + "protocol_version": 2 + }, + "schema": { + "additionalProperties": true, + "properties": { + "ingest_url": { + "type": "string" + }, + "key": { + "type": "string" + }, + "name": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "protocol_version": { + "type": "integer" + } + }, + "required": [ + "key", + "prefix", + "name", + "ingest_url", + "protocol_version" + ], + "type": "object" + } + } + }, + "description": "Observer registration descriptor." + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Register request failed validation.", + "x-reason-codes": [ + "invalid_segment_or_stream", + "missing_required_field" + ] + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Register caller was not trusted localhost or paired link.", + "x-reason-codes": [ + "local_request_only" + ] + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Observer record could not be saved.", + "x-reason-codes": [ + "settings_operation_failed" + ] + } + }, + "summary": "Register observer", + "tags": [ + "observer" + ] + } + }, + "/sse/events": { + "get": { + "description": "Open the root Server-Sent Events feed for Callosum events. Frames: heartbeat comments `: heartbeat\\n\\n`; data frames `data: {json}\\n\\n` carrying CallosumEvent JSON for all tracts, including chat. This stream emits no `event: error` frames.", + "operationId": "callosum.rootEvents", + "responses": { + "200": { + "content": { + "text/event-stream": { + "example": { + "event": "owner_message", + "message": "What changed?", + "tract": "chat", + "ts": 1781803200000 + }, + "schema": { + "$ref": "#/components/schemas/CallosumEvent" + } + } + }, + "description": "Callosum event stream. Heartbeat frames are comments `: heartbeat\\n\\n`; data frames are `data: {json}\\n\\n` and carry CallosumEvent JSON for all tracts, including chat.", + "x-chat-events": { + "classification": "extensible", + "description": "Native-client-interest subset of CallosumEvent kinds on tract 'chat'. This is not an exhaustive stream vocabulary; root SSE can carry other tracts and events. Payloads remain open (CallosumEvent.additionalProperties).", + "id": "root.chat.native_interest_kinds", + "kinds": [ + "owner_message", + "sol_message", + "talent_queued", + "talent_spawned", + "talent_finished", + "talent_errored", + "chat_queue_depth", + "result", + "chat_error" + ], + "stream_exhaustive": false, + "unknown_value_behavior": "preserve" + }, + "x-sse-frame-kinds": { + "classification": "closed", + "id": "callosum.rootEvents.sse_frames", + "unknown_value_behavior": "reject", + "values": [ + "data", + "heartbeat" + ] + } + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Access gate rejected a revoked paired-link identity before stream open.", + "x-reason-codes": [ + "pl_revoked" + ] + } + }, + "summary": "Stream root Callosum events", + "tags": [ + "callosum" + ] + } + } + }, + "x-callosum-registry": { + "activity": [ + "live", + "recorded" + ], + "chat": [ + "owner_message", + "sol_message", + "talent_queued", + "talent_spawned", + "talent_finished", + "talent_errored", + "reflection_ready", + "chat_queue_depth", + "chat_error", + "sol_chat_request", + "sol_chat_request_superseded", + "owner_chat_open", + "owner_chat_dismissed", + "support_draft", + "result", + "support_submit_claim" + ], + "cortex": [ + "request", + "start", + "thinking", + "tool_start", + "tool_end", + "finish", + "error", + "talent_updated", + "info", + "status" + ], + "importer": [ + "started", + "status", + "completed", + "error" + ], + "logs": [ + "exec", + "line", + "exit" + ], + "navigate": [ + "request" + ], + "notification": [ + "*" + ], + "observe": [ + "status", + "observing", + "detected", + "described", + "transcribed", + "observed" + ], + "supervisor": [ + "started", + "stopped", + "restarting", + "status", + "queue" + ], + "sync": [ + "status" + ], + "think": [ + "started", + "status", + "group_started", + "group_completed", + "talent_started", + "talent_completed", + "completed", + "segments_started", + "segments_completed" + ] + }, + "x-vocabularies": { + "callosum.tract_event": { + "classification": "extensible", + "known_registry": { + "activity": [ + "live", + "recorded" + ], + "chat": [ + "owner_message", + "sol_message", + "talent_queued", + "talent_spawned", + "talent_finished", + "talent_errored", + "reflection_ready", + "chat_queue_depth", + "chat_error", + "sol_chat_request", + "sol_chat_request_superseded", + "owner_chat_open", + "owner_chat_dismissed", + "support_draft", + "result", + "support_submit_claim" + ], + "cortex": [ + "request", + "start", + "thinking", + "tool_start", + "tool_end", + "finish", + "error", + "talent_updated", + "info", + "status" + ], + "importer": [ + "started", + "status", + "completed", + "error" + ], + "logs": [ + "exec", + "line", + "exit" + ], + "navigate": [ + "request" + ], + "notification": [ + "*" + ], + "observe": [ + "status", + "observing", + "detected", + "described", + "transcribed", + "observed" + ], + "supervisor": [ + "started", + "stopped", + "restarting", + "status", + "queue" + ], + "sync": [ + "status" + ], + "think": [ + "started", + "status", + "group_started", + "group_completed", + "talent_started", + "talent_completed", + "completed", + "segments_started", + "segments_completed" + ] + }, + "unknown_value_behavior": "preserve" + } + } +} diff --git a/docs/openapi/observer-client-contract/vectors.json b/docs/openapi/observer-client-contract/vectors.json new file mode 100644 index 000000000..46353fd84 --- /dev/null +++ b/docs/openapi/observer-client-contract/vectors.json @@ -0,0 +1,469 @@ +{ + "schema": "solstone.observer-client-contract-vectors.v1", + "tmp_token": "$TMP", + "vectors": [ + { + "decision": { + "action": "pass_through", + "frame_kind": "data", + "kind": "sse_frame", + "unknown_event_behavior": "preserve" + }, + "fixture_id": "recorded.sse.root.data_unknown_event", + "frame_kind": "data", + "id": "callosum.rootEvents.sse.data_unknown_event", + "kind": "recorded", + "pointer_hashes": { + "/event": "53331ee8a3d89d5e956086b9c9ffaa709efef795a17c43bbc9666bb622947f8e", + "/tract": "7d75b7beeeecfafd1fdf1c465813c01f0c794acf8e1b60a65b74a8675ad64215" + }, + "pointers": [ + "/tract", + "/event" + ] + }, + { + "decision": { + "action": "ignore_keepalive", + "frame_kind": "heartbeat", + "kind": "sse_frame" + }, + "fixture_id": "recorded.sse.root.heartbeat", + "frame_kind": "heartbeat", + "id": "callosum.rootEvents.sse.heartbeat", + "kind": "recorded", + "pointer_hashes": { + "": "d9526400820634cc9e4cd9ecc2b579ed41723071a82af69f696eda332bc17a89" + }, + "pointers": [ + "" + ] + }, + { + "decision": { + "accepted": false, + "kind": "chat_open_request", + "missing_field_behavior": "absent_malformed_empty_or_blank_rejected", + "reason_code": "missing_required_field" + }, + "fixture_id": "recorded.chat.openSolChatRequest.missing", + "id": "chat.openSolChatRequest.missing_required_field", + "kind": "recorded", + "observed_status": 400, + "pointer_hashes": { + "/reason_code": "c5e107146d3eafcb710f0f9e5f463aaf8b48e0b066d794a8405a81f6b131ca4c" + }, + "pointers": [ + "/reason_code" + ] + }, + { + "decision": { + "accepted": true, + "kind": "chat_open_request", + "missing_field_behavior": "non_empty_trimmed_request_id_required", + "result": "ok_true" + }, + "fixture_id": "recorded.chat.openSolChatRequest.ok", + "id": "chat.openSolChatRequest.ok", + "kind": "recorded", + "observed_status": 200, + "pointer_hashes": { + "/ok": "a17fcf0a2f50e2d495e4f90ce263410edc183add6c62699a2facbccf60410f74" + }, + "pointers": [ + "/ok" + ] + }, + { + "decision": { + "accepted": true, + "auth_form": "authorization_bearer", + "kind": "auth_header_form", + "precedence": "x_solstone_observer_preferred_when_both_present" + }, + "fixture_id": "recorded.auth.bearer.segments", + "id": "observer.auth.bearer", + "kind": "recorded", + "observed_status": 200, + "pointer_hashes": { + "/items": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "/protocol_version": "53c234e5e8472b6ac51c1ae1cab3fe06fad053beb8ebfd8977b010655bfdd3c3", + "/total": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "pointers": [ + "/items", + "/total", + "/protocol_version" + ] + }, + { + "decision": { + "accepted": true, + "auth_form": "x_solstone_observer", + "kind": "auth_header_form", + "precedence": "x_solstone_observer_preferred_when_both_present" + }, + "fixture_id": "recorded.auth.handle.segments", + "id": "observer.auth.handle", + "kind": "recorded", + "observed_status": 200, + "pointer_hashes": { + "/items": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "/protocol_version": "53c234e5e8472b6ac51c1ae1cab3fe06fad053beb8ebfd8977b010655bfdd3c3", + "/total": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "pointers": [ + "/items", + "/total", + "/protocol_version" + ] + }, + { + "decision": { + "action": "dispatch_callosum_event", + "frame_kind": "data", + "kind": "sse_frame", + "unknown_event_behavior": "preserve" + }, + "fixture_id": "recorded.sse.observer.data", + "frame_kind": "data", + "id": "observer.callosumStream.sse.data", + "kind": "recorded", + "pointer_hashes": { + "/event": "bedca5f5c3450db9f398aa7a86904a16e6fa794bdb256baadca050d80d4e7534", + "/tract": "137c165f7fe3ec3dfb24fffbefea32ad3186836233e02ebe226870cc6c1bd470" + }, + "pointers": [ + "/tract", + "/event" + ] + }, + { + "decision": { + "action": "surface_error_and_close", + "frame_kind": "error", + "kind": "sse_frame", + "reason_code": "pl_revoked" + }, + "fixture_id": "recorded.sse.observer.error", + "frame_kind": "error", + "id": "observer.callosumStream.sse.error", + "kind": "recorded", + "pointer_hashes": { + "/reason_code": "dcc3c59606299c038b842d61569ff85c6b57b9da41032257dc4e57a016972018" + }, + "pointers": [ + "/reason_code" + ] + }, + { + "decision": { + "action": "ignore_keepalive", + "frame_kind": "heartbeat", + "kind": "sse_frame" + }, + "fixture_id": "recorded.sse.observer.heartbeat", + "frame_kind": "heartbeat", + "id": "observer.callosumStream.sse.heartbeat", + "kind": "recorded", + "pointer_hashes": { + "": "d9526400820634cc9e4cd9ecc2b579ed41723071a82af69f696eda332bc17a89" + }, + "pointers": [ + "" + ] + }, + { + "decision": { + "holding_by_status": { + "missing": "not_held", + "present": "held", + "processed": "held" + }, + "kind": "custody_status", + "unknown_status": "reject" + }, + "fixture_id": "recorded.segments.custody_statuses", + "id": "observer.ingestSegments.custody_statuses", + "kind": "recorded", + "observed_status": 200, + "pointer_hashes": { + "/items/0/files/0/status": "16412508f5322b160fca89f86f8cd0f11d71deeb6dc114bd2fbfec57731d95e3", + "/items/0/files/1/status": "2781cbb6c3c0ce779a2e503364db12cb9f92bad3c740951e24e919e58b9e1186", + "/items/0/files/2/status": "87f7439876e4033adb5735dd9e8a4031ef2be2239574b1a83398efca11784151" + }, + "pointers": [ + "/items/0/files/0/status", + "/items/0/files/1/status", + "/items/0/files/2/status" + ] + }, + { + "decision": { + "kind": "custody_unknown", + "status": "unknown", + "unknown_status": "reject" + }, + "fixture_id": "declared.observer.ingestSegments.custody_unknown_rejected", + "id": "observer.ingestSegments.custody_unknown_rejected", + "kind": "declared", + "pointer_hashes": { + "/items/0/files/0/status": "53331ee8a3d89d5e956086b9c9ffaa709efef795a17c43bbc9666bb622947f8e" + }, + "pointers": [ + "/items/0/files/0/status" + ] + }, + { + "decision": { + "expected": "total_equals_items_length", + "kind": "envelope_integrity", + "valid": false + }, + "fixture_id": "declared.observer.ingestSegments.envelope_total_mismatch", + "id": "observer.ingestSegments.envelope_total_mismatch", + "kind": "declared", + "pointer_hashes": { + "/total": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + }, + "pointers": [ + "/total" + ] + }, + { + "decision": { + "absent_or_unparseable_uses": 1, + "header": "absent", + "kind": "protocol_variant", + "parsed_version": 1, + "response_variant": "legacy_array" + }, + "fixture_id": "recorded.segments.legacy.absent_header", + "id": "observer.ingestSegments.legacy_array.absent_header", + "kind": "recorded", + "observed_status": 200, + "pointer_hashes": { + "": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" + }, + "pointers": [ + "" + ] + }, + { + "decision": { + "absent_or_unparseable_uses": 1, + "header": "unparseable", + "kind": "protocol_variant", + "parsed_version": 1, + "response_variant": "legacy_array" + }, + "fixture_id": "recorded.segments.legacy.unparseable_header", + "id": "observer.ingestSegments.legacy_array.unparseable_header", + "kind": "recorded", + "observed_status": 200, + "pointer_hashes": { + "": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" + }, + "pointers": [ + "" + ] + }, + { + "decision": { + "fallback": "name", + "kind": "submitted_name_fallback", + "submitted_name_present": false + }, + "fixture_id": "recorded.segments.submitted_name_omitted", + "id": "observer.ingestSegments.submitted_name_fallback", + "kind": "recorded", + "observed_status": 200, + "pointer_hashes": { + "/items/0/files/0/name": "7773c30a843bf694ca3b00e7ba5ead99e6042ba78dadfe75431c0138f5b78287" + }, + "pointers": [ + "/items/0/files/0/name" + ] + }, + { + "decision": { + "current_protocol_version": 2, + "header": "2", + "kind": "protocol_variant", + "parsed_version": 2, + "response_variant": "v2_envelope" + }, + "fixture_id": "recorded.segments.v2.envelope", + "id": "observer.ingestSegments.v2_envelope", + "kind": "recorded", + "observed_status": 200, + "pointer_hashes": { + "/items": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "/protocol_version": "53c234e5e8472b6ac51c1ae1cab3fe06fad053beb8ebfd8977b010655bfdd3c3", + "/total": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa" + }, + "pointers": [ + "/items", + "/total", + "/protocol_version" + ] + }, + { + "decision": { + "accepted": true, + "client_action": "adopt_remapped_segment", + "http_status": 200, + "kind": "ingest_status", + "original_key_source": "segment_original", + "status": "collision", + "stored_key_precedence": [ + "segment", + "segment_original" + ], + "stored_key_source": "segment" + }, + "fixture_id": "recorded.ingestUpload.collision", + "id": "observer.ingestUpload.status.collision", + "kind": "recorded", + "observed_status": 200, + "pointer_hashes": { + "/segment": "fbd10688be6ee217abeacd21b42137e77be6a7445d4d7d30bf8c2e8b5f3453d9", + "/segment_original": "a7c937bcf529de76a07597c2f37951b1b775ae759c0446eb7335960f15c86a1b", + "/status": "130b3c5ac14097b6f596f0470a7a985e0582b8fca45e668fc0040590c306b305" + }, + "pointers": [ + "/status", + "/segment", + "/segment_original" + ] + }, + { + "decision": { + "accepted": false, + "client_action": "preserve_local_and_surface_conflict", + "http_status": 409, + "kind": "ingest_status", + "status": "conflict", + "stored_key_precedence": [ + "existing_segment" + ], + "stored_key_source": "existing_segment" + }, + "fixture_id": "recorded.ingestUpload.conflict", + "id": "observer.ingestUpload.status.conflict", + "kind": "recorded", + "observed_status": 409, + "pointer_hashes": { + "/conflicting_files": "41e76925274ddd172048699de40403b63a622b77ae075614db1f0039ba91af0b", + "/existing_segment": "a7c937bcf529de76a07597c2f37951b1b775ae759c0446eb7335960f15c86a1b", + "/reason_code": "c9cf4aec1407ba979c729691832dc66ddbfcebc87b3dd0b030de3ff8b2f6500f", + "/status": "2ebfc71bdbe3659bffaa77f7dd86fad4bf90586c93ad995d676dc1e5d4165b0b" + }, + "pointers": [ + "/status", + "/reason_code", + "/conflicting_files", + "/existing_segment" + ] + }, + { + "decision": { + "accepted": true, + "client_action": "adopt_existing_segment_without_reupload", + "http_status": 200, + "kind": "ingest_status", + "status": "duplicate", + "stored_key_precedence": [ + "existing_segment" + ], + "stored_key_source": "existing_segment" + }, + "fixture_id": "recorded.ingestUpload.duplicate", + "id": "observer.ingestUpload.status.duplicate", + "kind": "recorded", + "observed_status": 200, + "pointer_hashes": { + "/existing_segment": "a7c937bcf529de76a07597c2f37951b1b775ae759c0446eb7335960f15c86a1b", + "/message": "0e36bda88535e7cb2adc89ff06e730fbb87ac70f227a20a0bc279798060f2194", + "/status": "d949ce0c1c1a0ca6a9b37ca25e0fbd04bf25f9788767975e54ad48b8d36cc335" + }, + "pointers": [ + "/status", + "/existing_segment", + "/message" + ] + }, + { + "decision": { + "accepted": false, + "client_action": "preserve_local_and_surface_failure", + "http_status": 422, + "kind": "ingest_status", + "status": "failed", + "stored_key_precedence": [], + "stored_key_source": null + }, + "fixture_id": "recorded.ingestUpload.failed", + "id": "observer.ingestUpload.status.failed", + "kind": "recorded", + "observed_status": 422, + "pointer_hashes": { + "/failed_path": "d876240f5ba88940fb6ff0ba0142ff98deac01464429e87783648aabbdda1078", + "/reason_code": "9ad60b11f8bc9e9f5f55d955b9e5a00be65d59b15fd9f93cc25ecd5950d5bb3f", + "/status": "e9c6c7eab63b97ace1ed584f168beaba80eb77c85e4aef9d014e7242bef77f8e" + }, + "pointers": [ + "/status", + "/reason_code", + "/failed_path" + ] + }, + { + "decision": { + "accepted": true, + "client_action": "adopt_segment", + "http_status": 200, + "kind": "ingest_status", + "status": "ok", + "stored_key_precedence": [ + "segment" + ], + "stored_key_source": "segment" + }, + "fixture_id": "recorded.ingestUpload.ok", + "id": "observer.ingestUpload.status.ok", + "kind": "recorded", + "observed_status": 200, + "pointer_hashes": { + "/bytes": "aa67a169b0bba217aa0aa88a65346920c84c42447c36ba5f7ea65f422c1fe5d8", + "/files": "6959e999efaf728120be19b4161634a371611fe883b12033e02d97fd3f97641b", + "/segment": "a7c937bcf529de76a07597c2f37951b1b775ae759c0446eb7335960f15c86a1b", + "/status": "24a279376551117f31ed9d92797023d0f89b376a6392801c7d6626e4cb7877e5" + }, + "pointers": [ + "/status", + "/segment", + "/files", + "/bytes" + ] + }, + { + "decision": { + "kind": "closed_vocabulary_unknown", + "status": "unknown", + "unknown_value_behavior": "reject", + "vocabulary": "observer.ingestUpload.status" + }, + "fixture_id": "declared.observer.ingestUpload.status_unknown_rejected", + "id": "observer.ingestUpload.status_unknown_rejected", + "kind": "declared", + "pointer_hashes": { + "/status": "53331ee8a3d89d5e956086b9c9ffaa709efef795a17c43bbc9666bb622947f8e" + }, + "pointers": [ + "/status" + ] + } + ] +} diff --git a/scripts/build_openapi_contract.py b/scripts/build_openapi_contract.py index e36389d26..0fe20ded5 100644 --- a/scripts/build_openapi_contract.py +++ b/scripts/build_openapi_contract.py @@ -12,6 +12,13 @@ import sys from pathlib import Path from solstone.convey.contract.assemble import CALLOSUM_REGISTRY, build_document +from solstone.convey.contract.observer_bundle import ( + build_bundle_files, + stale_bundle_paths, +) +from solstone.convey.contract.observer_bundle_compatibility import ( + check_bundle_compatibility, +) ROOT = Path(__file__).resolve().parent.parent ARTIFACT_PATH = ROOT / "docs" / "openapi" / "convey-clients.json" @@ -48,11 +55,29 @@ def render_convey_doc(current: str) -> str: def write_outputs() -> None: - ARTIFACT_PATH.parent.mkdir(parents=True, exist_ok=True) - ARTIFACT_PATH.write_text(render_openapi_json(), encoding="utf-8") + openapi_text = render_openapi_json() current_doc = CONVEY_DOC_PATH.read_text(encoding="utf-8") - CONVEY_DOC_PATH.write_text(render_convey_doc(current_doc), encoding="utf-8") + convey_doc_text = render_convey_doc(current_doc) + bundle_files = build_bundle_files(ROOT) + deterministic_bundle_files = build_bundle_files(ROOT) + if deterministic_bundle_files != bundle_files: + raise RuntimeError( + "observer client contract bundle generation is not deterministic" + ) + compatibility_failures = check_bundle_compatibility(ROOT, bundle_files) + if compatibility_failures: + raise RuntimeError("\n".join(compatibility_failures)) + + ARTIFACT_PATH.parent.mkdir(parents=True, exist_ok=True) + ARTIFACT_PATH.write_text(openapi_text, encoding="utf-8") + CONVEY_DOC_PATH.write_text(convey_doc_text, encoding="utf-8") + for rel_path, text in bundle_files.items(): + output = ROOT / rel_path + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(text, encoding="utf-8") print(f"wrote {ARTIFACT_PATH.relative_to(ROOT)}") + for rel_path in sorted(bundle_files): + print(f"wrote {rel_path}") print(f"updated {CONVEY_DOC_PATH.relative_to(ROOT)}") @@ -71,6 +96,8 @@ def check_outputs() -> int: if current_doc != expected_doc: stale.append(str(CONVEY_DOC_PATH.relative_to(ROOT))) + stale.extend(str(path) for path in stale_bundle_paths(ROOT)) + if stale: paths = ", ".join(stale) print( diff --git a/solstone/convey/contract/observer_bundle.py b/solstone/convey/contract/observer_bundle.py new file mode 100644 index 000000000..6557a2cb6 --- /dev/null +++ b/solstone/convey/contract/observer_bundle.py @@ -0,0 +1,1241 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Build the observer-client OpenAPI contract bundle.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import os +import re +import stat +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from solstone.convey.contract.assemble import CALLOSUM_REGISTRY, build_document +from solstone.observe import protocol + +BUNDLE_SEMVER = "1.0.0" +GENERATOR_IDENTITY = "solstone.convey.contract.observer_bundle.v1" +BUNDLE_SCHEMA_IDENTITY = "solstone.observer-client-contract-bundle.schema.v1" +SCHEMA_DIALECT_URI = "https://json-schema.org/draft/2020-12/schema" + +BUNDLE_REL_DIR = Path("docs/openapi/observer-client-contract") +MANIFEST_REL = BUNDLE_REL_DIR / "manifest.json" +PROJECTION_REL = BUNDLE_REL_DIR / "projection.openapi.json" +VECTORS_REL = BUNDLE_REL_DIR / "vectors.json" +FIXTURES_REL = BUNDLE_REL_DIR / "fixtures/wire-behavior.json" +CONSUMER_AUDIT_REL = BUNDLE_REL_DIR / "consumer-audit.json" +MANIFEST_NAME = "manifest.json" + +OBSERVER_CLIENT_OPERATION_IDS: tuple[str, ...] = ( + "callosum.rootEvents", + "chat.openSolChatRequest", + "link.pair", + "observer.callosumStream", + "observer.ingestEvent", + "observer.ingestSegments", + "observer.ingestUpload", + "observer.register", +) + +CONSUMER_IDENTIFIERS = [ + "solstone-android", + "solstone-browser", + "solstone-linux", + "solstone-macos", + "solstone-swift", + "solstone-tmux", + "solstone-windows", +] + +AUDITED_CONSUMER_REVISIONS = [ + { + "consumer_identifier": "solstone-windows", + "revision": "19c972c4fea775176cea6421ac8b87f3bb20ab42", + }, + { + "consumer_identifier": "solstone-linux", + "revision": "1c679db1ce6f9a65db70c5aae0ca2fad677416ef", + }, + { + "consumer_identifier": "solstone-browser", + "revision": "998c1095cd8f766dd188bece5ad6527444f8dfac", + }, +] + +WINDOWS_LINUX_ROLLOUT_TARGETS = [ + { + "adoption_blocker_ids": [ + "linux.sol_voice.path", + "linux.sol_voice.linux_notify_send", + ], + "consumer_identifier": "solstone-linux", + }, + { + "adoption_blocker_ids": [], + "consumer_identifier": "solstone-windows", + }, +] + +_SEMVER_RE = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$") +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_WINDOWS_RESERVED_BASENAMES = { + "CON", + "PRN", + "AUX", + "NUL", + *(f"COM{index}" for index in range(1, 10)), + *(f"LPT{index}" for index in range(1, 10)), +} + +_SOURCE_INPUTS: tuple[tuple[str, Path, str], ...] = ( + ( + "bundle.projection_builder", + Path("solstone/convey/contract/observer_bundle.py"), + "projection_builder", + ), + ( + "bundle.recording", + Path("solstone/convey/contract/observer_bundle_recording.py"), + "fixture_vector_builder", + ), + ( + "bundle.recording_fixture_journal", + Path("tests/fixtures/journal"), + "recording_fixture_tree", + ), + ( + "openapi.assembler", + Path("solstone/convey/contract/assemble.py"), + "openapi_source", + ), + ( + "fragment.root", + Path("solstone/convey/root_contract.py"), + "code_adjacent_fragment", + ), + ( + "fragment.chat", + Path("solstone/convey/chat_contract.py"), + "code_adjacent_fragment", + ), + ( + "fragment.link", + Path("solstone/apps/network/contract.py"), + "code_adjacent_fragment", + ), + ( + "fragment.observer", + Path("solstone/apps/observer/contract.py"), + "code_adjacent_fragment", + ), + ( + "extension.root_chat_native_subset", + Path("solstone/convey/root_contract.py"), + "code_adjacent_extension", + ), + ( + "extension.observer_sse_error_frame", + Path("solstone/apps/observer/contract.py"), + "code_adjacent_extension", + ), + ("producer.root_sse", Path("solstone/convey/root.py"), "producer"), + ("producer.chat_routes", Path("solstone/convey/chat.py"), "producer"), + ( + "producer.chat_stream", + Path("solstone/convey/chat_stream.py"), + "producer", + ), + ( + "producer.chat_sol_initiated_copy", + Path("solstone/convey/sol_initiated/copy.py"), + "producer", + ), + ( + "producer.chat_sol_initiated_events", + Path("solstone/convey/sol_initiated/events.py"), + "producer", + ), + ( + "producer.observer_routes", + Path("solstone/apps/observer/routes.py"), + "producer", + ), + ( + "producer.observer_utils", + Path("solstone/apps/observer/utils.py"), + "producer", + ), + ("producer.protocol", Path("solstone/observe/protocol.py"), "producer"), + ("reason_codes", Path("solstone/convey/reasons.py"), "vocabulary_source"), +) + + +class ObserverBundleError(RuntimeError): + """Raised when the observer-client bundle cannot be built.""" + + +class BundleVerificationError(ObserverBundleError): + """Raised when a bundle directory or manifest is invalid.""" + + +class BundleExportRefused(ObserverBundleError): + """Raised when bundle export must refuse without mutating the target.""" + + +class BundleCompatibilityError(ObserverBundleError): + """Raised when bundle history compatibility cannot be accepted.""" + + +@dataclass(frozen=True) +class BundleSnapshot: + """Verified bundle files keyed by bundle-relative path.""" + + manifest: dict[str, Any] + files: dict[str, bytes] + + +def parse_semver(value: str) -> tuple[int, int, int]: + """Parse strict MAJOR.MINOR.PATCH SemVer.""" + + match = _SEMVER_RE.fullmatch(value) + if not match: + raise ValueError(f"invalid SemVer: {value!r}") + return tuple(int(part) for part in match.groups()) + + +def compare_semver(left: str, right: str) -> int: + """Compare two strict SemVer values.""" + + left_parts = parse_semver(left) + right_parts = parse_semver(right) + return (left_parts > right_parts) - (left_parts < right_parts) + + +def render_json(payload: object) -> str: + """Return canonical generated JSON text.""" + + return json.dumps(payload, indent=2, sort_keys=True) + "\n" + + +def build_projection_document(source: dict[str, Any] | None = None) -> dict[str, Any]: + """Project the full OpenAPI document to observer-client operations.""" + + source_document = source if source is not None else build_document() + selected_paths: dict[str, dict[str, Any]] = {} + found: set[str] = set() + + for path, methods in source_document.get("paths", {}).items(): + if not isinstance(methods, dict): + continue + for method, operation in methods.items(): + if not isinstance(operation, dict): + continue + operation_id = operation.get("operationId") + if operation_id not in OBSERVER_CLIENT_OPERATION_IDS: + continue + selected_paths.setdefault(path, {})[method] = copy.deepcopy(operation) + found.add(operation_id) + + missing = set(OBSERVER_CLIENT_OPERATION_IDS) - found + if missing: + raise ObserverBundleError( + "observer bundle projection missing operation(s): " + + ", ".join(sorted(missing)) + ) + + component_names = component_ref_closure(selected_paths, source_document) + source_schemas = source_document.get("components", {}).get("schemas", {}) + components = { + "schemas": { + name: copy.deepcopy(source_schemas[name]) for name in component_names + } + } + + projection = { + "openapi": source_document["openapi"], + "info": { + "title": "Solstone Observer Client Contract Projection", + "version": source_document["info"]["version"], + "x-generated": True, + "x-generated-by": GENERATOR_IDENTITY, + "description": ( + "Generated OpenAPI projection for observer-owned client " + "contract surfaces. Regenerate with make openapi." + ), + }, + "paths": {path: selected_paths[path] for path in sorted(selected_paths)}, + "components": components, + "x-callosum-registry": { + key: CALLOSUM_REGISTRY[key] for key in sorted(CALLOSUM_REGISTRY) + }, + } + if "x-vocabularies" in source_document: + projection["x-vocabularies"] = copy.deepcopy(source_document["x-vocabularies"]) + validate_projection_refs(projection) + return projection + + +def component_ref_closure( + selected_paths: dict[str, Any], source_document: dict[str, Any] +) -> list[str]: + """Return sorted component names reachable from selected paths.""" + + source_schemas = source_document.get("components", {}).get("schemas", {}) + closure: set[str] = set() + pending = list(_local_schema_refs(selected_paths)) + + while pending: + name = pending.pop() + if name in closure: + continue + if name not in source_schemas: + raise ObserverBundleError(f"projection references missing schema: {name}") + closure.add(name) + pending.extend(_local_schema_refs(source_schemas[name])) + + return sorted(closure) + + +def validate_projection_refs(projection: dict[str, Any]) -> None: + """Raise if any projection `$ref` does not resolve inside the projection.""" + + schemas = projection.get("components", {}).get("schemas", {}) + missing = sorted(set(_local_schema_refs(projection)) - set(schemas)) + if missing: + raise ObserverBundleError( + "projection has dangling schema reference(s): " + ", ".join(missing) + ) + + +def build_bundle_files(root: Path | None = None) -> dict[Path, str]: + """Build every generated observer-client bundle file as canonical text.""" + + from solstone.convey.contract.observer_bundle_recording import ( + build_fixture_and_vector_payloads, + ) + + repo_root = _repo_root(root) + projection = build_projection_document() + fixtures, vectors = build_fixture_and_vector_payloads(repo_root, projection) + consumer_audit = build_consumer_audit_payload() + + payload_texts = { + PROJECTION_REL: render_json(projection), + FIXTURES_REL: render_json(fixtures), + VECTORS_REL: render_json(vectors), + CONSUMER_AUDIT_REL: render_json(consumer_audit), + } + manifest = build_manifest_payload(repo_root, projection, payload_texts) + return { + MANIFEST_REL: render_json(manifest), + **payload_texts, + } + + +def write_bundle_files(root: Path | None = None) -> list[Path]: + """Write the generated bundle files into the repository tree.""" + + repo_root = _repo_root(root) + files = build_bundle_files(repo_root) + from solstone.convey.contract.observer_bundle_compatibility import ( + check_bundle_compatibility, + ) + + compatibility_failures = check_bundle_compatibility(repo_root, files) + if compatibility_failures: + raise BundleCompatibilityError("\n".join(compatibility_failures)) + written: list[Path] = [] + for rel_path, text in files.items(): + path = repo_root / rel_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + written.append(rel_path) + return sorted(written) + + +def stale_bundle_paths(root: Path | None = None) -> list[Path]: + """Return generated bundle paths whose committed text is stale.""" + + repo_root = _repo_root(root) + stale: list[Path] = [] + for rel_path, expected in build_bundle_files(repo_root).items(): + path = repo_root / rel_path + try: + current = path.read_text(encoding="utf-8") + except FileNotFoundError: + current = "" + if current != expected: + stale.append(rel_path) + return sorted(stale) + + +def build_manifest_payload( + root: Path, projection: dict[str, Any], payload_texts: dict[Path, str] +) -> dict[str, Any]: + """Build the deterministic manifest payload.""" + + parse_semver(BUNDLE_SEMVER) + return { + "audited_consumer_revisions": AUDITED_CONSUMER_REVISIONS, + "bundle_schema_identity": BUNDLE_SCHEMA_IDENTITY, + "bundle_semver": BUNDLE_SEMVER, + "component_closure": sorted( + projection.get("components", {}).get("schemas", {}).keys() + ), + "consumer_identifiers": CONSUMER_IDENTIFIERS, + "files": [ + { + "path": rel_path.relative_to(BUNDLE_REL_DIR).as_posix(), + "sha256": _sha256_text(payload_texts[rel_path]), + } + for rel_path in sorted(payload_texts) + ], + "generator_identity": GENERATOR_IDENTITY, + "generator_inputs": _generator_inputs(root), + "openapi_document_version": projection["info"]["version"], + "openapi_spec_version": projection["openapi"], + "observer_protocol_version": protocol.OBSERVER_PROTOCOL_VERSION, + "operation_ids": list(OBSERVER_CLIENT_OPERATION_IDS), + "projection_path": PROJECTION_REL.relative_to(BUNDLE_REL_DIR).as_posix(), + "schema_dialect_uri": SCHEMA_DIALECT_URI, + "supported_response_variants": [1, 2], + "vocabularies": build_vocabulary_inventory(projection), + "windows_linux_rollout_targets": WINDOWS_LINUX_ROLLOUT_TARGETS, + } + + +def build_consumer_audit_payload() -> dict[str, Any]: + """Return the committed frozen consumer audit data.""" + + windows_rev = "19c972c4fea775176cea6421ac8b87f3bb20ab42" + linux_rev = "1c679db1ce6f9a65db70c5aae0ca2fad677416ef" + browser_rev = "998c1095cd8f766dd188bece5ad6527444f8dfac" + windows_observer_files = [ + "crates/observer-pl/src/lib.rs", + "crates/observer-pl/src/wire.rs", + ] + linux_chat_file = "crates/solstone-linux/src/chat_bridge.rs" + browser_file = "extension/journal.js" + + return { + "audited_commits": [ + {"commit": windows_rev, "consumer": "solstone-windows"}, + {"commit": linux_rev, "consumer": "solstone-linux"}, + {"commit": browser_rev, "consumer": "solstone-browser"}, + ], + "direct_paths": [ + _audit_path( + "solstone-browser", + browser_rev, + [browser_file], + "/app/observer/register", + "bundled", + "Projected as observer.register for browser observer enrollment.", + ), + _audit_path( + "solstone-browser", + browser_rev, + [browser_file], + "/app/observer/ingest", + "bundled", + "Projected as observer.ingestUpload for browser upload.", + ), + _audit_path( + "solstone-browser", + browser_rev, + [browser_file], + "/app/observer/ingest/event", + "bundled", + "Projected as observer.ingestEvent for browser event relay.", + ), + _audit_path( + "solstone-browser", + browser_rev, + [browser_file], + "/app/observer/ingest/segments/{day}", + "bundled", + ( + "Projected as observer.ingestSegments; browser reconcile " + "consumes the legacy v1 array variant." + ), + ), + _audit_path( + "solstone-browser", + browser_rev, + [browser_file], + "/enroll/device", + "relay_session_control_excluded_from_journal_projection", + "Relay enrollment/session-control path, not journal-owned API.", + ), + _audit_path( + "solstone-linux", + linux_rev, + ["crates/solstone-linux/src/upload.rs"], + "/app/observer/register", + "bundled", + "Projected as observer.register for Linux observer enrollment.", + ), + _audit_path( + "solstone-linux", + linux_rev, + ["crates/solstone-linux/src/upload.rs"], + "/app/observer/ingest", + "bundled", + "Projected as observer.ingestUpload for Linux segment upload.", + ), + _audit_path( + "solstone-linux", + linux_rev, + ["crates/solstone-linux/src/upload.rs"], + "/app/observer/ingest/event", + "bundled", + "Projected as observer.ingestEvent for Linux event relay.", + ), + _audit_path( + "solstone-linux", + linux_rev, + ["crates/solstone-linux/src/upload.rs"], + "/app/observer/ingest/segments/{day}", + "bundled", + "Projected as observer.ingestSegments for Linux reconciliation.", + ), + _audit_path( + "solstone-linux", + linux_rev, + [linux_chat_file], + "/app/observer/callosum", + "bundled", + "Projected as observer.callosumStream for Linux chat bridge SSE.", + ), + _audit_path( + "solstone-linux", + linux_rev, + [linux_chat_file], + "/api/chat/sol_chat_request/open", + "bundled", + "Projected as chat.openSolChatRequest for Linux chat bridge.", + ), + _audit_path( + "solstone-linux", + linux_rev, + [linux_chat_file], + "/app/chat/{day}#event-{index}", + "browser_navigation_excluded_from_api_projection", + "Browser navigation anchor, not an HTTP API contract path.", + ), + _audit_path( + "solstone-linux", + linux_rev, + [linux_chat_file], + "/api/sol_voice", + "consumer_drift_adoption_blocker", + ( + "Consumer uses the old settings path and reads " + "linux_notify_send; journal serves /app/settings/api/sol_voice " + "and nests the Linux boolean under system_notifications.linux." + ), + ), + _audit_path( + "solstone-windows", + windows_rev, + windows_observer_files, + "/app/network/pair", + "bundled", + "Projected as link.pair for Windows pairing.", + ), + _audit_path( + "solstone-windows", + windows_rev, + windows_observer_files, + "/app/observer/register", + "bundled", + "Projected as observer.register for Windows observer enrollment.", + ), + _audit_path( + "solstone-windows", + windows_rev, + windows_observer_files, + "/app/observer/ingest", + "bundled", + "Projected as observer.ingestUpload for Windows segment upload.", + ), + _audit_path( + "solstone-windows", + windows_rev, + windows_observer_files, + "/app/observer/ingest/event", + "bundled", + "Projected as observer.ingestEvent for Windows event relay.", + ), + _audit_path( + "solstone-windows", + windows_rev, + windows_observer_files, + "/app/observer/ingest/segments/{day}", + "bundled", + "Projected as observer.ingestSegments for Windows reconciliation.", + ), + _audit_path( + "solstone-windows", + windows_rev, + ["crates/pl-transport-win/src/journal_bridge.rs"], + "/sse/events", + "bundled", + "Projected as callosum.rootEvents for Windows journal bridge SSE.", + ), + ], + "schema": "solstone.observer-client-consumer-audit.v1", + "searched_files": [ + _searched_file( + "solstone-browser", + browser_rev, + browser_file, + ), + _searched_file( + "solstone-linux", + linux_rev, + "crates/solstone-linux/src/upload.rs", + ), + _searched_file("solstone-linux", linux_rev, linux_chat_file), + _searched_file( + "solstone-windows", + windows_rev, + "crates/observer-pl/src/lib.rs", + ), + _searched_file( + "solstone-windows", + windows_rev, + "crates/observer-pl/src/wire.rs", + ), + _searched_file( + "solstone-windows", + windows_rev, + "crates/pl-transport-win/src/journal_bridge.rs", + ), + ], + "settings_drift_findings": [ + { + "consumer": "solstone-linux", + "id": "linux.sol_voice.path", + "rationale": ( + "Linux reads /api/sol_voice, but the journal route is " + "/app/settings/api/sol_voice." + ), + "status": "adoption_blocker", + "verified_citations": [ + "solstone/apps/settings/routes.py:86-89", + "solstone/apps/settings/routes.py:640-641", + ], + }, + { + "consumer": "solstone-linux", + "id": "linux.sol_voice.linux_notify_send", + "rationale": ( + "Linux reads top-level linux_notify_send, but the journal " + "response exposes system_notifications.linux." + ), + "status": "adoption_blocker", + "verified_citations": [ + "solstone/convey/sol_initiated/settings.py:41-42", + "solstone/convey/sol_initiated/settings.py:61-63", + "solstone/convey/sol_initiated/settings.py:211-213", + "solstone/apps/settings/tests/test_sol_voice_routes.py:58", + ], + }, + ], + } + + +def _audit_path( + consumer: str, + revision: str, + source_files: list[str], + path: str, + classification: str, + rationale: str, +) -> dict[str, Any]: + return { + "classification": classification, + "consumer": consumer, + "path": path, + "rationale": rationale, + "revision": revision, + "source_files": source_files, + } + + +def _searched_file(consumer: str, revision: str, path: str) -> dict[str, str]: + return { + "consumer": consumer, + "path": path, + "revision": revision, + "role": "production", + } + + +def build_vocabulary_inventory(projection: dict[str, Any]) -> list[dict[str, Any]]: + """Return reachable vocabulary classifications for the bundle manifest.""" + + vocabularies: dict[str, dict[str, Any]] = {} + for pointer, node in _walk_dict_nodes(projection): + vocabulary = node.get("x-vocabulary") + if isinstance(vocabulary, dict): + record = _vocabulary_record_from_extension(vocabulary, node) + record["source_pointer"] = _display_pointer(pointer) + vocabularies[record["id"]] = record + grouped = node.get("x-vocabularies") + if isinstance(grouped, dict): + for vocab_id, metadata in sorted(grouped.items()): + if not isinstance(metadata, dict): + continue + record = {"id": vocab_id, **copy.deepcopy(metadata)} + record["source_pointer"] = _display_pointer(pointer) + vocabularies[vocab_id] = record + chat_events = node.get("x-chat-events") + if isinstance(chat_events, dict): + record = { + "id": _extension_id(chat_events, "x-chat-events"), + "native_client_interest_subset": list(chat_events.get("kinds", [])), + "source_pointer": _display_pointer(pointer), + } + for key in ( + "classification", + "description", + "stream_exhaustive", + "unknown_value_behavior", + ): + if key in chat_events: + record[key] = copy.deepcopy(chat_events[key]) + vocabularies[record["id"]] = record + sse_frames = node.get("x-sse-frame-kinds") + if isinstance(sse_frames, dict): + record = { + "id": _extension_id(sse_frames, "x-sse-frame-kinds"), + **copy.deepcopy(sse_frames), + "source_pointer": _display_pointer(pointer), + } + vocabularies[record["id"]] = record + + for path, methods in projection["paths"].items(): + for method, operation in methods.items(): + for status, response in operation.get("responses", {}).items(): + response_pointer = _response_pointer(path, method, str(status)) + codes = response.get("x-reason-codes") + if isinstance(codes, list) and codes: + vocab_id = ( + f"{operation['operationId']}.responses.{status}.reason_code" + ) + vocabularies[vocab_id] = { + "classification": "closed", + "id": vocab_id, + "method": method.upper(), + "path": path, + "source_pointer": response_pointer, + "unknown_value_behavior": "reject", + "values": sorted(codes), + } + frame = response.get("x-sse-error-frame") + if isinstance(frame, dict): + frame_codes = frame.get("x-reason-codes", []) + vocab_id = ( + f"{operation['operationId']}.responses." + f"{status}.sse_error.reason_code" + ) + vocabularies[vocab_id] = { + "classification": "closed", + "id": vocab_id, + "method": method.upper(), + "path": path, + "source_pointer": _pointer_join( + response_pointer, + "x-sse-error-frame", + ), + "unknown_value_behavior": "reject", + "values": sorted(frame_codes), + } + + return [vocabularies[key] for key in sorted(vocabularies)] + + +def _iter_operations( + document: dict[str, Any], +) -> Iterator[tuple[str, str, dict[str, Any]]]: + for path, methods in document.get("paths", {}).items(): + if not isinstance(methods, dict): + continue + for method, operation in methods.items(): + if isinstance(operation, dict): + yield path, method, operation + + +def _walk_dict_nodes( + node: Any, pointer: str = "" +) -> Iterator[tuple[str, dict[str, Any]]]: + if isinstance(node, dict): + yield pointer, node + for key, value in node.items(): + yield from _walk_dict_nodes(value, _pointer_join(pointer, str(key))) + elif isinstance(node, list): + for index, value in enumerate(node): + yield from _walk_dict_nodes(value, _pointer_join(pointer, str(index))) + + +def _vocabulary_record_from_extension( + extension: dict[str, Any], schema_node: dict[str, Any] +) -> dict[str, Any]: + _extension_id(extension, "x-vocabulary") + record = copy.deepcopy(extension) + enum_values = schema_node.get("enum") + if isinstance(enum_values, list): + record["values"] = list(enum_values) + elif "const" in schema_node: + record["values"] = [schema_node["const"]] + return record + + +def _extension_id(extension: dict[str, Any], extension_name: str) -> str: + vocab_id = extension.get("id") + if not isinstance(vocab_id, str) or not vocab_id: + raise ObserverBundleError(f"{extension_name} extension missing id") + return vocab_id + + +def _response_pointer(path: str, method: str, status: str) -> str: + return _pointer_join( + _pointer_join( + _pointer_join(_pointer_join("/paths", path), method), + "responses", + ), + status, + ) + + +def _pointer_join(base: str, token: str) -> str: + escaped = token.replace("~", "~0").replace("/", "~1") + if not base: + return f"/{escaped}" + return f"{base}/{escaped}" + + +def _display_pointer(pointer: str) -> str: + return pointer or "/" + + +def _local_schema_refs(node: Any) -> Iterator[str]: + if isinstance(node, dict): + ref = node.get("$ref") + if isinstance(ref, str): + name = _schema_name_from_ref(ref) + if name is not None: + yield name + for value in node.values(): + yield from _local_schema_refs(value) + elif isinstance(node, list): + for item in node: + yield from _local_schema_refs(item) + + +def _schema_name_from_ref(ref: str) -> str | None: + prefix = "#/components/schemas/" + if not ref.startswith(prefix): + return None + return ref[len(prefix) :] + + +def _generator_inputs(root: Path) -> list[dict[str, Any]]: + bundle_root = (root / BUNDLE_REL_DIR).resolve() + records: list[dict[str, Any]] = [] + for input_id, rel_path, role in _SOURCE_INPUTS: + path = root / rel_path + try: + resolved_path = path.resolve(strict=True) + except FileNotFoundError as exc: + raise ObserverBundleError( + f"generator input path does not exist: {rel_path.as_posix()}" + ) from exc + if _is_relative_to(resolved_path, bundle_root): + raise ObserverBundleError( + f"generator input {input_id} points inside generated bundle: " + f"{rel_path.as_posix()}" + ) + records.append( + { + "id": input_id, + "path": rel_path.as_posix(), + "role": role, + "sha256": _sha256_path(path), + } + ) + return sorted(records, key=lambda item: item["id"]) + + +def _sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _sha256_path(path: Path) -> str: + parent_fd, leaf_name = _open_parent_dir_no_follow(path) + try: + try: + leaf_stat = os.stat(leaf_name, dir_fd=parent_fd, follow_symlinks=False) + except FileNotFoundError as exc: + raise ObserverBundleError( + f"generator input path does not exist: {path}" + ) from exc + mode = leaf_stat.st_mode + if stat.S_ISLNK(mode): + digest = _new_tree_digest() + _digest_record( + digest, + kind=b"symlink", + rel_parts=(), + value=_readlink_at_stable(parent_fd, leaf_name, leaf_stat), + ) + return digest.hexdigest() + if stat.S_ISREG(mode): + return _sha256_regular_at_no_follow(parent_fd, leaf_name, leaf_stat) + if not stat.S_ISDIR(mode): + raise ObserverBundleError(f"generator input is not regular: {path}") + root_fd = _open_dir_at_no_follow(parent_fd, leaf_name, leaf_stat) + finally: + os.close(parent_fd) + digest = _new_tree_digest() + try: + _digest_directory_fd(digest, root_fd, (), os.fstat(root_fd)) + finally: + os.close(root_fd) + return digest.hexdigest() + + +def _new_tree_digest() -> Any: + digest = hashlib.sha256() + _digest_frame(digest, b"domain", b"solstone-observer-bundle-input-tree-v1") + return digest + + +def _digest_directory_fd( + digest: Any, + dir_fd: int, + rel_parts: tuple[bytes, ...], + expected_stat: os.stat_result, +) -> None: + before_dir = _verify_opened_stat( + dir_fd, + expected_stat, + _display_rel_parts(rel_parts), + ) + before_entries = _directory_entry_snapshot(dir_fd) + for name, entry_stat in before_entries: + entry_parts = (*rel_parts, name) + mode = entry_stat.st_mode + if stat.S_ISLNK(mode): + _digest_record( + digest, + kind=b"symlink", + rel_parts=entry_parts, + value=_readlink_at_stable(dir_fd, name, entry_stat), + ) + continue + if stat.S_ISDIR(mode): + _digest_record(digest, kind=b"dir", rel_parts=entry_parts, value=b"") + child_fd = _open_dir_at_no_follow(dir_fd, name, entry_stat) + try: + _digest_directory_fd(digest, child_fd, entry_parts, os.fstat(child_fd)) + finally: + os.close(child_fd) + continue + if stat.S_ISREG(mode): + file_digest = _sha256_regular_at_no_follow(dir_fd, name, entry_stat) + _digest_record( + digest, + kind=b"file", + rel_parts=entry_parts, + value=file_digest.encode("ascii"), + ) + continue + raise ObserverBundleError( + "generator input is not regular: " + _display_rel_parts(entry_parts) + ) + after_entries = _directory_entry_snapshot(dir_fd) + if _entry_snapshot_identity(after_entries) != _entry_snapshot_identity( + before_entries + ): + raise ObserverBundleError( + "generator input directory entries changed while being read: " + + _display_rel_parts(rel_parts) + ) + if _stat_identity(os.fstat(dir_fd)) != _stat_identity(before_dir): + raise ObserverBundleError( + "generator input directory changed while being read: " + + _display_rel_parts(rel_parts) + ) + + +def _open_parent_dir_no_follow(path: Path) -> tuple[int, bytes]: + parts = path.parts + if not parts: + raise ObserverBundleError("generator input path is empty") + if path.is_absolute(): + fd = _open_trusted_dir(Path(path.anchor)) + component_parts = parts[1:] + else: + fd = _open_trusted_dir(Path(".")) + component_parts = parts + if not component_parts: + return fd, b"." + for component in component_parts[:-1]: + name = os.fsencode(component) + if name in {b"", b".", b".."}: + os.close(fd) + raise ObserverBundleError(f"generator input path is unsafe: {path}") + try: + entry_stat = os.stat(name, dir_fd=fd, follow_symlinks=False) + if stat.S_ISLNK(entry_stat.st_mode): + raise ObserverBundleError( + "generator input path component is a symlink: " + os.fsdecode(name) + ) + child_fd = _open_dir_at_no_follow(fd, name, entry_stat) + except Exception: + os.close(fd) + raise + os.close(fd) + fd = child_fd + leaf_name = os.fsencode(component_parts[-1]) + if leaf_name in {b"", b".", b".."}: + os.close(fd) + raise ObserverBundleError(f"generator input path is unsafe: {path}") + return fd, leaf_name + + +def _open_trusted_dir(path: Path) -> int: + try: + fd = os.open(path, _dir_open_flags()) + except OSError as exc: + raise ObserverBundleError( + f"trusted directory cannot be opened: {path}" + ) from exc + if not stat.S_ISDIR(os.fstat(fd).st_mode): + os.close(fd) + raise ObserverBundleError(f"trusted path is not a directory: {path}") + return fd + + +def _open_dir_at_no_follow( + parent_fd: int, name: bytes, expected_stat: os.stat_result +) -> int: + try: + fd = os.open(name, _dir_open_flags(), dir_fd=parent_fd) + except OSError as exc: + raise ObserverBundleError( + "generator input directory cannot be opened: " + os.fsdecode(name) + ) from exc + try: + _verify_opened_stat(fd, expected_stat, os.fsdecode(name)) + return fd + except Exception: + os.close(fd) + raise + + +def _sha256_regular_at_no_follow( + parent_fd: int, name: bytes, expected_stat: os.stat_result +) -> str: + payload = _read_regular_at_no_follow(parent_fd, name, expected_stat) + return hashlib.sha256(payload).hexdigest() + + +def _read_regular_at_no_follow( + parent_fd: int, name: bytes, expected_stat: os.stat_result +) -> bytes: + try: + fd = os.open(name, _file_open_flags(), dir_fd=parent_fd) + except OSError as exc: + raise ObserverBundleError( + "file cannot be opened without following symlinks: " + os.fsdecode(name) + ) from exc + try: + before = _verify_opened_stat(fd, expected_stat, os.fsdecode(name)) + chunks: list[bytes] = [] + while True: + chunk = os.read(fd, 1024 * 1024) + if not chunk: + break + chunks.append(chunk) + after = os.fstat(fd) + if _stat_identity(after) != _stat_identity(before): + raise ObserverBundleError( + "file changed while being read: " + os.fsdecode(name) + ) + return b"".join(chunks) + finally: + os.close(fd) + + +def _verify_opened_stat( + fd: int, expected_stat: os.stat_result, label: str +) -> os.stat_result: + opened_stat = os.fstat(fd) + expected_is_dir = stat.S_ISDIR(expected_stat.st_mode) + opened_matches_expected = ( + _node_identity(opened_stat) == _node_identity(expected_stat) + if expected_is_dir + else _stat_identity(opened_stat) == _stat_identity(expected_stat) + ) + if not opened_matches_expected: + raise ObserverBundleError(f"generator input changed before open: {label}") + if not (stat.S_ISREG(opened_stat.st_mode) or stat.S_ISDIR(opened_stat.st_mode)): + raise ObserverBundleError(f"generator input is not regular: {label}") + return opened_stat + + +def _readlink_at_stable( + parent_fd: int, name: bytes, expected_stat: os.stat_result +) -> bytes: + before = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if _stat_identity(before) != _stat_identity(expected_stat): + raise ObserverBundleError( + "symlink changed before readlink: " + os.fsdecode(name) + ) + try: + target = os.readlink(name, dir_fd=parent_fd) + except OSError as exc: + raise ObserverBundleError( + "symlink cannot be read: " + os.fsdecode(name) + ) from exc + after = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if _stat_identity(after) != _stat_identity(before): + raise ObserverBundleError( + "symlink changed while being read: " + os.fsdecode(name) + ) + return os.fsencode(target) + + +def _directory_entry_snapshot(dir_fd: int) -> list[tuple[bytes, os.stat_result]]: + entries: list[tuple[bytes, os.stat_result]] = [] + list_fd = _open_dir_at_no_follow(dir_fd, b".", os.fstat(dir_fd)) + try: + raw_names = os.listdir(list_fd) + finally: + os.close(list_fd) + for raw_name in raw_names: + name = os.fsencode(raw_name) + try: + entry_stat = os.stat(name, dir_fd=dir_fd, follow_symlinks=False) + except OSError as exc: + raise ObserverBundleError( + f"directory entry cannot be inspected: {os.fsdecode(name)}" + ) from exc + entries.append((name, entry_stat)) + return sorted(entries, key=lambda item: item[0]) + + +def _entry_snapshot_identity( + entries: list[tuple[bytes, os.stat_result]], +) -> tuple[tuple[bytes, tuple[int, int, int, int, int, int]], ...]: + return tuple((name, _stat_identity(entry_stat)) for name, entry_stat in entries) + + +def _stat_identity(value: os.stat_result) -> tuple[int, int, int, int, int, int]: + return ( + value.st_dev, + value.st_ino, + value.st_mode, + value.st_size, + value.st_mtime_ns, + value.st_ctime_ns, + ) + + +def _node_identity(value: os.stat_result) -> tuple[int, int, int]: + return (value.st_dev, value.st_ino, value.st_mode) + + +def _dir_open_flags() -> int: + return ( + os.O_RDONLY + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + + +def _file_open_flags() -> int: + return ( + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0) + ) + + +def _digest_record( + digest: Any, + *, + kind: bytes, + rel_parts: tuple[bytes, ...], + value: bytes, +) -> None: + _digest_frame(digest, b"record", b"") + _digest_frame(digest, b"kind", kind) + _digest_frame(digest, b"path-components", len(rel_parts).to_bytes(8, "big")) + for part in rel_parts: + _digest_frame(digest, b"path-component", part) + _digest_frame(digest, b"value", value) + + +def _digest_frame(digest: Any, label: bytes, value: bytes) -> None: + digest.update(len(label).to_bytes(8, "big")) + digest.update(label) + digest.update(len(value).to_bytes(8, "big")) + digest.update(value) + + +def _display_rel_parts(parts: tuple[bytes, ...]) -> str: + if not parts: + return "." + return "/".join(os.fsdecode(part) for part in parts) + + +def _repo_root(root: Path | None) -> Path: + if root is not None: + return Path(root).resolve() + return Path(__file__).resolve().parents[3] + + +def _is_relative_to(path: Path, parent: Path) -> bool: + try: + path.relative_to(parent) + except ValueError: + return False + return True + + +__all__ = [ + "BUNDLE_REL_DIR", + "BUNDLE_SEMVER", + "BundleCompatibilityError", + "BundleExportRefused", + "BundleSnapshot", + "BundleVerificationError", + "CONSUMER_AUDIT_REL", + "FIXTURES_REL", + "MANIFEST_REL", + "OBSERVER_CLIENT_OPERATION_IDS", + "PROJECTION_REL", + "VECTORS_REL", + "build_bundle_files", + "build_projection_document", + "compare_semver", + "component_ref_closure", + "parse_semver", + "render_json", + "stale_bundle_paths", + "validate_projection_refs", + "write_bundle_files", +] diff --git a/solstone/convey/contract/observer_bundle_compatibility.py b/solstone/convey/contract/observer_bundle_compatibility.py new file mode 100644 index 000000000..b49011743 --- /dev/null +++ b/solstone/convey/contract/observer_bundle_compatibility.py @@ -0,0 +1,641 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""History and SemVer compatibility checks for the observer-client bundle.""" + +from __future__ import annotations + +import copy +import os +import subprocess +from pathlib import Path +from typing import Any + +from solstone.convey.contract.observer_bundle import ( + BUNDLE_REL_DIR, + BUNDLE_SEMVER, + MANIFEST_NAME, + BundleCompatibilityError, + BundleSnapshot, + BundleVerificationError, + ObserverBundleError, + _repo_root, + build_bundle_files, + compare_semver, + parse_semver, + render_json, +) +from solstone.convey.contract.observer_bundle_verification import ( + _json_from_bytes, + _operation_locations, + _required_str, + _resolve_json_pointer, + _validate_bundle_snapshot, +) + +_SEVERITY_RANK = {"patch": 0, "minor": 1, "major": 2} +_BUNDLE_TREE_ABSENT = "absent" +_BUNDLE_TREE_PRESENT = "present" + + +def check_bundle_compatibility( + root: Path | None = None, + candidate_files: dict[Path, str] | None = None, + *, + enforce_current_contract: bool = True, +) -> list[str]: + """Return compatibility/history failures for an in-memory candidate bundle.""" + + repo_root = _repo_root(root) + try: + generated = ( + candidate_files + if candidate_files is not None + else build_bundle_files(repo_root) + ) + candidate = _snapshot_from_generated_files( + generated, + enforce_current_contract=enforce_current_contract, + ) + _check_bundle_history_compatibility(repo_root, candidate) + except ObserverBundleError as exc: + return [str(exc)] + return [] + + +def _snapshot_from_generated_files( + files: dict[Path, str], + *, + enforce_current_contract: bool, +) -> BundleSnapshot: + bundle_files: dict[str, bytes] = {} + for repo_rel_path, text in files.items(): + try: + bundle_rel = repo_rel_path.relative_to(BUNDLE_REL_DIR).as_posix() + except ValueError as exc: + raise BundleVerificationError( + f"generated file is outside bundle directory: {repo_rel_path}" + ) from exc + bundle_files[bundle_rel] = text.encode("utf-8") + return _validate_bundle_snapshot( + bundle_files, + source="candidate", + enforce_current_contract=enforce_current_contract, + ) + + +def _check_bundle_history_compatibility(root: Path, candidate: BundleSnapshot) -> None: + _ensure_complete_git_history(root) + commits = _earlier_history_commits(root) + candidate_version = _required_str(candidate.manifest, "bundle_semver", "candidate") + parse_semver(candidate_version) + found_bundle_history = False + bundle_commit_count = 0 + identical_commit_count = 0 + for commit in commits: + tree_state = _historical_bundle_tree_state(root, commit) + if tree_state == _BUNDLE_TREE_ABSENT: + continue + found_bundle_history = True + bundle_commit_count += 1 + historical = _load_historical_snapshot(root, commit) + if historical.files == candidate.files: + identical_commit_count += 1 + continue + severity, semantic = _classify_bundle_change(historical, candidate) + _enforce_bundle_version( + _required_str(historical.manifest, "bundle_semver", commit), + candidate_version, + severity, + semantic=semantic, + ) + return + if not found_bundle_history: + if candidate_version != BUNDLE_SEMVER: + raise BundleCompatibilityError( + "observer client contract history check failed: genuine first " + f"bundle must use version {BUNDLE_SEMVER}, got {candidate_version}. " + "Recovery: set bundle_semver to 1.0.0 before publishing the first bundle." + ) + return + if bundle_commit_count == identical_commit_count == 1 and ( + candidate_version != BUNDLE_SEMVER + ): + raise BundleCompatibilityError( + "observer client contract history check failed: genuine first " + f"bundle must use version {BUNDLE_SEMVER}, got {candidate_version}. " + "Recovery: set bundle_semver to 1.0.0 before publishing the first bundle." + ) + + +def _load_historical_snapshot(root: Path, commit: str) -> BundleSnapshot: + try: + files = _git_bundle_tree_files(root, commit) + return _validate_bundle_snapshot( + files, + source=f"history:{commit}", + enforce_current_contract=False, + ) + except (BundleVerificationError, ValueError, subprocess.CalledProcessError) as exc: + raise BundleCompatibilityError( + "observer client contract history check failed: historical bundle at " + f"{commit} is corrupt: {exc}. Recovery: restore or regenerate a valid " + "bundle at that baseline before changing the contract." + ) from exc + + +def _ensure_complete_git_history(root: Path) -> None: + shallow = _git_lines(root, ["rev-parse", "--is-shallow-repository"]) + if shallow == ["true"]: + raise BundleCompatibilityError( + "observer client contract history check failed: repository history is " + "shallow. Recovery: fetch complete history before checking bundle compatibility." + ) + grafts_path = _git_lines(root, ["rev-parse", "--git-path", "info/grafts"]) + if ( + grafts_path + and (root / grafts_path[0]).exists() + and (root / grafts_path[0]).stat().st_size + ): + raise BundleCompatibilityError( + "observer client contract history check failed: repository uses grafts. " + "Recovery: remove grafts and check against complete canonical history." + ) + replacements = _git_lines( + root, + ["for-each-ref", "--format=%(refname)", "refs/replace"], + ) + if replacements: + raise BundleCompatibilityError( + "observer client contract history check failed: repository uses replacement " + "objects. Recovery: remove replacement refs and check canonical history." + ) + try: + _git_bytes(root, ["rev-list", "--objects", "--missing=error", "HEAD"]) + _git_bytes(root, ["fsck", "--full", "--no-dangling"]) + except BundleCompatibilityError as exc: + raise BundleCompatibilityError( + "observer client contract history check failed: git object database is " + "corrupt or incomplete. Recovery: restore missing objects or fetch a " + "complete, uncorrupted history before checking bundle compatibility." + ) from exc + + +def _earlier_history_commits(root: Path) -> list[str]: + head = _git_lines(root, ["rev-parse", "--verify", "HEAD"]) + if not head: + raise BundleCompatibilityError( + "observer client contract history check failed: HEAD is missing. " + "Recovery: run inside a complete Git checkout." + ) + try: + return _git_lines(root, ["rev-list", "HEAD"]) + except BundleCompatibilityError as exc: + raise BundleCompatibilityError( + "observer client contract history check failed: missing ancestry. " + "Recovery: fetch complete history before checking bundle compatibility." + ) from exc + + +def _historical_bundle_tree_state(root: Path, commit: str) -> str: + _git_bytes(root, ["cat-file", "-e", f"{commit}^{{commit}}"]) + tree_output = _git_bytes( + root, + ["ls-tree", "-z", "--full-tree", commit, "--", BUNDLE_REL_DIR.as_posix()], + ) + if tree_output == b"": + return _BUNDLE_TREE_ABSENT + return _BUNDLE_TREE_PRESENT + + +def _git_bundle_tree_files(root: Path, commit: str) -> dict[str, bytes]: + tree_output = _git_bytes( + root, + ["ls-tree", "-r", "-z", "--full-tree", commit, BUNDLE_REL_DIR.as_posix()], + ) + if not tree_output: + raise BundleVerificationError("bundle tree is empty") + files: dict[str, bytes] = {} + for entry in tree_output.split(b"\0"): + if not entry: + continue + try: + metadata, repo_path_bytes = entry.split(b"\t", 1) + mode, object_type, object_id = metadata.decode("ascii").split() + repo_path = repo_path_bytes.decode("utf-8") + except ValueError as exc: + raise BundleVerificationError("bundle tree entry is malformed") from exc + if object_type != "blob" or not mode.startswith("100"): + raise BundleVerificationError( + f"bundle tree contains non-regular object: {repo_path}" + ) + rel_path = Path(repo_path).relative_to(BUNDLE_REL_DIR).as_posix() + files[rel_path] = _git_bytes(root, ["show", object_id]) + if MANIFEST_NAME not in files: + raise BundleVerificationError("manifest object is missing") + return files + + +def _classify_bundle_change( + previous: BundleSnapshot, candidate: BundleSnapshot +) -> tuple[str, bool]: + severity = "patch" + semantic = False + severity = _max_severity( + severity, + _classify_manifest_contract_change(previous.manifest, candidate.manifest), + ) + + previous_projection = _json_from_bytes( + previous.files[previous.manifest["projection_path"]], + "previous projection", + ) + candidate_projection = _json_from_bytes( + candidate.files[candidate.manifest["projection_path"]], + "candidate projection", + ) + severity = _max_severity( + severity, + _classify_projection_change(previous_projection, candidate_projection), + ) + + vocabulary_severity = _classify_vocabulary_change( + previous.manifest.get("vocabularies", []), + candidate.manifest.get("vocabularies", []), + ) + severity = _max_severity(severity, vocabulary_severity) + + vector_severity, vector_semantic = _classify_vector_fixture_change( + previous, + candidate, + ) + severity = _max_severity(severity, vector_severity) + semantic = semantic or vector_semantic + return severity, semantic + + +def _classify_manifest_contract_change( + previous: dict[str, Any], candidate: dict[str, Any] +) -> str: + severity = "patch" + major_fields = { + "bundle_schema_identity", + "generator_identity", + "observer_protocol_version", + "openapi_document_version", + "openapi_spec_version", + "projection_path", + "schema_dialect_uri", + "supported_response_variants", + } + for field in major_fields: + if previous.get(field) != candidate.get(field): + severity = _max_severity(severity, "major") + + for field in ( + "operation_ids", + "component_closure", + "consumer_identifiers", + "audited_consumer_revisions", + "windows_linux_rollout_targets", + ): + old_values = previous.get(field) + new_values = candidate.get(field) + if old_values == new_values: + continue + if not isinstance(old_values, list) or not isinstance(new_values, list): + severity = _max_severity(severity, "major") + continue + old_set = {render_json(item) for item in old_values} + new_set = {render_json(item) for item in new_values} + if old_set - new_set: + severity = _max_severity(severity, "major") + if new_set - old_set: + severity = _max_severity(severity, "minor") + return severity + + +def _classify_projection_change(previous: Any, candidate: Any) -> str: + previous_projection = _strip_nonsemantic_openapi(previous) + candidate_projection = _strip_nonsemantic_openapi(candidate) + if previous_projection == candidate_projection: + return "patch" + + severity = "patch" + previous_operations = _operation_locations(previous_projection) + candidate_operations = _operation_locations(candidate_projection) + removed = set(previous_operations) - set(candidate_operations) + added = set(candidate_operations) - set(previous_operations) + if removed: + severity = _max_severity(severity, "major") + if added: + severity = _max_severity(severity, "minor") + for operation_id in sorted(set(previous_operations) & set(candidate_operations)): + old_location = previous_operations[operation_id] + new_location = candidate_operations[operation_id] + if old_location[:2] != new_location[:2]: + severity = _max_severity(severity, "major") + continue + old_operation = old_location[2] + new_operation = new_location[2] + if old_operation != new_operation: + severity = _max_severity( + severity, + "minor" if _is_additive_only(old_operation, new_operation) else "major", + ) + + previous_components = previous_projection.get("components", {}).get("schemas", {}) + candidate_components = candidate_projection.get("components", {}).get("schemas", {}) + removed_components = set(previous_components) - set(candidate_components) + added_components = set(candidate_components) - set(previous_components) + if removed_components: + severity = _max_severity(severity, "major") + if added_components: + severity = _max_severity(severity, "minor") + for name in sorted(set(previous_components) & set(candidate_components)): + if previous_components[name] == candidate_components[name]: + continue + severity = _max_severity( + severity, + "minor" + if _is_additive_only(previous_components[name], candidate_components[name]) + else "major", + ) + return severity + + +def _classify_vocabulary_change(previous: object, candidate: object) -> str: + if not isinstance(previous, list) or not isinstance(candidate, list): + return "major" + old_vocab = { + item.get("id"): item + for item in previous + if isinstance(item, dict) and isinstance(item.get("id"), str) + } + new_vocab = { + item.get("id"): item + for item in candidate + if isinstance(item, dict) and isinstance(item.get("id"), str) + } + severity = "patch" + if set(old_vocab) - set(new_vocab): + severity = _max_severity(severity, "major") + if set(new_vocab) - set(old_vocab): + severity = _max_severity(severity, "minor") + for vocab_id in sorted(set(old_vocab) & set(new_vocab)): + old_item = old_vocab[vocab_id] + new_item = new_vocab[vocab_id] + old_class = old_item.get("classification") + new_class = new_item.get("classification") + if _is_extensible_class(old_class) and not _is_extensible_class(new_class): + severity = _max_severity(severity, "major") + old_values = _vocabulary_values(old_item) + new_values = _vocabulary_values(new_item) + if old_values - new_values: + severity = _max_severity(severity, "major") + added_values = new_values - old_values + if added_values: + if old_class == "closed" or new_class == "closed": + severity = _max_severity(severity, "major") + elif new_item.get("unknown_value_behavior") == "preserve": + severity = _max_severity(severity, "minor") + else: + severity = _max_severity(severity, "major") + return severity + + +def _classify_vector_fixture_change( + previous: BundleSnapshot, candidate: BundleSnapshot +) -> tuple[str, bool]: + old_fixtures = _fixtures_by_id(previous) + new_fixtures = _fixtures_by_id(candidate) + old_vectors = _vectors_by_id(previous) + new_vectors = _vectors_by_id(candidate) + severity = "patch" + semantic = False + for vector_id in sorted(set(old_vectors) & set(new_vectors)): + old_vector = old_vectors[vector_id] + new_vector = new_vectors[vector_id] + old_fixture = old_fixtures.get(old_vector.get("fixture_id")) + new_fixture = new_fixtures.get(new_vector.get("fixture_id")) + if old_fixture is None or new_fixture is None: + severity = _max_severity(severity, "major") + continue + if _vector_contract_meaning(old_vector, old_fixture) != ( + _vector_contract_meaning(new_vector, new_fixture) + ): + severity = _max_severity(severity, "major") + semantic = True + if set(old_vectors) - set(new_vectors): + severity = _max_severity(severity, "major") + if set(new_vectors) - set(old_vectors): + severity = _max_severity(severity, "minor") + return severity, semantic + + +def _vector_contract_meaning( + vector: dict[str, Any], + fixture: dict[str, Any], +) -> dict[str, Any]: + payload = fixture.get("payload") + pointers = vector.get("pointers") + if not isinstance(pointers, list): + pointers = [] + return { + "decision": copy.deepcopy(vector.get("decision")), + "fixture_id": vector.get("fixture_id"), + "frame_kind": vector.get("frame_kind"), + "kind": vector.get("kind"), + "observed_status": vector.get("observed_status"), + "pointer_values": { + pointer: _resolve_json_pointer(payload, pointer) + for pointer in pointers + if isinstance(pointer, str) + }, + "pointers": list(pointers), + } + + +def _enforce_bundle_version( + previous_version: str, + candidate_version: str, + severity: str, + *, + semantic: bool, +) -> None: + previous_parts = parse_semver(previous_version) + candidate_parts = parse_semver(candidate_version) + comparison = compare_semver(candidate_version, previous_version) + if comparison == 0: + raise BundleCompatibilityError( + "observer client contract compatibility failed: bundle content changed " + f"without a version bump from {previous_version}. Recovery: bump " + "bundle_semver according to the detected change severity." + ) + if comparison < 0: + raise BundleCompatibilityError( + "observer client contract compatibility failed: bundle_semver " + f"downgraded from {previous_version} to {candidate_version}. " + "Recovery: use a forward SemVer bump." + ) + if severity == "major": + if candidate_parts[0] <= previous_parts[0]: + raise BundleCompatibilityError( + "observer client contract compatibility failed: major change " + f"requires a major version bump from {previous_version} to " + f"{candidate_version}. Recovery: bump bundle_semver major." + ) + return + if severity == "minor" or semantic: + if ( + candidate_parts[0] == previous_parts[0] + and candidate_parts[1] <= previous_parts[1] + ): + reason = "semantic vector change" if semantic else "minor change" + raise BundleCompatibilityError( + "observer client contract compatibility failed: " + f"{reason} requires a minor or major version bump from " + f"{previous_version} to {candidate_version}. Recovery: bump " + "bundle_semver minor or major." + ) + + +def _strip_nonsemantic_openapi(node: Any) -> Any: + ignored = { + "description", + "example", + "examples", + "externalDocs", + "info", + "summary", + "title", + "x-generated", + "x-generated-by", + "x-chat-events", + "x-reason-codes", + "x-sse-frame-kinds", + "x-vocabularies", + "x-vocabulary", + } + if isinstance(node, dict): + return { + key: _strip_nonsemantic_openapi(value) + for key, value in node.items() + if key not in ignored + } + if isinstance(node, list): + return [_strip_nonsemantic_openapi(item) for item in node] + return node + + +def _is_additive_only(previous: Any, candidate: Any) -> bool: + if isinstance(previous, dict) and isinstance(candidate, dict): + for key, old_value in previous.items(): + if key not in candidate: + return False + if key == "required": + if isinstance(candidate[key], list) and isinstance(old_value, list): + if set(candidate[key]) != set(old_value): + return False + continue + if candidate[key] != old_value: + return False + continue + if key == "properties": + if not isinstance(old_value, dict) or not isinstance( + candidate[key], dict + ): + return False + for property_name, property_schema in old_value.items(): + if property_name not in candidate[key]: + return False + if not _is_additive_only( + property_schema, candidate[key][property_name] + ): + return False + continue + if not _is_additive_only(old_value, candidate[key]): + return False + return True + if isinstance(previous, list) and isinstance(candidate, list): + return previous == candidate + return previous == candidate + + +def _is_extensible_class(classification: object) -> bool: + return isinstance(classification, str) and classification.startswith("extensible") + + +def _vocabulary_values(vocabulary: dict[str, Any]) -> set[str]: + values = vocabulary.get("values") + if isinstance(values, list): + return {str(value) for value in values} + subset = vocabulary.get("native_client_interest_subset") + if isinstance(subset, list): + return {str(value) for value in subset} + registry = vocabulary.get("known_registry") + if isinstance(registry, dict): + return { + f"{tract}.{event}" + for tract, events in registry.items() + if isinstance(events, list) + for event in events + } + return set() + + +def _fixtures_by_id(snapshot: BundleSnapshot) -> dict[str, dict[str, Any]]: + payload = _json_from_bytes( + snapshot.files["fixtures/wire-behavior.json"], "fixtures" + ) + return { + item["id"]: item + for item in payload["fixtures"] + if isinstance(item, dict) and isinstance(item.get("id"), str) + } + + +def _vectors_by_id(snapshot: BundleSnapshot) -> dict[str, dict[str, Any]]: + payload = _json_from_bytes(snapshot.files["vectors.json"], "vectors") + return { + item["id"]: item + for item in payload["vectors"] + if isinstance(item, dict) and isinstance(item.get("id"), str) + } + + +def _max_severity(left: str, right: str) -> str: + return left if _SEVERITY_RANK[left] >= _SEVERITY_RANK[right] else right + + +def _git_lines(root: Path, args: list[str]) -> list[str]: + try: + output = _git_bytes(root, args).decode("utf-8") + except subprocess.CalledProcessError as exc: + raise BundleCompatibilityError( + "observer client contract history check failed: git " + f"{' '.join(args)} failed. Recovery: run inside a Git checkout." + ) from exc + return [line for line in output.splitlines() if line] + + +def _git_bytes(root: Path, args: list[str]) -> bytes: + env = { + **os.environ, + "GIT_NO_LAZY_FETCH": "1", + "GIT_NO_REPLACE_OBJECTS": "1", + } + try: + return subprocess.run( + ["git", *args], + cwd=str(root), + capture_output=True, + check=True, + env=env, + ).stdout + except subprocess.CalledProcessError as exc: + raise BundleCompatibilityError( + "observer client contract history check failed: git " + f"{' '.join(args)} failed. Recovery: run inside a complete, " + "uncorrupted Git checkout." + ) from exc diff --git a/solstone/convey/contract/observer_bundle_export.py b/solstone/convey/contract/observer_bundle_export.py new file mode 100644 index 000000000..940111c9d --- /dev/null +++ b/solstone/convey/contract/observer_bundle_export.py @@ -0,0 +1,392 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Export the verified observer-client contract bundle.""" + +from __future__ import annotations + +import ctypes +import errno +import os +import re +import stat +import uuid +from pathlib import Path + +from solstone.convey.contract.observer_bundle import ( + _WINDOWS_RESERVED_BASENAMES, + BUNDLE_REL_DIR, + BundleExportRefused, + BundleSnapshot, + BundleVerificationError, + ObserverBundleError, + _directory_entry_snapshot, + _entry_snapshot_identity, + _open_dir_at_no_follow, + _open_parent_dir_no_follow, + _repo_root, + _stat_identity, + stale_bundle_paths, +) +from solstone.convey.contract.observer_bundle_verification import ( + verify_bundle_directory, + verify_bundle_fd, + verify_committed_bundle, +) + +_RENAME_NOREPLACE = 1 + + +def export_bundle(destination: Path, root: Path | None = None) -> Path: + """Export the committed observer-client bundle to a new destination.""" + + repo_root = _repo_root(root) + source_dir = repo_root / BUNDLE_REL_DIR + stale = stale_bundle_paths(repo_root) + if stale: + raise _export_refused( + "committed bundle is stale: " + ", ".join(str(path) for path in stale), + "run make openapi before exporting", + ) + try: + source_snapshot = verify_committed_bundle(repo_root) + except ObserverBundleError as exc: + raise _export_refused( + str(exc), + "repair the bundle manifest and rerun make openapi", + ) from exc + return _publish_verified_bundle( + source_dir, + Path(destination), + repo_root, + source_snapshot, + ) + + +def publish_bundle_directory( + source_dir: Path, + destination: Path, + root: Path | None = None, +) -> Path: + """Publish a verified bundle directory to a new destination.""" + + repo_root = _repo_root(root) + source_snapshot = verify_bundle_directory(source_dir) + return _publish_verified_bundle( + Path(source_dir), + Path(destination), + repo_root, + source_snapshot, + ) + + +def _publish_verified_bundle( + source_dir: Path, + destination: Path, + repo_root: Path, + source_snapshot: BundleSnapshot, +) -> Path: + target = _resolve_export_destination(repo_root, Path(destination)) + parent_fd, destination_name = _open_destination_parent(target) + stage_name, stage_fd, stage_stat = _create_stage_dir(target, parent_fd) + finalized = False + try: + _populate_bundle_stage(stage_fd, source_snapshot) + staged_snapshot = _verify_bundle_stage(stage_fd) + if staged_snapshot.files != source_snapshot.files: + raise _export_refused( + "staged bundle bytes do not match the committed bundle", + "repair the bundle manifest and rerun make openapi", + ) + _finalize_bundle_publish(parent_fd, stage_name, destination_name, target) + finalized = True + except BundleVerificationError as exc: + _cleanup_stage(parent_fd, stage_name, stage_stat) + raise _export_refused( + str(exc), + "repair the bundle manifest and rerun make openapi", + ) from exc + except Exception: + if not finalized: + _cleanup_stage(parent_fd, stage_name, stage_stat) + raise + finally: + os.close(stage_fd) + os.close(parent_fd) + return target + + +def _populate_bundle_stage(stage_fd: int, snapshot: BundleSnapshot) -> None: + """Populate a fresh stage directory from verified source bundle files.""" + + for rel_path in sorted(snapshot.files): + _write_stage_file(stage_fd, rel_path, snapshot.files[rel_path]) + + +def _verify_bundle_stage(stage_fd: int) -> BundleSnapshot: + """Verify a populated staging directory before final publish.""" + + return verify_bundle_fd(stage_fd, "export-stage") + + +def _open_destination_parent(destination: Path) -> tuple[int, bytes]: + try: + parent_fd, destination_name = _open_parent_dir_no_follow(destination) + except ObserverBundleError as exc: + raise _export_refused( + str(exc), + "choose a destination whose parent components are normal directories", + ) from exc + try: + os.stat(destination_name, dir_fd=parent_fd, follow_symlinks=False) + except FileNotFoundError: + return parent_fd, destination_name + except OSError: + os.close(parent_fd) + raise + os.close(parent_fd) + raise _export_refused( + f"destination already exists: {destination}", + "choose a new empty destination path", + ) + + +def _create_stage_dir( + target: Path, parent_fd: int +) -> tuple[bytes, int, os.stat_result]: + for _attempt in range(100): + name = os.fsencode(f".{target.name}.staging.{uuid.uuid4().hex}") + try: + os.mkdir(name, mode=0o700, dir_fd=parent_fd) + except FileExistsError: + continue + except OSError as exc: + raise _export_refused( + f"staging directory cannot be created beside destination: {target}", + "choose a writable local Linux destination parent", + ) from exc + stage_stat = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + stage_fd = _open_dir_at_no_follow(parent_fd, name, stage_stat) + return name, stage_fd, stage_stat + raise _export_refused( + f"staging directory name collision beside destination: {target}", + "retry export with the same destination", + ) + + +def _finalize_bundle_publish( + parent_fd: int, + stage_name: bytes, + destination_name: bytes, + destination: Path, +) -> None: + """Publish a verified stage on Linux local filesystems without replacement.""" + + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = getattr(libc, "renameat2", None) + if renameat2 is None: + raise _export_refused( + "Linux renameat2(RENAME_NOREPLACE) is unavailable", + "export on a Linux local filesystem with renameat2 support", + ) + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + result = renameat2( + parent_fd, + stage_name, + parent_fd, + destination_name, + _RENAME_NOREPLACE, + ) + if result == 0: + return + err = ctypes.get_errno() + if err == errno.EEXIST: + raise _export_refused( + f"destination appeared before final rename: {destination}", + "choose a new empty destination path and rerun export", + ) + if err in { + errno.ENOSYS, + errno.EINVAL, + getattr(errno, "EOPNOTSUPP", errno.ENOTSUP), + errno.EXDEV, + }: + raise _export_refused( + "filesystem does not support Linux renameat2(RENAME_NOREPLACE) " + f"for local sibling publish: {destination.parent}", + "export on a supported local Linux filesystem; NFS and remote " + "filesystems are outside this atomic no-replace guarantee", + ) + raise OSError(err, os.strerror(err), str(destination)) + + +def _resolve_export_destination(repo_root: Path, destination: Path) -> Path: + target = destination if destination.is_absolute() else repo_root / destination + _validate_export_destination_text(target) + return target + + +def _validate_export_destination_text(target: Path) -> None: + raw = str(target) + if "\\" in raw: + raise _export_refused( + f"destination contains a backslash: {target}", + "choose a normal POSIX destination path", + ) + if re.search(r"(^|/)[A-Za-z]:", raw): + raise _export_refused( + f"destination contains a Windows drive prefix: {target}", + "choose a destination without drive-letter syntax", + ) + parts = [part for part in target.parts if part not in {target.anchor, ""}] + if not parts: + raise _export_refused( + f"destination has no path component: {target}", + "choose a named destination directory", + ) + for part in parts: + _validate_export_destination_component(part, target) + + +def _validate_export_destination_component(component: str, target: Path) -> None: + if component in {".", ".."}: + raise _export_refused( + f"destination has unsafe component: {target}", + "choose a destination without . or .. components", + ) + if any(ord(char) < 32 or ord(char) == 127 for char in component): + raise _export_refused( + f"destination contains a control character: {target!r}", + "choose a printable destination path", + ) + if ":" in component: + raise _export_refused( + f"destination contains a colon: {target}", + "choose a destination without Windows-unsafe characters", + ) + if any(char in component for char in "*?[]"): + raise _export_refused( + f"destination contains wildcard characters: {target}", + "choose a literal destination path", + ) + if component.endswith("."): + raise _export_refused( + f"destination component has trailing dot: {target}", + "choose a Windows-safe destination component", + ) + if component.endswith(" "): + raise _export_refused( + f"destination component has trailing space: {target}", + "choose a Windows-safe destination component", + ) + basename = component.split(".", 1)[0].upper() + if basename in _WINDOWS_RESERVED_BASENAMES: + raise _export_refused( + f"destination uses Windows-reserved device name: {target}", + "choose a Windows-safe destination component", + ) + + +def _write_stage_file(stage_fd: int, rel_path: str, payload: bytes) -> None: + parts = [os.fsencode(part) for part in rel_path.split("/")] + parent_fd = os.dup(stage_fd) + try: + for directory in parts[:-1]: + child_fd = _ensure_stage_directory(parent_fd, directory) + os.close(parent_fd) + parent_fd = child_fd + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) + file_fd = os.open(parts[-1], flags, 0o600, dir_fd=parent_fd) + try: + view = memoryview(payload) + while view: + written = os.write(file_fd, view) + view = view[written:] + finally: + os.close(file_fd) + finally: + os.close(parent_fd) + + +def _ensure_stage_directory(parent_fd: int, name: bytes) -> int: + try: + entry_stat = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + except FileNotFoundError: + os.mkdir(name, mode=0o700, dir_fd=parent_fd) + entry_stat = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if not stat.S_ISDIR(entry_stat.st_mode): + raise _export_refused( + f"staging path component is not a directory: {os.fsdecode(name)}", + "retry export with a fresh destination", + ) + return _open_dir_at_no_follow(parent_fd, name, entry_stat) + + +def _cleanup_stage( + parent_fd: int, stage_name: bytes, expected_stat: os.stat_result +) -> None: + try: + current_stat = os.stat(stage_name, dir_fd=parent_fd, follow_symlinks=False) + except FileNotFoundError: + return + if _node_identity(current_stat) != _node_identity(expected_stat): + return + stage_fd = _open_dir_at_no_follow(parent_fd, stage_name, current_stat) + try: + _remove_stage_contents(stage_fd) + finally: + os.close(stage_fd) + try: + current_stat = os.stat(stage_name, dir_fd=parent_fd, follow_symlinks=False) + except FileNotFoundError: + return + if _node_identity(current_stat) != _node_identity(expected_stat): + return + os.rmdir(stage_name, dir_fd=parent_fd) + + +def _remove_stage_contents(dir_fd: int) -> None: + before_entries = _directory_entry_snapshot(dir_fd) + for name, entry_stat in sorted(before_entries, key=lambda item: item[0]): + if stat.S_ISDIR(entry_stat.st_mode): + child_fd = _open_dir_at_no_follow(dir_fd, name, entry_stat) + try: + _remove_stage_contents(child_fd) + finally: + os.close(child_fd) + current_stat = os.stat(name, dir_fd=dir_fd, follow_symlinks=False) + if _node_identity(current_stat) == _node_identity(entry_stat): + os.rmdir(name, dir_fd=dir_fd) + continue + if stat.S_ISREG(entry_stat.st_mode): + current_stat = os.stat(name, dir_fd=dir_fd, follow_symlinks=False) + if _stat_identity(current_stat) == _stat_identity(entry_stat): + os.unlink(name, dir_fd=dir_fd) + continue + raise _export_refused( + f"staging path is not a regular file or directory: {os.fsdecode(name)}", + "retry export with a fresh destination", + ) + after_entries = _directory_entry_snapshot(dir_fd) + if _entry_snapshot_identity(after_entries) != (): + raise _export_refused( + "staging directory changed during cleanup", + "inspect and remove the staging directory manually", + ) + + +def _node_identity(value: os.stat_result) -> tuple[int, int, int]: + return (value.st_dev, value.st_ino, value.st_mode) + + +def _export_refused(reason: str, recovery: str) -> BundleExportRefused: + return BundleExportRefused( + f"observer client contract export refused: {reason}. Recovery: {recovery}." + ) diff --git a/solstone/convey/contract/observer_bundle_recording.py b/solstone/convey/contract/observer_bundle_recording.py new file mode 100644 index 000000000..b98a23302 --- /dev/null +++ b/solstone/convey/contract/observer_bundle_recording.py @@ -0,0 +1,1214 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Record observer-client wire behavior fixtures and vectors.""" + +from __future__ import annotations + +import copy +import json +import os +import shutil +import tempfile +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from io import BytesIO +from pathlib import Path +from typing import Any + +from jsonschema import Draft202012Validator + +from solstone.convey.contract.observer_bundle import ( + BundleVerificationError, + ObserverBundleError, + _iter_operations, + _schema_name_from_ref, + _sha256_text, + render_json, +) +from solstone.observe import protocol +from solstone.observe.processing_record import HANDLER_TRANSCRIBE, STATE_EMPTY +from solstone.observe.processing_record import SCHEMA as PROCESSING_SCHEMA + + +def build_fixture_and_vector_payloads( + root: Path, projection: dict[str, Any] +) -> tuple[dict[str, Any], dict[str, Any]]: + """Build fixture and vector payloads, validating fixture schemas.""" + + fixtures = _example_fixtures(projection) + recorded_fixtures, vectors = _record_behavior_vectors(root) + fixtures.extend(recorded_fixtures) + declared_fixtures = _declared_negative_fixtures() + fixtures.extend(declared_fixtures) + vectors.extend(_declared_negative_vectors(declared_fixtures)) + + fixture_payload = { + "fixtures": sorted(fixtures, key=lambda item: item["id"]), + "schema": "solstone.observer-client-contract-fixtures.v1", + } + _validate_fixtures(projection, fixture_payload["fixtures"]) + + vector_payload = { + "schema": "solstone.observer-client-contract-vectors.v1", + "tmp_token": "$TMP", + "vectors": sorted(vectors, key=lambda item: item["id"]), + } + return fixture_payload, vector_payload + + +def _example_fixtures(projection: dict[str, Any]) -> list[dict[str, Any]]: + fixtures: list[dict[str, Any]] = [] + for _path, _method, operation in _iter_operations(projection): + operation_id = operation["operationId"] + request_body = operation.get("requestBody", {}) + if isinstance(request_body, dict): + for media_type, media in request_body.get("content", {}).items(): + if isinstance(media, dict) and "example" in media: + fixtures.append( + _fixture( + fixture_id=_fixture_id( + "example", + operation_id, + "request", + "body", + media_type, + "default", + ), + kind="openapi-example", + operation_id=operation_id, + direction="request", + status=None, + media_type=media_type, + variant="default", + payload=copy.deepcopy(media["example"]), + validates=True, + ) + ) + + for status, response in operation.get("responses", {}).items(): + if not isinstance(response, dict): + continue + for media_type, media in response.get("content", {}).items(): + if not isinstance(media, dict): + continue + if "example" in media: + fixtures.append( + _fixture( + fixture_id=_fixture_id( + "example", + operation_id, + "response", + status, + media_type, + "default", + ), + kind="openapi-example", + operation_id=operation_id, + direction="response", + status=int(status), + media_type=media_type, + variant="default", + payload=copy.deepcopy(media["example"]), + validates=True, + ) + ) + examples = media.get("examples", {}) + if isinstance(examples, dict): + for variant, example in sorted(examples.items()): + if not isinstance(example, dict) or "value" not in example: + continue + fixtures.append( + _fixture( + fixture_id=_fixture_id( + "example", + operation_id, + "response", + status, + media_type, + variant, + ), + kind="openapi-example", + operation_id=operation_id, + direction="response", + status=int(status), + media_type=media_type, + variant=variant, + payload=copy.deepcopy(example["value"]), + validates=True, + ) + ) + return fixtures + + +def _record_behavior_vectors( + root: Path, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + fixtures: list[dict[str, Any]] = [] + vectors: list[dict[str, Any]] = [] + + with tempfile.TemporaryDirectory(prefix="solstone-observer-bundle-") as tmp_text: + tmp = Path(tmp_text) + journal = _prepare_recording_journal(root, tmp / "journal") + with _recording_env(journal): + from solstone.apps.observer import routes as observer_routes + from solstone.convey import bridge as convey_bridge + from solstone.convey import create_app + + with _temporary_attr(observer_routes, "now_ms", lambda: 1700000000000): + app = create_app(journal=str(journal.resolve())) + app.config["TESTING"] = True + client = app.test_client() + + auth_key = _register_observer(client, "vector-auth") + _record_response( + fixtures, + vectors, + fixture_id="recorded.auth.bearer.segments", + vector_id="observer.auth.bearer", + operation_id="observer.ingestSegments", + response=client.get( + "/app/observer/ingest/segments/20250103", + headers={ + "Authorization": f"Bearer {auth_key}", + protocol.OBSERVER_PROTOCOL_VERSION_HEADER: "2", + }, + ), + variant="bearer", + pointers=["/items", "/total", "/protocol_version"], + ) + _record_response( + fixtures, + vectors, + fixture_id="recorded.auth.handle.segments", + vector_id="observer.auth.handle", + operation_id="observer.ingestSegments", + response=client.get( + "/app/observer/ingest/segments/20250103", + headers={ + protocol.OBSERVER_HANDLE_HEADER: auth_key, + protocol.OBSERVER_PROTOCOL_VERSION_HEADER: "2", + }, + ), + variant="observer_handle", + pointers=["/items", "/total", "/protocol_version"], + ) + + segments_key = _register_observer(client, "vector-segments") + _record_response( + fixtures, + vectors, + fixture_id="recorded.segments.legacy.absent_header", + vector_id="observer.ingestSegments.legacy_array.absent_header", + operation_id="observer.ingestSegments", + response=client.get( + "/app/observer/ingest/segments/20250103", + headers={"Authorization": f"Bearer {segments_key}"}, + ), + variant="legacy_array_absent_header", + pointers=[""], + ) + _record_response( + fixtures, + vectors, + fixture_id="recorded.segments.legacy.unparseable_header", + vector_id=( + "observer.ingestSegments.legacy_array.unparseable_header" + ), + operation_id="observer.ingestSegments", + response=client.get( + "/app/observer/ingest/segments/20250103", + headers={ + "Authorization": f"Bearer {segments_key}", + protocol.OBSERVER_PROTOCOL_VERSION_HEADER: "bogus", + }, + ), + variant="legacy_array_unparseable_header", + pointers=[""], + ) + _record_response( + fixtures, + vectors, + fixture_id="recorded.segments.v2.envelope", + vector_id="observer.ingestSegments.v2_envelope", + operation_id="observer.ingestSegments", + response=client.get( + "/app/observer/ingest/segments/20250103", + headers={ + "Authorization": f"Bearer {segments_key}", + protocol.OBSERVER_PROTOCOL_VERSION_HEADER: "2", + }, + ), + variant="v2_envelope", + pointers=["/items", "/total", "/protocol_version"], + ) + + upload_key = _register_observer(client, "vector-upload") + ok_response = _upload( + client, + upload_key, + "20250104", + "120000_300", + [(b"ok audio", "120000_300_audio.flac")], + ) + _record_response( + fixtures, + vectors, + fixture_id="recorded.ingestUpload.ok", + vector_id="observer.ingestUpload.status.ok", + operation_id="observer.ingestUpload", + response=ok_response, + variant="ok", + pointers=["/status", "/segment", "/files", "/bytes"], + ) + duplicate_response = _upload( + client, + upload_key, + "20250104", + "120000_300", + [(b"ok audio", "120000_300_audio.flac")], + ) + _record_response( + fixtures, + vectors, + fixture_id="recorded.ingestUpload.duplicate", + vector_id="observer.ingestUpload.status.duplicate", + operation_id="observer.ingestUpload", + response=duplicate_response, + variant="duplicate", + pointers=["/status", "/existing_segment", "/message"], + ) + + collision_key = _register_observer(client, "vector-collision") + _upload( + client, + collision_key, + "20250105", + "120000_300", + [ + (b"collision audio", "audio.flac"), + (b"screen v1", "screen.mp4"), + ], + ) + collision_response = _upload( + client, + collision_key, + "20250105", + "120000_300", + [ + (b"collision audio", "audio.flac"), + (b"screen v2", "screen.mp4"), + ], + ) + _record_response( + fixtures, + vectors, + fixture_id="recorded.ingestUpload.collision", + vector_id="observer.ingestUpload.status.collision", + operation_id="observer.ingestUpload", + response=collision_response, + variant="collision", + pointers=["/status", "/segment", "/segment_original"], + ) + + conflict_key = _register_observer(client, "vector-conflict") + _upload( + client, + conflict_key, + "20250106", + "120000_300", + [(b"held audio", "audio.flac"), (b"notes v1", "notes.txt")], + ) + conflict_response = _upload( + client, + conflict_key, + "20250106", + "120000_300", + [(b"held audio", "audio.flac"), (b"notes v2", "notes.txt")], + ) + _record_response( + fixtures, + vectors, + fixture_id="recorded.ingestUpload.conflict", + vector_id="observer.ingestUpload.status.conflict", + operation_id="observer.ingestUpload", + response=conflict_response, + variant="conflict", + pointers=[ + "/status", + "/reason_code", + "/conflicting_files", + "/existing_segment", + ], + ) + + failed_key = _register_observer(client, "vector-failed") + failed_response = _upload( + client, + failed_key, + "20250107", + "120000_300", + [ + ( + b'{"raw":"audio.flac"}\n{"start":"00:00:00"}\n', + "120000_300_audio.jsonl", + ) + ], + ) + _record_response( + fixtures, + vectors, + fixture_id="recorded.ingestUpload.failed", + vector_id="observer.ingestUpload.status.failed", + operation_id="observer.ingestUpload", + response=failed_response, + variant="failed", + pointers=["/status", "/reason_code", "/failed_path"], + ) + + fallback_key = _register_observer(client, "vector-submitted") + _upload( + client, + fallback_key, + "20250108", + "120000_300", + [(b"same-name audio", "audio.flac")], + ) + _record_response( + fixtures, + vectors, + fixture_id="recorded.segments.submitted_name_omitted", + vector_id="observer.ingestSegments.submitted_name_fallback", + operation_id="observer.ingestSegments", + response=client.get( + "/app/observer/ingest/segments/20250108", + headers={ + "Authorization": f"Bearer {fallback_key}", + protocol.OBSERVER_PROTOCOL_VERSION_HEADER: "2", + }, + ), + variant="submitted_name_omitted", + pointers=["/items/0/files/0/name"], + ) + + custody_key = _register_observer(client, "vector-custody") + _upload( + client, + custody_key, + "20250109", + "120000_300", + [ + (b"custody audio", "audio.flac"), + (b"custody screen", "screen.mp4"), + (b"custody notes", "notes.txt"), + ], + ) + segment_dir = ( + journal / "chronicle" / "20250109" / "vector-custody" / "120000_300" + ) + _write_processing_sidecar(segment_dir, input_size=len(b"custody audio")) + (segment_dir / "audio.flac").unlink() + (segment_dir / "screen.mp4").unlink() + _record_response( + fixtures, + vectors, + fixture_id="recorded.segments.custody_statuses", + vector_id="observer.ingestSegments.custody_statuses", + operation_id="observer.ingestSegments", + response=client.get( + "/app/observer/ingest/segments/20250109", + headers={ + "Authorization": f"Bearer {custody_key}", + protocol.OBSERVER_PROTOCOL_VERSION_HEADER: "2", + }, + ), + variant="custody_statuses", + pointers=[ + "/items/0/files/0/status", + "/items/0/files/1/status", + "/items/0/files/2/status", + ], + ) + + _record_sse_vectors( + client, + observer_routes, + convey_bridge, + fixtures, + vectors, + ) + + _record_response( + fixtures, + vectors, + fixture_id="recorded.chat.openSolChatRequest.ok", + vector_id="chat.openSolChatRequest.ok", + operation_id="chat.openSolChatRequest", + response=client.post( + "/api/chat/sol_chat_request/open", + json={"request_id": "request-1"}, + ), + variant="ok", + pointers=["/ok"], + ) + _record_response( + fixtures, + vectors, + fixture_id="recorded.chat.openSolChatRequest.missing", + vector_id="chat.openSolChatRequest.missing_required_field", + operation_id="chat.openSolChatRequest", + response=client.post( + "/api/chat/sol_chat_request/open", + json={"request_id": " "}, + ), + variant="missing_required_field", + pointers=["/reason_code"], + ) + + return fixtures, vectors + + +def _record_sse_vectors( + client: Any, + observer_routes: Any, + convey_bridge: Any, + fixtures: list[dict[str, Any]], + vectors: list[dict[str, Any]], +) -> None: + root_response = client.get("/sse/events", buffered=False) + try: + root_heartbeat = _next_sse_chunk(root_response) + _record_sse_fixture( + fixtures, + vectors, + fixture_id="recorded.sse.root.heartbeat", + vector_id="callosum.rootEvents.sse.heartbeat", + operation_id="callosum.rootEvents", + variant="heartbeat", + payload=root_heartbeat, + frame_kind="heartbeat", + validates=None, + ) + convey_bridge._broadcast_to_sse_clients( + {"tract": "future", "event": "unknown", "ts": 0, "extra": "value"} + ) + root_data = _parse_sse_data(_next_sse_chunk(root_response)) + _record_sse_fixture( + fixtures, + vectors, + fixture_id="recorded.sse.root.data_unknown_event", + vector_id="callosum.rootEvents.sse.data_unknown_event", + operation_id="callosum.rootEvents", + variant="data_unknown_event", + payload=root_data, + frame_kind="data", + validates=True, + ) + finally: + root_response.close() + + sse_key = _register_observer(client, "vector-sse") + observer_response = client.get( + "/app/observer/callosum", + headers={"Authorization": f"Bearer {sse_key}"}, + buffered=False, + ) + try: + observer_heartbeat = _next_sse_chunk(observer_response) + _record_sse_fixture( + fixtures, + vectors, + fixture_id="recorded.sse.observer.heartbeat", + vector_id="observer.callosumStream.sse.heartbeat", + operation_id="observer.callosumStream", + variant="heartbeat", + payload=observer_heartbeat, + frame_kind="heartbeat", + validates=None, + ) + convey_bridge._broadcast_callosum_event( + {"tract": "observe", "event": "status", "ts": 0, "extra": "value"} + ) + observer_data = _parse_sse_data(_next_sse_chunk(observer_response)) + _record_sse_fixture( + fixtures, + vectors, + fixture_id="recorded.sse.observer.data", + vector_id="observer.callosumStream.sse.data", + operation_id="observer.callosumStream", + variant="data", + payload=observer_data, + frame_kind="data", + validates=True, + ) + finally: + observer_response.close() + + error_key = _register_observer(client, "vector-sse-error") + error_response = client.get( + "/app/observer/callosum", + headers={protocol.OBSERVER_HANDLE_HEADER: error_key}, + buffered=False, + ) + try: + _next_sse_chunk(error_response) + from solstone.apps.observer.utils import load_observer, save_observer + + observer = load_observer(error_key) + if observer is None: + raise ObserverBundleError("failed to reload observer for SSE error vector") + observer["revoked"] = True + if not save_observer(observer): + raise ObserverBundleError("failed to save revoked observer for SSE vector") + with _temporary_attr(observer_routes, "_SSE_HEARTBEAT_SECONDS", 0.01): + error_chunk = _next_sse_chunk(error_response) + error_payload = _parse_sse_error(error_chunk) + _record_sse_fixture( + fixtures, + vectors, + fixture_id="recorded.sse.observer.error", + vector_id="observer.callosumStream.sse.error", + operation_id="observer.callosumStream", + variant="error", + payload=error_payload, + frame_kind="error", + validates=True, + ) + finally: + error_response.close() + + +def _declared_negative_fixtures() -> list[dict[str, Any]]: + return [ + _fixture( + fixture_id="declared.observer.ingestSegments.envelope_total_mismatch", + kind="declared-negative", + operation_id="observer.ingestSegments", + direction="response", + status=200, + media_type="application/json", + variant="envelope_total_mismatch", + payload={ + "items": [], + "protocol_version": protocol.OBSERVER_PROTOCOL_VERSION, + "total": 1, + }, + validates=True, + ), + _fixture( + fixture_id="declared.observer.ingestSegments.custody_unknown_rejected", + kind="declared-negative", + operation_id="observer.ingestSegments", + direction="response", + status=200, + media_type="application/json", + variant="custody_unknown", + payload={ + "items": [ + { + "files": [ + { + "name": "audio.flac", + "sha256": "0" * 64, + "size": 1, + "status": "unknown", + } + ], + "key": "120000_300", + "observed": False, + } + ], + "protocol_version": protocol.OBSERVER_PROTOCOL_VERSION, + "total": 1, + }, + validates=False, + ), + _fixture( + fixture_id="declared.observer.ingestUpload.status_unknown_rejected", + kind="declared-negative", + operation_id="observer.ingestUpload", + direction="response", + status=200, + media_type="application/json", + variant="status_unknown", + payload={"status": "unknown"}, + validates=False, + ), + ] + + +def _declared_negative_vectors(fixtures: list[dict[str, Any]]) -> list[dict[str, Any]]: + pointers_by_fixture = { + "declared.observer.ingestSegments.custody_unknown_rejected": [ + "/items/0/files/0/status" + ], + "declared.observer.ingestSegments.envelope_total_mismatch": ["/total"], + "declared.observer.ingestUpload.status_unknown_rejected": ["/status"], + } + vectors: list[dict[str, Any]] = [] + fixtures_by_id = {fixture["id"]: fixture for fixture in fixtures} + for fixture_id, pointers in sorted(pointers_by_fixture.items()): + fixture = fixtures_by_id[fixture_id] + vector_id = fixture_id.removeprefix("declared.") + vectors.append( + { + "decision": _decision_for_declared_vector(vector_id), + "fixture_id": fixture_id, + "id": vector_id, + "kind": "declared", + "pointer_hashes": _pointer_hashes(fixture["payload"], pointers), + "pointers": pointers, + } + ) + return vectors + + +def _validate_fixtures( + projection: dict[str, Any], fixtures: Iterable[dict[str, Any]] +) -> None: + components = projection.get("components", {}).get("schemas", {}) + for fixture in fixtures: + validates = fixture.get("schema_validation", {}).get("validates") + if validates is None: + continue + schema = _schema_for_fixture(projection, fixture) + schema = _inline_refs(schema, components) + Draft202012Validator.check_schema(schema) + errors = list(Draft202012Validator(schema).iter_errors(fixture["payload"])) + if validates and errors: + raise ObserverBundleError( + f"fixture {fixture['id']} failed schema validation: {errors[0].message}" + ) + if not validates and not errors: + raise ObserverBundleError( + f"fixture {fixture['id']} unexpectedly passed schema validation" + ) + + +def _schema_for_fixture(projection: dict[str, Any], fixture: dict[str, Any]) -> dict: + operation = _operation_by_id(projection, fixture["provenance"]["operation_id"]) + direction = fixture["provenance"]["direction"] + media_type = fixture["provenance"]["media_type"] + if direction == "request": + content = operation["requestBody"]["content"] + elif direction == "response": + status = str(fixture["provenance"]["status"]) + response = operation["responses"][status] + if ( + fixture["provenance"]["media_type"] == "text/event-stream" + and fixture["provenance"]["named_variant"] == "error" + ): + frame = response.get("x-sse-error-frame", {}) + schema = frame.get("schema") if isinstance(frame, dict) else None + if not isinstance(schema, dict): + raise ObserverBundleError( + f"SSE error fixture {fixture['id']} has no error schema" + ) + return copy.deepcopy(schema) + content = response["content"] + else: + raise ObserverBundleError(f"unknown fixture direction: {direction}") + return copy.deepcopy(content[media_type]["schema"]) + + +def _operation_by_id(projection: dict[str, Any], operation_id: str) -> dict[str, Any]: + for _path, _method, operation in _iter_operations(projection): + if operation["operationId"] == operation_id: + return operation + raise ObserverBundleError(f"operation not found in projection: {operation_id}") + + +def _inline_refs(schema: Any, components: dict[str, Any]) -> Any: + if isinstance(schema, dict): + ref = schema.get("$ref") + if isinstance(ref, str): + name = _schema_name_from_ref(ref) + if name is None or name not in components: + raise ObserverBundleError(f"cannot inline ref: {ref}") + return _inline_refs(components[name], components) + return {key: _inline_refs(value, components) for key, value in schema.items()} + if isinstance(schema, list): + return [_inline_refs(item, components) for item in schema] + return schema + + +def _record_response( + fixtures: list[dict[str, Any]], + vectors: list[dict[str, Any]], + *, + fixture_id: str, + vector_id: str, + operation_id: str, + response: Any, + variant: str, + pointers: list[str], +) -> None: + payload = response.get_json() + if payload is None: + raise ObserverBundleError(f"{vector_id} did not return JSON") + stable_payload = _stable_payload(payload) + fixtures.append( + _fixture( + fixture_id=fixture_id, + kind="recorded-response", + operation_id=operation_id, + direction="response", + status=response.status_code, + media_type="application/json", + variant=variant, + payload=stable_payload, + validates=True, + ) + ) + vectors.append( + { + "decision": _decision_for_response_vector( + vector_id, + stable_payload, + response.status_code, + ), + "fixture_id": fixture_id, + "id": vector_id, + "kind": "recorded", + "observed_status": response.status_code, + "pointer_hashes": _pointer_hashes(stable_payload, pointers), + "pointers": pointers, + } + ) + + +def _record_sse_fixture( + fixtures: list[dict[str, Any]], + vectors: list[dict[str, Any]], + *, + fixture_id: str, + vector_id: str, + operation_id: str, + variant: str, + payload: object, + frame_kind: str, + validates: bool | None, +) -> None: + stable_payload = _stable_payload(payload) + if isinstance(payload, str): + pointers = [""] + elif frame_kind == "error": + pointers = ["/reason_code"] + else: + pointers = ["/tract", "/event"] + fixtures.append( + _fixture( + fixture_id=fixture_id, + kind="recorded-sse-frame", + operation_id=operation_id, + direction="response", + status=200, + media_type="text/event-stream", + variant=variant, + payload=stable_payload, + validates=validates, + ) + ) + vectors.append( + { + "decision": _decision_for_sse_vector(vector_id, frame_kind, stable_payload), + "fixture_id": fixture_id, + "frame_kind": frame_kind, + "id": vector_id, + "kind": "recorded", + "pointer_hashes": _pointer_hashes(stable_payload, pointers), + "pointers": pointers, + } + ) + + +def _decision_for_response_vector( + vector_id: str, + payload: object, + observed_status: int, +) -> dict[str, Any]: + if vector_id.startswith("observer.ingestSegments.legacy_array."): + header = "absent" if vector_id.endswith(".absent_header") else "unparseable" + return { + "absent_or_unparseable_uses": 1, + "header": header, + "kind": "protocol_variant", + "parsed_version": 1, + "response_variant": "legacy_array", + } + if not isinstance(payload, dict): + raise ObserverBundleError(f"{vector_id} response payload is not an object") + if vector_id.startswith("observer.ingestUpload.status."): + status = str(payload.get("status")) + decisions = { + "ok": { + "accepted": True, + "client_action": "adopt_segment", + "http_status": 200, + "kind": "ingest_status", + "status": "ok", + "stored_key_source": "segment", + "stored_key_precedence": ["segment"], + }, + "duplicate": { + "accepted": True, + "client_action": "adopt_existing_segment_without_reupload", + "http_status": 200, + "kind": "ingest_status", + "status": "duplicate", + "stored_key_source": "existing_segment", + "stored_key_precedence": ["existing_segment"], + }, + "collision": { + "accepted": True, + "client_action": "adopt_remapped_segment", + "http_status": 200, + "kind": "ingest_status", + "original_key_source": "segment_original", + "status": "collision", + "stored_key_source": "segment", + "stored_key_precedence": ["segment", "segment_original"], + }, + "conflict": { + "accepted": False, + "client_action": "preserve_local_and_surface_conflict", + "http_status": 409, + "kind": "ingest_status", + "status": "conflict", + "stored_key_source": "existing_segment", + "stored_key_precedence": ["existing_segment"], + }, + "failed": { + "accepted": False, + "client_action": "preserve_local_and_surface_failure", + "http_status": 422, + "kind": "ingest_status", + "status": "failed", + "stored_key_source": None, + "stored_key_precedence": [], + }, + } + decision = decisions.get(status) + if decision is None: + raise ObserverBundleError(f"{vector_id} has unknown ingest status {status}") + if decision["http_status"] != observed_status: + raise ObserverBundleError( + f"{vector_id} expected HTTP {decision['http_status']}, got {observed_status}" + ) + return decision + if vector_id == "observer.ingestSegments.submitted_name_fallback": + return { + "fallback": "name", + "kind": "submitted_name_fallback", + "submitted_name_present": False, + } + if vector_id == "observer.ingestSegments.custody_statuses": + return { + "holding_by_status": { + "missing": "not_held", + "present": "held", + "processed": "held", + }, + "kind": "custody_status", + "unknown_status": "reject", + } + if vector_id.startswith("observer.auth."): + auth_form = ( + "authorization_bearer" + if vector_id == "observer.auth.bearer" + else "x_solstone_observer" + ) + return { + "accepted": True, + "auth_form": auth_form, + "kind": "auth_header_form", + "precedence": "x_solstone_observer_preferred_when_both_present", + } + if vector_id == "observer.ingestSegments.v2_envelope": + return { + "current_protocol_version": protocol.OBSERVER_PROTOCOL_VERSION, + "header": "2", + "kind": "protocol_variant", + "parsed_version": 2, + "response_variant": "v2_envelope", + } + if vector_id == "chat.openSolChatRequest.ok": + return { + "accepted": True, + "kind": "chat_open_request", + "missing_field_behavior": "non_empty_trimmed_request_id_required", + "result": "ok_true", + } + if vector_id == "chat.openSolChatRequest.missing_required_field": + return { + "accepted": False, + "kind": "chat_open_request", + "missing_field_behavior": "absent_malformed_empty_or_blank_rejected", + "reason_code": "missing_required_field", + } + raise ObserverBundleError(f"no response decision declared for vector {vector_id}") + + +def _decision_for_sse_vector( + vector_id: str, + frame_kind: str, + payload: object, +) -> dict[str, Any]: + if frame_kind == "heartbeat": + return { + "action": "ignore_keepalive", + "frame_kind": "heartbeat", + "kind": "sse_frame", + } + if vector_id == "callosum.rootEvents.sse.data_unknown_event": + return { + "action": "pass_through", + "frame_kind": "data", + "kind": "sse_frame", + "unknown_event_behavior": "preserve", + } + if vector_id == "observer.callosumStream.sse.data": + return { + "action": "dispatch_callosum_event", + "frame_kind": "data", + "kind": "sse_frame", + "unknown_event_behavior": "preserve", + } + if vector_id == "observer.callosumStream.sse.error": + return { + "action": "surface_error_and_close", + "frame_kind": "error", + "kind": "sse_frame", + "reason_code": _required_payload_str(payload, "reason_code", vector_id), + } + raise ObserverBundleError(f"no SSE decision declared for vector {vector_id}") + + +def _decision_for_declared_vector(vector_id: str) -> dict[str, Any]: + if vector_id == "observer.ingestSegments.envelope_total_mismatch": + return { + "expected": "total_equals_items_length", + "kind": "envelope_integrity", + "valid": False, + } + if vector_id == "observer.ingestSegments.custody_unknown_rejected": + return { + "kind": "custody_unknown", + "status": "unknown", + "unknown_status": "reject", + } + if vector_id == "observer.ingestUpload.status_unknown_rejected": + return { + "kind": "closed_vocabulary_unknown", + "status": "unknown", + "unknown_value_behavior": "reject", + "vocabulary": "observer.ingestUpload.status", + } + raise ObserverBundleError(f"no declared decision for vector {vector_id}") + + +def _required_payload_str(payload: object, field: str, label: str) -> str: + if not isinstance(payload, dict) or not isinstance(payload.get(field), str): + raise ObserverBundleError(f"{label} payload missing string {field}") + return payload[field] + + +def _fixture( + *, + fixture_id: str, + kind: str, + operation_id: str, + direction: str, + status: int | None, + media_type: str, + variant: str, + payload: object, + validates: bool | None, +) -> dict[str, Any]: + validation = {"validates": validates} + if validates is None: + validation["reason"] = "SSE heartbeat comments are not JSON schema payloads" + return { + "id": fixture_id, + "kind": kind, + "payload": payload, + "provenance": { + "direction": direction, + "media_type": media_type, + "named_variant": variant, + "operation_id": operation_id, + "status": status, + }, + "schema_validation": validation, + } + + +def _fixture_id( + prefix: str, + operation_id: str, + direction: str, + status: str, + media_type: str, + variant: str, +) -> str: + media = media_type.replace("/", "-").replace("+", "-") + return f"{prefix}.{operation_id}.{direction}.{status}.{media}.{variant}" + + +def _register_observer(client: Any, name: str) -> str: + response = client.post( + "/app/observer/register", + json={ + "hostname": name, + "platform": "linux", + "stream_type": "desktop", + "version": "1", + }, + ) + if response.status_code != 200: + raise ObserverBundleError( + f"observer registration failed: {response.get_data(as_text=True)}" + ) + body = response.get_json() + if not isinstance(body, dict) or not body.get("key"): + raise ObserverBundleError("observer registration response missing key") + return str(body["key"]) + + +def _upload( + client: Any, + key: str, + day: str, + segment: str, + files: list[tuple[bytes, str]], +) -> Any: + return client.post( + "/app/observer/ingest", + headers={"Authorization": f"Bearer {key}"}, + data={ + "day": day, + "segment": segment, + "files": [(BytesIO(content), filename) for content, filename in files], + }, + ) + + +def _write_processing_sidecar(segment_dir: Path, *, input_size: int) -> None: + record = { + "handler": HANDLER_TRANSCRIBE, + "input_size": input_size, + "schema": PROCESSING_SCHEMA, + "state": STATE_EMPTY, + } + row = {"_solstone_processing": record, "raw": "audio.flac"} + (segment_dir / "audio.jsonl").write_text( + json.dumps(row, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def _next_sse_chunk(response: Any) -> str: + chunk = next(response.response) + if isinstance(chunk, bytes): + return chunk.decode("utf-8") + return str(chunk) + + +def _parse_sse_data(chunk: str) -> dict[str, Any]: + prefix = "data: " + if not chunk.startswith(prefix): + raise ObserverBundleError(f"expected SSE data frame, got {chunk!r}") + return json.loads(chunk[len(prefix) :].strip()) + + +def _parse_sse_error(chunk: str) -> dict[str, Any]: + lines = chunk.splitlines() + if lines[:1] != ["event: error"] or len(lines) < 2: + raise ObserverBundleError(f"expected SSE error frame, got {chunk!r}") + return json.loads(lines[1].removeprefix("data: ")) + + +def _prepare_recording_journal(root: Path, destination: Path) -> Path: + shutil.copytree(root / "tests" / "fixtures" / "journal", destination, symlinks=True) + _mark_setup_complete(destination) + return destination.resolve() + + +def _mark_setup_complete(journal: Path) -> None: + config_path = journal / "config" / "journal.json" + config_path.parent.mkdir(parents=True, exist_ok=True) + config = {} + if config_path.exists(): + config = json.loads(config_path.read_text(encoding="utf-8")) + config["setup"] = {"completed_at": 1700000000000} + config_path.write_text(render_json(config), encoding="utf-8") + + +@contextmanager +def _recording_env(journal: Path) -> Iterator[None]: + overrides = { + "SOLSTONE_DISABLE_CONVEY_SIDE_RUNTIMES": "1", + "SOLSTONE_JOURNAL": str(journal), + "SOL_SKIP_SUPERVISOR_CHECK": "1", + } + previous = {key: os.environ.get(key) for key in overrides} + os.environ.update(overrides) + try: + yield + finally: + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +@contextmanager +def _temporary_attr(target: object, name: str, value: object) -> Iterator[None]: + previous = getattr(target, name) + setattr(target, name, value) + try: + yield + finally: + setattr(target, name, previous) + + +def _stable_payload(payload: object) -> object: + if isinstance(payload, dict): + return {key: _stable_payload(payload[key]) for key in sorted(payload)} + if isinstance(payload, list): + return [_stable_payload(item) for item in payload] + return payload + + +def _pointer_hashes(payload: object, pointers: Iterable[str]) -> dict[str, str]: + return { + pointer: _sha256_text(render_json(_resolve_json_pointer(payload, pointer))) + for pointer in pointers + } + + +def _resolve_json_pointer(payload: Any, pointer: str) -> Any: + if pointer == "": + return payload + if not pointer.startswith("/"): + raise BundleVerificationError(f"invalid JSON pointer: {pointer}") + current = payload + for token in pointer[1:].split("/"): + token = token.replace("~1", "/").replace("~0", "~") + if isinstance(current, list): + try: + current = current[int(token)] + except (ValueError, IndexError) as exc: + raise BundleVerificationError( + f"JSON pointer does not resolve: {pointer}" + ) from exc + elif isinstance(current, dict): + if token not in current: + raise BundleVerificationError( + f"JSON pointer does not resolve: {pointer}" + ) + current = current[token] + else: + raise BundleVerificationError(f"JSON pointer does not resolve: {pointer}") + return current diff --git a/solstone/convey/contract/observer_bundle_verification.py b/solstone/convey/contract/observer_bundle_verification.py new file mode 100644 index 000000000..24eb33c2a --- /dev/null +++ b/solstone/convey/contract/observer_bundle_verification.py @@ -0,0 +1,1173 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Verify observer-client contract bundles and consumer-audit coverage.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import stat +from collections.abc import Iterable +from pathlib import Path, PurePosixPath +from typing import Any + +from solstone.convey.contract.observer_bundle import ( + _SHA256_RE, + _SOURCE_INPUTS, + _WINDOWS_RESERVED_BASENAMES, + AUDITED_CONSUMER_REVISIONS, + BUNDLE_REL_DIR, + BUNDLE_SCHEMA_IDENTITY, + CONSUMER_IDENTIFIERS, + GENERATOR_IDENTITY, + MANIFEST_NAME, + OBSERVER_CLIENT_OPERATION_IDS, + SCHEMA_DIALECT_URI, + WINDOWS_LINUX_ROLLOUT_TARGETS, + BundleSnapshot, + BundleVerificationError, + ObserverBundleError, + _directory_entry_snapshot, + _entry_snapshot_identity, + _is_relative_to, + _iter_operations, + _open_dir_at_no_follow, + _open_parent_dir_no_follow, + _read_regular_at_no_follow, + _repo_root, + _sha256_path, + _sha256_text, + _stat_identity, + parse_semver, + render_json, + validate_projection_refs, +) +from solstone.observe import protocol + +_CURRENT_VECTOR_DECISIONS: dict[str, dict[str, Any]] = { + "callosum.rootEvents.sse.data_unknown_event": { + "action": "pass_through", + "frame_kind": "data", + "kind": "sse_frame", + "unknown_event_behavior": "preserve", + }, + "callosum.rootEvents.sse.heartbeat": { + "action": "ignore_keepalive", + "frame_kind": "heartbeat", + "kind": "sse_frame", + }, + "chat.openSolChatRequest.missing_required_field": { + "accepted": False, + "kind": "chat_open_request", + "missing_field_behavior": "absent_malformed_empty_or_blank_rejected", + "reason_code": "missing_required_field", + }, + "chat.openSolChatRequest.ok": { + "accepted": True, + "kind": "chat_open_request", + "missing_field_behavior": "non_empty_trimmed_request_id_required", + "result": "ok_true", + }, + "observer.auth.bearer": { + "accepted": True, + "auth_form": "authorization_bearer", + "kind": "auth_header_form", + "precedence": "x_solstone_observer_preferred_when_both_present", + }, + "observer.auth.handle": { + "accepted": True, + "auth_form": "x_solstone_observer", + "kind": "auth_header_form", + "precedence": "x_solstone_observer_preferred_when_both_present", + }, + "observer.callosumStream.sse.data": { + "action": "dispatch_callosum_event", + "frame_kind": "data", + "kind": "sse_frame", + "unknown_event_behavior": "preserve", + }, + "observer.callosumStream.sse.error": { + "action": "surface_error_and_close", + "frame_kind": "error", + "kind": "sse_frame", + "reason_code": "pl_revoked", + }, + "observer.callosumStream.sse.heartbeat": { + "action": "ignore_keepalive", + "frame_kind": "heartbeat", + "kind": "sse_frame", + }, + "observer.ingestSegments.custody_statuses": { + "holding_by_status": { + "missing": "not_held", + "present": "held", + "processed": "held", + }, + "kind": "custody_status", + "unknown_status": "reject", + }, + "observer.ingestSegments.custody_unknown_rejected": { + "kind": "custody_unknown", + "status": "unknown", + "unknown_status": "reject", + }, + "observer.ingestSegments.envelope_total_mismatch": { + "expected": "total_equals_items_length", + "kind": "envelope_integrity", + "valid": False, + }, + "observer.ingestSegments.legacy_array.absent_header": { + "absent_or_unparseable_uses": 1, + "header": "absent", + "kind": "protocol_variant", + "parsed_version": 1, + "response_variant": "legacy_array", + }, + "observer.ingestSegments.legacy_array.unparseable_header": { + "absent_or_unparseable_uses": 1, + "header": "unparseable", + "kind": "protocol_variant", + "parsed_version": 1, + "response_variant": "legacy_array", + }, + "observer.ingestSegments.submitted_name_fallback": { + "fallback": "name", + "kind": "submitted_name_fallback", + "submitted_name_present": False, + }, + "observer.ingestSegments.v2_envelope": { + "current_protocol_version": 2, + "header": "2", + "kind": "protocol_variant", + "parsed_version": 2, + "response_variant": "v2_envelope", + }, + "observer.ingestUpload.status.collision": { + "accepted": True, + "client_action": "adopt_remapped_segment", + "http_status": 200, + "kind": "ingest_status", + "original_key_source": "segment_original", + "status": "collision", + "stored_key_precedence": ["segment", "segment_original"], + "stored_key_source": "segment", + }, + "observer.ingestUpload.status.conflict": { + "accepted": False, + "client_action": "preserve_local_and_surface_conflict", + "http_status": 409, + "kind": "ingest_status", + "status": "conflict", + "stored_key_precedence": ["existing_segment"], + "stored_key_source": "existing_segment", + }, + "observer.ingestUpload.status.duplicate": { + "accepted": True, + "client_action": "adopt_existing_segment_without_reupload", + "http_status": 200, + "kind": "ingest_status", + "status": "duplicate", + "stored_key_precedence": ["existing_segment"], + "stored_key_source": "existing_segment", + }, + "observer.ingestUpload.status.failed": { + "accepted": False, + "client_action": "preserve_local_and_surface_failure", + "http_status": 422, + "kind": "ingest_status", + "status": "failed", + "stored_key_precedence": [], + "stored_key_source": None, + }, + "observer.ingestUpload.status.ok": { + "accepted": True, + "client_action": "adopt_segment", + "http_status": 200, + "kind": "ingest_status", + "status": "ok", + "stored_key_precedence": ["segment"], + "stored_key_source": "segment", + }, + "observer.ingestUpload.status_unknown_rejected": { + "kind": "closed_vocabulary_unknown", + "status": "unknown", + "unknown_value_behavior": "reject", + "vocabulary": "observer.ingestUpload.status", + }, +} + + +def verify_bundle_directory(bundle_dir: Path) -> BundleSnapshot: + """Verify a bundle directory's manifest, file inventory, and references.""" + + bundle_root = Path(bundle_dir) + files = _read_bundle_directory_files(bundle_root) + return _validate_bundle_snapshot( + files, + source=str(bundle_root), + enforce_current_contract=True, + ) + + +def verify_committed_bundle(root: Path | None = None) -> BundleSnapshot: + """Verify the committed bundle and its current source-input digests.""" + + repo_root = _repo_root(root) + snapshot = verify_bundle_directory(repo_root / BUNDLE_REL_DIR) + _verify_generator_input_records(repo_root, snapshot.manifest) + return snapshot + + +def check_consumer_audit_coverage(root: Path | None = None) -> list[str]: + """Return Windows/Linux consumer-audit coverage failures.""" + + repo_root = _repo_root(root) + try: + snapshot = verify_bundle_directory(repo_root / BUNDLE_REL_DIR) + _validate_consumer_audit_coverage(snapshot) + except ObserverBundleError as exc: + return [str(exc)] + return [] + + +def _read_bundle_directory_files(bundle_root: Path) -> dict[str, bytes]: + parent_fd, leaf_name = _open_parent_dir_no_follow(bundle_root) + try: + try: + bundle_stat = os.stat(leaf_name, dir_fd=parent_fd, follow_symlinks=False) + except FileNotFoundError as exc: + raise BundleVerificationError( + f"bundle directory does not exist: {bundle_root}" + ) from exc + if stat.S_ISLNK(bundle_stat.st_mode): + raise BundleVerificationError( + f"bundle directory is a symlink: {bundle_root}" + ) + if not stat.S_ISDIR(bundle_stat.st_mode): + raise BundleVerificationError( + f"bundle path is not a directory: {bundle_root}" + ) + bundle_fd = _open_dir_at_no_follow(parent_fd, leaf_name, bundle_stat) + finally: + os.close(parent_fd) + try: + return _read_bundle_fd_files( + bundle_fd, + source=str(bundle_root), + rel_parts=(), + expected_stat=os.fstat(bundle_fd), + ) + finally: + os.close(bundle_fd) + + +def verify_bundle_fd(bundle_fd: int, source: str) -> BundleSnapshot: + """Verify a bundle directory already opened by the caller.""" + + files = _read_bundle_fd_files( + bundle_fd, + source=source, + rel_parts=(), + expected_stat=os.fstat(bundle_fd), + ) + return _validate_bundle_snapshot( + files, + source=source, + enforce_current_contract=True, + ) + + +def _read_bundle_fd_files( + dir_fd: int, + *, + source: str, + rel_parts: tuple[bytes, ...], + expected_stat: os.stat_result, +) -> dict[str, bytes]: + before_dir = os.fstat(dir_fd) + if _stat_identity(before_dir) != _stat_identity(expected_stat): + raise BundleVerificationError(f"{source}: bundle directory changed before read") + before_entries = _directory_entry_snapshot(dir_fd) + files: dict[str, bytes] = {} + for name, entry_stat in before_entries: + entry_parts = (*rel_parts, name) + rel_path = _bundle_rel_parts_to_posix(entry_parts) + mode = entry_stat.st_mode + if stat.S_ISLNK(mode): + raise BundleVerificationError(f"bundle path is a symlink: {rel_path}") + if stat.S_ISDIR(mode): + child_fd = _open_dir_at_no_follow(dir_fd, name, entry_stat) + try: + files.update( + _read_bundle_fd_files( + child_fd, + source=source, + rel_parts=entry_parts, + expected_stat=os.fstat(child_fd), + ) + ) + finally: + os.close(child_fd) + continue + if not stat.S_ISREG(mode): + raise BundleVerificationError( + f"bundle path is not a regular file: {rel_path}" + ) + _validate_manifest_relative_path(rel_path) + files[rel_path] = _read_regular_at_no_follow(dir_fd, name, entry_stat) + after_entries = _directory_entry_snapshot(dir_fd) + if _entry_snapshot_identity(after_entries) != _entry_snapshot_identity( + before_entries + ): + raise BundleVerificationError( + f"{source}: bundle directory entries changed during read" + ) + if _stat_identity(os.fstat(dir_fd)) != _stat_identity(before_dir): + raise BundleVerificationError(f"{source}: bundle directory changed during read") + return files + + +def _bundle_rel_parts_to_posix(parts: tuple[bytes, ...]) -> str: + return "/".join(os.fsdecode(part) for part in parts) + + +def _validate_bundle_snapshot( + files: dict[str, bytes], + *, + source: str, + enforce_current_contract: bool, +) -> BundleSnapshot: + if MANIFEST_NAME not in files: + raise BundleVerificationError(f"{source}: missing manifest.json") + manifest = _json_from_bytes(files[MANIFEST_NAME], f"{source}:manifest.json") + if not isinstance(manifest, dict): + raise BundleVerificationError(f"{source}: manifest.json is not an object") + + parse_semver(_required_str(manifest, "bundle_semver", source)) + file_paths = _validate_manifest_file_records(manifest.get("files"), source) + expected_file_sequence = [MANIFEST_NAME, *file_paths] + if sorted(files) != sorted(expected_file_sequence): + missing = sorted(set(expected_file_sequence) - set(files)) + extra = sorted(set(files) - set(expected_file_sequence)) + detail = [] + if missing: + detail.append("missing " + ", ".join(missing)) + if extra: + detail.append("unlisted " + ", ".join(extra)) + raise BundleVerificationError( + f"{source}: bundle file set mismatch: {'; '.join(detail)}" + ) + + for record in manifest["files"]: + rel_path = record["path"] + expected_sha = record["sha256"] + actual_sha = hashlib.sha256(files[rel_path]).hexdigest() + if actual_sha != expected_sha: + raise BundleVerificationError( + f"{source}: digest mismatch for {rel_path}: " + f"expected {expected_sha}, got {actual_sha}" + ) + + _verify_generator_inputs_do_not_self_reference(manifest) + projection = _validate_projection_payload(manifest, files, source) + _validate_fixture_vector_references( + files, + source, + enforce_current_contract=enforce_current_contract, + ) + snapshot = BundleSnapshot( + manifest=manifest, + files={path: files[path] for path in sorted(expected_file_sequence)}, + ) + if enforce_current_contract: + _validate_current_contract_requirements(snapshot, projection, source) + return snapshot + + +def _validate_manifest_file_records(records: object, source: str) -> list[str]: + if not isinstance(records, list): + raise BundleVerificationError(f"{source}: manifest.files is not a list") + paths: list[str] = [] + for index, record in enumerate(records): + if not isinstance(record, dict): + raise BundleVerificationError( + f"{source}: manifest.files[{index}] is not an object" + ) + path = record.get("path") + digest = record.get("sha256") + if not isinstance(path, str): + raise BundleVerificationError( + f"{source}: manifest.files[{index}].path is not a string" + ) + if not isinstance(digest, str) or not _SHA256_RE.fullmatch(digest): + raise BundleVerificationError( + f"{source}: manifest.files[{index}].sha256 is not a SHA-256 digest" + ) + paths.append(path) + _validate_manifest_relative_paths(paths) + if paths != sorted(paths): + raise BundleVerificationError(f"{source}: manifest.files is not sorted by path") + return paths + + +def _validate_manifest_relative_paths(paths: Iterable[str]) -> None: + seen: set[str] = set() + seen_casefolded: dict[str, str] = {} + for path in paths: + normalized = _validate_manifest_relative_path(path) + if normalized in seen: + raise BundleVerificationError(f"duplicate bundle path: {path}") + seen.add(normalized) + folded = normalized.casefold() + if folded in seen_casefolded: + raise BundleVerificationError( + f"case-fold-colliding bundle paths: {seen_casefolded[folded]} and {path}" + ) + seen_casefolded[folded] = normalized + + +def _validate_manifest_relative_path(path: str) -> str: + if path == "": + raise BundleVerificationError("empty bundle path") + if "\\" in path: + raise BundleVerificationError(f"bundle path contains backslash: {path}") + if any(ord(char) < 32 or ord(char) == 127 for char in path): + raise BundleVerificationError( + f"bundle path contains control character: {path!r}" + ) + if re.search(r"(^|/)[A-Za-z]:", path): + raise BundleVerificationError( + f"bundle path contains Windows drive prefix: {path}" + ) + if ":" in path: + raise BundleVerificationError(f"bundle path contains colon: {path}") + if any(char in path for char in "*?[]"): + raise BundleVerificationError( + f"bundle path contains wildcard character: {path}" + ) + raw_parts = path.split("/") + if any(part == "" for part in raw_parts): + raise BundleVerificationError(f"bundle path has empty component: {path}") + if any(part in {".", ".."} for part in raw_parts): + raise BundleVerificationError(f"bundle path has unsafe component: {path}") + pure_path = PurePosixPath(path) + if pure_path.is_absolute(): + raise BundleVerificationError(f"bundle path is absolute: {path}") + parts = pure_path.parts + if not parts: + raise BundleVerificationError(f"bundle path has empty component: {path}") + for part in parts: + if part.endswith("."): + raise BundleVerificationError( + f"bundle path component has trailing dot: {path}" + ) + if part.endswith(" "): + raise BundleVerificationError( + f"bundle path component has trailing space: {path}" + ) + basename = part.split(".", 1)[0].upper() + if basename in _WINDOWS_RESERVED_BASENAMES: + raise BundleVerificationError( + f"bundle path uses Windows-reserved device name: {path}" + ) + return pure_path.as_posix() + + +def _json_from_bytes(payload: bytes, label: str) -> Any: + try: + return json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise BundleVerificationError(f"{label}: invalid JSON") from exc + + +def _required_str(payload: dict[str, Any], field: str, source: str) -> str: + value = payload.get(field) + if not isinstance(value, str) or not value: + raise BundleVerificationError(f"{source}: manifest.{field} is required") + return value + + +def _validate_projection_payload( + manifest: dict[str, Any], files: dict[str, bytes], source: str +) -> dict[str, Any]: + projection_path = manifest.get("projection_path") + if not isinstance(projection_path, str): + raise BundleVerificationError(f"{source}: manifest.projection_path is required") + _validate_manifest_relative_path(projection_path) + if projection_path not in files: + raise BundleVerificationError(f"{source}: projection file is missing") + projection = _json_from_bytes(files[projection_path], f"{source}:{projection_path}") + if not isinstance(projection, dict): + raise BundleVerificationError(f"{source}: projection document is not an object") + try: + validate_projection_refs(projection) + except ObserverBundleError as exc: + raise BundleVerificationError(f"{source}: {exc}") from exc + + manifest_operations = manifest.get("operation_ids") + if not isinstance(manifest_operations, list) or not all( + isinstance(item, str) for item in manifest_operations + ): + raise BundleVerificationError(f"{source}: manifest.operation_ids is invalid") + projected_operations = sorted( + operation["operationId"] + for _path, _method, operation in _iter_operations(projection) + ) + if projected_operations != manifest_operations: + raise BundleVerificationError( + f"{source}: projection operation IDs do not match manifest.operation_ids" + ) + manifest_components = manifest.get("component_closure") + if not isinstance(manifest_components, list) or not all( + isinstance(item, str) for item in manifest_components + ): + raise BundleVerificationError( + f"{source}: manifest.component_closure is invalid" + ) + projected_components = sorted(projection.get("components", {}).get("schemas", {})) + if projected_components != manifest_components: + raise BundleVerificationError( + f"{source}: projection components do not match manifest.component_closure" + ) + return projection + + +def _validate_current_contract_requirements( + snapshot: BundleSnapshot, + projection: dict[str, Any], + source: str, +) -> None: + manifest = snapshot.manifest + if manifest.get("generator_identity") != GENERATOR_IDENTITY: + raise BundleVerificationError(f"{source}: unexpected generator_identity") + if manifest.get("bundle_schema_identity") != BUNDLE_SCHEMA_IDENTITY: + raise BundleVerificationError(f"{source}: unexpected bundle_schema_identity") + if manifest.get("schema_dialect_uri") != SCHEMA_DIALECT_URI: + raise BundleVerificationError(f"{source}: unexpected schema_dialect_uri") + if manifest.get("openapi_document_version") != projection.get("info", {}).get( + "version" + ): + raise BundleVerificationError( + f"{source}: manifest.openapi_document_version must match projection info.version" + ) + if manifest.get("openapi_spec_version") != projection.get("openapi"): + raise BundleVerificationError( + f"{source}: manifest.openapi_spec_version must match projection openapi" + ) + if manifest.get("observer_protocol_version") != protocol.OBSERVER_PROTOCOL_VERSION: + raise BundleVerificationError(f"{source}: unexpected observer_protocol_version") + if manifest.get("supported_response_variants") != [1, 2]: + raise BundleVerificationError( + f"{source}: unexpected supported_response_variants" + ) + if manifest.get("operation_ids") != list(OBSERVER_CLIENT_OPERATION_IDS): + raise BundleVerificationError(f"{source}: unexpected operation_ids") + if manifest.get("consumer_identifiers") != CONSUMER_IDENTIFIERS: + raise BundleVerificationError(f"{source}: unexpected consumer_identifiers") + if manifest.get("audited_consumer_revisions") != AUDITED_CONSUMER_REVISIONS: + raise BundleVerificationError( + f"{source}: unexpected audited_consumer_revisions" + ) + if manifest.get("windows_linux_rollout_targets") != WINDOWS_LINUX_ROLLOUT_TARGETS: + raise BundleVerificationError( + f"{source}: unexpected windows_linux_rollout_targets" + ) + if manifest.get("component_closure") != [ + "CallosumEvent", + "Error", + "SegmentFile", + "SegmentItem", + "SegmentsEnvelope", + ]: + raise BundleVerificationError(f"{source}: unexpected component_closure") + + operation_locations = _operation_locations(projection) + expected_locations = { + "callosum.rootEvents": ("/sse/events", "get"), + "chat.openSolChatRequest": ("/api/chat/sol_chat_request/open", "post"), + "link.pair": ("/app/network/pair", "post"), + "observer.callosumStream": ("/app/observer/callosum", "get"), + "observer.ingestEvent": ("/app/observer/ingest/event", "post"), + "observer.ingestSegments": ("/app/observer/ingest/segments/{day}", "get"), + "observer.ingestUpload": ("/app/observer/ingest", "post"), + "observer.register": ("/app/observer/register", "post"), + } + for operation_id, expected in expected_locations.items(): + location = operation_locations.get(operation_id) + if location is None or location[:2] != expected: + raise BundleVerificationError( + f"{source}: unexpected location for {operation_id}" + ) + + chat_operation = _operation_by_id(projection, "chat.openSolChatRequest") + request_schema = chat_operation["requestBody"]["content"]["application/json"][ + "schema" + ] + if request_schema.get("required") != ["request_id"]: + raise BundleVerificationError( + f"{source}: chat.openSolChatRequest request_id must be required" + ) + request_id = request_schema.get("properties", {}).get("request_id") + if request_id != {"minLength": 1, "pattern": "\\S", "type": "string"}: + raise BundleVerificationError( + f"{source}: chat.openSolChatRequest request_id must be a non-blank string" + ) + success_schema = chat_operation["responses"]["200"]["content"]["application/json"][ + "schema" + ] + if success_schema.get("required") != ["ok"]: + raise BundleVerificationError( + f"{source}: chat.openSolChatRequest ok must be required" + ) + if success_schema.get("properties", {}).get("ok") != {"type": "boolean"}: + raise BundleVerificationError( + f"{source}: chat.openSolChatRequest ok must be boolean" + ) + if chat_operation["responses"]["400"].get("x-reason-codes") != [ + "missing_required_field" + ]: + raise BundleVerificationError( + f"{source}: chat.openSolChatRequest 400 reason must be missing_required_field" + ) + if chat_operation["responses"]["403"].get("x-reason-codes") != ["pl_revoked"]: + raise BundleVerificationError( + f"{source}: chat.openSolChatRequest 403 reason must be pl_revoked" + ) + + upload_schema = _response_json_schema( + projection, + "observer.ingestUpload", + "200", + ) + upload_status = upload_schema.get("properties", {}).get("status", {}) + if upload_status.get("enum") != [ + "ok", + "duplicate", + "collision", + "conflict", + "failed", + ]: + raise BundleVerificationError(f"{source}: ingest upload status enum drifted") + upload_vocab = upload_status.get("x-vocabulary", {}) + if ( + upload_vocab.get("classification") != "closed" + or upload_vocab.get("unknown_value_behavior") != "reject" + ): + raise BundleVerificationError( + f"{source}: ingest upload status unknown-value behavior drifted" + ) + + segment_file = projection["components"]["schemas"]["SegmentFile"] + custody_status = segment_file["properties"]["status"] + if custody_status.get("enum") != ["present", "missing", "processed"]: + raise BundleVerificationError(f"{source}: SegmentFile.status enum drifted") + custody_vocab = custody_status.get("x-vocabulary", {}) + if ( + custody_vocab.get("classification") != "closed" + or custody_vocab.get("unknown_value_behavior") != "reject" + ): + raise BundleVerificationError( + f"{source}: SegmentFile.status unknown-value behavior drifted" + ) + + segments_operation = _operation_by_id(projection, "observer.ingestSegments") + protocol_vocab = segments_operation["responses"]["200"].get("x-vocabularies", {}) + if protocol_vocab.get("X-Solstone-Protocol-Version", {}).get("current") != 2: + raise BundleVerificationError( + f"{source}: X-Solstone-Protocol-Version current value drifted" + ) + + root_operation = _operation_by_id(projection, "callosum.rootEvents") + chat_events = root_operation["responses"]["200"].get("x-chat-events", {}) + if chat_events.get("classification") != "extensible": + raise BundleVerificationError( + f"{source}: root SSE chat events must be extensible" + ) + if chat_events.get("unknown_value_behavior") != "preserve": + raise BundleVerificationError( + f"{source}: root SSE chat events must preserve unknown values" + ) + + observer_sse = _operation_by_id(projection, "observer.callosumStream") + observer_frames = observer_sse["responses"]["200"].get("x-sse-frame-kinds", {}) + if observer_frames.get("values") != ["data", "error", "heartbeat"]: + raise BundleVerificationError(f"{source}: observer SSE frame kinds drifted") + + +def _response_json_schema( + projection: dict[str, Any], operation_id: str, status: str +) -> dict[str, Any]: + operation = _operation_by_id(projection, operation_id) + return operation["responses"][status]["content"]["application/json"]["schema"] + + +def _validate_fixture_vector_references( + files: dict[str, bytes], + source: str, + *, + enforce_current_contract: bool, +) -> None: + fixtures = _json_from_bytes( + files["fixtures/wire-behavior.json"], + f"{source}:fixtures/wire-behavior.json", + ) + vectors = _json_from_bytes(files["vectors.json"], f"{source}:vectors.json") + if not isinstance(fixtures, dict) or not isinstance(vectors, dict): + raise BundleVerificationError(f"{source}: fixtures/vectors must be objects") + if fixtures.get("schema") != "solstone.observer-client-contract-fixtures.v1": + raise BundleVerificationError(f"{source}: unknown fixture schema") + if vectors.get("schema") != "solstone.observer-client-contract-vectors.v1": + raise BundleVerificationError(f"{source}: unknown vector schema") + fixture_items = fixtures.get("fixtures") + vector_items = vectors.get("vectors") + if not isinstance(fixture_items, list) or not isinstance(vector_items, list): + raise BundleVerificationError(f"{source}: fixtures/vectors arrays are required") + fixture_ids: list[str] = [] + fixtures_by_id: dict[str, dict[str, Any]] = {} + for index, item in enumerate(fixture_items): + if not isinstance(item, dict): + raise BundleVerificationError( + f"{source}: fixture entry {index} is not an object" + ) + fixture_id = item.get("id") + if not isinstance(fixture_id, str) or not fixture_id: + raise BundleVerificationError(f"{source}: fixture entry {index} missing id") + if fixture_id in fixtures_by_id: + raise BundleVerificationError( + f"{source}: duplicate fixture id {fixture_id}" + ) + fixture_ids.append(fixture_id) + fixtures_by_id[fixture_id] = item + if fixture_ids != sorted(fixture_ids): + raise BundleVerificationError(f"{source}: fixture IDs are not sorted") + + vector_ids: list[str] = [] + seen_vector_ids: set[str] = set() + for vector in vector_items: + if not isinstance(vector, dict): + raise BundleVerificationError(f"{source}: vector entry is not an object") + vector_id = vector.get("id") + fixture_id = vector.get("fixture_id") + pointers = vector.get("pointers") + if not isinstance(vector_id, str) or not vector_id: + raise BundleVerificationError(f"{source}: vector missing id") + if vector_id in seen_vector_ids: + raise BundleVerificationError(f"{source}: duplicate vector id {vector_id}") + vector_ids.append(vector_id) + seen_vector_ids.add(vector_id) + if not isinstance(fixture_id, str) or fixture_id not in fixtures_by_id: + raise BundleVerificationError( + f"{source}: vector {vector_id} references missing fixture {fixture_id}" + ) + if not isinstance(pointers, list) or not all( + isinstance(pointer, str) for pointer in pointers + ): + raise BundleVerificationError( + f"{source}: vector {vector_id} pointers invalid" + ) + payload = fixtures_by_id[fixture_id].get("payload") + pointer_hashes = vector.get("pointer_hashes") + if pointer_hashes is not None and not isinstance(pointer_hashes, dict): + raise BundleVerificationError( + f"{source}: vector {vector_id} pointer_hashes invalid" + ) + for pointer in pointers: + value = _resolve_json_pointer(payload, pointer) + if isinstance(pointer_hashes, dict): + expected_hash = pointer_hashes.get(pointer) + actual_hash = _sha256_text(render_json(value)) + if expected_hash != actual_hash: + raise BundleVerificationError( + f"{source}: vector {vector_id} hash mismatch at {pointer}" + ) + _validate_vector_decision( + vector, + source, + enforce_current_contract=enforce_current_contract, + ) + if vector_ids != sorted(vector_ids): + raise BundleVerificationError(f"{source}: vector IDs are not sorted") + + +def _validate_vector_decision( + vector: dict[str, Any], + source: str, + *, + enforce_current_contract: bool, +) -> None: + vector_id = str(vector.get("id")) + decision = vector.get("decision") + if not isinstance(decision, dict): + raise BundleVerificationError(f"{source}: vector {vector_id} missing decision") + _validate_released_vector_decision_shape(vector_id, decision, source) + if not enforce_current_contract: + return + expected = _CURRENT_VECTOR_DECISIONS.get(vector_id) + if expected is None: + raise BundleVerificationError( + f"{source}: vector {vector_id} is not in the current policy table" + ) + if decision != expected: + raise BundleVerificationError(f"{source}: vector {vector_id} decision mismatch") + + +def _validate_released_vector_decision_shape( + vector_id: str, decision: dict[str, Any], source: str +) -> None: + kind = decision.get("kind") + if not isinstance(kind, str) or not kind: + raise BundleVerificationError( + f"{source}: vector {vector_id} has invalid decision kind" + ) + if kind == "independent_behavior": + _require_string(decision, "change_scope", source, vector_id) + _require_string(decision, "description", source, vector_id) + return + if kind == "ingest_status": + _require_string(decision, "status", source, vector_id) + _require_bool(decision, "accepted", source, vector_id) + _require_int(decision, "http_status", source, vector_id) + _require_string(decision, "client_action", source, vector_id) + _require_string_list(decision, "stored_key_precedence", source, vector_id) + if decision.get("stored_key_source") is not None: + _require_string(decision, "stored_key_source", source, vector_id) + if "original_key_source" in decision: + _require_string(decision, "original_key_source", source, vector_id) + return + if kind == "sse_frame": + _require_string(decision, "frame_kind", source, vector_id) + _require_string(decision, "action", source, vector_id) + for optional in ("unknown_event_behavior", "reason_code"): + if optional in decision: + _require_string(decision, optional, source, vector_id) + return + if kind == "protocol_variant": + _require_string(decision, "header", source, vector_id) + _require_int(decision, "parsed_version", source, vector_id) + _require_string(decision, "response_variant", source, vector_id) + for optional in ("absent_or_unparseable_uses", "current_protocol_version"): + if optional in decision: + _require_int(decision, optional, source, vector_id) + return + if kind == "custody_status": + status_map = decision.get("holding_by_status") + if not isinstance(status_map, dict) or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in status_map.items() + ): + raise BundleVerificationError( + f"{source}: vector {vector_id} custody status map invalid" + ) + _require_string(decision, "unknown_status", source, vector_id) + return + if kind in {"custody_unknown", "closed_vocabulary_unknown"}: + _require_string(decision, "status", source, vector_id) + if "unknown_status" in decision: + _require_string(decision, "unknown_status", source, vector_id) + if "unknown_value_behavior" in decision: + _require_string(decision, "unknown_value_behavior", source, vector_id) + if "vocabulary" in decision: + _require_string(decision, "vocabulary", source, vector_id) + return + if kind == "auth_header_form": + _require_string(decision, "auth_form", source, vector_id) + _require_bool(decision, "accepted", source, vector_id) + _require_string(decision, "precedence", source, vector_id) + return + if kind == "submitted_name_fallback": + _require_string(decision, "fallback", source, vector_id) + _require_bool(decision, "submitted_name_present", source, vector_id) + return + if kind == "chat_open_request": + _require_bool(decision, "accepted", source, vector_id) + _require_string(decision, "missing_field_behavior", source, vector_id) + if decision.get("accepted"): + _require_string(decision, "result", source, vector_id) + else: + _require_string(decision, "reason_code", source, vector_id) + return + if kind == "envelope_integrity": + _require_string(decision, "expected", source, vector_id) + _require_bool(decision, "valid", source, vector_id) + return + raise BundleVerificationError( + f"{source}: vector {vector_id} has unknown decision kind {kind}" + ) + + +def _require_string( + decision: dict[str, Any], field: str, source: str, vector_id: str +) -> str: + value = decision.get(field) + if not isinstance(value, str) or not value: + raise BundleVerificationError( + f"{source}: vector {vector_id} decision.{field} must be a string" + ) + return value + + +def _require_bool( + decision: dict[str, Any], field: str, source: str, vector_id: str +) -> bool: + value = decision.get(field) + if not isinstance(value, bool): + raise BundleVerificationError( + f"{source}: vector {vector_id} decision.{field} must be a boolean" + ) + return value + + +def _require_int( + decision: dict[str, Any], field: str, source: str, vector_id: str +) -> int: + value = decision.get(field) + if not isinstance(value, int) or isinstance(value, bool): + raise BundleVerificationError( + f"{source}: vector {vector_id} decision.{field} must be an integer" + ) + return value + + +def _require_string_list( + decision: dict[str, Any], field: str, source: str, vector_id: str +) -> list[str]: + value = decision.get(field) + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise BundleVerificationError( + f"{source}: vector {vector_id} decision.{field} must be a string list" + ) + return value + + +def _resolve_json_pointer(payload: Any, pointer: str) -> Any: + if pointer == "": + return payload + if not pointer.startswith("/"): + raise BundleVerificationError(f"invalid JSON pointer: {pointer}") + current = payload + for token in pointer[1:].split("/"): + token = token.replace("~1", "/").replace("~0", "~") + if isinstance(current, list): + try: + current = current[int(token)] + except (ValueError, IndexError) as exc: + raise BundleVerificationError( + f"JSON pointer does not resolve: {pointer}" + ) from exc + elif isinstance(current, dict): + if token not in current: + raise BundleVerificationError( + f"JSON pointer does not resolve: {pointer}" + ) + current = current[token] + else: + raise BundleVerificationError(f"JSON pointer does not resolve: {pointer}") + return current + + +def _verify_generator_inputs_do_not_self_reference(manifest: dict[str, Any]) -> None: + records = manifest.get("generator_inputs") + if not isinstance(records, list): + raise BundleVerificationError("manifest.generator_inputs is not a list") + bundle_prefix = BUNDLE_REL_DIR.as_posix() + "/" + for index, record in enumerate(records): + if not isinstance(record, dict): + raise BundleVerificationError( + f"manifest.generator_inputs[{index}] is not an object" + ) + path = record.get("path") + if not isinstance(path, str) or not path: + raise BundleVerificationError( + f"manifest.generator_inputs[{index}].path is not a string" + ) + if path == BUNDLE_REL_DIR.as_posix() or path.startswith(bundle_prefix): + raise BundleVerificationError( + f"generator input points inside generated bundle: {path}" + ) + + +def _verify_generator_input_records(root: Path, manifest: dict[str, Any]) -> None: + _verify_generator_inputs_do_not_self_reference(manifest) + records = manifest.get("generator_inputs") + if not isinstance(records, list): + raise BundleVerificationError("manifest.generator_inputs is not a list") + expected = sorted( + (input_id, rel_path.as_posix(), role) + for input_id, rel_path, role in _SOURCE_INPUTS + ) + actual: list[tuple[str, str, str]] = [] + seen: set[tuple[str, str, str]] = set() + for record in records: + if not isinstance(record, dict): + raise BundleVerificationError("manifest.generator_inputs entry is invalid") + input_id = record.get("id") + path = record.get("path") + role = record.get("role") + digest = record.get("sha256") + if not all(isinstance(item, str) and item for item in (input_id, path, role)): + raise BundleVerificationError( + "manifest.generator_inputs entry is incomplete" + ) + if not isinstance(digest, str) or not _SHA256_RE.fullmatch(digest): + raise BundleVerificationError( + f"manifest.generator_inputs[{input_id}].sha256 is invalid" + ) + triple = (input_id, path, role) + if triple in seen: + raise BundleVerificationError( + "duplicate generator input record: " + f"id={input_id} path={path} role={role}" + ) + seen.add(triple) + actual.append(triple) + source_path = root / path + try: + resolved_path = source_path.resolve(strict=True) + except FileNotFoundError as exc: + raise BundleVerificationError( + f"generator input path does not exist: {path}" + ) from exc + bundle_root = (root / BUNDLE_REL_DIR).resolve() + if _is_relative_to(resolved_path, bundle_root): + raise BundleVerificationError( + f"generator input points inside generated bundle: {path}" + ) + source_stat = source_path.lstat() + if not ( + stat.S_ISLNK(source_stat.st_mode) + or stat.S_ISREG(source_stat.st_mode) + or stat.S_ISDIR(source_stat.st_mode) + ): + raise BundleVerificationError( + f"generator input is not a regular file or directory: {path}" + ) + actual_digest = _sha256_path(source_path) + if actual_digest != digest: + raise BundleVerificationError( + f"generator input digest mismatch for {input_id}: " + f"expected {digest}, got {actual_digest}" + ) + if actual != expected: + actual_set = set(actual) + expected_set = set(expected) + if actual_set == expected_set: + raise BundleVerificationError( + "manifest.generator_inputs is not sorted by id/path/role" + ) + missing = sorted(expected_set - actual_set) + extra = sorted(actual_set - expected_set) + raise BundleVerificationError( + "manifest.generator_inputs do not match expected source sequence: " + f"missing={missing}; extra={extra}" + ) + + +def _validate_consumer_audit_coverage(snapshot: BundleSnapshot) -> None: + audit = _json_from_bytes( + snapshot.files["consumer-audit.json"], + "consumer-audit.json", + ) + if not isinstance(audit, dict): + raise BundleVerificationError("consumer-audit.json is not an object") + direct_paths = audit.get("direct_paths") + searched_files = audit.get("searched_files") + findings = audit.get("settings_drift_findings") + if not isinstance(direct_paths, list) or not isinstance(searched_files, list): + raise BundleVerificationError( + "consumer audit missing searched/direct path data" + ) + if not isinstance(findings, list): + raise BundleVerificationError("consumer audit missing settings drift findings") + + known_consumers = set(snapshot.manifest.get("consumer_identifiers", [])) + projection = _json_from_bytes( + snapshot.files[snapshot.manifest["projection_path"]], + "projection.openapi.json", + ) + projected_paths = set(projection.get("paths", {})) + rollout_consumers = { + item.get("consumer_identifier") + for item in snapshot.manifest.get("windows_linux_rollout_targets", []) + if isinstance(item, dict) + } + if rollout_consumers != {"solstone-linux", "solstone-windows"}: + raise BundleVerificationError( + "windows_linux_rollout_targets must be solstone-linux and solstone-windows" + ) + + direct_by_consumer: dict[str, list[dict[str, Any]]] = {} + for item in direct_paths: + if not isinstance(item, dict): + raise BundleVerificationError("consumer audit direct path entry is invalid") + consumer = item.get("consumer") + path = item.get("path") + classification = item.get("classification") + rationale = item.get("rationale") + if consumer not in known_consumers: + raise BundleVerificationError( + f"unknown consumer in audit direct path: {consumer}" + ) + if not isinstance(path, str) or not path: + raise BundleVerificationError("consumer audit direct path missing path") + if not isinstance(classification, str) or not classification: + raise BundleVerificationError( + "consumer audit direct path missing classification" + ) + if classification == "catch_all_exclusion": + raise BundleVerificationError( + "consumer audit uses forbidden catch-all exclusion" + ) + if not isinstance(rationale, str) or not rationale: + raise BundleVerificationError( + "consumer audit direct path missing rationale" + ) + if classification == "bundled" and path not in projected_paths: + raise BundleVerificationError( + f"bundled consumer path is absent from projection: {consumer} {path}" + ) + direct_by_consumer.setdefault(str(consumer), []).append(item) + + for consumer in rollout_consumers: + bundled = [ + item + for item in direct_by_consumer.get(str(consumer), []) + if item.get("classification") == "bundled" + ] + if not bundled: + raise BundleVerificationError( + f"rollout consumer has no bundled paths: {consumer}" + ) + + finding_ids = { + item.get("id") + for item in findings + if isinstance(item, dict) and item.get("status") == "adoption_blocker" + } + linux_blockers = next( + item.get("adoption_blocker_ids", []) + for item in snapshot.manifest["windows_linux_rollout_targets"] + if item.get("consumer_identifier") == "solstone-linux" + ) + missing_blockers = sorted(set(linux_blockers) - finding_ids) + if missing_blockers: + raise BundleVerificationError( + "linux rollout blockers missing from consumer audit: " + + ", ".join(missing_blockers) + ) + + +def _operation_locations(document: dict[str, Any]) -> dict[str, tuple[str, str, Any]]: + locations: dict[str, tuple[str, str, Any]] = {} + for path, method, operation in _iter_operations(document): + locations[operation["operationId"]] = (path, method, operation) + return locations + + +def _operation_by_id(projection: dict[str, Any], operation_id: str) -> dict[str, Any]: + for _path, _method, operation in _iter_operations(projection): + if operation.get("operationId") == operation_id: + return operation + raise BundleVerificationError(f"projection missing operation {operation_id}")