From de90a792d13a956ba4bd02f035b2cf7c09ce4a4e Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Fri, 3 Jul 2026 12:57:00 -0600 Subject: [PATCH] feat(observer): express transfer conventions 3-4 in the native-client OpenAPI contract Narrow the shared SegmentFile.status component to the enum [present, relocated, missing] and enrich three observer operation descriptions with the transfer contract's client obligations: - ingestSegments: proven-held iff status is present OR relocated AND sha256 matches the local file; missing/unaccounted is needs-upload and the local copy must never be deleted against it; relocated is a held state (re-upload only returns duplicate); confirm-before-delete matches per (filename via submitted_name-else-name, sha256). - ingestUpload 200: duplicate -> existing_segment authoritative; collision -> remapped segment authoritative. - ingestUpload: registry-derived media container list (from solstone.think.media FORMATS), headerless-raw rejected / raw PCM WAV-wrapped, video timestamps are boundary-relative real capture offsets. Description prose only; no ingest enforcement added. NOTE: the SegmentFile.status enum narrowing is a client-facing semantic narrowing that the OpenAPI breaking tripwire does NOT auto-catch (diff.py does not walk components.schemas / oneOf), so native-client owners must be notified out-of-band. Added conformance tests: declared-enum pin + live-listing membership, and the submitted_name-absent day-listing case. Regenerated convey-clients.json. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/openapi/convey-clients.json | 11 ++++-- solstone/apps/observer/contract.py | 43 +++++++++++++++++++-- solstone/apps/observer/tests/test_routes.py | 33 ++++++++++++++++ solstone/convey/contract/assemble.py | 2 +- tests/test_openapi_contract.py | 33 ++++++++++++++++ 5 files changed, 115 insertions(+), 7 deletions(-) diff --git a/docs/openapi/convey-clients.json b/docs/openapi/convey-clients.json index 748ed11b7..e6738a1f5 100644 --- a/docs/openapi/convey-clients.json +++ b/docs/openapi/convey-clients.json @@ -148,6 +148,11 @@ "type": "integer" }, "status": { + "enum": [ + "present", + "relocated", + "missing" + ], "type": "string" }, "submitted_name": { @@ -3140,7 +3145,7 @@ }, "/app/observer/ingest": { "post": { - "description": "Upload one capture segment as multipart form data and trigger local observe processing.", + "description": "Upload one capture segment as multipart form data and trigger local observe processing. Media parts must use a registry container format \u2014 audio: flac (audio/flac), opus (audio/opus), ogg (audio/ogg), m4a (audio/mp4), mp3 (audio/mpeg), wav (audio/wav); video: webm (video/webm), mp4 (video/mp4), mov (video/quicktime). Headerless raw streams are not accepted as media containers; raw PCM must be WAV-wrapped before upload. Video segment timestamps are boundary-relative real capture offsets (seconds from the segment start), never synthetic frame indices.", "operationId": "observer.ingestUpload", "parameters": [ { @@ -3271,7 +3276,7 @@ } } }, - "description": "Upload accepted, collision-adjusted, or duplicate." + "description": "Upload accepted, collision-adjusted, or duplicate. On `duplicate`, `existing_segment` is the authoritative stored segment key the client must adopt (do not re-upload). On `collision`, the returned `segment` is the authoritative remapped key the client must adopt for subsequent references." }, "400": { "content": { @@ -3749,7 +3754,7 @@ }, "/app/observer/ingest/segments/{day}": { "get": { - "description": "Return segment upload history for one day. Protocol version 2 and newer receive a collection envelope; older or absent protocol headers receive a legacy bare array.", + "description": "Return segment upload history for one day. Protocol version 2 and newer receive a collection envelope; older or absent protocol headers receive a legacy bare array. Per-file `status` is `present` (file at its recorded path), `relocated` (located by inode elsewhere; `current_path` gives the new journal-relative location), or `missing` (not found). A local file is proven held by the journal only when its listing entry is `present` or `relocated` AND the entry `sha256` matches the local file; a `missing` status \u2014 or any local file the listing does not positively account for \u2014 is needs-upload, and the client must never delete a local copy on that basis. `relocated` is a held state, not needs-upload: the journal provably holds the bytes (inode-verified, `current_path`), and re-uploading only returns `duplicate`. Confirm-before-delete matches every upload-eligible local file to a listing entry per (filename, sha256) \u2014 the listing filename is `submitted_name` when present, else `name` \u2014 and deletes only on a proven-held match. `submitted_name` appears only when the stored filename differs from the submitted one; segment-level `original_key` carries the client's originally requested segment key when a collision remapped it.", "operationId": "observer.ingestSegments", "parameters": [ { diff --git a/solstone/apps/observer/contract.py b/solstone/apps/observer/contract.py index 77de4d212..b1b816231 100644 --- a/solstone/apps/observer/contract.py +++ b/solstone/apps/observer/contract.py @@ -12,6 +12,7 @@ from solstone.convey.contract import ( RequestSpec, ResponseSpec, ) +from solstone.think.media import FORMATS _OBSERVER_AUTH_PARAMS = ( ParamSpec( @@ -56,6 +57,15 @@ def _observer_auth_errors() -> tuple[ResponseSpec, ResponseSpec]: ) +def _media_format_description() -> str: + by_kind: dict[str, list[str]] = {} + for ext, mime, kind in FORMATS: + by_kind.setdefault(kind, []).append(f"{ext.lstrip('.')} ({mime})") + return "; ".join( + f"{kind}: {', '.join(entries)}" for kind, entries in by_kind.items() + ) + + _DAY_SEGMENT_COUNT_MAP = { "type": "object", "additionalProperties": { @@ -181,7 +191,12 @@ OPERATIONS: list[OperationSpec] = [ summary="Upload observer segment files", description=( "Upload one capture segment as multipart form data and trigger local " - "observe processing." + "observe processing. Media parts must use a registry container format " + f"— {_media_format_description()}. Headerless raw streams are not " + "accepted as media containers; raw PCM must be WAV-wrapped before " + "upload. Video segment timestamps are boundary-relative real capture " + "offsets (seconds from the segment start), never synthetic frame " + "indices." ), parameters=_OBSERVER_AUTH_PARAMS, request=RequestSpec( @@ -214,7 +229,13 @@ OPERATIONS: list[OperationSpec] = [ responses=( ResponseSpec( status=200, - description="Upload accepted, collision-adjusted, or duplicate.", + description=( + "Upload accepted, collision-adjusted, or duplicate. On " + "`duplicate`, `existing_segment` is the authoritative stored " + "segment key the client must adopt (do not re-upload). On " + "`collision`, the returned `segment` is the authoritative " + "remapped key the client must adopt for subsequent references." + ), named_fields=( FieldSpec("status", "string", required=True), FieldSpec("segment", "string"), @@ -430,7 +451,23 @@ OPERATIONS: list[OperationSpec] = [ description=( "Return segment upload history for one day. Protocol version 2 and " "newer receive a collection envelope; older or absent protocol " - "headers receive a legacy bare array." + "headers receive a legacy bare array. Per-file `status` is `present` " + "(file at its recorded path), `relocated` (located by inode elsewhere; " + "`current_path` gives the new journal-relative location), or `missing` " + "(not found). A local file is proven held by the journal only when its " + "listing entry is `present` or `relocated` AND the entry `sha256` " + "matches the local file; a `missing` status — or any local file the " + "listing does not positively account for — is needs-upload, and the " + "client must never delete a local copy on that basis. `relocated` is a " + "held state, not needs-upload: the journal provably holds the bytes " + "(inode-verified, `current_path`), and re-uploading only returns " + "`duplicate`. Confirm-before-delete matches every upload-eligible " + "local file to a listing entry per (filename, sha256) — the listing " + "filename is `submitted_name` when present, else `name` — and deletes " + "only on a proven-held match. `submitted_name` appears only when the " + "stored filename differs from the submitted one; segment-level " + "`original_key` carries the client's originally requested segment key " + "when a collision remapped it." ), parameters=( *_OBSERVER_AUTH_PARAMS, diff --git a/solstone/apps/observer/tests/test_routes.py b/solstone/apps/observer/tests/test_routes.py index ad5184603..0b11f6160 100644 --- a/solstone/apps/observer/tests/test_routes.py +++ b/solstone/apps/observer/tests/test_routes.py @@ -1909,6 +1909,39 @@ def test_segments_endpoint_lists_uploads(observer_env): ) # Original name preserved +def test_segments_endpoint_omits_submitted_name_when_name_unchanged(observer_env): + """Test segments endpoint omits submitted_name when no filename rewrite occurred.""" + env = observer_env() + + resp = env.client.post( + "/app/observer/api/create", + json={"name": "segments-no-rewrite-test"}, + content_type="application/json", + ) + key = resp.get_json()["key"] + + test_data = b"test audio content" + resp = env.client.post( + "/app/observer/ingest", + headers={"Authorization": f"Bearer {key}"}, + data={ + "day": "20250103", + "segment": "120000_300", + "files": (io.BytesIO(test_data), "audio.flac"), + }, + ) + assert resp.status_code == 200 + + resp = env.client.get( + "/app/observer/ingest/segments/20250103", + headers={"Authorization": f"Bearer {key}"}, + ) + assert resp.status_code == 200 + data = resp.get_json() + file_info = data[0]["files"][0] + assert "submitted_name" not in file_info + + def test_segments_endpoint_v2_empty(observer_env): """Test v2 segments endpoint returns collection envelope for no uploads.""" env = observer_env() diff --git a/solstone/convey/contract/assemble.py b/solstone/convey/contract/assemble.py index 6d90ee99f..cd6697a19 100644 --- a/solstone/convey/contract/assemble.py +++ b/solstone/convey/contract/assemble.py @@ -254,7 +254,7 @@ def _components() -> dict[str, Any]: "name": {"type": "string"}, "size": {"type": "integer"}, "sha256": {"type": "string"}, - "status": {"type": "string"}, + "status": {"type": "string", "enum": ["present", "relocated", "missing"]}, "submitted_name": {"type": "string"}, "current_path": {"type": "string"}, }, diff --git a/tests/test_openapi_contract.py b/tests/test_openapi_contract.py index d7302089c..e92b30962 100644 --- a/tests/test_openapi_contract.py +++ b/tests/test_openapi_contract.py @@ -270,6 +270,39 @@ def test_segments_protocol_version_shape(contract_app): assert {"items", "total", "protocol_version"}.issubset(body) +def test_segment_file_status_enum_matches_live_day_listing(contract_app): + _app, client, _journal = contract_app + document = build_document() + status_enum = document["components"]["schemas"]["SegmentFile"]["properties"][ + "status" + ]["enum"] + assert status_enum == ["present", "relocated", "missing"] + key = _register_observer(client) + + upload = client.post( + "/app/observer/ingest", + headers={"X-Solstone-Observer": key}, + data={ + "day": "20250103", + "segment": "120000_300", + "files": (io.BytesIO(b"contract upload"), "audio.flac"), + }, + content_type="multipart/form-data", + ) + assert upload.status_code == 200 + + listing = client.get( + "/app/observer/ingest/segments/20250103", + headers={ + "Authorization": f"Bearer {key}", + "X-Solstone-Protocol-Version": "2", + }, + ) + items = listing.get_json()["items"] + statuses = [file_info["status"] for item in items for file_info in item["files"]] + assert statuses and all(status in status_enum for status in statuses) + + def test_multipart_and_json_parsing(contract_app): _app, client, _journal = contract_app document = build_document() -- 2.51.2