From 252729f437ca69edd8c28f45e5cd8ce53284e130 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 15:37:19 -0600 Subject: [PATCH] extract the raw-media release oracle from the reference, by executing it Two readers decide irreversibly about the owner's raw media and they do not apply the same predicate. Recording the disagreement as committed data rather than as prose, in the tree's established pattern for accepted divergence. 23 shapes, each built on a real filesystem and each reference verdict OBSERVED by running the reference over it -- never hand-typed. The unified verdicts are derived mechanically from one rule, so a row cannot be argued into agreement individually. The fixture's invariant is that no row may record the port RELEASING raw media the reference held. Writing the matrix out found the one row that would have: a record claiming "analyzed" whose analysis rows are gone. Terminal proof accepts it -- its question is whether a device may drop its copy -- and the reference's state derivation holds it, because the raw is then the only surviving copy of that content. Deriving the predicate from the conjunction alone would have dropped that, so it is a fifth condition: "analyzed" asserts derived output exists, and "empty" is the one terminal state that legitimately has none. With it, 13 rows narrow, 2 narrow onto disclosed pre-record row evidence, 8 agree, and none widen. Two rows are the still-image case: the gate globs audio and screen extraction names while the media test accepts all eight image formats, so an image is released today with no evidence it was processed -- including with no sidecar at all. The unified predicate resolves an expected handler before reading any record, so no handler in the closed set means no obtainable proof, which never releases. Verified the invariant is load-bearing: disabling the fifth condition fails the gate and names the row. --- Makefile | 13 +- core/fixtures/retention_release_oracle.json | 611 ++++++++++++++++++++ scripts/retention_release_oracle.py | 498 ++++++++++++++++ 3 files changed, 1121 insertions(+), 1 deletion(-) create mode 100644 core/fixtures/retention_release_oracle.json create mode 100644 scripts/retention_release_oracle.py diff --git a/Makefile b/Makefile index d3c6fa28f..aaa1d54d1 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ export TMPDIR := /var/tmp PYTEST_BASETEMP_INIT := BASETEMP=$$(mktemp -d /var/tmp/solstone-pytest-XXXXXX); trap 'rm -rf "$$BASETEMP"' EXIT INT TERM; PYTEST_BASETEMP_FLAG := --basetemp "$$BASETEMP" -.PHONY: install hopper-install uninstall test test-cov test-integration test-release release-checks test-performance test-app test-only format format-check install-checks ci clean clean-install coverage watch versions update update-prices preflight pre-commit skills render-packaging check-rust-fmt check-rust-msrv check-rust-clippy check-rust-test check-rust-ios check-rust-deny build check-release-advisory-liveness check-rust-release-manifest check-spl-dependency-pin audit openapi check-openapi check-openapi-observer-client-contract contract check-contract journal-resolution-vectors check-journal-resolution-vectors build-native-sol-grammar-oracle check-native-sol-grammar-oracle build-native-sol-root-contract check-native-sol-root-contract check-core-sdist-compile-inputs build-native-sol-journal-host-commands check-native-sol-journal-host-commands build-journal-access-rejection-inventory check-journal-access-rejection-inventory check-native-sol-python-manifest build-native-sol-inventory check-native-sol-inventory check-native-sol-architecture check-native-sol-contract-routes check-native-sol-conformance check-native-sol-coverage check-native-sol-no-python-spawn check-native-sol-compat check-native-sol-docs-links check-removed-time-parser-ready dev all sandbox sandbox-stop install-models speakers-analyze-helper parakeet-helper parakeet-helper-clean wheel-speakers-analyze-linux wheel-speakers-analyze-linux-x86_64 wheel-speakers-analyze-linux-aarch64 wheel-macos wheel-macos-clean verify verify-api verify-schemathesis update-api-baselines eval-schemas service-logs check-layer-hygiene check-api-conventions check-journal-io-access check-journal-io-mechanic check-journal-config-owner check-call-http-only check-no-legacy-chat check-channel-adapter-scrub check-brain-health-cutover check-tools-http-only check-access-imports-clean check-convey-bind-imports-clean check-schema-bounds check-thin-base-install check-extras-consistency check-cogitate-prompts smoke-cogitate release release-test publish-release publish-release-test FORCE +.PHONY: install hopper-install uninstall test test-cov test-integration test-release release-checks test-performance test-app test-only format format-check install-checks ci clean clean-install coverage watch versions update update-prices preflight pre-commit skills render-packaging check-rust-fmt check-rust-msrv check-rust-clippy check-rust-test check-rust-ios check-rust-deny build check-release-advisory-liveness check-rust-release-manifest check-spl-dependency-pin audit openapi check-openapi check-openapi-observer-client-contract contract check-contract journal-resolution-vectors check-journal-resolution-vectors build-native-sol-grammar-oracle check-native-sol-grammar-oracle build-native-sol-root-contract check-native-sol-root-contract check-core-sdist-compile-inputs build-native-sol-journal-host-commands check-native-sol-journal-host-commands build-journal-access-rejection-inventory check-journal-access-rejection-inventory check-native-sol-python-manifest build-native-sol-inventory check-native-sol-inventory check-native-sol-architecture check-native-sol-contract-routes check-native-sol-conformance check-native-sol-coverage check-native-sol-no-python-spawn check-native-sol-compat check-native-sol-docs-links check-removed-time-parser-ready dev all sandbox sandbox-stop install-models speakers-analyze-helper parakeet-helper parakeet-helper-clean wheel-speakers-analyze-linux wheel-speakers-analyze-linux-x86_64 wheel-speakers-analyze-linux-aarch64 wheel-macos wheel-macos-clean verify verify-api verify-schemathesis update-api-baselines eval-schemas service-logs check-layer-hygiene check-api-conventions check-journal-io-access check-journal-io-mechanic check-journal-config-owner check-call-http-only check-no-legacy-chat check-channel-adapter-scrub check-brain-health-cutover check-tools-http-only check-access-imports-clean check-convey-bind-imports-clean check-schema-bounds check-retention-release-oracle check-thin-base-install check-extras-consistency check-cogitate-prompts smoke-cogitate release release-test publish-release publish-release-test FORCE # Default target - build the native workspace during the Rust-conversion freeze all: build @@ -578,6 +578,9 @@ install-checks: .installed @echo "=== Checking conversion-wave retirements ===" @$(MAKE) check-conversion-retirements @echo "" + @echo "=== Checking the raw-media release oracle ===" + @$(MAKE) check-retention-release-oracle + @echo "" @echo "=== Checking spl health vocabulary ===" @$(MAKE) check-spl-health-vocabulary @echo "=== Running access-imports-clean check ===" @@ -780,6 +783,14 @@ check-spl-dependency-pin: check-conversion-retirements: python3 scripts/check_conversion_retirements.py +# Two readers decide irreversibly about the owner's raw media and they do not +# apply the same predicate. This regenerates the oracle from the reference and +# fails if the committed fixture has drifted -- or if any row records a port +# RELEASING raw media the reference held, which is a loosening of an +# irreversible path and needs its own argument. +check-retention-release-oracle: .installed + $(VENV_BIN)/python scripts/retention_release_oracle.py --check + # The spl link-health vocabulary spans two languages after the native cutover: # the native service emits the reason codes and the callosum event name, and the # web layer consumes them. Nothing else makes them agree, and drift is silent and diff --git a/core/fixtures/retention_release_oracle.json b/core/fixtures/retention_release_oracle.json new file mode 100644 index 000000000..0f4ecad67 --- /dev/null +++ b/core/fixtures/retention_release_oracle.json @@ -0,0 +1,611 @@ +{ + "_provenance": { + "generator": "scripts/retention_release_oracle.py", + "method": "the reference verdicts are OBSERVED by running the reference over constructed segments on a real filesystem; the unified verdicts are derived mechanically from one rule, not chosen per row", + "source_revision": "ba31b082b8588298a229df8d925cb42c1825a09d", + "reference": { + "segment_gate": "solstone/think/retention.py:190", + "state_derivation": "solstone/think/data_state.py:121", + "chunks_win": "solstone/think/data_state.py:145", + "empty_arm": "solstone/think/data_state.py:147", + "failed_arm": "solstone/think/data_state.py:139", + "terminal_proof": "solstone/apps/observer/processing_proof.py:24", + "raw_media_test": "solstone/think/retention.py:48", + "image_sidecar_writer": "solstone/observe/depict.py:49", + "extraction_globs": "solstone/think/retention.py:207" + }, + "reading": "reference_gate 'eligible' means the reference unlinks the owner's raw media. reference_proof is the four-condition conjunction the resolve path runs on the same bytes. They disagree in both directions, which is why this file exists.", + "invariant": "no row may carry disposition 'widened'. A port that releases raw the reference held has loosened an irreversible path, and each such row needs its own argument before it is recorded here." + }, + "summary": { + "agrees": 8, + "narrowed": 13, + "narrowed_legacy": 2 + }, + "rows": [ + { + "id": "record_terminal_no_row", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "schema": "solstone.processing.v1", + "state": "analyzed", + "reason_code": "ok", + "handler": "transcribe", + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": 104 + }, + "analysis_row": null + }, + "reference_gate": "incomplete", + "reference_releases_raw": false, + "reference_proof": true, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "the record claims analyzed and the analysis rows are gone; the raw is the only surviving copy" + }, + "disposition": "agrees", + "provenance": "observed", + "note": "a record claiming analyzed with no analysis row on disk -- the derived output is missing and the raw is the only surviving copy" + }, + { + "id": "record_terminal_with_row", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "schema": "solstone.processing.v1", + "state": "analyzed", + "reason_code": "ok", + "handler": "transcribe", + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": 104 + }, + "analysis_row": { + "start": 0.0, + "end": 1.0, + "text": "hello" + } + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": true, + "unified": { + "verdict": "releasable", + "evidence": "record", + "because": "the conjunction held and the derived output is on disk" + }, + "disposition": "agrees", + "provenance": "observed", + "note": "the ordinary processed shape" + }, + { + "id": "record_absent_with_row", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": "absent", + "analysis_row": { + "start": 0.0, + "end": 1.0, + "text": "hello" + } + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "releasable", + "evidence": "legacy_rows", + "because": "no processing record; analysis rows present" + }, + "disposition": "narrowed_legacy", + "provenance": "observed", + "note": "pre-record data: analysis rows, no processing record" + }, + { + "id": "record_absent_no_row", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": "absent", + "analysis_row": null + }, + "reference_gate": "incomplete", + "reference_releases_raw": false, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "no processing record and no analysis rows" + }, + "disposition": "agrees", + "provenance": "observed", + "note": "nothing has run" + }, + { + "id": "state_empty_bare", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "state": "empty" + }, + "analysis_row": null + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "schema is absent or unrecognized" + }, + "disposition": "narrowed", + "provenance": "observed", + "note": "the entire record is a state field" + }, + { + "id": "state_empty_wrong_schema", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "schema": "bogus.v9", + "state": "empty", + "reason_code": "ok", + "handler": "transcribe", + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": 104 + }, + "analysis_row": null + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "schema is absent or unrecognized" + }, + "disposition": "narrowed", + "provenance": "observed" + }, + { + "id": "state_empty_wrong_handler", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "schema": "solstone.processing.v1", + "state": "empty", + "reason_code": "ok", + "handler": "describe", + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": 104 + }, + "analysis_row": null + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "the recorded handler is not the one that claims this content" + }, + "disposition": "narrowed", + "provenance": "observed" + }, + { + "id": "state_empty_size_mismatch", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "schema": "solstone.processing.v1", + "state": "empty", + "reason_code": "ok", + "handler": "transcribe", + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": 999999 + }, + "analysis_row": null + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "the bytes on disk are not the bytes that were processed" + }, + "disposition": "narrowed", + "provenance": "observed", + "note": "the bytes on disk are not the bytes that were processed" + }, + { + "id": "state_empty_no_schema_key", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "state": "empty", + "reason_code": "ok", + "handler": "transcribe", + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": 104 + }, + "analysis_row": null + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "schema is absent or unrecognized" + }, + "disposition": "narrowed", + "provenance": "observed" + }, + { + "id": "state_empty_no_handler_key", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "schema": "solstone.processing.v1", + "state": "empty", + "reason_code": "ok", + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": 104 + }, + "analysis_row": null + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "the recorded handler is not the one that claims this content" + }, + "disposition": "narrowed", + "provenance": "observed" + }, + { + "id": "state_empty_no_size_key", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "schema": "solstone.processing.v1", + "state": "empty", + "reason_code": "ok", + "handler": "transcribe", + "attempted_at": "2026-08-05T00:00:00Z" + }, + "analysis_row": null + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "the bytes on disk are not the bytes that were processed" + }, + "disposition": "narrowed", + "provenance": "observed" + }, + { + "id": "state_analyzed_bare", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "state": "analyzed" + }, + "analysis_row": null + }, + "reference_gate": "incomplete", + "reference_releases_raw": false, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "schema is absent or unrecognized" + }, + "disposition": "agrees", + "provenance": "observed" + }, + { + "id": "state_unrecognized_with_row", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "state": "banana" + }, + "analysis_row": { + "start": 0.0, + "end": 1.0, + "text": "hello" + } + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "schema is absent or unrecognized" + }, + "disposition": "narrowed", + "provenance": "observed" + }, + { + "id": "record_empty_object_with_row", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": {}, + "analysis_row": { + "start": 0.0, + "end": 1.0, + "text": "hello" + } + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "schema is absent or unrecognized" + }, + "disposition": "narrowed", + "provenance": "observed", + "note": "an empty record object still releases the owner's raw today" + }, + { + "id": "marker_key_only_row", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": "absent", + "analysis_row": { + "start": 0.0 + } + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "releasable", + "evidence": "legacy_rows", + "because": "no processing record; analysis rows present" + }, + "disposition": "narrowed_legacy", + "provenance": "observed", + "note": "a row carrying the marker key and nothing else; a real transcript row carries start, end and text" + }, + { + "id": "non_analysis_row", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": "absent", + "analysis_row": { + "text": "hello" + } + }, + "reference_gate": "incomplete", + "reference_releases_raw": false, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "no processing record and no analysis rows" + }, + "disposition": "agrees", + "provenance": "observed" + }, + { + "id": "state_failed", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "schema": "solstone.processing.v1", + "state": "failed", + "reason_code": "ok", + "handler": "transcribe", + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": 104 + }, + "analysis_row": null + }, + "reference_gate": "failed", + "reference_releases_raw": false, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "failed", + "because": "processing failed; disclose, never release" + }, + "disposition": "agrees", + "provenance": "observed" + }, + { + "id": "state_failed_with_row", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "schema": "solstone.processing.v1", + "state": "failed", + "reason_code": "ok", + "handler": "transcribe", + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": 104 + }, + "analysis_row": { + "start": 0.0, + "end": 1.0, + "text": "hello" + } + }, + "reference_gate": "failed", + "reference_releases_raw": false, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "failed", + "because": "processing failed; disclose, never release" + }, + "disposition": "agrees", + "provenance": "observed", + "note": "a failed record beside rows that exist" + }, + { + "id": "record_terminal_wrong_handler_with_row", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "schema": "solstone.processing.v1", + "state": "analyzed", + "reason_code": "ok", + "handler": "describe", + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": 104 + }, + "analysis_row": { + "start": 0.0, + "end": 1.0, + "text": "hello" + } + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "the recorded handler is not the one that claims this content" + }, + "disposition": "narrowed", + "provenance": "observed", + "note": "the audio was consumed by the screen handler, which cannot be true" + }, + { + "id": "record_terminal_size_mismatch_with_row", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "schema": "solstone.processing.v1", + "state": "analyzed", + "reason_code": "ok", + "handler": "transcribe", + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": 999999 + }, + "analysis_row": { + "start": 0.0, + "end": 1.0, + "text": "hello" + } + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "incomplete", + "because": "the bytes on disk are not the bytes that were processed" + }, + "disposition": "narrowed", + "provenance": "observed", + "note": "rows exist but the file has changed size since it was processed" + }, + { + "id": "record_empty_full_no_row", + "shape": { + "media": "chunk_audio.flac", + "size": 104, + "record": { + "schema": "solstone.processing.v1", + "state": "empty", + "reason_code": "no_decodable_audio", + "handler": "transcribe", + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": 104 + }, + "analysis_row": null + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": true, + "unified": { + "verdict": "releasable", + "evidence": "record", + "because": "the conjunction held and the handler determined there was nothing to derive, so no analysis rows are expected" + }, + "disposition": "agrees", + "provenance": "observed", + "note": "terminal-empty: the handler ran, determined there was nothing to transcribe, and recorded it. This is the shape the raw hand-off produces" + }, + { + "id": "image_with_record", + "shape": { + "media": "photo.png", + "size": 104, + "record": { + "schema": "solstone.processing.v1", + "state": "empty", + "reason_code": "ok", + "handler": "depict", + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": 104 + }, + "analysis_row": null + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "unprovable", + "because": "no handler in the closed set claims this content, so no proof is obtainable for it at any point" + }, + "disposition": "narrowed", + "provenance": "observed", + "note": "an image whose sidecar names a handler that is not in the closed set" + }, + { + "id": "image_no_sidecar", + "shape": { + "media": "photo.png", + "size": 104, + "record": "absent", + "analysis_row": null + }, + "reference_gate": "eligible", + "reference_releases_raw": true, + "reference_proof": false, + "unified": { + "verdict": "held", + "blocker": "unprovable", + "because": "no handler in the closed set claims this content, so no proof is obtainable for it at any point" + }, + "disposition": "narrowed", + "provenance": "observed", + "note": "an image alone. The reference releases it with no evidence it was ever processed and none that it was ever looked at" + } + ] +} diff --git a/scripts/retention_release_oracle.py b/scripts/retention_release_oracle.py new file mode 100644 index 000000000..987599076 --- /dev/null +++ b/scripts/retention_release_oracle.py @@ -0,0 +1,498 @@ +"""Extract the raw-media release oracle from the reference, by executing it. + +The two readers that decide irreversibly about the owner's raw media do not +apply the same predicate, and they disagree in both directions. This emits the +disagreement as committed data so a port pins it rather than describing it. + +For each constructed on-disk shape the fixture records three verdicts: + + reference_gate what ``retention.resolve_segment_gate`` decides today. + ``eligible`` means the owner's raw media is unlinked. + reference_proof what ``processing_proof.has_terminal_processing_proof`` + decides for the same bytes -- the conjunction a device is + told it may drop its only copy on. + unified what a single predicate must decide, derived mechanically + from the rule below rather than chosen per row. + +and a ``disposition`` naming the relationship between the first and the third: + + agrees the port keeps the reference's verdict + narrowed the port HOLDS raw the reference released. Safe direction: + the cost is disk, not the owner's data + narrowed_legacy as ``agrees``, but the release rests on pre-record row + evidence rather than on a processing record, so the port + must disclose which files were released on it + widened the port RELEASES raw the reference held. ANY row here is + a defect until argued otherwise, one row at a time + +⛔ Verdicts are observed by running the reference on real files, never +hand-typed. The generator asserts ``widened`` is empty and exits non-zero if it +is not, so the fixture cannot silently record a loosening of an irreversible +path. + +Usage: python scripts/retention_release_oracle.py [--check] +""" + +import json +import shutil +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO)) + +from solstone.apps.observer.processing_proof import ( # noqa: E402 + has_terminal_processing_proof, +) +from solstone.observe.processing_record import ( # noqa: E402 + AUDIO_TRANSCRIPT_ROW_KEY, + HANDLER_TRANSCRIBE, + SCHEMA, + STATE_ANALYZED, + STATE_EMPTY, + STATE_FAILED, +) +from solstone.think.retention import resolve_segment_gate # noqa: E402 + +FIXTURE = REPO / "core" / "fixtures" / "retention_release_oracle.json" + +MEDIA = "chunk_audio.flac" +IMAGE = "photo.png" +SIZE = 104 + +# The extension -> handler map is the caller's, not the predicate's. Retention +# receives it rather than carrying a copy; this is the smallest stand-in that +# covers the arms the fixture exercises. `None` means no handler in the closed +# set claims the content, so no proof is obtainable for it at any point. +HANDLER_BY_EXTENSION = {"flac": HANDLER_TRANSCRIBE} + + +def sidecar_name(media: str) -> str: + """The same-stem sidecar both readers derive, per the reference.""" + return str(Path(media).with_suffix(".jsonl")) + + +def expected_handler_for(media: str) -> str | None: + return HANDLER_BY_EXTENSION.get(Path(media).suffix.lower().lstrip(".")) + + +TERMINAL_STATES = frozenset({STATE_ANALYZED, STATE_EMPTY}) + +# Cited so a reader can check any verdict against the line that decides it. +REFERENCE = { + "segment_gate": "solstone/think/retention.py:190", + "state_derivation": "solstone/think/data_state.py:121", + "chunks_win": "solstone/think/data_state.py:145", + "empty_arm": "solstone/think/data_state.py:147", + "failed_arm": "solstone/think/data_state.py:139", + "terminal_proof": "solstone/apps/observer/processing_proof.py:24", + "raw_media_test": "solstone/think/retention.py:48", + "image_sidecar_writer": "solstone/observe/depict.py:49", + "extraction_globs": "solstone/think/retention.py:207", +} + + +def full_record(**overrides: object) -> dict: + record = { + "schema": SCHEMA, + "state": STATE_ANALYZED, + "reason_code": "ok", + "handler": HANDLER_TRANSCRIBE, + "attempted_at": "2026-08-05T00:00:00Z", + "input_size": SIZE, + } + record.update(overrides) + return record + + +def without(key: str) -> dict: + record = full_record() + del record[key] + return record + + +ANALYSIS_ROW = {AUDIO_TRANSCRIPT_ROW_KEY: 0.0, "end": 1.0, "text": "hello"} +MARKER_ONLY_ROW = {AUDIO_TRANSCRIPT_ROW_KEY: 0.0} +NON_ANALYSIS_ROW = {"text": "hello"} + + +@dataclass(frozen=True) +class Shape: + id: str + record: dict | None + row: dict | None + note: str = "" + # `record` absent from the header entirely vs. present-but-empty are + # different on-disk shapes and the reference treats them differently. + omit_record_key: bool = False + # Non-audio media exercises the arm where no handler claims the content at + # all, so no proof is obtainable for it at any point. + media: str = MEDIA + # Omit the sidecar entirely rather than writing a header-only one. + omit_sidecar: bool = False + + +SHAPES = [ + Shape( + "record_terminal_no_row", + full_record(), + None, + "a record claiming analyzed with no analysis row on disk -- the derived " + "output is missing and the raw is the only surviving copy", + ), + Shape( + "record_terminal_with_row", + full_record(), + ANALYSIS_ROW, + "the ordinary processed shape", + ), + Shape( + "record_absent_with_row", + None, + ANALYSIS_ROW, + "pre-record data: analysis rows, no processing record", + omit_record_key=True, + ), + Shape("record_absent_no_row", None, None, "nothing has run", omit_record_key=True), + Shape( + "state_empty_bare", + {"state": STATE_EMPTY}, + None, + "the entire record is a state field", + ), + Shape( + "state_empty_wrong_schema", + full_record(state=STATE_EMPTY, schema="bogus.v9"), + None, + ), + Shape( + "state_empty_wrong_handler", + full_record(state=STATE_EMPTY, handler="describe"), + None, + ), + Shape( + "state_empty_size_mismatch", + full_record(state=STATE_EMPTY, input_size=999_999), + None, + "the bytes on disk are not the bytes that were processed", + ), + Shape( + "state_empty_no_schema_key", without("schema") | {"state": STATE_EMPTY}, None + ), + Shape( + "state_empty_no_handler_key", without("handler") | {"state": STATE_EMPTY}, None + ), + Shape( + "state_empty_no_size_key", without("input_size") | {"state": STATE_EMPTY}, None + ), + Shape("state_analyzed_bare", {"state": STATE_ANALYZED}, None), + Shape("state_unrecognized_with_row", {"state": "banana"}, ANALYSIS_ROW), + Shape( + "record_empty_object_with_row", + {}, + ANALYSIS_ROW, + "an empty record object still releases the owner's raw today", + ), + Shape( + "marker_key_only_row", + None, + MARKER_ONLY_ROW, + "a row carrying the marker key and nothing else; a real transcript row " + "carries start, end and text", + omit_record_key=True, + ), + Shape("non_analysis_row", None, NON_ANALYSIS_ROW, omit_record_key=True), + Shape("state_failed", full_record(state=STATE_FAILED), None), + Shape( + "state_failed_with_row", + full_record(state=STATE_FAILED), + ANALYSIS_ROW, + "a failed record beside rows that exist", + ), + Shape( + "record_terminal_wrong_handler_with_row", + full_record(handler="describe"), + ANALYSIS_ROW, + "the audio was consumed by the screen handler, which cannot be true", + ), + Shape( + "record_terminal_size_mismatch_with_row", + full_record(input_size=999_999), + ANALYSIS_ROW, + "rows exist but the file has changed size since it was processed", + ), + # The positive case the deletion ruling routes to retention: VAD found under + # a second of speech, the handler wrote a terminal-empty record, and the raw + # is handed over rather than unlinked in place. `empty` is the one terminal + # state that legitimately carries no analysis rows. + Shape( + "record_empty_full_no_row", + full_record(state=STATE_EMPTY, reason_code="no_decodable_audio"), + None, + "terminal-empty: the handler ran, determined there was nothing to " + "transcribe, and recorded it. This is the shape the raw hand-off produces", + ), + # Still images: no handler in the closed set claims them, so no proof is + # obtainable for them at any point. + Shape( + "image_with_record", + full_record(state=STATE_EMPTY, handler="depict"), + None, + "an image whose sidecar names a handler that is not in the closed set", + media=IMAGE, + ), + Shape( + "image_no_sidecar", + None, + None, + "an image alone. The reference releases it with no evidence it was ever " + "processed and none that it was ever looked at", + omit_record_key=True, + media=IMAGE, + omit_sidecar=True, + ), +] + + +def build(root: Path, shape: Shape) -> Path: + segment = root / shape.id + segment.mkdir(parents=True) + (segment / shape.media).write_bytes(b"f" * SIZE) + if shape.omit_sidecar: + return segment + header: dict = {"segment": shape.id} + if not shape.omit_record_key: + header["_solstone_processing"] = shape.record + lines = [json.dumps(header)] + if shape.row is not None: + lines.append(json.dumps(shape.row)) + (segment / sidecar_name(shape.media)).write_text("\n".join(lines) + "\n") + return segment + + +def unified(shape: Shape) -> dict: + """The single predicate, applied mechanically. + + Five conditions, in order. The first four are the terminal-proof + conjunction the resolve path already runs; the fifth is the lesson the + reference's state derivation encodes and a naive unification would drop. + """ + record = None if shape.omit_record_key else shape.record + has_row = shape.row is not None and AUDIO_TRANSCRIPT_ROW_KEY in shape.row + + # 0. A handler must claim the content, or no proof is obtainable for it at + # any point. This arm is what makes a still image unreleasable until its + # handler joins the closed set, and it is the whole reason the map is + # injected rather than carried here. + expected_handler = expected_handler_for(shape.media) + if expected_handler is None: + return { + "verdict": "held", + "blocker": "unprovable", + "because": "no handler in the closed set claims this content, so no " + "proof is obtainable for it at any point", + } + + if record is None: + # Read old: analysis rows are real evidence the media was consumed, + # weaker than a record. Honoured, tagged, disclosed. + if has_row: + return { + "verdict": "releasable", + "evidence": "legacy_rows", + "because": "no processing record; analysis rows present", + } + return { + "verdict": "held", + "blocker": "incomplete", + "because": "no processing record and no analysis rows", + } + + if not isinstance(record, dict): + return { + "verdict": "held", + "blocker": "incomplete", + "because": "the record is not an object", + } + + # 1-4: the conjunction, evaluated exactly as the resolve path evaluates it. + # The size supplied is the size ON DISK, not the manifest size -- this + # caller is about to delete these bytes, so it must prove these bytes. + if record.get("schema") != SCHEMA: + return { + "verdict": "held", + "blocker": "incomplete", + "because": "schema is absent or unrecognized", + } + state = record.get("state") + if state == STATE_FAILED: + return { + "verdict": "held", + "blocker": "failed", + "because": "processing failed; disclose, never release", + } + if state not in TERMINAL_STATES: + return { + "verdict": "held", + "blocker": "incomplete", + "because": "state is not terminal", + } + if record.get("handler") != expected_handler: + return { + "verdict": "held", + "blocker": "incomplete", + "because": "the recorded handler is not the one that claims this content", + } + if record.get("input_size") != SIZE: + return { + "verdict": "held", + "blocker": "incomplete", + "because": "the bytes on disk are not the bytes that were processed", + } + + # 5. A record claiming `analyzed` asserts derived output exists. If the rows + # are gone the raw is the only surviving copy of that content, and + # releasing it is unrecoverable. `empty` is the terminal state that + # legitimately has no rows. + # This is what the reference's chunks_win ordering encodes, and a + # unification built only from the conjunction drops it. + if state == STATE_ANALYZED and not has_row: + return { + "verdict": "held", + "blocker": "incomplete", + "because": "the record claims analyzed and the analysis rows are gone; " + "the raw is the only surviving copy", + } + + if state == STATE_EMPTY: + because = ( + "the conjunction held and the handler determined there was nothing to " + "derive, so no analysis rows are expected" + ) + else: + because = "the conjunction held and the derived output is on disk" + return {"verdict": "releasable", "evidence": "record", "because": because} + + +def disposition(gate: str, decision: dict) -> str: + released_by_reference = gate == "eligible" + releases = decision["verdict"] == "releasable" + if released_by_reference == releases: + if releases and decision.get("evidence") == "legacy_rows": + return "narrowed_legacy" + return "agrees" + return "narrowed" if released_by_reference else "widened" + + +def main() -> int: + check = "--check" in sys.argv[1:] + root = Path(tempfile.mkdtemp(prefix="retention-oracle-")) + rows = [] + try: + for shape in SHAPES: + segment = build(root, shape) + gate = resolve_segment_gate(segment).verdict + proof = has_terminal_processing_proof(segment / MEDIA, SIZE) + decision = unified(shape) + row = { + "id": shape.id, + "shape": { + "media": shape.media, + "size": SIZE, + "record": "absent" if shape.omit_record_key else shape.record, + "analysis_row": shape.row, + }, + "reference_gate": gate, + "reference_releases_raw": gate == "eligible", + "reference_proof": proof, + "unified": decision, + "disposition": disposition(gate, decision), + "provenance": "observed", + } + if shape.note: + row["note"] = shape.note + rows.append(row) + finally: + shutil.rmtree(root, ignore_errors=True) + + widened = [row["id"] for row in rows if row["disposition"] == "widened"] + + document = { + "_provenance": { + "generator": "scripts/retention_release_oracle.py", + "method": ( + "the reference verdicts are OBSERVED by running the reference over " + "constructed segments on a real filesystem; the unified verdicts are " + "derived mechanically from one rule, not chosen per row" + ), + "source_revision": revision(), + "reference": REFERENCE, + "reading": ( + "reference_gate 'eligible' means the reference unlinks the owner's raw " + "media. reference_proof is the four-condition conjunction the resolve " + "path runs on the same bytes. They disagree in both directions, which is " + "why this file exists." + ), + "invariant": ( + "no row may carry disposition 'widened'. A port that releases raw the " + "reference held has loosened an irreversible path, and each such row " + "needs its own argument before it is recorded here." + ), + }, + "summary": tally(rows), + "rows": rows, + } + + rendered = json.dumps(document, indent=2, sort_keys=False) + "\n" + + if widened: + print( + f"FAIL: rows widen the irreversible path: {', '.join(widened)}", + file=sys.stderr, + ) + return 1 + + if check: + if not FIXTURE.exists(): + print(f"FAIL: {FIXTURE} does not exist", file=sys.stderr) + return 1 + current = FIXTURE.read_text() + if current != rendered: + print(f"FAIL: {FIXTURE} is stale; regenerate it", file=sys.stderr) + return 1 + print(f"ok: {FIXTURE.relative_to(REPO)} matches the reference") + return 0 + + FIXTURE.parent.mkdir(parents=True, exist_ok=True) + FIXTURE.write_text(rendered) + print(f"wrote {FIXTURE.relative_to(REPO)}: {len(rows)} rows") + for name, count in tally(rows).items(): + print(f" {name}: {count}") + return 0 + + +def tally(rows: list[dict]) -> dict[str, int]: + counts: dict[str, int] = {} + for row in rows: + counts[row["disposition"]] = counts.get(row["disposition"], 0) + 1 + return dict(sorted(counts.items())) + + +def revision() -> str: + try: + return subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=REPO, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "unknown" + + +if __name__ == "__main__": + sys.exit(main()) -- 2.51.2