From ee0e68a4e8df8bba08e982d915e6a4e33a0c805e Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Mon, 13 Jul 2026 05:25:26 -0600 Subject: [PATCH] Enforce health raw retention decisions The approval artifact's raw-retention decision is now operative. The validated PreSaveGateDecision is the sole authority, and save paths do not re-read the artifact after the gate passes. Apple Health discard writes no raw/ at all. retain_parsed retains only raw/export.xml, streamed from the ZIP member so GPX workout routes, ECG CSVs, and clinical CDA records are not copied along, for both ZIP and directory inputs. retain_complete is the only branch that may copy the original archive or full export tree. Oura discard writes no raw page JSONL, while retain_parsed writes pages as before. Under discard, normalized rows and manifests claim no raw path and newly inserted dedupe rows carry a null raw_ref; updates preserve an earlier still-valid raw_ref so historic provenance is never erased and never dangles into the current no-raw bundle. Manifests and returned save metadata name the applied choice. The window-independent raw_ref ordinal contract, the Oura quiet-run cursor-only contract, and catalog-writes-nothing behavior are preserved. (cherry picked from commit f80534d202a128f0a50932c6e521071e51461098) --- docs/design/oura-import.md | 5 +- docs/health_imports.md | 8 +- solstone/think/importers/apple_health.py | 54 +++++- solstone/think/importers/file_importer.py | 1 + solstone/think/importers/oura.py | 43 +++-- tests/test_apple_health_importer.py | 190 ++++++++++++++++++++-- tests/test_health_dedupe.py | 41 +++++ tests/test_importer_presave_gate.py | 15 ++ tests/test_oura_importer.py | 53 ++++++ 9 files changed, 379 insertions(+), 31 deletions(-) diff --git a/docs/design/oura-import.md b/docs/design/oura-import.md index 4c1e6cd84..d908c8eea 100644 --- a/docs/design/oura-import.md +++ b/docs/design/oura-import.md @@ -129,15 +129,18 @@ Mirrors `apple_health` exactly; all writes live under `imports/**` plus (optiona ``` imports// - raw/oura//.json # verbatim API page documents (save phase) + raw/oura/.jsonl # verbatim API pages when raw_retention=retain_parsed normalized/.jsonl # monthly shards, schema solstone.health.oura.v1 manifest.json # shared.write_manifest, source_type "oura" content_manifest.jsonl # shared.write_content_manifest + fetch_windows.json # fetched window evidence for the chunker imports/health-dedupe.sqlite # shared dedupe DB (existing) imports/oura.json # sync cursor (phase O3; never tokens) chronicle//import.oura/000000_300/day_summary_transcript.md # optional, save phase ``` +Importer-owned files under `imports/` are private (`0600`) and importer-owned directories under `imports/` are created or repaired as `0700`. Oura API sync applies the validated `raw_retention.decision`: `retain_parsed` keeps raw API pages, while `discard` writes no raw page JSONL and stores no new `raw_ref` values. + **Normalized row** (implemented in the skeleton): ```json diff --git a/docs/health_imports.md b/docs/health_imports.md index db37b74c0..322602b6c 100644 --- a/docs/health_imports.md +++ b/docs/health_imports.md @@ -9,7 +9,7 @@ The current Apple Health importer supports a gated synthetic/test-week save path - Preview Apple Health `export.xml` data from a directory or zip. - Filter previews and save runs with `--date-from YYYY-MM-DD` and `--date-to YYYY-MM-DD`. - Require the health pre-save gate before any non-dry-run Apple Health write. -- Install the source export under `imports//raw/`. +- Apply the approved raw-retention decision before installing raw source material. - Write normalized monthly JSONL under `imports//normalized/`. - Keep importer-owned record dedupe in `imports/health-dedupe.sqlite`. - Optionally write small factual day-summary transcript files with `--with-day-summaries`. @@ -18,10 +18,14 @@ The live journal and real Apple Health export remain outside automated tests. Us ## Apple Health Local Save Path -Apple Health has a concrete importer save path for orchestrated use after privacy preflight. The importer writes only under the provided `journal_root`: raw source material under `imports//raw/`, normalized monthly JSONL under `imports//normalized/`, importer-owned dedupe rows in `imports/health-dedupe.sqlite`, and optional factual day-summary transcript files under `chronicle/YYYYMMDD/import.apple_health/000000_300/`. +Apple Health has a concrete importer save path for orchestrated use after privacy preflight. The importer writes only under the provided `journal_root`: approved raw source material under `imports//raw/`, normalized monthly JSONL under `imports//normalized/`, importer-owned dedupe rows in `imports/health-dedupe.sqlite`, and optional factual day-summary transcript files under `chronicle/YYYYMMDD/import.apple_health/000000_300/`. Dense normalized JSONL shards are not returned in `ImportResult.files_created`; only optional day-summary transcript files are returned there so indexers do not ingest per-sample health rows. +Raw retention is enforced from the validated gate decision. `discard` writes no `raw/` directory and normalized rows carry no `raw_ref`. `retain_parsed` installs only `raw/export.xml` for Apple Health, whether the input was a zip or an export directory. `retain_complete` is the only Apple Health branch that copies the original zip or full export tree. Oura API sync accepts `discard` and `retain_parsed`: parsed retention keeps the raw API page JSONL files, while discard writes normalized shards, manifests, dedupe rows, fetch windows, and cursor state without raw page files or raw refs. + +All files written under `imports/` by the shared importer writers are installed as `0600`. Import-owned directories under `imports/` are created or repaired as `0700`. The approval-artifact directory `imports/_approvals/` remains manually owner-managed and is not created or repaired by the read-only gate. + ## Source Strategy Use Apple Health as the broad local bus: diff --git a/solstone/think/importers/apple_health.py b/solstone/think/importers/apple_health.py index 7a41d9eb1..0f223870d 100644 --- a/solstone/think/importers/apple_health.py +++ b/solstone/think/importers/apple_health.py @@ -31,9 +31,13 @@ from solstone.think.importers.health_schema import ( health_value_hash, pick_day_sleep, ) -from solstone.think.importers.pre_save_gate import enforce_pre_save_gate +from solstone.think.importers.pre_save_gate import ( + RawRetentionDecision, + enforce_pre_save_gate, +) from solstone.think.importers.shared import ( install_source_file, + install_source_stream, windowed_source_hash, write_content_manifest, write_jsonl_records, @@ -258,6 +262,7 @@ class AppleHealthImporter: summary=f"Dry run only: {preview.summary}", date_range=preview.date_range, ) + assert _gate_decision.raw_retention is not None resolved_import_id = import_id or dt.datetime.now().strftime("%Y%m%d_%H%M%S") result = _save_export( @@ -267,6 +272,7 @@ class AppleHealthImporter: date_window=date_window, with_day_summaries=with_day_summaries, progress_callback=progress_callback, + retention=_gate_decision.raw_retention, ) return ImportResult( entries_written=result["entries_written"], @@ -276,6 +282,7 @@ class AppleHealthImporter: summary=result["summary"], segments=result["segments"] or None, date_range=result["date_range"], + raw_retention=result["raw_retention"], ) @@ -524,10 +531,11 @@ def _save_export( date_window: _DateWindow, with_day_summaries: bool, progress_callback: Callable | None, + retention: RawRetentionDecision, ) -> dict[str, Any]: journal_root = Path(journal_root) import_dir = Path(journal_root) / "imports" / import_id - raw_ref = _install_raw_source(path, import_dir) + raw_ref = _install_raw_source(path, import_dir, retention) normalized_items = _parse_normalized_items( path, import_id=import_id, @@ -591,6 +599,7 @@ def _save_export( len(normalized_items), files_created, days_affected=sorted(summaries), + raw_retention=retention.value, ) date_range = _date_range_from_days(summaries) @@ -599,6 +608,7 @@ def _save_export( "files_created": files_created, "segments": segments, "date_range": date_range, + "raw_retention": retention.value, "summary": ( "Saved Apple Health import: " f"records={len(normalized_items)}, " @@ -651,8 +661,35 @@ def _write_day_summaries( return files, segments -def _install_raw_source(path: Path, import_dir: Path) -> str: +def _install_raw_source( + path: Path, + import_dir: Path, + retention: RawRetentionDecision, +) -> str | None: + if retention == RawRetentionDecision.DISCARD: + return None + raw_dir = import_dir / "raw" + if retention == RawRetentionDecision.RETAIN_PARSED: + raw_path = raw_dir / "export.xml" + if path.is_file(): + with zipfile.ZipFile(path) as archive: + member = _find_export_xml_in_zip(archive.namelist()) + if member is None: + raise FileNotFoundError( + f"No Apple Health export.xml found in {path}" + ) + with archive.open(member) as handle: + install_source_stream(handle, raw_path) + else: + export_xml = _find_export_xml_in_directory(path) + if export_xml is None: + raise FileNotFoundError( + f"No Apple Health export.xml found under {path}" + ) + install_source_file(export_xml, raw_path) + return f"imports/{import_dir.name}/raw/export.xml" + if path.is_file(): raw_path = raw_dir / path.name install_source_file(path, raw_path) @@ -674,7 +711,7 @@ def _parse_normalized_items( *, import_id: str, date_window: _DateWindow, - raw_ref: str, + raw_ref: str | None, progress_callback: Callable | None, ) -> list[_NormalizedItem]: items: list[_NormalizedItem] = [] @@ -703,12 +740,17 @@ def _parse_normalized_items( if elem.tag == "Workout": for key, value in _workout_statistics_metadata(elem).items(): metadata.setdefault(key, value) + item_raw_ref = ( + f"{raw_ref}#{elem.tag.lower()}-{scanned}" + if raw_ref is not None + else None + ) items.append( _normalize_element( elem.tag, attrib, import_id=import_id, - raw_ref=f"{raw_ref}#{elem.tag.lower()}-{scanned}", + raw_ref=item_raw_ref, day=day or "", metadata=metadata, identity_metadata=identity_metadata, @@ -739,7 +781,7 @@ def _normalize_element( attrib: dict[str, str], *, import_id: str, - raw_ref: str, + raw_ref: str | None, day: str, metadata: dict[str, str] | None = None, identity_metadata: dict[str, str] | None = None, diff --git a/solstone/think/importers/file_importer.py b/solstone/think/importers/file_importer.py index 7a4f2c50a..5d7151e67 100644 --- a/solstone/think/importers/file_importer.py +++ b/solstone/think/importers/file_importer.py @@ -36,6 +36,7 @@ class ImportResult: principal_collision: dict[str, Any] | None = None merge_log_path: str | None = None merge_staging_path: str | None = None + raw_retention: str | None = None @runtime_checkable diff --git a/solstone/think/importers/oura.py b/solstone/think/importers/oura.py index 67088de3c..1e4e6cbc0 100644 --- a/solstone/think/importers/oura.py +++ b/solstone/think/importers/oura.py @@ -90,6 +90,7 @@ from solstone.think.importers.health_schema import ( health_value_hash, ) from solstone.think.importers.pre_save_gate import ( + RawRetentionDecision, ScheduledSyncConsent, enforce_oura_sync_gate, enforce_pre_save_gate, @@ -461,7 +462,7 @@ def normalize_bundle( bundle: Mapping[str, Iterable[Mapping[str, Any]]], *, import_id: str, - raw_ref_root: str, + raw_ref_root: str | None, owner_timezone: dt.tzinfo | None = None, ) -> list[OuraNormalizedItem]: """Normalize parsed endpoint items into rows with stable dedupe keys.""" @@ -470,7 +471,11 @@ def normalize_bundle( items: list[OuraNormalizedItem] = [] for endpoint in sorted(bundle): for index, item in enumerate(bundle[endpoint], start=1): - raw_ref = f"{raw_ref_root}#{endpoint}-{index}" + raw_ref = ( + f"{raw_ref_root}#{endpoint}-{index}" + if raw_ref_root is not None + else None + ) items.extend( _normalize_item( endpoint, @@ -488,7 +493,7 @@ def _normalize_item( item: dict[str, Any], *, import_id: str, - raw_ref: str, + raw_ref: str | None, owner_timezone: dt.tzinfo, ) -> list[OuraNormalizedItem]: day = parse_oura_day(item.get(_DOCUMENT_DAY_FIELDS.get(endpoint, "day"))) or "" @@ -872,7 +877,7 @@ def _build_item( unit: str | None, metadata: dict[str, Any], import_id: str, - raw_ref: str, + raw_ref: str | None, ) -> OuraNormalizedItem: dedupe_key = health_record_dedupe_key( HealthRecordIdentity( @@ -1518,6 +1523,7 @@ class OuraSyncBackend: ) assert _gate_decision is not None + assert _gate_decision.raw_retention is not None # Quiet-run check: classify the fetch against the dedupe ledger # BEFORE allocating an import id or writing anything. Dedupe keys @@ -1564,15 +1570,21 @@ class OuraSyncBackend: scheduled_sync=_gate_decision.scheduled_sync, read_artifact=False, ), + raw_retention=_gate_decision.raw_retention.value, quiet_run=True, errors=errors, ) import_id = _new_import_id(journal_root) + raw_ref_root = ( + f"imports/{import_id}/raw/oura" + if _gate_decision.raw_retention == RawRetentionDecision.RETAIN_PARSED + else None + ) items = normalize_bundle( bundle, import_id=import_id, - raw_ref_root=f"imports/{import_id}/raw/oura", + raw_ref_root=raw_ref_root, owner_timezone=owner_timezone, ) saved = _save_sync_bundle( @@ -1582,6 +1594,7 @@ class OuraSyncBackend: raw_pages=raw_pages, bundle=bundle, windows=windows, + retention=_gate_decision.raw_retention, ) # The cursor advances only after every bundle write succeeded; a @@ -1615,6 +1628,7 @@ class OuraSyncBackend: updated=saved["updated"], months=saved["months"], cron_hint=cron_hint, + raw_retention=_gate_decision.raw_retention.value, errors=errors, ) @@ -1715,6 +1729,7 @@ def _sync_result( updated: int, months: list[str], cron_hint: str | None, + raw_retention: str | None = None, known_rows: int = 0, quiet_run: bool = False, errors: list[str] | None = None, @@ -1766,6 +1781,8 @@ def _sync_result( result["import_id"] = import_id if cron_hint is not None: result["cron_hint"] = cron_hint + if raw_retention is not None: + result["raw_retention"] = raw_retention return result @@ -1965,6 +1982,7 @@ def _save_sync_bundle( raw_pages: Mapping[str, list[dict[str, Any]]], bundle: Mapping[str, list[dict[str, Any]]], windows: Mapping[str, tuple[str, str]], + retention: RawRetentionDecision, ) -> dict[str, Any]: """Write one sync run's import bundle, mirroring apple_health save mode. @@ -1978,12 +1996,13 @@ def _save_sync_bundle( # Raw page documents land first so every row's raw_ref points at bytes # that already exist: one JSONL per endpoint, one verbatim API page # per line, under imports//raw/oura/. - for endpoint in sorted(raw_pages): - pages = raw_pages[endpoint] - if pages: - write_jsonl_records( - import_dir / "raw" / "oura" / f"{endpoint}.jsonl", pages - ) + if retention == RawRetentionDecision.RETAIN_PARSED: + for endpoint in sorted(raw_pages): + pages = raw_pages[endpoint] + if pages: + write_jsonl_records( + import_dir / "raw" / "oura" / f"{endpoint}.jsonl", pages + ) normalized_by_month: dict[str, list[dict[str, Any]]] = defaultdict(list) dedupe_records: list[HealthDedupeRecord] = [] @@ -2035,12 +2054,14 @@ def _save_sync_bundle( len(items), files_created=[], days_affected=sorted(days), + raw_retention=retention.value, ) return { "inserted": dedupe_result.inserted, "updated": dedupe_result.updated, "months": [path.stem for path in normalized_paths], + "raw_retention": retention.value, } diff --git a/tests/test_apple_health_importer.py b/tests/test_apple_health_importer.py index 07580bcae..f6277b11d 100644 --- a/tests/test_apple_health_importer.py +++ b/tests/test_apple_health_importer.py @@ -29,7 +29,10 @@ from solstone.think.importers.health_schema import ( pick_day_sleep, pick_main_session, ) -from solstone.think.importers.pre_save_gate import PreSaveGateError +from solstone.think.importers.pre_save_gate import ( + PreSaveGateError, + RawRetentionDecision, +) from tests.conftest import write_health_approval_artifact FIXTURE_ROOT = ( @@ -84,8 +87,22 @@ Glucose 105 mg/dL, 1 workout. """ -def _process_with_approval(path: Path, journal: Path, **kwargs: Any) -> ImportResult: - write_health_approval_artifact(journal, importers=["apple_health"]) +def _process_with_approval( + path: Path, + journal: Path, + *, + raw_retention_decision: str = RawRetentionDecision.RETAIN_PARSED.value, + unparsed_sensitive_modalities_acknowledged: bool | None = None, + **kwargs: Any, +) -> ImportResult: + write_health_approval_artifact( + journal, + importers=["apple_health"], + raw_retention_decision=raw_retention_decision, + unparsed_sensitive_modalities_acknowledged=( + unparsed_sensitive_modalities_acknowledged + ), + ) return AppleHealthImporter().process( path, journal, @@ -347,14 +364,7 @@ def test_save_mode_writes_raw_source_normalized_rows_and_dedupe_to_journal_root( date_to="20260102", ) - raw_export = ( - journal - / "imports" - / "20260103_120000" - / "raw" - / "apple_health_export" - / "export.xml" - ) + raw_export = journal / "imports" / "20260103_120000" / "raw" / "export.xml" normalized = ( journal / "imports" / "20260103_120000" / "normalized" / "2026-01.jsonl" ) @@ -372,6 +382,9 @@ def test_save_mode_writes_raw_source_normalized_rows_and_dedupe_to_journal_root( assert result.segments is None assert result.date_range == ("20260102", "20260102") assert raw_export.read_text(encoding="utf-8").startswith(" None: + journal = tmp_path / f"journal-{source_label}-{retention.value}" + import_id = f"20260103_12{len(source_label):02d}{len(retention.value):02d}" + monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) + + result = _process_with_approval( + source_path, + journal, + import_id=import_id, + dry_run=False, + date_from="2026-01-02", + date_to="2026-01-02", + raw_retention_decision=retention.value, + unparsed_sensitive_modalities_acknowledged=( + True if retention == RawRetentionDecision.RETAIN_COMPLETE else None + ), + ) + + import_dir = journal / "imports" / import_id + files = _relative_files(import_dir) + shard = import_dir / "normalized" / "2026-01.jsonl" + rows = _read_jsonl(shard) + manifest = json.loads((import_dir / "manifest.json").read_text()) + dedupe_row = get_health_dedupe_record(journal, rows[0]["dedupe_key"]) + + assert result.raw_retention == retention.value + assert manifest["raw_retention"] == retention.value + assert dedupe_row is not None + + if retention == RawRetentionDecision.DISCARD: + assert not (import_dir / "raw").exists() + assert all("raw_ref" not in row for row in rows) + assert dedupe_row["raw_ref"] is None + assert "raw/" not in json.dumps(manifest) + return + + assert all(row["raw_ref"] == dedupe_row["raw_ref"] for row in rows[:1]) + assert all("raw_ref" in row for row in rows) + if retention == RawRetentionDecision.RETAIN_PARSED: + assert "raw/export.xml" in files + assert not any( + "workout-routes" in file + or "electrocardiograms" in file + or file.endswith("export_cda.xml") + for file in files + ) + assert rows[0]["raw_ref"].startswith(f"imports/{import_id}/raw/export.xml#") + else: + if source_label == "zip": + assert f"raw/{ZIP_FIXTURE.name}" in files + assert rows[0]["raw_ref"].startswith( + f"imports/{import_id}/raw/{ZIP_FIXTURE.name}#" + ) + else: + assert "raw/apple_health_export/export.xml" in files + assert "raw/apple_health_export/workout-routes/synthetic-route.gpx" in files + assert rows[0]["raw_ref"].startswith( + f"imports/{import_id}/raw/apple_health_export/export.xml#" + ) + + +def test_apple_health_retain_parsed_excludes_cda_and_ecg_files( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = tmp_path / "journal" + monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) + + _process_with_approval( + DTD_FIXTURE_ROOT, + journal, + import_id="20260412_120000", + dry_run=False, + raw_retention_decision=RawRetentionDecision.RETAIN_PARSED.value, + ) + + files = _relative_files(journal / "imports" / "20260412_120000") + + assert "raw/export.xml" in files + assert not any( + "electrocardiograms" in file or file.endswith("export_cda.xml") + for file in files + ) + + def test_apple_health_save_repairs_private_import_modes_under_permissive_umask( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -422,6 +538,58 @@ def test_apple_health_save_repairs_private_import_modes_under_permissive_umask( assert _mode(file_path) == 0o600 +def test_apple_health_discard_update_preserves_historic_dedupe_raw_ref( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = tmp_path / "journal" + monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) + + _process_with_approval( + FIXTURE_ROOT, + journal, + import_id="20260103_120000", + dry_run=False, + date_from="2026-01-02", + date_to="2026-01-02", + raw_retention_decision=RawRetentionDecision.RETAIN_PARSED.value, + ) + first_rows = _read_jsonl( + journal / "imports" / "20260103_120000" / "normalized" / "2026-01.jsonl" + ) + first_glucose = next( + row + for row in first_rows + if row["record_type"] == "HKQuantityTypeIdentifierBloodGlucose" + ) + historic_raw_ref = first_glucose["raw_ref"] + + _process_with_approval( + FIXTURE_ROOT, + journal, + import_id="20260104_120000", + dry_run=False, + date_from="2026-01-02", + date_to="2026-01-02", + raw_retention_decision=RawRetentionDecision.DISCARD.value, + ) + second_rows = _read_jsonl( + journal / "imports" / "20260104_120000" / "normalized" / "2026-01.jsonl" + ) + second_glucose = next( + row + for row in second_rows + if row["record_type"] == "HKQuantityTypeIdentifierBloodGlucose" + ) + dedupe_row = get_health_dedupe_record(journal, second_glucose["dedupe_key"]) + + assert "raw_ref" not in second_glucose + assert not (journal / "imports" / "20260104_120000" / "raw").exists() + assert dedupe_row is not None + assert dedupe_row["raw_ref"] == historic_raw_ref + assert dedupe_row["raw_ref"].startswith("imports/20260103_120000/raw/export.xml#") + + def test_workout_statistics_children_land_in_metadata_without_changing_dedupe( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_health_dedupe.py b/tests/test_health_dedupe.py index 29aea8984..cef0a8f49 100644 --- a/tests/test_health_dedupe.py +++ b/tests/test_health_dedupe.py @@ -261,6 +261,47 @@ def test_existing_broad_dedupe_db_is_repaired_on_open(tmp_path: Path): assert _mode(db_path.parent) == 0o700 +def test_batch_upsert_with_null_raw_ref_preserves_historic_raw_ref(tmp_path: Path): + key = "sha256:historic-raw-ref" + historic_raw_ref = "imports/20260103_120000/raw/export.xml#record-4" + upsert_health_dedupe_records( + tmp_path, + [ + HealthDedupeRecord( + dedupe_key=key, + source_family="apple_health", + record_type="HKQuantityTypeIdentifierBloodGlucose", + start_time="2026-01-02T12:30:00-07:00", + first_import_id="20260103_120000", + last_seen_import_id="20260103_120000", + raw_ref=historic_raw_ref, + ) + ], + ) + + upsert_health_dedupe_records( + tmp_path, + [ + HealthDedupeRecord( + dedupe_key=key, + source_family="apple_health", + record_type="HKQuantityTypeIdentifierBloodGlucose", + start_time="2026-01-02T12:30:00-07:00", + first_import_id="20260104_120000", + last_seen_import_id="20260104_120000", + normalized_ref="imports/20260104_120000/normalized/2026-01.jsonl#L1", + raw_ref=None, + ) + ], + ) + + row = get_health_dedupe_record(tmp_path, key) + + assert row is not None + assert row["raw_ref"] == historic_raw_ref + assert row["last_seen_import_id"] == "20260104_120000" + + def test_upsert_health_dedupe_records_handles_duplicate_keys_in_batch(tmp_path: Path): result = upsert_health_dedupe_records( tmp_path, diff --git a/tests/test_importer_presave_gate.py b/tests/test_importer_presave_gate.py index 1e254c534..db2ef8369 100644 --- a/tests/test_importer_presave_gate.py +++ b/tests/test_importer_presave_gate.py @@ -185,6 +185,21 @@ def test_apple_health_save_missing_artifact_blocks_before_setup( assert not (journal / "imports" / "20260102_123000").exists() +def test_blocked_health_gate_creates_no_directories(tmp_path: Path): + journal = tmp_path / "missing-journal" + + with pytest.raises(PreSaveGateError) as exc_info: + enforce_pre_save_gate( + "apple_health", + dry_run=False, + confirm_health_save=True, + journal_root=journal, + ) + + assert exc_info.value.to_dict()["gate_reason"] == "missing_approval_artifact" + assert not journal.exists() + + def test_cli_apple_health_save_missing_artifact_blocks_before_setup( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_oura_importer.py b/tests/test_oura_importer.py index 2fbb5fbd5..1152a7d28 100644 --- a/tests/test_oura_importer.py +++ b/tests/test_oura_importer.py @@ -1621,6 +1621,7 @@ def test_first_sync_save_writes_bundle_dedupe_and_cursor(tmp_path: Path, monkeyp assert manifest["source_type"] == SOURCE_OURA_API assert manifest["entry_count"] == _SYNC_ROW_COUNT assert manifest["days_affected"] == ["20260102", "20260103"] + assert manifest["raw_retention"] == RawRetentionDecision.RETAIN_PARSED.value content_lines = (import_dir / "content_manifest.jsonl").read_text().splitlines() assert json.loads(content_lines[0])["type"] == "health_normalized_month" @@ -1668,6 +1669,58 @@ def test_first_sync_save_writes_bundle_dedupe_and_cursor(tmp_path: Path, monkeyp assert "cron_hint" not in result +@pytest.mark.parametrize( + ("retention", "expect_raw_pages"), + [ + (RawRetentionDecision.RETAIN_PARSED, True), + (RawRetentionDecision.DISCARD, False), + ], +) +def test_oura_sync_applies_raw_retention_choice( + tmp_path: Path, + monkeypatch, + retention: RawRetentionDecision, + expect_raw_pages: bool, +) -> None: + journal = _use_journal(tmp_path, monkeypatch) + _write_sync_artifact( + journal, + _sync_artifact(journal, raw_retention_decision=retention.value), + ) + + result = oura.backend.sync( + journal, + dry_run=False, + confirm_health_save=True, + client=_canned_client(_fixture_transport()), + today=dt.date(2026, 1, 10), + ) + + import_dir = journal / "imports" / result["import_id"] + raw_dir = import_dir / "raw" + rows = [ + json.loads(line) + for line in (import_dir / "normalized" / "2026-01.jsonl") + .read_text() + .splitlines() + ] + manifest = json.loads((import_dir / "manifest.json").read_text()) + dedupe_row = get_health_dedupe_record(journal, rows[0]["dedupe_key"]) + + assert result["raw_retention"] == retention.value + assert manifest["raw_retention"] == retention.value + assert dedupe_row is not None + if expect_raw_pages: + assert (raw_dir / "oura" / "daily_readiness.jsonl").is_file() + assert rows[0]["raw_ref"].startswith(f"imports/{result['import_id']}/raw/oura#") + assert dedupe_row["raw_ref"] == rows[0]["raw_ref"] + else: + assert not raw_dir.exists() + assert all("raw_ref" not in row for row in rows) + assert dedupe_row["raw_ref"] is None + assert "raw/" not in json.dumps(manifest) + + def test_oura_sync_private_modes_under_permissive_umask( tmp_path: Path, monkeypatch, -- 2.51.2