diff --git a/docs/openapi/convey-clients.json b/docs/openapi/convey-clients.json index 8f93d7b1c..84b077235 100644 --- a/docs/openapi/convey-clients.json +++ b/docs/openapi/convey-clients.json @@ -1293,6 +1293,9 @@ "schema": { "additionalProperties": true, "properties": { + "deterministic_only": { + "type": "boolean" + }, "facet": { "type": "string" }, @@ -1327,7 +1330,10 @@ "facet": null, "path": "/journal/imports/20260618_143022/source.txt", "setting": null, - "timestamp": "20260618_143022" + "timestamp": "20260618_143022", + "timestamp_detection_method": "deterministic", + "timestamp_detection_model_called": false, + "timestamp_detection_no_match_reason": null }, "schema": { "additionalProperties": true, @@ -1349,13 +1355,29 @@ }, "timestamp": { "type": "string" + }, + "timestamp_detection_method": { + "description": "Timestamp detection method: deterministic, model, upload_fallback, or explicit.", + "type": "string" + }, + "timestamp_detection_model_called": { + "type": "boolean" + }, + "timestamp_detection_no_match_reason": { + "type": [ + "string", + "null" + ] } }, "required": [ "path", "timestamp", "facet", - "setting" + "setting", + "timestamp_detection_method", + "timestamp_detection_model_called", + "timestamp_detection_no_match_reason" ], "type": "object" } diff --git a/solstone/apps/import/contract.py b/solstone/apps/import/contract.py index 939de06e7..94491c625 100644 --- a/solstone/apps/import/contract.py +++ b/solstone/apps/import/contract.py @@ -46,6 +46,7 @@ OPERATIONS: list[OperationSpec] = [ FieldSpec("setting", "string"), FieldSpec("imported_via", "string"), FieldSpec("observer_handle", "string"), + FieldSpec("deterministic_only", "boolean"), ), description="Multipart body with either file or text.", ), @@ -68,12 +69,35 @@ OPERATIONS: list[OperationSpec] = [ required=True, raw_schema=_NULLABLE_STRING, ), + FieldSpec( + "timestamp_detection_method", + "string", + required=True, + description=( + "Timestamp detection method: deterministic, model, " + "upload_fallback, or explicit." + ), + ), + FieldSpec( + "timestamp_detection_model_called", + "boolean", + required=True, + ), + FieldSpec( + "timestamp_detection_no_match_reason", + "string", + required=True, + raw_schema=_NULLABLE_STRING, + ), ), example={ "path": "/journal/imports/20260618_143022/source.txt", "timestamp": "20260618_143022", "facet": None, "setting": None, + "timestamp_detection_method": "deterministic", + "timestamp_detection_model_called": False, + "timestamp_detection_no_match_reason": None, }, ), _json_error( diff --git a/solstone/apps/import/routes.py b/solstone/apps/import/routes.py index 764227a19..b9b811119 100644 --- a/solstone/apps/import/routes.py +++ b/solstone/apps/import/routes.py @@ -31,7 +31,7 @@ from solstone.convey.utils import ( respond_collection, success_response, ) -from solstone.think.detect_created import detect_created +from solstone.think.detect_created import detect_created, resolve_created_deterministic from solstone.think.importers.utils import ( build_import_info, generate_content_manifest, @@ -218,6 +218,10 @@ def _link_id_from_identity() -> str | None: ) +def _form_bool(value: str | None) -> bool: + return value.strip().lower() in {"true", "1", "yes"} if value else False + + @import_bp.route("/api/save", methods=["POST"]) def import_save() -> Any: from datetime import datetime @@ -226,6 +230,7 @@ def import_save() -> Any: text = request.form.get("text", "").strip() facet = request.form.get("facet", "").strip() or None setting = request.form.get("setting", "").strip() or None + deterministic_only = _form_bool(request.form.get("deterministic_only")) # Generate timestamp for folder name timestamp_ms = now_ms() @@ -241,6 +246,9 @@ def import_save() -> Any: # Detect timestamp from content first (need temporary save for detection) ts = None detection_result = None + timestamp_detection_method = "upload_fallback" + timestamp_detection_model_called = False + timestamp_detection_no_match_reason = None # Create temporary file for detection if needed if upload: @@ -264,17 +272,47 @@ def import_save() -> Any: temp_path = tmp.name try: - # Pass original filename for better timestamp detection - original_name = upload.filename if upload else None - detection_result = detect_created(temp_path, original_filename=original_name) - if ( - detection_result - and detection_result.get("day") - and detection_result.get("time") - ): - ts = f"{detection_result['day']}_{detection_result['time']}" - except Exception: - ts = None + try: + original_name = upload.filename if upload else None + detection_result = resolve_created_deterministic( + temp_path, + original_filename=original_name, + ) + if ( + detection_result + and detection_result.get("day") + and detection_result.get("time") + ): + ts = f"{detection_result['day']}_{detection_result['time']}" + timestamp_detection_method = "deterministic" + except Exception: + detection_result = None + + if not ts: + if deterministic_only: + timestamp_detection_no_match_reason = "no_deterministic_match" + else: + try: + # Pass original filename for better timestamp detection + original_name = upload.filename if upload else None + detection_result = detect_created( + temp_path, + original_filename=original_name, + ) + timestamp_detection_model_called = True + if ( + detection_result + and detection_result.get("day") + and detection_result.get("time") + ): + ts = f"{detection_result['day']}_{detection_result['time']}" + timestamp_detection_method = "model" + else: + timestamp_detection_no_match_reason = "model_no_match" + except Exception: + detection_result = None + timestamp_detection_model_called = True + timestamp_detection_no_match_reason = "model_no_match" finally: # Clean up temporary file Path(temp_path).unlink(missing_ok=True) @@ -320,6 +358,9 @@ def import_save() -> Any: "detection_result": detection_result, "detected_timestamp": ts, "user_timestamp": folder_timestamp, # The timestamp used for the folder + "timestamp_detection_method": timestamp_detection_method, + "timestamp_detection_model_called": timestamp_detection_model_called, + "timestamp_detection_no_match_reason": timestamp_detection_no_match_reason, "file_size": file_path.stat().st_size if file_path.exists() else 0, "mime_type": upload.content_type if upload else "text/plain", "facet": facet, # Include selected facet @@ -358,6 +399,9 @@ def import_save() -> Any: "timestamp": folder_timestamp, "facet": facet, "setting": setting, + "timestamp_detection_method": timestamp_detection_method, + "timestamp_detection_model_called": timestamp_detection_model_called, + "timestamp_detection_no_match_reason": timestamp_detection_no_match_reason, } if dedup: result["dedup"] = dedup diff --git a/solstone/think/detect_created.py b/solstone/think/detect_created.py index 63f6c0386..6edc76d3b 100644 --- a/solstone/think/detect_created.py +++ b/solstone/think/detect_created.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import re import subprocess from datetime import datetime, timezone from pathlib import Path @@ -13,6 +14,35 @@ from typing import Optional from .prompts import load_prompt +DETERMINISTIC_SOURCE_FILENAME = "filename_local" +DETERMINISTIC_SOURCE_FILENAME_UTC = "filename_utc" +DETERMINISTIC_SOURCE_METADATA_LOCAL = "metadata_local" +DETERMINISTIC_SOURCE_METADATA_UTC = "metadata_utc" + +_LIMITLESS_FILENAME_RE = re.compile( + r"^limitless_pendant_(\d{4})-(\d{2})-(\d{2})T" + r"(\d{2})-(\d{2})-(\d{2})_to_.*$" +) +_LOCAL_FILENAME_RES = ( + re.compile(r"^(\d{4})-(\d{2})-(\d{2})_(\d{2})_(\d{2})_(\d{2})(?!\d)"), + re.compile(r"^(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})(?!\d)"), + re.compile(r"^(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})(?!\d)"), + re.compile(r"^(\d{4})(\d{2})(\d{2})_(\d{2})(\d{2})(\d{2})(?!\d)"), + re.compile(r"^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})(?!\d)"), +) +_METADATA_CREATION_FIELDS = ( + "SubSecCreateDate", + "SubSecDateTimeOriginal", + "CreateDate", + "CreationDate", + "DateTimeOriginal", + "MediaCreateDate", + "TrackCreateDate", + "ContentCreateDate", +) +_OFFSET_RE = re.compile(r"(Z|[+-]\d{2}:\d{2})$") +_SUBSECOND_RE = re.compile(r"(\d{2}:\d{2}:\d{2})\.\d+") + _SCHEMA = json.loads( (Path(__file__).parent / "detect_created.schema.json").read_text(encoding="utf-8") ) @@ -37,6 +67,163 @@ def _extract_metadata(path: str) -> str: return f"Error extracting metadata: {exc}" +def _extract_metadata_json(path: str) -> dict: + """Return JSON metadata for *path* using exiftool if available.""" + cmd = [ + "exiftool", + "-json", + path, + ] + try: + proc = subprocess.run(cmd, capture_output=True, text=True, check=True) + metadata = json.loads(proc.stdout) + except Exception: # pragma: no cover - exiftool optional + return {} + if not isinstance(metadata, list) or not metadata: + return {} + first = metadata[0] + return first if isinstance(first, dict) else {} + + +def _result_from_datetime( + value: datetime, + *, + source: str, + utc: bool, +) -> dict: + return { + "day": value.strftime("%Y%m%d"), + "time": value.strftime("%H%M%S"), + "confidence": "high", + "source": source, + "utc": utc, + } + + +def _parse_datetime_parts(parts: tuple[str, ...]) -> datetime | None: + try: + return datetime(*(int(part) for part in parts)) + except ValueError: + return None + + +def _filename_stem(path: str, original_filename: Optional[str]) -> str: + name = original_filename if original_filename else path + return Path(Path(name).name).stem + + +def _resolve_limitless_filename(stem: str) -> dict | None: + match = _LIMITLESS_FILENAME_RE.match(stem) + if match is None: + return None + parsed = _parse_datetime_parts(match.groups()) + if parsed is None: + return None + utc_dt = parsed.replace(tzinfo=timezone.utc) + return _result_from_datetime( + utc_dt.astimezone(), + source=DETERMINISTIC_SOURCE_FILENAME_UTC, + utc=True, + ) + + +def _resolve_local_filename(stem: str) -> dict | None: + for pattern in _LOCAL_FILENAME_RES: + match = pattern.match(stem) + if match is None: + continue + parsed = _parse_datetime_parts(match.groups()) + if parsed is None: + return None + return _result_from_datetime( + parsed, + source=DETERMINISTIC_SOURCE_FILENAME, + utc=False, + ) + return None + + +def _normalize_metadata_datetime(value: object) -> tuple[str, str, bool] | None: + if not isinstance(value, str): + return None + raw = value.strip() + if not raw or raw.startswith("0000:") or raw.startswith("0000-"): + return None + normalized = raw.replace("T", " ") + normalized = _SUBSECOND_RE.sub(r"\1", normalized) + if len(normalized) >= 10 and normalized[4] == ":" and normalized[7] == ":": + normalized = f"{normalized[:4]}-{normalized[5:7]}-{normalized[8:]}" + normalized = normalized.replace(" ", "T", 1) + + offset_bearing = bool(_OFFSET_RE.search(normalized)) + if normalized.endswith("Z"): + normalized = f"{normalized[:-1]}+00:00" + + try: + parsed = datetime.fromisoformat(normalized) + except ValueError: + return None + if parsed.year == 0: + return None + if offset_bearing: + parsed = parsed.astimezone() + return parsed.strftime("%Y%m%d"), parsed.strftime("%H%M%S"), offset_bearing + + +def _resolve_metadata(path: str) -> dict | None: + try: + metadata = _extract_metadata_json(path) + except Exception: + return None + pairs: set[tuple[str, str]] = set() + saw_offset = False + for field in _METADATA_CREATION_FIELDS: + parsed = _normalize_metadata_datetime(metadata.get(field)) + if parsed is None: + continue + day, time, offset_bearing = parsed + pairs.add((day, time)) + saw_offset = saw_offset or offset_bearing + if len(pairs) != 1: + return None + day, time = next(iter(pairs)) + return { + "day": day, + "time": time, + "confidence": "high", + "source": ( + DETERMINISTIC_SOURCE_METADATA_UTC + if saw_offset + else DETERMINISTIC_SOURCE_METADATA_LOCAL + ), + "utc": saw_offset, + } + + +def resolve_created_deterministic( + path: str, original_filename: Optional[str] = None +) -> Optional[dict]: + """Return deterministic creation time information for *path* when unambiguous. + + Direct-source timestamps, such as Plaud recording start times, bypass this resolver + by passing an explicit timestamp into the importer. + """ + stem = _filename_stem(path, original_filename) + limitless_candidate = _resolve_limitless_filename(stem) + if limitless_candidate is not None: + return limitless_candidate + + filename_candidate = _resolve_local_filename(stem) + metadata_candidate = _resolve_metadata(path) + if filename_candidate is not None and metadata_candidate is not None: + filename_pair = (filename_candidate["day"], filename_candidate["time"]) + metadata_pair = (metadata_candidate["day"], metadata_candidate["time"]) + if filename_pair != metadata_pair: + return None + return filename_candidate + return filename_candidate or metadata_candidate + + def detect_created( path: str, original_filename: Optional[str] = None, guidance: Optional[str] = None ) -> Optional[dict]: diff --git a/solstone/think/import_client.py b/solstone/think/import_client.py index 9cd9a8da2..6e0a94de8 100644 --- a/solstone/think/import_client.py +++ b/solstone/think/import_client.py @@ -30,6 +30,7 @@ MODE_DISPOSITIONS = { "--source": "http-client", "--force": "http-client", "--auto": "http-client", + "--deterministic-only": "http-client", "--dry-run": "reject-journal-host", "--json": "client-output", "-v/--verbose": "client-logging", @@ -70,6 +71,11 @@ def _build_parser() -> argparse.ArgumentParser: default=None, help="Accept the server-detected timestamp", ) + parser.add_argument( + "--deterministic-only", + action="store_true", + help="Use only deterministic timestamp detection; skip model detection", + ) parser.add_argument( "--dry-run", action="store_true", @@ -169,6 +175,8 @@ def _save_media(client: ConveyClient, args: argparse.Namespace) -> dict[str, Any }.items() if value is not None } + if args.deterministic_only: + data["deterministic_only"] = "true" if media_path.exists() and media_path.is_file(): return client.upload( f"{IMPORT_API}/save", diff --git a/solstone/think/importers/cli.py b/solstone/think/importers/cli.py index b96b96e52..ffc66e0c5 100644 --- a/solstone/think/importers/cli.py +++ b/solstone/think/importers/cli.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Any from solstone.think.callosum import CallosumConnection -from solstone.think.detect_created import detect_created +from solstone.think.detect_created import detect_created, resolve_created_deterministic from solstone.think.importers.audio import _get_audio_duration, prepare_audio_segments from solstone.think.importers.shared import ( _get_relative_path, @@ -375,6 +375,7 @@ def import_one( json_output: bool = False, verbose: bool = False, wait_for_processing: bool = True, + deterministic_only: bool = False, ) -> dict[str, Any] | None: """When False, returns after segment creation without awaiting transcription completion; failed_segments is omitted from the result and created_segments is the durable @@ -392,6 +393,7 @@ def import_one( json=json_output, verbose=verbose, wait_for_processing=wait_for_processing, + deterministic_only=deterministic_only, ) return _import_one_from_args(args) @@ -474,13 +476,32 @@ def _import_one_from_args(args: argparse.Namespace) -> dict[str, Any] | None: # File importers don't need an external timestamp — auto-generate for metadata args.timestamp = dt.datetime.now().strftime("%Y%m%d_%H%M%S") elif not args.timestamp: - # If no timestamp provided, detect it - # Pass the original filename for better detection - detection_result = detect_created( + detection_result = resolve_created_deterministic( args.media, original_filename=os.path.basename(args.media), - guidance=args.auto if isinstance(args.auto, str) else None, ) + if ( + not detection_result + or not detection_result.get("day") + or not detection_result.get("time") + ): + if args.deterministic_only: + print( + "No deterministic timestamp found. Provide --timestamp " + "YYYYMMDD_HHMMSS, or omit --deterministic-only to use model " + "detection." + ) + return { + "skipped": True, + "reason": "no_deterministic_match", + } + # If no deterministic timestamp exists, fall back to model detection. + # Pass the original filename for better detection. + detection_result = detect_created( + args.media, + original_filename=os.path.basename(args.media), + guidance=args.auto if isinstance(args.auto, str) else None, + ) if ( detection_result and detection_result.get("day") @@ -1345,6 +1366,11 @@ def main() -> None: action="store_true", help="Show what would be imported without writing to the journal", ) + parser.add_argument( + "--deterministic-only", + action="store_true", + help="Use only deterministic timestamp detection; skip model detection", + ) parser.add_argument( "--backends", action="store_true", @@ -1470,6 +1496,7 @@ def main() -> None: dry_run=args.dry_run, json_output=args.json, verbose=args.verbose, + deterministic_only=args.deterministic_only, ) except Exception as exc: raise SystemExit(str(exc)) from exc diff --git a/tests/test_detect_created_deterministic.py b/tests/test_detect_created_deterministic.py new file mode 100644 index 000000000..ceb26b342 --- /dev/null +++ b/tests/test_detect_created_deterministic.py @@ -0,0 +1,227 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +import importlib +import os +import time + +import pytest + +detect_created_mod = importlib.import_module("solstone.think.detect_created") + + +@pytest.fixture +def tz_los_angeles(monkeypatch): + original_tz = os.environ.get("TZ") + monkeypatch.setenv("TZ", "America/Los_Angeles") + time.tzset() + yield + if original_tz is None: + os.environ.pop("TZ", None) + else: + os.environ["TZ"] = original_tz + time.tzset() + + +@pytest.mark.parametrize( + "filename", + [ + "2024-01-15_10_30_00_copy.m4a", + "2024-01-15 10:30:00.wav", + "2024-01-15_10-30-00.mp3", + "20240115_103000_2.m4a", + "20240115103000.mov", + ], +) +def test_filename_local_formats_and_suffixes(monkeypatch, filename): + monkeypatch.setattr(detect_created_mod, "_extract_metadata_json", lambda path: {}) + + result = detect_created_mod.resolve_created_deterministic( + "/tmp/source", + original_filename=filename, + ) + + assert result == { + "day": "20240115", + "time": "103000", + "confidence": "high", + "source": detect_created_mod.DETERMINISTIC_SOURCE_FILENAME, + "utc": False, + } + + +@pytest.mark.parametrize( + "filename", + [ + "2024-01-15.m4a", + "01-15.m4a", + "2024-13-15_10_30_00.m4a", + "2024-02-30_10_30_00.m4a", + "2024-01-15_25_30_00.m4a", + ], +) +def test_filename_non_matches_and_invalid_values(monkeypatch, filename): + monkeypatch.setattr(detect_created_mod, "_extract_metadata_json", lambda path: {}) + + assert ( + detect_created_mod.resolve_created_deterministic( + "/tmp/source", + original_filename=filename, + ) + is None + ) + + +def test_limitless_filename_utc_converts_to_local(monkeypatch, tz_los_angeles): + def fail_metadata(path): + raise AssertionError("metadata should not be read for limitless filenames") + + monkeypatch.setattr(detect_created_mod, "_extract_metadata_json", fail_metadata) + + result = detect_created_mod.resolve_created_deterministic( + "/tmp/source", + original_filename=( + "limitless_pendant_2024-01-15T18-30-00_to_2024-01-15T18-45-00.m4a" + ), + ) + + assert result == { + "day": "20240115", + "time": "103000", + "confidence": "high", + "source": detect_created_mod.DETERMINISTIC_SOURCE_FILENAME_UTC, + "utc": True, + } + + +def test_limitless_rule_is_source_scoped(monkeypatch): + monkeypatch.setattr(detect_created_mod, "_extract_metadata_json", lambda path: {}) + + result = detect_created_mod.resolve_created_deterministic( + "/tmp/source", + original_filename="other_2024-01-15T18-30-00_to_2024-01-15T18-45-00.m4a", + ) + + assert result is None + + +def test_metadata_local_unambiguous(monkeypatch): + monkeypatch.setattr( + detect_created_mod, + "_extract_metadata_json", + lambda path: { + "CreateDate": "2024:01:15 10:30:00", + "DateTimeOriginal": "2024:01:15 10:30:00", + }, + ) + + result = detect_created_mod.resolve_created_deterministic( + "/tmp/source", + original_filename="voice.m4a", + ) + + assert result == { + "day": "20240115", + "time": "103000", + "confidence": "high", + "source": detect_created_mod.DETERMINISTIC_SOURCE_METADATA_LOCAL, + "utc": False, + } + + +def test_metadata_utc_offset_unambiguous(monkeypatch, tz_los_angeles): + monkeypatch.setattr( + detect_created_mod, + "_extract_metadata_json", + lambda path: {"CreationDate": "2024:01:15 18:30:00+00:00"}, + ) + + result = detect_created_mod.resolve_created_deterministic( + "/tmp/source", + original_filename="voice.m4a", + ) + + assert result == { + "day": "20240115", + "time": "103000", + "confidence": "high", + "source": detect_created_mod.DETERMINISTIC_SOURCE_METADATA_UTC, + "utc": True, + } + + +def test_metadata_ambiguous_voice_memo_falls_through(monkeypatch, tz_los_angeles): + monkeypatch.setattr( + detect_created_mod, + "_extract_metadata_json", + lambda path: { + "CreationDate": "2024:01:15 10:30:00-08:00", + "CreateDate": "2024:01:15 18:30:00", + }, + ) + + assert ( + detect_created_mod.resolve_created_deterministic( + "/tmp/source", + original_filename="voice.m4a", + ) + is None + ) + + +def test_filename_metadata_conflict_falls_through(monkeypatch): + monkeypatch.setattr( + detect_created_mod, + "_extract_metadata_json", + lambda path: {"CreateDate": "2024:01:15 10:30:01"}, + ) + + assert ( + detect_created_mod.resolve_created_deterministic( + "/tmp/source", + original_filename="2024-01-15_10_30_00.m4a", + ) + is None + ) + + +def test_filename_metadata_agree_keeps_filename_source(monkeypatch): + monkeypatch.setattr( + detect_created_mod, + "_extract_metadata_json", + lambda path: {"CreateDate": "2024:01:15 10:30:00"}, + ) + + result = detect_created_mod.resolve_created_deterministic( + "/tmp/source", + original_filename="2024-01-15_10_30_00.m4a", + ) + + assert result["source"] == detect_created_mod.DETERMINISTIC_SOURCE_FILENAME + assert result["day"] == "20240115" + assert result["time"] == "103000" + + +def test_metadata_failure_does_not_block_filename(monkeypatch): + def fail_metadata(path): + raise AssertionError("metadata failed") + + monkeypatch.setattr(detect_created_mod, "_extract_metadata_json", fail_metadata) + + result = detect_created_mod.resolve_created_deterministic( + "/tmp/source", + original_filename="2024-01-15_10_30_00.m4a", + ) + + assert result["source"] == detect_created_mod.DETERMINISTIC_SOURCE_FILENAME + assert result["day"] == "20240115" + assert result["time"] == "103000" + + +def test_extract_metadata_json_returns_empty_on_failure(monkeypatch): + def fail_run(*args, **kwargs): + raise FileNotFoundError("exiftool") + + monkeypatch.setattr(detect_created_mod.subprocess, "run", fail_run) + + assert detect_created_mod._extract_metadata_json("/tmp/source") == {} diff --git a/tests/test_import_client.py b/tests/test_import_client.py index da9031ba2..521bfdb2b 100644 --- a/tests/test_import_client.py +++ b/tests/test_import_client.py @@ -66,6 +66,7 @@ def test_mode_disposition_table_covers_d5_modes() -> None: "--source": "http-client", "--force": "http-client", "--auto": "http-client", + "--deterministic-only": "http-client", "--dry-run": "reject-journal-host", "--json": "client-output", "-v/--verbose": "client-logging", @@ -185,6 +186,21 @@ def test_metadata_and_start_options_forward(tmp_path: Path) -> None: } +def test_deterministic_only_forwards_only_on_save_data(tmp_path: Path) -> None: + media = tmp_path / "sample.txt" + media.write_text("hello", encoding="utf-8") + client = FakeClient() + + code = import_client.main( + [str(media), "--deterministic-only"], + client=client, # type: ignore[arg-type] + ) + + assert code == 0 + assert client.uploads[0]["data"] == {"deterministic_only": "true"} + assert "deterministic_only" not in client.requests[0]["json"] + + def test_json_output_shape(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: media = tmp_path / "sample.txt" media.write_text("hello", encoding="utf-8") diff --git a/tests/test_importer.py b/tests/test_importer.py index 9579d7d9f..cc7529e12 100644 --- a/tests/test_importer.py +++ b/tests/test_importer.py @@ -101,6 +101,9 @@ def _import_route_client( monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) think_utils._journal_path_cache = None monkeypatch.setattr(import_routes, "detect_created", lambda *args, **kwargs: None) + monkeypatch.setattr( + import_routes, "resolve_created_deterministic", lambda *args, **kwargs: None + ) monkeypatch.setattr(import_routes, "now_ms", lambda: 1_765_000_000_000) stamped = identity or ConveyIdentity( @@ -133,6 +136,109 @@ def _post_import_save(client, data: dict): ) +def _read_import_metadata(journal_root: Path, timestamp: str) -> dict: + return json.loads( + (journal_root / "imports" / timestamp / "import.json").read_text( + encoding="utf-8" + ) + ) + + +def test_importer_deterministic_success_skips_model_without_flag(tmp_path, monkeypatch): + mod = importlib.import_module("solstone.think.importers.cli") + media = tmp_path / "note.txt" + media.write_text("meeting notes", encoding="utf-8") + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + think_utils._journal_path_cache = None + monkeypatch.setattr( + mod, + "resolve_created_deterministic", + lambda *args, **kwargs: {"day": "20240115", "time": "103000"}, + ) + + def fail_detect(*args, **kwargs): + raise AssertionError("model detection should not be called") + + monkeypatch.setattr(mod, "detect_created", fail_detect) + + result = mod.import_one(media) + + assert result == { + "skipped": True, + "reason": "timestamp_required", + "detected_timestamp": "20240115_103000", + } + + +def test_importer_deterministic_only_no_match_never_calls_model( + tmp_path, monkeypatch, capsys +): + mod = importlib.import_module("solstone.think.importers.cli") + media = tmp_path / "note.txt" + media.write_text("meeting notes", encoding="utf-8") + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + think_utils._journal_path_cache = None + monkeypatch.setattr( + mod, "resolve_created_deterministic", lambda *args, **kwargs: None + ) + + def fail_detect(*args, **kwargs): + raise AssertionError("model detection should not be called") + + monkeypatch.setattr(mod, "detect_created", fail_detect) + + result = mod.import_one(media, deterministic_only=True) + + assert result == {"skipped": True, "reason": "no_deterministic_match"} + assert "No deterministic timestamp found" in capsys.readouterr().out + + +def test_importer_falls_back_to_model_when_deterministic_missing(tmp_path, monkeypatch): + mod = importlib.import_module("solstone.think.importers.cli") + media = tmp_path / "note.txt" + media.write_text("meeting notes", encoding="utf-8") + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + think_utils._journal_path_cache = None + monkeypatch.setattr( + mod, "resolve_created_deterministic", lambda *args, **kwargs: None + ) + monkeypatch.setattr( + mod, + "detect_created", + lambda *args, **kwargs: {"day": "20240115", "time": "103000"}, + ) + + result = mod.import_one(media) + + assert result == { + "skipped": True, + "reason": "timestamp_required", + "detected_timestamp": "20240115_103000", + } + + +def test_import_one_explicit_timestamp_bypasses_resolver_and_model( + tmp_path, monkeypatch +): + mod = importlib.import_module("solstone.think.importers.cli") + transcript = "hello\nworld" + txt = tmp_path / "sample.txt" + txt.write_text(transcript, encoding="utf-8") + monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) + think_utils._journal_path_cache = None + _configure_text_import_runtime(monkeypatch, mod) + + def fail_detect(*args, **kwargs): + raise AssertionError("timestamp detection should not be called") + + monkeypatch.setattr(mod, "resolve_created_deterministic", fail_detect) + monkeypatch.setattr(mod, "detect_created", fail_detect) + + result = mod.import_one(txt, timestamp="20240101_120000") + + assert result["processed_timestamp"] == "20240101_120000" + + def test_import_save_stamps_web_dashboard_provenance(tmp_path, monkeypatch): client = _import_route_client(tmp_path, monkeypatch) @@ -185,6 +291,126 @@ def test_import_save_stamps_pl_link_id(tmp_path, monkeypatch): assert metadata["link_id"] == fingerprint +def test_import_save_deterministic_success_skips_model_and_audits( + tmp_path, monkeypatch +): + client = _import_route_client(tmp_path, monkeypatch) + import_routes = importlib.import_module("solstone.apps.import.routes") + deterministic_result = { + "day": "20240115", + "time": "103000", + "confidence": "high", + "source": "filename_local", + "utc": False, + } + monkeypatch.setattr( + import_routes, + "resolve_created_deterministic", + lambda *args, **kwargs: deterministic_result, + ) + + def fail_detect(*args, **kwargs): + raise AssertionError("model detection should not be called") + + monkeypatch.setattr(import_routes, "detect_created", fail_detect) + + response = _post_import_save(client, {}) + + assert response.status_code == 200 + body = response.get_json() + assert body["timestamp"] == "20240115_103000" + assert body["timestamp_detection_method"] == "deterministic" + assert body["timestamp_detection_model_called"] is False + assert body["timestamp_detection_no_match_reason"] is None + metadata = _read_import_metadata(tmp_path, body["timestamp"]) + assert metadata["detection_result"] == deterministic_result + assert metadata["detected_timestamp"] == "20240115_103000" + assert metadata["timestamp_detection_method"] == "deterministic" + assert metadata["timestamp_detection_model_called"] is False + assert metadata["timestamp_detection_no_match_reason"] is None + + +def test_import_save_deterministic_only_no_match_uses_upload_fallback_and_audit( + tmp_path, monkeypatch +): + client = _import_route_client(tmp_path, monkeypatch) + import_routes = importlib.import_module("solstone.apps.import.routes") + monkeypatch.setattr( + import_routes, "resolve_created_deterministic", lambda *args, **kwargs: None + ) + + def fail_detect(*args, **kwargs): + raise AssertionError("model detection should not be called") + + monkeypatch.setattr(import_routes, "detect_created", fail_detect) + + response = _post_import_save(client, {"deterministic_only": "true"}) + + assert response.status_code == 200 + body = response.get_json() + expected_timestamp = dt.datetime.fromtimestamp(1_765_000_000_000 / 1000).strftime( + "%Y%m%d_%H%M%S" + ) + assert body["timestamp"] == expected_timestamp + assert body["timestamp_detection_method"] == "upload_fallback" + assert body["timestamp_detection_model_called"] is False + assert body["timestamp_detection_no_match_reason"] == "no_deterministic_match" + metadata = _read_import_metadata(tmp_path, body["timestamp"]) + assert metadata["detected_timestamp"] is None + assert metadata["user_timestamp"] == body["timestamp"] + assert metadata["timestamp_detection_method"] == "upload_fallback" + assert metadata["timestamp_detection_model_called"] is False + assert metadata["timestamp_detection_no_match_reason"] == "no_deterministic_match" + + +def test_import_save_model_success_audits(tmp_path, monkeypatch): + client = _import_route_client(tmp_path, monkeypatch) + import_routes = importlib.import_module("solstone.apps.import.routes") + model_result = {"day": "20240115", "time": "103000"} + monkeypatch.setattr( + import_routes, "resolve_created_deterministic", lambda *args, **kwargs: None + ) + monkeypatch.setattr( + import_routes, "detect_created", lambda *args, **kwargs: model_result + ) + + response = _post_import_save(client, {}) + + assert response.status_code == 200 + body = response.get_json() + assert body["timestamp"] == "20240115_103000" + assert body["timestamp_detection_method"] == "model" + assert body["timestamp_detection_model_called"] is True + assert body["timestamp_detection_no_match_reason"] is None + metadata = _read_import_metadata(tmp_path, body["timestamp"]) + assert metadata["detection_result"] == model_result + assert metadata["timestamp_detection_method"] == "model" + assert metadata["timestamp_detection_model_called"] is True + assert metadata["timestamp_detection_no_match_reason"] is None + + +def test_import_save_model_no_match_audits_upload_fallback(tmp_path, monkeypatch): + client = _import_route_client(tmp_path, monkeypatch) + import_routes = importlib.import_module("solstone.apps.import.routes") + monkeypatch.setattr( + import_routes, "resolve_created_deterministic", lambda *args, **kwargs: None + ) + monkeypatch.setattr(import_routes, "detect_created", lambda *args, **kwargs: None) + + response = _post_import_save(client, {}) + + assert response.status_code == 200 + body = response.get_json() + assert body["timestamp_detection_method"] == "upload_fallback" + assert body["timestamp_detection_model_called"] is True + assert body["timestamp_detection_no_match_reason"] == "model_no_match" + metadata = _read_import_metadata(tmp_path, body["timestamp"]) + assert metadata["detected_timestamp"] is None + assert metadata["timestamp_detection_method"] == "upload_fallback" + assert metadata["timestamp_detection_model_called"] is True + assert metadata["timestamp_detection_no_match_reason"] == "model_no_match" + + def test_cli_import_provenance_defaults(tmp_path, monkeypatch): shared = importlib.import_module("solstone.think.importers.shared") monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) @@ -284,9 +510,12 @@ def test_importer_text(tmp_path, monkeypatch): txt.write_text(transcript) monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) - monkeypatch.setattr( - mod, "detect_created", lambda p, **kw: {"day": "20240101", "time": "120000"} - ) + + def fail_detect(*args, **kwargs): + raise AssertionError("timestamp detection should not be called") + + monkeypatch.setattr(mod, "resolve_created_deterministic", fail_detect) + monkeypatch.setattr(mod, "detect_created", fail_detect) # Mock segment detection: returns (start_at, text) tuples with absolute times def mock_detect_segment(text, start_time):