diff --git a/tests/test_import_dedup.py b/tests/test_import_dedup.py index d433439b3..858535422 100644 --- a/tests/test_import_dedup.py +++ b/tests/test_import_dedup.py @@ -3,12 +3,14 @@ """Tests for import deduplication — manifests, source-level dedup, entry merge.""" +import hashlib import json import os import tempfile from pathlib import Path from think.importers.shared import ( + _build_import_manifest, _entry_content_key, _load_existing_entries, find_manifest_by_hash, @@ -68,6 +70,37 @@ def test_hash_source_directory_changes(): # --- manifest tests --- +def test_build_import_manifest(): + with tempfile.TemporaryDirectory() as tmpdir: + import_dir = Path(tmpdir) + alpha = import_dir / "alpha.txt" + alpha_bytes = b"alpha payload" + alpha.write_bytes(alpha_bytes) + beta = import_dir / "nested" / "beta.bin" + beta.parent.mkdir(parents=True) + beta_bytes = b"\x00\x01beta" + beta.write_bytes(beta_bytes) + + manifest = _build_import_manifest(import_dir) + + assert manifest["import_dir"] == str(import_dir) + assert manifest["file_count"] == 2 + assert manifest["total_bytes"] == len(alpha_bytes) + len(beta_bytes) + assert manifest["files"] == [ + { + "name": "alpha.txt", + "bytes": len(alpha_bytes), + "hash": hashlib.sha256(alpha_bytes).hexdigest(), + }, + { + "name": "nested/beta.bin", + "bytes": len(beta_bytes), + "hash": hashlib.sha256(beta_bytes).hexdigest(), + }, + ] + assert manifest["timestamp"].endswith("+00:00") + + def test_write_and_find_manifest(): with tempfile.TemporaryDirectory() as journal: manifest_path = write_manifest( diff --git a/tests/test_importer.py b/tests/test_importer.py index babdc1345..bdf7eba33 100644 --- a/tests/test_importer.py +++ b/tests/test_importer.py @@ -2,6 +2,7 @@ # Copyright (c) 2026 sol pbc import datetime as dt +import hashlib import importlib import json import subprocess @@ -9,6 +10,8 @@ import zipfile from pathlib import Path from unittest.mock import ANY, MagicMock, patch +import pytest + from think.importers.file_importer import ImportPreview, ImportResult from think.utils import day_path @@ -37,6 +40,47 @@ def _make_mock_file_importer(name="ics", display_name="ICS Calendar"): return mock_imp +def _configure_text_import_runtime(monkeypatch, mod): + """Patch text import processing and callosum helpers for CLI tests.""" + text_mod = importlib.import_module("think.importers.text") + + monkeypatch.setattr( + text_mod, + "detect_transcript_segment", + lambda text, start_time: [("12:00:00", text)], + ) + monkeypatch.setattr( + text_mod, + "detect_transcript_json", + lambda text, segment_start: { + "entries": [ + { + "start": segment_start, + "speaker": "Unknown", + "text": text, + } + ], + "topics": "", + "setting": "", + }, + ) + monkeypatch.setattr(mod, "CallosumConnection", lambda **kwargs: MagicMock()) + monkeypatch.setattr(mod, "_status_emitter", lambda: None) + + +def _read_action_entries(journal_root: Path) -> list[dict]: + """Read journal-level app action log entries for today.""" + today = dt.datetime.now().strftime("%Y%m%d") + log_path = journal_root / "config" / "actions" / f"{today}.jsonl" + if not log_path.exists(): + return [] + return [ + json.loads(line) + for line in log_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + def test_slice_audio_segment(tmp_path): """Test slice_audio_segment extracts audio with stream copy.""" mod = importlib.import_module("think.importers.audio") @@ -924,6 +968,121 @@ def test_importer_dry_run_auto(tmp_path, monkeypatch, capsys): assert not (tmp_path / "chronicle" / "20240315").exists() +def test_importer_force_reimport_logs_manifest_and_replaces_directory( + tmp_path, monkeypatch +): + """--force logs a manifest, removes the old import dir, and writes the new file.""" + mod = importlib.import_module("think.importers.cli") + + timestamp = "20240101_120000" + old_import_dir = tmp_path / "imports" / timestamp + old_import_dir.mkdir(parents=True) + stale_file = old_import_dir / "stale.txt" + stale_bytes = b"old import payload" + stale_file.write_bytes(stale_bytes) + nested_file = old_import_dir / "nested" / "extra.bin" + nested_file.parent.mkdir(parents=True) + nested_bytes = b"\x00\x01nested" + nested_file.write_bytes(nested_bytes) + + txt = tmp_path / "replacement.txt" + txt.write_text("replacement transcript") + + monkeypatch.setenv("_SOLSTONE_JOURNAL_OVERRIDE", str(tmp_path)) + _configure_text_import_runtime(monkeypatch, mod) + monkeypatch.setattr( + "sys.argv", + ["sol import", str(txt), "--timestamp", timestamp, "--force"], + ) + + mod.main() + + action_entries = _read_action_entries(tmp_path) + assert len(action_entries) == 1 + entry = action_entries[0] + assert entry["action"] == "import_force_reimport" + assert entry["actor"] == "import" + assert entry["params"]["dry_run"] is False + assert entry["params"]["import_dir"] == str(old_import_dir) + assert entry["params"]["file_count"] == 2 + assert entry["params"]["total_bytes"] == len(stale_bytes) + len(nested_bytes) + assert entry["params"]["files"] == [ + { + "name": "nested/extra.bin", + "bytes": len(nested_bytes), + "hash": hashlib.sha256(nested_bytes).hexdigest(), + }, + { + "name": "stale.txt", + "bytes": len(stale_bytes), + "hash": hashlib.sha256(stale_bytes).hexdigest(), + }, + ] + + assert not stale_file.exists() + assert not nested_file.exists() + new_imported_file = old_import_dir / "replacement.txt" + assert new_imported_file.exists() + assert new_imported_file.read_text() == "replacement transcript" + + +def test_importer_force_dry_run_logs_manifest_without_deleting(tmp_path, monkeypatch): + """--force --dry-run logs the manifest but leaves the old import dir untouched.""" + mod = importlib.import_module("think.importers.cli") + + timestamp = "20240101_120000" + old_import_dir = tmp_path / "imports" / timestamp + old_import_dir.mkdir(parents=True) + stale_file = old_import_dir / "stale.txt" + stale_file.write_text("old import payload") + + txt = tmp_path / "replacement.txt" + txt.write_text("replacement transcript") + + monkeypatch.setenv("_SOLSTONE_JOURNAL_OVERRIDE", str(tmp_path)) + monkeypatch.setattr( + "sys.argv", + ["sol import", str(txt), "--timestamp", timestamp, "--force", "--dry-run"], + ) + + mod.main() + + action_entries = _read_action_entries(tmp_path) + assert len(action_entries) == 1 + entry = action_entries[0] + assert entry["action"] == "import_force_reimport" + assert entry["params"]["dry_run"] is True + assert entry["params"]["import_dir"] == str(old_import_dir) + assert entry["params"]["file_count"] == 1 + assert stale_file.exists() + assert stale_file.read_text() == "old import payload" + assert not (old_import_dir / "replacement.txt").exists() + + +def test_importer_existing_import_without_force_still_errors(tmp_path, monkeypatch): + """Existing imports still error without --force and do not log a reimport action.""" + mod = importlib.import_module("think.importers.cli") + + timestamp = "20240101_120000" + existing_import_dir = tmp_path / "imports" / timestamp + existing_import_dir.mkdir(parents=True) + (existing_import_dir / "stale.txt").write_text("old import payload") + + txt = tmp_path / "replacement.txt" + txt.write_text("replacement transcript") + + monkeypatch.setenv("_SOLSTONE_JOURNAL_OVERRIDE", str(tmp_path)) + monkeypatch.setattr( + "sys.argv", + ["sol import", str(txt), "--timestamp", timestamp], + ) + + with pytest.raises(SystemExit, match="Import already exists"): + mod.main() + + assert _read_action_entries(tmp_path) == [] + + def test_file_importer_without_timestamp(tmp_path, monkeypatch, capsys): """File importers auto-generate timestamp and skip import setup.""" mod = importlib.import_module("think.importers.cli") diff --git a/think/importers/cli.py b/think/importers/cli.py index 8fc6b7aa1..3bc515b2c 100644 --- a/think/importers/cli.py +++ b/think/importers/cli.py @@ -497,6 +497,12 @@ def main() -> None: import_source = "audio" stream = stream_name(import_source=import_source) + needs_setup = _file_importer is None and not _is_in_imports(args.media) + force_reimport_preview = ( + args.force + and needs_setup + and (Path(get_journal()) / "imports" / args.timestamp).exists() + ) if args.dry_run and _file_importer is not None: preview = _file_importer.preview(Path(args.media)) @@ -618,11 +624,18 @@ def main() -> None: print() print(" Duration: unknown (ffprobe failed)") print() + if force_reimport_preview: + _setup_import( + args.media, + args.timestamp, + args.facet, + args.setting, + detection_result, + force=True, + dry_run=True, + ) return - # Check if file needs setup (not already in imports/) - needs_setup = _file_importer is None and not _is_in_imports(args.media) - # Copy to imports/ if file is not already there if needs_setup: args.media = _setup_import( @@ -632,6 +645,7 @@ def main() -> None: args.setting, detection_result, force=args.force, + dry_run=args.dry_run, ) print("Starting import...") diff --git a/think/importers/shared.py b/think/importers/shared.py index 13411271d..fad5b58cc 100644 --- a/think/importers/shared.py +++ b/think/importers/shared.py @@ -298,6 +298,36 @@ def _is_in_imports(media_path: str) -> bool: return abs_media.startswith(abs_imports + os.sep) +def _build_import_manifest(import_dir: Path) -> dict[str, Any]: + """Return a hash manifest for files currently stored in an import directory.""" + files = sorted(path for path in import_dir.rglob("*") if path.is_file()) + entries: list[dict[str, Any]] = [] + total_bytes = 0 + + for path in files: + size = path.stat().st_size + digest = hashlib.sha256() + with open(path, "rb") as handle: + while chunk := handle.read(64 * 1024): + digest.update(chunk) + entries.append( + { + "name": path.relative_to(import_dir).as_posix(), + "bytes": size, + "hash": digest.hexdigest(), + } + ) + total_bytes += size + + return { + "timestamp": dt.datetime.now(dt.timezone.utc).isoformat(), + "import_dir": str(import_dir), + "total_bytes": total_bytes, + "file_count": len(entries), + "files": entries, + } + + def _setup_import( media_path: str, timestamp: str, @@ -305,14 +335,28 @@ def _setup_import( setting: str | None, detection_result: dict | None, force: bool = False, + dry_run: bool = False, ) -> str: """Copy file to imports/ and write metadata. Returns new file path.""" journal_root = Path(get_journal()) import_dir = journal_root / "imports" / timestamp + filename = os.path.basename(media_path) + dry_run_target_path = str(import_dir / filename) # Check for conflict if import_dir.exists(): if force: + from apps.utils import log_app_action + + manifest = _build_import_manifest(import_dir) + log_app_action( + app="import", + facet=None, + action="import_force_reimport", + params={**manifest, "dry_run": dry_run}, + ) + if dry_run: + return dry_run_target_path logger.info(f"Removing existing import directory: {import_dir}") shutil.rmtree(import_dir) else: @@ -322,7 +366,6 @@ def _setup_import( ) # Copy file to imports/ - filename = os.path.basename(media_path) new_path = save_import_file( journal_root=journal_root, timestamp=timestamp,