diff --git a/apps/activities/routes.py b/apps/activities/routes.py index 6a015b394..c3dcb1701 100644 --- a/apps/activities/routes.py +++ b/apps/activities/routes.py @@ -266,7 +266,7 @@ def _dev_activities_screens_detail( return "", 404 # Check if the screen.jsonl file exists in segment - segment_dir = str(get_segment_path(day, timestamp, stream)) + segment_dir = str(get_segment_path(day, timestamp, stream, create=False)) jsonl_path = os.path.join(segment_dir, filename) if not os.path.isfile(jsonl_path): return "", 404 @@ -354,7 +354,7 @@ def _dev_screen_frames( if not filename.endswith("screen.jsonl"): return "", 404 - segment_dir = str(get_segment_path(day, timestamp, stream)) + segment_dir = str(get_segment_path(day, timestamp, stream, create=False)) jsonl_path = os.path.join(segment_dir, filename) if not os.path.isfile(jsonl_path): diff --git a/apps/import/ingest.py b/apps/import/ingest.py index 7cafe14e0..9887911a2 100644 --- a/apps/import/ingest.py +++ b/apps/import/ingest.py @@ -31,7 +31,7 @@ from think.entities.journal import ( save_journal_entity, ) from think.entities.matching import find_matching_entity -from think.utils import DEFAULT_STREAM, day_path +from think.utils import DEFAULT_STREAM, STREAM_RE, day_path from .journal_sources import ( get_state_directory, @@ -43,7 +43,6 @@ logger = logging.getLogger(__name__) _DAY_RE = re.compile(r"^\d{8}$") _SEGMENT_RE = re.compile(r"^\d{6}_\d+$") -_STREAM_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$") _FACET_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$") _IMPORT_ID_RE = re.compile(r"^\d{8}_\d{6}$") @@ -150,7 +149,7 @@ def register_ingest_routes(bp) -> None: if not _DAY_RE.match(day): raise ValueError("Invalid day format") - if stream != DEFAULT_STREAM and not _STREAM_RE.match(stream): + if stream != DEFAULT_STREAM and not STREAM_RE.fullmatch(stream): raise ValueError("Invalid stream format") if not _SEGMENT_RE.match(segment_key): raise ValueError("Invalid segment_key format") diff --git a/apps/speakers/attribution.py b/apps/speakers/attribution.py index 280ca05bd..b44973a18 100644 --- a/apps/speakers/attribution.py +++ b/apps/speakers/attribution.py @@ -564,7 +564,7 @@ def accumulate_voiceprints( return {} owner_centroid, owner_threshold = centroid_data - seg_dir = segment_path(day, segment_key, stream) + seg_dir = segment_path(day, segment_key, stream, create=False) emb_data = load_embeddings_file(seg_dir / f"{source}.npz") if emb_data is None: return {} diff --git a/apps/speakers/discovery.py b/apps/speakers/discovery.py index 818c7a9ee..8d7ce531b 100644 --- a/apps/speakers/discovery.py +++ b/apps/speakers/discovery.py @@ -113,7 +113,7 @@ def discover_unknown_speakers() -> dict[str, Any]: for segment in scan_segment_embeddings(day): stream = segment["stream"] seg_key = segment["key"] - seg_dir = segment_path(day, seg_key, stream) + seg_dir = segment_path(day, seg_key, stream, create=False) labels_data = load_speaker_labels(seg_dir) attributed_sids: set[int] = set() @@ -221,7 +221,7 @@ def discover_unknown_speakers() -> dict[str, Any]: continue seen_segments.add(seg_triplet) seg_dir = segment_path( - record["day"], record["segment_key"], record["stream"] + record["day"], record["segment_key"], record["stream"], create=False ) samples.append( { @@ -247,7 +247,7 @@ def discover_unknown_speakers() -> dict[str, Any]: for pos in sorted_positions: record = provenance[int(cluster_indices[int(pos)])] seg_dir = segment_path( - record["day"], record["segment_key"], record["stream"] + record["day"], record["segment_key"], record["stream"], create=False ) sample = { **record, diff --git a/apps/speakers/owner.py b/apps/speakers/owner.py index 41aa8f750..3f303fa5b 100644 --- a/apps/speakers/owner.py +++ b/apps/speakers/owner.py @@ -342,7 +342,7 @@ def classify_sentences( centroid, threshold = centroid_data emb_data = load_embeddings_file( - segment_path(day, segment_key, stream) / f"{source}.npz" + segment_path(day, segment_key, stream, create=False) / f"{source}.npz" ) if emb_data is None: return [] diff --git a/apps/speakers/routes.py b/apps/speakers/routes.py index 59952426a..8538977cb 100644 --- a/apps/speakers/routes.py +++ b/apps/speakers/routes.py @@ -433,7 +433,7 @@ def _load_sentences( - emb_data: Tuple of (embeddings, statement_ids) or None if no embeddings """ if stream: - segment_dir = get_segment_path(day, segment_key, stream) + segment_dir = get_segment_path(day, segment_key, stream, create=False) else: segment_dir = day_path(day) / segment_key @@ -493,7 +493,7 @@ def _get_sentence_embedding( ) -> np.ndarray | None: """Get a specific sentence's embedding, normalized.""" if stream: - segment_dir = get_segment_path(day, segment_key, stream) + segment_dir = get_segment_path(day, segment_key, stream, create=False) else: segment_dir = day_path(day) / segment_key npz_path = segment_dir / f"{source}.npz" @@ -571,7 +571,7 @@ def api_segments(day: str) -> Any: principal = get_journal_principal() principal_id = principal["id"] if principal else None for seg in segments: - seg_dir = get_segment_path(day, seg["key"], seg["stream"]) + seg_dir = get_segment_path(day, seg["key"], seg["stream"], create=False) labels_data = _load_speaker_labels(seg_dir) if labels_data: labels = labels_data.get("labels", []) @@ -612,7 +612,7 @@ def api_segment_speakers(day: str, stream: str, segment_key: str) -> Any: return error_response("Invalid segment key", 400) # Load speakers from speakers.json - segment_dir = get_segment_path(day, segment_key, stream) + segment_dir = get_segment_path(day, segment_key, stream, create=False) speakers = _load_segment_speakers(segment_dir) if not speakers: return jsonify({"matched": [], "unmatched": []}) @@ -658,7 +658,7 @@ def api_review(day: str, stream: str, segment_key: str, source: str) -> Any: if not sentences: return error_response("No transcript found", 404) - segment_dir = get_segment_path(day, segment_key, stream) + segment_dir = get_segment_path(day, segment_key, stream, create=False) labels_data = _load_speaker_labels(segment_dir) label_map: dict[int, dict] = {} if labels_data: diff --git a/apps/transcripts/routes.py b/apps/transcripts/routes.py index c1f406459..f13389656 100644 --- a/apps/transcripts/routes.py +++ b/apps/transcripts/routes.py @@ -34,7 +34,8 @@ from observe.utils import AUDIO_EXTENSIONS, VIDEO_EXTENSIONS from think.cluster import cluster_scan, cluster_segments, scan_day from think.entities.journal import get_journal_principal, load_journal_entity from think.models import get_usage_cost -from think.utils import day_dirs, day_path, segment_path +from think.supervisor import is_supervisor_up +from think.utils import STREAM_RE, day_dirs, day_path, segment_path from think.utils import segment_key as validate_segment_key logger = logging.getLogger(__name__) @@ -248,10 +249,13 @@ def segment_content(day: str, stream: str, segment_key: str) -> Any: if not DATE_RE.fullmatch(day): return error_response("Invalid day format", 404) + if not STREAM_RE.fullmatch(stream): + return error_response("Invalid stream format", 404) + if not validate_segment_key(segment_key): return error_response("Invalid segment key format", 404) - segment_dir = str(segment_path(day, segment_key, stream)) + segment_dir = str(segment_path(day, segment_key, stream, create=False)) if not os.path.isdir(segment_dir): return error_response("Segment directory not found", 404) @@ -505,8 +509,11 @@ def delete_segment(day: str, stream: str, segment_key: str) -> Any: if not validate_segment_key(segment_key): return error_response("Invalid segment key format", 400) - day_dir = str(day_path(day)) - segment_dir = str(segment_path(day, segment_key, stream)) + if not STREAM_RE.fullmatch(stream): + return error_response("Invalid stream format", 400) + + day_dir = str(day_path(day, create=False)) + segment_dir = str(segment_path(day, segment_key, stream, create=False)) # Verify segment exists if not os.path.isdir(segment_dir): @@ -529,6 +536,10 @@ def delete_segment(day: str, stream: str, segment_key: str) -> Any: day=day, ) + payload = {"deleted": segment_key} + if not is_supervisor_up(): + payload["search_index_warning"] = True + # Trigger indexer rescan to remove deleted segment from search index # Supervisor queues by command name, serializing concurrent indexer requests emit( @@ -537,7 +548,7 @@ def delete_segment(day: str, stream: str, segment_key: str) -> Any: cmd=["sol", "indexer", "--rescan-full"], ) - return success_response({"deleted": segment_key}) + return success_response(payload) except OSError as e: return error_response(f"Failed to delete segment: {e}", 500) diff --git a/apps/transcripts/tests/conftest.py b/apps/transcripts/tests/conftest.py index 5d6c79520..5b79a1f02 100644 --- a/apps/transcripts/tests/conftest.py +++ b/apps/transcripts/tests/conftest.py @@ -4,14 +4,45 @@ """Fixtures for transcripts app tests.""" import os +import sys +from pathlib import Path import pytest +from convey import create_app + +ROOT = Path(__file__).resolve().parents[3] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from tests._baseline_harness import copytree_tracked + @pytest.fixture(autouse=True) -def _journal_env(monkeypatch): - """Point _SOLSTONE_JOURNAL_OVERRIDE at the test fixtures.""" +def _journal_env(request, monkeypatch): + """Point tests at a copied journal when needed, otherwise the tracked fixture.""" + if "journal_copy" in request.fixturenames: + journal_copy = request.getfixturevalue("journal_copy") + monkeypatch.setenv("_SOLSTONE_JOURNAL_OVERRIDE", str(journal_copy)) + return + monkeypatch.setenv( "_SOLSTONE_JOURNAL_OVERRIDE", os.path.join(os.getcwd(), "tests", "fixtures", "journal"), ) + + +@pytest.fixture +def client(journal_copy, monkeypatch): + monkeypatch.setenv("_SOLSTONE_JOURNAL_OVERRIDE", str(journal_copy)) + app = create_app(str(journal_copy)) + return app.test_client() + + +@pytest.fixture +def journal_copy(tmp_path, monkeypatch): + src = Path(__file__).resolve().parents[3] / "tests" / "fixtures" / "journal" + dst = tmp_path / "journal" + copytree_tracked(src, dst) + monkeypatch.setenv("_SOLSTONE_JOURNAL_OVERRIDE", str(dst.resolve())) + return dst diff --git a/apps/transcripts/tests/test_segment_routes.py b/apps/transcripts/tests/test_segment_routes.py new file mode 100644 index 000000000..b2477ba19 --- /dev/null +++ b/apps/transcripts/tests/test_segment_routes.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +import pytest + +FIXTURE_DAY = "20260304" +FIXTURE_STREAM = "default" +FIXTURE_SEGMENT = "090000_300" + + +@pytest.mark.parametrize("stream", ["-bad", "Upper", "..bad"]) +def test_segment_content_rejects_invalid_stream(client, stream): + response = client.get( + f"/app/transcripts/api/segment/{FIXTURE_DAY}/{stream}/{FIXTURE_SEGMENT}" + ) + + assert response.status_code == 404 + assert response.get_json() == {"error": "Invalid stream format"} + + +@pytest.mark.parametrize("stream", ["-bad", "Upper", "..bad"]) +def test_delete_segment_rejects_invalid_stream(client, stream): + response = client.delete( + f"/app/transcripts/api/segment/{FIXTURE_DAY}/{stream}/{FIXTURE_SEGMENT}" + ) + + assert response.status_code == 400 + assert response.get_json() == {"error": "Invalid stream format"} + + +def test_segment_content_missing_segment_does_not_create_phantom_directory( + client, journal_copy +): + response = client.get("/app/transcripts/api/segment/29990101/default/090000_300") + + assert response.status_code == 404 + assert response.get_json() == {"error": "Segment directory not found"} + assert not (journal_copy / "chronicle" / "29990101").exists() + assert not ( + journal_copy / "chronicle" / "29990101" / "default" / "090000_300" + ).exists() + + +def test_delete_missing_segment_does_not_create_phantom_directory(client, journal_copy): + response = client.delete("/app/transcripts/api/segment/29990101/default/090000_300") + + assert response.status_code == 404 + assert response.get_json() == {"error": "Segment not found"} + assert not (journal_copy / "chronicle" / "29990101").exists() + assert not ( + journal_copy / "chronicle" / "29990101" / "default" / "090000_300" + ).exists() + + +def test_segment_content_happy_path_returns_segment_payload(client): + response = client.get( + f"/app/transcripts/api/segment/{FIXTURE_DAY}/{FIXTURE_STREAM}/{FIXTURE_SEGMENT}" + ) + + assert response.status_code == 200 + data = response.get_json() + assert data["segment_key"] == FIXTURE_SEGMENT + assert data["chunks"] + assert "media_sizes" in data + + +def test_delete_segment_happy_path_removes_segment_directory( + client, journal_copy, monkeypatch +): + monkeypatch.setattr("apps.transcripts.routes.is_supervisor_up", lambda: True) + segment_dir = ( + journal_copy / "chronicle" / FIXTURE_DAY / FIXTURE_STREAM / FIXTURE_SEGMENT + ) + + response = client.delete( + f"/app/transcripts/api/segment/{FIXTURE_DAY}/{FIXTURE_STREAM}/{FIXTURE_SEGMENT}" + ) + + assert response.status_code == 200 + assert response.get_json() == {"success": True, "deleted": FIXTURE_SEGMENT} + assert not segment_dir.exists() + + +def test_delete_segment_includes_search_index_warning_when_supervisor_is_down(client): + response = client.delete( + f"/app/transcripts/api/segment/{FIXTURE_DAY}/{FIXTURE_STREAM}/{FIXTURE_SEGMENT}" + ) + + assert response.status_code == 200 + assert response.get_json() == { + "success": True, + "deleted": FIXTURE_SEGMENT, + "search_index_warning": True, + } + + +def test_delete_segment_omits_search_index_warning_when_supervisor_is_up( + client, monkeypatch +): + monkeypatch.setattr("apps.transcripts.routes.is_supervisor_up", lambda: True) + + response = client.delete( + f"/app/transcripts/api/segment/{FIXTURE_DAY}/{FIXTURE_STREAM}/{FIXTURE_SEGMENT}" + ) + + assert response.status_code == 200 + assert response.get_json() == {"success": True, "deleted": FIXTURE_SEGMENT} diff --git a/apps/transcripts/workspace.html b/apps/transcripts/workspace.html index 193c8cea5..61bfa3500 100644 --- a/apps/transcripts/workspace.html +++ b/apps/transcripts/workspace.html @@ -787,6 +787,143 @@ body.has-date-nav .workspace:has(.tr-wrap) { flex-shrink: 0; } +#trDeleteSegmentModal { + position: fixed; + inset: 0; + display: none; + align-items: center; + justify-content: center; + padding: 24px; + background: rgba(15, 23, 42, 0.55); + z-index: 1100; +} + +#trDeleteSegmentModal .modal-content { + width: min(520px, 100%); + background: #ffffff; + border-radius: 16px; + box-shadow: 0 24px 48px rgba(15, 23, 42, 0.22); + overflow: hidden; +} + +#trDeleteSegmentModal .modal-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 20px 24px 16px; + border-bottom: 1px solid #e5e7eb; +} + +#trDeleteSegmentModal .modal-header h3 { + margin: 0; + font-size: 20px; + font-weight: 700; + color: #111827; +} + +#trDeleteSegmentModal .modal-header.danger h3 { + color: #991b1b; +} + +#trDeleteSegmentModal .modal-body { + padding: 20px 24px; +} + +#trDeleteSegmentModal .modal-footer { + display: flex; + justify-content: flex-end; + gap: 12px; + padding: 0 24px 24px; +} + +#trDeleteSegmentModal .close { + color: #9ca3af; + font-size: 28px; + line-height: 1; + cursor: pointer; + transition: color 0.15s ease; +} + +#trDeleteSegmentModal .close:hover { + color: #4b5563; +} + +#trDeleteSegmentModal .close:active { + color: #111827; +} + +#trDeleteSegmentModal .btn-secondary, +#trDeleteSegmentModal .btn-danger { + border: none; + border-radius: 10px; + padding: 10px 18px; + font-size: 14px; + font-weight: 600; + cursor: pointer; + transition: background 0.15s ease, color 0.15s ease, transform 0.15s ease; +} + +#trDeleteSegmentModal .btn-secondary { + background: #f3f4f6; + color: #374151; +} + +#trDeleteSegmentModal .btn-secondary:hover { + background: #e5e7eb; +} + +#trDeleteSegmentModal .btn-secondary:active { + transform: translateY(1px); +} + +#trDeleteSegmentModal .btn-danger { + background: #dc2626; + color: #ffffff; +} + +#trDeleteSegmentModal .btn-danger:hover { + background: #b91c1c; +} + +#trDeleteSegmentModal .btn-danger:active { + transform: translateY(1px); +} + +#trDeleteSegmentModal .delete-warning { + padding: 16px 18px; + border-radius: 12px; + background: #fef2f2; + border-left: 4px solid #dc2626; + color: #7f1d1d; +} + +#trDeleteSegmentModal .delete-warning p { + margin: 0 0 12px; +} + +#trDeleteSegmentModal .delete-warning p:last-child { + margin-bottom: 0; +} + +#trDeleteSegmentModal .tr-delete-segment-meta { + font-size: 14px; + color: #991b1b; +} + +#trDeleteSegmentModal .tr-delete-segment-list { + margin: 0 0 12px; + padding-left: 20px; +} + +#trDeleteSegmentModal .tr-delete-segment-list li + li { + margin-top: 6px; +} + +#trDeleteSegmentModal .tr-delete-segment-size { + font-size: 13px; + color: #7f1d1d; +} + /* Unified timeline view */ .tr-unified { display: flex; @@ -1290,6 +1427,22 @@ body.has-date-nav .workspace:has(.tr-wrap) { + +