From cc8fed8f89c09441dff12b75464e20197ff8a85b Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Sun, 5 Apr 2026 21:43:26 -0600 Subject: [PATCH] feat: consolidate transcripts dual traversal into single scan_day() Replaces two parallel iter_segments() passes (cluster_scan + cluster_segments) with a single scan_day() function. Adds combined /api/day/ endpoint and updates frontend to one fetch. day_path() gains create=False to prevent mkdir side effects on read-only requests. --- apps/transcripts/routes.py | 14 ++- apps/transcripts/workspace.html | 27 +++--- tests/test_cluster.py | 59 ++++++++++++ think/cluster.py | 154 +++++++++++++++----------------- think/utils.py | 9 +- 5 files changed, 159 insertions(+), 104 deletions(-) diff --git a/apps/transcripts/routes.py b/apps/transcripts/routes.py index dfa5fcf25..96c549dc8 100644 --- a/apps/transcripts/routes.py +++ b/apps/transcripts/routes.py @@ -29,7 +29,7 @@ from convey.utils import DATE_RE, error_response, format_date, success_response from observe.hear import format_audio from observe.screen import format_screen from observe.utils import AUDIO_EXTENSIONS, VIDEO_EXTENSIONS -from think.cluster import cluster_scan, cluster_segments +from think.cluster import cluster_scan, cluster_segments, scan_day from think.entities.journal import get_journal_principal, load_journal_entity from think.models import get_usage_cost from think.utils import day_dirs, day_path, segment_path @@ -91,6 +91,16 @@ def transcript_segments(day: str) -> Any: return jsonify({"segments": segments}) +@transcripts_bp.route("/api/day/") +def transcript_day_data(day: str) -> Any: + """Return combined ranges and segments for a day in a single response.""" + if not DATE_RE.fullmatch(day): + return "", 404 + + audio_ranges, screen_ranges, segments = scan_day(day) + return jsonify({"audio": audio_ranges, "screen": screen_ranges, "segments": segments}) + + @transcripts_bp.route("/api/serve_file//") def serve_file(day: str, encoded_path: str) -> Any: """Serve actual media files for embedding.""" @@ -101,7 +111,7 @@ def serve_file(day: str, encoded_path: str) -> Any: rel_path = encoded_path.replace("__", "/") full_path = os.path.join(state.journal_root, day, rel_path) - day_dir = str(day_path(day)) + day_dir = str(day_path(day, create=False)) if not os.path.commonpath([full_path, day_dir]) == day_dir: return "", 403 diff --git a/apps/transcripts/workspace.html b/apps/transcripts/workspace.html index fd2c611e1..0176bffc9 100644 --- a/apps/transcripts/workspace.html +++ b/apps/transcripts/workspace.html @@ -2266,20 +2266,15 @@ body.has-date-nav .workspace:has(.tr-wrap) { updateZoom(); }).observe(zoom); - // Load transcript ranges and segments in parallel, initialize after both resolve - const rangesFetch = fetch(`/app/transcripts/api/ranges/${day}`).then(r => { - if (!r.ok) throw new Error(`Ranges failed: ${r.status}`); - return r.json(); - }); - const segmentsFetch = fetch(`/app/transcripts/api/segments/${day}`).then(r => { - if (!r.ok) throw new Error(`Segments failed: ${r.status}`); - return r.json(); - }); - - Promise.all([rangesFetch, segmentsFetch]) - .then(([rangesData, segmentsData]) => { + // Load combined transcript data + fetch(`/app/transcripts/api/day/${day}`) + .then(r => { + if (!r.ok) throw new Error(`Day data failed: ${r.status}`); + return r.json(); + }) + .then(data => { // Apply dynamic timeline bounds from ranges - const bounds = computeTimelineBounds(rangesData); + const bounds = computeTimelineBounds(data); timelineStart = bounds.start; timelineEnd = bounds.end; @@ -2296,17 +2291,17 @@ body.has-date-nav .workspace:has(.tr-wrap) { renderTimeline(); // Add segment indicators from ranges - (rangesData.audio || []).forEach(rg => { + (data.audio || []).forEach(rg => { const [s, e] = rg.map(parseTime); addSegmentIndicator('audio', s, e, 0); }); - (rangesData.screen || []).forEach(rg => { + (data.screen || []).forEach(rg => { const [s, e] = rg.map(parseTime); addSegmentIndicator('screen', s, e, 1); }); // Store segments and update zoom - allSegments = segmentsData.segments || []; + allSegments = data.segments || []; updateZoom(); // Check for hash fragment to auto-select segment diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 09523d7e9..ddf1e83c1 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -576,3 +576,62 @@ def test_agent_matches_filter(): assert _agent_matches_filter("meetings", filter_dict) is False assert _agent_matches_filter("_todos_review", filter_dict) is True assert _agent_matches_filter("flow", filter_dict) is False # Not in filter + + +def test_scan_day_combined(tmp_path, monkeypatch): + monkeypatch.setenv("_SOLSTONE_JOURNAL_OVERRIDE", str(tmp_path)) + day_dir = day_path("20240101") + + mod = importlib.import_module("think.cluster") + + first = day_dir / "default" / "090000_300" + first.mkdir(parents=True) + (first / "audio.jsonl").write_text("{}\n") + (first / "screen.jsonl").write_text('{"raw": "screen.webm"}\n') + + second = day_dir / "default" / "093000_300" + second.mkdir(parents=True) + (second / "audio.jsonl").write_text("{}\n") + + audio_ranges, screen_ranges, segments = mod.scan_day("20240101") + expected_ranges = mod.cluster_scan("20240101") + expected_segments = mod.cluster_segments("20240101") + + assert audio_ranges == [("09:00", "09:15"), ("09:30", "09:45")] + assert screen_ranges == [("09:00", "09:15")] + assert segments == [ + { + "key": "090000_300", + "start": "09:00", + "end": "09:05", + "types": ["audio", "screen"], + "stream": "default", + }, + { + "key": "093000_300", + "start": "09:30", + "end": "09:35", + "types": ["audio"], + "stream": "default", + }, + ] + assert (audio_ranges, screen_ranges) == expected_ranges + assert segments == expected_segments + + +def test_scan_day_empty(tmp_path, monkeypatch): + monkeypatch.setenv("_SOLSTONE_JOURNAL_OVERRIDE", str(tmp_path)) + + mod = importlib.import_module("think.cluster") + + assert mod.scan_day("20250101") == ([], [], []) + + +def test_day_path_create_false(tmp_path, monkeypatch): + monkeypatch.setenv("_SOLSTONE_JOURNAL_OVERRIDE", str(tmp_path)) + + missing = day_path("29990101", create=False) + assert not missing.exists() + + created = day_path("29990101") + assert created.exists() diff --git a/think/cluster.py b/think/cluster.py index dbc5ffe7e..adb4bd2e0 100644 --- a/think/cluster.py +++ b/think/cluster.py @@ -394,59 +394,95 @@ def _slots_to_ranges(slots: list[datetime]) -> list[tuple[str, str]]: return ranges -def cluster_scan(day: str) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]: - """Return 15-minute ranges with transcript and screen content for ``day``. +def _detect_content_types(seg_path: Path) -> list[str]: + """Detect content types present in a segment directory.""" + types = [] + if ( + (seg_path / "audio.jsonl").exists() + or any(seg_path.glob("*_audio.jsonl")) + or any(seg_path.glob("*_transcript.jsonl")) + or any(seg_path.glob("*_transcript.md")) + or (seg_path / "imported.md").exists() + ): + types.append("audio") + if (seg_path / "screen.jsonl").exists() or any(seg_path.glob("*_screen.jsonl")): + types.append("screen") + return types + + +def scan_day( + day: str, +) -> tuple[list[tuple[str, str]], list[tuple[str, str]], list[dict[str, Any]]]: + """Single-pass scan returning both range aggregation and segment list. + + Combines the work of ``cluster_scan()`` and ``cluster_segments()`` + into one ``iter_segments()`` traversal. Args: day: Day folder in ``YYYYMMDD`` format. Returns: - Two lists containing ``(start, end)`` pairs (``HH:MM``) for transcript and - screen content respectively. + Tuple of (audio_ranges, screen_ranges, segments) where ranges are + ``(start, end)`` pairs in ``HH:MM`` format and segments is a list + of dicts with ``key``, ``start``, ``end``, ``types``, and ``stream``. """ + from think.utils import iter_segments, segment_parse - day_dir = str(day_path(day)) - # day_path now ensures dir exists - if not os.path.isdir(day_dir): - return [], [] + day_dir = day_path(day, create=False) + if not day_dir.is_dir(): + return [], [], [] - date_str = _date_str(day_dir) + date_str = _date_str(str(day_dir)) + day_date = datetime.strptime(date_str, "%Y%m%d").date() transcript_slots: set[datetime] = set() percept_slots: set[datetime] = set() - day_path_obj = Path(day_dir) + segments: list[dict[str, Any]] = [] - # Check timestamp subdirectories for content files - from think.utils import iter_segments, segment_parse + for stream_name, _, seg_path in iter_segments(day_dir): + start_time, end_time = segment_parse(seg_path.name) - for _stream, _seg_key, seg_path in iter_segments(day_path_obj): - start_time, _ = segment_parse(seg_path.name) - if start_time: - # Found segment - combine with date to get datetime - day_date = datetime.strptime(date_str, "%Y%m%d").date() + types = _detect_content_types(seg_path) if start_time else [] + + if start_time and types: dt = datetime.combine(day_date, start_time) slot = dt.replace( minute=dt.minute - (dt.minute % 15), second=0, microsecond=0 ) - - # Check for transcript content (legacy audio + new transcript convention) - if ( - (seg_path / "audio.jsonl").exists() - or any(seg_path.glob("*_audio.jsonl")) - or any(seg_path.glob("*_transcript.jsonl")) - or any(seg_path.glob("*_transcript.md")) - or (seg_path / "imported.md").exists() - ): + if "audio" in types: transcript_slots.add(slot) - - # Check for screen content - if (seg_path / "screen.jsonl").exists() or any( - seg_path.glob("*_screen.jsonl") - ): + if "screen" in types: percept_slots.add(slot) - transcript_ranges = _slots_to_ranges(sorted(transcript_slots)) - percept_ranges = _slots_to_ranges(sorted(percept_slots)) - return transcript_ranges, percept_ranges + if start_time and end_time and types: + segments.append( + { + "key": seg_path.name, + "start": start_time.strftime("%H:%M"), + "end": end_time.strftime("%H:%M"), + "types": types, + "stream": stream_name, + } + ) + + audio_ranges = _slots_to_ranges(sorted(transcript_slots)) + screen_ranges = _slots_to_ranges(sorted(percept_slots)) + segments.sort(key=lambda s: s["start"]) + return audio_ranges, screen_ranges, segments + + +def cluster_scan(day: str) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]: + """Return 15-minute ranges with transcript and screen content for ``day``. + + Args: + day: Day folder in ``YYYYMMDD`` format. + + Returns: + Two lists containing ``(start, end)`` pairs (``HH:MM``) for transcript and + screen content respectively. + """ + + audio_ranges, screen_ranges, _ = scan_day(day) + return audio_ranges, screen_ranges def cluster_segments(day: str) -> list[dict[str, Any]]: @@ -465,55 +501,7 @@ def cluster_segments(day: str) -> list[dict[str, Any]]: - end: end time as HH:MM - types: list of content types present ("audio", "screen", or both) """ - from think.utils import segment_parse - - day_dir = str(day_path(day)) - if not os.path.isdir(day_dir): - return [] - - from think.utils import iter_segments - - day_path_obj = Path(day_dir) - segments: list[dict[str, Any]] = [] - - for stream_name, seg_key, seg_path in iter_segments(day_path_obj): - start_time, end_time = segment_parse(seg_path.name) - if not (start_time and end_time): - continue - - types = [] - # Check for transcript content (legacy audio + new transcript convention) - if ( - (seg_path / "audio.jsonl").exists() - or any(seg_path.glob("*_audio.jsonl")) - or any(seg_path.glob("*_transcript.jsonl")) - or any(seg_path.glob("*_transcript.md")) - or (seg_path / "imported.md").exists() - ): - types.append("audio") - - # Check for screen content - if (seg_path / "screen.jsonl").exists() or any(seg_path.glob("*_screen.jsonl")): - types.append("screen") - - if not types: - continue - - start_str = start_time.strftime("%H:%M") - end_str = end_time.strftime("%H:%M") - - segments.append( - { - "key": seg_path.name, - "start": start_str, - "end": end_str, - "types": types, - "stream": stream_name, - } - ) - - # Sort by start time - segments.sort(key=lambda s: s["start"]) + _, _, segments = scan_day(day) return segments diff --git a/think/utils.py b/think/utils.py index 3a0ab6f98..b9466ef5a 100644 --- a/think/utils.py +++ b/think/utils.py @@ -118,13 +118,15 @@ def get_journal() -> str: return journal -def day_path(day: Optional[str] = None) -> Path: +def day_path(day: Optional[str] = None, *, create: bool = True) -> Path: """Return absolute path for a day directory within the journal. Parameters ---------- day : str, optional Day in YYYYMMDD format. If None, uses today's date. + create : bool, optional + Create the day directory if it does not exist. Defaults to True. Returns ------- @@ -145,7 +147,8 @@ def day_path(day: Optional[str] = None) -> Path: raise ValueError("day must be in YYYYMMDD format") path = Path(journal) / day - path.mkdir(parents=True, exist_ok=True) + if create: + path.mkdir(parents=True, exist_ok=True) return path @@ -268,7 +271,7 @@ def iter_segments(day: str | Path) -> list[tuple[str, str, Path]]: if isinstance(day, Path): day_dir = day else: - day_dir = day_path(day) + day_dir = day_path(day, create=False) if not day_dir.exists(): return [] -- 2.51.2