diff --git a/apps/import/journal_sources.py b/apps/import/journal_sources.py index cfdc6907e..85ef0f714 100644 --- a/apps/import/journal_sources.py +++ b/apps/import/journal_sources.py @@ -14,6 +14,7 @@ from pathlib import Path from flask import abort, g, request from apps.utils import get_app_storage_path +from convey import state logger = logging.getLogger(__name__) @@ -22,7 +23,9 @@ STATE_AREAS = ("segments", "entities", "facets", "imports", "config") def is_valid_journal_source_name(name: str) -> bool: - return bool(name) and name not in {".", ".."} and "/" not in name and "\\" not in name + return ( + bool(name) and name not in {".", ".."} and "/" not in name and "\\" not in name + ) def generate_key() -> str: @@ -98,6 +101,10 @@ def create_state_directory(journal_root: Path, key_prefix: str) -> Path: return state_dir +def get_state_directory(key_prefix: str) -> Path: + return Path(state.journal_root) / "imports" / key_prefix + + def require_journal_source(f): @wraps(f) def wrapped(*args, **kwargs): diff --git a/apps/import/routes.py b/apps/import/routes.py index 4f5cceb87..99bd82b54 100644 --- a/apps/import/routes.py +++ b/apps/import/routes.py @@ -9,17 +9,9 @@ import time from pathlib import Path from typing import Any -from flask import Blueprint, jsonify, render_template, request +from flask import Blueprint, abort, g, jsonify, render_template, request from werkzeug.utils import secure_filename -from .journal_sources import ( - create_state_directory, - find_journal_source_by_name, - generate_key, - is_valid_journal_source_name, - list_journal_sources, - save_journal_source, -) from apps.utils import log_app_action from convey import emit, state from media import MEDIA_EXTENSIONS @@ -37,6 +29,18 @@ from think.importers.utils import ( ) from think.utils import now_ms +from .journal_sources import ( + STATE_AREAS, + create_state_directory, + find_journal_source_by_name, + generate_key, + get_state_directory, + is_valid_journal_source_name, + list_journal_sources, + require_journal_source, + save_journal_source, +) + import_bp = Blueprint( "app:import", __name__, @@ -929,3 +933,18 @@ def api_journal_source_status(name: str) -> Any: "stats": source.get("stats", {}), } ) + + +@import_bp.route("/journal//manifest/") +@require_journal_source +def journal_source_manifest(key_prefix: str, area: str) -> Any: + if g.journal_source["key"][:8] != key_prefix: + abort(403, description="Key prefix mismatch") + if area not in STATE_AREAS: + abort(404, description="Unknown manifest area") + state_path = get_state_directory(key_prefix) / area / "state.json" + try: + data = json.loads(state_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + data = {} + return jsonify(data) diff --git a/convey/root.py b/convey/root.py index e7221796e..f5ec53ed6 100644 --- a/convey/root.py +++ b/convey/root.py @@ -98,6 +98,8 @@ def require_login() -> Any: "app:observer.ingest_transfer", "app:observer.ingest_manifest", "app:observer.ingest_manifest_day", + # Journal-source manifest endpoints use key-based auth, not session + "app:import.journal_source_manifest", }: return None diff --git a/tests/test_journal_sources.py b/tests/test_journal_sources.py index d00e58b8f..8f00257a7 100644 --- a/tests/test_journal_sources.py +++ b/tests/test_journal_sources.py @@ -9,7 +9,7 @@ import stat from importlib import import_module import pytest -from flask import Flask, g, jsonify +from flask import Flask, abort, g, jsonify import convey.state from think.utils import now_ms @@ -19,6 +19,7 @@ STATE_AREAS = journal_sources.STATE_AREAS create_state_directory = journal_sources.create_state_directory find_journal_source_by_name = journal_sources.find_journal_source_by_name generate_key = journal_sources.generate_key +get_state_directory = journal_sources.get_state_directory is_valid_journal_source_name = journal_sources.is_valid_journal_source_name list_journal_sources = journal_sources.list_journal_sources load_journal_source = journal_sources.load_journal_source @@ -29,7 +30,9 @@ save_journal_source = journal_sources.save_journal_source @pytest.fixture def journal_env(tmp_path, monkeypatch): monkeypatch.setattr(convey.state, "journal_root", str(tmp_path), raising=False) - (tmp_path / "apps" / "import" / "journal_sources").mkdir(parents=True, exist_ok=True) + (tmp_path / "apps" / "import" / "journal_sources").mkdir( + parents=True, exist_ok=True + ) return tmp_path @@ -51,6 +54,37 @@ def _source(name: str, key: str, created_at: int = 0) -> dict: } +@pytest.fixture +def manifest_env(journal_env): + """Journal env with a saved source and state directory.""" + key = generate_key() + source = _source("manifest-test", key, created_at=123) + save_journal_source(source) + create_state_directory(journal_env, key[:8]) + return {"root": journal_env, "key": key, "source": source} + + +@pytest.fixture +def manifest_app(manifest_env): + app = Flask(__name__) + + @app.route("/journal//manifest/") + @require_journal_source + def journal_source_manifest(key_prefix: str, area: str): + if g.journal_source["key"][:8] != key_prefix: + abort(403, description="Key prefix mismatch") + if area not in STATE_AREAS: + abort(404, description="Unknown manifest area") + state_path = get_state_directory(key_prefix) / area / "state.json" + try: + data = json.loads(state_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + data = {} + return jsonify(data) + + return app + + def test_generate_key(): key = generate_key() assert len(key) == 43 @@ -124,7 +158,10 @@ def test_duplicate_name_rejected(journal_env): def test_invalid_name_rejected(journal_env): assert is_valid_journal_source_name("../alpha") is False - assert save_journal_source(_source("../alpha", generate_key(), created_at=123)) is False + assert ( + save_journal_source(_source("../alpha", generate_key(), created_at=123)) + is False + ) assert find_journal_source_by_name("../alpha") is None assert not (journal_env.parent / "alpha.json").exists() @@ -215,3 +252,112 @@ def test_auth_decorator_revoked_key(journal_env): ) assert response.status_code == 403 + + +@pytest.mark.parametrize("area", STATE_AREAS) +def test_manifest_empty_state(manifest_app, manifest_env, area): + response = manifest_app.test_client().get( + f"/journal/{manifest_env['key'][:8]}/manifest/{area}", + headers={"Authorization": f"Bearer {manifest_env['key']}"}, + ) + + assert response.status_code == 200 + assert response.get_json() == {} + + +def test_manifest_populated_state(manifest_app, manifest_env): + data = {"days": {"2026-04-01": {"count": 5}}} + state_path = ( + get_state_directory(manifest_env["key"][:8]) / "segments" / "state.json" + ) + state_path.write_text(json.dumps(data), encoding="utf-8") + + response = manifest_app.test_client().get( + f"/journal/{manifest_env['key'][:8]}/manifest/segments", + headers={"Authorization": f"Bearer {manifest_env['key']}"}, + ) + + assert response.status_code == 200 + assert response.get_json() == data + + +def test_manifest_missing_state_file(manifest_app, manifest_env): + state_path = ( + get_state_directory(manifest_env["key"][:8]) / "segments" / "state.json" + ) + state_path.unlink() + + response = manifest_app.test_client().get( + f"/journal/{manifest_env['key'][:8]}/manifest/segments", + headers={"Authorization": f"Bearer {manifest_env['key']}"}, + ) + + assert response.status_code == 200 + assert response.get_json() == {} + + +def test_manifest_malformed_json(manifest_app, manifest_env): + state_path = ( + get_state_directory(manifest_env["key"][:8]) / "segments" / "state.json" + ) + state_path.write_text("not valid json{{{", encoding="utf-8") + + response = manifest_app.test_client().get( + f"/journal/{manifest_env['key'][:8]}/manifest/segments", + headers={"Authorization": f"Bearer {manifest_env['key']}"}, + ) + + assert response.status_code == 200 + assert response.get_json() == {} + + +def test_manifest_invalid_area(manifest_app, manifest_env): + response = manifest_app.test_client().get( + f"/journal/{manifest_env['key'][:8]}/manifest/invalid_area", + headers={"Authorization": f"Bearer {manifest_env['key']}"}, + ) + + assert response.status_code == 404 + + +def test_manifest_key_prefix_mismatch(manifest_app, manifest_env): + other_prefix = "deadbeef" + assert other_prefix != manifest_env["key"][:8] + + response = manifest_app.test_client().get( + f"/journal/{other_prefix}/manifest/segments", + headers={"Authorization": f"Bearer {manifest_env['key']}"}, + ) + + assert response.status_code == 403 + + +def test_manifest_auth_missing(manifest_app, manifest_env): + response = manifest_app.test_client().get( + f"/journal/{manifest_env['key'][:8]}/manifest/segments" + ) + + assert response.status_code == 401 + + +def test_manifest_auth_invalid(manifest_app, manifest_env): + response = manifest_app.test_client().get( + f"/journal/{manifest_env['key'][:8]}/manifest/segments", + headers={"Authorization": "Bearer does-not-exist"}, + ) + + assert response.status_code == 401 + + +def test_manifest_auth_revoked(manifest_app, manifest_env): + source = manifest_env["source"] + source["revoked"] = True + source["revoked_at"] = now_ms() + assert save_journal_source(source) is True + + response = manifest_app.test_client().get( + f"/journal/{manifest_env['key'][:8]}/manifest/segments", + headers={"Authorization": f"Bearer {manifest_env['key']}"}, + ) + + assert response.status_code == 403