From f32e5f5ed4daa7d1fb22d42b87d8ffb335370237 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Fri, 24 Jul 2026 12:42:14 -0600 Subject: [PATCH] refactor(indexer): retire the Python command-write path Every write-bearing journal indexer invocation now runs solstone-core indexer; query, count, display, and interactive search stay Python. Delete the core.indexer / core.indexer_on_decline selector, the 69 decline-fallback, and the Python CLI write blocks rather than deprecating them; the command no longer reads either key. Remove reset_journal_index and its export. Write-bearing commands on hosts with no compatible solstone-core wheel, including musl Linux and Intel macOS, exit 78 before native execution, Python writes, or query. Make --rescan-file with --rescan/--rescan-full exit 64 before any side effect, replacing the old partial-side-effect ordering. Rename native_seam.py to native.py. Teach the indexer differential harness the native edge-extraction-skip stderr wording, mirroring the existing native markdown-sanitize rule, and point the fixture-corpus commands at the retired-selector-free journal indexer invocation. --- docs/PORTING.md | 105 +-- docs/deletion-sites-inventory.md | 6 - docs/design/indexer-native-atomicity.md | 15 +- docs/design/indexer-native-selection-seam.md | 6 + solstone/think/indexer/__init__.py | 2 - solstone/think/indexer/cli.py | 75 +- solstone/think/indexer/journal.py | 11 +- solstone/think/indexer/native.py | 253 ++++++ solstone/think/indexer/native_seam.py | 335 ------- tests/test_indexer_cli.py | 789 +++++++++-------- tests/test_indexer_differential.py | 42 +- tests/test_indexer_native.py | 372 ++++++++ tests/test_indexer_native_seam.py | 862 ------------------- tests/test_journal_index.py | 93 +- tests/verify_indexer_differential.py | 7 +- 15 files changed, 1229 insertions(+), 1744 deletions(-) create mode 100644 solstone/think/indexer/native.py delete mode 100644 solstone/think/indexer/native_seam.py create mode 100644 tests/test_indexer_native.py delete mode 100644 tests/test_indexer_native_seam.py diff --git a/docs/PORTING.md b/docs/PORTING.md index 319366776..7eb372f1a 100644 --- a/docs/PORTING.md +++ b/docs/PORTING.md @@ -83,7 +83,7 @@ pin, and rerun `make audit`. ## Owner Timezone -The Python owner-timezone fallback is effectively `identity.timezone` from +The Python owner-timezone resolution is effectively `identity.timezone` from `config/journal.json`, then UTC. The apparent host-local branches in `get_owner_timezone()` are dead because CPython `astimezone()` returns a fixed-offset `datetime.timezone` without a `.key`. Reproducing host-local time @@ -172,50 +172,39 @@ lode documents that hazard but does not change Python behavior. ## Unsupported Inputs -Ports use this vocabulary for unsupported behavior: - -- `on-unsupported = abort`: fail loudly; this is the default. -- `on-unsupported = abort-silent`: decline without noisy logging when a caller - has explicitly requested quiet refusal. -- `on-unsupported = fallback`: use the Python path while the port is active. - -The reserved declined process exit code is 69, matching the sysexits meaning -"service unavailable"; it means this port declines to handle the input. It is -distinct from existing success, usage, empty-input, and temporary-failure codes. - -The first declined-exit wave is the indexer selection seam in the Python -`journal indexer` wrapper. When the seam routes a write-only invocation to -`solstone-core indexer`, the native indexer returns 69 when it declines an -unsupported input. The wrapper handles that code according to -`config/journal.json` key `core.indexer_on_decline`: `abort` reports the decline -and exits 69, while `fallback` reruns the same operation on the Python indexer. -Usage errors (64) and temporary failures (75) are never retried in Python. -Signal death is normalized to temporary failure (75). The supervisor -intentionally keeps mapping non-zero scheduled-task exits to `error`; -abort-by-default makes that classification correct, and decline visibility lives -in the wrapper's stderr and logs. - -### Indexer Selection Seam - -`journal indexer` has a temporary Python/native selection seam for the native -indexer migration. It reads `config/journal.json` once at command launch. -Query-only and mixed write+query invocations stay on Python for the whole -invocation and do not read selection config. - -For write-only native-eligible invocations, explicit `core.indexer = "python"` -runs the Python indexer and remains the rollback switch. Explicit -`core.indexer = "rust"` selects the sibling `solstone-core indexer` binary -everywhere and keeps its loud handshake-failure behavior. When `core.indexer` is -unset, hosts covered by the probe module's solstone-core package predicate -default to Rust; uncovered hosts keep Python. - -When 69 fallback is selected, the wrapper reruns the full operation set in -Python; any native operations that succeeded before the decline are repeated. +Native ports reserve process exit code 69 for inputs the native command cannot +process. Wrappers should surface that code unless a command-specific design says +otherwise. It is distinct from success, usage errors (64), empty-input codes, and +temporary failures (75). Signal death is normalized to temporary failure (75). +The supervisor intentionally keeps mapping non-zero scheduled-task exits to +`error`; command stderr carries the operator-facing detail. + +### Indexer Native Write Routing + +`journal indexer` routes command writes (`--reset`, `--rebuild-edges`, +`--rescan`, `--rescan-full`, and `--rescan-file`) to the sibling +`solstone-core indexer` binary. Query-only invocations remain in Python. Mixed +write+query invocations run native writes first; on native success they enter +the Python query path, and on native non-zero they return that code without +querying. The command no longer reads journal config for write routing, so stale +old routing keys in `config/journal.json` are inert. + +Before launching the native helper, the wrapper checks that the current runtime +has a compatible `solstone-core` wheel: the normalized host tuple must be in +`probe.SOLSTONE_CORE_COVERED_PLATFORMS`, and the platform tags advertised by +`packaging.tags.sys_tags()` must intersect the tag set recorded in +`probe.SOLSTONE_CORE_PLATFORM_TAGS`. Linux x86_64 and Linux aarch64 require the +manylinux 2.17 / manylinux2014 glibc floor. macOS requires +`macosx_14_0_arm64`; older arm64 macOS hosts therefore report no compatible +wheel. A covered source checkout without `solstone-core` distribution metadata +also returns 78 for every write-bearing command until the developer runs +`make install`. + Backup-restore full rescans, direct `index_file()` callers, chat stream appends, importers, day-accumulator writes, and index-mutating deletes bypass -`journal indexer` and stay on the Python indexer during the dual window. +`journal indexer` and continue to use the named Python in-process writers. -The seam normalizes `--rescan-file` to an absolute path with the same Python +The wrapper normalizes `--rescan-file` to an absolute path with the same Python journal-path resolver used by `index_file()` before passing it to native. This keeps `chronicle/`-prefixed relative paths from being interpreted differently by the Rust relative-path resolver. @@ -230,12 +219,11 @@ and writes both watermarks as one unit. Reset is SQLite-native: it drops and recreates index objects transactionally and does not unlink the database, WAL, or SHM files. -During the Python/native dual window, journals containing edge source files -whose extraction fails can show differing `edge_files` rows between Python and -native. Python may delete prior edge rows and advance `edge_files` for the -failed source; native preserves the prior rows and mtime so the unchanged file -retries on the next scan. Journals with no failing edge sources must remain -byte-identical between Python and native index output. +Command writes now use the native path only. Journals containing edge source +files whose extraction fails preserve prior native `edge_files` rows and mtime +so the unchanged file retries on the next scan. The remaining Python in-process +bypass consumers keep their existing Python semantics because they do not enter +`journal indexer`. The detailed native atomicity design is in `docs/design/indexer-native-atomicity.md`. @@ -249,20 +237,11 @@ Native sol client design records: `docs/design/native-sol-client/05-raw-body-parity.md`, and `docs/design/native-sol-client/06-cutover-design.md`. -The selection keys are intentionally absent from `journal_default.json`. They -are a two-release-lifetime migration control: release N kept Python as the -absent-key default and allowed opt-in Rust; release N+1 defaults absent-key -native-eligible writes to Rust on covered hosts while still honoring explicit -Python and keeping uncovered hosts on Python; release N+2 may delete the Python -orchestration path and remove `core.indexer` / `core.indexer_on_decline` only -after a completed normal alpha interval and an explicit uncovered-host -disposition. - ## Dual Paths And Shims -The repository no-shims rule still stands. During an active port, a config-gated -old/new selection is a deliberate, time-boxed, per-change exception. Each dual -path needs a named deletion schedule. Do not add fallback aliases, +The repository no-shims rule still stands. During an active port, a temporary +old/new route is a deliberate, time-boxed, per-change exception. Each dual path +needs a named deletion schedule. Do not add compatibility aliases, deprecated-parameter handling, or compatibility re-exports. The native `sol` cutover has one sanctioned temporary delegation boundary: the @@ -287,17 +266,17 @@ The first behavior port is `get_journal_info()` / `get_journal()` from `solstone/think/utils.py`, backed by `solstone/think/user_config.py`. 1. **MSRV is 1.95 for the locked native dependency set.** Rust 1.87 is enough - for the safe home fallback, but the current bundled SQLite dependency line + for the safe home path, but the current bundled SQLite dependency line requires Rust 1.95. The journal resolver uses the hybrid shape: literal `HOME` when present, and `std::env::home_dir()` only when `HOME` is absent. This avoids a hand-rolled unsafe `getpwuid_r` implementation. 2. **No unsafe passwd FFI.** Keeping the old 1.85 floor would require libc - fallback code with buffer sizing and retry behavior for a home-directory + backup code with buffer sizing and retry behavior for a home-directory lookup. That defect surface is not justified for this port. 3. **Home normalization follows `str(Path.home() / "journal")`, not just `os.path.expanduser("~")`.** The port reproduces the observed layers needed by `user_config.default_journal()`: present-but-empty `HOME` becomes `/`, - trailing slashes are stripped with an or-root fallback, repeated separators + trailing slashes are stripped with an or-root default, repeated separators and `.` components are collapsed lexically, exactly two leading slashes are preserved, `..` is not collapsed, and `.` joined with `journal` renders as `journal`. If the expanded home still starts with `~`, the port raises the diff --git a/docs/deletion-sites-inventory.md b/docs/deletion-sites-inventory.md index 10e8cedd3..4abff96bc 100644 --- a/docs/deletion-sites-inventory.md +++ b/docs/deletion-sites-inventory.md @@ -70,12 +70,6 @@ Inventory of every non-test, non-scratch, non-atomic-tmp destructive removal (`s | --- | --- | --- | --- | --- | --- | --- | --- | | `solstone/think/facets.py:907` | `facets//` directory | `delete_facet()` | facet path resolves under `journal/facets`, with existing-facet checks before delete | yes (`solstone/think/facets.py:899-906`) | no | `⚠️` | audited write-owner delete path; deferred rather than expanded in this lode | -## think/indexer - -| file:line | target | trigger | path validation | audit log | dry-run | class | why | -| --- | --- | --- | --- | --- | --- | --- | --- | -| `solstone/think/indexer/journal.py:876` | SQLite index database file | `reset_journal_index()` | fixed `journal/indexer/` path | no | no | `⚠️` | index artifact reset; infrastructure is out of scope for retention-style parity | - ## think/identity | file:line | target | trigger | path validation | audit log | dry-run | class | why | diff --git a/docs/design/indexer-native-atomicity.md b/docs/design/indexer-native-atomicity.md index 26227b84d..208564372 100644 --- a/docs/design/indexer-native-atomicity.md +++ b/docs/design/indexer-native-atomicity.md @@ -348,9 +348,9 @@ Because D4 rejects the `hooks` feature, this design avoids a Cargo feature or lock-file change. If implementation unexpectedly changes Cargo metadata or `Cargo.lock`, escalate to full `make ci`. -## PORTING.md Divergence Text +## PORTING.md Native Routing Text -Add this text to `docs/PORTING.md` under `Indexer Selection Seam`: +Add this text to `docs/PORTING.md` under `Indexer Native Write Routing`: > Native indexer compound writes are atomic at the logical replacement-unit > boundary. A content file replacement deletes old chunks, inserts new chunks, @@ -362,12 +362,11 @@ Add this text to `docs/PORTING.md` under `Indexer Selection Seam`: > recreates index objects transactionally and does not unlink the database, WAL, > or SHM files. > -> During the Python/native dual window, journals containing edge source files -> whose extraction fails can show differing `edge_files` rows between Python and -> native. Python may delete prior edge rows and advance `edge_files` for the -> failed source; native preserves the prior rows and mtime so the unchanged file -> retries on the next scan. Journals with no failing edge sources must remain -> byte-identical between Python and native index output. +> Command writes now use the native path only. Journals containing edge source +> files whose extraction fails preserve prior native `edge_files` rows and mtime +> so the unchanged file retries on the next scan. The remaining Python +> in-process bypass consumers keep their existing Python semantics because they +> do not enter `journal indexer`. ## Risks And Open Questions diff --git a/docs/design/indexer-native-selection-seam.md b/docs/design/indexer-native-selection-seam.md index 1545a5ed2..c5ca84a89 100644 --- a/docs/design/indexer-native-selection-seam.md +++ b/docs/design/indexer-native-selection-seam.md @@ -1,5 +1,11 @@ # Config-Gated Native Indexer Selection Seam +> Retired historical design. This document is preserved only as context for the +> removed config-gated native indexer migration path. Active behavior is +> documented in `docs/PORTING.md` and implemented in +> `solstone/think/indexer/native.py`; do not follow the decisions below for new +> work. + This is the review-gate design for routing selected `journal indexer` write-only invocations to `solstone-core indexer`. It does not implement the seam. diff --git a/solstone/think/indexer/__init__.py b/solstone/think/indexer/__init__.py index 1c61de672..a433c69b4 100644 --- a/solstone/think/indexer/__init__.py +++ b/solstone/think/indexer/__init__.py @@ -14,7 +14,6 @@ from .journal import ( ScanReport, get_journal_index, index_file, - reset_journal_index, sanitize_fts_query, scan_journal, search_counts, @@ -28,7 +27,6 @@ __all__ = [ "ScanReport", "get_journal_index", "index_file", - "reset_journal_index", "sanitize_fts_query", "scan_journal", "search_counts", diff --git a/solstone/think/indexer/cli.py b/solstone/think/indexer/cli.py index ec5df57a5..5ae35d4a6 100644 --- a/solstone/think/indexer/cli.py +++ b/solstone/think/indexer/cli.py @@ -4,27 +4,20 @@ """CLI functionality for the indexer.""" import argparse -import logging +import sys from typing import Any from solstone.think.utils import get_journal, require_solstone, setup_cli -from .edges import rebuild_edges from .journal import ( - index_file, - reset_journal_index, - scan_journal, search_counts, search_journal, ) -from .native_seam import maybe_run_native_indexer +from .native import EXIT_USAGE, run_native_indexer -logger = logging.getLogger(__name__) - -ZERO_EDGE_HINT = ( - "Zero edges indexed: edges are talent-derived, and the --rescan-full edge phase " - "remains modification-time incremental — run journal indexer --rebuild-edges to " - "force full edge re-extraction." +INVALID_RESCAN_FILE_SCAN_MESSAGE = ( + "journal indexer usage error: --rescan-file cannot be combined with --rescan " + "or --rescan-full." ) @@ -89,6 +82,16 @@ def _display_search_results( print(f"{idx}. {meta.get('day')} {label}{facet_str}: {snippet}") +def _has_write_operation(args: argparse.Namespace) -> bool: + return bool( + args.reset + or args.rebuild_edges + or args.rescan + or args.rescan_full + or args.rescan_file + ) + + def main() -> int | None: """Main CLI entry point for the indexer.""" parser = argparse.ArgumentParser( @@ -171,8 +174,6 @@ def main() -> int | None: ) args = setup_cli(parser) - require_solstone() - journal = get_journal() if ( not args.rescan @@ -185,45 +186,17 @@ def main() -> int | None: parser.print_help() return - native_exit = maybe_run_native_indexer(args, journal) - if native_exit is not None: - return native_exit - - if args.reset: - reset_journal_index(journal) + if args.rescan_file and (args.rescan or args.rescan_full): + print(INVALID_RESCAN_FILE_SCAN_MESSAGE, file=sys.stderr) + return EXIT_USAGE - if args.rebuild_edges: - result = rebuild_edges(journal) - logger.info( - "indexer edges rebuilt: files=%s rows=%s drops=%s failed=%s skipped=%s", - result.get("files", 0), - result.get("rows", 0), - result.get("drops", 0), - result.get("failed", 0), - result.get("skipped", 0), - ) + require_solstone() + journal = get_journal() - if args.rescan_file: - # Single file indexing (incompatible with --rescan/--rescan-full) - if args.rescan or args.rescan_full: - parser.error("--rescan-file cannot be used with --rescan or --rescan-full") - try: - index_file(journal, args.rescan_file, verbose=args.verbose) - logger.info("indexer file indexed: %s", args.rescan_file) - except (ValueError, FileNotFoundError) as e: - parser.error(str(e)) - elif args.rescan or args.rescan_full: - report = scan_journal(journal, verbose=args.verbose, full=args.rescan_full) - if report.changed: - logger.info("indexer journal rescan ok") - should_emit_zero_edge_hint = ( - args.rescan_full - and not args.rebuild_edges - and not args.reset - and report.edge_rows_inserted == 0 - ) - if should_emit_zero_edge_hint: - print(ZERO_EDGE_HINT) + if _has_write_operation(args): + native_exit = run_native_indexer(args, journal) + if native_exit != 0 or args.query is None: + return native_exit if args.query is not None: query_kwargs: dict[str, Any] = {} diff --git a/solstone/think/indexer/journal.py b/solstone/think/indexer/journal.py index d4b9765c9..5fa77aa1d 100644 --- a/solstone/think/indexer/journal.py +++ b/solstone/think/indexer/journal.py @@ -155,15 +155,6 @@ def get_journal_index(journal: str | None = None) -> tuple[sqlite3.Connection, s return conn, db_path -def reset_journal_index(journal: str) -> None: - """Remove the journal index database file.""" - db_path = os.path.join(journal, INDEX_DIR, DB_NAME) - try: - os.unlink(db_path) - except FileNotFoundError: - pass - - def prune_chunks_by_stream(stream: str, journal: str | None = None) -> dict: """Remove all index chunks for a stream and their files rows. @@ -1032,7 +1023,7 @@ def sanitize_fts_query( quotes, apostrophes, and *. For plain multi-word queries (no explicit operators or quotes), produces - a NEAR-proximity formulation with AND fallback: + a NEAR-proximity formulation with an AND alternative: NEAR(term1 term2, 10) OR (term1 AND term2) Returns: diff --git a/solstone/think/indexer/native.py b/solstone/think/indexer/native.py new file mode 100644 index 000000000..d5420eebc --- /dev/null +++ b/solstone/think/indexer/native.py @@ -0,0 +1,253 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +"""Native journal-indexer command runner. + +The journal indexer CLI sends command write operations to `solstone-core +indexer`. Query and interactive search stay in Python, and in-process index +writers such as backup-restore rescans, direct `index_file()` callers, chat +stream appends, importers, day-accumulator writes, observer prune, share delete, +and entity-merge edge maintenance continue to call the Python indexer APIs +directly. +""" + +from __future__ import annotations + +import argparse +import logging +import subprocess +import sys +from collections.abc import Callable, Iterable +from pathlib import Path +from typing import Any + +from packaging import tags + +from solstone.think import core_handshake, probe +from solstone.think.utils import resolve_journal_path + +logger = logging.getLogger(__name__) + +EXIT_UNAVAILABLE = 69 +EXIT_USAGE = 64 +EXIT_TEMPFAIL = 75 + +UNSUPPORTED_HOST_MESSAGE = ( + "journal indexer requires a compatible solstone-core wheel for command " + "writes. Supported wheel platforms: Linux x86_64 glibc 2.17+ " + "(manylinux2014); Linux aarch64 glibc 2.17+ (manylinux2014); macOS 14.0+ " + "arm64. This host has no compatible solstone-core wheel. Use a supported " + "host or install a compatible solstone-core wheel, then rerun journal " + "indexer." +) +HANDSHAKE_SKIP_MESSAGE = ( + "journal indexer requires solstone-core for command writes, but " + "solstone-core distribution metadata is missing in this source checkout. " + "Run make install to restore the journal-host environment, then rerun " + "journal indexer." +) +HANDSHAKE_FAIL_MESSAGE = "journal indexer cannot run native command writes: {message}" +NATIVE_USAGE_MESSAGE = ( + "journal indexer usage error: native solstone-core indexer rejected the " + "command arguments with exit 64. This is a command argument-construction " + "bug; update solstone or report the full journal indexer command." +) +NATIVE_UNAVAILABLE_MESSAGE = ( + "journal indexer native command exited 69 (unsupported input). The Python " + "command-write path has been retired; remove or change the unsupported " + "input and rerun journal indexer." +) +NATIVE_TEMPFAIL_MESSAGE = ( + "journal indexer native command exited 75 (temporary failure). Fix the " + "reported cause and rerun journal indexer." +) +NATIVE_LAUNCH_FAILED_MESSAGE = ( + "journal indexer failed to launch solstone-core indexer: {error}. Run make " + "install in a source checkout or reinstall solstone-journal, then rerun " + "journal indexer." +) +NATIVE_SIGNAL_MESSAGE = ( + "journal indexer native command died from signal {signal_number} " + "(returncode {returncode}); treating as temporary failure. Fix the cause " + "and rerun journal indexer." +) +NATIVE_OTHER_NONZERO_MESSAGE = ( + "journal indexer native command exited {returncode}. Fix the reported " + "cause and rerun journal indexer." +) +EMPTY_TAIL_MESSAGE = ( + "journal indexer native command had no write operation flags to pass. This " + "is a command argument-construction bug." +) +COMPOSED_COMMAND_WARNING = ( + "journal indexer warning: this command combined multiple write operations " + "and the native command did not complete. Some earlier operations may " + "already have run. Fix the reported cause, then rerun the whole journal " + "indexer command." +) + +HandshakeChecker = Callable[[], core_handshake.CoreHandshakeResult] +HelperLocator = Callable[[], Path] +NativeRunner = Callable[..., subprocess.CompletedProcess[Any]] +PlatformReader = Callable[[], probe.CorePlatform] +PlatformTagReader = Callable[[], Iterable[str]] + + +def _packaging_platform_tags() -> set[str]: + return {tag.platform for tag in tags.sys_tags()} + + +def runtime_has_solstone_core_wheel_coverage( + *, + platform_reader: PlatformReader = probe.current_solstone_core_platform, + platform_tag_reader: PlatformTagReader = _packaging_platform_tags, +) -> bool: + """Return whether this runtime can install the packaged helper wheel.""" + platform_tuple = platform_reader() + if platform_tuple not in probe.SOLSTONE_CORE_COVERED_PLATFORMS: + return False + expected_platforms = probe.SOLSTONE_CORE_PLATFORM_TAGS.get(platform_tuple) + if expected_platforms is None: + return False + + expected = set(expected_platforms.split(".")) + actual = set(platform_tag_reader()) + return not expected.isdisjoint(actual) + + +def run_native_indexer( + args: argparse.Namespace, + journal: str, + *, + handshake_checker: HandshakeChecker = core_handshake.check_solstone_core_handshake, + helper_locator: HelperLocator = core_handshake.helper_path_for_executable, + native_runner: NativeRunner = subprocess.run, + platform_reader: PlatformReader = probe.current_solstone_core_platform, + platform_tag_reader: PlatformTagReader = _packaging_platform_tags, +) -> int: + """Run the native indexer command and return its process-style status code.""" + if not runtime_has_solstone_core_wheel_coverage( + platform_reader=platform_reader, + platform_tag_reader=platform_tag_reader, + ): + _emit_error(UNSUPPORTED_HOST_MESSAGE) + return core_handshake.EX_CONFIG + + handshake = handshake_checker() + if handshake.status == "skip": + _emit_error(HANDSHAKE_SKIP_MESSAGE) + return core_handshake.EX_CONFIG + if handshake.status == "fail": + _emit_error( + HANDSHAKE_FAIL_MESSAGE.format( + message=handshake.message or "unknown reason", + ) + ) + return core_handshake.EX_CONFIG + + operation_flags = _build_operation_flags(args, journal) + if not operation_flags: + raise RuntimeError(EMPTY_TAIL_MESSAGE) + + helper_path = helper_locator() + argv = [str(helper_path), "indexer", "--journal", journal, *operation_flags] + try: + completed = native_runner(argv, check=False) + except OSError as exc: + _emit_error( + NATIVE_LAUNCH_FAILED_MESSAGE.format( + error=exc, + ) + ) + return EXIT_TEMPFAIL + + return _map_native_returncode(completed.returncode, _operation_count(args)) + + +def _operation_count(args: argparse.Namespace) -> int: + count = 0 + if args.reset: + count += 1 + if args.rebuild_edges: + count += 1 + if args.rescan_file or args.rescan_full or args.rescan: + count += 1 + return count + + +def _build_operation_flags(args: argparse.Namespace, journal: str) -> list[str]: + flags: list[str] = [] + if args.reset: + flags.append("--reset") + if args.rebuild_edges: + flags.append("--rebuild-edges") + + if args.rescan_file: + flags.extend( + ["--rescan-file", _normalize_rescan_file(journal, args.rescan_file)] + ) + elif args.rescan_full: + flags.append("--rescan-full") + elif args.rescan: + flags.append("--rescan") + + return flags + + +def _normalize_rescan_file(journal: str, file_path: str) -> str: + journal_path = Path(journal).resolve() + path = Path(file_path) + if path.is_absolute(): + return str(path.resolve()) + return str(resolve_journal_path(journal_path, file_path).resolve()) + + +def _map_native_returncode( + returncode: int, + operation_count: int, +) -> int: + if returncode == 0: + return 0 + if returncode < 0: + _emit_error( + NATIVE_SIGNAL_MESSAGE.format( + signal_number=abs(returncode), + returncode=returncode, + ) + ) + _emit_composed_warning(operation_count) + return EXIT_TEMPFAIL + if returncode == EXIT_USAGE: + _emit_error(NATIVE_USAGE_MESSAGE) + return EXIT_USAGE + if returncode == EXIT_UNAVAILABLE: + _emit_error(NATIVE_UNAVAILABLE_MESSAGE) + _emit_composed_warning(operation_count) + return EXIT_UNAVAILABLE + if returncode == EXIT_TEMPFAIL: + _emit_error(NATIVE_TEMPFAIL_MESSAGE) + _emit_composed_warning(operation_count) + return EXIT_TEMPFAIL + + _emit_error( + NATIVE_OTHER_NONZERO_MESSAGE.format( + returncode=returncode, + ) + ) + _emit_composed_warning(operation_count) + return returncode + + +def _emit_composed_warning(operation_count: int) -> None: + if operation_count > 1: + _emit_warning(COMPOSED_COMMAND_WARNING) + + +def _emit_error(message: str) -> None: + print(message, file=sys.stderr) + logger.error(message) + + +def _emit_warning(message: str) -> None: + print(message, file=sys.stderr) + logger.warning(message) diff --git a/solstone/think/indexer/native_seam.py b/solstone/think/indexer/native_seam.py deleted file mode 100644 index ca2699262..000000000 --- a/solstone/think/indexer/native_seam.py +++ /dev/null @@ -1,335 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright (c) 2026 sol pbc - -"""Config-gated native-indexer selection for the journal indexer CLI. - -This module is the only command-level seam between the Python indexer and -`solstone-core indexer`. It is deliberately narrower than the full in-process -indexer API: backup-restore full rescans, per-file `index_file()` calls from -segment finish, chat stream appends, importers, and day-accumulator writes, plus -index-mutating deletes such as observer prune, share-delete, and entity-merge -edge folds, bypass `journal indexer` and stay on the Python indexer during the -dual window. - -Selection is read once from `config/journal.json` at command launch. Explicit -`python` stays on Python and explicit `rust` selects the native path. When -`core.indexer` is unset, write-only native-eligible invocations default to Rust -on hosts covered by the probe module's solstone-core package predicate and to -Python on uncovered hosts. Query and mixed write+query invocations stay in -Python. - -When 69 fallback is enabled, the command reruns the full operation set in -Python; any native operations that completed before the decline are repeated. -""" - -from __future__ import annotations - -import argparse -import logging -import subprocess -import sys -from collections.abc import Callable -from pathlib import Path -from typing import Any - -from solstone.think import core_handshake, probe -from solstone.think.journal_config import read_journal_config -from solstone.think.utils import resolve_journal_path - -logger = logging.getLogger(__name__) - -EXIT_UNAVAILABLE = 69 -EXIT_USAGE = 64 -EXIT_TEMPFAIL = 75 - -INVALID_INDEXER_MESSAGE = ( - "journal indexer selected implementation 'invalid' from config key " - "core.indexer; found {value!r}; expected 'python' or 'rust'. " - "Set core.indexer to 'python' to revert." -) -INVALID_CORE_SECTION_MESSAGE = ( - "journal indexer selected implementation 'invalid' from config key " - "core.indexer, but config section core has invalid value {value!r}; " - "expected an object. Set core.indexer to 'python' to revert." -) -INVALID_DECLINE_MESSAGE = ( - "{provenance}, but config key core.indexer_on_decline has invalid value " - "{value!r}; expected 'abort' or 'fallback'. Set core.indexer to 'python' to " - "revert." -) -HANDSHAKE_SKIP_MESSAGE = ( - "{provenance}, but solstone-core handshake returned 'skip': {message}. Set " - "core.indexer to 'python' to revert." -) -HANDSHAKE_FAIL_MESSAGE = ( - "{provenance}, but solstone-core handshake returned 'fail': {message}. Set " - "core.indexer to 'python' to revert." -) -NATIVE_DECLINE_ABORT_MESSAGE = ( - "{provenance}, but solstone-core indexer declined this input with exit 69. " - "Set core.indexer_on_decline to 'fallback' to retry unsupported inputs " - "through Python, or set core.indexer to 'python' to revert." -) -NATIVE_DECLINE_FALLBACK_MESSAGE = ( - "{provenance}, but solstone-core indexer declined this input with exit 69; " - "falling back to Python because core.indexer_on_decline is 'fallback'. Set " - "core.indexer to 'python' to revert." -) -NATIVE_USAGE_MESSAGE = ( - "{provenance}, but solstone-core indexer exited 64 (usage error). This is a " - "seam argument-construction bug; set core.indexer to 'python' to revert." -) -NATIVE_TEMPFAIL_MESSAGE = ( - "{provenance}, but solstone-core indexer exited 75 (temporary failure). Set " - "core.indexer to 'python' to revert." -) -NATIVE_LAUNCH_FAILED_MESSAGE = ( - "{provenance}, but launching solstone-core indexer failed: {error}. Set " - "core.indexer to 'python' to revert." -) -NATIVE_SIGNAL_MESSAGE = ( - "{provenance}, but solstone-core indexer died from signal {signal_number} " - "(returncode {returncode}); treating as temporary failure. Set core.indexer " - "to 'python' to revert." -) -NATIVE_OTHER_NONZERO_MESSAGE = ( - "{provenance}, but solstone-core indexer exited {returncode}. Set " - "core.indexer to 'python' to revert." -) -EMPTY_TAIL_MESSAGE = ( - "{provenance}, but found no native-supported operation flags to pass. This " - "is a seam bug; set core.indexer to 'python' to revert." -) - -ConfigReader = Callable[[str | Path | None], dict[str, Any]] -HandshakeChecker = Callable[[], core_handshake.CoreHandshakeResult] -HelperLocator = Callable[[], Path] -NativeRunner = Callable[..., subprocess.CompletedProcess[Any]] -CoverageChecker = Callable[[], bool] - - -def _platform_has_core_coverage() -> bool: - system, machine = probe.current_solstone_core_platform() - return probe.is_solstone_core_covered_platform(system, machine) - - -def _provenance_clause( - selected: str, - *, - explicit: bool, - covered: bool | None, -) -> str: - if explicit: - return ( - f"journal indexer selected implementation {selected!r} from config key " - "core.indexer" - ) - if covered: - return ( - f"journal indexer defaulted to implementation {selected!r} because " - "config key core.indexer is unset and solstone-core is packaged for " - "this platform" - ) - return ( - f"journal indexer defaulted to implementation {selected!r} because config " - "key core.indexer is unset and solstone-core is not packaged for this " - "platform" - ) - - -def maybe_run_native_indexer( - args: argparse.Namespace, - journal: str, - *, - config_reader: ConfigReader = read_journal_config, - handshake_checker: HandshakeChecker = core_handshake.check_solstone_core_handshake, - helper_locator: HelperLocator = core_handshake.helper_path_for_executable, - native_runner: NativeRunner = subprocess.run, - coverage_checker: CoverageChecker = _platform_has_core_coverage, -) -> int | None: - """Run native indexer when selected, else continue in Python.""" - if args.query is not None: - return None - - if not _has_write_operation(args): - return None - - if args.rescan_file and (args.rescan or args.rescan_full): - return None - - selected, decline_policy, provenance, error_message = _resolve_config( - config_reader(journal), - coverage_checker=coverage_checker, - ) - if error_message is not None: - _emit_error(error_message) - return core_handshake.EX_CONFIG - - if selected == "python": - return None - - handshake = handshake_checker() - if handshake.status == "skip": - _emit_error( - HANDSHAKE_SKIP_MESSAGE.format( - provenance=provenance, - message=handshake.message, - ) - ) - return core_handshake.EX_CONFIG - if handshake.status == "fail": - _emit_error( - HANDSHAKE_FAIL_MESSAGE.format( - provenance=provenance, - message=handshake.message, - ) - ) - return core_handshake.EX_CONFIG - - operation_flags = _build_operation_flags(args, journal) - if not operation_flags: - raise RuntimeError(EMPTY_TAIL_MESSAGE.format(provenance=provenance)) - - helper_path = helper_locator() - argv = [str(helper_path), "indexer", "--journal", journal, *operation_flags] - try: - completed = native_runner(argv, check=False) - except OSError as exc: - _emit_error( - NATIVE_LAUNCH_FAILED_MESSAGE.format( - provenance=provenance, - error=exc, - ) - ) - return EXIT_TEMPFAIL - - return _map_native_returncode(completed.returncode, decline_policy, provenance) - - -def _has_write_operation(args: argparse.Namespace) -> bool: - return bool( - args.reset - or args.rebuild_edges - or args.rescan - or args.rescan_full - or args.rescan_file - ) - - -def _resolve_config( - config: dict[str, Any], - *, - coverage_checker: CoverageChecker, -) -> tuple[str, str, str, str | None]: - core = config.get("core", {}) - if not isinstance(core, dict): - return ( - "invalid", - "abort", - "", - INVALID_CORE_SECTION_MESSAGE.format(value=core), - ) - - if "indexer" in core: - selected = core["indexer"] - if selected not in ("python", "rust"): - return ( - "invalid", - "abort", - "", - INVALID_INDEXER_MESSAGE.format(value=selected), - ) - provenance = _provenance_clause(selected, explicit=True, covered=None) - else: - covered = coverage_checker() - selected = "rust" if covered else "python" - provenance = _provenance_clause(selected, explicit=False, covered=covered) - - decline_policy = core.get("indexer_on_decline", "abort") - if decline_policy not in ("abort", "fallback"): - return ( - selected, - "abort", - provenance, - INVALID_DECLINE_MESSAGE.format( - provenance=provenance, - value=decline_policy, - ), - ) - - return selected, decline_policy, provenance, None - - -def _build_operation_flags(args: argparse.Namespace, journal: str) -> list[str]: - flags: list[str] = [] - if args.reset: - flags.append("--reset") - if args.rebuild_edges: - flags.append("--rebuild-edges") - - if args.rescan_file: - flags.extend( - ["--rescan-file", _normalize_rescan_file(journal, args.rescan_file)] - ) - elif args.rescan_full: - flags.append("--rescan-full") - elif args.rescan: - flags.append("--rescan") - - return flags - - -def _normalize_rescan_file(journal: str, file_path: str) -> str: - journal_path = Path(journal).resolve() - path = Path(file_path) - if path.is_absolute(): - return str(path.resolve()) - return str(resolve_journal_path(journal_path, file_path).resolve()) - - -def _map_native_returncode( - returncode: int, - decline_policy: str, - provenance: str, -) -> int | None: - if returncode == 0: - return 0 - if returncode < 0: - _emit_error( - NATIVE_SIGNAL_MESSAGE.format( - provenance=provenance, - signal_number=abs(returncode), - returncode=returncode, - ) - ) - return EXIT_TEMPFAIL - if returncode == EXIT_USAGE: - _emit_error(NATIVE_USAGE_MESSAGE.format(provenance=provenance)) - return EXIT_USAGE - if returncode == EXIT_UNAVAILABLE: - if decline_policy == "fallback": - _emit_warning(NATIVE_DECLINE_FALLBACK_MESSAGE.format(provenance=provenance)) - return None - _emit_error(NATIVE_DECLINE_ABORT_MESSAGE.format(provenance=provenance)) - return EXIT_UNAVAILABLE - if returncode == EXIT_TEMPFAIL: - _emit_error(NATIVE_TEMPFAIL_MESSAGE.format(provenance=provenance)) - return EXIT_TEMPFAIL - - _emit_error( - NATIVE_OTHER_NONZERO_MESSAGE.format( - provenance=provenance, - returncode=returncode, - ) - ) - return returncode - - -def _emit_error(message: str) -> None: - print(message, file=sys.stderr) - logger.error(message) - - -def _emit_warning(message: str) -> None: - print(message, file=sys.stderr) - logger.warning(message) diff --git a/tests/test_indexer_cli.py b/tests/test_indexer_cli.py index d6743e050..8e91a6cb6 100644 --- a/tests/test_indexer_cli.py +++ b/tests/test_indexer_cli.py @@ -5,24 +5,23 @@ from __future__ import annotations +import json import runpy import subprocess +import sys +from collections import Counter from pathlib import Path +from typing import Any import pytest +import solstone.think.journal_config as journal_config import solstone.think.utils as think_utils -from solstone.think import core_handshake +from solstone.think import core_handshake, sol_cli from solstone.think.indexer import cli as indexer_cli -from solstone.think.indexer import native_seam +from solstone.think.indexer import native as indexer_native from solstone.think.indexer.journal import ScanReport -EXPECTED_ZERO_EDGE_HINT = ( - "Zero edges indexed: edges are talent-derived, and the --rescan-full edge phase " - "remains modification-time incremental — run journal indexer --rebuild-edges to " - "force full edge re-extraction." -) - def test_module_entrypoint_propagates_main_return(monkeypatch): monkeypatch.setattr(indexer_cli, "main", lambda: 75) @@ -33,494 +32,520 @@ def test_module_entrypoint_propagates_main_return(monkeypatch): assert exc_info.value.code == 75 +def _setup_cli_for(args: list[str]): + def setup_cli(parser): + parsed = parser.parse_args(args) + if not hasattr(parsed, "verbose"): + parsed.verbose = False + if not hasattr(parsed, "debug"): + parsed.debug = False + return parsed + + return setup_cli + + def _run_indexer_cli( - monkeypatch, + monkeypatch: pytest.MonkeyPatch, journal: Path, args: list[str], - *, - core_indexer: str | None = None, -) -> None: +) -> int | None: monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) think_utils._journal_path_cache = None monkeypatch.setattr(indexer_cli, "get_journal", lambda: str(journal)) monkeypatch.setattr(indexer_cli, "require_solstone", lambda: None) - config_path = journal / "config" / "journal.json" - if core_indexer is not None and not config_path.exists(): - config_path.parent.mkdir(parents=True) - config_path.write_text( - f'{{"core": {{"indexer": "{core_indexer}"}}}}\n', - encoding="utf-8", - ) - - def setup_cli(parser): - parsed = parser.parse_args(args) - if not hasattr(parsed, "verbose"): - parsed.verbose = False - return parsed - - monkeypatch.setattr(indexer_cli, "setup_cli", setup_cli) - indexer_cli.main() + monkeypatch.setattr(indexer_cli, "setup_cli", _setup_cli_for(args)) + return indexer_cli.main() -def test_rescan_full_zero_edge_rows_prints_hint(tmp_path, monkeypatch, capsys): - journal = tmp_path / "journal" - journal.mkdir() - scans: list[tuple[str, bool, bool]] = [] +def _install_python_write_recorders(monkeypatch: pytest.MonkeyPatch) -> list[str]: + calls: list[str] = [] def scan_journal( - journal_arg: str, + _journal: str, *, verbose: bool = False, full: bool = False, ) -> ScanReport: - scans.append((journal_arg, verbose, full)) - return ScanReport(changed=True, edge_rows_inserted=0) + calls.append(f"scan_journal:{verbose}:{full}") + return ScanReport(changed=True, edge_rows_inserted=1) - monkeypatch.setattr(indexer_cli, "scan_journal", scan_journal) + def index_file(_journal: str, _path: str, *, verbose: bool = False) -> bool: + calls.append(f"index_file:{verbose}") + return True - _run_indexer_cli(monkeypatch, journal, ["--rescan-full"], core_indexer="python") + def rebuild_edges(_journal: str) -> dict[str, int]: + calls.append("rebuild_edges") + return {"files": 1, "rows": 1, "drops": 0, "failed": 0} + + monkeypatch.setattr(indexer_cli, "scan_journal", scan_journal, raising=False) + monkeypatch.setattr(indexer_cli, "index_file", index_file, raising=False) + monkeypatch.setattr(indexer_cli, "rebuild_edges", rebuild_edges, raising=False) + return calls + + +def _native_argv_from_args(args: Any, journal: str) -> list[str]: + flags: list[str] = [] + if args.reset: + flags.append("--reset") + if args.rebuild_edges: + flags.append("--rebuild-edges") + if args.rescan_file: + flags.extend(["--rescan-file", args.rescan_file]) + elif args.rescan_full: + flags.append("--rescan-full") + elif args.rescan: + flags.append("--rescan") + return ["/tmp/bin/solstone-core", "indexer", "--journal", journal, *flags] + + +def _install_native_stub( + monkeypatch: pytest.MonkeyPatch, + native_argvs: list[list[str]], + *, + returncode: int = 0, +) -> None: + def run_native_indexer(args: Any, journal: str) -> int: + native_argvs.append(_native_argv_from_args(args, journal)) + return returncode + + monkeypatch.setattr(indexer_cli, "run_native_indexer", run_native_indexer) + + +def _install_query_stubs(monkeypatch: pytest.MonkeyPatch) -> list[str]: + calls: list[str] = [] + + def search_counts(query: str, **_kwargs: Any) -> dict[str, Any]: + calls.append(f"counts:{query}") + return { + "total": 1, + "facets": Counter({"work": 1}), + "agents": Counter({"flow": 1}), + "days": Counter({"20260101": 1}), + } + + def search_journal( + query: str, + limit: int, + offset: int, + **_kwargs: Any, + ) -> tuple[int, list[dict[str, Any]]]: + calls.append(f"search:{query}:{limit}:{offset}") + return ( + 1, + [ + { + "metadata": { + "day": "20260101", + "agent": "flow", + "facet": "work", + }, + "text": f"found {query}", + } + ], + ) - captured = capsys.readouterr() - assert scans == [(str(journal), False, True)] - assert captured.out == EXPECTED_ZERO_EDGE_HINT + "\n" - assert captured.err == "" + monkeypatch.setattr(indexer_cli, "search_counts", search_counts) + monkeypatch.setattr(indexer_cli, "search_journal", search_journal) + return calls -def test_rescan_full_nonzero_edge_rows_suppresses_hint(tmp_path, monkeypatch, capsys): - journal = tmp_path / "journal" - journal.mkdir() +def _install_no_config_handshake_or_subprocess( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fail(name: str): + def raise_unexpected(*_args: Any, **_kwargs: Any): + raise AssertionError(f"{name} should not be called") - def scan_journal( - _journal_arg: str, - *, - verbose: bool = False, - full: bool = False, - ) -> ScanReport: - assert verbose is False - assert full is True - return ScanReport(changed=True, edge_rows_inserted=1) + return raise_unexpected - monkeypatch.setattr(indexer_cli, "scan_journal", scan_journal) + monkeypatch.setattr( + journal_config, + "read_journal_config", + fail("read_journal_config"), + ) + monkeypatch.setattr( + core_handshake, + "check_solstone_core_handshake", + fail("check_solstone_core_handshake"), + ) + monkeypatch.setattr(subprocess, "run", fail("subprocess.run")) + monkeypatch.setattr(indexer_cli, "run_native_indexer", fail("run_native_indexer")) + + +def test_help_no_work_uses_python_help_without_config_handshake_or_subprocess( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + journal = tmp_path / "journal" + journal.mkdir() + _install_no_config_handshake_or_subprocess(monkeypatch) - _run_indexer_cli(monkeypatch, journal, ["--rescan-full"], core_indexer="python") + result = _run_indexer_cli(monkeypatch, journal, []) captured = capsys.readouterr() - assert captured.out == "" + assert result is None + assert "usage:" in captured.out + assert "Index journal content" in captured.out assert captured.err == "" -def test_rebuild_edges_rescan_full_suppresses_zero_edge_hint( - tmp_path, - monkeypatch, - capsys, -): +def test_query_only_uses_python_search_without_config_handshake_or_subprocess( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: journal = tmp_path / "journal" journal.mkdir() - calls: list[tuple[str, str] | tuple[str, str, bool, bool]] = [] - - def rebuild_edges(journal_arg: str) -> dict[str, int]: - calls.append(("rebuild_edges", journal_arg)) - return {"files": 0, "rows": 0, "drops": 0, "failed": 0} - - def scan_journal( - journal_arg: str, - *, - verbose: bool = False, - full: bool = False, - ) -> ScanReport: - calls.append(("scan", journal_arg, verbose, full)) - return ScanReport(changed=True, edge_rows_inserted=0) + _install_no_config_handshake_or_subprocess(monkeypatch) + query_calls = _install_query_stubs(monkeypatch) - monkeypatch.setattr(indexer_cli, "rebuild_edges", rebuild_edges) - monkeypatch.setattr(indexer_cli, "scan_journal", scan_journal) - - _run_indexer_cli( - monkeypatch, - journal, - ["--rebuild-edges", "--rescan-full"], - core_indexer="python", - ) + result = _run_indexer_cli(monkeypatch, journal, ["-q", "needle"]) captured = capsys.readouterr() - assert calls == [ - ("rebuild_edges", str(journal)), - ("scan", str(journal), False, True), - ] - assert captured.out == "" + assert result is None + assert query_calls == ["counts:needle", "search:needle:10:0"] + assert "Total: 1 chunks" in captured.out + assert "found needle" in captured.out assert captured.err == "" -def test_reset_rescan_full_suppresses_zero_edge_hint(tmp_path, monkeypatch, capsys): +def test_interactive_query_uses_python_search_without_config_handshake_or_subprocess( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: journal = tmp_path / "journal" journal.mkdir() - calls: list[tuple[str, str] | tuple[str, str, bool, bool]] = [] + _install_no_config_handshake_or_subprocess(monkeypatch) + query_calls = _install_query_stubs(monkeypatch) + inputs = iter(["needle", ""]) + monkeypatch.setattr("builtins.input", lambda _prompt: next(inputs)) - def reset_journal_index(journal_arg: str) -> None: - calls.append(("reset", journal_arg)) - - def scan_journal( - journal_arg: str, - *, - verbose: bool = False, - full: bool = False, - ) -> ScanReport: - calls.append(("scan", journal_arg, verbose, full)) - return ScanReport(changed=True, edge_rows_inserted=0) - - monkeypatch.setattr(indexer_cli, "reset_journal_index", reset_journal_index) - monkeypatch.setattr(indexer_cli, "scan_journal", scan_journal) - - _run_indexer_cli( - monkeypatch, - journal, - ["--reset", "--rescan-full"], - core_indexer="python", - ) + result = _run_indexer_cli(monkeypatch, journal, ["-q"]) captured = capsys.readouterr() - assert calls == [ - ("reset", str(journal)), - ("scan", str(journal), False, True), - ] - assert captured.out == "" + assert result is None + assert query_calls == ["counts:needle", "search:needle:10:0"] + assert "Total: 1 chunks" in captured.out + assert "found needle" in captured.out assert captured.err == "" -def test_rescan_file_does_not_create_root_task_log(tmp_path, monkeypatch): +def test_write_only_routes_to_native_without_python_writes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: journal = tmp_path / "journal" journal.mkdir() - indexed: list[tuple[str, str, bool]] = [] + native_argvs: list[list[str]] = [] + python_write_calls = _install_python_write_recorders(monkeypatch) + _install_native_stub(monkeypatch, native_argvs) - def index_file(journal_arg: str, path_arg: str, *, verbose: bool = False) -> bool: - indexed.append((journal_arg, path_arg, verbose)) - return True + result = _run_indexer_cli(monkeypatch, journal, ["--rescan"]) - monkeypatch.setattr(indexer_cli, "index_file", index_file) + assert result == 0 + assert native_argvs == [ + ["/tmp/bin/solstone-core", "indexer", "--journal", str(journal), "--rescan"] + ] + assert python_write_calls == [] + + +def test_mixed_write_query_runs_query_after_native_success( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + journal = tmp_path / "journal" + journal.mkdir() + native_argvs: list[list[str]] = [] + python_write_calls = _install_python_write_recorders(monkeypatch) + _install_native_stub(monkeypatch, native_argvs) + query_calls = _install_query_stubs(monkeypatch) - _run_indexer_cli( + result = _run_indexer_cli( monkeypatch, journal, - ["--rescan-file", "chronicle/today.md"], - core_indexer="python", + ["--rescan", "--query", "needle"], ) - assert indexed == [(str(journal), "chronicle/today.md", False)] - assert not (journal / "task_log.txt").exists() + captured = capsys.readouterr() + assert result is None + assert native_argvs == [ + ["/tmp/bin/solstone-core", "indexer", "--journal", str(journal), "--rescan"] + ] + assert python_write_calls == [] + assert query_calls == ["counts:needle", "search:needle:10:0"] + assert "Total: 1 chunks" in captured.out + assert "found needle" in captured.out -def test_rebuild_edges_does_not_create_root_task_log(tmp_path, monkeypatch): +def test_mixed_empty_query_enters_interactive_after_native_success( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: journal = tmp_path / "journal" journal.mkdir() - rebuilt: list[str] = [] - - def rebuild_edges(journal_arg: str) -> dict[str, int]: - rebuilt.append(journal_arg) - return {"files": 1, "rows": 2, "drops": 0, "failed": 0} - - monkeypatch.setattr(indexer_cli, "rebuild_edges", rebuild_edges) + native_argvs: list[list[str]] = [] + python_write_calls = _install_python_write_recorders(monkeypatch) + _install_native_stub(monkeypatch, native_argvs) + query_calls = _install_query_stubs(monkeypatch) + inputs = iter(["needle", ""]) + monkeypatch.setattr("builtins.input", lambda _prompt: next(inputs)) - _run_indexer_cli(monkeypatch, journal, ["--rebuild-edges"], core_indexer="python") + result = _run_indexer_cli(monkeypatch, journal, ["--rescan", "-q"]) - assert rebuilt == [str(journal)] - assert not (journal / "task_log.txt").exists() + captured = capsys.readouterr() + assert result is None + assert native_argvs == [ + ["/tmp/bin/solstone-core", "indexer", "--journal", str(journal), "--rescan"] + ] + assert python_write_calls == [] + assert query_calls == ["counts:needle", "search:needle:10:0"] + assert "found needle" in captured.out -@pytest.mark.parametrize("core_indexer", [None, "python", "rust"]) -def test_rescan_file_with_rescan_errors_after_reset_and_rebuild( - tmp_path, - monkeypatch, - capsys, - core_indexer, -): +def test_mixed_write_query_native_failure_skips_query( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: journal = tmp_path / "journal" - calls: list[tuple[str, str]] = [] + journal.mkdir() + native_argvs: list[list[str]] = [] + python_write_calls = _install_python_write_recorders(monkeypatch) + _install_native_stub(monkeypatch, native_argvs, returncode=75) + query_calls = _install_query_stubs(monkeypatch) - def reset_journal_index(journal_arg: str) -> None: - calls.append(("reset", journal_arg)) + result = _run_indexer_cli( + monkeypatch, + journal, + ["--rescan", "--query", "needle"], + ) + + captured = capsys.readouterr() + assert result == 75 + assert native_argvs == [ + ["/tmp/bin/solstone-core", "indexer", "--journal", str(journal), "--rescan"] + ] + assert python_write_calls == [] + assert query_calls == [] + assert captured.out == "" - def rebuild_edges(journal_arg: str) -> dict[str, int]: - calls.append(("rebuild_edges", journal_arg)) - return {"files": 1, "rows": 2, "drops": 0, "failed": 0} - monkeypatch.setattr(indexer_cli, "reset_journal_index", reset_journal_index) - monkeypatch.setattr(indexer_cli, "rebuild_edges", rebuild_edges) +def test_rescan_file_with_scan_exits_64_before_side_effects( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + python_write_calls = _install_python_write_recorders(monkeypatch) + query_calls = _install_query_stubs(monkeypatch) monkeypatch.setattr( indexer_cli, - "index_file", - lambda *args, **kwargs: pytest.fail("index_file should not run"), + "setup_cli", + _setup_cli_for( + ["--rescan", "--rescan-file", "chronicle/today.md", "--query", "needle"] + ), ) monkeypatch.setattr( indexer_cli, - "scan_journal", - lambda *args, **kwargs: pytest.fail("scan_journal should not run"), + "require_solstone", + lambda: pytest.fail("require_solstone should not run"), ) - - with pytest.raises(SystemExit) as exc_info: - _run_indexer_cli( - monkeypatch, - journal, - [ - "--reset", - "--rebuild-edges", - "--rescan", - "--rescan-file", - "chronicle/today.md", - ], - core_indexer=core_indexer, - ) - - assert exc_info.value.code == 2 - assert calls == [("reset", str(journal)), ("rebuild_edges", str(journal))] - assert ( - "--rescan-file cannot be used with --rescan or --rescan-full" - in capsys.readouterr().err + monkeypatch.setattr( + indexer_cli, + "get_journal", + lambda: pytest.fail("get_journal should not run"), + ) + monkeypatch.setattr( + indexer_cli, + "run_native_indexer", + lambda *_args, **_kwargs: pytest.fail("native should not run"), ) + result = indexer_cli.main() -def _raise_unexpected(name: str): - def fail(*_args, **_kwargs): - raise AssertionError(f"{name} should not be called") - - return fail + captured = capsys.readouterr() + assert result == 64 + assert python_write_calls == [] + assert query_calls == [] + assert captured.err.strip() == ( + "journal indexer usage error: --rescan-file cannot be combined with " + "--rescan or --rescan-full." + ) -def test_cli_covered_absent_indexer_routes_write_to_native( - tmp_path, - monkeypatch, -): +@pytest.mark.parametrize( + "args", + [ + ["--rescan"], + ["--rescan", "--query", "needle"], + ], +) +def test_stale_core_indexer_keys_are_inert( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + args: list[str], +) -> None: journal = tmp_path / "journal" journal.mkdir() - python_calls: list[str] = [] native_argvs: list[list[str]] = [] + _install_python_write_recorders(monkeypatch) + _install_native_stub(monkeypatch, native_argvs) + _install_query_stubs(monkeypatch) - def scan_journal( - journal_arg: str, - *, - verbose: bool = False, - full: bool = False, - ) -> ScanReport: - python_calls.append(journal_arg) - return ScanReport(changed=True, edge_rows_inserted=1) + def fail_read_journal_config(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + raise AssertionError("read_journal_config should not be touched") - def seam(args, journal_arg: str) -> int | None: - return native_seam.maybe_run_native_indexer( - args, - journal_arg, - coverage_checker=lambda: True, - handshake_checker=lambda: core_handshake.CoreHandshakeResult("ok"), - helper_locator=lambda: Path("/tmp/bin/solstone-core"), - native_runner=lambda argv, *, check=False: ( - native_argvs.append(argv) or subprocess.CompletedProcess(argv, 0) - ), - ) - - monkeypatch.setattr(indexer_cli, "maybe_run_native_indexer", seam) - monkeypatch.setattr(indexer_cli, "scan_journal", scan_journal) - - _run_indexer_cli(monkeypatch, journal, ["--rescan"], core_indexer=None) - - assert native_argvs == [ - ["/tmp/bin/solstone-core", "indexer", "--journal", str(journal), "--rescan"] - ] - assert python_calls == [] - - -def test_cli_uncovered_absent_indexer_routes_write_to_python_without_native_boundaries( - tmp_path, - monkeypatch, -): - journal = tmp_path / "journal" - journal.mkdir() - python_calls: list[tuple[str, bool, bool]] = [] - native_argvs: list[list[str]] = [] + monkeypatch.setattr( + journal_config, + "read_journal_config", + fail_read_journal_config, + ) - def scan_journal( - journal_arg: str, - *, - verbose: bool = False, - full: bool = False, - ) -> ScanReport: - python_calls.append((journal_arg, verbose, full)) - return ScanReport(changed=True, edge_rows_inserted=1) + first_result = _run_indexer_cli(monkeypatch, journal, args) + first_output = capsys.readouterr() - def seam(args, journal_arg: str) -> int | None: - return native_seam.maybe_run_native_indexer( - args, - journal_arg, - coverage_checker=lambda: False, - handshake_checker=_raise_unexpected("handshake_checker"), - helper_locator=_raise_unexpected("helper_locator"), - native_runner=_raise_unexpected("native_runner"), + config_path = journal / "config" / "journal.json" + config_path.parent.mkdir(parents=True) + config_path.write_text( + json.dumps( + { + "core": { + "indexer": "python", + "indexer_on_decline": "fallback", + } + } ) + + "\n", + encoding="utf-8", + ) - monkeypatch.setattr(indexer_cli, "maybe_run_native_indexer", seam) - monkeypatch.setattr(indexer_cli, "scan_journal", scan_journal) - - _run_indexer_cli(monkeypatch, journal, ["--rescan"], core_indexer=None) + second_result = _run_indexer_cli(monkeypatch, journal, args) + second_output = capsys.readouterr() - assert python_calls == [(str(journal), False, False)] - assert native_argvs == [] + assert first_result == second_result + assert native_argvs[0] == native_argvs[1] + assert first_output.out == second_output.out + assert first_output.err == second_output.err -def test_cli_explicit_python_routes_write_to_python_without_coverage_or_native( - tmp_path, - monkeypatch, -): +@pytest.mark.parametrize( + "args", + [ + ["--rescan"], + ["--rescan-full"], + ["--rebuild-edges"], + ["--rescan-file", "chronicle/today.md"], + ], +) +def test_rescan_leaves_existing_root_task_log_unchanged( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + args: list[str], +) -> None: journal = tmp_path / "journal" journal.mkdir() - python_calls: list[tuple[str, bool, bool]] = [] + root_log = journal / "task_log.txt" + original = b"123\tkeep\n" + root_log.write_bytes(original) native_argvs: list[list[str]] = [] + python_write_calls = _install_python_write_recorders(monkeypatch) + _install_native_stub(monkeypatch, native_argvs) - def scan_journal( - journal_arg: str, - *, - verbose: bool = False, - full: bool = False, - ) -> ScanReport: - python_calls.append((journal_arg, verbose, full)) - return ScanReport(changed=True, edge_rows_inserted=1) - - def seam(args, journal_arg: str) -> int | None: - return native_seam.maybe_run_native_indexer( - args, - journal_arg, - coverage_checker=_raise_unexpected("coverage_checker"), - handshake_checker=_raise_unexpected("handshake_checker"), - helper_locator=_raise_unexpected("helper_locator"), - native_runner=_raise_unexpected("native_runner"), - ) - - monkeypatch.setattr(indexer_cli, "maybe_run_native_indexer", seam) - monkeypatch.setattr(indexer_cli, "scan_journal", scan_journal) - - _run_indexer_cli(monkeypatch, journal, ["--rescan"], core_indexer="python") + result = _run_indexer_cli(monkeypatch, journal, args) - assert python_calls == [(str(journal), False, False)] - assert native_argvs == [] + assert result == 0 + assert native_argvs == [ + ["/tmp/bin/solstone-core", "indexer", "--journal", str(journal), *args] + ] + assert python_write_calls == [] + assert root_log.read_bytes() == original -def test_cli_explicit_rust_routes_write_to_native_without_coverage( - tmp_path, - monkeypatch, +def _dispatcher_native_impl( + *, native_returncode: int | None = None, launch_error=False ): - journal = tmp_path / "journal" - journal.mkdir() - python_calls: list[str] = [] - native_argvs: list[list[str]] = [] - - def scan_journal( - journal_arg: str, - *, - verbose: bool = False, - full: bool = False, - ) -> ScanReport: - python_calls.append(journal_arg) - return ScanReport(changed=True, edge_rows_inserted=1) - - def seam(args, journal_arg: str) -> int | None: - return native_seam.maybe_run_native_indexer( + def run_native_indexer(args: Any, journal: str) -> int: + def native_runner(argv: list[str], *, check: bool = False): + assert check is False + if launch_error: + raise OSError("missing helper") + return subprocess.CompletedProcess(argv, native_returncode) + + return indexer_native.run_native_indexer( args, - journal_arg, - coverage_checker=_raise_unexpected("coverage_checker"), + journal, handshake_checker=lambda: core_handshake.CoreHandshakeResult("ok"), helper_locator=lambda: Path("/tmp/bin/solstone-core"), - native_runner=lambda argv, *, check=False: ( - native_argvs.append(argv) or subprocess.CompletedProcess(argv, 0) - ), + native_runner=native_runner, + platform_reader=lambda: ("linux", "x86_64"), + platform_tag_reader=lambda: {"manylinux2014_x86_64"}, ) - monkeypatch.setattr(indexer_cli, "maybe_run_native_indexer", seam) - monkeypatch.setattr(indexer_cli, "scan_journal", scan_journal) + return run_native_indexer - _run_indexer_cli(monkeypatch, journal, ["--rescan"], core_indexer="rust") - assert native_argvs == [ - ["/tmp/bin/solstone-core", "indexer", "--journal", str(journal), "--rescan"] - ] - assert python_calls == [] +def _run_dispatcher( + monkeypatch: pytest.MonkeyPatch, + journal: Path, + args: list[str], + run_native_indexer, +) -> int: + monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) + think_utils._journal_path_cache = None + monkeypatch.setattr(indexer_cli, "get_journal", lambda: str(journal)) + monkeypatch.setattr(indexer_cli, "require_solstone", lambda: None) + monkeypatch.setattr(indexer_cli, "setup_cli", _setup_cli_for(args)) + monkeypatch.setattr(indexer_cli, "run_native_indexer", run_native_indexer) + monkeypatch.setattr(sys, "argv", ["journal", "indexer", *args]) + return sol_cli.run_command("solstone.think.indexer") -def test_native_decline_fallback_runs_python_write_blocks(tmp_path, monkeypatch): - journal = tmp_path / "journal" +@pytest.mark.parametrize( + ("case", "run_native_indexer", "expected"), + [ + ("success", _dispatcher_native_impl(native_returncode=0), 0), + ("native_69", _dispatcher_native_impl(native_returncode=69), 69), + ("native_tempfail", _dispatcher_native_impl(native_returncode=75), 75), + ("native_signal", _dispatcher_native_impl(native_returncode=-9), 75), + ("native_launch", _dispatcher_native_impl(launch_error=True), 75), + ("native_other", _dispatcher_native_impl(native_returncode=12), 12), + ], +) +def test_dispatcher_exit_contract_for_native_results( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + case: str, + run_native_indexer, + expected: int, +) -> None: + journal = tmp_path / case journal.mkdir() - seam_calls: list[tuple[str, bool, bool, bool]] = [] - calls: list[tuple[str, str] | tuple[str, str, bool, bool]] = [] - - def fallback_after_native_decline(args, journal_arg: str) -> None: - seam_calls.append( - (journal_arg, args.reset, args.rebuild_edges, args.rescan_full) - ) - return None - - def reset_journal_index(journal_arg: str) -> None: - calls.append(("reset", journal_arg)) - def rebuild_edges(journal_arg: str) -> dict[str, int]: - calls.append(("rebuild_edges", journal_arg)) - return {"files": 1, "rows": 2, "drops": 0, "failed": 0} - - def scan_journal( - journal_arg: str, - *, - verbose: bool = False, - full: bool = False, - ) -> ScanReport: - calls.append(("scan", journal_arg, verbose, full)) - return ScanReport(changed=True, edge_rows_inserted=0) - - monkeypatch.setattr( - indexer_cli, "maybe_run_native_indexer", fallback_after_native_decline - ) - monkeypatch.setattr(indexer_cli, "reset_journal_index", reset_journal_index) - monkeypatch.setattr(indexer_cli, "rebuild_edges", rebuild_edges) - monkeypatch.setattr(indexer_cli, "scan_journal", scan_journal) - - _run_indexer_cli( + result = _run_dispatcher( monkeypatch, journal, - ["--reset", "--rebuild-edges", "--rescan-full"], + ["--rescan"], + run_native_indexer, ) - assert seam_calls == [(str(journal), True, True, True)] - assert calls == [ - ("reset", str(journal)), - ("rebuild_edges", str(journal)), - ("scan", str(journal), False, True), - ] + assert result == expected -@pytest.mark.parametrize( - ("args", "full"), - [ - (["--rescan"], False), - (["--rescan-full"], True), - ], -) -def test_rescan_leaves_existing_root_task_log_unchanged( - tmp_path, - monkeypatch, - args, - full, -): +def test_dispatcher_exit_contract_for_usage_64( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: journal = tmp_path / "journal" journal.mkdir() - root_log = journal / "task_log.txt" - original = b"123\tkeep\n" - root_log.write_bytes(original) - scans: list[tuple[str, bool, bool]] = [] - - def scan_journal( - journal_arg: str, - *, - verbose: bool = False, - full: bool = False, - ) -> ScanReport: - scans.append((journal_arg, verbose, full)) - return ScanReport(changed=True, edge_rows_inserted=1) - - monkeypatch.setattr(indexer_cli, "scan_journal", scan_journal) - _run_indexer_cli(monkeypatch, journal, args, core_indexer="python") + result = _run_dispatcher( + monkeypatch, + journal, + ["--rescan", "--rescan-file", "chronicle/today.md"], + lambda *_args, **_kwargs: pytest.fail("native should not run"), + ) - assert scans == [(str(journal), False, full)] - assert root_log.read_bytes() == original + assert result == 64 diff --git a/tests/test_indexer_differential.py b/tests/test_indexer_differential.py index 3f3f71a14..828b2fdda 100644 --- a/tests/test_indexer_differential.py +++ b/tests/test_indexer_differential.py @@ -112,21 +112,14 @@ def _command(tmp_path: Path, mode: str) -> str: return _quote_command(sys.executable, _writer_script(tmp_path), mode) -def _python_journal_indexer_command(tmp_path: Path) -> str: - script = tmp_path / "run_python_journal_indexer.py" +def _journal_indexer_command(tmp_path: Path) -> str: + script = tmp_path / "run_journal_indexer.py" script.write_text( """ -import json -import os import subprocess import sys from pathlib import Path -journal = Path(os.environ["SOLSTONE_JOURNAL"]) -config_path = journal / "config" / "journal.json" -config = json.loads(config_path.read_text(encoding="utf-8")) -config.setdefault("core", {})["indexer"] = "python" -config_path.write_text(json.dumps(config, indent=2, sort_keys=True) + "\\n", encoding="utf-8") journal_bin = Path(sys.executable).with_name("journal") completed = subprocess.run([str(journal_bin), "indexer", "--rescan-full"], check=False) raise SystemExit(completed.returncode) @@ -413,6 +406,27 @@ def test_stderr_classifier_allows_native_markdown_sanitize_warning() -> None: assert classified["unclassified"] == [] +def test_stderr_classifier_allows_native_edge_skip_warning() -> None: + stderr = "\n".join( + [ + f"{harness.NATIVE_EDGE_SKIP_PREFIX}" + "20240102/default/234567_300/screen.jsonl" + ": invalid segment key 234567_300", + "unexpected diagnostic", + ] + ) + + classified = harness.classify_stderr(stderr) + native_edge_rule = next( + rule + for rule in classified["rules"] + if rule["name"] == harness.NATIVE_EDGE_SKIP_RULE + ) + + assert native_edge_rule["count"] == 1 + assert classified["unclassified"] == ["unexpected diagnostic"] + + def test_stderr_classifier_rejects_markdown_near_misses() -> None: stderr = "\n".join( [ @@ -660,7 +674,7 @@ def test_command_failure_is_distinct_and_cli_nonzero(tmp_path: Path, capsys) -> def test_fixture_corpus_reports_equal_with_visible_edge_skips(tmp_path: Path) -> None: - command = _python_journal_indexer_command(tmp_path) + command = _journal_indexer_command(tmp_path) report = harness.run_differential( journal=FIXTURE_JOURNAL, @@ -674,7 +688,11 @@ def test_fixture_corpus_reports_equal_with_visible_edge_skips(tmp_path: Path) -> for table in report["canonical"]["tables"] } skip_counts = [ - command_report["stderr_classification"]["rules"][0]["count"] + next( + rule["count"] + for rule in command_report["stderr_classification"]["rules"] + if rule["name"] == harness.NATIVE_EDGE_SKIP_RULE + ) for command_report in report["commands"] ] corpus = report["provenance"]["corpus"] @@ -865,7 +883,7 @@ def test_functional_fixture_cases_are_non_empty_on_reference_index( def test_fixture_corpus_reports_functionally_equal(tmp_path: Path) -> None: - command = _python_journal_indexer_command(tmp_path) + command = _journal_indexer_command(tmp_path) report = harness.run_differential( journal=FIXTURE_JOURNAL, diff --git a/tests/test_indexer_native.py b/tests/test_indexer_native.py new file mode 100644 index 000000000..e41ac970c --- /dev/null +++ b/tests/test_indexer_native.py @@ -0,0 +1,372 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright (c) 2026 sol pbc + +from __future__ import annotations + +import argparse +import re +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from solstone.think import core_handshake +from solstone.think.indexer import native + + +def _args(**overrides: Any) -> argparse.Namespace: + values: dict[str, Any] = { + "rescan": False, + "rescan_full": False, + "rescan_file": None, + "rebuild_edges": False, + "reset": False, + "day": None, + "day_from": None, + "day_to": None, + "facet": None, + "agent": None, + "stream": None, + "query": None, + "limit": 10, + "offset": 0, + "top": 5, + "verbose": False, + "debug": False, + } + values.update(overrides) + return argparse.Namespace(**values) + + +def _ok() -> core_handshake.CoreHandshakeResult: + return core_handshake.CoreHandshakeResult("ok") + + +def _raise_unexpected(name: str): + def fail(*_args, **_kwargs): + raise AssertionError(f"{name} should not be called") + + return fail + + +def _native_argv(*flags: str, journal: str = "/tmp/journal") -> list[str]: + return ["/tmp/bin/solstone-core", "indexer", "--journal", journal, *flags] + + +def _route( + args: argparse.Namespace, + *, + native_returncode: int = 0, + native_runner=None, + handshake_checker=None, + helper_path: Path | None = None, + journal: str = "/tmp/journal", + platform_tuple: tuple[str, str] = ("linux", "x86_64"), + platform_tags: set[str] | None = None, +) -> tuple[int, list[list[str]]]: + native_argvs: list[list[str]] = [] + + def default_native_runner(argv: list[str], *, check: bool = False): + assert check is False + native_argvs.append(argv) + return subprocess.CompletedProcess(argv, native_returncode) + + result = native.run_native_indexer( + args, + journal, + handshake_checker=handshake_checker or _ok, + helper_locator=lambda: helper_path or Path("/tmp/bin/solstone-core"), + native_runner=native_runner or default_native_runner, + platform_reader=lambda: platform_tuple, + platform_tag_reader=lambda: platform_tags or {"manylinux2014_x86_64"}, + ) + return result, native_argvs + + +@pytest.mark.parametrize( + ("platform_tuple", "platform_tags", "expected"), + [ + (("linux", "x86_64"), {"manylinux2014_x86_64"}, True), + (("linux", "x86_64"), {"musllinux_1_2_x86_64"}, False), + (("linux", "aarch64"), {"musllinux_1_2_aarch64"}, False), + (("darwin", "x86_64"), {"macosx_14_0_x86_64"}, False), + (("linux", "ppc64le"), {"manylinux_2_17_ppc64le"}, False), + ], +) +def test_runtime_coverage_uses_platform_tuple_and_packaging_tags( + platform_tuple: tuple[str, str], + platform_tags: set[str], + expected: bool, +) -> None: + assert ( + native.runtime_has_solstone_core_wheel_coverage( + platform_reader=lambda: platform_tuple, + platform_tag_reader=lambda: platform_tags, + ) + is expected + ) + + +def test_runtime_coverage_splits_compressed_linux_tag_set() -> None: + assert native.runtime_has_solstone_core_wheel_coverage( + platform_reader=lambda: ("linux", "x86_64"), + platform_tag_reader=lambda: {"manylinux_2_17_x86_64"}, + ) + + +def test_uncovered_runtime_returns_78_without_handshake( + capsys: pytest.CaptureFixture[str], +) -> None: + result, native_argvs = _route( + _args(rescan=True), + platform_tags={"musllinux_1_2_x86_64"}, + handshake_checker=_raise_unexpected("handshake_checker"), + ) + + assert result == core_handshake.EX_CONFIG + assert native_argvs == [] + assert "This host has no compatible solstone-core wheel" in capsys.readouterr().err + + +def test_handshake_skip_reports_source_checkout_guidance( + capsys: pytest.CaptureFixture[str], +) -> None: + result, native_argvs = _route( + _args(rescan=True), + handshake_checker=lambda: core_handshake.CoreHandshakeResult("skip", "reason"), + ) + + assert result == core_handshake.EX_CONFIG + assert native_argvs == [] + assert capsys.readouterr().err.strip() == ( + "journal indexer requires solstone-core for command writes, but " + "solstone-core distribution metadata is missing in this source checkout. " + "Run make install to restore the journal-host environment, then rerun " + "journal indexer." + ) + + +def test_handshake_fail_passes_through_context( + capsys: pytest.CaptureFixture[str], +) -> None: + result, native_argvs = _route( + _args(rescan=True), + handshake_checker=lambda: core_handshake.CoreHandshakeResult("fail", "reason"), + ) + + assert result == core_handshake.EX_CONFIG + assert native_argvs == [] + assert capsys.readouterr().err.strip() == ( + "journal indexer cannot run native command writes: reason" + ) + + +@pytest.mark.parametrize( + ("overrides", "expected_flags"), + [ + ({"reset": True}, ["--reset"]), + ({"rebuild_edges": True}, ["--rebuild-edges"]), + ({"rescan": True}, ["--rescan"]), + ({"rescan_full": True}, ["--rescan-full"]), + ( + {"rescan_file": "chronicle/today.md"}, + ["--rescan-file", str(Path("/tmp/journal/chronicle/today.md").resolve())], + ), + ], +) +def test_write_operations_invoke_native_with_expected_flags( + overrides: dict[str, Any], + expected_flags: list[str], +) -> None: + result, native_argvs = _route(_args(**overrides)) + + assert result == 0 + assert native_argvs == [_native_argv(*expected_flags)] + + +@pytest.mark.parametrize( + ("overrides", "expected_flags"), + [ + ( + {"reset": True, "rebuild_edges": True, "rescan_full": True}, + ["--reset", "--rebuild-edges", "--rescan-full"], + ), + ( + {"rebuild_edges": True, "rescan_full": True}, + ["--rebuild-edges", "--rescan-full"], + ), + ({"reset": True, "rescan_full": True}, ["--reset", "--rescan-full"]), + ({"rescan": True, "rescan_full": True}, ["--rescan-full"]), + ], +) +def test_composed_write_order( + overrides: dict[str, Any], + expected_flags: list[str], +) -> None: + result, native_argvs = _route(_args(**overrides)) + + assert result == 0 + assert native_argvs == [_native_argv(*expected_flags)] + + +def test_native_tail_drops_verbose_debug_and_query_filters() -> None: + result, native_argvs = _route( + _args( + rescan=True, + verbose=True, + debug=True, + day="20260101", + facet="work", + agent="flow", + stream="archon", + limit=99, + offset=4, + top=7, + query="needle", + ) + ) + + assert result == 0 + assert native_argvs == [_native_argv("--rescan")] + + +def test_rescan_file_normalizes_chronicle_prefixed_relative_to_absolute( + tmp_path: Path, +) -> None: + journal = tmp_path / "journal" + rel = "chronicle/20240101/talents/flow.md" + (journal / "chronicle" / "20240101" / "talents").mkdir(parents=True) + + result, native_argvs = _route( + _args(rescan_file=rel), + journal=str(journal), + ) + + assert result == 0 + assert native_argvs == [ + [ + "/tmp/bin/solstone-core", + "indexer", + "--journal", + str(journal), + "--rescan-file", + str((journal / rel).resolve()), + ] + ] + + +def test_empty_tail_raises_runtime_error(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(native, "_build_operation_flags", lambda _args, _journal: []) + + with pytest.raises(RuntimeError, match=re.escape(native.EMPTY_TAIL_MESSAGE)): + _route( + _args(rescan=True), + native_runner=_raise_unexpected("native_runner"), + ) + + +@pytest.mark.parametrize( + ("returncode", "expected_result", "expected_error"), + [ + ( + 64, + 64, + "journal indexer usage error: native solstone-core indexer rejected " + "the command arguments with exit 64. This is a command " + "argument-construction bug; update solstone or report the full " + "journal indexer command.", + ), + ( + 69, + 69, + "journal indexer native command exited 69 (unsupported input). The " + "Python command-write path has been retired; remove or change the " + "unsupported input and rerun journal indexer.", + ), + ( + 75, + 75, + "journal indexer native command exited 75 (temporary failure). Fix " + "the reported cause and rerun journal indexer.", + ), + ( + -9, + 75, + "journal indexer native command died from signal 9 (returncode -9); " + "treating as temporary failure. Fix the cause and rerun journal " + "indexer.", + ), + ( + 12, + 12, + "journal indexer native command exited 12. Fix the reported cause " + "and rerun journal indexer.", + ), + ], +) +def test_native_exit_mappings( + returncode: int, + expected_result: int, + expected_error: str, + capsys: pytest.CaptureFixture[str], +) -> None: + result, native_argvs = _route( + _args(rescan=True), + native_returncode=returncode, + ) + + assert result == expected_result + assert native_argvs == [_native_argv("--rescan")] + assert capsys.readouterr().err.strip() == expected_error + + +def test_native_launch_oserror_maps_to_tempfail_without_composed_warning( + capsys: pytest.CaptureFixture[str], +) -> None: + def native_runner(_argv: list[str], *, check: bool = False): + assert check is False + raise OSError("missing helper") + + result, native_argvs = _route( + _args(reset=True, rebuild_edges=True, rescan=True), + native_runner=native_runner, + ) + + assert result == 75 + assert native_argvs == [] + err = capsys.readouterr().err.strip() + assert err == ( + "journal indexer failed to launch solstone-core indexer: missing helper. " + "Run make install in a source checkout or reinstall solstone-journal, " + "then rerun journal indexer." + ) + assert native.COMPOSED_COMMAND_WARNING not in err + + +@pytest.mark.parametrize("returncode", [69, 75, -9, 12]) +def test_composed_nonzero_after_process_start_warns( + returncode: int, + capsys: pytest.CaptureFixture[str], +) -> None: + result, native_argvs = _route( + _args(reset=True, rebuild_edges=True, rescan=True), + native_returncode=returncode, + ) + + assert result == (75 if returncode < 0 else returncode) + assert native_argvs == [_native_argv("--reset", "--rebuild-edges", "--rescan")] + assert native.COMPOSED_COMMAND_WARNING in capsys.readouterr().err + + +def test_composed_native_usage_64_does_not_warn( + capsys: pytest.CaptureFixture[str], +) -> None: + result, native_argvs = _route( + _args(reset=True, rebuild_edges=True, rescan=True), + native_returncode=64, + ) + + assert result == 64 + assert native_argvs == [_native_argv("--reset", "--rebuild-edges", "--rescan")] + assert native.COMPOSED_COMMAND_WARNING not in capsys.readouterr().err diff --git a/tests/test_indexer_native_seam.py b/tests/test_indexer_native_seam.py deleted file mode 100644 index 49340b984..000000000 --- a/tests/test_indexer_native_seam.py +++ /dev/null @@ -1,862 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright (c) 2026 sol pbc - -from __future__ import annotations - -import argparse -import re -import subprocess -import sys -from pathlib import Path -from typing import Any - -import pytest - -from solstone.think import core_handshake, probe, sol_cli -from solstone.think.indexer import cli as indexer_cli -from solstone.think.indexer import native_seam - - -def _args(**overrides: Any) -> argparse.Namespace: - values: dict[str, Any] = { - "rescan": False, - "rescan_full": False, - "rescan_file": None, - "rebuild_edges": False, - "reset": False, - "day": None, - "day_from": None, - "day_to": None, - "facet": None, - "agent": None, - "stream": None, - "query": None, - "limit": 10, - "offset": 0, - "top": 5, - "verbose": False, - "debug": False, - } - values.update(overrides) - return argparse.Namespace(**values) - - -def _ok() -> core_handshake.CoreHandshakeResult: - return core_handshake.CoreHandshakeResult("ok") - - -COVERED_PROVENANCE = ( - "journal indexer defaulted to implementation 'rust' because config key " - "core.indexer is unset and solstone-core is packaged for this platform" -) -UNCOVERED_PROVENANCE = ( - "journal indexer defaulted to implementation 'python' because config key " - "core.indexer is unset and solstone-core is not packaged for this platform" -) -EXPLICIT_RUST_PROVENANCE = ( - "journal indexer selected implementation 'rust' from config key core.indexer" -) - - -def _route( - args: argparse.Namespace, - *, - config: dict[str, Any] | None = None, - native_returncode: int = 0, - config_reader=None, - handshake_checker=None, - coverage_checker=lambda: False, - helper_path: Path | None = None, - journal: str = "/tmp/journal", -) -> tuple[int | None, list[list[str]]]: - native_argvs: list[list[str]] = [] - - def default_config_reader(_journal: str) -> dict[str, Any]: - return config if config is not None else {} - - def native_runner(argv: list[str], *, check: bool = False): - assert check is False - native_argvs.append(argv) - return subprocess.CompletedProcess(argv, native_returncode) - - result = native_seam.maybe_run_native_indexer( - args, - journal, - config_reader=config_reader or default_config_reader, - handshake_checker=handshake_checker or _ok, - helper_locator=lambda: helper_path or Path("/tmp/bin/solstone-core"), - native_runner=native_runner, - coverage_checker=coverage_checker, - ) - return result, native_argvs - - -def _raise_unexpected(name: str): - def fail(*_args, **_kwargs): - raise AssertionError(f"{name} should not be called") - - return fail - - -def _native_argv(*flags: str, journal: str = "/tmp/journal") -> list[str]: - return ["/tmp/bin/solstone-core", "indexer", "--journal", journal, *flags] - - -@pytest.mark.parametrize( - ("platform_tuple", "expected"), - [ - (probe.SOLSTONE_CORE_COVERED_PLATFORMS[0], True), - (("linux", "riscv64"), False), - ], -) -def test_platform_has_core_coverage_resolves_through_probe_predicate( - platform_tuple: tuple[str, str], - expected: bool, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(probe, "current_solstone_core_platform", lambda: platform_tuple) - - assert native_seam._platform_has_core_coverage() is expected - - -@pytest.mark.parametrize( - ("platform_tuple", "expected_native"), - [ - (probe.SOLSTONE_CORE_COVERED_PLATFORMS[0], True), - (("linux", "riscv64"), False), - ], -) -def test_absent_indexer_default_coverage_checker_uses_probe( - platform_tuple: tuple[str, str], - expected_native: bool, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(probe, "current_solstone_core_platform", lambda: platform_tuple) - native_argvs: list[list[str]] = [] - - if expected_native: - result = native_seam.maybe_run_native_indexer( - _args(rescan=True), - "/tmp/journal", - config_reader=lambda _journal: {}, - handshake_checker=_ok, - helper_locator=lambda: Path("/tmp/bin/solstone-core"), - native_runner=lambda argv, *, check=False: ( - native_argvs.append(argv) or subprocess.CompletedProcess(argv, 0) - ), - ) - - assert result == 0 - assert native_argvs == [_native_argv("--rescan")] - return - - result = native_seam.maybe_run_native_indexer( - _args(rescan=True), - "/tmp/journal", - config_reader=lambda _journal: {}, - handshake_checker=_raise_unexpected("handshake_checker"), - helper_locator=_raise_unexpected("helper_locator"), - native_runner=_raise_unexpected("native_runner"), - ) - - assert result is None - assert native_argvs == [] - - -@pytest.mark.parametrize("config", [{}, {"core": {}}]) -@pytest.mark.parametrize( - ("overrides", "expected_flags"), - [ - ({"reset": True}, ["--reset"]), - ({"rebuild_edges": True}, ["--rebuild-edges"]), - ({"rescan": True}, ["--rescan"]), - ({"rescan_full": True}, ["--rescan-full"]), - ( - {"rescan_file": "chronicle/today.md"}, - ["--rescan-file", str(Path("/tmp/journal/chronicle/today.md").resolve())], - ), - ], -) -def test_absent_indexer_covered_host_write_operations_select_native( - config: dict[str, Any], - overrides: dict[str, Any], - expected_flags: list[str], -) -> None: - result, native_argvs = _route( - _args(**overrides), - config=config, - coverage_checker=lambda: True, - ) - - assert result == 0 - assert native_argvs == [_native_argv(*expected_flags)] - - -@pytest.mark.parametrize("config", [{}, {"core": {}}]) -@pytest.mark.parametrize( - ("overrides", "expected_flags"), - [ - ( - {"reset": True, "rebuild_edges": True, "rescan_full": True}, - ["--reset", "--rebuild-edges", "--rescan-full"], - ), - ( - {"rebuild_edges": True, "rescan_full": True}, - ["--rebuild-edges", "--rescan-full"], - ), - ({"reset": True, "rescan_full": True}, ["--reset", "--rescan-full"]), - ({"rescan": True, "rescan_full": True}, ["--rescan-full"]), - ], -) -def test_absent_indexer_covered_host_compositions_select_native( - config: dict[str, Any], - overrides: dict[str, Any], - expected_flags: list[str], -) -> None: - result, native_argvs = _route( - _args(**overrides), - config=config, - coverage_checker=lambda: True, - ) - - assert result == 0 - assert native_argvs == [_native_argv(*expected_flags)] - - -@pytest.mark.parametrize("config", [{}, {"core": {}}]) -def test_absent_indexer_uncovered_host_runs_python_without_native_boundaries( - config: dict[str, Any], -) -> None: - result = native_seam.maybe_run_native_indexer( - _args(rescan=True), - "/tmp/journal", - config_reader=lambda _journal: config, - coverage_checker=lambda: False, - handshake_checker=_raise_unexpected("handshake_checker"), - helper_locator=_raise_unexpected("helper_locator"), - native_runner=_raise_unexpected("native_runner"), - ) - - assert result is None - - -def test_explicit_python_runs_python() -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer": "python"}}, - coverage_checker=_raise_unexpected("coverage_checker"), - ) - - assert result is None - assert native_argvs == [] - - -def test_rust_rescan_invokes_native_with_explicit_journal() -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer": "rust"}}, - journal="/var/journal", - helper_path=Path("/venv/bin/solstone-core"), - ) - - assert result == 0 - assert native_argvs == [ - ["/venv/bin/solstone-core", "indexer", "--journal", "/var/journal", "--rescan"] - ] - - -def test_rust_tail_drops_verbose_debug_and_query_filters() -> None: - result, native_argvs = _route( - _args( - rescan=True, - verbose=True, - debug=True, - day="20260101", - facet="work", - agent="flow", - stream="archon", - limit=99, - offset=4, - top=7, - ), - config={"core": {"indexer": "rust"}}, - ) - - assert result == 0 - assert native_argvs == [ - ["/tmp/bin/solstone-core", "indexer", "--journal", "/tmp/journal", "--rescan"] - ] - - -def test_rust_composed_write_order() -> None: - result, native_argvs = _route( - _args(reset=True, rebuild_edges=True, rescan_full=True), - config={"core": {"indexer": "rust"}}, - ) - - assert result == 0 - assert native_argvs == [ - [ - "/tmp/bin/solstone-core", - "indexer", - "--journal", - "/tmp/journal", - "--reset", - "--rebuild-edges", - "--rescan-full", - ] - ] - - -def test_rust_prefers_rescan_full_when_both_scan_flags_are_set() -> None: - result, native_argvs = _route( - _args(rescan=True, rescan_full=True), - config={"core": {"indexer": "rust"}}, - ) - - assert result == 0 - assert native_argvs == [ - [ - "/tmp/bin/solstone-core", - "indexer", - "--journal", - "/tmp/journal", - "--rescan-full", - ] - ] - - -@pytest.mark.parametrize( - ("overrides", "expected_flags"), - [ - ({"rescan_full": True}, ["--rescan-full"]), - ({"rescan": True}, ["--rescan"]), - ( - {"rebuild_edges": True, "rescan_full": True}, - ["--rebuild-edges", "--rescan-full"], - ), - ({"reset": True, "rescan_full": True}, ["--reset", "--rescan-full"]), - ], -) -def test_rust_zero_edge_hint_suppression_options_are_forwarded_in_native_argv( - overrides: dict[str, Any], - expected_flags: list[str], -) -> None: - # The seam inherits native stdio, so direct Rust tests assert hint output. - result, native_argvs = _route( - _args(**overrides), - config={"core": {"indexer": "rust"}}, - ) - - assert result == 0 - assert native_argvs == [ - [ - "/tmp/bin/solstone-core", - "indexer", - "--journal", - "/tmp/journal", - *expected_flags, - ] - ] - - -def test_rust_rescan_file_normalizes_chronicle_prefixed_relative_to_absolute( - tmp_path: Path, -) -> None: - journal = tmp_path / "journal" - rel = "chronicle/20240101/talents/flow.md" - (journal / "chronicle" / "20240101" / "talents").mkdir(parents=True) - - result, native_argvs = _route( - _args(rescan_file=rel), - config={"core": {"indexer": "rust"}}, - journal=str(journal), - ) - - assert result == 0 - assert native_argvs == [ - [ - "/tmp/bin/solstone-core", - "indexer", - "--journal", - str(journal), - "--rescan-file", - str((journal / rel).resolve()), - ] - ] - - -def test_rust_rescan_file_with_rescan_stays_python() -> None: - result, native_argvs = _route( - _args( - reset=True, - rebuild_edges=True, - rescan_file="20240101/talents/flow.md", - rescan=True, - ), - config_reader=_raise_unexpected("config_reader"), - ) - - assert result is None - assert native_argvs == [] - - -def test_query_only_rust_selection_stays_python_without_reading_config() -> None: - result, native_argvs = _route( - _args(query="foo"), - config_reader=_raise_unexpected("config_reader"), - ) - - assert result is None - assert native_argvs == [] - - -def test_mixed_write_query_rust_selection_stays_python_without_reading_config() -> None: - result, native_argvs = _route( - _args(rescan=True, query="foo"), - config_reader=_raise_unexpected("config_reader"), - ) - - assert result is None - assert native_argvs == [] - - -def test_invalid_indexer_value_returns_ex_config( - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer": "go"}}, - ) - - assert result == core_handshake.EX_CONFIG - assert native_argvs == [] - assert capsys.readouterr().err.strip() == ( - "journal indexer selected implementation 'invalid' from config key " - "core.indexer; found 'go'; expected 'python' or 'rust'. Set core.indexer " - "to 'python' to revert." - ) - - -def test_invalid_indexer_on_decline_value_returns_ex_config( - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer": "python", "indexer_on_decline": "retry"}}, - ) - - assert result == core_handshake.EX_CONFIG - assert native_argvs == [] - assert capsys.readouterr().err.strip() == ( - "journal indexer selected implementation 'python' from config key " - "core.indexer, but config key core.indexer_on_decline has invalid value " - "'retry'; expected 'abort' or 'fallback'. Set core.indexer to 'python' " - "to revert." - ) - - -@pytest.mark.parametrize("core_value", [[], "yes"]) -def test_non_object_core_returns_ex_config( - core_value: object, - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": core_value}, - ) - - assert result == core_handshake.EX_CONFIG - assert native_argvs == [] - assert capsys.readouterr().err.strip() == ( - "journal indexer selected implementation 'invalid' from config key " - f"core.indexer, but config section core has invalid value {core_value!r}; " - "expected an object. Set core.indexer to 'python' to revert." - ) - - -def test_explicit_rust_ignores_coverage_and_handshake_skip_aborts( - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer": "rust"}}, - handshake_checker=lambda: core_handshake.CoreHandshakeResult("skip", "reason"), - coverage_checker=_raise_unexpected("coverage_checker"), - ) - - assert result == core_handshake.EX_CONFIG - assert native_argvs == [] - assert capsys.readouterr().err.strip() == ( - "journal indexer selected implementation 'rust' from config key " - "core.indexer, but solstone-core handshake returned 'skip': reason. " - "Set core.indexer to 'python' to revert." - ) - - -def test_explicit_rust_ignores_coverage_and_handshake_fail_aborts( - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer": "rust"}}, - handshake_checker=lambda: core_handshake.CoreHandshakeResult("fail", "reason"), - coverage_checker=_raise_unexpected("coverage_checker"), - ) - - assert result == core_handshake.EX_CONFIG - assert native_argvs == [] - assert capsys.readouterr().err.strip() == ( - "journal indexer selected implementation 'rust' from config key " - "core.indexer, but solstone-core handshake returned 'fail': reason. " - "Set core.indexer to 'python' to revert." - ) - - -@pytest.mark.parametrize( - ("status", "expected"), - [ - ( - "skip", - f"{COVERED_PROVENANCE}, but solstone-core handshake returned " - "'skip': reason. Set core.indexer to 'python' to revert.", - ), - ( - "fail", - f"{COVERED_PROVENANCE}, but solstone-core handshake returned " - "'fail': reason. Set core.indexer to 'python' to revert.", - ), - ], -) -def test_absent_indexer_covered_host_handshake_abort_messages( - status: core_handshake.HandshakeStatus, - expected: str, - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={}, - coverage_checker=lambda: True, - handshake_checker=lambda: core_handshake.CoreHandshakeResult(status, "reason"), - ) - - assert result == core_handshake.EX_CONFIG - assert native_argvs == [] - assert capsys.readouterr().err.strip() == expected - - -def test_native_decline_abort_returns_69_without_python( - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer": "rust"}}, - native_returncode=69, - ) - - assert result == 69 - assert native_argvs != [] - assert capsys.readouterr().err.strip() == ( - "journal indexer selected implementation 'rust' from config key " - "core.indexer, but solstone-core indexer declined this input with exit " - "69. Set core.indexer_on_decline to 'fallback' to retry unsupported " - "inputs through Python, or set core.indexer to 'python' to revert." - ) - - -def test_native_decline_fallback_continues_to_python( - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer": "rust", "indexer_on_decline": "fallback"}}, - native_returncode=69, - ) - - assert result is None - assert native_argvs != [] - assert capsys.readouterr().err.strip() == ( - "journal indexer selected implementation 'rust' from config key " - "core.indexer, but solstone-core indexer declined this input with exit " - "69; falling back to Python because core.indexer_on_decline is " - "'fallback'. Set core.indexer to 'python' to revert." - ) - - -def test_absent_indexer_covered_host_decline_abort_uses_default_policy( - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={}, - coverage_checker=lambda: True, - native_returncode=69, - ) - - assert result == 69 - assert len(native_argvs) == 1 - assert capsys.readouterr().err.strip() == ( - f"{COVERED_PROVENANCE}, but solstone-core indexer declined this input " - "with exit 69. Set core.indexer_on_decline to 'fallback' to retry " - "unsupported inputs through Python, or set core.indexer to 'python' to " - "revert." - ) - - -def test_absent_indexer_covered_host_decline_fallback_warns_once( - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer_on_decline": "fallback"}}, - coverage_checker=lambda: True, - native_returncode=69, - ) - - assert result is None - assert len(native_argvs) == 1 - assert capsys.readouterr().err.strip() == ( - f"{COVERED_PROVENANCE}, but solstone-core indexer declined this input " - "with exit 69; falling back to Python because core.indexer_on_decline is " - "'fallback'. Set core.indexer to 'python' to revert." - ) - - -def test_native_usage_error_64_never_fallbacks( - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer": "rust", "indexer_on_decline": "fallback"}}, - native_returncode=64, - ) - - assert result == 64 - assert native_argvs != [] - assert capsys.readouterr().err.strip() == ( - "journal indexer selected implementation 'rust' from config key " - "core.indexer, but solstone-core indexer exited 64 (usage error). This " - "is a seam argument-construction bug; set core.indexer to 'python' to " - "revert." - ) - - -def test_native_tempfail_75_never_fallbacks( - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer": "rust", "indexer_on_decline": "fallback"}}, - native_returncode=75, - ) - - assert result == 75 - assert native_argvs != [] - assert capsys.readouterr().err.strip() == ( - "journal indexer selected implementation 'rust' from config key " - "core.indexer, but solstone-core indexer exited 75 (temporary failure). " - "Set core.indexer to 'python' to revert." - ) - - -def test_native_signal_death_maps_to_tempfail( - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer": "rust", "indexer_on_decline": "fallback"}}, - native_returncode=-9, - ) - - assert result == 75 - assert native_argvs != [] - assert capsys.readouterr().err.strip() == ( - "journal indexer selected implementation 'rust' from config key " - "core.indexer, but solstone-core indexer died from signal 9 (returncode " - "-9); treating as temporary failure. Set core.indexer to 'python' to " - "revert." - ) - - -@pytest.mark.parametrize( - ("returncode", "expected_result", "expected_error"), - [ - ( - 64, - 64, - f"{COVERED_PROVENANCE}, but solstone-core indexer exited 64 " - "(usage error). This is a seam argument-construction bug; set " - "core.indexer to 'python' to revert.", - ), - ( - 75, - 75, - f"{COVERED_PROVENANCE}, but solstone-core indexer exited 75 " - "(temporary failure). Set core.indexer to 'python' to revert.", - ), - ( - -9, - 75, - f"{COVERED_PROVENANCE}, but solstone-core indexer died from signal " - "9 (returncode -9); treating as temporary failure. Set core.indexer " - "to 'python' to revert.", - ), - ], -) -def test_absent_indexer_covered_host_native_exit_mappings( - returncode: int, - expected_result: int, - expected_error: str, - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={}, - coverage_checker=lambda: True, - native_returncode=returncode, - ) - - assert result == expected_result - assert native_argvs != [] - assert capsys.readouterr().err.strip() == expected_error - - -def test_native_other_nonzero_returns_code(capsys: pytest.CaptureFixture[str]) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer": "rust"}}, - native_returncode=12, - ) - - assert result == 12 - assert native_argvs != [] - assert capsys.readouterr().err.strip() == ( - "journal indexer selected implementation 'rust' from config key " - "core.indexer, but solstone-core indexer exited 12. Set core.indexer to " - "'python' to revert." - ) - - -def test_native_launch_oserror_maps_to_tempfail( - capsys: pytest.CaptureFixture[str], -) -> None: - def native_runner(_argv: list[str], *, check: bool = False): - assert check is False - raise OSError("missing helper") - - result = native_seam.maybe_run_native_indexer( - _args(rescan=True), - "/tmp/journal", - config_reader=lambda _journal: {"core": {"indexer": "rust"}}, - handshake_checker=_ok, - helper_locator=lambda: Path("/tmp/bin/solstone-core"), - native_runner=native_runner, - ) - - assert result == 75 - assert capsys.readouterr().err.strip() == ( - "journal indexer selected implementation 'rust' from config key " - "core.indexer, but launching solstone-core indexer failed: missing " - "helper. Set core.indexer to 'python' to revert." - ) - - -def test_absent_indexer_covered_host_launch_oserror_maps_to_tempfail( - capsys: pytest.CaptureFixture[str], -) -> None: - def native_runner(_argv: list[str], *, check: bool = False): - assert check is False - raise OSError("missing helper") - - result = native_seam.maybe_run_native_indexer( - _args(rescan=True), - "/tmp/journal", - config_reader=lambda _journal: {}, - coverage_checker=lambda: True, - handshake_checker=_ok, - helper_locator=lambda: Path("/tmp/bin/solstone-core"), - native_runner=native_runner, - ) - - assert result == 75 - assert capsys.readouterr().err.strip() == ( - f"{COVERED_PROVENANCE}, but launching solstone-core indexer failed: " - "missing helper. Set core.indexer to 'python' to revert." - ) - - -@pytest.mark.parametrize( - ("covered", "expected_provenance"), - [(True, COVERED_PROVENANCE), (False, UNCOVERED_PROVENANCE)], -) -def test_absent_indexer_invalid_decline_renders_host_provenance( - covered: bool, - expected_provenance: str, - capsys: pytest.CaptureFixture[str], -) -> None: - result, native_argvs = _route( - _args(rescan=True), - config={"core": {"indexer_on_decline": "retry"}}, - coverage_checker=lambda: covered, - ) - - assert result == core_handshake.EX_CONFIG - assert native_argvs == [] - assert capsys.readouterr().err.strip() == ( - f"{expected_provenance}, but config key core.indexer_on_decline has " - "invalid value 'retry'; expected 'abort' or 'fallback'. Set " - "core.indexer to 'python' to revert." - ) - - -def test_empty_tail_raises_runtime_error() -> None: - with pytest.MonkeyPatch.context() as monkeypatch: - monkeypatch.setattr(native_seam, "_has_write_operation", lambda _args: True) - monkeypatch.setattr( - native_seam, "_build_operation_flags", lambda _args, _journal: [] - ) - expected = ( - f"{EXPLICIT_RUST_PROVENANCE}, but found no native-supported operation " - "flags to pass. This is a seam bug; set core.indexer to 'python' to " - "revert." - ) - - with pytest.raises(RuntimeError, match=re.escape(expected)): - native_seam.maybe_run_native_indexer( - _args(), - "/tmp/journal", - config_reader=lambda _journal: {"core": {"indexer": "rust"}}, - coverage_checker=_raise_unexpected("coverage_checker"), - handshake_checker=_ok, - helper_locator=lambda: Path("/tmp/bin/solstone-core"), - native_runner=_raise_unexpected("native_runner"), - ) - - -def test_run_command_propagates_native_nonzero_indexer_return(monkeypatch) -> None: - def setup_cli(parser): - parsed = parser.parse_args(["--rescan"]) - parsed.verbose = False - parsed.debug = False - return parsed - - def seam(args: argparse.Namespace, journal: str) -> int | None: - return native_seam.maybe_run_native_indexer( - args, - journal, - config_reader=lambda _journal: {"core": {"indexer": "rust"}}, - handshake_checker=_ok, - helper_locator=lambda: Path("/tmp/bin/solstone-core"), - native_runner=lambda argv, *, check=False: subprocess.CompletedProcess( - argv, - 75, - ), - ) - - monkeypatch.setattr(indexer_cli, "setup_cli", setup_cli) - monkeypatch.setattr(indexer_cli, "require_solstone", lambda: None) - monkeypatch.setattr(indexer_cli, "get_journal", lambda: "/tmp/journal") - monkeypatch.setattr(indexer_cli, "maybe_run_native_indexer", seam) - monkeypatch.setattr(sys, "argv", ["journal indexer", "--rescan"]) - - assert sol_cli.run_command("solstone.think.indexer") == 75 diff --git a/tests/test_journal_index.py b/tests/test_journal_index.py index 70c9b5c88..ecb77fb68 100644 --- a/tests/test_journal_index.py +++ b/tests/test_journal_index.py @@ -769,18 +769,6 @@ def test_time_bucket_non_segment_empty(journal_fixture): conn.close() -def test_reset_journal_index(journal_fixture): - """Test resetting the journal index.""" - from solstone.think.indexer.journal import reset_journal_index, scan_journal - - scan_journal(str(journal_fixture)) - index_path = journal_fixture / "indexer" / "journal.sqlite" - assert index_path.exists() - - reset_journal_index(str(journal_fixture)) - assert not index_path.exists() - - def test_index_caching(journal_fixture): """Test that unchanged files are not re-indexed.""" from solstone.think.indexer.journal import scan_journal @@ -1797,6 +1785,87 @@ def test_prune_chunks_by_stream(monkeypatch, tmp_path): conn.close() +def test_delete_segment_index_rows_removes_only_target_segment(monkeypatch, tmp_path): + """delete_segment_index_rows remains an in-process index maintenance helper.""" + from solstone.think.indexer.journal import ( + delete_segment_index_rows, + get_journal_index, + scan_journal, + ) + from solstone.think.streams import write_segment_stream + + journal = tmp_path / "journal" + monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) + + target_rel = "20240101/default/100000_300" + target_seg = journal / "chronicle" / target_rel + target_talents = target_seg / "talents" + target_talents.mkdir(parents=True) + (target_talents / "screen.md").write_text( + "# Target\n\nTarget segment content.\n", + encoding="utf-8", + ) + write_segment_stream(target_seg, "default", None, None, 1) + + other_rel = "20240101/default/101000_300" + other_seg = journal / "chronicle" / other_rel + other_talents = other_seg / "talents" + other_talents.mkdir(parents=True) + (other_talents / "screen.md").write_text( + "# Other\n\nOther segment content.\n", + encoding="utf-8", + ) + write_segment_stream(other_seg, "default", None, None, 1) + + scan_journal(str(journal), full=True) + conn, _ = get_journal_index(str(journal)) + target_chunks = conn.execute( + "SELECT count(*) FROM chunks WHERE path = ? OR path LIKE ?", + (target_rel, f"{target_rel}/%"), + ).fetchone()[0] + target_files = conn.execute( + "SELECT count(*) FROM files WHERE path LIKE ?", + (f"{target_rel}/%",), + ).fetchone()[0] + other_chunks = conn.execute( + "SELECT count(*) FROM chunks WHERE path = ? OR path LIKE ?", + (other_rel, f"{other_rel}/%"), + ).fetchone()[0] + conn.close() + + assert target_chunks > 0 + assert target_files > 0 + assert other_chunks > 0 + + result = delete_segment_index_rows(str(journal), target_rel) + + assert result == {"chunks": target_chunks, "files": target_files, "error": None} + + conn, _ = get_journal_index(str(journal)) + assert ( + conn.execute( + "SELECT count(*) FROM chunks WHERE path = ? OR path LIKE ?", + (target_rel, f"{target_rel}/%"), + ).fetchone()[0] + == 0 + ) + assert ( + conn.execute( + "SELECT count(*) FROM files WHERE path LIKE ?", + (f"{target_rel}/%",), + ).fetchone()[0] + == 0 + ) + assert ( + conn.execute( + "SELECT count(*) FROM chunks WHERE path = ? OR path LIKE ?", + (other_rel, f"{other_rel}/%"), + ).fetchone()[0] + == other_chunks + ) + conn.close() + + class TestSegmentChunks: """Tests for segment-level concatenated FTS5 chunks.""" diff --git a/tests/verify_indexer_differential.py b/tests/verify_indexer_differential.py index 5f9d42890..7421d8754 100644 --- a/tests/verify_indexer_differential.py +++ b/tests/verify_indexer_differential.py @@ -94,6 +94,8 @@ NATIVE_MARKDOWN_SANITIZE_RULE = "native_markdown_sanitize_drop" NATIVE_MARKDOWN_SANITIZE_RE = re.compile( r"^warning: Dropped \d+ line\(s\) exceeding 2048 chars during markdown sanitization$" ) +NATIVE_EDGE_SKIP_RULE = "native_edge_extraction_skip" +NATIVE_EDGE_SKIP_PREFIX = "warning: Skipping edge extraction for " EXCLUDED_SHADOW_TABLES = [ "chunks_config", "chunks_content", @@ -295,6 +297,7 @@ def classify_stderr(stderr: str) -> dict[str, Any]: "count": 0, "examples": [], } + native_edge_rule = {"name": NATIVE_EDGE_SKIP_RULE, "count": 0, "examples": []} unclassified: list[str] = [] for line in stderr.splitlines(): if not line.strip(): @@ -305,10 +308,12 @@ def classify_stderr(stderr: str) -> dict[str, Any]: _record_rule_hit(markdown_rule, line) elif NATIVE_MARKDOWN_SANITIZE_RE.match(line): _record_rule_hit(native_markdown_rule, line) + elif line.startswith(NATIVE_EDGE_SKIP_PREFIX): + _record_rule_hit(native_edge_rule, line) else: unclassified.append(line) return { - "rules": [edge_rule, markdown_rule, native_markdown_rule], + "rules": [edge_rule, markdown_rule, native_markdown_rule, native_edge_rule], "unclassified": unclassified, } -- 2.51.2