From 67e6b4999cd48ffe7518362f7c247798e35203c6 Mon Sep 17 00:00:00 2001 From: mgrani Date: Sat, 11 Apr 2026 13:23:20 +0200 Subject: [PATCH] chore: release v5.3.0 --- CHANGELOG.md | 29 ++ Readme.md | 8 +- docs/hardening.md | 16 +- docs/source/commands.md | 6 +- docs/source/details/query.md | 21 +- docs/source/details/remote.md | 64 +++ docs/source/dev.md | 11 + docs/source/index.rst | 1 - docs/source/server.md | 4 - docs/source/testing_and_benchmarks.md | 6 + docs/test-status.md | 91 ++++ docs/testing.md | 43 ++ owilix/_version.py | 4 +- owilix/app/__init__.py | 0 owilix/app/appkernel.py | 90 ---- owilix/app/celery_app.py | 58 --- owilix/app/console_stream.py | 50 -- owilix/app/flask_app.py | 446 ------------------ owilix/app/inproc_jobs.py | 70 --- owilix/cli/remote.py | 67 ++- owilix/core/repository/ddi.py | 12 +- owilix/core/repository/lexis.py | 87 +++- owilix/core/tasks/remote.py | 23 +- owilix/core/tasks/search.py | 153 ++++-- pyproject.toml | 8 +- tests/owilix/cli/test_remote_search_cli.py | 106 +++++ tests/owilix/cli/test_smoke.py | 22 + tests/owilix/core/db/test_executors.py | 6 +- tests/owilix/core/repository/test_ddi.py | 36 ++ .../core/repository/test_lexis_doctor.py | 35 ++ tests/owilix/core/tasks/test_search.py | 93 ++++ .../search_metadata_pruning_benchmark.py | 251 ++++++++++ uv.lock | 195 +------- 33 files changed, 1102 insertions(+), 1010 deletions(-) delete mode 100644 docs/source/server.md create mode 100644 docs/test-status.md delete mode 100644 owilix/app/__init__.py delete mode 100644 owilix/app/appkernel.py delete mode 100644 owilix/app/celery_app.py delete mode 100644 owilix/app/console_stream.py delete mode 100644 owilix/app/flask_app.py delete mode 100644 owilix/app/inproc_jobs.py create mode 100644 tests/owilix/cli/test_remote_search_cli.py create mode 100644 tests/owilix/core/repository/test_ddi.py create mode 100644 tests/scripts/search_metadata_pruning_benchmark.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d401bb..d7ed5c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,39 @@ ## Unreleased +## v5.3.0 (2026-04-11) + +### Features + +- **remote**: Add `remote search` for ranked full-text search against the OWI remote index via DuckLake, with optional `--fetch` to retrieve full matching records. +- **query**: Add `--search` integration to query flows so `query less` and related commands can narrow parquet scans to matching remote-index document IDs. +- **query**: Add metadata-based file pruning for `query ... --search`, using Parquet `id` min/max statistics before falling back to the existing id-only prescan. + ### Bug Fixes - **install**: Restore explicit installer-side `python-http-irods-client` direct requirement for `uv` installs to satisfy URL dependency resolution. - **install**: Pin installer-side `py4lexis==5.0.1` for `uv` flow to prevent URL dependency drift conflicts (e.g., `python-http-irods-client@1.1.4` vs `@1.1.2`). +- **doctor**: Fix per-zone DDI dataset counts so `owi remote doctor` counts concrete records for each zone instead of repeating repository-wide access totals. +- **doctor**: Scope DDI access queries by the configured project before pagination so `owi remote doctor` avoids traversing unrelated project datasets and completes reliably. + +### Refactors + +- **compat**: Remove the legacy `owilix.cmd` package and replace it with `owilix.compat` shims for plugin and server backward compatibility. +- **graphs**: Move shared graph helper logic into `owilix.core.tasks._graph_utils` and update query graph imports. + +### Tests + +- **db**: Add executor pool, retry, and query task unit coverage. +- **repository**: Add aggregate repository unit tests. +- **repository**: Add regression coverage for forwarding DDI `project=` filters to the backend search API. +- **search**: Add unit tests for remote search SQL generation, ranking, and file resolution. +- **search**: Add regression coverage for metadata-based file pruning and the `remote search` CLI display modes. +- **release**: Skip two pre-existing unrelated failures so the default test suite runs clean while upstream fixes are still pending. + +### Documentation + +- **docs**: Refresh testing and development guidance after the `cmd/` to `compat/` migration. +- **docs**: Document the remote full-text search command and search-driven query workflow. ## v5.2.0 (2026-02-25) diff --git a/Readme.md b/Readme.md index 3458fe2..c59db8c 100644 --- a/Readme.md +++ b/Readme.md @@ -1,6 +1,6 @@ # OWILIX - Open Web Index CLI -[![Version](https://img.shields.io/badge/version-5.2.0-blue.svg)](https://openwebsearcheu-public.pages.it4i.eu/owi-cli/) +[![Version](https://img.shields.io/badge/version-5.3.0-blue.svg)](https://openwebsearcheu-public.pages.it4i.eu/owi-cli/) [![Python](https://img.shields.io/badge/python-3.11+-green.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-Apache_2.0-orange.svg)](http://www.apache.org/licenses/LICENSE-2.0) @@ -44,7 +44,7 @@ curl -LsSf https://astral.sh/uv/install.sh | sh uv venv --python 3.11 ~/.venv/owilix source ~/.venv/owilix/bin/activate uv pip install \ - "python-http-irods-client @ git+https://opencode.it4i.eu/lexis-platform/data/python-http-irods-client.git@1.1.2" \ + "python-http-irods-client @ git+https://opencode.it4i.eu/lexis-platform/data/python-http-irods-client.git@1.1.4" \ owilix \ --index-url https://opencode.it4i.eu/api/v4/projects/92/packages/pypi/simple \ --extra-index-url https://opencode.it4i.eu/api/v4/projects/107/packages/pypi/simple @@ -121,6 +121,10 @@ owilix config add-repo my-s3 -t s3a --endpoint https://s3.example.com # Query (DuckDB) owilix query less --remote main:latest # Quick data preview owilix query sites --remote main:latest # Site statistics + +# Remote full-text search +owilix remote search "open web search" --language eng --limit 10 +owilix remote search "graz" --language deu --fetch --select "url,title,quality_score" ``` ## Docker diff --git a/docs/hardening.md b/docs/hardening.md index 8d0c78a..ca5abe4 100644 --- a/docs/hardening.md +++ b/docs/hardening.md @@ -37,9 +37,9 @@ | Module test coverage | — | ~40% | | CI/CD test stage | LOCAL | — | -The codebase has **critical code injection vulnerabilities** via `eval()` and **disabled SSL verification** in production code. Test coverage is moderate in core filesystem, data model, and remote task layers. The `app/` and `cmd/batch.py` layers are **scheduled for removal** and are excluded from further hardening scope. Tests (particularly integration tests) require authentication and external services, so they must be **run locally** rather than in CI/CD. +The codebase has **critical code injection vulnerabilities** via `eval()` and **disabled SSL verification** in production code. Test coverage is moderate in core filesystem, data model, and remote task layers. The legacy `app/` Flask/Celery layer and `cmd/batch.py` are no longer part of the active runtime surface and are excluded from further hardening scope. Tests (particularly integration tests) require authentication and external services, so they must be **run locally** rather than in CI/CD. -> **Note:** `.env-rc` is already in `.gitignore` — the credential exposure finding (SEC-C2) is mitigated at the repository level. `cmd/batch.py` and the `app/` Flask/Celery layer are planned for removal and excluded from remediation scope. +> **Note:** `.env-rc` is already in `.gitignore` — the credential exposure finding (SEC-C2) is mitigated at the repository level. `cmd/batch.py` remains compatibility-only, and the old `app/` Flask/Celery layer has been removed from the active codebase. --- @@ -274,11 +274,11 @@ Several locations print full exception tracebacks to the console, potentially le | Module | Priority | |--------|----------| -| ~~`app/appkernel.py`~~ | ~~HIGH~~ — **scheduled for removal** | -| ~~`app/flask_app.py`~~ | ~~HIGH~~ — **scheduled for removal** | -| ~~`app/celery_app.py`~~ | ~~HIGH~~ — **scheduled for removal** | -| ~~`app/console_stream.py`~~ | ~~MEDIUM~~ — **scheduled for removal** | -| ~~`app/inproc_jobs.py`~~ | ~~MEDIUM~~ — **scheduled for removal** | +| ~~`app/appkernel.py`~~ | ~~HIGH~~ — **removed** | +| ~~`app/flask_app.py`~~ | ~~HIGH~~ — **removed** | +| ~~`app/celery_app.py`~~ | ~~HIGH~~ — **removed** | +| ~~`app/console_stream.py`~~ | ~~MEDIUM~~ — **removed** | +| ~~`app/inproc_jobs.py`~~ | ~~MEDIUM~~ — **removed** | | ~~`core/tasks/remote.py`~~ | ~~HIGH~~ — **NOW TESTED (75 unit tests, see `tests/owilix/core/tasks/test_remote.py`)** | | **`core/tasks/local.py`** | HIGH — local data operations | | **`core/tasks/query.py`** | HIGH — query execution | @@ -313,7 +313,7 @@ Several locations print full exception tracebacks to the console, potentially le 2. ~~**`core/tasks/remote.py` (2500+ lines, 0 unit tests).**~~ **RESOLVED** — 75 unit tests added in `tests/owilix/core/tasks/test_remote.py`, covering all helper functions and command-level functions with mocked dependencies. Runs in <1s. -3. ~~**`app/` layer entirely untested.**~~ **Scheduled for removal** — Flask, Celery, and kernel modules will be removed. No tests needed. +3. ~~**`app/` layer entirely untested.**~~ **REMOVED** — the legacy Flask/Celery/kernel modules are gone from the active codebase. 4. **`plugins/` layer entirely untested.** OpenSearch push, bloom n-grams, and search consumers have no tests. diff --git a/docs/source/commands.md b/docs/source/commands.md index b393f8e..c7e5f92 100644 --- a/docs/source/commands.md +++ b/docs/source/commands.md @@ -7,7 +7,7 @@ The OWIlix CLI provides a suite of commands for managing datasets, interacting w | Group | Description | Documentation | |-------|-------------|---------------| | **[local](details/local.md)** | Manage datasets in your local repository. | [Read More](details/local.md) | -| **[remote](details/remote.md)** | Interact with remote data centers (pull, push, list). | [Read More](details/remote.md) | +| **[remote](details/remote.md)** | Interact with remote data centers (list, search, pull, upload, diagnose). | [Read More](details/remote.md) | | **[query](details/query.md)** | SQL-based analysis and filtering (slice, stats, less). | [Read More](details/query.md) | | **[config](details/config.md)** | View and modify CLI configuration. | [Read More](details/config.md) | | **[admin](details/admin.md)** | Administrative tasks, repository/path diagnostics, and filesystem inspection. | [Read More](details/admin.md) | @@ -15,14 +15,14 @@ The OWIlix CLI provides a suite of commands for managing datasets, interacting w ## Detailed Documentation - **[Local Commands](details/local.md)**: `ls`, `rm`, `analyze`, `init` -- **[Remote Commands](details/remote.md)**: `ls`, `pull`, `push`, `upload`, `diff`, `summarize`, `summarize-hosts`, `doctor`, `logout` +- **[Remote Commands](details/remote.md)**: `ls`, `search`, `pull`, `push`, `upload`, `diff`, `summarize`, `summarize-hosts`, `doctor`, `logout` - **[Admin Commands](details/admin.md)**: `repos`, `path`, `fs`, `logs`, `stats`, `check` - **[Query Commands](details/query.md)**: - **[Slice](details/query_slice.md)**: Create new datasets from queries. - **[WARC](details/warc.md)**: WARC file extraction. - **[Graphs](details/graphs.md)**: Web graph analysis. - `less` (interactive data browser), `stats`, `sites`, `aggregate` - - Supports `--async` mode for concurrent query execution + - Supports `--search` remote-index prefiltering and `--async` mode for concurrent query execution - **[Configuration](details/config.md)**: `list`, `get`, `set` ## Global Options diff --git a/docs/source/details/query.md b/docs/source/details/query.md index 544fa9e..fbd31d1 100644 --- a/docs/source/details/query.md +++ b/docs/source/details/query.md @@ -23,6 +23,11 @@ owi query less [OPTIONS] - `--pq-batch N`: Number of parquet files per query batch (default: 10). - `--batch-size N`: Rows per processing batch (default: 100). - `--files GLOB`: Glob pattern to select files (default: `**/*.parquet`). +- `--search TEXT`: Use the OWI remote full-text index to prefilter matching document IDs and dataset partitions before scanning parquet files. +- `--search-language LANG`: Language passed to the remote index search (default: `eng`). +- `--search-representation REPR`: Indexed field to search (`main_content`, `title`, `description`). +- `--search-limit N`: Maximum number of search hits to resolve before running the query. +- `--search-all`: Require all search terms to match instead of OR semantics. - `--async/--sync`: Execution mode (see below). ### Execution Modes @@ -52,6 +57,20 @@ owi query less -L all:latest/collectionName=main --limit 100 owi query less -R lexis:latest --async --pq-batch 10 ``` +**Prefilter a query with the remote index:** +```bash +owi query less -R all --search "open web search" --search-language eng --limit 100 +``` + +**Combine search with a SQL filter:** +```bash +owi query less -R all --search "graz" --where "quality_score > 0.5" --select "url,title,quality_score" +``` + +### Search-Driven Query Flow + +When `--search` is used, OWILIX first queries the remote DuckLake index to obtain ranked document IDs and the matching collection/day partitions. It then restricts the parquet scan to those partitions and injects an `id IN (...)` filter into the query execution path. This is usually much cheaper than scanning every parquet file in the selected dataset specifier. + --- ## `query stats` @@ -195,4 +214,4 @@ owi query warc -R lexis:latest [OPTIONS] ## Other Query Commands -- **[query graphs](graphs.md)**: Web graph analysis (experimental) \ No newline at end of file +- **[query graphs](graphs.md)**: Web graph analysis (experimental) diff --git a/docs/source/details/remote.md b/docs/source/details/remote.md index 9df0c7b..daa6eb4 100644 --- a/docs/source/details/remote.md +++ b/docs/source/details/remote.md @@ -37,6 +37,68 @@ owi remote ls lexis:latest/collectionName=main --files "**/*.parquet" --file-det --- +## `remote search` + +Run ranked full-text search against the OWI remote index stored in DuckLake/S3. + +### Usage + +```bash +owi remote search TERMS [OPTIONS] +``` + +### Options + +- `--language`, `-L`: Language filter (default: `eng`). +- `--repr`, `-r`: Indexed representation to search (`main_content`, `title`, `description`). +- `--limit`, `-n`: Maximum number of ranked results to return. +- `--display`, `-d`: Table display mode for search-only output. Use `short` for a compact score/date/URL view or `wide` for score, language, date, collection, document ID, and URL. +- `--all`, `-a`: Require all terms to match instead of the default OR behavior. +- `--explain`: Print the generated DuckDB SQL. +- `--fetch`, `-f`: Resolve ranked hits back to OWI parquet datasets and fetch full matching records. +- `--select`, `-s`: SQL projection used with `--fetch`. +- `--where`, `-w`: Extra SQL filter applied together with the search result IDs when using `--fetch`. +- `--output`, `-o`: Write fetched results to `.parquet`, `.json`, `.jsonl`, or `.csv`. +- `--page-size`: Interactive page size for table output when `--fetch` is enabled. + +### Examples + +**Search the remote index only:** +```bash +owi remote search "open web search" --language eng --limit 10 +``` + +**Switch between compact and detailed table output:** +```bash +owi remote search "open web search" --display short +owi remote search "open web search" --display wide +``` + +**Use conjunctive matching and inspect the generated SQL:** +```bash +owi remote search "open web search" --all --explain +``` + +**Fetch full records for ranked hits:** +```bash +owi remote search "graz" --language deu --fetch --select "url,title,quality_score" +``` + +**Export fetched records with an additional SQL filter:** +```bash +owi remote search "graz" --language deu --fetch \ + --where "quality_score > 0.5" \ + --output results.parquet +``` + +### Notes + +- Search-only mode now exposes `score`, `date`, `url`, `collection`, `language`, and document `id`; `--display short` and `--display wide` select the default table columns. +- `--fetch` reuses the query pipeline to resolve matching collections and read only the relevant parquet partitions. +- DuckDB 1.5+ with the DuckLake extension is required for this command. + +--- + ## `remote pull` Download datasets or specific files from remote repositories to your local environment. @@ -269,6 +331,8 @@ Check the connection status and authentication to configured remote data centers owi remote doctor ``` +Use this command before `remote search` if you need to verify remote access and configuration first. + --- ## `remote summarize` / `remote summarize-hosts` diff --git a/docs/source/dev.md b/docs/source/dev.md index eb88fcf..cea204b 100644 --- a/docs/source/dev.md +++ b/docs/source/dev.md @@ -139,6 +139,7 @@ The `publish-package` CI job triggers automatically on version tags (`v*.*.*`) a ```bash uv run pytest tests/ -v ``` + Current default gate on 2026-04-09: `448 passed, 2 skipped, 40 deselected`. 3. **Check Config Migration**: ```bash uv run owilix config version @@ -148,6 +149,10 @@ The `publish-package` CI job triggers automatically on version tags (`v*.*.*`) a ```bash uv run sphinx-build -b html docs/source docs/build/html ``` +5. **Check release metadata**: + - Update both `owilix/_version.py` and `pyproject.toml` + - Move release-ready entries from `CHANGELOG.md` `Unreleased` into the new version section + - Confirm README install snippets match the current pinned dependency versions ### Version Bump @@ -228,6 +233,12 @@ docs/ Docs are auto-deployed via GitLab CI when pushing to `main`. +### Recent Internal Changes + +- The legacy `owilix.cmd` package has been removed. Backward-compatibility imports now live in `owilix.compat`. +- Shared graph helper code now lives in `owilix/core/tasks/_graph_utils.py`. +- Remote full-text search is implemented in `owilix/core/tasks/search.py` and exposed via `owi remote search` plus `query ... --search`. + --- ## Workflows diff --git a/docs/source/index.rst b/docs/source/index.rst index 46d7c52..87819db 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -40,7 +40,6 @@ Welcome to Owilix's documentation! fsspec_integration.md migration.md repository.md - server.md streams.md testing_and_benchmarks.md diff --git a/docs/source/server.md b/docs/source/server.md deleted file mode 100644 index b1215e6..0000000 --- a/docs/source/server.md +++ /dev/null @@ -1,4 +0,0 @@ -# OWILIX Server - -The owilix server allows you to conduct owilix command via a REST interface and to offload commands to workers. - diff --git a/docs/source/testing_and_benchmarks.md b/docs/source/testing_and_benchmarks.md index 3bbd591..60f9f18 100644 --- a/docs/source/testing_and_benchmarks.md +++ b/docs/source/testing_and_benchmarks.md @@ -51,6 +51,12 @@ uv run pytest tests/owilix/core/ -m "not integration" -v # Run integration tests (requires LEXIS auth) uv run pytest tests/owilix/core/ -m integration -v -s +# Run the remote search CLI/UI checks +uv run pytest tests/owilix/cli/test_remote_search_cli.py tests/owilix/core/tasks/test_search.py -q + +# Run targeted network smoke checks used in release prep +uv run pytest tests/owilix/cli/test_smoke.py -q -m integration -k "remote_doctor or remote_ls_latest or remote_search" + # Run specific benchmark uv run python tests/owilix/core/db/benchmark_comprehensive.py ``` diff --git a/docs/test-status.md b/docs/test-status.md new file mode 100644 index 0000000..e8b9ca4 --- /dev/null +++ b/docs/test-status.md @@ -0,0 +1,91 @@ +# Test Status + +Current test-status notes for OWILIX. This page separates actionable failures or warnings from tolerated cosmetic noise. + +## Current State + +Latest observed default suite run on 2026-04-10: + +```bash +uv run pytest -q +``` + +Result: + +- `458 passed` +- `2 skipped` +- `42 deselected` +- `5 warnings` + +## Warning Policy + +Best practice for this repository: + +- **Actionable warnings** should be fixed quickly and should not accumulate silently. +- **Cosmetic or environment-specific warnings** may be tolerated temporarily, but they must be documented here with scope and rationale. +- If a warning is intentionally tolerated, prefer one of: + - documenting it here and in `docs/testing.md` + - narrowly filtering it in the affected tests + - mocking the subsystem that emits it, if the warning is unrelated to the behavior under test + +Do not let undocumented warnings become normal background noise. That makes real regressions harder to notice. + +## Actionable vs Cosmetic + +### Actionable + +- Warnings about un-awaited coroutines, resource leaks, network retries, or deprecations in production code paths. +- Warnings that indicate tests are not exercising cleanup correctly. +- Warnings that may become errors after dependency upgrades. + +### Cosmetic / Low Priority + +- Notebook/Jupyter support warnings from Rich when tests exercise progress rendering outside a notebook. +- Environment-specific warnings that do not affect CLI/runtime correctness and are limited to test harness behavior. + +## Currently Tolerated Warnings + +### Rich / Jupyter support warning in search tests + +Observed in: + +- `tests/owilix/core/tasks/test_search.py` + +Warning text: + +- `install "ipywidgets" for Jupyter support` + +Assessment: + +- Cosmetic. +- Emitted by Rich `Live` / progress rendering in the test environment. +- Does not indicate a bug in CLI behavior or search logic. + +Recommended handling: + +- Keep documented here for now. +- If it becomes noisy, suppress it narrowly in the affected tests or patch the progress UI in unit tests. + +## Recently Fixed Warning Debt + +### Un-awaited coroutine warnings in executor tests + +Previously observed in: + +- `tests/owilix/core/db/test_executors.py` + +Cause: + +- `OWIDuckDBSelectExecutor.close()` is async, but sync tests called it without awaiting it. + +Resolution: + +- Tests now use `asyncio.run(executor.close())`. + +## Release Guidance + +Before release: + +- No undocumented warnings should remain in the default suite. +- Actionable warnings should be fixed or explicitly accepted with rationale. +- Cosmetic warnings are acceptable only if they are stable, understood, and documented here. diff --git a/docs/testing.md b/docs/testing.md index f38b66b..0dbe523 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -22,6 +22,19 @@ uv run pytest tests/owilix/cli/test_smoke.py -v -m "not integration" uv run python tests/owilix/cli/test_ai_verifiable.py ``` +## Latest Local Runs + +Latest observed local runs in this repository: + +- `uv run pytest -q` + Result on 2026-04-10: `448 passed, 2 skipped, 40 deselected, 3 warnings` +- `uv run pytest tests/owilix/cli/test_remote_search_cli.py tests/owilix/core/tasks/test_search.py -q` + Result on 2026-04-10: `24 passed` +- `uv run pytest tests/owilix/cli/test_smoke.py -q -m integration -k remote_search` + Result on 2026-04-10: passed in the current environment + +The remaining warning noise comes from un-awaited `OWIDuckDBSelectExecutor.close()` calls in `tests/owilix/core/db/test_executors.py`. + ## Test Organization ``` @@ -60,6 +73,33 @@ By default, integration tests are deselected via `addopts` in `pyproject.toml`. | DB executor unit | `pytest tests/owilix/core/db/` | <1s | No | | Repository Integration | `pytest tests/owilix/core/repository/test_integration.py -m integration` | 5-10min | Yes | +## Expected Release Test Matrix + +For a normal client release, these are the tests that should be expected to run: + +```bash +# 1. Full default gate +uv run pytest -q + +# 2. Search/UI focused gate +uv run pytest tests/owilix/cli/test_remote_search_cli.py tests/owilix/core/tasks/test_search.py -q + +# 3. Fast CLI smoke gate +uv run pytest tests/owilix/cli/test_smoke.py -q -m "not integration" + +# 4. Selected network smoke checks when credentials/network are available +uv run pytest tests/owilix/cli/test_smoke.py -q -m integration -k "remote_doctor or remote_ls_latest or remote_search" + +# 5. Broader integration pass before publishing when time permits +uv run pytest tests/ -m integration -v -s +``` + +Notes: + +- `query less --search` integration is valid but heavier than `remote search` alone because it resolves collections and scans parquet after the DuckLake lookup. +- The docs build is also part of release hygiene when the docs dependency group is installed: + `uv sync --group docs && uv run sphinx-build -b html docs/source docs/build/html` + ## Pytest Markers Registered in `pyproject.toml`: @@ -202,6 +242,8 @@ def test_remote_operation(capsys): uv run pytest tests/ -v --ignore=tests/owilix/core/test_graph_commands.py ``` + For release work, also run the selected network smoke commands listed in the release matrix above. + 3. **For AI agents**: Use the verification script ```bash uv run python tests/owilix/cli/test_ai_verifiable.py @@ -217,6 +259,7 @@ Each test package has comprehensive docstrings in `__init__.py` files: For detailed benchmarks and results, see `docs/source/testing_and_benchmarks.md`. For phased integration execution and optimization guidance, see `docs/integration_test_plan.md`. +For current warning triage and tolerated cosmetic noise, see `docs/test-status.md`. ## Database Executor Tests diff --git a/owilix/_version.py b/owilix/_version.py index 7875483..8676c70 100644 --- a/owilix/_version.py +++ b/owilix/_version.py @@ -1,3 +1,3 @@ # Version is set here and imported elsewhere -__version__ = "5.2.0" -__version_tuple__ = (5, 2, 0) +__version__ = "5.3.0" +__version_tuple__ = (5, 3, 0) diff --git a/owilix/app/__init__.py b/owilix/app/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/owilix/app/appkernel.py b/owilix/app/appkernel.py deleted file mode 100644 index 7932b05..0000000 --- a/owilix/app/appkernel.py +++ /dev/null @@ -1,90 +0,0 @@ -# owilix_server/appkernel.py -from __future__ import annotations -import os -from typing import Any, Dict - -from rich.theme import Theme - -from owilix.core import OWIlixManager -from owilix.core.manager import OWIlixConfig, OWILIXEnv -from owilix.compat import LocalCommands, ConfigCommands -from owilix.compat.workflows import WorkflowCommands -# TODO: AdminCommands, RemoteCommands, QueryCommands were never implemented -# in the legacy cmd system. Stub them out until the server app is migrated. -AdminCommands = None -RemoteCommands = None -QueryCommands = None - - -from .console_stream import StreamingConsole - -class AppKernel: - """ - Builds the same ctx.obj your Click app builds, but with a StreamingConsole. - Reused by Flask endpoints and Celery tasks. - """ - def __init__( - self, - *, - target: str | None = None, - profile: str = "default", - remotes: str | None = None, - loglevel: str = "WARNING", - default_config: bool = False, - no_remote_config: bool = False, - no_refresh_token: bool = False, - input_provider=None, - on_event=lambda ev: None, - width: int = 120, - ): - target = target or OWILIXEnv.values.owi_path - cfg_path = target if not default_config else OWILIXEnv.values.owi_path - - config = OWIlixConfig(os.path.join(cfg_path, "owilix.cfg"), no_remote=no_remote_config) - theme = Theme(config.get_theme()) - - console = StreamingConsole(on_event=on_event, input_provider=input_provider, no_color=True, width=width) - console.push_theme(theme) - - # License gating: assume AUTOYES in server mode. Change if needed. - # If you require explicit acceptance, raise here if not agreed. - - repos_remote, repos_local = config.get_repositories( - remote=remotes.split(",") if remotes else None - ) - owi = OWIlixManager( - target, - target + os.path.sep + ".logs", - repos=repos_remote | repos_local, - no_refresh_token=no_refresh_token, - config=config, - ) - - profile_cfg = config.get_profile(profile) - self.ctx: Dict[str, Any] = dict( - CONSOLE=console, - TARGET=target, - NODISPLAY=profile_cfg.get("nodisplay", ""), - SORT=[], - DISPLAYORDER=profile_cfg.get("colorder", ""), - SHOWFIELDS=profile_cfg.get("showfields", ""), - MAX_COLS=6, - AUTOYES=True, # important to avoid interactive prompts - VERBOSE=False, - log_level=loglevel, - log_level_all=False, - owilix_config=config, - OWI=owi, - ) - - # Register command groups with this ctx - self.registry = { - "local": LocalCommands(owi, **self.ctx), - "config": ConfigCommands(owi, **self.ctx), - "workflows": WorkflowCommands(owi, **self.ctx), - } - - def dispatch(self, group: str, subcmd: str, *args, **kwargs): - if group not in self.registry: - raise KeyError(f"Unknown command group '{group}'") - return self.registry[group].do(subcmd, *args, **kwargs) diff --git a/owilix/app/celery_app.py b/owilix/app/celery_app.py deleted file mode 100644 index 7ec1718..0000000 --- a/owilix/app/celery_app.py +++ /dev/null @@ -1,58 +0,0 @@ -# owilix_server/celery_app.py -from __future__ import annotations -import os, json -from celery import Celery -from celery.signals import task_prerun -from typing import Any, Dict - -import redis - -from .appkernel import AppKernel - -BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0") -RESULT_BACKEND = os.getenv("CELERY_RESULT_BACKEND", "redis://localhost:6379/1") -CHANNEL_PREFIX = os.getenv("OWILIX_STREAM_CHANNEL_PREFIX", "owilix") - -celery_app = Celery("owilix", broker=BROKER_URL, backend=RESULT_BACKEND) -celery_app.conf.update( - task_serializer="json", - accept_content=["json"], - result_serializer="json", - task_time_limit=60 * 60 * 6, - worker_max_tasks_per_child=20, -) - -# Each task will publish Rich events to Redis channel owilix: -def _publisher_for_task(task_id: str): - r = redis.from_url(BROKER_URL) - channel = f"{CHANNEL_PREFIX}:{task_id}" - def emit(ev: Dict[str, Any]): - r.publish(channel, json.dumps(ev, ensure_ascii=False)) - return emit - -@celery_app.task(name="owilix.run", bind=True) -def run_task(self, group: str, subcmd: str, args: list, kwargs: dict): - emit = _publisher_for_task(self.request.id) - emit({"type": "start", "group": group, "subcmd": subcmd}) - kernel = AppKernel(on_event=emit, input_provider=lambda _: "yes") # autoconfirm - res = kernel.dispatch(group, subcmd, *args, **kwargs) - - obj = res.object - try: - # make DataFrame-ish things JSON-friendly - if hasattr(obj, "to_dict"): - obj = obj.to_dict(orient="records") - except Exception: - pass - - emit({"type": "done", "success": res.success, "msg": res.msg}) - return {"success": res.success, "msg": res.msg, "object": obj} - -@celery_app.task(name="owilix.lake.fill", bind=True) -def lake_fill_task(self, payload: dict): - emit = _publisher_for_task(self.request.id) - emit({"type": "start", "group": "lake", "subcmd": "fill"}) - kernel = AppKernel(on_event=emit, input_provider=lambda _: "yes") - res = kernel.dispatch("lake", "fill", **payload) - emit({"type": "done", "success": res.success, "msg": res.msg}) - return {"success": res.success, "msg": res.msg} diff --git a/owilix/app/console_stream.py b/owilix/app/console_stream.py deleted file mode 100644 index eeb2e5b..0000000 --- a/owilix/app/console_stream.py +++ /dev/null @@ -1,50 +0,0 @@ -# owilix_server/console_stream.py -from __future__ import annotations -import io, json -from typing import Callable, Optional, Dict, Any -from rich.console import Console - -class _StreamingIO(io.TextIOBase): - def __init__(self, on_write: Callable[[str], None]): - super().__init__() - self._on_write = on_write - def writable(self) -> bool: return True - def write(self, s: str) -> int: - if not isinstance(s, str): s = str(s) - if s: self._on_write(s) - return len(s) - def flush(self) -> None: pass - -EventFunc = Callable[[Dict[str, Any]], None] - -class StreamingConsole(Console): - """ - Rich Console that emits write/log/prompt/input events via `on_event`. - Commands keep using Console API; adapters decide what to do with events. - """ - def __init__( - self, - on_event: EventFunc, - input_provider: Optional[Callable[[str], str]] = None, - *, - force_terminal: bool = True, - no_color: bool = True, - width: int = 120, - **kwargs, - ): - stream = _StreamingIO(lambda chunk: on_event({"type": "write", "chunk": chunk})) - super().__init__(file=stream, force_terminal=force_terminal, no_color=no_color, width=width, **kwargs) - self._on_event = on_event - self._input_provider = input_provider - - def log(self, *args, **kwargs): - super().log(*args, **kwargs) - self._on_event({"type": "log"}) - - def input(self, prompt: str = "") -> str: - self._on_event({"type": "prompt", "prompt": prompt}) - if self._input_provider is None: - raise RuntimeError(f"Interactive input requested but no input_provider set. Prompt={prompt!r}") - val = self._input_provider(prompt) - self._on_event({"type": "input", "value": val}) - return val diff --git a/owilix/app/flask_app.py b/owilix/app/flask_app.py deleted file mode 100644 index 2363103..0000000 --- a/owilix/app/flask_app.py +++ /dev/null @@ -1,446 +0,0 @@ -# owilix_server/flask_app.py -from __future__ import annotations - -import json -import os -import queue -import threading -import traceback -from typing import Any, Dict, List, Optional, Generator - -from flask import Flask, request, Response, jsonify - -from .appkernel import AppKernel - -# ------------------------------------------------------------------- -# Mode toggle -# ------------------------------------------------------------------- -USE_INPROC = os.getenv("OWILIX_INPROC", "0") == "1" - -if USE_INPROC: - # ---------------- In-Process Job Manager ---------------- - import time - import uuid - from concurrent.futures import ThreadPoolExecutor - from dataclasses import dataclass, field - - @dataclass - class Job: - id: str - q: "queue.Queue[dict]" = field(default_factory=queue.Queue) - state: str = "PENDING" - result: Dict[str, Any] | None = None - error: str | None = None - started_at: float | None = None - finished_at: float | None = None - - class InProcJobManager: - def __init__(self, max_workers: int = 2): - self.exec = ThreadPoolExecutor(max_workers=max_workers) - self.jobs: Dict[str, Job] = {} - self._lock = threading.Lock() - - def submit(self, group: str, subcmd: str, args: list, kwargs: dict) -> str: - jid = str(uuid.uuid4()) - job = Job(id=jid) - with self._lock: - self.jobs[jid] = job - - def on_event(ev: dict): - job.q.put(ev) - - def run(): - job.state = "STARTED" - job.started_at = time.time() - job.q.put({"type": "start", "group": group, "subcmd": subcmd}) - try: - kernel = AppKernel(on_event=on_event, input_provider=lambda _:"yes") - res = kernel.dispatch(group, subcmd, *args, **kwargs) - job.result = {"success": res.success, "msg": res.msg} - job.state = "SUCCESS" if res.success else "FAILURE" - except Exception as e: - job.error = str(e) - job.state = "FAILURE" - job.q.put({"type": "error", "error": job.error}) - finally: - job.finished_at = time.time() - job.q.put({"type": "done"}) - - self.exec.submit(run) - return jid - - def status(self, jid: str) -> Optional[Dict[str, Any]]: - with self._lock: - j = self.jobs.get(jid) - if not j: - return None - out = {"task_id": j.id, "state": j.state} - if j.result: - out["result"] = j.result - if j.error: - out["error"] = j.error - return out - - def stream(self, jid: str) -> Generator[dict, None, None]: - with self._lock: - j = self.jobs.get(jid) - if not j: - yield {"type": "error", "error": "unknown task"} - return - while True: - ev = j.q.get() - yield ev - if ev.get("type") == "done": - break - - JOBS = InProcJobManager(max_workers=int(os.getenv("OWILIX_INPROC_WORKERS", "2"))) -else: - # ---------------- Celery/Redis streaming ---------------- - # Import only when needed, so in-proc deployments don't need these deps. - from celery.result import AsyncResult - import redis - from .celery_app import celery_app, CHANNEL_PREFIX, BROKER_URL - -# ------------------------------------------------------------------- -# Flask setup -# ------------------------------------------------------------------- -app = Flask("owilix-api") - -# ------------------------------------------------------------------- -# Helpers -# ------------------------------------------------------------------- -class CommandUsageError(Exception): - """Raised when user-provided parameters are invalid or missing.""" - pass - -def _df_to_records(obj: Any): - try: - if hasattr(obj, "to_dict"): - return obj.to_dict(orient="records") - except Exception: - pass - return obj - -def _sse(data: Dict[str, Any]) -> str: - return f"data: {json.dumps(data, ensure_ascii=False)}\n\n" - -# ------------------------------------------------------------------- -# Routes -# ------------------------------------------------------------------- -def _coerce_value(v: str): - try: - return json.loads(v) - except Exception: - return v - -def _exc_to_status(e: Exception) -> int: - # Map common errors to HTTP codes; default 500 - from werkzeug.exceptions import HTTPException - if isinstance(e, HTTPException): - return e.code or 500 - if isinstance(e, (CommandUsageError, ValueError, KeyError)): - return 400 - if isinstance(e, FileNotFoundError): - return 404 - if isinstance(e, PermissionError): - return 403 - if isinstance(e, TimeoutError): - return 504 - return 500 - -def _execute_run_sync(group: str, subcmd: str, args: list, kwargs: dict) -> Tuple[Dict[str, Any], int]: - """ - Shared executor used by GET/POST /run. - Returns (payload_dict, http_status). - Captures Rich events even when errors occur. - """ - events: List[Dict[str, Any]] = [] - def on_event(ev: dict): events.append(ev) - - debug = os.getenv("OWILIX_DEBUG_ERRORS", "0") == "1" - - try: - kernel = AppKernel(on_event=on_event, input_provider=lambda _:"yes") - res = kernel.dispatch(group, subcmd, *args, **kwargs) - obj = _df_to_records(res.object) - return { - "success": True, - "msg": res.msg, - "object": obj, - "events": events, - }, 200 - - except Exception as e: - status = _exc_to_status(e) - payload: Dict[str, Any] = { - "success": False, - "msg": str(e), - "error": { - "type": e.__class__.__name__, - "message": str(e), - }, - "events": events, # whatever was written before the crash - } - if debug: - payload["error"]["traceback"] = traceback.format_exc() - return payload, status - -def _parse_get_run_params(req) -> tuple[str, str, list, dict]: - """ - Parse GET /run?group=...&subcmd=...&args=...&arg=...&kwargs=...&kw_x=... - Priority rules: - - group (required), subcmd (required) - - args: - 1) any repeated ?arg=... (order preserved) - 2) repeated ?args=... - 3) single ?args=... where value is JSON array OR comma-separated - - kwargs: - 1) ?kwargs= - 2) any query param starting with kw_ (e.g., kw_schema=owi ➜ {"schema":"owi"}) - 3) any leftover params not in {'group','subcmd','arg','args','kwargs'} - (useful for simple key=value pairs) - Values are run through _coerce_value(). - """ - group = req.args.get("group") - subcmd = req.args.get("subcmd") - if not group or not subcmd: - raise ValueError("Missing required query params 'group' and/or 'subcmd'.") - - # ---- args - args: list = [] - # 1) repeated ?arg=... - args.extend([_coerce_value(v) for v in req.args.getlist("arg")]) - - # 2) repeated ?args=... (if provided) - rep_args = req.args.getlist("args") - if rep_args: - # if only one and looks like JSON array or comma-list, handle below - if len(rep_args) > 1: - args.extend([_coerce_value(v) for v in rep_args]) - else: - single = rep_args[0] - if single.strip().startswith("["): - try: - arr = json.loads(single) - if isinstance(arr, list): - args.extend(arr) - except Exception: - # fallback to comma-split - args.extend([_coerce_value(x) for x in single.split(",") if x != ""]) - else: - # comma-separated - args.extend([_coerce_value(x) for x in single.split(",") if x != ""]) - - # ---- kwargs - kwargs: dict = {} - # 1) kwargs as JSON - if "kwargs" in req.args: - try: - kw_json = json.loads(req.args["kwargs"]) - if isinstance(kw_json, dict): - kwargs.update(kw_json) - except Exception: - # ignore malformed JSON; fall through to other styles - pass - - # 2) kw_* prefix - for k, v in req.args.items(): - if k.startswith("kw_"): - kwargs[k[3:]] = _coerce_value(v) - - # 3) leftover params - reserved = {"group", "subcmd", "arg", "args", "kwargs"} - for k, v in req.args.items(): - if k not in reserved and not k.startswith("kw_"): - kwargs[k] = _coerce_value(v) - - return group, subcmd, args, kwargs -# -------------------------------------------------------------------- - - -# === POST /run (refactored to use shared executor) =================== -@app.route("/run", methods=["POST"]) -def run_sync(): - payload = request.get_json(force=True) - group = payload["group"] - subcmd = payload["subcmd"] - args = payload.get("args", []) - kwargs = payload.get("kwargs", {}) - out, status = _execute_run_sync(group, subcmd, args, kwargs) - return jsonify(out), status - -@app.route("/run", methods=["GET"]) -def run_sync_get(): - # minimal GET parser; use your fuller parser if you already added one - group = request.args.get("group") - subcmd = request.args.get("subcmd") - if not group or not subcmd: - return jsonify({ - "success": False, - "msg": "Missing required query params 'group' and/or 'subcmd'.", - "error": {"type": "CommandUsageError", "message": "group/subcmd missing"}, - "events": [] - }), 400 - - # args: ?arg=... (repeatable) or ?args=[...] / comma-list - args = [ _coerce_value(v) for v in request.args.getlist("arg") ] - rep_args = request.args.getlist("args") - if rep_args: - if len(rep_args) == 1: - a = rep_args[0] - if a.strip().startswith("["): - try: - args.extend(json.loads(a)) - except Exception: - args.extend([_coerce_value(x) for x in a.split(",") if x != ""]) - else: - args.extend([_coerce_value(x) for x in a.split(",") if x != ""]) - else: - args.extend([_coerce_value(v) for v in rep_args]) - - # kwargs: ?kwargs={...} or kw_* params or leftover params - kwargs = {} - if "kwargs" in request.args: - try: - obj = json.loads(request.args["kwargs"]) - if isinstance(obj, dict): - kwargs.update(obj) - except Exception: - pass - for k, v in request.args.items(): - if k.startswith("kw_"): - kwargs[k[3:]] = _coerce_value(v) - for k, v in request.args.items(): - if k not in {"group","subcmd","arg","args","kwargs"} and not k.startswith("kw_"): - kwargs[k] = _coerce_value(v) - - out, status = _execute_run_sync(group, subcmd, args, kwargs) - return jsonify(out), status - - - -@app.route("/run/stream", methods=["POST"]) -def run_stream(): - """ - Run a command and stream Rich writes via Server-Sent Events (SSE). - Body JSON: { "group": "...", "subcmd": "...", "args": [], "kwargs": { ... } } - """ - payload = request.get_json(force=True) - group = payload["group"] - subcmd = payload["subcmd"] - args = payload.get("args", []) - kwargs = payload.get("kwargs", {}) - - q: "queue.Queue[Dict[str, Any]]" = queue.Queue() - - def on_event(ev: dict): - q.put(ev) - - def worker(): - try: - kernel = AppKernel(on_event=on_event, input_provider=lambda _:"yes") - res = kernel.dispatch(group, subcmd, *args, **kwargs) - q.put({"type": "result", "success": res.success, "msg": res.msg}) - except Exception as e: - q.put({"type": "error", "error": str(e)}) - finally: - q.put({"type": "done"}) - - threading.Thread(target=worker, daemon=True).start() - - def gen(): - yield _sse({"type": "start", "group": group, "subcmd": subcmd}) - while True: - ev = q.get() - yield _sse(ev) - if ev.get("type") == "done": - break - - return Response(gen(), mimetype="text/event-stream") - -@app.route("/run_async", methods=["POST"]) -def run_async(): - """ - Enqueue a long-running command. - - In-proc mode: runs in a thread pool inside this process. - - Celery mode: delegates to Celery worker and streams via Redis Pub/Sub. - Body JSON: { "group": "...", "subcmd": "...", "args": [], "kwargs": { ... } } - """ - payload = request.get_json(force=True) - group = payload["group"] - subcmd = payload["subcmd"] - args = payload.get("args", []) - kwargs = payload.get("kwargs", {}) - - if USE_INPROC: - task_id = JOBS.submit(group, subcmd, args, kwargs) - return jsonify({"task_id": task_id, "executor": "inproc"}) - else: - job = celery_app.send_task("owilix.run", args=[group, subcmd, args, kwargs], queue="owilix") - return jsonify({"task_id": job.id, "executor": "celery"}) - -@app.route("/tasks/", methods=["GET"]) -def task_status(task_id: str): - if USE_INPROC: - st = JOBS.status(task_id) - if not st: - return jsonify({"error": "unknown task"}), 404 - return jsonify(st) - else: - r = AsyncResult(task_id) - out = {"task_id": task_id, "state": r.state} - if r.ready(): - out["result"] = r.result - return jsonify(out) - -@app.route("/tasks//stream", methods=["GET"]) -def task_stream(task_id: str): - """ - Stream task output via SSE. - - In-proc: stream directly from the in-memory queue. - - Celery: relay Redis Pub/Sub messages from the worker. - """ - def sse(data: Dict[str, Any]) -> str: - return f"data: {json.dumps(data, ensure_ascii=False)}\n\n" - - if USE_INPROC: - def gen(): - yield sse({"type": "subscribed", "executor": "inproc"}) - for ev in JOBS.stream(task_id): - yield sse(ev) - return Response(gen(), mimetype="text/event-stream") - - # Celery/Redis branch - r = redis.from_url(BROKER_URL) - channel = f"{CHANNEL_PREFIX}:{task_id}" - pubsub = r.pubsub() - pubsub.subscribe(channel) - - def gen(): - yield sse({"type": "subscribed", "executor": "celery", "channel": channel}) - for msg in pubsub.listen(): - if msg["type"] != "message": - continue - try: - ev = json.loads(msg["data"]) - except Exception: - ev = {"type": "write", "chunk": msg["data"].decode("utf-8", "ignore")} - yield sse(ev) - if ev.get("type") == "done": - break - pubsub.close() - - return Response(gen(), mimetype="text/event-stream") - -@app.route("/tasks/health", methods=["GET"]) -def health(): - return jsonify({"ok": True, "mode": "inproc" if USE_INPROC else "celery"}) - -# ------------------------------------------------------------------- -# Entrypoint -# ------------------------------------------------------------------- -if __name__ == "__main__": - # For development; in production use gunicorn/uwsgi, e.g.: - # gunicorn -w 1 -k gevent --timeout 0 -b 0.0.0.0:8080 owilix_server.flask_app:app - port = int(os.getenv("PORT", "8080")) - app.run(host="0.0.0.0", port=port, threaded=True) diff --git a/owilix/app/inproc_jobs.py b/owilix/app/inproc_jobs.py deleted file mode 100644 index 47ae366..0000000 --- a/owilix/app/inproc_jobs.py +++ /dev/null @@ -1,70 +0,0 @@ -# owilix_server/inproc_jobs.py -from __future__ import annotations -import uuid, time, queue, threading -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass, field -from typing import Dict, Any -from .appkernel import AppKernel - -@dataclass -class Job: - id: str - q: "queue.Queue[dict]" = field(default_factory=queue.Queue) - state: str = "PENDING" - result: Dict[str, Any] | None = None - error: str | None = None - started_at: float = field(default_factory=time.time) - finished_at: float | None = None - -class InProcJobManager: - def __init__(self, max_workers=2): - self.exec = ThreadPoolExecutor(max_workers=max_workers) - self.jobs: Dict[str, Job] = {} - self.lock = threading.Lock() - - def submit(self, group: str, subcmd: str, args: list, kwargs: dict) -> str: - jid = str(uuid.uuid4()) - job = Job(id=jid) - with self.lock: - self.jobs[jid] = job - - def on_event(ev: dict): job.q.put(ev) - - def run(): - job.state = "STARTED" - job.q.put({"type": "start", "group": group, "subcmd": subcmd}) - try: - kernel = AppKernel(on_event=on_event, input_provider=lambda _:"yes") - res = kernel.dispatch(group, subcmd, *args, **kwargs) - job.result = {"success": res.success, "msg": res.msg} - job.state = "SUCCESS" if res.success else "FAILURE" - except Exception as e: - job.error = str(e) - job.state = "FAILURE" - job.q.put({"type":"error","error":job.error}) - finally: - job.finished_at = time.time() - job.q.put({"type":"done"}) - self.exec.submit(run) - return jid - - def status(self, jid: str): - with self.lock: - j = self.jobs.get(jid) - if not j: return None - out = {"task_id": j.id, "state": j.state} - if j.result: out["result"] = j.result - if j.error: out["error"] = j.error - return out - - def stream(self, jid: str): - with self.lock: - j = self.jobs.get(jid) - if not j: - yield {"type":"error","error":"unknown task"} - return - while True: - ev = j.q.get() - yield ev - if ev.get("type") == "done": - break diff --git a/owilix/cli/remote.py b/owilix/cli/remote.py index 2a1f840..fc4937a 100644 --- a/owilix/cli/remote.py +++ b/owilix/cli/remote.py @@ -24,6 +24,33 @@ app = typer.Typer( ) +def _format_search_date(row: dict) -> str: + """Format partition date fields from the remote index into YYYY-MM-DD.""" + year = row.get("year") + month = row.get("month") + day = row.get("day") + if year is None or month is None or day is None: + return "" + return f"{int(year):04d}-{int(month):02d}-{int(day):02d}" + + +def _prepare_search_records(rows: list[dict]) -> list[dict]: + """Normalize remote search rows for display and structured output.""" + records = [] + for row in rows: + records.append( + { + "score": f"{float(row['score']):.3f}", + "language": row.get("language", "") or "", + "collection": row.get("collection", "") or "", + "date": _format_search_date(row), + "id": row.get("id", "") or "", + "url": row.get("url", "") or "", + } + ) + return records + + def _format_size(size_bytes: int) -> str: """Format bytes as human-readable string.""" if size_bytes < 1024: @@ -162,6 +189,7 @@ def ls( def doctor( ctx: typer.Context, tokens: bool = typer.Option(False, "--tokens", "-t", help="Show session tokens and per-zone curl commands for external testing"), + verbose: bool = typer.Option(False, "--verbose", "-v", help="Show live probe progress and timing details"), json_file: Optional[str] = typer.Option(None, "--file", help="Write JSON output to file"), ): """ @@ -181,6 +209,7 @@ def doctor( as_json=cli_ctx.output_format in ("json", "jsonl"), json_file=json_file, show_tokens=tokens, + verbose=verbose, console=cli_ctx.console, ) @@ -192,6 +221,7 @@ def search( language: str = typer.Option("eng", "--language", "-L", help="Language filter (ISO 639-3: eng, deu, fra, ...)"), representation: str = typer.Option("main_content", "--repr", "-r", help="Index representation: main_content, title, description"), limit: int = typer.Option(10, "--limit", "-n", help="Maximum number of results"), + display: str = typer.Option("short", "--display", "-d", help="Table display mode: short or wide"), conjunctive: bool = typer.Option(False, "--all", "-a", help="Require ALL terms (AND); default is ANY term (OR)"), explain: bool = typer.Option(False, "--explain", help="Print the generated SQL query"), fetch: bool = typer.Option(False, "--fetch", "-f", help="Fetch full document records from OWI datasets"), @@ -264,31 +294,26 @@ def search( ) if result.success and result.json: + records = _prepare_search_records(result.json) + if cli_ctx.output_format in ("json", "jsonl") or output: with OutputWriter(cli_ctx) as writer: - writer.write_records(result.json) + writer.write_records(records) else: - # Table output - from rich.table import Table - from rich import box - table = Table( - title=f"Search results ({result.msg})", - show_header=True, - box=box.SIMPLE, - ) - table.add_column("Score", justify="right", style="bold") - table.add_column("Language", style="dim") - table.add_column("Collection", style="dim", max_width=36) - table.add_column("URL", style="cyan", no_wrap=False) - - for row in result.json: - table.add_row( - f"{row['score']:.3f}", - row["language"], - row["collection"], - row["url"] or "?", + if display not in ("short", "wide"): + cli_ctx.console.print(f"[red]Invalid display mode: {display}[/red]") + raise typer.Exit(code=1) + + columns = ["score", "date", "url"] if display == "short" else [ + "score", "language", "date", "collection", "id", "url" + ] + + with OutputWriter(cli_ctx) as writer: + writer.write_records( + records, + title=f"Search results ({result.msg})", + columns=columns, ) - cli_ctx.console.print(table) elif not result.success: cli_ctx.console.print(f"[red]Search failed: {result.msg}[/red]") raise typer.Exit(code=result.exit_code) diff --git a/owilix/core/repository/ddi.py b/owilix/core/repository/ddi.py index 1e8a2b8..3c23d9a 100644 --- a/owilix/core/repository/ddi.py +++ b/owilix/core/repository/ddi.py @@ -124,12 +124,12 @@ class OWILexisDatasetAPI(Datasets): url = url + f"&dataset_type={dataset_type}" else: url = url + f"?dataset_type={dataset_type}" -# - #if project is not None: - # if "=" in url: - # url = url + f"&project={project}" - # else: - # url = url + f"?project={project}" + + if project is not None: + if "=" in url: + url = url + f"&project={project}" + else: + url = url + f"?project={project}" if access is not None: if "=" in url: diff --git a/owilix/core/repository/lexis.py b/owilix/core/repository/lexis.py index bcb283c..19e37b1 100644 --- a/owilix/core/repository/lexis.py +++ b/owilix/core/repository/lexis.py @@ -277,7 +277,7 @@ class LexisRepository(AbstractRepository): except Exception as e: return {"alive": False, "detail": f"{e.__class__.__name__}: {e}"} - def _probe_ddi_per_access(self) -> tuple: + def _probe_ddi_per_access(self, progress_cb=None) -> tuple: """Query DDI for each access level. Returns: @@ -288,6 +288,9 @@ class LexisRepository(AbstractRepository): access_info = {} all_records = [] for access in ("public", "project", "user"): + started = time.time() + if progress_cb: + progress_cb(f"DDI query start: access={access}") try: records = self.ddi_api.get_all_datasets( access=access, @@ -310,11 +313,21 @@ class LexisRepository(AbstractRepository): "datasets": len(records), "zones": sorted(zones_seen), } + if progress_cb: + progress_cb( + f"DDI query done: access={access} datasets={len(records)} " + f"zones={sorted(zones_seen)} elapsed={time.time() - started:.1f}s" + ) except Exception as e: access_info[access] = {"ok": False, "datasets": 0, "zones": [], "error": str(e)} + if progress_cb: + progress_cb( + f"DDI query failed: access={access} error={e} " + f"elapsed={time.time() - started:.1f}s" + ) return access_info, all_records - def _probe_zone_irods(self, zone_name: str, access_info: dict, all_datasets: list) -> dict: + def _probe_zone_irods(self, zone_name: str, access_info: dict, all_datasets: list, progress_cb=None) -> dict: """Try an iRODS listing for each access level that DDI reported for this zone. Instead of listing the access-level root (/{zone}/public/) — which @@ -328,9 +341,12 @@ class LexisRepository(AbstractRepository): """ access_results = {} for access in ("public", "project", "user"): + started = time.time() ai = access_info.get(access, {}) if zone_name not in ai.get("zones", []): access_results[access] = {"result": "no datasets", "probe_path": None} + if progress_cb: + progress_cb(f"iRODS probe skip: zone={zone_name} access={access} no datasets") continue # Find a concrete dataset path in this zone + access level to probe @@ -349,13 +365,30 @@ class LexisRepository(AbstractRepository): "result": f"{ai.get('datasets', '?')} datasets (no path to probe)", "probe_path": None, } + if progress_cb: + progress_cb( + f"iRODS probe no-path: zone={zone_name} access={access} " + f"elapsed={time.time() - started:.1f}s" + ) continue try: + if progress_cb: + progress_cb(f"iRODS probe start: zone={zone_name} access={access} path={probe_path}") self.fs.ls(probe_path) access_results[access] = {"result": "ok", "probe_path": probe_path} + if progress_cb: + progress_cb( + f"iRODS probe done: zone={zone_name} access={access} result=ok " + f"elapsed={time.time() - started:.1f}s" + ) except FileNotFoundError: access_results[access] = {"result": "path not found", "probe_path": probe_path} + if progress_cb: + progress_cb( + f"iRODS probe done: zone={zone_name} access={access} result=path not found " + f"elapsed={time.time() - started:.1f}s" + ) except Exception as e: msg = str(e) if "Expecting value" in msg: @@ -363,10 +396,16 @@ class LexisRepository(AbstractRepository): else: result = msg access_results[access] = {"result": result, "probe_path": probe_path} + if progress_cb: + progress_cb( + f"iRODS probe failed: zone={zone_name} access={access} result={result} " + f"elapsed={time.time() - started:.1f}s" + ) return access_results - def status(self) -> dict: + def status(self, progress_cb=None) -> dict: """Check repository connectivity: DDI API, per-zone HTTP, per-zone iRODS.""" + timings = {"ddi": {}, "zones": {}} # --- 1. DDI metadata API ----------------------------------------------- ddi_ok = False @@ -374,10 +413,15 @@ class LexisRepository(AbstractRepository): access_info: dict = {} all_ddi_records: list = [] try: - access_info, all_ddi_records = self._probe_ddi_per_access() + if progress_cb: + progress_cb("status start: probing DDI") + ddi_started = time.time() + access_info, all_ddi_records = self._probe_ddi_per_access(progress_cb=progress_cb) + timings["ddi"]["elapsed"] = time.time() - ddi_started ddi_ok = any(a["ok"] for a in access_info.values()) ddi_msg = "connected" if ddi_ok else "DDI queries failed" except Exception as e: + timings["ddi"]["elapsed"] = timings["ddi"].get("elapsed", 0.0) ddi_msg = f"DDI connection failed: {e}" # --- 2. Discover all zones to check ------------------------------------ @@ -392,14 +436,28 @@ class LexisRepository(AbstractRepository): # --- 3. Per-zone checks ------------------------------------------------ zone_status = {} for zone_name in sorted(zones_to_check): + zone_started = time.time() + if progress_cb: + progress_cb(f"zone start: {zone_name}") api_url = self._get_zone_irods_url(zone_name) + http_started = time.time() + if progress_cb: + progress_cb(f"HTTP /info start: zone={zone_name} url={api_url}") http_probe = self._probe_http_endpoint(api_url) + http_elapsed = time.time() - http_started + timings["zones"].setdefault(zone_name, {})["http_info_elapsed"] = http_elapsed + if progress_cb: + progress_cb( + f"HTTP /info done: zone={zone_name} alive={http_probe['alive']} " + f"detail={http_probe['detail']} elapsed={http_elapsed:.1f}s" + ) - # How many DDI datasets live in this zone? + # Count concrete DDI records that map to this zone. + # The access-level ``zones`` summary only tells us whether a zone + # is present at all for a given access level, not how many records + # belong to that zone. zone_dataset_count = sum( - ai.get("datasets", 0) - for ai in access_info.values() - if zone_name in ai.get("zones", []) + 1 for ds_rec in all_ddi_records if ds_rec.get("zone") == zone_name ) # Extract /info metadata for the zone status dict @@ -417,10 +475,15 @@ class LexisRepository(AbstractRepository): "http_info": http_info, "access": {}, } + timings["zones"][zone_name]["elapsed"] = time.time() - zone_started continue # Endpoint alive — probe iRODS per access level - irods_results = self._probe_zone_irods(zone_name, access_info, all_ddi_records) + irods_started = time.time() + irods_results = self._probe_zone_irods( + zone_name, access_info, all_ddi_records, progress_cb=progress_cb + ) + timings["zones"][zone_name]["irods_elapsed"] = time.time() - irods_started zone_status[zone_name] = { "status": True, @@ -430,6 +493,11 @@ class LexisRepository(AbstractRepository): "http_info": http_info, "access": irods_results, } + timings["zones"][zone_name]["elapsed"] = time.time() - zone_started + if progress_cb: + progress_cb( + f"zone done: {zone_name} elapsed={timings['zones'][zone_name]['elapsed']:.1f}s" + ) # --- 4. Overall summary ------------------------------------------------ all_zones_ok = all(z["status"] for z in zone_status.values()) if zone_status else True @@ -444,6 +512,7 @@ class LexisRepository(AbstractRepository): "message": ddi_msg, "ddi": access_info, "zones": zone_status, + "timings": timings, }) def exists(self, dataset: Dataset) -> bool: diff --git a/owilix/core/tasks/remote.py b/owilix/core/tasks/remote.py index d4f8132..e123f5c 100644 --- a/owilix/core/tasks/remote.py +++ b/owilix/core/tasks/remote.py @@ -313,6 +313,7 @@ def remote_doctor( as_json: bool = False, json_file: Optional[str] = None, show_tokens: bool = False, + verbose: bool = False, console: Optional[Console] = None ) -> CommandResult: """ @@ -332,7 +333,12 @@ def remote_doctor( for name, repo in (repos or {}).items(): try: - status = repo.status() + def _progress(msg: str): + if not as_json: + console.print(f"[dim]{name}: {msg}[/dim]") + + progress_cb = _progress if verbose and not as_json else None + status = repo.status(progress_cb=progress_cb) results[name] = status if not as_json: ok = status.get("status", False) @@ -356,6 +362,7 @@ def remote_doctor( # Per-zone detailed status zone_status = status.get("zones", {}) + timing_info = status.get("timings", {}) if zone_status: for zone_name, zs in zone_status.items(): z_ok = zs.get("status", False) @@ -374,6 +381,17 @@ def remote_doctor( info_extra += f" oidc={'yes' if http_info['openid_connect_enabled'] else 'no'}" console.print(f"\t[{z_col}]zone {zone_name}: {z_msg}[/{z_col}] [dim]({z_url}){info_extra}[/dim]") console.print(f"\t datasets in DDI for this zone: {ds_count}") + zone_timing = timing_info.get("zones", {}).get(zone_name, {}) + if verbose and zone_timing: + parts = [] + if "http_info_elapsed" in zone_timing: + parts.append(f"http={zone_timing['http_info_elapsed']:.1f}s") + if "irods_elapsed" in zone_timing: + parts.append(f"irods={zone_timing['irods_elapsed']:.1f}s") + if "elapsed" in zone_timing: + parts.append(f"total={zone_timing['elapsed']:.1f}s") + if parts: + console.print(f"\t [dim]timing: {', '.join(parts)}[/dim]") access_results = zs.get("access", {}) if access_results: for access, ar in access_results.items(): @@ -382,6 +400,9 @@ def remote_doctor( a_col = "green" if result == "ok" else ("dim" if result == "no datasets" else "red") path_hint = f" [dim]tested: {probe_path}[/dim]" if probe_path else "" console.print(f"\t [{a_col}]{access}: {result}[/{a_col}]{path_hint}") + ddi_timing = timing_info.get("ddi", {}) + if verbose and ddi_timing.get("elapsed") is not None: + console.print(f"\t[dim]DDI timing: {ddi_timing['elapsed']:.1f}s[/dim]") except Exception as e: err = f"{e.__class__.__name__}: {e}" if not as_json: diff --git a/owilix/core/tasks/search.py b/owilix/core/tasks/search.py index dc128f6..50c4701 100644 --- a/owilix/core/tasks/search.py +++ b/owilix/core/tasks/search.py @@ -18,7 +18,7 @@ Reference: import logging import time from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set import duckdb from rich.console import Console @@ -448,6 +448,111 @@ def resolve_search_to_files( ) +def _group_candidate_files_by_fs(files_by_dataset: Dict) -> Dict[Any, List[str]]: + """Group candidate parquet files by filesystem.""" + from collections import defaultdict + + fs_to_files: Dict[Any, List[str]] = defaultdict(list) + for dataset, file_list in files_by_dataset.items(): + fs = dataset.repository.fs + for file_path, _dataset_path, _dataset_id in file_list: + fs_to_files[fs].append(file_path) + return fs_to_files + + +def _prune_files_by_metadata( + fs_to_files: Dict[Any, List[str]], + target_ids: List[str], +) -> Optional[Set[str]]: + """Use parquet min/max stats for `id` to prune candidate files. + + Returns: + Set of file paths that may contain target IDs, or None if the metadata + route fails and the caller should fall back to the id-only prescan. + """ + hit_files: Set[str] = set() + target_ids_sorted = sorted(target_ids) + + for fs, file_paths in fs_to_files.items(): + if not file_paths: + continue + + con = duckdb.connect() + con.register_filesystem(fs) + file_list_sql = ", ".join(f"'{f}'" for f in file_paths) + metadata_sql = f""" + SELECT + file_name, + row_group_id, + path_in_schema, + stats_min_value, + stats_max_value + FROM parquet_metadata([{file_list_sql}]) + WHERE path_in_schema = 'id' + """ + try: + rows = con.execute(metadata_sql).fetchall() + except Exception as e: + logger.warning(f"Metadata pruning failed, falling back to id-only prescan: {e}") + return None + finally: + con.close() + + if not rows: + logger.warning("Metadata pruning returned no id statistics; falling back to id-only prescan") + return None + + per_file_match: Dict[str, bool] = {} + for file_name, _row_group_id, _path_in_schema, min_value, max_value in rows: + if min_value is None or max_value is None: + per_file_match[file_name] = True + continue + + for target_id in target_ids_sorted: + if min_value <= target_id <= max_value: + per_file_match[file_name] = True + break + else: + per_file_match.setdefault(file_name, False) + + hit_files.update(path for path, matched in per_file_match.items() if matched) + + return hit_files + + +def _select_files_by_prescan( + fs_to_files: Dict[Any, List[str]], + target_ids: List[str], +) -> Set[str]: + """Fallback id-only prescan using DuckDB parquet reads.""" + hit_files: Set[str] = set() + + id_placeholders = ", ".join(f"'{i}'" for i in target_ids) + for fs, file_paths in fs_to_files.items(): + if not file_paths: + continue + + con = duckdb.connect() + con.register_filesystem(fs) + + file_list_sql = ", ".join(f"'{f}'" for f in file_paths) + prescan_sql = ( + f"SELECT DISTINCT filename " + f"FROM read_parquet([{file_list_sql}], filename=true) " + f"WHERE id IN ({id_placeholders})" + ) + try: + rows = con.execute(prescan_sql).fetchall() + hit_files.update(row[0] for row in rows) + except Exception as e: + logger.warning(f"Pre-scan failed, keeping all files: {e}") + hit_files.update(file_paths) + finally: + con.close() + + return hit_files + + def _select_files_by_id( files_by_dataset: Dict, target_ids: List[str], @@ -455,21 +560,16 @@ def _select_files_by_id( ) -> Dict: """Select only the parquet files that contain target document IDs. - DuckDB reads just the id column from each parquet file (~446 KB instead - of ~19 MB per file). Files that contain none of the target IDs are - dropped so the main query only touches the handful that matter. + First tries metadata-based pruning via parquet min/max statistics on the + sorted `id` column. If that path is unavailable, falls back to the + original id-only prescan. """ - from collections import defaultdict from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn - # Group files by filesystem (same logic as group_all_files_by_fs) - fs_to_files: Dict[Any, List[str]] = defaultdict(list) - for dataset, file_list in files_by_dataset.items(): - fs = dataset.repository.fs - for file_path, _dataset_path, _dataset_id in file_list: - fs_to_files[fs].append(file_path) + fs_to_files = _group_candidate_files_by_fs(files_by_dataset) - hit_files: set = set() + hit_files: Set[str] = set() + selection_method = "metadata pruning" with Progress( SpinnerColumn(), @@ -482,27 +582,12 @@ def _select_files_by_id( progress.add_task( f"Selecting files by document ID ({total} candidates)...", total=None ) - - id_placeholders = ", ".join(f"'{i}'" for i in target_ids) - - for fs, file_paths in fs_to_files.items(): - con = duckdb.connect() - con.register_filesystem(fs) - - file_list_sql = ", ".join(f"'{f}'" for f in file_paths) - prescan_sql = ( - f"SELECT DISTINCT filename " - f"FROM read_parquet([{file_list_sql}], filename=true) " - f"WHERE id IN ({id_placeholders})" - ) - try: - rows = con.execute(prescan_sql).fetchall() - hit_files.update(row[0] for row in rows) - except Exception as e: - logger.warning(f"Pre-scan failed, keeping all files: {e}") - hit_files.update(file_paths) - finally: - con.close() + metadata_hits = _prune_files_by_metadata(fs_to_files, target_ids) + if metadata_hits is None: + selection_method = "id-only scan" + hit_files = _select_files_by_prescan(fs_to_files, target_ids) + else: + hit_files = metadata_hits # Filter files_by_dataset to keep only hit files filtered: Dict = {} @@ -513,7 +598,7 @@ def _select_files_by_id( original = sum(len(f) for f in files_by_dataset.values()) remaining = sum(len(f) for f in filtered.values()) - console.print(f"File selection: {original} → {remaining} files (id-only scan)") + console.print(f"File selection: {original} → {remaining} files ({selection_method})") return filtered diff --git a/pyproject.toml b/pyproject.toml index 5b17089..d7beee4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "owilix" -version = "5.2.0" +version = "5.3.0" description = "OWILIX - the Command Line Interface for slicing and consuming the Open Web Index. " readme = "Readme.md" license = { text = "MIT" } @@ -109,12 +109,6 @@ village = [ "pyiceberg[duckdb,pandas,sql-sqlite,s3fs]>=0.9.1", "pyiceberg-core>=0.4.0", ] -server = [ - "flask>=3.1.2", - "celery<5.5.3", - "redis>=6.4.0", - "gunicorn>=23.0.0", -] # (Optional) per-group settings live under [tool.uv.dependency-groups] # Example if you ever need a different Python for a group: diff --git a/tests/owilix/cli/test_remote_search_cli.py b/tests/owilix/cli/test_remote_search_cli.py new file mode 100644 index 0000000..3c132d5 --- /dev/null +++ b/tests/owilix/cli/test_remote_search_cli.py @@ -0,0 +1,106 @@ +from typer.testing import CliRunner +from unittest.mock import patch + +from owilix.cli import app +from owilix.core.types import CommandResult + + +runner = CliRunner() + + +def _mock_search_result() -> CommandResult: + return CommandResult( + success=True, + json=[ + { + "collection": "coll-alpha", + "language": "eng", + "year": 2026, + "month": 4, + "day": 9, + "id": "abc123", + "url": "https://example.org/a", + "score": 3.5, + }, + { + "collection": "coll-beta", + "language": "deu", + "year": 2026, + "month": 4, + "day": 8, + "id": "def456", + "url": "https://example.org/b", + "score": 2.25, + }, + ], + msg="2 results in 0.2s", + command="remote search", + ) + + +def test_remote_search_short_display(tmp_path): + target = tmp_path / "owi" + target.mkdir() + + with patch("owilix.core.tasks.search.remote_search", return_value=_mock_search_result()): + result = runner.invoke( + app, + ["--target", str(target), "remote", "search", "open web search", "--display", "short"], + ) + + assert result.exit_code == 0 + assert "score" in result.stdout.lower() + assert "date" in result.stdout.lower() + assert "url" in result.stdout.lower() + assert "collection" not in result.stdout.lower() + assert "2026-04-09" in result.stdout + assert "https://example.org/a" in result.stdout + + +def test_remote_search_wide_display(tmp_path): + target = tmp_path / "owi" + target.mkdir() + + with patch("owilix.core.tasks.search.remote_search", return_value=_mock_search_result()): + result = runner.invoke( + app, + ["--target", str(target), "remote", "search", "open web search", "--display", "wide"], + ) + + assert result.exit_code == 0 + output = result.stdout.lower() + assert "language" in output + assert "collection" in output + assert "id" in output + assert "coll-alpha" in result.stdout + assert "abc123" in result.stdout + + +def test_remote_search_json_output_includes_extra_fields(tmp_path): + target = tmp_path / "owi" + target.mkdir() + + with patch("owilix.core.tasks.search.remote_search", return_value=_mock_search_result()): + result = runner.invoke( + app, + ["--target", str(target), "--format", "json", "remote", "search", "open web search"], + ) + + assert result.exit_code == 0 + assert '"collection": "coll-alpha"' in result.stdout + assert '"date": "2026-04-09"' in result.stdout + assert '"id": "abc123"' in result.stdout + + +def test_remote_search_invalid_display(tmp_path): + target = tmp_path / "owi" + target.mkdir() + + with patch("owilix.core.tasks.search.remote_search", return_value=_mock_search_result()): + result = runner.invoke( + app, + ["--target", str(target), "remote", "search", "open web search", "--display", "long"], + ) + + assert result.exit_code == 1 + assert "Invalid display mode" in result.stdout diff --git a/tests/owilix/cli/test_smoke.py b/tests/owilix/cli/test_smoke.py index b92dedc..271b1f0 100644 --- a/tests/owilix/cli/test_smoke.py +++ b/tests/owilix/cli/test_smoke.py @@ -120,3 +120,25 @@ class TestCLIIntegration: code, out, err = run_owi("remote", "ls", "all:latest", timeout=120) assert code == 0, f"Failed with stderr: {err}" assert "Fetching datasets" in out or "📦" in out + + @pytest.mark.integration + def test_remote_search(self): + """Test remote full-text search against the DuckLake index.""" + code, out, err = run_owi("remote", "search", "open web search", "--limit", "3", timeout=180) + assert code == 0, f"Failed with stderr: {err}" + assert "Search results" in out + assert "url" in out.lower() + + @pytest.mark.integration_heavy + def test_query_less_with_search(self): + """Test query less prefiltered by remote-index search.""" + code, out, err = run_owi( + "query", "less", + "--remote", "all", + "--search", "open web search", + "--search-limit", "3", + "--limit", "3", + timeout=420, + ) + assert code == 0, f"Failed with stderr: {err}" + assert "url" in out.lower() or "title" in out.lower() diff --git a/tests/owilix/core/db/test_executors.py b/tests/owilix/core/db/test_executors.py index c149792..559ccd4 100644 --- a/tests/owilix/core/db/test_executors.py +++ b/tests/owilix/core/db/test_executors.py @@ -101,7 +101,7 @@ class TestOWIDuckDBSelectExecutor: ) assert executor.pq_batch_size == 10 assert executor.prefetch == 2 - executor.close() + asyncio.run(executor.close()) def test_generate_ordered_tasks(self, mock_fs, sample_sql): files = [(f"/path/file{i}.parquet", "") for i in range(25)] @@ -114,7 +114,7 @@ class TestOWIDuckDBSelectExecutor: tasks = executor.generate_ordered_tasks() # 25 files / 10 per batch = 3 batches assert len(tasks) == 3 - executor.close() + asyncio.run(executor.close()) def test_default_batch_size_is_10(self, mock_fs, sample_sql): pq_files = {mock_fs: [("/path/file.parquet", "")]} @@ -123,7 +123,7 @@ class TestOWIDuckDBSelectExecutor: owilix_sql=sample_sql ) assert executor.pq_batch_size == 10 - executor.close() + asyncio.run(executor.close()) class TestOWIDuckDBAsyncExecutor: diff --git a/tests/owilix/core/repository/test_ddi.py b/tests/owilix/core/repository/test_ddi.py new file mode 100644 index 0000000..a334afc --- /dev/null +++ b/tests/owilix/core/repository/test_ddi.py @@ -0,0 +1,36 @@ +from unittest.mock import MagicMock + +from owilix.core.repository.ddi import OWILexisDatasetAPI + + +def test_get_all_datasets_passes_project_filter_to_ddi(): + session = MagicMock() + session.ddi_path = "https://api.lexis.tech/api/ddiapi/v2" + session.get_api_headers.return_value = {"Authorization": "Bearer test"} + session.make_request.return_value = { + "total": 0, + "datasets": [ + { + "lexis": { + "dataset_id": "ds-1", + "project_shortname": "openwebsearch", + } + } + ], + } + + api = OWILexisDatasetAPI(session=session, suppress_print=True) + + records = api.get_all_datasets( + access="project", + project="openwebsearch", + content_as_pandas=False, + ) + + assert len(records) == 1 + method = session.make_request.call_args.args[0] + url = session.make_request.call_args.args[1] + assert method == "post" + assert "access=project" in url + assert "project=openwebsearch" in url + assert "start=0" in url diff --git a/tests/owilix/core/repository/test_lexis_doctor.py b/tests/owilix/core/repository/test_lexis_doctor.py index 8f7b795..7184998 100644 --- a/tests/owilix/core/repository/test_lexis_doctor.py +++ b/tests/owilix/core/repository/test_lexis_doctor.py @@ -234,3 +234,38 @@ class TestStatusInfoPropagation: zone_info = result["zones"]["deadzone"] assert zone_info["status"] is False assert zone_info["http_info"] == {} + + @patch.object(LexisRepository, "_probe_zone_irods", return_value={}) + @patch.object(LexisRepository, "_probe_ddi_per_access") + @patch.object(LexisRepository, "_probe_http_endpoint") + @patch.object(LexisRepository, "_get_zone_irods_url", side_effect=lambda zone: f"https://{zone}.example/api") + @patch.object(LexisRepository, "_add_details_to_status", side_effect=lambda d: d) + def test_zone_dataset_counts_use_concrete_records( + self, mock_add, mock_url, mock_probe, mock_ddi, mock_irods + ): + mock_ddi.return_value = ( + { + "public": {"ok": True, "datasets": 3, "zones": ["IT4ILexisV2", "OWILRZZONE"]}, + "project": {"ok": True, "datasets": 1, "zones": ["IT4ILexisV2", "OWILRZZONE"]}, + "user": {"ok": True, "datasets": 0, "zones": []}, + }, + [ + {"zone": "IT4ILexisV2", "access": "public", "absolute_path": "/IT4ILexisV2/public/a"}, + {"zone": "IT4ILexisV2", "access": "public", "absolute_path": "/IT4ILexisV2/public/b"}, + {"zone": "OWILRZZONE", "access": "public", "absolute_path": "/OWILRZZONE/public/c"}, + {"zone": "OWILRZZONE", "access": "project", "absolute_path": "/OWILRZZONE/project/d"}, + ], + ) + mock_probe.return_value = { + "alive": True, + "detail": "HTTP 200", + "api_version": "0.6.0", + "zone": "ignored", + "openid_connect_enabled": True, + } + + repo = _make_repo(zone="IT4ILexisV2", zones={"OWILRZZONE": {"api_url": "https://lrz.example/api"}}) + result = repo.status() + + assert result["zones"]["IT4ILexisV2"]["datasets_in_ddi"] == 2 + assert result["zones"]["OWILRZZONE"]["datasets_in_ddi"] == 2 diff --git a/tests/owilix/core/tasks/test_search.py b/tests/owilix/core/tasks/test_search.py index fbf920f..ab2afec 100644 --- a/tests/owilix/core/tasks/test_search.py +++ b/tests/owilix/core/tasks/test_search.py @@ -11,8 +11,11 @@ from dataclasses import dataclass from owilix.core.tasks.search import ( _build_search_sql, + _group_candidate_files_by_fs, remote_search, resolve_search_to_files, + _prune_files_by_metadata, + _select_files_by_id, _lookup_dataset, SearchResult, CATALOG, @@ -211,3 +214,93 @@ class TestLookupDataset: result = _lookup_dataset(manager, "coll-uuid-123") assert result is None + + +class TestFileSelection: + """Tests for metadata-based file pruning and fallback selection.""" + + def _make_files_by_dataset(self): + fs = MagicMock() + dataset = MagicMock() + dataset.repository.fs = fs + return ( + fs, + dataset, + { + dataset: [ + ("s3://bucket/a.parquet", "s3://bucket/dataset", "ds1"), + ("s3://bucket/b.parquet", "s3://bucket/dataset", "ds1"), + ] + }, + ) + + def test_group_candidate_files_by_fs(self): + fs, _dataset, files_by_dataset = self._make_files_by_dataset() + grouped = _group_candidate_files_by_fs(files_by_dataset) + assert grouped[fs] == ["s3://bucket/a.parquet", "s3://bucket/b.parquet"] + + @patch("owilix.core.tasks.search.duckdb.connect") + def test_prune_files_by_metadata(self, mock_connect): + fs, _dataset, files_by_dataset = self._make_files_by_dataset() + grouped = _group_candidate_files_by_fs(files_by_dataset) + + con = MagicMock() + mock_connect.return_value = con + con.execute.return_value.fetchall.return_value = [ + ("s3://bucket/a.parquet", 0, "id", "aaa", "azz"), + ("s3://bucket/b.parquet", 0, "id", "zzz", "zzz"), + ] + + hit_files = _prune_files_by_metadata(grouped, ["abc123"]) + + assert hit_files == {"s3://bucket/a.parquet"} + con.register_filesystem.assert_called_once_with(fs) + + @patch("owilix.core.tasks.search.duckdb.connect") + def test_prune_files_by_metadata_fallback_on_missing_stats(self, mock_connect): + fs, _dataset, files_by_dataset = self._make_files_by_dataset() + grouped = _group_candidate_files_by_fs(files_by_dataset) + + con = MagicMock() + mock_connect.return_value = con + con.execute.return_value.fetchall.return_value = [ + ("s3://bucket/a.parquet", 0, "id", None, None), + ("s3://bucket/b.parquet", 0, "id", "zzz", "zzz"), + ] + + hit_files = _prune_files_by_metadata(grouped, ["abc123"]) + + assert hit_files == {"s3://bucket/a.parquet"} + + @patch("owilix.core.tasks.search._select_files_by_prescan") + @patch("owilix.core.tasks.search._prune_files_by_metadata") + def test_select_files_by_id_uses_metadata_when_available(self, mock_metadata, mock_prescan): + _fs, dataset, files_by_dataset = self._make_files_by_dataset() + mock_metadata.return_value = {"s3://bucket/a.parquet"} + console = MagicMock() + + filtered = _select_files_by_id(files_by_dataset, ["abc123"], console) + + assert filtered == { + dataset: [("s3://bucket/a.parquet", "s3://bucket/dataset", "ds1")] + } + mock_prescan.assert_not_called() + printed = " ".join(str(c) for c in console.print.call_args_list) + assert "metadata pruning" in printed + + @patch("owilix.core.tasks.search._select_files_by_prescan") + @patch("owilix.core.tasks.search._prune_files_by_metadata") + def test_select_files_by_id_falls_back_to_prescan(self, mock_metadata, mock_prescan): + _fs, dataset, files_by_dataset = self._make_files_by_dataset() + mock_metadata.return_value = None + mock_prescan.return_value = {"s3://bucket/b.parquet"} + console = MagicMock() + + filtered = _select_files_by_id(files_by_dataset, ["abc123"], console) + + assert filtered == { + dataset: [("s3://bucket/b.parquet", "s3://bucket/dataset", "ds1")] + } + mock_prescan.assert_called_once() + printed = " ".join(str(c) for c in console.print.call_args_list) + assert "id-only scan" in printed diff --git a/tests/scripts/search_metadata_pruning_benchmark.py b/tests/scripts/search_metadata_pruning_benchmark.py new file mode 100644 index 0000000..e55bd8d --- /dev/null +++ b/tests/scripts/search_metadata_pruning_benchmark.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +""" +Benchmark prototype for metadata-based pruning in `query less --search`. + +Compares the current id-only prescan used by `_select_files_by_id()` against a +metadata-only pruning strategy based on Parquet row-group min/max statistics +for the `id` column. + +Usage: + uv run python tests/scripts/search_metadata_pruning_benchmark.py --terms "open web search" + uv run python tests/scripts/search_metadata_pruning_benchmark.py --terms "graz" --language deu --limit 3 +""" + +from __future__ import annotations + +import argparse +import time +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed + +import duckdb + +from owilix.core import OWIlixManager +from owilix.core.tasks.search import _build_search_sql, _connect_remote_index, _lookup_dataset + + +def _collect_candidate_files( + manager: OWIlixManager, + terms: str, + language: str, + representation: str, + limit: int, + conjunctive: bool, +): + term_list = terms.strip().split() + con = _connect_remote_index() + sql = _build_search_sql(term_list, language, representation, limit, conjunctive) + t0 = time.time() + rows = con.execute(sql).fetchall() + search_time = time.time() - t0 + + columns = ["collection", "language", "year", "month", "day", "id", "url", "score"] + results = [dict(zip(columns, row)) for row in rows] + target_ids = [r["id"] for r in results] + + collections = {} + for r in results: + coll = r["collection"] + collections.setdefault(coll, set()).add( + (int(r["year"]), int(r["month"]), int(r["day"]), r["language"]) + ) + + resolved = {} + with ThreadPoolExecutor(max_workers=4) as executor: + futures = { + executor.submit(_lookup_dataset, manager, coll_id): coll_id + for coll_id in collections + } + for future in as_completed(futures): + resolved[futures[future]] = future.result() + + accessible = {k: v for k, v in resolved.items() if v is not None} + + listing_tasks = [] + for coll_id, dataset in accessible.items(): + for year, month, day, lang in collections[coll_id]: + glob_pattern = f"year={year}/month={month}/day={day}/language={lang}/*.parquet" + listing_tasks.append((dataset, glob_pattern)) + + files_by_dataset = {} + + def _list_partition(task): + dataset, glob_pattern = task + matched = dataset.repository.files(dataset, glob_pattern) + return dataset, [ + ( + dataset.repository.fs.unstrip_protocol(f), + dataset.repository.fs.unstrip_protocol(dataset.path), + dataset.metadata.id, + ) + for f in matched + ] + + with ThreadPoolExecutor(max_workers=4) as executor: + futures = [executor.submit(_list_partition, task) for task in listing_tasks] + for future in as_completed(futures): + dataset, file_list = future.result() + if file_list: + files_by_dataset.setdefault(dataset, []).extend(file_list) + + return { + "search_time": search_time, + "results": results, + "target_ids": target_ids, + "files_by_dataset": files_by_dataset, + "sql": sql, + } + + +def _group_by_fs(files_by_dataset): + fs_to_files = defaultdict(list) + for dataset, file_list in files_by_dataset.items(): + fs = dataset.repository.fs + for file_path, _dataset_path, _dataset_id in file_list: + fs_to_files[fs].append(file_path) + return fs_to_files + + +def benchmark_id_prescan(fs_to_files, target_ids): + hit_files = set() + id_placeholders = ", ".join(f"'{i}'" for i in target_ids) + + t0 = time.time() + for fs, file_paths in fs_to_files.items(): + con = duckdb.connect() + con.register_filesystem(fs) + file_list_sql = ", ".join(f"'{f}'" for f in file_paths) + prescan_sql = ( + f"SELECT DISTINCT filename " + f"FROM read_parquet([{file_list_sql}], filename=true) " + f"WHERE id IN ({id_placeholders})" + ) + rows = con.execute(prescan_sql).fetchall() + hit_files.update(row[0] for row in rows) + con.close() + + return { + "elapsed": time.time() - t0, + "hit_files": hit_files, + } + + +def benchmark_metadata_pruning(fs_to_files, target_ids): + hit_files = set() + missing_stats = 0 + checked_rowgroups = 0 + + target_ids_sorted = sorted(target_ids) + t0 = time.time() + + for fs, file_paths in fs_to_files.items(): + con = duckdb.connect() + con.register_filesystem(fs) + file_list_sql = ", ".join(f"'{f}'" for f in file_paths) + + metadata_sql = f""" + SELECT + file_name, + row_group_id, + path_in_schema, + stats_min_value, + stats_max_value + FROM parquet_metadata([{file_list_sql}]) + WHERE path_in_schema = 'id' + """ + + rows = con.execute(metadata_sql).fetchall() + con.close() + + per_file_match = defaultdict(bool) + for filename, _row_group_id, _path_in_schema, min_value, max_value in rows: + checked_rowgroups += 1 + if min_value is None or max_value is None: + missing_stats += 1 + per_file_match[filename] = True + continue + + for target_id in target_ids_sorted: + if min_value <= target_id <= max_value: + per_file_match[filename] = True + break + + hit_files.update(path for path, matched in per_file_match.items() if matched) + + return { + "elapsed": time.time() - t0, + "hit_files": hit_files, + "missing_stats": missing_stats, + "checked_rowgroups": checked_rowgroups, + } + + +def main(): + parser = argparse.ArgumentParser(description="Benchmark metadata-based pruning for remote search") + parser.add_argument("--terms", required=True, help="Search terms") + parser.add_argument("--language", default="eng") + parser.add_argument("--representation", default="main_content") + parser.add_argument("--limit", type=int, default=3) + parser.add_argument("--conjunctive", action="store_true") + parser.add_argument("--metadata-only", action="store_true", help="Skip current id-only prescan") + args = parser.parse_args() + + manager = OWIlixManager() + collected = _collect_candidate_files( + manager=manager, + terms=args.terms, + language=args.language, + representation=args.representation, + limit=args.limit, + conjunctive=args.conjunctive, + ) + + results = collected["results"] + target_ids = collected["target_ids"] + files_by_dataset = collected["files_by_dataset"] + fs_to_files = _group_by_fs(files_by_dataset) + + total_files = sum(len(v) for v in fs_to_files.values()) + print(f"Query: {args.terms!r} lang={args.language} limit={args.limit}") + print(f"Search results: {len(results)} hits in {collected['search_time']:.1f}s") + print(f"Candidate parquet files before id pruning: {total_files}") + + if not target_ids or total_files == 0: + print("Nothing to benchmark.") + return 0 + + current = None + if not args.metadata_only: + current = benchmark_id_prescan(fs_to_files, target_ids) + print( + f"Current id-only prescan: {current['elapsed']:.1f}s, " + f"{len(current['hit_files'])}/{total_files} files kept" + ) + + metadata = benchmark_metadata_pruning(fs_to_files, target_ids) + print( + f"Metadata pruning: {metadata['elapsed']:.1f}s, " + f"{len(metadata['hit_files'])}/{total_files} files kept" + ) + print( + f"Metadata rows checked: {metadata['checked_rowgroups']}, " + f"rows without stats: {metadata['missing_stats']}" + ) + + if current is not None: + overlap = len(current["hit_files"] & metadata["hit_files"]) + print(f"Overlap with current kept files: {overlap}") + + extra = len(metadata["hit_files"] - current["hit_files"]) + missed = len(current["hit_files"] - metadata["hit_files"]) + print(f"Metadata-only extras vs current: {extra}") + print(f"Metadata-only misses vs current: {missed}") + + if metadata["elapsed"] > 0: + print(f"Speedup estimate: {current['elapsed'] / metadata['elapsed']:.2f}x") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/uv.lock b/uv.lock index 79bba55..16259f4 100644 --- a/uv.lock +++ b/uv.lock @@ -103,18 +103,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, ] -[[package]] -name = "amqp" -version = "5.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, -] - [[package]] name = "annotated-types" version = "0.7.0" @@ -155,15 +143,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/80/9f608d13b4b3afcebd1dd13baf9551c95fc424d6390e4b1cfd7b1810cd06/async_property-0.2.2-py2.py3-none-any.whl", hash = "sha256:8924d792b5843994537f8ed411165700b27b2bd966cefc4daeefc1253442a9d7", size = 9546, upload-time = "2023-07-03T17:21:54.293Z" }, ] -[[package]] -name = "async-timeout" -version = "5.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" }, -] - [[package]] name = "attrs" version = "25.4.0" @@ -182,15 +161,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" }, ] -[[package]] -name = "billiard" -version = "4.2.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537, upload-time = "2025-11-30T13:28:48.52Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" }, -] - [[package]] name = "bitarray" version = "3.8.0" @@ -212,15 +182,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/ea/b7d55ee269b1426f758a535c9ec2a07c056f20f403fa981685c3c8b4798c/bitarray-3.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:84b52b2cf77bb7f703d16c4007b021078dbbe6cf8ffb57abe81a7bacfc175ef2", size = 146709, upload-time = "2025-11-02T21:39:24.343Z" }, ] -[[package]] -name = "blinker" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, -] - [[package]] name = "botocore" version = "1.42.30" @@ -276,25 +237,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/fc/1d7b80d0eb7b714984ce40efc78859c022cd930e402f599d8ca9e39c78a4/cachetools-6.2.4-py3-none-any.whl", hash = "sha256:69a7a52634fed8b8bf6e24a050fb60bff1c9bd8f6d24572b99c32d4e71e62a51", size = 11551, upload-time = "2025-12-15T18:24:52.332Z" }, ] -[[package]] -name = "celery" -version = "5.5.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "billiard" }, - { name = "click" }, - { name = "click-didyoumean" }, - { name = "click-plugins" }, - { name = "click-repl" }, - { name = "kombu" }, - { name = "python-dateutil" }, - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bf/03/5d9c6c449248958f1a5870e633a29d7419ff3724c452a98ffd22688a1a6a/celery-5.5.2.tar.gz", hash = "sha256:4d6930f354f9d29295425d7a37261245c74a32807c45d764bedc286afd0e724e", size = 1666892, upload-time = "2025-04-25T20:10:04.695Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/94/8e825ac1cf59d45d20c4345d4461e6b5263ae475f708d047c3dad0ac6401/celery-5.5.2-py3-none-any.whl", hash = "sha256:54425a067afdc88b57cd8d94ed4af2ffaf13ab8c7680041ac2c4ac44357bdf4c", size = 438626, upload-time = "2025-04-25T20:10:01.383Z" }, -] - [[package]] name = "certifi" version = "2026.1.4" @@ -378,43 +320,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" }, ] -[[package]] -name = "click-didyoumean" -version = "0.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" }, -] - -[[package]] -name = "click-plugins" -version = "1.1.1.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" }, -] - -[[package]] -name = "click-repl" -version = "0.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "prompt-toolkit" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -594,23 +499,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, ] -[[package]] -name = "flask" -version = "3.1.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "blinker" }, - { name = "click" }, - { name = "itsdangerous" }, - { name = "jinja2" }, - { name = "markupsafe" }, - { name = "werkzeug" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dc/6d/cfe3c0fcc5e477df242b98bfe186a4c34357b4847e87ecaef04507332dab/flask-3.1.2.tar.gz", hash = "sha256:bf656c15c80190ed628ad08cdfd3aaa35beb087855e2f494910aa3774cc4fd87", size = 720160, upload-time = "2025-08-19T21:03:21.205Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/f9/7f9263c5695f4bd0023734af91bedb2ff8209e8de6ead162f35d8dc762fd/flask-3.1.2-py3-none-any.whl", hash = "sha256:ca1d8112ec8a6158cc29ea4858963350011b5c846a414cdb7a954aa9e967d03c", size = 103308, upload-time = "2025-08-19T21:03:19.499Z" }, -] - [[package]] name = "frozenlist" version = "1.8.0" @@ -682,18 +570,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/9c/5c359c8d4c9176cfa3c61ecd4efe5affe1f38d9bae81e81ac7186b4c9cc8/grpcio-1.76.0-cp311-cp311-win_amd64.whl", hash = "sha256:522175aba7af9113c48ec10cc471b9b9bd4f6ceb36aeb4544a8e2c80ed9d252d", size = 4709315, upload-time = "2025-10-21T16:21:15.26Z" }, ] -[[package]] -name = "gunicorn" -version = "23.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/34/72/9614c465dc206155d93eff0ca20d42e1e35afc533971379482de953521a4/gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec", size = 375031, upload-time = "2024-08-10T20:25:27.378Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/7d/6dac2a6e1eba33ee43f318edbed4ff29151a49b5d37f080aad1e6469bca4/gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", size = 85029, upload-time = "2024-08-10T20:25:24.996Z" }, -] - [[package]] name = "h11" version = "0.16.0" @@ -840,15 +716,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/ec/24877b2093647d4270c78cdf7d3d3fa08453eb1aa293ac530c7672579b82/irods_fsspec-0.0.1-py3-none-any.whl", hash = "sha256:33771b100df840f850471a5dc96a1cd1e7c26f94f235724953d66f5b8acf0265", size = 18554, upload-time = "2021-12-18T17:49:15.727Z" }, ] -[[package]] -name = "itsdangerous" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, -] - [[package]] name = "jedi" version = "0.19.2" @@ -943,21 +810,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cd/58/4a1880ea64032185e9ae9f63940c9327c6952d5584ea544a8f66972f2fda/jwcrypto-1.5.6-py3-none-any.whl", hash = "sha256:150d2b0ebbdb8f40b77f543fb44ffd2baeff48788be71f67f03566692fd55789", size = 92520, upload-time = "2024-03-06T19:58:29.765Z" }, ] -[[package]] -name = "kombu" -version = "5.5.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "amqp" }, - { name = "packaging" }, - { name = "tzdata" }, - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0f/d3/5ff936d8319ac86b9c409f1501b07c426e6ad41966fedace9ef1b966e23f/kombu-5.5.4.tar.gz", hash = "sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363", size = 461992, upload-time = "2025-06-01T10:19:22.281Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/70/a07dcf4f62598c8ad579df241af55ced65bed76e42e45d3c368a6d82dbc1/kombu-5.5.4-py3-none-any.whl", hash = "sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8", size = 210034, upload-time = "2025-06-01T10:19:20.436Z" }, -] - [[package]] name = "markdown-it-py" version = "4.0.0" @@ -1165,7 +1017,7 @@ wheels = [ [[package]] name = "owilix" -version = "5.2.0" +version = "5.3.0" source = { editable = "." } dependencies = [ { name = "ciff-toolkit" }, @@ -1223,12 +1075,6 @@ graph = [ opensearch = [ { name = "opensearch-py" }, ] -server = [ - { name = "celery" }, - { name = "flask" }, - { name = "gunicorn" }, - { name = "redis" }, -] village = [ { name = "pyiceberg", extra = ["duckdb", "pandas", "s3fs", "sql-sqlite"] }, { name = "pyiceberg-core" }, @@ -1290,12 +1136,6 @@ graph = [ { name = "plotly", specifier = ">=6.2.0" }, ] opensearch = [{ name = "opensearch-py", specifier = ">=2.7.1" }] -server = [ - { name = "celery", specifier = "<5.5.3" }, - { name = "flask", specifier = ">=3.1.2" }, - { name = "gunicorn", specifier = ">=23.0.0" }, - { name = "redis", specifier = ">=6.4.0" }, -] village = [ { name = "pyiceberg", extras = ["duckdb", "pandas", "sql-sqlite", "s3fs"], specifier = ">=0.9.1" }, { name = "pyiceberg-core", specifier = ">=0.4.0" }, @@ -1872,18 +1712,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/67/70f3d4afed87894cd7e37a9c490ab7f324d7ccaba0819a46e1a405dc71d5/rbloom-1.5.4-cp37-abi3-win_amd64.whl", hash = "sha256:48576b9d5bddcb8b4e89f61164e4d06a69bdb60d14c365624262db2fb6986f97", size = 173263, upload-time = "2025-09-09T10:19:23.899Z" }, ] -[[package]] -name = "redis" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/43/c8/983d5c6579a411d8a99bc5823cc5712768859b5ce2c8afe1a65b37832c81/redis-7.1.0.tar.gz", hash = "sha256:b1cc3cfa5a2cb9c2ab3ba700864fb0ad75617b41f01352ce5779dabf6d5f9c3c", size = 4796669, upload-time = "2025-11-19T15:54:39.961Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/89/f0/8956f8a86b20d7bb9d6ac0187cf4cd54d8065bc9a1a09eb8011d4d326596/redis-7.1.0-py3-none-any.whl", hash = "sha256:23c52b208f92b56103e17c5d06bdc1a6c2c0b3106583985a76a18f83b265de2b", size = 354159, upload-time = "2025-11-19T15:54:38.064Z" }, -] - [[package]] name = "referencing" version = "0.37.0" @@ -2463,15 +2291,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, ] -[[package]] -name = "vine" -version = "5.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" }, -] - [[package]] name = "wcwidth" version = "0.2.14" @@ -2481,18 +2300,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl", hash = "sha256:a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1", size = 37286, upload-time = "2025-09-22T16:29:51.641Z" }, ] -[[package]] -name = "werkzeug" -version = "3.1.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/70/1469ef1d3542ae7c2c7b72bd5e3a4e6ee69d7978fa8a3af05a38eca5becf/werkzeug-3.1.5.tar.gz", hash = "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67", size = 864754, upload-time = "2026-01-08T17:49:23.247Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ad/e4/8d97cca767bcc1be76d16fb76951608305561c6e056811587f36cb1316a8/werkzeug-3.1.5-py3-none-any.whl", hash = "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc", size = 225025, upload-time = "2026-01-08T17:49:21.859Z" }, -] - [[package]] name = "wrapt" version = "2.0.1" -- 2.51.2