From 86fec1b6dca8cfc892e35a9e61e99b9d05ec716d Mon Sep 17 00:00:00 2001 From: mgrani Date: Thu, 9 Apr 2026 16:01:58 +0200 Subject: [PATCH] =?UTF-8?q?docs:=20update=20cmd/=20=E2=86=92=20compat/=20r?= =?UTF-8?q?eferences=20and=20refresh=20testing=20guide?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - dev.md: update architecture tree to show compat/ shim and note that new code should not import from it - testing.md: refresh test organization to include new test files (test_search, test_aggregate, test_query_unit, test_executors pool/retry) and document default integration skip via addopts - hardening.md: mark SEC-C3 resolved (cmd/ removed), update SEC-M5 and Appendix B file paths to compat/batch.py Co-Authored-By: Claude Opus 4.6 --- docs/hardening.md | 10 +++++----- docs/source/dev.md | 10 +++++----- docs/testing.md | 23 +++++++++++++++++------ 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/docs/hardening.md b/docs/hardening.md index d2f32ed..8d0c78a 100644 --- a/docs/hardening.md +++ b/docs/hardening.md @@ -77,9 +77,9 @@ files_pattern = [f.strip() for f in files.split(",")] if files else None `.env-rc` is already listed in `.gitignore` and is not tracked in version control. Credentials are local-only. Consider running `git-filter-repo` if secrets were ever committed to history in earlier branches. -#### ~~SEC-C3: Shell Injection via `subprocess` with `shell=True`~~ (SCHEDULED FOR REMOVAL) +#### ~~SEC-C3: Shell Injection via `subprocess` with `shell=True`~~ (RESOLVED) -**File:** `owilix/cmd/batch.py:204` — This module is planned for removal. No remediation needed. +The vulnerable code lived in `owilix/cmd/batch.py:204`. The `cmd/` package has been removed; the equivalent module now lives at `owilix/compat/batch.py` and still retains `shell=True` for plugin backward compatibility. New code must not copy this pattern. #### SEC-C4: SQL/GenQuery Injection @@ -204,7 +204,7 @@ Temporary files are created in multiple locations without consistent cleanup gua | File | Lines | |------|-------| -| `owilix/cmd/batch.py` | 396 | +| `owilix/compat/batch.py` | 396 | | `owilix/core/stream.py` | 106 | | `owilix/core/db/duckdb_executor.py` | 49 | | `owilix/core/tasks/query.py` | 585 | @@ -443,10 +443,10 @@ owilix/core/tasks/remote.py:91,362,2416 owilix/core/tasks/warc/query_warc.py:2359 ``` -### B. Files Containing `shell=True` (scheduled for removal) +### B. Files Containing `shell=True` ``` -owilix/cmd/batch.py:204 (module scheduled for removal) +owilix/compat/batch.py:204 (legacy plugin compat shim; do not copy) ``` ### C. Files with `verify=False` diff --git a/docs/source/dev.md b/docs/source/dev.md index 9d2ec63..eb88fcf 100644 --- a/docs/source/dev.md +++ b/docs/source/dev.md @@ -7,23 +7,23 @@ This guide covers development workflow, testing, release procedures, and documen OWILIX distinguishes between: - **Core packages** (`owilix/core/`): Managers, repositories, models, and the DB query engine - **CLI packages** (`owilix/cli/`): Typer-based command-line interface -- **Legacy CLI** (`owilix/cmd/`): Deprecated Click-based commands +- **Compat shim** (`owilix/compat/`): `BaseCommand`/`SubCommand` kept for plugin backward compatibility. New code should not import from here. ``` owilix/ -├── cli/ # Typer CLI (new) +├── cli/ # Typer CLI │ ├── remote.py # Remote repository commands -│ ├── local.py # Local repository commands +│ ├── local.py # Local repository commands │ ├── query.py # DuckDB query commands │ └── config.py # Configuration commands ├── core/ -│ ├── manager/ # OWIlixManager, session, config +│ ├── manager/ # OWIlixManager, session, config, UI helpers │ ├── repository/ # Repository implementations │ ├── db/ # DuckDB executors │ ├── fsspec/ # iRODS filesystem (Http2IrodsFileSystem) │ ├── models/ # Dataset, metadata models │ └── tasks/ # Query and data tasks -└── cmd/ # Legacy Click CLI (deprecated) +└── compat/ # Legacy BaseCommand/SubCommand for plugins ``` --- diff --git a/docs/testing.md b/docs/testing.md index ec2f057..f38b66b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -28,26 +28,37 @@ uv run python tests/owilix/cli/test_ai_verifiable.py tests/ ├── owilix/ │ ├── cli/ # CLI smoke tests (fast, subprocess-based) -│ │ ├── test_smoke.py # 7 quick tests (~8s total) +│ │ ├── test_smoke.py │ │ └── test_ai_verifiable.py # AI-interpretable JSON output │ └── core/ +│ ├── db/ # DuckDB executor & pool tests +│ │ ├── test_executors.py # SQL query, pool, retry logic +│ │ ├── test_duckdb.py +│ │ └── test_copy_executor.py │ ├── repository/ # Repository tests -│ │ ├── test_integration.py # CLI integration (slow, network) +│ │ ├── test_aggregate.py # Multi-datacenter aggregation +│ │ ├── test_integration.py # CLI integration (slow, network) │ │ └── test_repository_legacy_ported.py -│ ├── fsspec/ # fsspec tests (19/21 passing) +│ ├── tasks/ # Task function tests +│ │ ├── test_search.py # Full-text search (SQL builder, remote) +│ │ └── test_query_unit.py # query_less / query_sites / query_analyze +│ ├── fsspec/ # fsspec tests │ │ ├── test_core_fsspec_unit.py │ │ └── benchmark_async.py │ └── test_metadata.py └── data/ # Test fixtures ``` +By default, integration tests are deselected via `addopts` in `pyproject.toml`. The full unit suite runs in ~25 seconds. + ## Test Types & Timing | Test Suite | Command | Duration | Network | |------------|---------|----------|---------| -| CLI Smoke | `pytest tests/owilix/cli/test_smoke.py -m "not integration"` | ~8s | No | -| fsspec Unit | `pytest tests/owilix/core/fsspec/` | ~8s | No | -| Repository Integration | `pytest tests/owilix/core/repository/test_integration.py` | 5-10min | Yes | +| Full unit suite | `pytest tests/` | ~25s | No | +| CLI Smoke | `pytest tests/owilix/cli/test_smoke.py` | ~8s | No | +| DB executor unit | `pytest tests/owilix/core/db/` | <1s | No | +| Repository Integration | `pytest tests/owilix/core/repository/test_integration.py -m integration` | 5-10min | Yes | ## Pytest Markers -- 2.51.2