diff --git a/docs/integration_test_plan.md b/docs/integration_test_plan.md new file mode 100644 index 0000000..ef7fe57 --- /dev/null +++ b/docs/integration_test_plan.md @@ -0,0 +1,136 @@ +# Integration Test Plan + +Date: 2026-02-17 +Scope: `pytest -m integration` for `owi-cli` + +## Current Inventory + +- Total integration tests: `26` +- Collected with: + - `uv run pytest tests/ -m integration --collect-only -q` +- Current distribution: + - CLI integration/smoke: `7` + - Core repository integration: `3` + - fsspec integration: `14` + - DB executor integration: `2` + +## Authentication Strategy (Reuse Existing `.owi` Token) + +Goal: avoid re-login prompts and reuse the token already available on the machine. + +1. Verify token exists: + - `ls -l ~/.owi/.tokens/refresh_token` +2. Export token for subprocess-based tests: + - `export PY4LEXIS_TOKEN="$(cat ~/.owi/.tokens/refresh_token)"` +3. Ensure default OWI path is used (or set explicitly): + - `export OWS_OWI_PATH="$HOME/.owi"` +4. Optional quick auth sanity check before running integration: + - `uv run owi config version` + - `uv run owi remote ls all:latest --limit 1` + +Notes: +- `OWIlixManager` reads token from `PY4LEXIS_TOKEN` first, then `~/.owi/.tokens/refresh_token`. +- Keep tokens out of logs and shell history where possible. + +## Execution Plan + +Run integration in phases so failures are isolated and reruns are fast. + +### Phase 0: Preflight (fast) + +- `uv run pytest tests/owilix/cli/test_smoke.py -m "not integration" -q` +- `uv run pytest tests/owilix/core/fsspec/test_core_fsspec_unit.py -q` + +### Phase 1: Fast Integration Smoke (target 2-5 min) + +- `uv run pytest -m integration tests/owilix/cli/test_smoke.py tests/owilix/core/repository/test_integration.py::TestIntegration::test_config_version -v -s` + +### Phase 2: API/Auth Integration (target 5-10 min) + +- `uv run pytest -m integration tests/owilix/core/repository/test_integration.py::TestIntegration::test_remote_ls tests/owilix/cli/test_smoke.py::TestCLIIntegration::test_remote_ls_latest -v -s` + +### Phase 3: fsspec Integration (target 10-20 min) + +- `uv run pytest -m integration tests/owilix/core/fsspec/test_core_fsspec_integration.py -v -s` + +### Phase 4: Heavy/Performance Integration (target 15-30+ min) + +- `uv run pytest -m integration tests/owilix/core/fsspec/test_genquery_find.py::TestGenQueryFind::test_find_large_dataset_performance -v -s` +- `uv run pytest -m integration tests/owilix/core/db/test_executors.py -v -s` + +### Optional Full Run + +- `uv run pytest tests/ -m integration -v -s` + +## Speed Optimization Strategy + +Keep integration real, but reduce redundant remote work. + +## Immediate (No Code Changes) + +- Run targeted subsets before full suite; only run full integration before release tags. +- Prefer smaller datasets for default integration path; reserve large dataset tests for nightly/manual. +- Use `--maxfail=1` in pre-release gate to fail fast: + - `uv run pytest tests/ -m integration -v -s --maxfail=1` +- Keep `PY4LEXIS_TOKEN` exported once per shell session. + +## Near-Term Test Refactors (Recommended) + +1. Shared authenticated session fixture (`session` scope): + - Reuse one Lexis/OWI session across tests/modules. + - Avoid repeated token reads and repeated auth handshake. + +2. Shared dataset fixture registry: + - Centralize known test dataset IDs in `tests/conftest.py`. + - Distinguish: + - `SMALL_DATASET_ID` for functional integration + - `LARGE_DATASET_ID` for performance-only checks + +3. Split marker taxonomy: + - Keep `integration` for functional E2E. + - Add `integration_heavy` for expensive tests. + - Add `integration_perf` for throughput/perf assertions. + - Then run release gate with: + - `-m "integration and not integration_heavy and not integration_perf"` + +4. Reduce redundant subprocess invocations: + - Current CLI integration repeatedly executes `uv run owi ...`. + - For many assertion variants, use one command + multiple assertions on parsed output where feasible. + +5. Add explicit test time budgets: + - Use `pytest-timeout` or local timeout wrappers for known long-running tests. + - Mark tests with expected runtime in docstrings. + +## Proposed Release Gates + +### Gate A (required on merge to `main`) + +- Non-integration + selected fast integration: + - `uv run pytest tests/ -m "not integration" -q` + - `uv run pytest -m integration tests/owilix/cli/test_smoke.py tests/owilix/core/repository/test_integration.py::TestIntegration::test_remote_ls -v -s` + +### Gate B (required before tag/release) + +- Full functional integration excluding heavy perf: + - `uv run pytest tests/ -m "integration and not integration_heavy and not integration_perf" -v -s` + +### Gate C (nightly/manual) + +- Heavy/performance integration: + - `uv run pytest tests/ -m "integration_heavy or integration_perf" -v -s` + +## Risks and Controls + +- Token expiry during run: + - Control: preflight remote command and fail early. +- Remote service instability: + - Control: retries in fixtures for setup calls; avoid strict timing asserts on shared infra. +- Dataset drift: + - Control: pin canonical dataset IDs and validate existence in a dedicated preflight test. + +## Backlog Items to Implement + +- Add `tests/conftest.py` integration fixtures for token/session/datasets. +- Introduce `integration_heavy` and `integration_perf` markers in `pyproject.toml`. +- Refactor `tests/owilix/core/fsspec/test_genquery_find.py` token loading (`~/tmp/refresh_token.txt`) to use shared OWI token path. +- Update `docs/testing.md` to include phased integration execution and gating. diff --git a/docs/testing.md b/docs/testing.md index f7f7f62..ec2f057 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -205,6 +205,7 @@ Each test package has comprehensive docstrings in `__init__.py` files: - `tests/owilix/core/repository/__init__.py` - Repository tests For detailed benchmarks and results, see `docs/source/testing_and_benchmarks.md`. +For phased integration execution and optimization guidance, see `docs/integration_test_plan.md`. ## Database Executor Tests @@ -217,4 +218,3 @@ uv run pytest tests/owilix/core/db/test_executors.py -v # Run comprehensive benchmarks (sync vs async) uv run python tests/owilix/core/db/benchmark_comprehensive.py ``` - diff --git a/pyproject.toml b/pyproject.toml index f0267cd..c27c92c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -128,6 +128,8 @@ testpaths = ["tests"] norecursedirs = ["tests/scripts", "tests/benchmarks", "tests/examples"] markers = [ "integration: marks tests as integration tests (require network, may be slow)", + "integration_heavy: long-running integration tests excluded from normal release gates", + "integration_perf: performance-oriented integration tests", "slow: marks tests as slow (deselect with '-m \"not slow\"')", ] filterwarnings = [ diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e1ce1d1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,63 @@ +"""Shared pytest fixtures for OWILIX tests.""" + +import os +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="session") +def integration_refresh_token() -> str: + """ + Return refresh token for integration tests. + + Lookup order: + 1) PY4LEXIS_TOKEN environment variable + 2) ${OWS_OWI_PATH:-~/.owi}/.tokens/refresh_token + """ + env_token = os.environ.get("PY4LEXIS_TOKEN", "").strip() + if env_token: + return env_token + + owi_path = Path(os.path.expanduser(os.environ.get("OWS_OWI_PATH", "~/.owi"))) + token_file = owi_path / ".tokens" / "refresh_token" + if token_file.exists(): + token = token_file.read_text(encoding="utf-8").strip() + if token: + return token + + pytest.skip( + "No integration refresh token found. Set PY4LEXIS_TOKEN or place token in " + "${OWS_OWI_PATH:-~/.owi}/.tokens/refresh_token." + ) + + +@pytest.fixture(scope="session") +def integration_irods_session(integration_refresh_token): + """Create an authenticated iRODS wrapper for integration tests.""" + from py4lexis.core.lexis_irods import iRODS + from py4lexis.session import LexisSession + + class OWIIrods(iRODS): + def irods(self): + self._iRODS__check_access_token() + return self._irds + + session = LexisSession( + suppress_print=True, + login_method="token", + refresh_token=integration_refresh_token, + ) + return OWIIrods(session=session, suppress_print=True) + + +@pytest.fixture(scope="session") +def integration_dataset_id_small() -> str: + """Small dataset for functional integration tests.""" + return "0350fecc-e58b-11f0-a8c9-8ebf6bb2cab9" + + +@pytest.fixture(scope="session") +def integration_dataset_id_large() -> str: + """Large dataset for heavy/performance integration tests.""" + return "f6ea5756-2e0b-11ef-b336-0242ac1d0004" diff --git a/tests/owilix/core/fsspec/test_genquery_find.py b/tests/owilix/core/fsspec/test_genquery_find.py index b44303c..537e140 100644 --- a/tests/owilix/core/fsspec/test_genquery_find.py +++ b/tests/owilix/core/fsspec/test_genquery_find.py @@ -12,7 +12,6 @@ The -s flag is important to see progress output during slow operations. import pytest import time -import os from typing import Set # Mark all tests as integration tests (require network + auth) @@ -27,44 +26,18 @@ def elapsed(start: float) -> str: class TestGenQueryFind: """Integration tests for GenQuery-based find() implementation.""" - # Known dataset IDs for testing - DATASET_ID_SMALL = "0350fecc-e58b-11f0-a8c9-8ebf6bb2cab9" # ~22 files - DATASET_ID_LARGE = "f6ea5756-2e0b-11ef-b336-0242ac1d0004" # ~1524 files - - @pytest.fixture - def irods_session(self): - """Create an authenticated iRODS session.""" - from py4lexis.session import LexisSession - from py4lexis.core.lexis_irods import iRODS - - class OWIIrods(iRODS): - def irods(self): - self._iRODS__check_access_token() - return self._irds - - token_path = os.path.expanduser("~/tmp/refresh_token.txt") - try: - with open(token_path, "r") as f: - refresh_token = f.read().strip() - session = LexisSession(login_method="token", refresh_token=refresh_token) - except FileNotFoundError: - pytest.skip("No refresh token found, skipping integration test") - - irods = OWIIrods(session=session, suppress_print=True) - return irods - @pytest.fixture - def fs(self, irods_session): + def fs(self, integration_irods_session): """Create Http2IrodsFileSystem instance.""" from owilix.core.fsspec.http2irods import Http2IrodsFileSystem return Http2IrodsFileSystem( - irods_client=irods_session.irods(), - url_base=irods_session.irods().url_base + irods_client=integration_irods_session.irods(), + url_base=integration_irods_session.irods().url_base ) @pytest.fixture - def reference_find(self, irods_session): + def reference_find(self, integration_irods_session): """ Get reference results using the old iRODSCollecion approach. This is used to verify semantic correctness. @@ -73,8 +46,8 @@ class TestGenQueryFind: from irods_http_client.models.collection import iRODSCollecion collections = Collections( - irods_session.irods(), - url_base=irods_session.irods().url_base + integration_irods_session.irods(), + url_base=integration_irods_session.irods().url_base ) def find_recursive(coll_path: str) -> Set[str]: @@ -94,14 +67,14 @@ class TestGenQueryFind: return results - return find_recursive, irods_session + return find_recursive, integration_irods_session - def test_find_returns_correct_file_count(self, fs, irods_session, capsys): + def test_find_returns_correct_file_count(self, fs, integration_irods_session, integration_dataset_id_small, capsys): """Test that find() returns the expected number of files.""" start = time.time() print(f"\n{elapsed(start)} Starting find file count test...") - coll = irods_session.get_dataset_collection(self.DATASET_ID_SMALL) + coll = integration_irods_session.get_dataset_collection(integration_dataset_id_small) print(f"{elapsed(start)} Dataset path: {coll.path}") results = fs.find(coll.path, withdirs=False) @@ -114,13 +87,13 @@ class TestGenQueryFind: print(f"{elapsed(start)} ✅ File count test passed") - def test_find_semantic_equivalence_small(self, fs, reference_find, capsys): + def test_find_semantic_equivalence_small(self, fs, reference_find, integration_dataset_id_small, capsys): """Test that GenQuery find() matches iRODSCollecion find() for small dataset.""" start = time.time() print(f"\n{elapsed(start)} Starting semantic equivalence test (small dataset)...") find_recursive, irods_session = reference_find - coll = irods_session.get_dataset_collection(self.DATASET_ID_SMALL) + coll = irods_session.get_dataset_collection(integration_dataset_id_small) # Get results from GenQuery-based find() print(f"{elapsed(start)} Running GenQuery find()...") @@ -151,12 +124,12 @@ class TestGenQueryFind: print(f"{elapsed(start)} ✅ Semantic equivalence verified") print(f" Speedup: {reference_time / genquery_time:.1f}x") - def test_find_with_detail_returns_metadata(self, fs, irods_session, capsys): + def test_find_with_detail_returns_metadata(self, fs, integration_irods_session, integration_dataset_id_small, capsys): """Test that find(detail=True) returns proper metadata.""" start = time.time() print(f"\n{elapsed(start)} Starting detail mode test...") - coll = irods_session.get_dataset_collection(self.DATASET_ID_SMALL) + coll = integration_irods_session.get_dataset_collection(integration_dataset_id_small) results = fs.find(coll.path, withdirs=False, detail=True) @@ -179,12 +152,12 @@ class TestGenQueryFind: print(f"{elapsed(start)} Sample result: {first_result}") print(f"{elapsed(start)} ✅ Detail mode test passed") - def test_find_with_withdirs_includes_directories(self, fs, irods_session, capsys): + def test_find_with_withdirs_includes_directories(self, fs, integration_irods_session, integration_dataset_id_small, capsys): """Test that find(withdirs=True) includes directories.""" start = time.time() print(f"\n{elapsed(start)} Starting withdirs test...") - coll = irods_session.get_dataset_collection(self.DATASET_ID_SMALL) + coll = integration_irods_session.get_dataset_collection(integration_dataset_id_small) # Get files only files_only = fs.find(coll.path, withdirs=False) @@ -206,12 +179,12 @@ class TestGenQueryFind: print(f"{elapsed(start)} ✅ withdirs test passed") - def test_info_uses_cache_after_find(self, fs, irods_session, capsys): + def test_info_uses_cache_after_find(self, fs, integration_irods_session, integration_dataset_id_small, capsys): """Test that info() uses cached metadata from find().""" start = time.time() print(f"\n{elapsed(start)} Starting info cache test...") - coll = irods_session.get_dataset_collection(self.DATASET_ID_SMALL) + coll = integration_irods_session.get_dataset_collection(integration_dataset_id_small) # Clear cache first fs.invalidate_cache() @@ -241,12 +214,14 @@ class TestGenQueryFind: print(f"{elapsed(start)} ✅ Info cache test passed") - def test_find_large_dataset_performance(self, fs, irods_session, capsys): + @pytest.mark.integration_heavy + @pytest.mark.integration_perf + def test_find_large_dataset_performance(self, fs, integration_irods_session, integration_dataset_id_large, capsys): """Test that find() performs well on large datasets.""" start = time.time() print(f"\n{elapsed(start)} Starting large dataset performance test...") - coll = irods_session.get_dataset_collection(self.DATASET_ID_LARGE) + coll = integration_irods_session.get_dataset_collection(integration_dataset_id_large) print(f"{elapsed(start)} Dataset path: {coll.path}") find_start = time.time()