From db9a4dabb2ce876faf639cbc0e859176f113f169 Mon Sep 17 00:00:00 2001 From: mgrani Date: Sat, 11 Apr 2026 18:17:12 +0200 Subject: [PATCH] chore: release v5.3.1 --- CHANGELOG.md | 25 ++++++ Readme.md | 2 +- docs/source/commands.md | 20 ++++- docs/source/dataset-structure-and-metadata.md | 30 +++++-- docs/source/details/admin.md | 28 ++++++- docs/source/details/local.md | 21 +++++ docs/source/details/remote.md | 23 ++++++ docs/test-status.md | 6 +- docs/testing.md | 32 +++++++- owilix/_version.py | 4 +- owilix/cli/__init__.py | 5 ++ owilix/cli/_common/context.py | 5 ++ owilix/cli/admin.py | 71 ++++++++++------ owilix/cli/local.py | 19 +++-- owilix/cli/remote.py | 28 +++++-- owilix/core/db/duckdb_executor.py | 8 +- owilix/core/manager/manager.py | 69 ++++++++++------ owilix/core/models/dataset.py | 28 +++++-- owilix/core/sql_logging.py | 46 +++++++++++ owilix/core/stream.py | 4 +- owilix/core/tasks/local.py | 16 ++-- owilix/core/tasks/query.py | 3 +- owilix/core/tasks/query_utils.py | 14 ++-- owilix/core/tasks/remote.py | 81 +++++++++++-------- owilix/core/tasks/search.py | 13 ++- owilix/core/utils.py | 18 ++++- pyproject.toml | 2 +- tests/owilix/cli/test_smoke.py | 14 ++++ tests/owilix/core/db/test_executors.py | 29 +++++++ tests/owilix/core/test_dataset_api.py | 68 ++++++++++++++++ tests/owilix/core/test_specifier_parser.py | 56 +++++++++++++ tests/owilix/core/test_sql_logging.py | 19 +++++ uv.lock | 2 +- 33 files changed, 667 insertions(+), 142 deletions(-) create mode 100644 owilix/core/sql_logging.py create mode 100644 tests/owilix/core/test_specifier_parser.py create mode 100644 tests/owilix/core/test_sql_logging.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d7ed5c9..9a28008 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,31 @@ ## Unreleased +## v5.3.1 (2026-04-11) + +### Features + +- **specifier**: Add explicit `start..end` date range syntax while keeping the existing `#days` form for backward compatibility. +- **specifier**: Treat legacy remote scopes `it4i` and `lrz` as zone aliases (`IT4ILexisV2`, `OWILRZZONE`) instead of assuming separate backends. +- **specifier**: Add comma-separated OR values within one field and `field-=` exact exclusion filters while keeping `field*=` as the compact regex-match alias. +- **debug**: Add a global `--sql-log` CLI flag that writes executed SQL statements to `.owi/.logs/sql.log` for debugging. +- **admin**: Extend `owi admin logs` with explicit log selection (`latest`, `events`, `sql`, filenames) and a `location` mode that prints resolved log paths. + +### Bug Fixes + +- **local**: Apply parsed specifier date windows consistently in `local ls`, `local rm`, and related helper paths instead of ignoring the time component. +- **query**: Normalize specifier `access=` handling so only plain single-value access filters are lifted into repository listing and richer filters stay in metadata matching. + +### Tests + +- **specifier**: Add parser regression coverage for legacy syntax, explicit ranges, zone aliases, and the new filter operators. +- **dataset**: Add filter regression coverage for AND semantics, comma-separated OR values, regex matching, and exclusion matching. + +### Documentation + +- **docs**: Clarify specifier syntax in command help and reference pages, including scope aliases, `;` AND semantics, comma-separated OR values, regex filters, and exclusion filters. +- **docs**: Document the SQL debug logging workflow, the location of `sql.log`, and the new admin log-browsing commands. + ## v5.3.0 (2026-04-11) ### Features diff --git a/Readme.md b/Readme.md index c59db8c..8436a5b 100644 --- a/Readme.md +++ b/Readme.md @@ -1,6 +1,6 @@ # OWILIX - Open Web Index CLI -[![Version](https://img.shields.io/badge/version-5.3.0-blue.svg)](https://openwebsearcheu-public.pages.it4i.eu/owi-cli/) +[![Version](https://img.shields.io/badge/version-5.3.1-blue.svg)](https://openwebsearcheu-public.pages.it4i.eu/owi-cli/) [![Python](https://img.shields.io/badge/python-3.11+-green.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-Apache_2.0-orange.svg)](http://www.apache.org/licenses/LICENSE-2.0) diff --git a/docs/source/commands.md b/docs/source/commands.md index c7e5f92..4e2d37d 100644 --- a/docs/source/commands.md +++ b/docs/source/commands.md @@ -40,6 +40,7 @@ The following flags can be used with almost any command: | `--no-display` | | Suppress console output (useful for scripting). | | `--target` | `-t` | Override the target directory for data. | | `--no-progress` | `-N` | Suppress progress bars. | +| `--sql-log` | | Write executed SQL statements to `~/.owi/.logs/sql.log` for debugging. | ## Defaults & Specifiers @@ -49,19 +50,30 @@ The following flags can be used with almost any command: ### Specifier Format Access datasets using the following specifier format: -`{datacenter|all}:{YYYY-MM-DD|latest}#{days}/{key=value;key=value}` +`{scope|all}[:{YYYY-MM-DD|latest|YYYY-MM-DD#days|YYYY-MM-DD..YYYY-MM-DD}][/{key=value;key=value}]` + +Rules: +- `all` searches all configured sources. +- `it4i` and `lrz` remain valid legacy scope aliases and now translate to zone filters (`IT4ILexisV2` and `OWILRZZONE`). +- `;` combines metadata filters with AND. +- `value1,value2` means OR within one field. +- `key*=` performs a regex match. +- `key-=` excludes exact values. Examples: -- `all:latest`: Latest datasets from all datacenters. +- `all:latest`: Latest datasets from all configured sources. - `lexis:2025-01-01`: Datasets from LEXIS on a specific date. +- `all:2025-01-01..2025-01-10`: Explicit inclusive date range. +- `it4i:latest`: Latest datasets in zone `IT4ILexisV2`. - `all/collectionName=main`: Filter by collection name. -- `all/access=public`: Filter by access level. +- `all/zone=IT4ILexisV2,OWILRZZONE`: Match either zone. +- `all/title*=.*Open.*;zone-=OWILRZZONE`: Regex title match while excluding LRZ. ## Admin Commands Admin commands are used for troubleshooting and system monitoring. -- `owi admin logs [level]`: View logs. +- `owi admin logs [selector]`: View logs by selector (`latest`, `events`, `sql`, filename) or print paths with `location`. - `owi admin stats`: View system statistics. - `owi admin repos`: List configured repositories and their status. - `owi admin path [--repository ]`: Check whether a path exists in one/all repositories. diff --git a/docs/source/dataset-structure-and-metadata.md b/docs/source/dataset-structure-and-metadata.md index 12697bf..10df0ca 100644 --- a/docs/source/dataset-structure-and-metadata.md +++ b/docs/source/dataset-structure-and-metadata.md @@ -671,23 +671,34 @@ Specifiers provide flexible filtering and selection of datasets: **Format:** ``` -:#/= +[:|latest|#|..][/] ``` **Components:** -- **``**: Repository name (`all`, `it4i`, `lrz`, `lexis`) or `all` for all repositories +- **``**: Scope selector + - `all`: all configured sources + - `lexis`: unified LEXIS repository + - `it4i`: legacy alias for zone `IT4ILexisV2` + - `lrz`: legacy alias for zone `OWILRZZONE` - **``**: Date filter - `latest`: Most recent dataset - `YYYY-MM-DD`: Specific date - `YYYY-MM`: Month (defaults to first day) -- **`#`**: Date range in days (e.g., `#30` for ±30 days from date) -- **`/=`**: Metadata filters (can chain multiple with `/`) +- **`#`**: Backward-compatible trailing date window (e.g., `#30`) +- **`..`**: Explicit inclusive date range +- **`/`**: Metadata filters + - `;` combines filters with AND + - `value1,value2` means OR within one field + - `field*=` means regex match + - `field-=` means exact exclusion **Filters:** - `id=`: Specific dataset ID - `collectionName=`: Collection filter - `access=`: Access level (`public`, `project`, `private`) +- `zone=`: Zone match (`IT4ILexisV2`, `OWILRZZONE`) +- `zone-=`: Exclude a zone - `resourceType=`: Dataset type (`owi`, `owip`, `warc`, etc.) - `startDate=`: Datasets starting on/after date - `endDate=`: Datasets ending on/before date @@ -701,20 +712,23 @@ owi remote ls all # Latest dataset from IT4I owi remote ls it4i:latest -# Datasets from January 2024 ±30 days +# Datasets in an explicit date range +owi remote ls all:2024-01-01..2024-01-31 + +# Backward-compatible date window syntax owi remote ls all:2024-01-01#30 # Specific dataset by ID owi remote ls all/id=b313af04-f101-11f0-89ba-02a47ca5d9fd # All project-level datasets in "cefal" collection -owi remote ls all/access=project/collectionName=cefal +owi remote ls all/access=project;collectionName=cefal # All WARC datasets owi remote ls all/resourceType=warc -# Combined filters -owi remote ls it4i:2024-01/access=public/collectionName=main +# Combined filters with OR and exclusion +owi remote ls all/zone=IT4ILexisV2,OWILRZZONE;access=public;zone-=OWILRZZONE ``` ### Authentication and Access Control diff --git a/docs/source/details/admin.md b/docs/source/details/admin.md index 0aee637..a79b32f 100644 --- a/docs/source/details/admin.md +++ b/docs/source/details/admin.md @@ -82,16 +82,38 @@ View application logs. ### Usage ```bash -owi admin logs [LOG_TYPE] [OPTIONS] +owi admin logs [LOG_SELECTOR] [OPTIONS] ``` +### Log selectors + +- `latest`: most recently modified `.log` or `.json` log file in `~/.owi/.logs` +- `location`: print the log directory and known log file locations +- `events`: show `events.json` +- `sql`: show `sql.log` +- `lexis`, `errors`: show named log files when present +- any explicit filename in `~/.owi/.logs` + +### Options + +- `--lines`, `-n`: Number of lines to show (default: `50`) +- `--follow`, `-f`: Reserved follow mode flag + ### Examples ```bash -owi admin logs error -owi admin logs all --lines 100 +owi admin logs latest +owi admin logs sql --lines 100 +owi admin logs events --lines 20 +owi admin logs location ``` +### Notes + +- `events` is stored as JSON lines in `events.json`. +- SQL debugging output is only written when commands are run with the global `--sql-log` flag. +- `location` is the quickest way to confirm where `events.json`, `sql.log`, and other logs live in the active OWI target directory. + --- ## `admin stats` diff --git a/docs/source/details/local.md b/docs/source/details/local.md index 8e16163..9948ff5 100644 --- a/docs/source/details/local.md +++ b/docs/source/details/local.md @@ -2,6 +2,17 @@ Local commands (`owi local`) are used to manage datasets stored in your local repository. The local repository path defaults to `~/.owi` but can be configured. +Specifier syntax summary: + +```bash +[:|latest|#|..][/] +``` + +- `;` means AND between metadata filters. +- `value1,value2` means OR within one field. +- `field*=` performs regex matching. +- `field-=` excludes exact values. + ## `local ls` List datasets available in the local repository. @@ -33,6 +44,16 @@ owi local ls "all/title*=.*Web.*" owi local ls all --sort totalSize --reverse ``` +**List datasets in an explicit date range:** +```bash +owi local ls all:2026-04-01..2026-04-10 +``` + +**Filter with OR and exclusion:** +```bash +owi local ls all/collectionName=main,source;zone-=OWILRZZONE +``` + **List all parquet files in a specific dataset:** ```bash owi local ls all/id=123... --files "**/*.parquet" diff --git a/docs/source/details/remote.md b/docs/source/details/remote.md index daa6eb4..b1a3ec7 100644 --- a/docs/source/details/remote.md +++ b/docs/source/details/remote.md @@ -2,6 +2,19 @@ Remote commands (`owi remote`) allow you to interact with datasets stored in remote data centers (e.g., LEXIS, LRZ, IT4I). +Specifier syntax summary: + +```bash +[:|latest|#|..][/] +``` + +- `all`, `lexis`, `it4i`, and `lrz` are valid scopes. +- `it4i` and `lrz` now behave as legacy zone aliases rather than separate backends. +- `;` means AND between metadata filters. +- `value1,value2` means OR within one field. +- `field*=` performs regex matching. +- `field-=` excludes exact values. + ## `remote ls` List datasets available in remote repositories. @@ -25,6 +38,11 @@ owi remote ls [SPECIFIER] [OPTIONS] owi remote ls lexis:latest ``` +**List datasets in an explicit date range for the IT4I zone alias:** +```bash +owi remote ls it4i:2026-04-01..2026-04-10 +``` + **List all parquet files in a remote dataset:** ```bash owi remote ls lexis:latest/collectionName=main --files "**/*.parquet" @@ -35,6 +53,11 @@ owi remote ls lexis:latest/collectionName=main --files "**/*.parquet" owi remote ls lexis:latest/collectionName=main --files "**/*.parquet" --file-details ``` +**Combine AND, OR, regex, and exclusion filters:** +```bash +owi remote ls all/title*=.*Open.*;zone=IT4ILexisV2,OWILRZZONE;zone-=OWILRZZONE +``` + --- ## `remote search` diff --git a/docs/test-status.md b/docs/test-status.md index e8b9ca4..457116e 100644 --- a/docs/test-status.md +++ b/docs/test-status.md @@ -4,7 +4,7 @@ Current test-status notes for OWILIX. This page separates actionable failures or ## Current State -Latest observed default suite run on 2026-04-10: +Latest observed default suite run on 2026-04-11: ```bash uv run pytest -q @@ -12,10 +12,10 @@ uv run pytest -q Result: -- `458 passed` +- `467 passed` - `2 skipped` - `42 deselected` -- `5 warnings` +- `2 warnings` ## Warning Policy diff --git a/docs/testing.md b/docs/testing.md index 0dbe523..e751c12 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -27,13 +27,41 @@ uv run python tests/owilix/cli/test_ai_verifiable.py Latest observed local runs in this repository: - `uv run pytest -q` - Result on 2026-04-10: `448 passed, 2 skipped, 40 deselected, 3 warnings` + Result on 2026-04-11: `471 passed, 2 skipped, 42 deselected, 2 warnings` - `uv run pytest tests/owilix/cli/test_remote_search_cli.py tests/owilix/core/tasks/test_search.py -q` Result on 2026-04-10: `24 passed` - `uv run pytest tests/owilix/cli/test_smoke.py -q -m integration -k remote_search` Result on 2026-04-10: passed in the current environment +- `uv run pytest tests/owilix/core/test_sql_logging.py tests/owilix/core/db/test_executors.py -q` + Result on 2026-04-11: `22 passed, 2 deselected` +- `uv run pytest tests/owilix/cli/test_smoke.py -q -k 'admin_logs_location or admin_logs_sql' -o addopts=''` + Result on 2026-04-11: `2 passed` -The remaining warning noise comes from un-awaited `OWIDuckDBSelectExecutor.close()` calls in `tests/owilix/core/db/test_executors.py`. +The remaining warning noise is the documented Rich/Jupyter cosmetic warning from `tests/owilix/core/tasks/test_search.py`. + +## SQL Debug Logging + +To capture executed SQL statements during CLI runs, enable the global SQL log: + +```bash +uv run owi --sql-log query less -R lexis:latest --limit 10 +``` + +The statements are written to: + +```bash +~/.owi/.logs/sql.log +``` + +This log is opt-in and intended for debugging query construction and execution. + +To inspect the logs from the CLI: + +```bash +uv run owi admin logs location +uv run owi admin logs sql --lines 20 +uv run owi admin logs events --lines 20 +``` ## Test Organization diff --git a/owilix/_version.py b/owilix/_version.py index 8676c70..471fcbf 100644 --- a/owilix/_version.py +++ b/owilix/_version.py @@ -1,3 +1,3 @@ # Version is set here and imported elsewhere -__version__ = "5.3.0" -__version_tuple__ = (5, 3, 0) +__version__ = "5.3.1" +__version_tuple__ = (5, 3, 1) diff --git a/owilix/cli/__init__.py b/owilix/cli/__init__.py index 95a5317..2c191e8 100644 --- a/owilix/cli/__init__.py +++ b/owilix/cli/__init__.py @@ -72,6 +72,10 @@ def main( False, "--no-remote-config", help="Disable server config updates and use local config as-is" ), + sql_log: bool = typer.Option( + False, "--sql-log", + help="Write executed SQL statements to .owi/.logs/sql.log for debugging" + ), ): """ OWILIX - Open Web Index Client @@ -103,6 +107,7 @@ def main( autoyes=yes, loglevel=effective_loglevel, no_remote_config=no_remote_config, + sql_log=sql_log, ) # Suppress progress bars globally when --no-progress is set diff --git a/owilix/cli/_common/context.py b/owilix/cli/_common/context.py index 1ee25df..8269ae7 100644 --- a/owilix/cli/_common/context.py +++ b/owilix/cli/_common/context.py @@ -38,6 +38,7 @@ class CLIContext(BaseModel): target: Optional[str] = Field(default=None, description="Target directory") no_progress: bool = Field(default=False, description="Suppress progress bars") fields: Optional[str] = Field(default=None, description="Custom fields (+field,-field)") + sql_log: bool = Field(default=False, description="Write executed SQL to sql.log") # Additional context from legacy CLI showfields: Optional[str] = Field(default=None, description="Fields to show in display") @@ -57,6 +58,7 @@ def create_context( autoyes: bool = False, loglevel: Optional[str] = None, no_remote_config: bool = False, + sql_log: bool = False, ) -> CLIContext: """ Create a CLI context with initialized OWIlixManager. @@ -79,6 +81,7 @@ def create_context( from owilix.core import OWIlixManager from owilix.core.manager import OWIlixConfig, OWILIXEnv from owilix.core.manager.ui import OWILIXConsole + from owilix.core.sql_logging import configure_sql_logging # Determine target path owi_path = target or OWILIXEnv.values.owi_path @@ -101,6 +104,7 @@ def create_context( # Create manager with owi_path and config owi = OWIlixManager(owi_path=owi_path, config=config) + configure_sql_logging(sql_log, owi.sql_logfile) return CLIContext( owi=owi, @@ -113,6 +117,7 @@ def create_context( no_progress=no_progress, fields=fields, auto_yes=autoyes, + sql_log=sql_log, ) diff --git a/owilix/cli/admin.py b/owilix/cli/admin.py index 487583a..3b22be7 100644 --- a/owilix/cli/admin.py +++ b/owilix/cli/admin.py @@ -8,6 +8,7 @@ from rich.table import Table from ._common.context import CLIContext from ._common.ui import ask_yes_no +from owilix.core.utils import split_query_access # Create sub-app for admin commands app = typer.Typer( @@ -221,44 +222,65 @@ def filesystem( @app.command() def logs( ctx: typer.Context, - log_type: str = typer.Argument("error", help="Log type: error, info, all"), + log_type: str = typer.Argument("latest", help="Log selector: latest, location, events, sql, or a concrete filename"), lines: int = typer.Option(50, "--lines", "-n", help="Number of lines to show"), follow: bool = typer.Option(False, "--follow", "-f", help="Follow log output"), ): """ View application logs. - Shows recent log entries filtered by type. - + Shows recent log entries from the selected log target. + Examples: - owi admin logs error - owi admin logs all --lines 100 + owi admin logs latest + owi admin logs sql --lines 100 + owi admin logs events + owi admin logs location """ cli_ctx: CLIContext = ctx.obj - # Determine log file path - # TODO: is this really good to have .logs hard coded here? where do we define it? log_dir = Path(cli_ctx.target) / ".logs" if not log_dir.exists(): cli_ctx.console.print(f"[yellow]No logs directory found at {log_dir}[/yellow]") return - - # Find log files - log_files = list(log_dir.glob("*.log")) - if not log_files: + + known_logs = { + "events": log_dir / "events.json", + "sql": log_dir / "sql.log", + "lexis": log_dir / "lexis.log", + "errors": log_dir / "errors.log", + } + + if log_type == "location": + cli_ctx.console.print(f"[bold]Log directory:[/bold] {log_dir}") + for name, path in known_logs.items(): + status = "exists" if path.exists() else "missing" + cli_ctx.console.print(f" • {name}: {path} [{status}]") + return + + available_logs = sorted({p for p in log_dir.iterdir() if p.is_file() and p.suffix in {".log", ".json"}}) + if not available_logs: cli_ctx.console.print("[yellow]No log files found[/yellow]") return - + cli_ctx.console.print(f"[bold]Available logs in {log_dir}:[/bold]") - for lf in log_files: + for lf in available_logs: cli_ctx.console.print(f" • {lf.name}") - - # Show most recent log - latest = max(log_files, key=lambda p: p.stat().st_mtime) - cli_ctx.console.print(f"\n[bold]Showing last {lines} lines from {latest.name}:[/bold]\n") - - with open(latest, 'r') as f: + + if log_type == "latest": + selected = max(available_logs, key=lambda p: p.stat().st_mtime) + else: + selected = known_logs.get(log_type) + if selected is None: + selected = log_dir / log_type + if not selected.exists(): + cli_ctx.console.print(f"[red]Log file not found:[/red] {selected}") + return + + cli_ctx.console.print(f"\n[bold]Showing last {lines} lines from {selected.name}:[/bold]\n") + + with open(selected, 'r', encoding='utf-8') as f: all_lines = f.readlines() for line in all_lines[-lines:]: # Color based on level @@ -423,11 +445,12 @@ def check( # And logic suggests it patches metadata. spec = cli_ctx.owi.parse_specifier(specifier) - datasets = cli_ctx.owi.remote_data.list(spec.pop("data_center",None), - spec.get("query", {}).pop("access", "public"), - day=spec.pop("day", None), - duration= spec.pop("duration", 0), - query=spec.pop("query", {})) + access, query = split_query_access(spec.get("query")) + datasets = cli_ctx.owi.remote_data.list(spec.get("data_center"), + access, + day=spec.get("day"), + duration=spec.get("duration", 0), + query=query) cli_ctx.console.print(f"Found {len(datasets)} datasets to check.") diff --git a/owilix/cli/local.py b/owilix/cli/local.py index fec18bf..ec64203 100644 --- a/owilix/cli/local.py +++ b/owilix/cli/local.py @@ -13,6 +13,7 @@ from ._common.context import CLIContext from ._common.output import OutputWriter from ._common.progress import simple_progress from owilix.core.tasks.local import insert_local_dataset, remove_local_dataset, export_and_merge +from owilix.core.utils import split_query_access # Create sub-app for local commands @@ -37,7 +38,7 @@ def _format_size(size_bytes: int) -> str: @app.command() def ls( ctx: typer.Context, - specifier: str = typer.Argument("all", help="Dataset specifier (e.g., 'main:latest')"), + specifier: str = typer.Argument("all", help="Dataset specifier (e.g., 'all:latest', 'all:2026-04-01..2026-04-10/zone=IT4ILexisV2')"), display_format: str = typer.Option("short", "--display", "-d", help="Display format: short, wide, markdown"), sort_by: str = typer.Option("", "--sort", "-s", help="Sort by field (e.g., startDate, totalSize)"), reverse: bool = typer.Option(False, "--reverse", "-r", help="Reverse sort order"), @@ -59,6 +60,8 @@ def ls( Examples: owi local ls all owi local ls main:latest --display wide + owi local ls all:2026-04-01..2026-04-10 + owi local ls all/collectionName=main;zone=IT4ILexisV2,OWILRZZONE owi local ls all --sort totalSize --reverse owi local ls all/id=abc123 --files "**/*.parquet" """ @@ -67,9 +70,12 @@ def ls( # Parse specifier and list datasets spec = cli_ctx.owi.parse_specifier(specifier) + access, query = split_query_access(spec.get("query")) datasets_list = cli_ctx.owi.local.list( - access=spec.get("query", {}).get("access", "public"), - query={k: v for k, v in spec.get("query", {}).items() if k != "access"}, + access=access, + day=spec.get("day"), + duration=spec.get("duration") or 0, + query=query, ) # If --files is specified, list files instead of datasets @@ -166,9 +172,12 @@ def rm( # Parse specifier and list datasets spec = cli_ctx.owi.parse_specifier(specifier) + access, query = split_query_access(spec.get("query")) datasets = cli_ctx.owi.local.list( - access=spec.get("query", {}).get("access", "public"), - query={k: v for k, v in spec.get("query", {}).items() if k != "access"}, + access=access, + day=spec.get("day"), + duration=spec.get("duration") or 0, + query=query, ) if not datasets: diff --git a/owilix/cli/remote.py b/owilix/cli/remote.py index fc4937a..e167311 100644 --- a/owilix/cli/remote.py +++ b/owilix/cli/remote.py @@ -15,6 +15,7 @@ import sys from ._common.context import CLIContext, get_context from ._common.output import OutputWriter from owilix.cli._common.executor import execute_command +from owilix.core.utils import split_query_access # Create sub-app for remote commands app = typer.Typer( @@ -66,7 +67,7 @@ def _format_size(size_bytes: int) -> str: @app.command() def ls( ctx: typer.Context, - specifier: str = typer.Argument("all", help="Dataset specifier (e.g., 'lrz:latest')"), + specifier: str = typer.Argument("all", help="Dataset specifier (e.g., 'all:latest', 'it4i:latest', 'all:2026-04-01..2026-04-10/zone-=OWILRZZONE')"), display_format: str = typer.Option("short", "--display", "-d", help="Display format: short, wide, markdown"), sort_by: str = typer.Option("", "--sort", "-s", help="Sort by field (e.g., startDate, totalSize)"), reverse: bool = typer.Option(False, "--reverse", "-r", help="Reverse sort order"), @@ -78,7 +79,21 @@ def ls( """ List remote datasets matching SPECIFIER. - Specifier format: :#/= + Specifier format: + [:|latest|#|..][/] + + Scope aliases: + all - all configured remote sources + lexis - the unified LEXIS repository + it4i - shorthand for zone=IT4ILexisV2 + lrz - shorthand for zone=OWILRZZONE + + Filter syntax: + key=value exact match + key=v1,v2 exact match against any listed value + key*=pattern regex match + key-=value exclude exact match + ; combines filters with AND Display formats: short - Compact ls-style (default) @@ -88,7 +103,9 @@ def ls( Examples: owi remote ls all owi remote ls lexis:latest --display wide - owi remote ls it4i:2024-01 --sort totalSize --reverse + owi remote ls it4i:2026-04-01..2026-04-10 --sort totalSize --reverse + owi remote ls all/collectionName=main;zone=IT4ILexisV2,OWILRZZONE + owi remote ls all/title*=.*Web.*; owi remote ls all/id=abc123 --files "**/*.parquet" """ cli_ctx: CLIContext = ctx.obj @@ -96,12 +113,13 @@ def ls( # Parse specifier and list datasets spec = cli_ctx.owi.parse_specifier(specifier) + access, query = split_query_access(spec.get("query")) datasets_list = cli_ctx.owi.remote_data.list( datacenter=spec.get("data_center"), - access=spec.get("query", {}).get("access", "public"), + access=access, day=spec.get("day"), duration=spec.get("duration") or 0, - query={k: v for k, v in spec.get("query", {}).items() if k != "access"}, + query=query, ) # If --files is specified, list files instead of datasets diff --git a/owilix/core/db/duckdb_executor.py b/owilix/core/db/duckdb_executor.py index 598e008..8df8058 100644 --- a/owilix/core/db/duckdb_executor.py +++ b/owilix/core/db/duckdb_executor.py @@ -10,6 +10,7 @@ import fsspec from .models import OWIlixSQLQuery, ParquetBatch, ParquetBatchResult from .utils import partitionByPath +from owilix.core.sql_logging import log_sql_query # Handle python-irodsclient version differences try: @@ -286,6 +287,7 @@ class OWIDuckDBSelectExecutor: # Execute with retry try: + log_sql_query(_query.sql, source="OWIDuckDBSelectExecutor.run_query_on_batch") cursor = self._retry_query(conn.execute, _query.sql, fs=fs) self.logger.debug(f"run_query_on_batch: query executed, fetching results...") except Exception as e: @@ -568,7 +570,9 @@ class OWIDuckDBCopyExecutor(OWIDuckDBSelectExecutor): # Optionally do an EXPLAIN if self.explain: self.logger.info("==== EXPLAIN PLAN ====") - explain_cursor = self._retry_query(conn.execute, f"EXPLAIN {_query.sql}", fs=fs) + explain_sql = f"EXPLAIN {_query.sql}" + log_sql_query(explain_sql, source="OWIDuckDBCopyExecutor.explain") + explain_cursor = self._retry_query(conn.execute, explain_sql, fs=fs) plan = explain_cursor.fetchall() for p in plan: self.logger.info(p) @@ -747,6 +751,7 @@ class OWIDuckDBAsyncExecutor: _query = _query.format(**pq_batch.query_args) # Execute + log_sql_query(_query.sql, source="OWIDuckDBAsyncExecutor._sync_run_query") cursor = conn.execute(_query.sql) # Fetch results @@ -888,4 +893,3 @@ class OWIDuckDBAsyncExecutor: return results return asyncio.run(collect()) - diff --git a/owilix/core/manager/manager.py b/owilix/core/manager/manager.py index da6fc1a..c7aa612 100644 --- a/owilix/core/manager/manager.py +++ b/owilix/core/manager/manager.py @@ -627,8 +627,10 @@ class OWIlixManager: os.makedirs(self.logpath) self.logfile = os.path.join(self.logpath, "events.json") + self.sql_logfile = os.path.join(self.logpath, "sql.log") self._repo_stats_file = os.path.join(self.logpath, "repository_stats.json") rotate_log_file(self.logfile) + rotate_log_file(self.sql_logfile) # Initialize repositories repo_classes, AggregatedRepository = _get_repository_classes() @@ -855,13 +857,11 @@ class OWIlixManager: Parses a specifier string to extract data query parameters. Args: - specifier: Format `:#/=`. + specifier: Format `: