diff --git a/docs/source/commands.md b/docs/source/commands.md index 93c8584..ef0ab15 100644 --- a/docs/source/commands.md +++ b/docs/source/commands.md @@ -230,18 +230,34 @@ Remote commands operate on the datacenters specified. ```bash owi remote ls all -owi remote ls lrz:latest --display wide -owi remote ls all:latest/id=0350fecc-e58b-11f0-a8c9-8ebf6bb2cab9 +owi remote ls lexis:latest --display wide +owi remote ls all/id=0350fecc-e58b-11f0-a8c9-8ebf6bb2cab9 + +# List files in a dataset (new --files flag) +owi remote ls lexis:latest/collectionName=cefal --files "**/*.parquet" +``` + +##### Listing files in datasets + +Use `--files` (`-f`) to list files matching a glob pattern: + +```bash +# List parquet files in a local dataset +owi local ls all/id=abc123 --files "**/*.parquet" + +# List parquet files in a remote dataset +owi remote ls lexis:latest/collectionName=cefal --files "**/*.parquet" ``` ##### Pulling datasets ```bash -owi remote pull lrz:latest/access=public +owi remote pull lexis:latest/access=public owi remote pull all/id=abc123... --files "**/*.parquet" owi remote pull it4i:latest --language eng --threads 4 ``` + ##### Pushing datasets ```bash diff --git a/owilix/cli/local.py b/owilix/cli/local.py index d8f2e9d..c53ea7f 100644 --- a/owilix/cli/local.py +++ b/owilix/cli/local.py @@ -32,6 +32,7 @@ def ls( reverse: bool = typer.Option(False, "--reverse", "-r", help="Reverse sort order"), no_summary: bool = typer.Option(False, "--no-summary", help="Skip summary table"), fields: Optional[str] = typer.Option(None, "--fields", help="Customize fields (+field, -field)"), + files_glob: Optional[str] = typer.Option(None, "--files", "-f", help="List files matching glob pattern (e.g., '**/*.parquet')"), ): """ List local datasets matching SPECIFIER. @@ -47,6 +48,7 @@ def ls( owi local ls all owi local ls main:latest --display wide owi local ls all --sort totalSize --reverse + owi local ls all/id=abc123 --files "**/*.parquet" """ cli_ctx: CLIContext = ctx.obj cli_ctx.fields = fields @@ -58,6 +60,27 @@ def ls( query={k: v for k, v in spec.get("query", {}).items() if k != "access"}, ) + # If --files is specified, list files instead of datasets + if files_glob: + from rich.table import Table + for ds in datasets_list: + cli_ctx.console.print(f"\n[bold]Dataset:[/bold] {ds.metadata.get('title', 'Unknown')} ({ds.metadata.id})") + cli_ctx.console.print(f"[dim]Path: {ds.path}[/dim]") + + files = ds.files(files_glob, relative=True) + if files: + table = Table(title=f"Files matching '{files_glob}'", show_header=True) + table.add_column("File", style="cyan") + for f in sorted(files)[:50]: # Limit to 50 files + table.add_row(f) + if len(files) > 50: + table.add_row(f"... and {len(files) - 50} more files") + cli_ctx.console.print(table) + cli_ctx.console.print(f"[green]Total: {len(files)} files[/green]") + else: + cli_ctx.console.print(f"[yellow]No files matching '{files_glob}'[/yellow]") + return + # Sort if requested if sort_by: def sort_key(ds): diff --git a/owilix/cli/remote.py b/owilix/cli/remote.py index cb0d96c..793bec2 100644 --- a/owilix/cli/remote.py +++ b/owilix/cli/remote.py @@ -31,6 +31,7 @@ def ls( reverse: bool = typer.Option(False, "--reverse", "-r", help="Reverse sort order"), no_summary: bool = typer.Option(False, "--no-summary", help="Skip summary table"), fields: Optional[str] = typer.Option(None, "--fields", help="Customize fields (+field, -field)"), + files_glob: Optional[str] = typer.Option(None, "--files", "-f", help="List files matching glob pattern (e.g., '**/*.parquet')"), ): """ List remote datasets matching SPECIFIER. @@ -44,8 +45,9 @@ def ls( Examples: owi remote ls all - owi remote ls lrz:latest --display wide + owi remote ls lexis:latest --display wide owi remote ls it4i:2024-01 --sort totalSize --reverse + owi remote ls all/id=abc123 --files "**/*.parquet" """ cli_ctx: CLIContext = ctx.obj cli_ctx.fields = fields @@ -60,6 +62,30 @@ def ls( query={k: v for k, v in spec.get("query", {}).items() if k != "access"}, ) + # If --files is specified, list files instead of datasets + if files_glob: + from rich.table import Table + for ds in datasets_list: + cli_ctx.console.print(f"\n[bold]Dataset:[/bold] {ds.metadata.get('title', 'Unknown')} ({ds.metadata.id})") + cli_ctx.console.print(f"[dim]DataCenter: {getattr(ds, 'dataCenter', 'unknown')}[/dim]") + + try: + files = ds.files(files_glob, relative=True) + if files: + table = Table(title=f"Files matching '{files_glob}'", show_header=True) + table.add_column("File", style="cyan") + for f in sorted(files)[:50]: # Limit to 50 files + table.add_row(f) + if len(files) > 50: + table.add_row(f"... and {len(files) - 50} more files") + cli_ctx.console.print(table) + cli_ctx.console.print(f"[green]Total: {len(files)} files[/green]") + else: + cli_ctx.console.print(f"[yellow]No files matching '{files_glob}'[/yellow]") + except Exception as e: + cli_ctx.console.print(f"[red]Error listing files: {e}[/red]") + return + # Sort if requested if sort_by: def sort_key(ds): diff --git a/owilix/core/db/duckdb_executor.py b/owilix/core/db/duckdb_executor.py index 48a3108..7a0a5c8 100644 --- a/owilix/core/db/duckdb_executor.py +++ b/owilix/core/db/duckdb_executor.py @@ -259,7 +259,10 @@ class OWIDuckDBSelectExecutor: conn, tmp_dir = conn_tuple # Register the filesystem with the connection if needed - if not conn.filesystem_is_registered(fs.protocol) and not conn.filesystem_is_registered(fs.fsid) : + # fs.protocol can be a string or tuple (e.g., ('file', 'local')) + protocol_name = fs.protocol[0] if isinstance(fs.protocol, tuple) else fs.protocol + fsid = getattr(fs, 'fsid', None) + if not conn.filesystem_is_registered(protocol_name) and (fsid is None or not conn.filesystem_is_registered(fsid)): # Note: fs.protocol usually tuple ('http2irods', 'irods'). DuckDB might expect a string. # Fsspec backends usually register by protocol. # Let's try registering. DuckDB fsspec integration uses the protocol. @@ -497,7 +500,10 @@ class OWIDuckDBCopyExecutor(OWIDuckDBSelectExecutor): conn_tuple = self.pool.acquire_connection() conn, tmp_dir = conn_tuple # Register the filesystem with the DuckDB connection - if not conn.filesystem_is_registered(fs.protocol) and not conn.filesystem_is_registered(fs.fsid) : + # fs.protocol can be a string or tuple (e.g., ('file', 'local')) + protocol_name = fs.protocol[0] if isinstance(fs.protocol, tuple) else fs.protocol + fsid = getattr(fs, 'fsid', None) + if not conn.filesystem_is_registered(protocol_name) and (fsid is None or not conn.filesystem_is_registered(fsid)): try: conn.register_filesystem(fs) except Exception as e: @@ -675,7 +681,10 @@ class OWIDuckDBAsyncExecutor: conn, tmp_dir = conn_tuple # Register filesystem - if not conn.filesystem_is_registered(fs.protocol) and not conn.filesystem_is_registered(fs.fsid): + # fs.protocol can be a string or tuple (e.g., ('file', 'local')) + protocol_name = fs.protocol[0] if isinstance(fs.protocol, tuple) else fs.protocol + fsid = getattr(fs, 'fsid', None) + if not conn.filesystem_is_registered(protocol_name) and (fsid is None or not conn.filesystem_is_registered(fsid)): try: conn.register_filesystem(fs) except Exception as e: diff --git a/owilix/core/repository/file.py b/owilix/core/repository/file.py index 4c4fa29..a17bfb4 100644 --- a/owilix/core/repository/file.py +++ b/owilix/core/repository/file.py @@ -198,7 +198,11 @@ class FileBasedRepository(AbstractRepository): files_glob: str | Sequence[str] | None = None, ) -> Sequence[str]: """Return a list of files under the dataset path matching glob patterns.""" - dataset_path = self._get_path(dataset) + # Use the actual dataset.path rather than reconstructing it, + # since the path may differ from what _get_path() computes + # (e.g., if collectionName in metadata differs from actual directory) + # TODO: Check LexisRepository.files() for the same issue with remote datasets + dataset_path = dataset.path if dataset.path else self._get_path(dataset) if files_glob is None: patterns: Sequence[str] = ["**/*"]