diff --git a/owilix/cli/__init__.py b/owilix/cli/__init__.py index 140546d..03b5ce5 100644 --- a/owilix/cli/__init__.py +++ b/owilix/cli/__init__.py @@ -44,6 +44,10 @@ def main( None, "-t", "--target", help="Target directory for data storage" ), + no_progress: bool = typer.Option( + False, "-N", "--no-progress", + help="Suppress progress bars globally" + ), ): """ OWILIX - Open Web Index Client @@ -57,7 +61,14 @@ def main( output_file=output_file, no_display=no_display, target=target, + no_progress=no_progress, ) + + # Patch legacy progress bars if globally suppressed + if no_progress: + import owilix.cmd.base + from ._common.progress import DummyProgress + owilix.cmd.base.currentItemProgress = lambda *args, **kwargs: DummyProgress() # Import and register sub-apps diff --git a/owilix/cli/_common/context.py b/owilix/cli/_common/context.py index 21704ce..835ea18 100644 --- a/owilix/cli/_common/context.py +++ b/owilix/cli/_common/context.py @@ -36,6 +36,8 @@ class CLIContext(BaseModel): output_file: Optional[str] = Field(default=None, description="Write output to file") no_display: bool = Field(default=False, description="Suppress console output") target: Optional[str] = Field(default=None, description="Target directory") + no_progress: bool = Field(default=False, description="Suppress progress bars") + fields: Optional[str] = Field(default=None, description="Custom fields (+field,-field)") # Additional context from legacy CLI showfields: Optional[str] = Field(default=None, description="Fields to show in display") @@ -50,6 +52,8 @@ def create_context( output_file: Optional[str] = None, no_display: bool = False, target: Optional[str] = None, + no_progress: bool = False, + fields: Optional[str] = None, ) -> CLIContext: """ Create a CLI context with initialized OWIlixManager. @@ -61,6 +65,8 @@ def create_context( output_file: Optional output file path no_display: Suppress console output target: Target directory for data + no_progress: Suppress progress bars + fields: Custom field selection Returns: Initialized CLIContext @@ -95,6 +101,8 @@ def create_context( output_file=output_file, no_display=no_display, target=owi_path, + no_progress=no_progress, + fields=fields, ) diff --git a/owilix/cli/_common/output.py b/owilix/cli/_common/output.py index 472ae9a..6a9bf5e 100644 --- a/owilix/cli/_common/output.py +++ b/owilix/cli/_common/output.py @@ -42,19 +42,38 @@ class OutputWriter: def __init__(self, ctx: "CLIContext"): """ Initialize output writer from CLI context. - - Args: - ctx: CLI context with output preferences """ self.ctx = ctx self.format = OutputFormat(ctx.output_format) self.console = ctx.console self._file = None + # Parse field customization from context + self.show_fields = [] + self.hide_fields = [] + if getattr(ctx, 'fields', None): + for f in ctx.fields.split(','): + f = f.strip() + if f.startswith('+'): + self.show_fields.append(f[1:].lower()) + elif f.startswith('-'): + self.hide_fields.append(f[1:].lower()) + else: + self.show_fields.append(f.lower()) + # Open output file if specified if ctx.output_file: self._file = open(ctx.output_file, 'w', encoding='utf-8') + def _is_field_visible(self, field_name: str, default_visible: bool = True) -> bool: + """Check if field should be shown based on customization.""" + field_name = field_name.lower() + if field_name in self.hide_fields: + return False + if field_name in self.show_fields: + return True + return default_visible + def write_records( self, records: Union[Iterator[Dict[str, Any]], List[Dict[str, Any]]], @@ -75,6 +94,12 @@ class OutputWriter: elif self.format == OutputFormat.JSON: self._write_json(list(records) if not isinstance(records, list) else records) elif self.format == OutputFormat.TABLE: + # Filter columns if customized + if columns is None and records: + # Get all keys but filter + all_keys = list(records[0].keys()) if isinstance(records, list) else [] + columns = [k for k in all_keys if self._is_field_visible(k)] + self._write_table( list(records) if not isinstance(records, list) else records, title=title, @@ -151,6 +176,8 @@ class OutputWriter: self._write_datasets_short(datasets) elif display_format == "wide": self._write_datasets_wide(datasets) + elif display_format == "full": + self._write_datasets_full(datasets) elif display_format == "markdown": self._write_datasets_markdown(datasets) else: @@ -162,7 +189,10 @@ class OutputWriter: self.console.print(f"\n[bold]Summary:[/bold] {len(datasets)} datasets, {total_size/1e9:.1f} GB, {total_files:,} files") def _write_datasets_short(self, datasets): - """Compact ls-style display format: 📦 id title """ + """ + Compact single-line format: 📦 ID TITLE n=COUNT SIZE [ACCESS] + Grep-friendly (no emojis/colors if not terminal). + """ for d in datasets: ds_id = d.metadata.get("internalID", "N/A") or "N/A" title = d.metadata.get("title", "Untitled") or "Untitled" @@ -184,38 +214,126 @@ class OutputWriter: size_gib = float(total_size) / (1024**3) access = getattr(d, 'access', None) or d.metadata.get("access", "?") - # Last change - last_change = d.metadata.get("lastChanged") or d.metadata.get("modificationDate", "") - if last_change: - last_change = str(last_change).split("T")[0] if "T" in str(last_change) else str(last_change)[:10] + # Single line format + line = f"[bold cyan]📦 {ds_id}[/] [bold]{title}[/] [dim]<{date_range};{coll};{dc}>[/] n={int(obj_count):,} {size_gib:.2f}GiB [{access}]" + self.console.print(line) + + def _write_datasets_wide(self, datasets): + """ + Rich Panel format with detailed metadata. + Uses indented key-value pairs for better readability. + """ + from rich.panel import Panel + from rich.text import Text + from rich.box import ROUNDED + + for d in datasets: + md = d.metadata + content = Text() + + # Helper for fields + def add_field(key, value, indent=0, style="dim", field_id=None): + fid = field_id or key.lower() + if self._is_field_visible(fid) and value: + prefix = " " * indent + content.append(f"{prefix}{key}: ", style="bold " + style) + content.append(str(value) + "\n", style=style) + + # Core Info + add_field("Title", md.get("title"), style="cyan") + add_field("ID", md.get("internalID")) - # Build output - line1 = f"[bold cyan]📦 {ds_id}[/] [bold]{title}[/]" - line2 = f" [dim]<{date_range};{coll};{dc}>[/] n={int(obj_count):,} {size_gib:.2f}GiB [{access}]" - if last_change: - line2 += f" [dim]🔄{last_change}[/]" + # Detailed sections + add_field("Path", d.path, indent=1) + add_field("Dates", f"{md.get('startDate')} - {md.get('endDate')}", indent=1) + add_field("Size", f"{md.get('totalSize', 0)/1e9:.2f} GB ({md.get('fileCount', 0)} files, {md.get('objectCount', 0)} objects)", indent=1) - self.console.print(line1) - self.console.print(line2) - self.console.print() - - def _write_datasets_wide(self, datasets): - """Full table format with all fields.""" - records = [ - { - "id": d.metadata.get('internalID', '')[:12] + "..." if len(d.metadata.get('internalID', '')) > 12 else d.metadata.get('internalID', ''), - "title": d.metadata.get('title', '') or "", - "collection": d.metadata.get('collectionName', '') or "", - "dc": getattr(d, 'dataCenter', '') or "", - "start": str(d.metadata.startDate)[:10] if d.metadata.startDate else "?", - "end": str(d.metadata.endDate)[:10] if d.metadata.endDate else "?", - "size": f"{d.metadata.get('totalSize', 0) / 1e9:.1f}G", - "files": d.metadata.get('fileCount', 0) or 0, - "access": getattr(d, 'access', '') or "", - } - for d in datasets - ] - self._write_table(records, title=f"Datasets ({len(datasets)})") + # Collection/Location + add_field("Collection", md.get("collectionName"), indent=1) + add_field("DataCenter", getattr(d, 'dataCenter', None), indent=1) + add_field("Access", getattr(d, 'access', None) or md.get("access"), indent=1) + + # Descriptions + if self._is_field_visible("descriptions"): + descriptions = md.get("descriptions", []) + if isinstance(descriptions, str): + descriptions = [descriptions] + + if descriptions: + content.append(" Descriptions:\n", style="bold") + for desc in descriptions[:3]: # Show first 3 + if isinstance(desc, dict): + txt = desc.get("description", "") + dtype = desc.get("type", "") + content.append(f" - [{dtype}] {txt[:100]}{'...' if len(txt)>100 else ''}\n") + else: + content.append(f" - {str(desc)[:100]}\n") + + # Creators + if self._is_field_visible("creators"): + creators = md.get("creators", []) + if isinstance(creators, str): + creators = [creators] + + if creators: + names = [] + for c in creators: + if isinstance(c, dict): + names.append(c.get("name", str(c))) + else: + names.append(str(c)) + add_field("Creators", ", ".join(names), indent=1, field_id="creators") + + # Render panel + self.console.print(Panel( + content, + title=f"[bold]{md.get('title')}[/] ({md.get('internalID')})", + expand=True, + border_style="blue", + box=ROUNDED + )) + + def _write_datasets_full(self, datasets): + """Show ALL metadata in a pager.""" + import yaml + + # Collect all data + all_output = [] + for d in datasets: + # Combine object attrs + metadata + try: + # Try explicit to_dict first + if hasattr(d.metadata, 'to_dict'): + data = d.metadata.to_dict() + # Try internal dict storage + elif hasattr(d.metadata, '_metadata'): + data = dict(d.metadata._metadata) + # Try direct iteration (if dict-like) + else: + data = dict(d.metadata) + except Exception: + # Fallback if metadata is opaque + data = {"error": "Could not serialize metadata", "raw": str(d.metadata)} + + # Add implicit fields + data['_path'] = str(d.path) + data['_access'] = getattr(d, 'access', None) + data['_dataCenter'] = getattr(d, 'dataCenter', None) + + # Format as YAML block + try: + yaml_str = yaml.dump(data, sort_keys=False, default_flow_style=False, allow_unicode=True) + except Exception as e: + yaml_str = f"Error dumping YAML: {e}\n{data}" + + all_output.append(f"---\n# Dataset: {data.get('title', 'Unknown')}\n{yaml_str}") + + # Page it + if all_output: + with self.console.pager(): + self.console.print("\n".join(all_output)) + else: + self.console.print("[dim]No datasets to display[/dim]") def _write_datasets_markdown(self, datasets): """Markdown table format.""" diff --git a/owilix/cli/_common/progress.py b/owilix/cli/_common/progress.py index 73a4e48..0c919b1 100644 --- a/owilix/cli/_common/progress.py +++ b/owilix/cli/_common/progress.py @@ -3,25 +3,39 @@ Progress utilities for CLI commands. Re-exports ui.py progress bar helpers for consistent usage across CLI. """ +from typing import Optional, Any +from contextlib import contextmanager + from owilix.core.manager.ui import EnhancedProgressDisplay, ErrorCollector # Import simple progress from base.py for simpler operations from rich.progress import Progress, BarColumn, TextColumn, TimeRemainingColumn -def simple_progress(): +class DummyProgress: + """A no-op progress bar compatible with Rich Progress interface.""" + def __init__(self, *args, **kwargs): pass + def __enter__(self): return self + def __exit__(self, *args, **kwargs): pass + def add_task(self, *args, **kwargs): return 0 + def update(self, *args, **kwargs): pass + def start(self, *args, **kwargs): pass + def stop(self, *args, **kwargs): pass + + +def simple_progress(ctx: Optional[Any] = None) -> Progress: """ Create a simple progress bar for operations like ls, pull, push. - Usage: - with simple_progress() as progress: - task = progress.add_task("Downloading...", total=10, current_item="") - for item in items: - progress.update(task, advance=1, current_item=item.name) - + Args: + ctx: Optional CLIContext. If provided and ctx.no_progress is True, returns DummyProgress. + Returns: - Rich Progress context manager + Rich Progress context manager or DummyProgress """ + if ctx and getattr(ctx, 'no_progress', False): + return DummyProgress() + return Progress( TextColumn("[progress.description]{task.description}"), BarColumn(), @@ -31,20 +45,23 @@ def simple_progress(): ) -def create_enhanced_progress(console, task_name: str = "records processed"): +def create_enhanced_progress(console, task_name: str = "records processed", ctx: Optional[Any] = None): """ Create an enhanced progress display for complex operations with error tracking. - Used for query commands that need error tracking and multiple metrics. - Args: console: OWILIXConsole or Rich Console - task_name: Description of what's being counted (e.g., "records processed") + task_name: Description of what's being counted + ctx: Optional CLIContext. If provided and ctx.no_progress is True, returns dummies. Returns: - tuple: (EnhancedProgressDisplay, ErrorCollector) + tuple: (EnhancedProgressDisplay/Dummy, ErrorCollector) """ error_collector = ErrorCollector(max_recent_errors=10) + + if ctx and getattr(ctx, 'no_progress', False): + return DummyProgress(), error_collector + progress = EnhancedProgressDisplay( console, error_collector, task_name=task_name ) diff --git a/owilix/cli/local.py b/owilix/cli/local.py index 8f7d550..7b72e8f 100644 --- a/owilix/cli/local.py +++ b/owilix/cli/local.py @@ -29,6 +29,7 @@ def ls( sort_by: str = typer.Option("", "--sort", "-s", help="Sort by field (e.g., startDate, totalSize)"), reverse: bool = typer.Option(False, "--reverse", "-r", help="Reverse sort order"), no_summary: bool = typer.Option(False, "--no-summary", help="Skip summary table"), + fields: Optional[str] = typer.Option(None, "--fields", help="Customize fields (+field, -field)"), ): """ List local datasets matching SPECIFIER. @@ -46,6 +47,7 @@ def ls( owi local ls all --sort totalSize --reverse """ cli_ctx: CLIContext = ctx.obj + cli_ctx.fields = fields # Parse specifier and list datasets spec = cli_ctx.owi.parse_specifier(specifier) @@ -131,7 +133,7 @@ def rm( return # Remove datasets with progress - with simple_progress() as progress: + with simple_progress(cli_ctx) as progress: task = progress.add_task("Removing datasets...", total=len(datasets), current_item="") for ds in datasets: diff --git a/owilix/cli/remote.py b/owilix/cli/remote.py index 80d88c1..35fccbf 100644 --- a/owilix/cli/remote.py +++ b/owilix/cli/remote.py @@ -25,82 +25,50 @@ app = typer.Typer( def ls( ctx: typer.Context, specifier: str = typer.Argument("all", help="Dataset specifier (e.g., 'lrz:latest')"), - details: bool = typer.Option(False, "-l", "--details", help="Show detailed information"), - files: Optional[str] = typer.Option(None, "--files", help="File glob pattern"), - groups: Optional[int] = typer.Option(None, "--groups", help="Group by size"), + display_format: str = typer.Option("short", "--display", "-d", help="Display format: short, wide, markdown"), + sort_by: str = typer.Option("", "--sort", "-s", help="Sort by field (e.g., startDate, totalSize)"), + reverse: bool = typer.Option(False, "--reverse", "-r", help="Reverse sort order"), no_summary: bool = typer.Option(False, "--no-summary", help="Skip summary table"), + fields: Optional[str] = typer.Option(None, "--fields", help="Customize fields (+field, -field)"), ): """ List remote datasets matching SPECIFIER. Specifier format: :#/= + Display formats: + short - Compact ls-style (default) + wide - Full table with all fields + markdown - Markdown table format + Examples: owi remote ls all - owi remote ls lrz:latest - owi remote ls it4i:2024-01#7/access=public + owi remote ls lrz:latest --display wide + owi remote ls it4i:2024-01 --sort totalSize --reverse """ cli_ctx: CLIContext = ctx.obj + cli_ctx.fields = fields + + # Parse specifier and list datasets + spec = cli_ctx.owi.parse_specifier(specifier) + datasets_list = cli_ctx.owi.remote_data.list( + datacenter=spec.get("data_center"), + access=spec.get("query", {}).get("access", "public"), + day=spec.get("day"), + duration=spec.get("duration") or 0, + query={k: v for k, v in spec.get("query", {}).items() if k != "access"}, + ) + + # Sort if requested + if sort_by: + def sort_key(ds): + val = ds.metadata.get(sort_by) + return (val is not None, val) + datasets_list.sort(key=sort_key, reverse=reverse) with OutputWriter(cli_ctx) as writer: - # Parse specifier and list datasets using manager API - spec = cli_ctx.owi.parse_specifier(specifier) - datasets_list = cli_ctx.owi.remote_data.list( - datacenter=spec.get("data_center"), - access=spec.get("query", {}).get("access", "public"), - day=spec.get("day"), - duration=spec.get("duration") or 0, - query={k: v for k, v in spec.get("query", {}).items() if k != "access"}, - ) - - if cli_ctx.output_format == "table": - # Use table display - if not datasets_list: - writer.write_message("[dim]No datasets found[/dim]") - return - - # Access attributes via metadata - def get_id(ds): - return ds.metadata.get('internalID') or ds.metadata.get('id') or '' - - records = [ - { - "id": get_id(ds)[:8] + "..." if len(get_id(ds)) > 8 else get_id(ds), - "title": ds.metadata.get('title', '') or "", - "collection": ds.metadata.get('collectionName', '') or "", - "dc": getattr(ds, 'dataCenter', '') or "", - "dates": f"{ds.metadata.startDate.strftime('%Y-%m-%d') if ds.metadata.startDate else '?'}-{ds.metadata.endDate.strftime('%Y-%m-%d') if ds.metadata.endDate else '?'}", - "size": f"{ds.metadata.get('totalSize', 0) / 1e9:.1f}G" if ds.metadata.get('totalSize') else "?", - "files": ds.metadata.get('fileCount', 0) or 0, - } - for ds in datasets_list - ] - - if details: - # Show full IDs and more details - records = [ - { - "id": get_id(ds), - "title": ds.metadata.get('title', '') or "", - "collectionName": ds.metadata.get('collectionName', '') or "", - "dataCenter": getattr(ds, 'dataCenter', '') or "", - "startDate": str(ds.metadata.startDate) if ds.metadata.startDate else None, - "endDate": str(ds.metadata.endDate) if ds.metadata.endDate else None, - "size": ds.metadata.get('totalSize'), - "fileCount": ds.metadata.get('fileCount'), - "access": getattr(ds, 'access', '') or "", - } - for ds in datasets_list - ] - - writer.write_records(records, title=f"Remote Datasets ({len(records)})") - - if not no_summary: - total_size = sum(ds.metadata.get('totalSize', 0) or 0 for ds in datasets_list) - total_files = sum(ds.metadata.get('fileCount', 0) or 0 for ds in datasets_list) - writer.write_message(f"\n[bold]Summary:[/bold] {len(datasets_list)} datasets, {total_size/1e9:.1f} GB, {total_files:,} files") - else: - # JSON/JSONL output + # For JSON output, use records format + if cli_ctx.output_format in ("json", "jsonl"): def get_id(ds): return ds.metadata.get('internalID') or ds.metadata.get('id') @@ -119,6 +87,13 @@ def ls( for ds in datasets_list ] writer.write_records(records) + else: + # Use display format for table output + writer.write_datasets( + datasets_list, + display_format=display_format, + show_summary=not no_summary, + ) @app.command()