diff --git a/owilix/__init__.py b/owilix/__init__.py index 2d581ee..76ed8bd 100644 --- a/owilix/__init__.py +++ b/owilix/__init__.py @@ -13,9 +13,9 @@ Usage: The CLI provides functionalities to pull and list dataset slices from specified data centers on specified dates or periods. Advanced options allow for filtering and detailed querying with the aid of DuckDB. The default path for storing data can be modified using the OWS_OWI_PATH environment variable. Commands include: -- `owi pull ` to retrieve data. -- `owi ls ` for listing and filtering available files. -- `owi duckdb ` to perform SQL queries on retrieved data. +- `owi remote pull ` to retrieve data. +- `owi remote ls ` for listing and filtering available files. +- `owi remote duckdb ` to perform SQL queries on retrieved data. Advanced Usage: --------------- @@ -29,4 +29,4 @@ For detailed documentation and a screencast demo, visit [OpenWebIndex documentat Developed with Poetry, this module also includes Sphinx documentation setup for easy maintenance and extension of usage docs. """ -__version__ = "0.0.0.post70.dev0+79ef75e" \ No newline at end of file +from ._version import __version__ \ No newline at end of file diff --git a/owilix/_version.py b/owilix/_version.py index e69de29..9ff3f0f 100644 --- a/owilix/_version.py +++ b/owilix/_version.py @@ -0,0 +1 @@ +__version__ = "0.4.1-post.6+e0da862" \ No newline at end of file diff --git a/owilix/cli.py b/owilix/cli.py index 80a8f72..f48dded 100644 --- a/owilix/cli.py +++ b/owilix/cli.py @@ -150,7 +150,7 @@ def config(ctx, option, default): @click.argument('args', nargs=-1) @click.pass_context def local(ctx, subcmd, specifier, args): - kwargs = {k[0]:k[1] for k in [f.split("=") for f in [args for args in args if "=" in args]]} + kwargs = {k[0]:k[1] for k in [f.split("=",1) for f in [args for args in args if "=" in args]]} args = [args for args in args if "=" not in args] LocalCommands(ctx.obj['OWI'], **ctx.obj).do(subcmd, specifier, *args, **kwargs) @@ -160,7 +160,7 @@ def local(ctx, subcmd, specifier, args): @click.argument('args', nargs=-1) @click.pass_context def admin(ctx, subcmd, args): - kwargs = {k[0]:k[1] for k in [f.split("=") for f in [args for args in args if "=" in args]]} + kwargs = {k[0]:k[1] for k in [f.split("=",1 ) for f in [args for args in args if "=" in args]]} args = [args for args in args if "=" not in args] result = AdminCommands(ctx.obj['OWI'], **ctx.obj).do(subcmd, *args, **kwargs) if isinstance(result, pd.DataFrame): @@ -174,7 +174,7 @@ def admin(ctx, subcmd, args): @click.argument('args', nargs=-1) @click.pass_context def remote(ctx, subcmd, specifier, args): - kwargs = {k[0]:k[1] for k in [f.split("=") for f in [args for args in args if "=" in args]]} + kwargs = {k[0]:k[1] for k in [f.split("=",1) for f in [args for args in args if "=" in args]]} args = [args for args in args if "=" not in args] RemoteCommands(ctx.obj['OWI'], **ctx.obj).do(subcmd, specifier, *args, **kwargs) diff --git a/owilix/cmd/base.py b/owilix/cmd/base.py index 56ef527..4785a1c 100644 --- a/owilix/cmd/base.py +++ b/owilix/cmd/base.py @@ -1,8 +1,10 @@ import os import re from collections import Counter +from typing import Iterable import fsspec +from rich.columns import Columns from rich.console import Console from rich.progress import Progress, TextColumn, BarColumn, TimeRemainingColumn @@ -126,7 +128,16 @@ class BaseCommand(metaclass=SubCommandMeta): render_table(list_of_dict, self.console, True, ignore=ignore, colorder=order, tablefmt=tablefmt, roworder=roworder, showfields=showfields) - def show_details(self, data: dict | list, non_indented_key=None): + def show_columns(self, data :Iterable): + """ + show the data as columns + :param data: + :return: + """ + columns = Columns(data, equal=True, expand=True) + self.console.print(columns) + + def show_details(self, data: dict , non_indented_key=None): # Create a table with no header table = Table(show_header=False, box=None) columns = set(data.keys()) @@ -354,7 +365,7 @@ def render_table(data: list, console: Console, show_header: bool = True, for column in columns: table.add_column(column) - if len(roworder)>0: + if roworder is not None and len(roworder)>0: # Custom sort key function def sort_key(item): return [(item.get(key,""), order) for order, key in roworder] diff --git a/owilix/cmd/remote.py b/owilix/cmd/remote.py index 415b9ec..4115d77 100644 --- a/owilix/cmd/remote.py +++ b/owilix/cmd/remote.py @@ -64,6 +64,7 @@ def ls(self, specifier, *args, **kwargs): for d in datasets: _to_show = d.metadata _files = self.owi.remote_data.files(d, kwargs["files"]) + self.console.print(f"Found {len(_files)} files.") if "groups" in kwargs: groups = group_and_count_files(_files, d.path, int(kwargs["groups"])) _to_show["Groups"]=groups @@ -190,8 +191,8 @@ def diff(self, specifier, *args, **kwargs): _local_datasets = self.list_local_datasets_by_specifier(specifier) _remote_datasets = self.list_remote_dataset_by_specifier(specifier) - self.console.print(f"Found {len(_local_datasets)} and {len(_remote_datasets)}") - _ids = set([_d.internalID for _d in _local_datasets]).intersection([[_d.internalID for _d in _remote_datasets]]) + self.console.print(f"Found {len(_local_datasets)} local and {len(_remote_datasets)} remote") + _ids = set([_d.internalID for _d in _local_datasets]).intersection([_d.internalID for _d in _remote_datasets]) if len(_ids)1: - self.console("Local files only:") - self.show_details(_lonly) + self.console.print("Local files only:") + self.show_columns(_lonly) _ronly = set(_remote_files) - set(_local_files) if len(_ronly) > 1: - self.console("Remote files only:") - self.show_details(_ronly) - - + self.console.print("Remote files only:") + self.show_columns(_ronly) @RemoteCommands.register def sync(self, specifier, *args, **kwargs): diff --git a/owilix/core/repository.py b/owilix/core/repository.py index f9a0554..ce4c568 100644 --- a/owilix/core/repository.py +++ b/owilix/core/repository.py @@ -196,6 +196,7 @@ class AggregatedRepository: def create(self, datacenter:str, access:str = "public", collectionName:str="main", + internalID=None, title=None, **metadata): """ Create a new, empty dataset with the given metadata. @@ -210,7 +211,7 @@ class AggregatedRepository: _r = self.get_single_repo(datacenter) if _r is None: return None - return _r.create(access, collectionName, title, **metadata) + return _r.create(access, collectionName, internalID, title, **metadata) def delete(self, dataset:Dataset): _r = self.get_single_repo(dataset.dataCenter) @@ -486,6 +487,7 @@ class AbstractRepository: @abstractmethod def create(self, access:str = "public", collectionName:str="main", + internalID=None, title=None, **metadata): """ Create a new, empty dataset with the given metadata. @@ -493,6 +495,7 @@ class AbstractRepository: Args: uuid_: the uuid of the dataset. If None, a new uuid is generated metadata: a dictionary with the metadata of the dataset + internalID: internalID to use or None, if a new one should be created Returns: the created dataset @@ -563,8 +566,9 @@ class FileBasedRepository(AbstractRepository): def create(self, access:str = "public", collectionName:str="main", + internalID=None, title=None, **metadata): - uuid_ = str(uuid.uuid4()) + uuid_ = internalID if internalID is not None else str(uuid.uuid4()) _ds = Dataset(self, None, internalID=uuid_, collectionName=collectionName, access=access, title=title, **metadata) _p = self._get_path(_ds) _ds.path = _p # weird but necessary @@ -792,7 +796,10 @@ class IRODSRepository(FileBasedRepository): def create(self, access:str = "public", collectionName:str="main", + internalID=None, title=None, **metadata): + if internalID is not None: + raise ValueError("Cannot create dataset in Lexis with given internalID") _lexis = OWILexisDataset(self.manager.session) if title is None: title = "UNKNOWN TITLE" try: diff --git a/pyproject.toml b/pyproject.toml index 344590e..eaf6e92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ build-backend = "poetry_dynamic_versioning.backend" [tool.poetry.scripts] owi = "owilix.cli:main" owilix = "owilix.cli:main" - +update_version = "update_version:main" [tool.git-changelog] repository = "."