From 1d96b31214db3be8bf1875ffe4babe6790981b6f Mon Sep 17 00:00:00 2001 From: mgrani Date: Thu, 21 Nov 2024 08:17:04 +0100 Subject: [PATCH] fix: removed domain_label from some commands, as it is no longer named this way@ --- Readme.md | 4 ++-- owilix/cli.py | 8 +++++++- owilix/cmd/admin.py | 5 +++-- owilix/cmd/query.py | 8 ++++---- owilix/cmd/remote.py | 4 ++++ owilix/core/duckdb.py | 2 +- owilix/core/utils.py | 4 ++-- tests/do_oa.py | 4 ++-- tests/lexis_test.py | 6 +++++- tests/test_cli.py | 2 +- 10 files changed, 31 insertions(+), 16 deletions(-) diff --git a/Readme.md b/Readme.md index b33ffb9..3e6b598 100644 --- a/Readme.md +++ b/Readme.md @@ -242,9 +242,9 @@ So depending on the query (e.g. not very specific select or where statements), i ```sh owilix --remotes it4i query less --remote all:2023-12-4 limit=500 "where=url_suffix='at'" ``` -- list all ".at" websites but fetch 10 batches as buffer and select only url, domain_label and title (should be faster) +- list all ".at" websites but fetch 10 batches as buffer and select only url and title (should be faster) ```sh - owilix --remotes it4i query less --remote it4i:2023-12-3 limit=500 "where=url_suffix='at'" select=url,domain_label,title page_size=30 prefetch=10 + owilix --remotes it4i query less --remote it4i:2023-12-3 limit=500 "where=url_suffix='at'" select=url,title page_size=30 prefetch=10 ``` - list all ".at" websites but only consider files under partition "language=deu". diff --git a/owilix/cli.py b/owilix/cli.py index 36e546a..d58a107 100644 --- a/owilix/cli.py +++ b/owilix/cli.py @@ -277,8 +277,14 @@ def remote(ctx, subcmd, specifier, args): args (list): Additional arguments for the remote command. Example: - owilix remote fetch + owilix remote ls + + owilix remote pull all:latest + + owilix remote pull all/collectionName=legal + owilix remote help + owilix remote help """ args, kwargs = extract_args(args) diff --git a/owilix/cmd/admin.py b/owilix/cmd/admin.py index 62649e1..2b9db30 100644 --- a/owilix/cmd/admin.py +++ b/owilix/cmd/admin.py @@ -77,13 +77,14 @@ def set_irods_metadata_deprecated(self, path, **kwargs): self.show_details(_md) @AdminCommands.register -def set_irods_metadata(self, path, dataset_root:bool =False, do_infer:bool =True, **kwargs): +def set_irods_metadata(self, path, dataset_root:bool =False, do_infer:bool =True, do_count:bool=True, **kwargs): """ sets the metadata for the dataset at the provided path. path must start with irods:// Args: dataset_root (bool): indicates whether the provided file is the rot for a number of datasets or a dataset itself (false). if it is a dataset root, metadata will be set for all datasets contained in the root do_infer (bool): indicates whether some metadata (e.g. subResourceType, counts) should be inferred from the file structure. Might take long as parquet row_count is estimated. inferred metadata overwrites existing metadata. + do_count (bool): indicates, whether inference of metadata also includes counting objects. kwargs (dict): kwargs with additional metdata (usually provided as key=value). if value is empty, the metadata will be deleted. kwargs overwrite all other metadata """ # instantiate sessions here @@ -115,7 +116,7 @@ def set_irods_metadata(self, path, dataset_root:bool =False, do_infer:bool =True task = progress.add_task(f"Filling File Details", total=1) def fn_cb_progress(_path, current, total): progress.update(task, completed=current / total, description=f"Getting file details {_path}") - _file_details = fill_file_details(_fs, _files, _p,fn_cb_progress ) + _file_details = fill_file_details(_fs, _files, _p,fn_cb_progress, do_count=do_count) with Progress(transient=True) as progress: task = progress.add_task(f"Inferring Metadata", total=1) diff --git a/owilix/cmd/query.py b/owilix/cmd/query.py index 442b9bf..5958079 100644 --- a/owilix/cmd/query.py +++ b/owilix/cmd/query.py @@ -88,7 +88,7 @@ class QueryCommands(BaseCommand): @QueryCommands.register def less( self, local_specifier: str, remote_specifier: str, - select: str = "url,domain_label,title,plain_text", + select: str = "url,title,plain_text", where: Optional[str] = "", limit: Optional[int] = None, files: str = "**/*.parquet", explain: bool = False, as_json=False, pq_batch_size: int = 1, batch_size: int = 100, prefetch: int = 2, page_size: int = 10 @@ -100,7 +100,7 @@ def less( self, local_specifier: str, remote_specifier: str, Args: local_specifier (str): The specifier to filter datasets locally. remote_specifier (str): The specifier to filter datasets remotely. - select (str): Comma-separated list of fields specifying the columns to select. Defaults to "url,domain_label,title,plain_text". + select (str): Comma-separated list of fields specifying the columns to select. Defaults to "url,title,plain_text". where (str, optional): WHERE clause to be applied in the SELECT statement. Defaults to an empty string. limit (int, optional): Limit on the number of rows to return. Defaults to None. files (str): Glob pattern for selecting files in both local and remote locations. Defaults to "**/*.parquet". @@ -271,7 +271,7 @@ def slice(self, local_specifier, remote_specifier, @QueryCommands.register def stream(self, local_specifier: str, remote_specifier: str, - select: str = "url,domain_label,title,plain_text", + select: str = "url,title,plain_text", where: Optional[str] = "", limit: Optional[int] = None, files: str = "**/*.parquet", verbose: bool = False, pq_batch_size: int = 1, batch_size: int = 100, prefetch: int = 2, queue_size:int =5, @@ -284,7 +284,7 @@ def stream(self, local_specifier: str, remote_specifier: str, Args: local_specifier (str): The specifier to filter datasets locally. remote_specifier (str): The specifier to filter datasets remotely. - select (str): Comma-separated list of fields specifying the columns to select. Defaults to "url,domain_label,title,plain_text". + select (str): Comma-separated list of fields specifying the columns to select. Defaults to "url,title,plain_text". where (str, optional): WHERE clause to be applied in the SELECT statement. Defaults to an empty string. limit (int, optional): Limit on the number of rows to return. Defaults to None. files (str): Glob pattern for selecting files in both local and remote locations. Defaults to "**/*.parquet". diff --git a/owilix/cmd/remote.py b/owilix/cmd/remote.py index 4269cb3..ef9159b 100644 --- a/owilix/cmd/remote.py +++ b/owilix/cmd/remote.py @@ -127,6 +127,10 @@ def pull(self, specifier, files = "**/*", language = None, overwrite = False): @RemoteCommands.register def doctor(self, *args, **kwargs): + """ + checks the connection status of the remote connections. If there are problems, you can + remove a remote with the flag --remotes + """ self.console.print("Checking connections") self.console.print("Repository status:") for s in [f"\t{k}: {v}" for k, v in self.owi.remote_data.status().items()]: diff --git a/owilix/core/duckdb.py b/owilix/core/duckdb.py index fdc2948..5dd4215 100644 --- a/owilix/core/duckdb.py +++ b/owilix/core/duckdb.py @@ -561,7 +561,7 @@ class OWIDuckDBCopy (OWIDuckDBSelect): # Extract and write the chunk directly to a file, excluding the source_file column copy_query = f""" COPY ( - SELECT url, domain_label, title, plain_text + SELECT url, title, plain_text FROM owi_slice LIMIT {self.chunk_size} OFFSET {offset} ) diff --git a/owilix/core/utils.py b/owilix/core/utils.py index 520b399..ebe847f 100644 --- a/owilix/core/utils.py +++ b/owilix/core/utils.py @@ -102,7 +102,7 @@ def get_filesystem(path): return filepath, fs, protocol -def fill_file_details(fs, files, root=None, cb=None): +def fill_file_details(fs, files, root=None, cb=None, do_count:bool=True): """ Fill the details of the files provided in the files list using the filesystem fs. @@ -127,7 +127,7 @@ def fill_file_details(fs, files, root=None, cb=None): }) if cb is not None: cb(f,ix, _total) - if f.endswith(".parquet"): # count the number of objects in the parquet file + if f.endswith(".parquet") and do_count: # count the number of objects in the parquet file _returns[-1]["objects"] = 0 if f.endswith('.parquet'): try: diff --git a/tests/do_oa.py b/tests/do_oa.py index 1697cef..1e85bef 100644 --- a/tests/do_oa.py +++ b/tests/do_oa.py @@ -78,7 +78,7 @@ def convert_args_kwargs_with_pydantic(func, args: Tuple[Any, ...], kwargs: Dict[ def example_function( local_specifier: str, remote_specifier: str, - select: str = "url,domain_label,title,plain_text", + select: str = "url,title,plain_text", where: Optional[str] = "", limit: Optional[int] = None, files: str = "**/*.parquet", @@ -96,7 +96,7 @@ def example_function( # Example usage args = ("example_local", "example_remote") kwargs = { - "select": "url,domain_label,title", + "select": "url,title", "where": "url_suffix='at'", "limit": "10", "files": "**/*.parquet", diff --git a/tests/lexis_test.py b/tests/lexis_test.py index 99d828f..c222499 100644 --- a/tests/lexis_test.py +++ b/tests/lexis_test.py @@ -1,3 +1,7 @@ +""" +Simple test script to test connection statuses. +""" + from py4lexis.session import LexisSession from py4lexis.lexis_irods import iRODS @@ -11,7 +15,7 @@ from owilix.core.irods import IRODSOWI iowi = IRODSOWI(session) print("Collections at IT4I") -print(iowi.filesystem.ls("/IT4ILexisV2/public")) +print(iowi.filesystem.ls("/IT4ILexisV2/public/proj862c5962623246664c1fda27b7afb108")) print("Collections at LRZ") print(iowi.filesystem.ls("/OWSLRZZONE/public")) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2fe4eec..a50566c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -129,7 +129,7 @@ class TestCLI: def test_config_command(self, runner, temp_dir): - result = runner.invoke(owilix.cli.cli, ['config', 'set', 'showfields=url,domain_label', '--target', str(temp_dir)], prog_name='owilix.cli') + result = runner.invoke(owilix.cli.cli, ['config', 'set', 'showfields=url', '--target', str(temp_dir)], prog_name='owilix.cli') assert result.exit_code == 0 assert "config" in result.output # Adjust this check based on expected output # Verify that configuration was correctly set in the target directory -- 2.51.2