From d6582d8ee25c91894b1236c96d4fd09e5cf02f0b Mon Sep 17 00:00:00 2001 From: mgrani Date: Tue, 17 Dec 2024 08:32:17 +0100 Subject: [PATCH 1/3] docs: improved help message on remote pull@ --- owilix/cmd/remote.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/owilix/cmd/remote.py b/owilix/cmd/remote.py index ef9159b..9d84f89 100644 --- a/owilix/cmd/remote.py +++ b/owilix/cmd/remote.py @@ -88,9 +88,16 @@ def ls(self, specifier, *args, details=False, **kwargs): def pull(self, specifier, files = "**/*", language = None, overwrite = False): """"pulls the datasets with the given specifier to the local repository + Additional Notes: + - wildcards like * are expanded on the linux command line. So you need to escape them. e.g. `owilix --target . remote pull it4i:latest files='**/language=eng/*' + - if the files have been already downloaded, they will not be downloaded again (as with git). Use overwrite to ignore it + + Args: overwrite(bool) : if True, files are overwritten if they exists (default False) - files(str): a glob pattern to select files within the datasets to be pulled + files(str): a glob pattern to select files within the datasets to be pulled. consider using "" or '' to escape wildcards like * + language(str): Language to filter on. Will be deprecated as it is to special. Use files for filtering on language on oiw datasets + overwrite(bool): if true, files will be overwritten. """ self.console.print(f"Fetching datasets for specifier {specifier}") datasets = self.list_remote_dataset_by_specifier(specifier) -- 2.51.2 From 8e0e0f2416404c2d80d8bf549f053f6e3c39d11a Mon Sep 17 00:00:00 2001 From: mgrani Date: Thu, 19 Dec 2024 09:28:10 +0100 Subject: [PATCH 2/3] fix: switchted to py4lexis 3.0.0 on main --- owilix/core/irods.py | 4 ++-- pyproject.toml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/owilix/core/irods.py b/owilix/core/irods.py index 8272fbc..5001f77 100644 --- a/owilix/core/irods.py +++ b/owilix/core/irods.py @@ -33,11 +33,11 @@ class IRODSOWI(iRODS): @property def zone(self): - return self._iRODS__irods.zone + return self._iRODS__irds.zone @property def session(self): - return self._iRODS__irods + return self._iRODS__irds def get_path(self, zone, access, lexis_id, internalId=None): if internalId is not None: diff --git a/pyproject.toml b/pyproject.toml index 334dc2c..b3339f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,21 +24,21 @@ initial-content = """ [tool.poetry.dependencies] python = ">=3.10,<3.12" -py4lexis = {version = "^2.2.3", source = "py4lexis"} +py4lexis = {version = ">2.2.3", source = "py4lexis"} pyarrow = "^16.0.0" pyyaml="^6.0.1" tabulate = "^0.9.0" fsspec = "^2024.3.1" pandas = "^2.0.3" tqdm = "^4.66.2" -click= "8.1.6" +click= "^8.1.6" irods-fsspec="^0.0.1" duckdb = {version = "^1.0.0"} ipython = {version = "^8.24.0"} pydantic = "^2.5.3" rich = "^13.7.0" numpy = ">2.0.0" -typer = "^0.12.5" +typer = "^0.13.1" requests = "^2.32.3" python-dateutil = "^2.9.0.post0" -- 2.51.2 From 91632b2cdeb6801cfc2fa582ce51a9fae02a090e Mon Sep 17 00:00:00 2001 From: semantic-release-mgrani Date: Thu, 19 Dec 2024 09:28:29 +0100 Subject: [PATCH 3/3] 0.16.4 Automatically generated by python-semantic-release --- CHANGELOG.md | 131 +++++++++++++++++++++++++-------------------- owilix/_version.py | 2 +- pyproject.toml | 2 +- 3 files changed, 74 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f48b9a..f951626 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # CHANGELOG +## v0.16.4 (2024-12-19) + +### Bug Fixes + +- Switchted to py4lexis 3.0.0 on main + ([`8e0e0f2`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/8e0e0f2416404c2d80d8bf549f053f6e3c39d11a)) + +### Documentation + +- Improved help message on remote pull@ + ([`d6582d8`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/d6582d8ee25c91894b1236c96d4fd09e5cf02f0b)) + + ## v0.16.3 (2024-11-28) ### Bug Fixes @@ -72,12 +85,12 @@ ### Features -- Added server side configuration and license agreement - ([`25ba173`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/25ba1730707272f47fbffc5b2088635097849533)) - - Add option to turn common ngrams count off ([`4b781d2`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/4b781d2afea16389513b49c91d1773cd51bf0939)) +- Added server side configuration and license agreement + ([`25ba173`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/25ba1730707272f47fbffc5b2088635097849533)) + - Bloom filter querying now working (including a test example). Still slow as rust does not use memory mapped files. ([`535a16e`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/535a16e367f0bb4525eebec4ebd826041c57da9d)) @@ -91,12 +104,6 @@ ### Bug Fixes -- Alpha version for bloom filter - ([`b7d090d`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/b7d090d1c0201c4cbf863869c1038e61c56d5a49)) - -- Improved default configuration - ([`21d55f1`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/21d55f1169834187e45266f8233a10136b1ab67c)) - - Added bloom filter plugin create command ([`289fda6`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/289fda6852eee7d5ac5e50404cdd41864242916b)) @@ -115,6 +122,12 @@ - Added test for creating bloom ngram filters for text reuse detection ([`3aee073`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/3aee0734b2d5c1b8a340c323a0a04f1142edcef2)) +- Alpha version for bloom filter + ([`b7d090d`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/b7d090d1c0201c4cbf863869c1038e61c56d5a49)) + +- Improved default configuration + ([`21d55f1`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/21d55f1169834187e45266f8233a10136b1ab67c)) + ## v0.12.0 (2024-07-29) @@ -206,6 +219,13 @@ ### Bug Fixes +- Nodisplay parameter adds to the list of fields to not display instead of replacing it. + ([`d725c51`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/d725c51ad691ea088bf78e92c295b5b1e627012f)) + +- Non existing metadata did raise an Attribute Error, which becomes problematic if metadata is not + fully correctly filled. Now dataset returns None if an attribute/metadata is not existing. + ([`8b9abcf`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/8b9abcfeb913fc27f9de0633b4a9eb00b67a0964)) + - Set irods metadata now has infer flag. irods url config supports now user+zone syntax ([`b37a4d3`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/b37a4d3b76cf0aec7e2bf4bf8f52d25898181afd)) @@ -215,27 +235,20 @@ - Set irods metadata now has infer flag. irods url config supports now user+zone syntax ([`b77575e`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/b77575eaaa64c86278935f2758bc9980ea4f59d5)) -- Nodisplay parameter adds to the list of fields to not display instead of replacing it. - ([`d725c51`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/d725c51ad691ea088bf78e92c295b5b1e627012f)) - -- Non existing metadata did raise an Attribute Error, which becomes problematic if metadata is not - fully correctly filled. Now dataset returns None if an attribute/metadata is not existing. - ([`8b9abcf`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/8b9abcfeb913fc27f9de0633b4a9eb00b67a0964)) - ## v0.11.0 (2024-06-24) ### Bug Fixes +- Added typecasting for parameter calls + ([`06b6e1e`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/06b6e1e7c34abe51a8aebef1e38aeecb5f084938)) + - Corrected error reporting when slicing ([`7fc1500`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7fc15002e394b31edced5bc2896cf06850378ebe)) - Corrected error reporting when slicing ([`55eecfb`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/55eecfba7d2e69afe8a0cece08bbc3cfc0a856ea)) -- Added typecasting for parameter calls - ([`06b6e1e`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/06b6e1e7c34abe51a8aebef1e38aeecb5f084938)) - ### Chores - Changelog now writes the correct commit url @@ -259,18 +272,18 @@ ### Bug Fixes -- Retry count for duckdb arrow command - ([`e9c4a25`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e9c4a255b97dd1a988c3ee540c606b999e91e8de)) - -- Corrected merge problem - ([`8c35a11`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/8c35a11501567fcbe3067eff76ea661359249a3f)) - - Added retry count for duckdb query to avoid server errors ([`6b83a2a`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/6b83a2ad35b12aa0fae1ee5f3ac7d4a134c45367)) - Added retry count for duckdb query to avoid server errors ([`2e9eb80`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/2e9eb80c0ef8d02c313221eb04fba3c2ad2bf651)) +- Corrected merge problem + ([`8c35a11`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/8c35a11501567fcbe3067eff76ea661359249a3f)) + +- Retry count for duckdb arrow command + ([`e9c4a25`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e9c4a255b97dd1a988c3ee540c606b999e91e8de)) + ### Chores - Added gitlab as vcs for build process @@ -292,8 +305,8 @@ ### Bug Fixes -- Non-existing collection folder yielded an exception when pulling the first time. - ([`3aae39f`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/3aae39f7c479c677deda9695c1ec5c678b68fecf)) +- Documentation was not shown properly + ([`050d5a1`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/050d5a1b99c0b1e0eb982cd31414afadb067ef30)) - Irods works now when client_zone needs to be selected. ([`4b8f706`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/4b8f706b13d227c012de5f75d736922e4f6f3ccb)) @@ -301,36 +314,36 @@ - Irods works now when client_zone needs to be selected. ([`2477de8`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/2477de80813e7050ddb02ebc4c23bc338b0eaae8)) -- Documentation was not shown properly - ([`050d5a1`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/050d5a1b99c0b1e0eb982cd31414afadb067ef30)) - - Lexis error log not found message implemented. ([`5891717`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/589171740853d85cf278fc22ed40adb5b262dc9d)) -### Chores +- Non-existing collection folder yielded an exception when pulling the first time. + ([`3aae39f`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/3aae39f7c479c677deda9695c1ec5c678b68fecf)) -- Optimized imports - ([`89a7c78`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/89a7c78303d88e9f15c3f04e78eee95fa6c50bcd)) +### Chores - Numpy 2.0.0 yielded a problem with libraries in ubuntu ([`1d4ba04`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/1d4ba043c06c8f873c4af6a9aa09a1f0dd36ae84)) -### Documentation +- Optimized imports + ([`89a7c78`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/89a7c78303d88e9f15c3f04e78eee95fa6c50bcd)) -- Updated doc (a bit) - ([`0fe822c`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/0fe822cdaaf23aed33c0ba8f4553a944568133dd)) +### Documentation - Improved documentation ([`7becf4a`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7becf4a7cc2a9eda707de44ef80887fa455f8919)) -### Features +- Updated doc (a bit) + ([`0fe822c`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/0fe822cdaaf23aed33c0ba8f4553a944568133dd)) -- New command: local free to free space (i.e. remove local datasets) - ([`3b75b8a`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/3b75b8a133f05aedec20510e040b7c510c9e4e28)) +### Features - Added --default_config flag to not load stored config. ([`ff339bf`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/ff339bfc7f8d8e6f8d3b46462e923971f2a50bc4)) +- New command: local free to free space (i.e. remove local datasets) + ([`3b75b8a`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/3b75b8a133f05aedec20510e040b7c510c9e4e28)) + ## v0.7.4 (2024-06-17) @@ -369,36 +382,36 @@ ### Bug Fixes -- Local ls returns dataset over all collections per default - ([`7d037ab`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7d037ab148821aab8dac41b4940e84cc4b57d42d)) - - Configuration now supports query federation via zone_path config. Datasets in lrz become available with that setting ([`d6b5813`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/d6b5813e9ecf3d8b16939c69119634d4c9c687f0)) +- Local ls returns dataset over all collections per default + ([`7d037ab`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7d037ab148821aab8dac41b4940e84cc4b57d42d)) + - Metadata is now casted after update to avoid list vs. value problems ([`7d4fcb2`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7d4fcb279b72ba2c226d67557718402a70dd206c)) ### Chores -- Updated py4lexis to 2.1.3 and duckdb to 1.0.0 - ([`02e672c`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/02e672cf882099006eade0501d0565823bd07344)) - - Moved build to dev dependency group ([`0734502`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/07345027ab96040ea8e190b9259cee2973e10a40)) - Removed python tuple version ([`cf78677`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/cf7867766f9e43c55f51d51951890277fe04b509)) -### Features +- Updated py4lexis to 2.1.3 and duckdb to 1.0.0 + ([`02e672c`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/02e672cf882099006eade0501d0565823bd07344)) -- Update_metadata command available (but untested due to rights error) - ([`39d0408`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/39d04080610b07f98d00b6c12239481264f43d67)) +### Features - Slicing finished - when slicing, provenance chain is established and a new dataset is locally created. ([`006f439`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/006f439187c10a4239755692e5cf2695d4d55551)) +- Update_metadata command available (but untested due to rights error) + ([`39d0408`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/39d04080610b07f98d00b6c12239481264f43d67)) + ### Refactoring - Removed slice command and field @@ -474,28 +487,28 @@ ### Chores -- Added upate of versioning in code - ([`a97a386`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/a97a3862096b3e61a03437db9c6b570c11f339a3)) - -- Remove obsolete, old code. cleanup of the module. - ([`565bf18`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/565bf18a4a2f198d10133e136c74644ac27aeeaa)) - - Added git-changelog; poetry dev and docs groups ([`7a3a076`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7a3a076e87ed0c09df35cf11bec9f9cac7448144)) - Added git-changelog; poetry dev and docs groups ([`0d550fc`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/0d550fc1282c42aeca611d928a7a3380f85291df)) -### Features +- Added upate of versioning in code + ([`a97a386`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/a97a3862096b3e61a03437db9c6b570c11f339a3)) -- Inserting datasets loally from path - ([`a3ef490`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/a3ef490eaf6826427c852ff8e16f352d9d530ee7)) +- Remove obsolete, old code. cleanup of the module. + ([`565bf18`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/565bf18a4a2f198d10133e136c74644ac27aeeaa)) -- Loca/remote diffs between datasets and files - ([`8f9af3d`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/8f9af3d66e0f081fa2df9a5e789545c28f448b01)) +### Features - Added regexp based query in specifier. Field needs to end with * ([`e0da862`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e0da862019593b3853b2035a607b1b865f75c058)) - Improved control over displayed columns and sort order ([`b90ff3f`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/b90ff3f5999cd20c8a4b04a7f762ef0067cf4108)) + +- Inserting datasets loally from path + ([`a3ef490`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/a3ef490eaf6826427c852ff8e16f352d9d530ee7)) + +- Loca/remote diffs between datasets and files + ([`8f9af3d`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/8f9af3d66e0f081fa2df9a5e789545c28f448b01)) diff --git a/owilix/_version.py b/owilix/_version.py index 33df529..31d4281 100644 --- a/owilix/_version.py +++ b/owilix/_version.py @@ -1,3 +1,3 @@ # These version placeholders will be replaced later during substitution. -__version__ = "0.16.3" +__version__ = "0.16.4" __version_tuple__ = (0, 15, 0) diff --git a/pyproject.toml b/pyproject.toml index b3339f2..ae56a50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "owilix" -version = "0.16.3" +version = "0.16.4" description = "OWILIX - the Command Line Interface for slicing and consuming the Open Web Index. " authors = ["Michael Granitzer "] license = "MIT" -- 2.51.2