diff --git a/CHANGELOG.md b/CHANGELOG.md index ce8f7af..81b43a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,452 +2,459 @@ ## v0.10.0 (2024-06-21) -### Unknown - -* Merge branch 'main' of https://opencode.it4i.eu/openwebsearcheu-public/owi-cli ([`98a381a`](https://github.com/openwebsearcheu-public/owi-cli/commit/98a381a3d1772064082b2c60735e8d4586db9878)) - -## v0.9.0 (2024-06-21) - ### Chore -* chore: added gitlab as vcs for build process ([`9d9a2b7`](https://github.com/openwebsearcheu-public/owi-cli/commit/9d9a2b7bcb7976c51af901071fe1a83e3f9ae197)) +* chore: added gitlab as vcs for build process ([`9d9a2b7`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/9d9a2b7bcb7976c51af901071fe1a83e3f9ae197)) ### Feature -* feat: stream now writes to socket ([`31a2c16`](https://github.com/openwebsearcheu-public/owi-cli/commit/31a2c16974c2e3deacb3233dd657aca0c1e7e427)) +* feat: stream now writes to socket ([`31a2c16`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/31a2c16974c2e3deacb3233dd657aca0c1e7e427)) -* feat: added query stream command - owilix now provides a stream of pyarrow instances from duckdb results ([`1921086`](https://github.com/openwebsearcheu-public/owi-cli/commit/19210866b14c41cadf0f2ea45e532849b408f241)) +* feat: added query stream command - owilix now provides a stream of pyarrow instances from duckdb results ([`1921086`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/19210866b14c41cadf0f2ea45e532849b408f241)) -* feat: added query stream command - owilix now provides a stream of pyarrow instances from duckdb results ([`4afd05f`](https://github.com/openwebsearcheu-public/owi-cli/commit/4afd05ff1b34254c4b54f3de314eabf8162fbd85)) +* feat: added query stream command - owilix now provides a stream of pyarrow instances from duckdb results ([`4afd05f`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/4afd05ff1b34254c4b54f3de314eabf8162fbd85)) -* feat: slice now allows to add to existing dataset and considers proveancne syncing ([`02e24fe`](https://github.com/openwebsearcheu-public/owi-cli/commit/02e24febf5951f308faae9afde373f7ddc78a25d)) +* feat: slice now allows to add to existing dataset and considers proveancne syncing ([`02e24fe`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/02e24febf5951f308faae9afde373f7ddc78a25d)) ### Fix -* fix: corrected merge problem ([`8c35a11`](https://github.com/openwebsearcheu-public/owi-cli/commit/8c35a11501567fcbe3067eff76ea661359249a3f)) +* fix: corrected merge problem ([`8c35a11`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/8c35a11501567fcbe3067eff76ea661359249a3f)) -* fix: retry count for duckdb arrow command ([`e9c4a25`](https://github.com/openwebsearcheu-public/owi-cli/commit/e9c4a255b97dd1a988c3ee540c606b999e91e8de)) +* fix: retry count for duckdb arrow command ([`e9c4a25`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e9c4a255b97dd1a988c3ee540c606b999e91e8de)) -* fix: added retry count for duckdb query to avoid server errors ([`6b83a2a`](https://github.com/openwebsearcheu-public/owi-cli/commit/6b83a2ad35b12aa0fae1ee5f3ac7d4a134c45367)) +* fix: added retry count for duckdb query to avoid server errors ([`6b83a2a`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/6b83a2ad35b12aa0fae1ee5f3ac7d4a134c45367)) -* fix: added retry count for duckdb query to avoid server errors ([`2e9eb80`](https://github.com/openwebsearcheu-public/owi-cli/commit/2e9eb80c0ef8d02c313221eb04fba3c2ad2bf651)) +* fix: added retry count for duckdb query to avoid server errors ([`2e9eb80`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/2e9eb80c0ef8d02c313221eb04fba3c2ad2bf651)) ### Unknown -* Merge remote-tracking branch 'origin/main' ([`7fc6a50`](https://github.com/openwebsearcheu-public/owi-cli/commit/7fc6a5059e1d765d7100ca25b683e4fb80dcb936)) +* Merge branch 'main' of https://opencode.it4i.eu/openwebsearcheu-public/owi-cli ([`98a381a`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/98a381a3d1772064082b2c60735e8d4586db9878)) -* merge: conflict when arrro duckdb resolved ([`2fa5f62`](https://github.com/openwebsearcheu-public/owi-cli/commit/2fa5f62289744c09502c0771def09d0d6e45359b)) +* Merge remote-tracking branch 'origin/main' ([`7fc6a50`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7fc6a5059e1d765d7100ca25b683e4fb80dcb936)) -* added retry count ([`4d721c6`](https://github.com/openwebsearcheu-public/owi-cli/commit/4d721c660dbc2c2b62e7b8f8f24b60c7950e610a)) +* merge: conflict when arrro duckdb resolved ([`2fa5f62`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/2fa5f62289744c09502c0771def09d0d6e45359b)) -* Merge remote-tracking branch 'origin/main' ([`59f2ca3`](https://github.com/openwebsearcheu-public/owi-cli/commit/59f2ca383d63b9b1d3df3e503083f526b323df71)) +* added retry count ([`4d721c6`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/4d721c660dbc2c2b62e7b8f8f24b60c7950e610a)) + +* Merge remote-tracking branch 'origin/main' ([`59f2ca3`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/59f2ca383d63b9b1d3df3e503083f526b323df71)) ## v0.8.0 (2024-06-18) ### Chore -* chore: optimized imports ([`89a7c78`](https://github.com/openwebsearcheu-public/owi-cli/commit/89a7c78303d88e9f15c3f04e78eee95fa6c50bcd)) +* chore: optimized imports ([`89a7c78`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/89a7c78303d88e9f15c3f04e78eee95fa6c50bcd)) -* chore: numpy 2.0.0 yielded a problem with libraries in ubuntu ([`1d4ba04`](https://github.com/openwebsearcheu-public/owi-cli/commit/1d4ba043c06c8f873c4af6a9aa09a1f0dd36ae84)) +* chore: numpy 2.0.0 yielded a problem with libraries in ubuntu ([`1d4ba04`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/1d4ba043c06c8f873c4af6a9aa09a1f0dd36ae84)) ### Documentation -* docs: updated doc (a bit) ([`0fe822c`](https://github.com/openwebsearcheu-public/owi-cli/commit/0fe822cdaaf23aed33c0ba8f4553a944568133dd)) +* docs: updated doc (a bit) ([`0fe822c`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/0fe822cdaaf23aed33c0ba8f4553a944568133dd)) -* docs: improved documentation ([`7becf4a`](https://github.com/openwebsearcheu-public/owi-cli/commit/7becf4a7cc2a9eda707de44ef80887fa455f8919)) +* docs: improved documentation ([`7becf4a`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7becf4a7cc2a9eda707de44ef80887fa455f8919)) ### Feature -* feat: new command: local free to free space (i.e. remove local datasets) ([`3b75b8a`](https://github.com/openwebsearcheu-public/owi-cli/commit/3b75b8a133f05aedec20510e040b7c510c9e4e28)) +* feat: new command: local free to free space (i.e. remove local datasets) ([`3b75b8a`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/3b75b8a133f05aedec20510e040b7c510c9e4e28)) -* feat: added --default_config flag to not load stored config. ([`ff339bf`](https://github.com/openwebsearcheu-public/owi-cli/commit/ff339bfc7f8d8e6f8d3b46462e923971f2a50bc4)) +* feat: added --default_config flag to not load stored config. ([`ff339bf`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/ff339bfc7f8d8e6f8d3b46462e923971f2a50bc4)) ### Fix -* fix: non-existing collection folder yielded an exception when pulling the first time. ([`3aae39f`](https://github.com/openwebsearcheu-public/owi-cli/commit/3aae39f7c479c677deda9695c1ec5c678b68fecf)) +* fix: non-existing collection folder yielded an exception when pulling the first time. ([`3aae39f`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/3aae39f7c479c677deda9695c1ec5c678b68fecf)) -* fix: irods works now when client_zone needs to be selected. ([`2477de8`](https://github.com/openwebsearcheu-public/owi-cli/commit/2477de80813e7050ddb02ebc4c23bc338b0eaae8)) +* fix: irods works now when client_zone needs to be selected. ([`2477de8`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/2477de80813e7050ddb02ebc4c23bc338b0eaae8)) -* fix: irods works now when client_zone needs to be selected. ([`4b8f706`](https://github.com/openwebsearcheu-public/owi-cli/commit/4b8f706b13d227c012de5f75d736922e4f6f3ccb)) +* fix: irods works now when client_zone needs to be selected. ([`4b8f706`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/4b8f706b13d227c012de5f75d736922e4f6f3ccb)) -* fix: documentation was not shown properly ([`050d5a1`](https://github.com/openwebsearcheu-public/owi-cli/commit/050d5a1b99c0b1e0eb982cd31414afadb067ef30)) +* fix: documentation was not shown properly ([`050d5a1`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/050d5a1b99c0b1e0eb982cd31414afadb067ef30)) -* fix: lexis error log not found message implemented. ([`5891717`](https://github.com/openwebsearcheu-public/owi-cli/commit/589171740853d85cf278fc22ed40adb5b262dc9d)) +* fix: lexis error log not found message implemented. ([`5891717`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/589171740853d85cf278fc22ed40adb5b262dc9d)) ### Unknown -* Merge branch 'main' of https://opencode.it4i.eu/openwebsearcheu-public/owi-cli ([`7b1d576`](https://github.com/openwebsearcheu-public/owi-cli/commit/7b1d5763a0fe93029ddbc59923e5c4762ff9b863)) +* Merge branch 'main' of https://opencode.it4i.eu/openwebsearcheu-public/owi-cli ([`7b1d576`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7b1d5763a0fe93029ddbc59923e5c4762ff9b863)) -* doc: documentation improved ([`1fd05fc`](https://github.com/openwebsearcheu-public/owi-cli/commit/1fd05fcca8225b01d2fb3c6dededc9f3928d6965)) +* doc: documentation improved ([`1fd05fc`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/1fd05fcca8225b01d2fb3c6dededc9f3928d6965)) -* doc: documentation improved ([`8969e26`](https://github.com/openwebsearcheu-public/owi-cli/commit/8969e262a090aee32986d8ac0af52395bbe777f6)) +* doc: documentation improved ([`8969e26`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/8969e262a090aee32986d8ac0af52395bbe777f6)) ## v0.7.4 (2024-06-17) ### Fix -* fix: query now writes to the right collection ([`e85c893`](https://github.com/openwebsearcheu-public/owi-cli/commit/e85c8930f530ad2b27dd0ec06cb102bb474d9123)) +* fix: query now writes to the right collection ([`e85c893`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e85c8930f530ad2b27dd0ec06cb102bb474d9123)) ### Unknown -* doc: added todo ([`25a6e96`](https://github.com/openwebsearcheu-public/owi-cli/commit/25a6e961ab8a523e286e72ee7c3b2914409d0be0)) +* doc: added todo ([`25a6e96`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/25a6e961ab8a523e286e72ee7c3b2914409d0be0)) + +* doc: small change in metadata default value ([`cf6875c`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/cf6875cdc99c9c8ce5c0c4366f620359aa2e6c75)) -* doc: small change in metadata default value ([`cf6875c`](https://github.com/openwebsearcheu-public/owi-cli/commit/cf6875cdc99c9c8ce5c0c4366f620359aa2e6c75)) ## v0.7.3 (2024-06-16) ### Fix -* fix: speed improvement for irods using queries and direct session for listing objects, bypasing fsspec_irods ([`2f842a1`](https://github.com/openwebsearcheu-public/owi-cli/commit/2f842a16036b48c9456a5f95706e267f4f1bcd9d)) +* fix: speed improvement for irods using queries and direct session for listing objects, bypasing fsspec_irods ([`2f842a1`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/2f842a16036b48c9456a5f95706e267f4f1bcd9d)) ## v0.7.2 (2024-06-16) ### Fix -* fix: slice exits, when no files are found ([`9083708`](https://github.com/openwebsearcheu-public/owi-cli/commit/9083708687d1fd5c6251cb904700e0fdcc76f73a)) +* fix: slice exits, when no files are found ([`9083708`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/9083708687d1fd5c6251cb904700e0fdcc76f73a)) ## v0.7.1 (2024-06-16) ### Fix -* fix: optimze import; pandas no longer on cli start required ([`9bf446b`](https://github.com/openwebsearcheu-public/owi-cli/commit/9bf446bd4c9bc4bd4ab77b4bd515090d88bba825)) +* fix: optimze import; pandas no longer on cli start required ([`9bf446b`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/9bf446bd4c9bc4bd4ab77b4bd515090d88bba825)) ### Unknown -* version update ([`75986f8`](https://github.com/openwebsearcheu-public/owi-cli/commit/75986f8284ccff5bc15e5ff5622a134009d6547a)) +* version update ([`75986f8`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/75986f8284ccff5bc15e5ff5622a134009d6547a)) -* version update ([`7b05707`](https://github.com/openwebsearcheu-public/owi-cli/commit/7b057070969c1d8d84649c931ed0b45e2057ce23)) +* version update ([`7b05707`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7b057070969c1d8d84649c931ed0b45e2057ce23)) ## v0.7.0 (2024-06-16) ### Chore -* chore: updated py4lexis to 2.1.3 and duckdb to 1.0.0 ([`02e672c`](https://github.com/openwebsearcheu-public/owi-cli/commit/02e672cf882099006eade0501d0565823bd07344)) +* chore: updated py4lexis to 2.1.3 and duckdb to 1.0.0 ([`02e672c`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/02e672cf882099006eade0501d0565823bd07344)) -* chore: moved build to dev dependency group ([`0734502`](https://github.com/openwebsearcheu-public/owi-cli/commit/07345027ab96040ea8e190b9259cee2973e10a40)) +* chore: moved build to dev dependency group ([`0734502`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/07345027ab96040ea8e190b9259cee2973e10a40)) -* chore: removed python tuple version ([`cf78677`](https://github.com/openwebsearcheu-public/owi-cli/commit/cf7867766f9e43c55f51d51951890277fe04b509)) +* chore: removed python tuple version ([`cf78677`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/cf7867766f9e43c55f51d51951890277fe04b509)) ### Feature -* feat: update_metadata command available (but untested due to rights error) ([`39d0408`](https://github.com/openwebsearcheu-public/owi-cli/commit/39d04080610b07f98d00b6c12239481264f43d67)) +* feat: update_metadata command available (but untested due to rights error) ([`39d0408`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/39d04080610b07f98d00b6c12239481264f43d67)) -* feat: slicing finished - when slicing, provenance chain is established and a new dataset is locally created. ([`006f439`](https://github.com/openwebsearcheu-public/owi-cli/commit/006f439187c10a4239755692e5cf2695d4d55551)) +* feat: slicing finished - when slicing, provenance chain is established and a new dataset is locally created. ([`006f439`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/006f439187c10a4239755692e5cf2695d4d55551)) ### Fix -* fix: local ls returns dataset over all collections per default ([`7d037ab`](https://github.com/openwebsearcheu-public/owi-cli/commit/7d037ab148821aab8dac41b4940e84cc4b57d42d)) +* fix: local ls returns dataset over all collections per default ([`7d037ab`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7d037ab148821aab8dac41b4940e84cc4b57d42d)) -* fix: configuration now supports query federation via zone_path config. Datasets in lrz become available with that setting ([`d6b5813`](https://github.com/openwebsearcheu-public/owi-cli/commit/d6b5813e9ecf3d8b16939c69119634d4c9c687f0)) +* fix: configuration now supports query federation via zone_path config. Datasets in lrz become available with that setting ([`d6b5813`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/d6b5813e9ecf3d8b16939c69119634d4c9c687f0)) -* fix: metadata is now casted after update to avoid list vs. value problems ([`7d4fcb2`](https://github.com/openwebsearcheu-public/owi-cli/commit/7d4fcb279b72ba2c226d67557718402a70dd206c)) +* fix: metadata is now casted after update to avoid list vs. value problems ([`7d4fcb2`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7d4fcb279b72ba2c226d67557718402a70dd206c)) ### Refactor -* refactor: removed slice command and field ([`57e5637`](https://github.com/openwebsearcheu-public/owi-cli/commit/57e5637e2de8b3179411f9820a127b6afd840ead)) +* refactor: removed slice command and field ([`57e5637`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/57e5637e2de8b3179411f9820a127b6afd840ead)) ### Unknown -* doc: metadata todos ([`8026ef6`](https://github.com/openwebsearcheu-public/owi-cli/commit/8026ef6e5dbcfcf3fb9255b829ad6b2a84996229)) +* doc: metadata todos ([`8026ef6`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/8026ef6e5dbcfcf3fb9255b829ad6b2a84996229)) -* add: summary for dataset table ([`f8faf74`](https://github.com/openwebsearcheu-public/owi-cli/commit/f8faf744d94a1729badd1d321ce2715c53c09e0c)) +* add: summary for dataset table ([`f8faf74`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/f8faf744d94a1729badd1d321ce2715c53c09e0c)) -* add: statistic summary table for datastes ([`b6dac15`](https://github.com/openwebsearcheu-public/owi-cli/commit/b6dac1593cd80a87534588b6c8d3aab69b94ea15)) +* add: statistic summary table for datastes ([`b6dac15`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/b6dac1593cd80a87534588b6c8d3aab69b94ea15)) -* doc: added roadmap; minor formatting of readme ([`6c63d9d`](https://github.com/openwebsearcheu-public/owi-cli/commit/6c63d9de6fd0b446006b709e922b0787ceca3606)) +* doc: added roadmap; minor formatting of readme ([`6c63d9d`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/6c63d9de6fd0b446006b709e922b0787ceca3606)) ## v0.6.1 (2024-06-13) ### Fix -* fix: validation error in metadata when pulling datasets@ ([`f0d467d`](https://github.com/openwebsearcheu-public/owi-cli/commit/f0d467d4d0488fe942a47a1dfbad0ef0a3bc1fee)) +* fix: validation error in metadata when pulling datasets@ ([`f0d467d`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/f0d467d4d0488fe942a47a1dfbad0ef0a3bc1fee)) ### Unknown -* merge with slice changes@ ([`94a44c9`](https://github.com/openwebsearcheu-public/owi-cli/commit/94a44c9f108c7ce6178453c0c60a05c9cf2103cd)) +* merge with slice changes@ ([`94a44c9`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/94a44c9f108c7ce6178453c0c60a05c9cf2103cd)) -* small update ([`7469310`](https://github.com/openwebsearcheu-public/owi-cli/commit/7469310965ee3a70c8dcd84efda7289f732d1949)) +* small update ([`7469310`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7469310965ee3a70c8dcd84efda7289f732d1949)) -* add: metadata got section for default dataset metadata configuration ([`83c82a5`](https://github.com/openwebsearcheu-public/owi-cli/commit/83c82a57386e53bfa2e1def4496cd2f8c96bb03e)) +* add: metadata got section for default dataset metadata configuration ([`83c82a5`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/83c82a57386e53bfa2e1def4496cd2f8c96bb03e)) -* renamed Spark Indexer in relatedSoftware to OWI Indexer ([`5437933`](https://github.com/openwebsearcheu-public/owi-cli/commit/54379334fa695f52bf6c02239b2a0db4b18a9f2e)) +* renamed Spark Indexer in relatedSoftware to OWI Indexer ([`5437933`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/54379334fa695f52bf6c02239b2a0db4b18a9f2e)) -* add: parquet batch when using duckdb, which allows to have parquet specific template parameters. ([`501128d`](https://github.com/openwebsearcheu-public/owi-cli/commit/501128dbc55ec3b9de30ab6ad401cf45a9d76fed)) +* add: parquet batch when using duckdb, which allows to have parquet specific template parameters. ([`501128d`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/501128dbc55ec3b9de30ab6ad401cf45a9d76fed)) ## v0.6.0 (2024-06-11) ### Feature -* feat: insert with sub-path is possible now. Datasets can be now inserted from larger directories (e.g. year=2023/month=12/day=X) ([`5928228`](https://github.com/openwebsearcheu-public/owi-cli/commit/59282285c9836396feba190f0fa4c278129e89dc)) - -### Fix - -* fix: fixing attempt for semantic release ([`d27a930`](https://github.com/openwebsearcheu-public/owi-cli/commit/d27a930e64c734da85c1c34de9952795827635a1)) +* feat: insert with sub-path is possible now. Datasets can be now inserted from larger directories (e.g. year=2023/month=12/day=X) ([`5928228`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/59282285c9836396feba190f0fa4c278129e89dc)) -* fix: problem with changelog when inserting ([`be0c1d1`](https://github.com/openwebsearcheu-public/owi-cli/commit/be0c1d1ccd09a72932527d79864a0b6b12690830)) +* feat: first version for multithread slicing via duckdb into a single file ([`2221b29`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/2221b299eb623ae56cf8646fa2b3382d7807cae7)) -## v0.5.0 (2024-06-09) +### Fix -### Feature +* fix: fixing attempt for semantic release ([`d27a930`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/d27a930e64c734da85c1c34de9952795827635a1)) -* feat: first version for multithread slicing via duckdb into a single file ([`2221b29`](https://github.com/openwebsearcheu-public/owi-cli/commit/2221b299eb623ae56cf8646fa2b3382d7807cae7)) +* fix: problem with changelog when inserting ([`be0c1d1`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/be0c1d1ccd09a72932527d79864a0b6b12690830)) ## v0.4.0 (2024-06-09) ### Documentation -* docs: readme improved on semantic versioning ([`108fb47`](https://github.com/openwebsearcheu-public/owi-cli/commit/108fb4759e46af2349880286f85a2046846b9b99)) +* docs: readme improved on semantic versioning ([`108fb47`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/108fb4759e46af2349880286f85a2046846b9b99)) ### Feature -* feat: Less command for querying via duckdb ([`086b444`](https://github.com/openwebsearcheu-public/owi-cli/commit/086b444a449c217634511f483baf225cb8a0b2fd)) +* feat: Less command for querying via duckdb ([`086b444`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/086b444a449c217634511f483baf225cb8a0b2fd)) ### Unknown -* doc: documentatoin of duckdb functions improved ([`3369a6b`](https://github.com/openwebsearcheu-public/owi-cli/commit/3369a6b50971ef34b071defc9eada5358713279c)) +* doc: documentatoin of duckdb functions improved ([`3369a6b`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/3369a6b50971ef34b071defc9eada5358713279c)) ## v0.3.0 (2024-06-07) ### Feature -* feat: semantic versioning tested. used this commit to bump ([`eea6e55`](https://github.com/openwebsearcheu-public/owi-cli/commit/eea6e5524eecf369efc8f5f1a0061a8af8cfb7e6)) +* feat: semantic versioning tested. used this commit to bump ([`eea6e55`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/eea6e5524eecf369efc8f5f1a0061a8af8cfb7e6)) ## v0.2.0 (2024-06-07) ### Feature -* feat: semantic versioning ([`ebac113`](https://github.com/openwebsearcheu-public/owi-cli/commit/ebac113fadfa71e750e29ac0d678a21b39595cc1)) +* feat: semantic versioning ([`ebac113`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/ebac113fadfa71e750e29ac0d678a21b39595cc1)) ### Unknown -* gitigore change ([`f9ee469`](https://github.com/openwebsearcheu-public/owi-cli/commit/f9ee46934929bc2bc724e36211a5526bbc17f6f1)) +* gitigore change ([`f9ee469`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/f9ee46934929bc2bc724e36211a5526bbc17f6f1)) ## v0.1.0 (2024-06-07) ### Unknown -* readme example had an error@ ([`7d13a10`](https://github.com/openwebsearcheu-public/owi-cli/commit/7d13a10f458ff492e1940c1a15fc9cf6c45448dc)) +* readme example had an error@ ([`7d13a10`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7d13a10f458ff492e1940c1a15fc9cf6c45448dc)) -* post build changes ([`5601e9f`](https://github.com/openwebsearcheu-public/owi-cli/commit/5601e9fca586a4dee462138339d503471ea62cd6)) +* post build changes ([`5601e9f`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/5601e9fca586a4dee462138339d503471ea62cd6)) -* poetry problems ([`d9ad3b0`](https://github.com/openwebsearcheu-public/owi-cli/commit/d9ad3b0c4edff66aa5047018a9f701d32efcca94)) +* poetry problems ([`d9ad3b0`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/d9ad3b0c4edff66aa5047018a9f701d32efcca94)) -## v0.5.1 (2024-06-07) +* added config commands and changed metdata field provenance to a list ([`a627ba8`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/a627ba89e155cc426e9c013f290e5f4ff716e107)) -### Chore +* readme updated ([`209ecff`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/209ecff56fad6ed1f2094df66914cb4d646259ab)) -* chore: added upate of versioning in code ([`a97a386`](https://github.com/openwebsearcheu-public/owi-cli/commit/a97a3862096b3e61a03437db9c6b570c11f339a3)) +* fixed error in query not being and for all keys. Missing keys are interpreted as false. ([`51ae9ff`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/51ae9ff5fe062d163cd7ef769b0b37c5857ace50)) -* chore: remove obsolete, old code. cleanup of the module. ([`565bf18`](https://github.com/openwebsearcheu-public/owi-cli/commit/565bf18a4a2f198d10133e136c74644ac27aeeaa)) +* change metadata extraction from title supports multiple title formats. Consequence: more robustness when using lexishttp@ ([`72c7f33`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/72c7f33fe8ae8c039e4ac10cc55cfe2828be830e)) -* chore: added git-changelog; poetry dev and docs groups ([`7a3a076`](https://github.com/openwebsearcheu-public/owi-cli/commit/7a3a076e87ed0c09df35cf11bec9f9cac7448144)) +## v0.5.0 (2024-06-07) -* chore: added git-changelog; poetry dev and docs groups ([`0d550fc`](https://github.com/openwebsearcheu-public/owi-cli/commit/0d550fc1282c42aeca611d928a7a3380f85291df)) +### Unknown -### Feature +* small updates ([`af4b346`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/af4b3468cf9ccd55131144fe3d3bcb947ac1ac03)) -* feat: inserting datasets loally from path ([`a3ef490`](https://github.com/openwebsearcheu-public/owi-cli/commit/a3ef490eaf6826427c852ff8e16f352d9d530ee7)) +* added return value for commands and integrated into logging ([`c4c50c1`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/c4c50c16ea8534559b03f724fb1c5008104c4b51)) -* feat: loca/remote diffs between datasets and files ([`8f9af3d`](https://github.com/openwebsearcheu-public/owi-cli/commit/8f9af3d66e0f081fa2df9a5e789545c28f448b01)) +* added lastChanged metadata and provenance became a list ([`3cab29d`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/3cab29d7e79a7eef26d54509641ee5309f2b6716)) -* feat: added regexp based query in specifier. Field needs to end with * ([`e0da862`](https://github.com/openwebsearcheu-public/owi-cli/commit/e0da862019593b3853b2035a607b1b865f75c058)) +* changed license to Apache 2.0 ([`d8bf773`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/d8bf773b0d2346ca158957aaa3b99fc134fb4331)) -* feat: improved control over displayed columns and sort order ([`b90ff3f`](https://github.com/openwebsearcheu-public/owi-cli/commit/b90ff3f5999cd20c8a4b04a7f762ef0067cf4108)) +* added loging of commands and reading out different log files via ([`02d1479`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/02d14796809cafaa93350692a6d55aa0106d59a9)) -### Unknown +* added HTTPLexisRepository as backup in case irods ports are closed. However, only works for pull / ls. ([`99e65ef`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/99e65efafc25d5671d116fa88ab7a5a163a8fe06)) + +* documented advanced usage ([`7e2023f`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7e2023f2d86271186027fe0ede0254deffec9c4f)) + +* add pyyaml as dependency for configuration file ([`9b32fa9`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/9b32fa98f913d7ebcc54c6e2c8a2d1ad27ab7e3b)) + +* added relatedSoftware and alternateIdentifier to metadata ([`63c551f`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/63c551ffd140d41f7a16c84f8ac09c9a5eef8606)) + +* added configuration of client zones and zones for enabling federated access to irods zones ([`0a50067`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/0a500679950f157211ea3c405a7b5741d084e758)) + +* intern: fixed path based start_transfer ([`e417de6`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e417de61ede5ad201f7993587f108a784a0560a3)) + +* added --log_level flag. You can now get all Debug information by using `--log_level=DEBUG` ([`2ca2f5a`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/2ca2f5a9c793d43596419a45e0af629300e0b8f3)) -* added config commands and changed metdata field provenance to a list ([`a627ba8`](https://github.com/openwebsearcheu-public/owi-cli/commit/a627ba89e155cc426e9c013f290e5f4ff716e107)) +* added "owilix" logger ([`b918f4f`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/b918f4f8b3b04e6b679d6fad5fa04111949879b1)) -* readme updated ([`209ecff`](https://github.com/openwebsearcheu-public/owi-cli/commit/209ecff56fad6ed1f2094df66914cb4d646259ab)) +* change ! flag --exclude changed to --remotes ([`deeaf68`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/deeaf68dd9928047181dec3214ef75d64b5b3e23)) -* fixed error in query not being and for all keys. Missing keys are interpreted as false. ([`51ae9ff`](https://github.com/openwebsearcheu-public/owi-cli/commit/51ae9ff5fe062d163cd7ef769b0b37c5857ace50)) +* change config being stored in yaml ([`0f55b49`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/0f55b492de9386a13ea3700fd826a68cb3fd97d8)) -* change metadata extraction from title supports multiple title formats. Consequence: more robustness when using lexishttp@ ([`72c7f33`](https://github.com/openwebsearcheu-public/owi-cli/commit/72c7f33fe8ae8c039e4ac10cc55cfe2828be830e)) +* change config being stored in yaml ([`e85bf6b`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e85bf6b457a6f348a06e85d74de3433bc75393a9)) -* small updates ([`af4b346`](https://github.com/openwebsearcheu-public/owi-cli/commit/af4b3468cf9ccd55131144fe3d3bcb947ac1ac03)) +* Merge branch 'main' of https://opencode.it4i.eu/openwebsearcheu-public/owi-cli ([`f6d4f71`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/f6d4f71460f656a9cdff760bc7cdbb96d8175a4d)) -* added return value for commands and integrated into logging ([`c4c50c1`](https://github.com/openwebsearcheu-public/owi-cli/commit/c4c50c16ea8534559b03f724fb1c5008104c4b51)) +* add doctor command for checking network connnections configured ([`aa0afd4`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/aa0afd45ae9f0e8cf1ba793555cbedd5026824e8)) -* added lastChanged metadata and provenance became a list ([`3cab29d`](https://github.com/openwebsearcheu-public/owi-cli/commit/3cab29d7e79a7eef26d54509641ee5309f2b6716)) +* fix of a bug on date based selection. ([`bc7a1f9`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/bc7a1f993bb88e4d4228d76ffe345395c12f3c51)) -* changed license to Apache 2.0 ([`d8bf773`](https://github.com/openwebsearcheu-public/owi-cli/commit/d8bf773b0d2346ca158957aaa3b99fc134fb4331)) +* fix dataset creation which is public requires list of rightsURI and list of rights ([`3d3107e`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/3d3107e55734b4af788481128a692b10153647ae)) -* added loging of commands and reading out different log files via ([`02d1479`](https://github.com/openwebsearcheu-public/owi-cli/commit/02d14796809cafaa93350692a6d55aa0106d59a9)) +* intermediate fix for rightsURI requirement on public datasets ([`cdb676d`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/cdb676d8a59dfda92bf3af42b32ae28424085644)) -* added HTTPLexisRepository as backup in case irods ports are closed. However, only works for pull / ls. ([`99e65ef`](https://github.com/openwebsearcheu-public/owi-cli/commit/99e65efafc25d5671d116fa88ab7a5a163a8fe06)) +* update for version 0.4.3 ([`ec08289`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/ec08289664d72a5c03cc20bf43a6784f88c1661f)) -* documented advanced usage ([`7e2023f`](https://github.com/openwebsearcheu-public/owi-cli/commit/7e2023f2d86271186027fe0ede0254deffec9c4f)) +## v0.4.3 (2024-06-03) -* add pyyaml as dependency for configuration file ([`9b32fa9`](https://github.com/openwebsearcheu-public/owi-cli/commit/9b32fa98f913d7ebcc54c6e2c8a2d1ad27ab7e3b)) +### Feature -* added relatedSoftware and alternateIdentifier to metadata ([`63c551f`](https://github.com/openwebsearcheu-public/owi-cli/commit/63c551ffd140d41f7a16c84f8ac09c9a5eef8606)) +* feat: inserting datasets loally from path ([`a3ef490`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/a3ef490eaf6826427c852ff8e16f352d9d530ee7)) -* added configuration of client zones and zones for enabling federated access to irods zones ([`0a50067`](https://github.com/openwebsearcheu-public/owi-cli/commit/0a500679950f157211ea3c405a7b5741d084e758)) +### Unknown -* intern: fixed path based start_transfer ([`e417de6`](https://github.com/openwebsearcheu-public/owi-cli/commit/e417de61ede5ad201f7993587f108a784a0560a3)) +* fix small bug in pushing to remote ([`78114bb`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/78114bb026a759308bd7d78756da85432c345819)) -* added --log_level flag. You can now get all Debug information by using `--log_level=DEBUG` ([`2ca2f5a`](https://github.com/openwebsearcheu-public/owi-cli/commit/2ca2f5a9c793d43596419a45e0af629300e0b8f3)) +## v0.4.2 (2024-06-01) -* added "owilix" logger ([`b918f4f`](https://github.com/openwebsearcheu-public/owi-cli/commit/b918f4f8b3b04e6b679d6fad5fa04111949879b1)) +### Chore -* change ! flag --exclude changed to --remotes ([`deeaf68`](https://github.com/openwebsearcheu-public/owi-cli/commit/deeaf68dd9928047181dec3214ef75d64b5b3e23)) +* chore: added upate of versioning in code ([`a97a386`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/a97a3862096b3e61a03437db9c6b570c11f339a3)) -* change config being stored in yaml ([`0f55b49`](https://github.com/openwebsearcheu-public/owi-cli/commit/0f55b492de9386a13ea3700fd826a68cb3fd97d8)) +* chore: remove obsolete, old code. cleanup of the module. ([`565bf18`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/565bf18a4a2f198d10133e136c74644ac27aeeaa)) -* change config being stored in yaml ([`e85bf6b`](https://github.com/openwebsearcheu-public/owi-cli/commit/e85bf6b457a6f348a06e85d74de3433bc75393a9)) +* chore: added git-changelog; poetry dev and docs groups ([`7a3a076`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7a3a076e87ed0c09df35cf11bec9f9cac7448144)) -* Merge branch 'main' of https://opencode.it4i.eu/openwebsearcheu-public/owi-cli ([`f6d4f71`](https://github.com/openwebsearcheu-public/owi-cli/commit/f6d4f71460f656a9cdff760bc7cdbb96d8175a4d)) +* chore: added git-changelog; poetry dev and docs groups ([`0d550fc`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/0d550fc1282c42aeca611d928a7a3380f85291df)) -* add doctor command for checking network connnections configured ([`aa0afd4`](https://github.com/openwebsearcheu-public/owi-cli/commit/aa0afd45ae9f0e8cf1ba793555cbedd5026824e8)) +### Feature -* fix of a bug on date based selection. ([`bc7a1f9`](https://github.com/openwebsearcheu-public/owi-cli/commit/bc7a1f993bb88e4d4228d76ffe345395c12f3c51)) +* feat: loca/remote diffs between datasets and files ([`8f9af3d`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/8f9af3d66e0f081fa2df9a5e789545c28f448b01)) -* fix dataset creation which is public requires list of rightsURI and list of rights ([`3d3107e`](https://github.com/openwebsearcheu-public/owi-cli/commit/3d3107e55734b4af788481128a692b10153647ae)) +* feat: added regexp based query in specifier. Field needs to end with * ([`e0da862`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e0da862019593b3853b2035a607b1b865f75c058)) -* intermediate fix for rightsURI requirement on public datasets ([`cdb676d`](https://github.com/openwebsearcheu-public/owi-cli/commit/cdb676d8a59dfda92bf3af42b32ae28424085644)) +* feat: improved control over displayed columns and sort order ([`b90ff3f`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/b90ff3f5999cd20c8a4b04a7f762ef0067cf4108)) -* update for version 0.4.3 ([`ec08289`](https://github.com/openwebsearcheu-public/owi-cli/commit/ec08289664d72a5c03cc20bf43a6784f88c1661f)) +### Unknown + +* added versionoing ([`63377ff`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/63377ff0b49b5239f3bf7b8b04e6078aaf356183)) -* fix small bug in pushing to remote ([`78114bb`](https://github.com/openwebsearcheu-public/owi-cli/commit/78114bb026a759308bd7d78756da85432c345819)) +## v0.4.1 (2024-05-31) -* added versionoing ([`63377ff`](https://github.com/openwebsearcheu-public/owi-cli/commit/63377ff0b49b5239f3bf7b8b04e6078aaf356183)) +### Unknown -* compatibility with py4lexis-2.1.1 ([`79ef75e`](https://github.com/openwebsearcheu-public/owi-cli/commit/79ef75e954491b412479de5e083eebf8a79e51fb)) +* compatibility with py4lexis-2.1.1 ([`79ef75e`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/79ef75e954491b412479de5e083eebf8a79e51fb)) -* readme update for admin command ([`b231051`](https://github.com/openwebsearcheu-public/owi-cli/commit/b2310515bf59dbe4752d3ff78a1216be313432a2)) +* readme update for admin command ([`b231051`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/b2310515bf59dbe4752d3ff78a1216be313432a2)) -* readme update ([`de2c6ed`](https://github.com/openwebsearcheu-public/owi-cli/commit/de2c6eddef8949bb3e4c338dde18c01ecef64f7c)) +* readme update ([`de2c6ed`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/de2c6eddef8949bb3e4c338dde18c01ecef64f7c)) -* bad merge ([`6c71f56`](https://github.com/openwebsearcheu-public/owi-cli/commit/6c71f569de4b21f5600af4d6932340c7b456c1fd)) +* bad merge ([`6c71f56`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/6c71f569de4b21f5600af4d6932340c7b456c1fd)) -* added admin set_irods_metadata command ([`45910b5`](https://github.com/openwebsearcheu-public/owi-cli/commit/45910b5655ae3df3ff313414943df5f1e46d9ee3)) +* added admin set_irods_metadata command ([`45910b5`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/45910b5655ae3df3ff313414943df5f1e46d9ee3)) -* intermediate status for getting all lexis to work ([`3686911`](https://github.com/openwebsearcheu-public/owi-cli/commit/3686911fc50e4a435b8ebb8b758239fce4d375fb)) +* intermediate status for getting all lexis to work ([`3686911`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/3686911fc50e4a435b8ebb8b758239fce4d375fb)) -* dataset creation alpha. API not working ([`6c67198`](https://github.com/openwebsearcheu-public/owi-cli/commit/6c671987e4fd7fd05b17bf5936cbedf8af2d7fc1)) +* dataset creation alpha. API not working ([`6c67198`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/6c671987e4fd7fd05b17bf5936cbedf8af2d7fc1)) -* update local ls; readme ([`d21412b`](https://github.com/openwebsearcheu-public/owi-cli/commit/d21412b99611350093d129d5f146fb662205ddc7)) +* update local ls; readme ([`d21412b`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/d21412b99611350093d129d5f146fb662205ddc7)) -* bit of cleanup ([`743dd22`](https://github.com/openwebsearcheu-public/owi-cli/commit/743dd22002c79425fe4cf7d62c635663e61d26ca)) +* bit of cleanup ([`743dd22`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/743dd22002c79425fe4cf7d62c635663e61d26ca)) -* insert /update of local dataset form path down ([`f29a130`](https://github.com/openwebsearcheu-public/owi-cli/commit/f29a13038fe5b19ec47ea3dac4fb1e1017fa492c)) +* insert /update of local dataset form path down ([`f29a130`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/f29a13038fe5b19ec47ea3dac4fb1e1017fa492c)) -* Merge branch 'develop' of https://opencode.it4i.eu/openwebsearcheu-public/owi-cli into develop ([`00492e2`](https://github.com/openwebsearcheu-public/owi-cli/commit/00492e266a8c0631b4e51ab487e6c14ac9059bdb)) +* Merge branch 'develop' of https://opencode.it4i.eu/openwebsearcheu-public/owi-cli into develop ([`00492e2`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/00492e266a8c0631b4e51ab487e6c14ac9059bdb)) -* swichted to repositories ([`2b933aa`](https://github.com/openwebsearcheu-public/owi-cli/commit/2b933aa5c92bf7c27f132c2a5e00fee428ebbfd0)) +* swichted to repositories ([`2b933aa`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/2b933aa5c92bf7c27f132c2a5e00fee428ebbfd0)) -* small changes. ([`3a10529`](https://github.com/openwebsearcheu-public/owi-cli/commit/3a10529fddfd21bfdcd635c4ee12dc7547bbcfe5)) +* small changes. ([`3a10529`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/3a10529fddfd21bfdcd635c4ee12dc7547bbcfe5)) -* error in install ([`63ee514`](https://github.com/openwebsearcheu-public/owi-cli/commit/63ee5142dd8aed67f13d02efc4a738e192207636)) +* error in install ([`63ee514`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/63ee5142dd8aed67f13d02efc4a738e192207636)) -* Merge branch 'main' into develop ([`c0e28e8`](https://github.com/openwebsearcheu-public/owi-cli/commit/c0e28e8f425d8880f8da618914164a31312ac468)) +* Merge branch 'main' into develop ([`c0e28e8`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/c0e28e8f425d8880f8da618914164a31312ac468)) -* docu update ([`b1d28e1`](https://github.com/openwebsearcheu-public/owi-cli/commit/b1d28e128a44ae93b0b906810a2d9a6dff3e22ba)) +* docu update ([`b1d28e1`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/b1d28e128a44ae93b0b906810a2d9a6dff3e22ba)) -* udpate readme install ([`df9f687`](https://github.com/openwebsearcheu-public/owi-cli/commit/df9f687f23934da24eec6945c31b97fd5c9bcb35)) +* udpate readme install ([`df9f687`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/df9f687f23934da24eec6945c31b97fd5c9bcb35)) -* added prepration flag ([`82b5171`](https://github.com/openwebsearcheu-public/owi-cli/commit/82b5171d619ed433f5d274329a89e15a838c8abb)) +* added prepration flag ([`82b5171`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/82b5171d619ed433f5d274329a89e15a838c8abb)) -* added bug in pulling datasets ([`86e10a0`](https://github.com/openwebsearcheu-public/owi-cli/commit/86e10a038c43e6dd6f959aecaad37a0fe3a6abf7)) +* added bug in pulling datasets ([`86e10a0`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/86e10a038c43e6dd6f959aecaad37a0fe3a6abf7)) -* added dataset creation and metadta admin panel which metadata chekc. Bot incomplete, since server api is not working properly ([`e559d6b`](https://github.com/openwebsearcheu-public/owi-cli/commit/e559d6bfdcee85d8dbda93d22f5acc14f8ecdedf)) +* added dataset creation and metadta admin panel which metadata chekc. Bot incomplete, since server api is not working properly ([`e559d6b`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e559d6bfdcee85d8dbda93d22f5acc14f8ecdedf)) -* added metadata management; added create v0.01 ([`40912de`](https://github.com/openwebsearcheu-public/owi-cli/commit/40912de7a5367a94e35b0fba623e37f78bb7272c)) +* added metadata management; added create v0.01 ([`40912de`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/40912de7a5367a94e35b0fba623e37f78bb7272c)) -* reworked commands. Converted to recent metadta format ([`e8a35e0`](https://github.com/openwebsearcheu-public/owi-cli/commit/e8a35e0cdf97b9fb5a039ba056d4443fac8734eb)) +* reworked commands. Converted to recent metadta format ([`e8a35e0`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e8a35e0cdf97b9fb5a039ba056d4443fac8734eb)) -* reworked commands. Converted to recent metadta format ([`bf55504`](https://github.com/openwebsearcheu-public/owi-cli/commit/bf55504202d245073b9935381e5f92a4668f2522)) +* reworked commands. Converted to recent metadta format ([`bf55504`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/bf55504202d245073b9935381e5f92a4668f2522)) -* removed path from stats ([`cc400d6`](https://github.com/openwebsearcheu-public/owi-cli/commit/cc400d6891d3ec346b62ad23ab4ee52cab45576e)) +* removed path from stats ([`cc400d6`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/cc400d6891d3ec346b62ad23ab4ee52cab45576e)) -* added todo ([`b4da4fa`](https://github.com/openwebsearcheu-public/owi-cli/commit/b4da4fa43cea51622b693b660790373c00ca0530)) +* added todo ([`b4da4fa`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/b4da4fa43cea51622b693b660790373c00ca0530)) -* docu;improved get_dataset; irods imp; test; fillet cmd; ([`f32db96`](https://github.com/openwebsearcheu-public/owi-cli/commit/f32db965d570341d932257716a843858cfb1513e)) +* docu;improved get_dataset; irods imp; test; fillet cmd; ([`f32db96`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/f32db965d570341d932257716a843858cfb1513e)) -* added documentatoin ([`76df3e9`](https://github.com/openwebsearcheu-public/owi-cli/commit/76df3e9ab33fb171d638cb4226f5d1c9e123b9ed)) +* added documentatoin ([`76df3e9`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/76df3e9ab33fb171d638cb4226f5d1c9e123b9ed)) -* update commands + documentaiton; update dependencies with +rich -tqdm ([`41ccc99`](https://github.com/openwebsearcheu-public/owi-cli/commit/41ccc9919c4881a856ea60e8cea5cb28558d59e1)) +* update commands + documentaiton; update dependencies with +rich -tqdm ([`41ccc99`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/41ccc9919c4881a856ea60e8cea5cb28558d59e1)) -* added first sql template mechanisms plus scripts ([`dd7df70`](https://github.com/openwebsearcheu-public/owi-cli/commit/dd7df7015a59b0093fa219a0c9b643cf7900fc45)) +* added first sql template mechanisms plus scripts ([`dd7df70`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/dd7df7015a59b0093fa219a0c9b643cf7900fc45)) -* update commands. middle of change ([`af4f5ac`](https://github.com/openwebsearcheu-public/owi-cli/commit/af4f5ac79420f6860d3d9ed0eaad0ac5b6f58776)) +* update commands. middle of change ([`af4f5ac`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/af4f5ac79420f6860d3d9ed0eaad0ac5b6f58776)) -* prpeared pyproject-local-dev.toml to work with py4lexis locally (needed to use python 3.12) ([`667889b`](https://github.com/openwebsearcheu-public/owi-cli/commit/667889b9753f05f142277c49b6dd6765d19ea4c5)) +* prpeared pyproject-local-dev.toml to work with py4lexis locally (needed to use python 3.12) ([`667889b`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/667889b9753f05f142277c49b6dd6765d19ea4c5)) -* updated readme. Added changelog ([`0956f65`](https://github.com/openwebsearcheu-public/owi-cli/commit/0956f655d4a91cc95632cc5465d2a7a19eac4fd0)) +* updated readme. Added changelog ([`0956f65`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/0956f655d4a91cc95632cc5465d2a7a19eac4fd0)) -* typo ([`fb80821`](https://github.com/openwebsearcheu-public/owi-cli/commit/fb8082194a6266e5f6dccb2384b41c9faaf32ab4)) +* typo ([`fb80821`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/fb8082194a6266e5f6dccb2384b41c9faaf32ab4)) -* update poetry config and bump to version ([`d3fcfea`](https://github.com/openwebsearcheu-public/owi-cli/commit/d3fcfea00cbc0ee2589958b84da0b628694178d4)) +* update poetry config and bump to version ([`d3fcfea`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/d3fcfea00cbc0ee2589958b84da0b628694178d4)) -* added duckdb shell ([`e068e76`](https://github.com/openwebsearcheu-public/owi-cli/commit/e068e76fca3503d127b9ce258d5bcaa47138d65c)) +* added duckdb shell ([`e068e76`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e068e76fca3503d127b9ce258d5bcaa47138d65c)) -* final alpha version for pull ([`b0d6bfd`](https://github.com/openwebsearcheu-public/owi-cli/commit/b0d6bfd6787c3b37251b8ad622bba44a26eebe90)) +* final alpha version for pull ([`b0d6bfd`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/b0d6bfd6787c3b37251b8ad622bba44a26eebe90)) -* first version working ([`e223f92`](https://github.com/openwebsearcheu-public/owi-cli/commit/e223f92c4151fc569d86c9ca6e2487fc412d05dd)) +* first version working ([`e223f92`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e223f92c4151fc569d86c9ca6e2487fc412d05dd)) -* Threading. in the middle ([`29cab7e`](https://github.com/openwebsearcheu-public/owi-cli/commit/29cab7e9074d0efeb7cb6dc686e6c19348b0d453)) +* Threading. in the middle ([`29cab7e`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/29cab7e9074d0efeb7cb6dc686e6c19348b0d453)) -* added file based transfer ([`199bded`](https://github.com/openwebsearcheu-public/owi-cli/commit/199bded0599c6503e0db6052e1c3bcc72780f96f)) +* added file based transfer ([`199bded`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/199bded0599c6503e0db6052e1c3bcc72780f96f)) -* added duckdb plugin (first test); changed download to thread based download ([`ad7d3bc`](https://github.com/openwebsearcheu-public/owi-cli/commit/ad7d3bc26f1000dc7ec77c934f25f55fbc1d36ec)) +* added duckdb plugin (first test); changed download to thread based download ([`ad7d3bc`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/ad7d3bc26f1000dc7ec77c934f25f55fbc1d36ec)) -* swichted to click and rich ([`ac16ea5`](https://github.com/openwebsearcheu-public/owi-cli/commit/ac16ea558fe9b60eb114df7a6908885c343f0ab1)) +* swichted to click and rich ([`ac16ea5`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/ac16ea558fe9b60eb114df7a6908885c343f0ab1)) -* added optional duckdb dependency ([`b0590ce`](https://github.com/openwebsearcheu-public/owi-cli/commit/b0590ce5b62d036c8dcae79ffdc0a255a1f0dcd2)) +* added optional duckdb dependency ([`b0590ce`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/b0590ce5b62d036c8dcae79ffdc0a255a1f0dcd2)) -* update readme; some testing with poetry ([`9941e00`](https://github.com/openwebsearcheu-public/owi-cli/commit/9941e0055decb8c6c6791ac602b3f9f648959c42)) +* update readme; some testing with poetry ([`9941e00`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/9941e0055decb8c6c6791ac602b3f9f648959c42)) -* added local commands;swithc to poetry;first build ([`564e437`](https://github.com/openwebsearcheu-public/owi-cli/commit/564e4377c43c05cf2cea1383bb00aaee9fd55a49)) +* added local commands;swithc to poetry;first build ([`564e437`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/564e4377c43c05cf2cea1383bb00aaee9fd55a49)) -* added plugin for testing ([`0072b4a`](https://github.com/openwebsearcheu-public/owi-cli/commit/0072b4af71a31d16a17a0db9479b27a36c9db85c)) +* added plugin for testing ([`0072b4a`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/0072b4af71a31d16a17a0db9479b27a36c9db85c)) -* path corrected ([`c0fb2ac`](https://github.com/openwebsearcheu-public/owi-cli/commit/c0fb2acf619475693d22f06792e5768173b9e2ef)) +* path corrected ([`c0fb2ac`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/c0fb2acf619475693d22f06792e5768173b9e2ef)) -* bug fix for tar extraction ([`8290610`](https://github.com/openwebsearcheu-public/owi-cli/commit/82906107aaa556ad381ffe5f32af4807e060882c)) +* bug fix for tar extraction ([`8290610`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/82906107aaa556ad381ffe5f32af4807e060882c)) -* bug-fix: dataset get ([`77a722c`](https://github.com/openwebsearcheu-public/owi-cli/commit/77a722c67d5e146c53f88f660252cb05dcb4965e)) +* bug-fix: dataset get ([`77a722c`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/77a722c67d5e146c53f88f660252cb05dcb4965e)) -* added field list for showing dataset dataframe; first version for extract tar.gz file ([`d001e67`](https://github.com/openwebsearcheu-public/owi-cli/commit/d001e67d0886e8420b4df471ebcfca9f7df74f45)) +* added field list for showing dataset dataframe; first version for extract tar.gz file ([`d001e67`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/d001e67d0886e8420b4df471ebcfca9f7df74f45)) -* added more verbose messages in pull ([`79b07c6`](https://github.com/openwebsearcheu-public/owi-cli/commit/79b07c6a9a88a788176c220966959693655e182a)) +* added more verbose messages in pull ([`79b07c6`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/79b07c6a9a88a788176c220966959693655e182a)) -* added command line option for default path ([`a604506`](https://github.com/openwebsearcheu-public/owi-cli/commit/a60450609d9c15101f1d08a644cc8f139f6639b1)) +* added command line option for default path ([`a604506`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/a60450609d9c15101f1d08a644cc8f139f6639b1)) -* file based filtering (but not download) ([`da59c2b`](https://github.com/openwebsearcheu-public/owi-cli/commit/da59c2b885fe3bd495fdf25dfc687c30325f5247)) +* file based filtering (but not download) ([`da59c2b`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/da59c2b885fe3bd495fdf25dfc687c30325f5247)) -* update setup ([`a23e4d2`](https://github.com/openwebsearcheu-public/owi-cli/commit/a23e4d29b7de97dcb42a77f487600f066bac97ee)) +* update setup ([`a23e4d2`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/a23e4d29b7de97dcb42a77f487600f066bac97ee)) -* setup and command line ([`78e7675`](https://github.com/openwebsearcheu-public/owi-cli/commit/78e767560bd23fadaf281563a453808a39b0d62f)) +* setup and command line ([`78e7675`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/78e767560bd23fadaf281563a453808a39b0d62f)) -* udpated towards specifier ([`01af31e`](https://github.com/openwebsearcheu-public/owi-cli/commit/01af31e0a5c84443556e1b763ae5152893634068)) +* udpated towards specifier ([`01af31e`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/01af31e0a5c84443556e1b763ae5152893634068)) -* rmoeved submodule ([`51f5815`](https://github.com/openwebsearcheu-public/owi-cli/commit/51f581556bf46733c51f14c5bd8b03032ed1a5c8)) +* rmoeved submodule ([`51f5815`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/51f581556bf46733c51f14c5bd8b03032ed1a5c8)) -* rmoeved submodule ([`472b5a1`](https://github.com/openwebsearcheu-public/owi-cli/commit/472b5a15a9216abf3669f5dbbacef4f93d4ac19c)) +* rmoeved submodule ([`472b5a1`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/472b5a15a9216abf3669f5dbbacef4f93d4ac19c)) -* Removed submodule ([`4d2deb1`](https://github.com/openwebsearcheu-public/owi-cli/commit/4d2deb1da4e7aaecf9a250d035f1d88a0e2dd42e)) +* Removed submodule ([`4d2deb1`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/4d2deb1da4e7aaecf9a250d035f1d88a0e2dd42e)) -* first cli version: dataset list, stats ([`e271636`](https://github.com/openwebsearcheu-public/owi-cli/commit/e271636964def01fee8de7f516dc7509f04e77f1)) +* first cli version: dataset list, stats ([`e271636`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e271636964def01fee8de7f516dc7509f04e77f1)) -* first cli version: dataset list, stats ([`e4bff1c`](https://github.com/openwebsearcheu-public/owi-cli/commit/e4bff1cf7e7f541bccb9be4cce5e1db27430c97b)) +* first cli version: dataset list, stats ([`e4bff1c`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e4bff1cf7e7f541bccb9be4cce5e1db27430c97b)) -* submodule change ([`5e57239`](https://github.com/openwebsearcheu-public/owi-cli/commit/5e57239e9890fc9c49e632ffb2d4e49b1ef3db87)) +* submodule change ([`5e57239`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/5e57239e9890fc9c49e632ffb2d4e49b1ef3db87)) -* added workflow example from Lexis plus authentication ([`7ce7052`](https://github.com/openwebsearcheu-public/owi-cli/commit/7ce70522179994cf705e848e9d70f9912bf776a0)) +* added workflow example from Lexis plus authentication ([`7ce7052`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/7ce70522179994cf705e848e9d70f9912bf776a0)) -* renamed main to datasets ([`e88f553`](https://github.com/openwebsearcheu-public/owi-cli/commit/e88f5536825dc781f3471163db012d69b6a2aac2)) +* renamed main to datasets ([`e88f553`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/e88f5536825dc781f3471163db012d69b6a2aac2)) -* datasets api working ([`c9f742d`](https://github.com/openwebsearcheu-public/owi-cli/commit/c9f742d241426e3a291649dca30abd6babca43ba)) +* datasets api working ([`c9f742d`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/c9f742d241426e3a291649dca30abd6babca43ba)) -* added openwebsearch/py4lexis as submodule; update readme and test ([`fa36598`](https://github.com/openwebsearcheu-public/owi-cli/commit/fa36598886c5a1ce8e7d2c66ce57196f8de64892)) +* added openwebsearch/py4lexis as submodule; update readme and test ([`fa36598`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/fa36598886c5a1ce8e7d2c66ce57196f8de64892)) -* added username /pwd configuration via Enviornment Variables LEXIS_USERNAME and LEXIS_PASSWORD ([`093ef31`](https://github.com/openwebsearcheu-public/owi-cli/commit/093ef31eac636b235695a4e6365b712995ee3919)) +* added username /pwd configuration via Enviornment Variables LEXIS_USERNAME and LEXIS_PASSWORD ([`093ef31`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/093ef31eac636b235695a4e6365b712995ee3919)) -* removed Py4Lexis in order to fork the repo ([`55a61ef`](https://github.com/openwebsearcheu-public/owi-cli/commit/55a61efc30458fd7901e1c78c78561b95ba4808c)) +* removed Py4Lexis in order to fork the repo ([`55a61ef`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/55a61efc30458fd7901e1c78c78561b95ba4808c)) -* added git modules ([`2f7a8eb`](https://github.com/openwebsearcheu-public/owi-cli/commit/2f7a8eb0764770f18733dc96290f00adc917b3ab)) +* added git modules ([`2f7a8eb`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/2f7a8eb0764770f18733dc96290f00adc917b3ab)) -* added howto for getting account password ([`db0a408`](https://github.com/openwebsearcheu-public/owi-cli/commit/db0a408e00400060045d0d619d7b58d2e4430c78)) +* added howto for getting account password ([`db0a408`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/db0a408e00400060045d0d619d7b58d2e4430c78)) -* Initial commit ([`d06f6f7`](https://github.com/openwebsearcheu-public/owi-cli/commit/d06f6f74a37270dd9d50814783eaf84a97541f4f)) +* Initial commit ([`d06f6f7`](https://opencode.it4i.eu/openwebsearcheu-public/owi-cli/-/commit/d06f6f74a37270dd9d50814783eaf84a97541f4f)) diff --git a/owilix/cmd/query.py b/owilix/cmd/query.py index 5e24ffb..bc58805 100644 --- a/owilix/cmd/query.py +++ b/owilix/cmd/query.py @@ -42,7 +42,7 @@ class QueryCommands(BaseCommand): Fetches all files from the datasets selected by local and remote specifiers and grouped by filesystem. Returns: - Dict[AbstractFileSystem, List[(str,str)]]: A dictionary with filesystems as keys and list of tuples + Dict[AbstractFileSystem, List[(str,str)]], List[Datasets]: A dictionary with filesystems as keys and list of tuples with file and dataset path as values. """ @@ -339,6 +339,44 @@ def stream(self, local_specifier: str, remote_specifier: str, if writer: writer.close() + def configure_kafka_producer(): + from confluent_kafka import Producer + import json + + # Kafka Configuration + KAFKA_BOOTSTRAP_SERVERS = 'localhost:9092' + KAFKA_TOPIC = 'pyarrow_stream' + + # Kafka Producer configuration + producer_config = { + 'bootstrap.servers': KAFKA_BOOTSTRAP_SERVERS, + 'client.id': 'pyarrow-stream-producer' + } + + # Create a Kafka producer + return Producer(producer_config) + + def stream_to_kafka(buffer_queue, topic): + while True: + batch = buffer_queue.get() + if batch is None: + break # End of stream + + # Serialize the batch to a bytes buffer + sink = pa.BufferOutputStream() + writer = ipc.RecordBatchStreamWriter(sink, batch.schema) + writer.write_table(batch) + writer.close() + + # Get the serialized data as bytes + data = sink.getvalue().to_pybytes() + + # Send serialized data to Kafka + producer.produce(topic, data) + producer.flush() # Ensure the message is sent + + buffer_queue.task_done() + all_files = self.get_all_files(files, local_specifier, remote_specifier, print_it=False) _datasets = all_files.keys() all_files = self.group_all_files_by_fs(all_files) diff --git a/pyproject.toml b/pyproject.toml index 5c6d0a0..e24e42f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,11 +74,11 @@ version_pattern = [ ] major_on_zero = false branch = "main" -upload_to_PyPI = true -upload_to_release = true +upload_to_vcs_release = true build_command = "pip install poetry && poetry build" -repository_url = "https://opencode.it4i.eu/openwebsearcheu-public/owi-cli" -vcs_provider = "gitlab" + +[tool.semantic_release.remote] +domain = "https://opencode.it4i.eu" type="gitlab"