From e4bff1cf7e7f541bccb9be4cce5e1db27430c97b Mon Sep 17 00:00:00 2001 From: Michael Granitzer Date: Sat, 20 Apr 2024 09:30:21 +0200 Subject: [PATCH] first cli version: dataset list, stats --- .gitignore | 2 +- LICENSE | 21 ++++++++++++++ Readme.md | 51 +++++++++++++-------------------- datasets.py | 34 ---------------------- modules/Py4Lexis | 2 +- owi/__init__.py | 0 owi/cli.py | 48 +++++++++++++++++++++++++++++++ owi/core/__init__.py | 1 + owi/core/datasets.py | 68 ++++++++++++++++++++++++++++++++++++++++++++ owi/core/irods.py | 13 +++++++++ owi/core/project.py | 44 ++++++++++++++++++++++++++++ requirements.txt | 4 +++ test.py | 31 ++++++++++++++++++++ 13 files changed, 252 insertions(+), 67 deletions(-) create mode 100644 LICENSE delete mode 100644 datasets.py create mode 100644 owi/__init__.py create mode 100644 owi/cli.py create mode 100644 owi/core/__init__.py create mode 100644 owi/core/datasets.py create mode 100644 owi/core/irods.py create mode 100644 owi/core/project.py create mode 100644 requirements.txt create mode 100644 test.py diff --git a/.gitignore b/.gitignore index e489bd0..05e83c2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ outputs mlruns flagged tmp/ -lexis_logs.log +lexis_logs-failed.log # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..037a300 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 OpenWebSearch.eu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Readme.md b/Readme.md index d52a243..95b5521 100644 --- a/Readme.md +++ b/Readme.md @@ -1,51 +1,40 @@ -# OWSEU Lexis Workflows +# OWI CLI -Repositories to explore the use of Lexis in OWSEU +Command line interface for managing [OpenWebIndex](http://openwebindex.eu) slices. -## Installation -Via conda +## Examples ```bash -conda activate -cd ~/tmp -git clone https://github.com/OpenWebSearch/Py4Lexis -cd Py4Lexis -python -m pip install . +PYTHONPATH=. python owi/cli.py ls --start 2024-01-01 --days 10 +PYTHONPATH=. python owi/cli.py stat --start 2024-01-01 --days 10 ``` -as source +Repositories to explore the use of Lexis in OWSEU -```bash -git submodule update --init -``` -then add the `modules/Py4Lexis` to your `PYTHONPATH` +## Troubleshooting -note that we worked on a forked version of the Py4Lexis client. The fork is available at OpenWebSearch +- `Error occured in py4lexis.core.session.__check_token_expiration(): SESSION -- TOKENS ARE NONE -- FAILED`: remove `.env` or `~/.tokens_lxs` -## Getting your account / setting the password +## Installation -you need to obtain a password for your user account and/or create a user account +Usually `pip install -r requirements.txt` should do the trick. -- Go to the [AAI Lexis Keycloack Realm](https://aai.lexis.tech/auth/realms/LEXIS_AAI/account/#/security/signingin). Login with [B2ACCESS](https://b2access.eudat.eu/home/). We encourage to use your university login or your ORCID account or your Github account -- In the AAI Lexis Keycloak Realm portal, navigate to "Account Security" "Signing In" and set a password for your account. -- Under personal information you find your username. +### Manual -## Using the python client +You need tabulate, pandas and py4lexis -the original client will prompt for user name and password: +Py4lexis Via Pip: -```python -from py4lexis.session import LexisSession -session = LexisSession() -Welcome to the Py4Lexis! -Please provide your credentials... -Username: +```bash +pip install py4lexis --index-url https://opencode.it4i.eu/api/v4/projects/107/packages/pypi/simple ``` -The forked version of the [client at OpenWebSearch](https://github.com/OpenWebSearch/Py4Lexis) allows to pass the credentials as environment variables: +Or as source ```bash -LEXIS_USERNAME= LEXIS_PASSWORD= python -c "from py4lexis.session import LexisSession; session = LexisSession()" -You have been successfully logged in LEXIS session. +git submodule update --init ``` +then add the `modules/Py4Lexis` to your `PYTHONPATH` + + diff --git a/datasets.py b/datasets.py deleted file mode 100644 index c09188a..0000000 --- a/datasets.py +++ /dev/null @@ -1,34 +0,0 @@ -# This is a sample Python script. - -# Press Shift+F10 to execute it or replace it with your code. -# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings. - - -# Press the green button in the gutter to run the script. -if __name__ == '__main__': - from py4lexis.session import LexisSession - from py4lexis.cli.datasets import DatasetsCLI - - session = LexisSession() - dscli = DatasetsCLI(session) - datasets, req_status = dscli.datasets.get_all_datasets(content_as_pandas=True) - if req_status== 200: - print(f"Found {len(datasets)} datasets") - else: - print("Error {req_status}") - exit(1) - if len(datasets)>0: - dataset_id= datasets["InternalID"][0] - access = datasets["Access"][0] - project = datasets["Project"][0] - print("Using dataset", dataset_id, access, project) - files, req_status = dscli.datasets.get_list_of_files_in_dataset(dataset_id,access, project, session.DFLT_Z ,path='',content_as_pandas=True) - print(f"found {len(files)} files") - # get temporary directory - import tempfile - dir = tempfile.gettempdir() - print(f"Downloading to {dir}") - dscli.download_dataset(dataset_id=dataset_id, access=access, project=project, path=dir) - - -# See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/modules/Py4Lexis b/modules/Py4Lexis index 59514c4..4003c72 160000 --- a/modules/Py4Lexis +++ b/modules/Py4Lexis @@ -1 +1 @@ -Subproject commit 59514c48acf42bdac4d9a17004ad13ec5ce8a86f +Subproject commit 4003c726d30bea67ee2eae93009e7587ec89a58b diff --git a/owi/__init__.py b/owi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/owi/cli.py b/owi/cli.py new file mode 100644 index 0000000..9ab0e69 --- /dev/null +++ b/owi/cli.py @@ -0,0 +1,48 @@ +import argparse + +from tabulate import tabulate +from owi.core import OWSProject + +def project_stats(owi, args): + df = owi.stats(args.start, args.days, args.public_only) + print("Available Datasets:") + print(tabulate(df[df["Kind"]=="Datasets"], headers='keys', tablefmt=args.tablefmt)) + +def list_datasets(owi, args): + datasets = owi.datasets.ls(args.start, args.days, args.public_only, args.query) + print(tabulate(datasets, headers='keys', tablefmt=args.tablefmt)) + +def main(): + parser = argparse.ArgumentParser(description="CLI tool for handling OWI data.") + parser.add_argument('--tablefmt', type=str, default="psql", help='Output format') + subparsers = parser.add_subparsers(help='commands', dest='command') + subparsers.required = True + + # Sub-parser for project_stats + parser_project_stats = subparsers.add_parser('stats', help='Display project statistics') + parser_project_stats.add_argument('--start', type=str, default=None, help='Start date in YYYY-MM-DD') + parser_project_stats.add_argument('--days', type=int, default=None, help='Duration in days') + parser_project_stats.add_argument('--public_only', action='store_true', help='Flag to only show public datasets') + parser_project_stats.add_argument('--query', type=str, default=None, help='Query to filter datasets') + + # Sub-parser for list_datasets + parser_list_datasets = subparsers.add_parser('ls', help='List available datasets') + parser_list_datasets.add_argument('--start', type=str, default=None, help='Start date in YYYY-MM-DD') + parser_list_datasets.add_argument('--days', type=int, default=None, help='Duration in days') + parser_list_datasets.add_argument('--public_only', action='store_true', help='Flag to only show public datasets') + parser_list_datasets.add_argument('--query', type=str, default=None, help='Query to filter datasets') + + # Parse arguments + args = parser.parse_args() + + # Instantiate OWI object + owi = OWSProject() + + # Execute the appropriate function based on the command + if args.command == 'stats': + project_stats(owi, args) + elif args.command == 'ls': + list_datasets(owi, args) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/owi/core/__init__.py b/owi/core/__init__.py new file mode 100644 index 0000000..82d06cb --- /dev/null +++ b/owi/core/__init__.py @@ -0,0 +1 @@ +from .project import OWSProject \ No newline at end of file diff --git a/owi/core/datasets.py b/owi/core/datasets.py new file mode 100644 index 0000000..c2e195d --- /dev/null +++ b/owi/core/datasets.py @@ -0,0 +1,68 @@ +import logging +from typing import List + +from pandas import DataFrame + +from py4lexis.session import LexisSession +from py4lexis.cli.datasets import Datasets +from datetime import datetime, timedelta +import pandas as pd + +def filter_by_date_range(df, start_date=None, end_date=None): + if start_date and end_date: + mask = (df['Date'] >= start_date) & (df['Date'] <= end_date) + elif start_date: + mask = (df['Date'] >= start_date) + elif end_date: + mask = (df['Date'] <= end_date) + else: + return df # No filtering, return all + return df[mask] + +class OWSDatasets: + + def __init__(self, datasets): + self.datasets + +class OWSData: + + def __init__(self, project_name, session=None): + self.session =session if session is not None else LexisSession(in_cli=True) + self.dscli = Datasets(self.session) + self.__project_name = project_name + + def _to_datetime(self, date: str|datetime): + return datetime.strptime(date, "%Y-%m-%d") if isinstance(date,str) else date + + def ls(self, startdate: str|datetime|None=None, duration:int = 0, public_only =True, query=None): + startdate = self._to_datetime(startdate) + enddate = None if startdate is None else startdate + timedelta(days=duration) + df = self.dscli.get_all_datasets(content_as_pandas=True) + df= df[df["Project"] == self.__project_name] + pattern = r'@(.+?)[-\s](\d{4}/\d{1,2}/\d{1,2})$' + df[['DataCenter', 'Date']] = df['Title'].str.extract(pattern) + df['Date'] = pd.to_datetime(df["Date"]) + df = filter_by_date_range(df, pd.to_datetime(startdate), pd.to_datetime(enddate)) + if query is not None: + df = df.query(query) + + if public_only: + return df[df["Access"]=="public"] + else: + return df + + def files(self, datasets, filter=None): + pass + + def download(self, datasets:DataFrame, language: List[str]|None = None): + + pass + + def download_files(self, files: DataFrame): + + pass + + + + + diff --git a/owi/core/irods.py b/owi/core/irods.py new file mode 100644 index 0000000..8f40545 --- /dev/null +++ b/owi/core/irods.py @@ -0,0 +1,13 @@ +from py4lexis.session import LexisSession +from py4lexis.lexis_irods import iRODS + +## IRODS Client for the project. Allows maybe faster and more direct operations, including accessing iRODS metadata +## but needs to be tested + +class IRODSOWI(iRODS): + + + def __init__(selfl,session: LexisSession, + suppress_print: bool=True): + super().__init__(session, suppress_print) + diff --git a/owi/core/project.py b/owi/core/project.py new file mode 100644 index 0000000..bc5f8fb --- /dev/null +++ b/owi/core/project.py @@ -0,0 +1,44 @@ +import logging +import os +from datetime import datetime + +from owi.core.datasets import OWSData +from py4lexis.session import LexisSession +import pandas as pd + + +def _flatten_and_count(df, column): + # flatten a column [a,b,c] and count the occurences of values. + if df[column].apply(lambda x: isinstance(x, list)).any(): + if df[column].explode().apply(lambda x: isinstance(x, list)).any(): + return df[column].explode().explode().value_counts().reset_index() # coudl be made generic, but no need yet + return df[column].explode().value_counts().reset_index() + else: + return df[column].value_counts().reset_index() + + +class OWSProject: + + def __init__(self): + self.name = os.getenv("OWI_LEXIS_PROJECT_NAME", "openwebsearch") + self.session = LexisSession(in_cli=True) + self.datasets = OWSData(self.name, self.session) + self.logger = logging.getLogger(__name__) + + def stats(self, startdate: str|datetime|None=None, duration:int = 0, public_only =True): + df = self.datasets.ls(startdate, duration, public_only) + # Initialize an empty DataFrame for the summary + summary_df = pd.DataFrame({'Kind':["Datasets"], "Key":["Total"], "Value":["Number Datasets"], "Count":[len(df)]}) + if df is None: return None + df['Month'] = pd.to_datetime(df['Date']).dt.to_period('M') + columns_to_exclude = ["Title", "InternalID", "CreationDate", "Date"] + df = df[[col for col in df.columns if col not in columns_to_exclude]] + # Loop through columns and append results + for column in df.columns: + temp_df = _flatten_and_count(df, column) + temp_df.columns = ['Value', 'Count'] + temp_df['Kind'] = "Datasets" + temp_df['Key'] = column + summary_df = pd.concat([summary_df, temp_df], ignore_index=True) + return summary_df + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9654ea0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +tabulate +pandas +--extra-index-url https://opencode.it4i.eu/api/v4/projects/107/packages/pypi/simple +py4lexis \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..197673e --- /dev/null +++ b/test.py @@ -0,0 +1,31 @@ +from py4lexis.session import LexisSession +from py4lexis.ddi.datasets import Datasets +session = LexisSession() +ds = Datasets(session, suppress_print=False) +content: list[dict] = ds.get_all_datasets(content_as_pandas=True) +print(content) + + + +#2024-02-24 14:45:56,572 -- INFO -- py4lexis.core.session -- _check_api -- Initialise API path 'https://api.lexis.tech/api/v0.2' -- OK +#2024-02-24 14:45:56,574 -- INFO -- py4lexis.core.lexis_irods -- _validate_irods -- iRODS -- VALIDATE TOKEN -- PROGRESS +#2024-02-24 14:45:56,575 -- INFO -- py4lexis.core.session -- make_request -- Preparing 'GET' request with the following payload: +# None +#2024-02-24 14:45:57,072 -- INFO -- py4lexis.core.lexis_irods -- _validate_irods -- iRODS -- VALIDATE TOKEN -- OK +#2024-02-24 14:45:57,073 -- INFO -- py4lexis.core.lexis_irods -- _get_irods_session -- iRODS -- INITIALISED -- OK +#2024-02-24 14:46:04,732 -- INFO -- py4lexis.core.utils -- assemble_dataset_path -- Assembled irods path: public/proj862c5962623246664c1fda27b7afb108 +#2024-02-24 14:46:04,732 -- INFO -- py4lexis.core.lexis_irods -- get_project_collection -- Getting directory (collection): /IT4ILexisV2/public/proj862c5962623246664c1fda27b7afb108 +#2024-02-24 14:46:05,637 -- ERROR -- py4lexis.core.lexis_irods -- log_and_print_errors -- Unhandled Exception occurred. Exception: + +from py4lexis.session import LexisSession +from py4lexis.lexis_irods import iRODS +from irods.models import DataObject, DataObjectMeta, Collection, CollectionMeta +from irods.column import Criterion + +session = LexisSession() + +irods = iRODS(session=session, + suppress_print=False) +isession = irods._iRODS__irods +isession.query(Collection, CollectionMeta).filter(Criterion('=', CollectionMeta.name, 'type')).filter(Criterion('like', CollectionMeta.value, '%Project%')) +irods.get_project_collection(access="public", project="openwebsearch", zone="IT4ILexisV2") \ No newline at end of file -- 2.51.2