From af4b3468cf9ccd55131144fe3d3bcb947ac1ac03 Mon Sep 17 00:00:00 2001 From: Michael Granitzer Date: Fri, 7 Jun 2024 08:51:09 +0200 Subject: [PATCH] small updates --- Readme.md | 11 +++++++- owilix/core/repository.py | 59 ++++++++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/Readme.md b/Readme.md index ae4182f..0e5be0a 100644 --- a/Readme.md +++ b/Readme.md @@ -5,6 +5,14 @@ The OWI (Open Web Index) CLI is a command-line tool designed for managing [OpenWebIndex](http://openwebindex.eu) data slices. This documentation will guide you through its setup, usage, and troubleshooting. +## Data License + +By using this tool you agree to our [Open Web Index Licnse (OWIL)](https://ows.eu/owil/current), to our [Terms of Use](https://dashboard.ows.eu/terms-of-use) as well as to the terms of use of the used data centers (via [B2ACCESS Login](https://b2access.eudat.eu/)). + +## Software License + +`owilix` itself is licensed under the [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0). + ## Installation ### Requirements @@ -286,4 +294,5 @@ fix Fixed change Changed remove Removed merge Merged -doc Documented \ No newline at end of file +doc Documented + diff --git a/owilix/core/repository.py b/owilix/core/repository.py index fc820b9..91bc018 100644 --- a/owilix/core/repository.py +++ b/owilix/core/repository.py @@ -31,12 +31,6 @@ class AggregatedRepository: @staticmethod def single_repo_from_url(url): parsed = urllib.parse.urlparse(url) - options = { - 'host': parsed.hostname, - 'port': parsed.port, - 'user': parsed.username, - 'password': parsed.password - } if parsed.scheme =="file": return FileBasedRepository(None, parsed.path) elif parsed.scheme == "irods": @@ -271,6 +265,7 @@ class AbstractRepository: Returns: tuple (DataFrame, DataFrame): a DataFrame with the metadata of the datasets and a second Datframe with the files per dataset (key=internalID) """ + raise NotImplementedError() @abstractmethod def exists(self, dataset: Dataset): @@ -281,12 +276,22 @@ class AbstractRepository: :param dataset: :return: """ - pass + raise NotImplementedError() @abstractmethod def files_details(self, dataset:Dataset, files_glob=None): """like files, but also aims to esimate filesizes and object counts in files. might be slow""" - pass + raise NotImplementedError() + + @abstractmethod + def readlines(self, dataset: Dataset, file_name): + """reads the text file `file_name` in the dataset if it exits. Otherwise returns None""" + raise NotImplementedError() + + @abstractmethod + def writelines(self, dataset: Dataset, file_name, content): + """writes the content into `file_name` in the dataset.""" + raise NotImplementedError() @abstractmethod def files(self, dataset:Dataset, files_glob=None): @@ -324,17 +329,17 @@ class AbstractRepository: that fetches the dataset's content based on given parameters and returns it as a pandas DataFrame. """ - pass + raise NotImplementedError() @abstractmethod def delete(self, dataset:Dataset): - pass + raise NotImplementedError() @abstractmethod def delete_by_id(self, internalID:str): - pass + raise NotImplementedError() @abstractmethod def put(self, dataset:Dataset, local_path:str, rel_filename:str): @@ -349,7 +354,7 @@ class AbstractRepository: Returns: filename of the uploaded file """ - pass + raise NotImplementedError() @abstractmethod def get(self, dataset, file, local_path): @@ -361,14 +366,14 @@ class AbstractRepository: file: the file to get. If file can be either relative to dataset or prefixed with the dataset path local_path: the local path to store the file (without the filename itself() """ - pass + raise NotImplementedError() @abstractmethod def rm(self, dataset, files): """ Remove a file from the dataset. """ - pass + raise NotImplementedError() def checks(self, access:str="public", day: str | datetime | None=None, @@ -391,10 +396,10 @@ class AbstractRepository: Returns: a list of errors for the datasets """ - pass + raise NotImplementedError() def update_metadata(self, dataset): - pass + raise NotImplementedError() @abstractmethod def create(self, access:str = "public", @@ -413,7 +418,7 @@ class AbstractRepository: Returns: the created dataset """ - pass + raise NotImplementedError() @@ -576,6 +581,7 @@ class FileBasedRepository(AbstractRepository): if len(_md_errors)>0: raise ValueError (f"Metadata errors for dataset {dataset.internalID}: "+",".join(_md_errors)) try: + dataset.update_lastchanged() with self.fs.open(_p+".json", "w") as _fh: json.dump(dataset.metadata, _fh) except Exception as e: @@ -583,6 +589,24 @@ class FileBasedRepository(AbstractRepository): self.fs.rm(_p+".json") return [(None, str(e))] + def readlines(self, dataset: Dataset, file_name): + """reads the text file `file_name` in the dataset if it exits. Otherwise returns None""" + _p = self._get_path(dataset) + file_path =os.path.join(_p,file_name) + if self.fs.exists(file_path): + # Open the file and read its content + with self.fs.open(file_path, 'r') as file: + content = file.read() + return content + else: + return None + + def writelines(self, dataset: Dataset, file_name, content): + """writes the content into `file_name` in the dataset.""" + _p = self._get_path(dataset) + file_path = os.path.join(_p, file_name) + with self.fs.open(file_path, 'w') as file: + file.write(content) class LocalRepository(FileBasedRepository): pass @@ -680,6 +704,7 @@ class IRODSRepository(FileBasedRepository): _md_errors = validate_metadata(dataset.metadata) if len(_md_errors) > 0: raise ValueError(f"Dataset {dataset.internalID} has validation errors:"+ ",".join(_md_errors)) + dataset.update_lastchanged() return update_metadata_for_irods_collection(self.session, _p, dataset.metadata) def change_id(self, dataset, new_id): -- 2.51.2