diff --git a/Readme.md b/Readme.md index 9d16aea..c71e275 100644 --- a/Readme.md +++ b/Readme.md @@ -111,20 +111,29 @@ Local commands `owi local` are used to manage local datasets. - **Creating a local dataset and inserting files into local datasets** Creating a dataset from a local directory. Note that the dataset is only moved / copied to the configured owilix directory and not pushed to the server (which needs to be done separately) - If no InternalID is provided, the dataset is newly created - ```bash - owilix local insert file:///data/owseu/owilix/it4i/231203 access=public collectionName="mgrani" move=False - ``` - If an intenralID is provided, only files will be inserted - ```bash - owilix local insert file:///data/owseu/owilix/it4i/231203 access=public collectionName="mgrani" internalID=33ad44a3-8a85-41bf-8118-1647987c4e52 move=False - ``` - - Creating datasets on the project level and test the diffs then: - ```bash - owilix local insert /Users/username/mydata//2023-12-03 access=project collectionName="main" move=False - owilix --profile M remote diff all/access=project - ``` + - If no InternalID is provided, the dataset is newly created + ```bash + owilix local insert file:///data/owseu/owilix/it4i/231203 access=public collectionName="mgrani" move=False + ``` + - If an intenralID is provided, only files will be inserted + ```bash + owilix local insert file:///data/owseu/owilix/it4i/231203 access=public collectionName="mgrani" internalID=33ad44a3-8a85-41bf-8118-1647987c4e52 move=False + ``` + + - Creating datasets on the project level and test the diffs then: + ```bash + owilix local insert /Users/username/mydata//2023-12-03 access=project collectionName="main" move=False + owilix --profile M remote diff all/access=project + ``` + - Inserting using sub path selector (e.g. only parts of a directory): + ```bash + local insert file:///data/migration/it4i/2023-12-5 'sub_path=year=2023/month=12/day=6/**/*' access=public collectionName="main" move=False owner="OpenWebSearch.eu Consortium" creator="OpenWebSearch.eu Consortium" publisher="OpenWebSearch.eu Consortium" + ``` + - Listing all datasets that have been inserted, but not pushed (i.e. data center is unknown) + ```sh + local ls all/dataCenter=unknown + ``` + #### Remote Commands Remote commans operate on the datacenters specified. You can use the `--exclude dc1,dc2`flag to exclude some data centers diff --git a/owilix/cmd/local.py b/owilix/cmd/local.py index 99a07ea..6aca4ee 100644 --- a/owilix/cmd/local.py +++ b/owilix/cmd/local.py @@ -77,7 +77,7 @@ def ls(self, specifier, *args, **kwargs): @LocalCommands.register def insert(self, path, access="public", collectionName="userslice", internalID=None, inferred=True, - move=True, **kwargs): + move=True, sub_path=None, **kwargs): """ insert files from a filesystem. Command structure (note that the path specifier replaces the usual specifier): `local insert access=[access] collectionName=[collectionName] internalId=[internalId] move=[move] @@ -98,16 +98,19 @@ def insert(self, path, access="public", collectionName="userslice", internalID=N - internalId: str, the internal id of the dataset. If not provided, a new dataset is created - move: bool, whether to move the dataset or copy it. - inferred:bool (default False), whether to overwrite inferred metadata with provided metadata + - sub_path: str, glob specifying the sub path to be imported ignoring the rest. - kwargs: dict, metadata for the dataset. supports + """ # 0. prepare and inferre metdata overwrite_inferred = inferred if isinstance(inferred, bool) else inferred.lower() in ("true", "t", "1") move = move if isinstance(move, bool) else move.lower() in ("true", "t", "1") filepath, fs, protocol = get_filesystem(path) if not fs.isdir(filepath): raise ValueError(f"Path {filepath} is not a directory on filesystem {protocol}") - self.console.print(f"Looking for files to insert under {filepath}") - _file_details = fill_file_details(fs, [f for f in fs.glob(filepath + "/**/*") if not fs.isdir(f)], root=filepath) + sub_path = sub_path or "**/*" + self.console.print(f"Looking for files to insert under {filepath} with glob {sub_path}") + _file_details = fill_file_details(fs, [f for f in fs.glob(os.path.join(filepath,sub_path)) if not fs.isdir(f)], root=filepath) if internalID is not None: # create internalId and fail in the unlikely event of a double id _ds_local = self.owi.local.list(access=access, query={"internalID": internalID, "collectionName": collectionName})