diff --git a/docs/source/architecture.md b/docs/source/architecture.md new file mode 100644 index 0000000..88973a9 --- /dev/null +++ b/docs/source/architecture.md @@ -0,0 +1,5 @@ +# Architecture + +`owilix` dinstinguishes between the core packages, consisting of managers and repositories, and +the command packages, realising several CLI commands. + diff --git a/owilix/cli.py b/owilix/cli.py index 8b193f9..68c5fc0 100644 --- a/owilix/cli.py +++ b/owilix/cli.py @@ -132,7 +132,7 @@ def local(ctx, subcmd, specifier, args): args, kwargs = extract_args(args) LocalCommands(ctx.obj['OWI'], **ctx.obj).do(subcmd, specifier, *args, **kwargs) -@click.command(help='Executes local commands. See local help for available subcommands. ', context_settings={"ignore_unknown_options": True}) +@click.command(help='Executes sql query commands. See local help for available subcommands. ', context_settings={"ignore_unknown_options": True}) @click.argument('subcmd', required=True) @click.option('--local', default="", help="Local specifier for datasets to query") @click.option('--remote', default="", help="Remote specifier for datasets to query") diff --git a/owilix/cmd/__init__.py b/owilix/cmd/__init__.py index 26de59d..d7525a0 100644 --- a/owilix/cmd/__init__.py +++ b/owilix/cmd/__init__.py @@ -3,12 +3,14 @@ Package containing the different commands for OWIlix. Commands are intended to be run via the CLI, receive context objects from the CLI and can interact with the user while using the owilix.core API, whereas the rich cmd python library is used for interaction. -- BaseCommand: Base class for all commands. It contains common methods and attributes used by all commands, particularly in resolviing context parameters like getting the rich console etc. -- LexisDownloadManager: class for managing download of partitoins from the Open Web Index. It uses the Lexis Rest API and does not rely on iRODS for file access. -- IRODSBasedCommands: class for accessing the iRODS datasets and defining a set of commands using duckdb plus IRODS. + +`owilix.cmd.base.BaseCommands` defines a base class for implementing commands. It thereby defines a register method, +that allows to register methods as commands for this class. + + """ -from .remote import IRODSBasedCommands, RemoteCommands -from .base import show_dataframe +from .remote import RemoteCommands from .local import LocalCommands from .admin import AdminCommands -from .config import ConfigCommands \ No newline at end of file +from .config import ConfigCommands +from .query import QueryCommands, OWIlixSQLQuery \ No newline at end of file diff --git a/owilix/cmd/admin.py b/owilix/cmd/admin.py index 66aa438..a33590a 100644 --- a/owilix/cmd/admin.py +++ b/owilix/cmd/admin.py @@ -2,13 +2,15 @@ import os from collections import defaultdict import pandas as pd +from irods.column import Criterion +from irods.models import Collection, DataObject from rich.prompt import Prompt from owilix.cmd.base import BaseCommand, input_dict, SubCommand, ask_yes_no from owilix.core.metadata import fill_metadata, cast_metadata, format_metadata, \ extract_metadata_from_title, validate_metadata, metadata_in_schema from owilix.core.repository import load_metadata_from_irods, \ - update_metadata_for_irods_collection + update_metadata_for_irods_collection, get_metadata_from_irods_collection from owilix.core.utils import get_filesystem, fill_file_details @@ -45,7 +47,6 @@ def correct_ui(metadata): edited = True return choice, metadata - @AdminCommands.register def set_irods_metadata(self, path, **kwargs): """ @@ -69,11 +70,49 @@ def set_irods_metadata(self, path, **kwargs): else: _md = format_metadata(_md) self.console.print("Saving metadata") - update_metadata_for_irods_collection(_fs.session, _p, _md) + update_metadata_for_irods_collection(_fs.session.collections.get(_p), _md) self.console.print("Saved. Loading metadata back and showing it:") _md = load_metadata_from_irods(_fs.session, _p) self.show_details(_md) +@AdminCommands.register +def set_irods_metadata_new(self, path, dataset_root=False, **kwargs): + """ + sets the metadata for the dataset at the provided path. path must start with irods:// + """ + # instantiate sessions here + dataset_root = dataset_root if isinstance(dataset_root,bool) else dataset_root.lower() in ("true", "t", "1") + _p, _fs, _prot = get_filesystem(path) + _update = {k:v for k,v in kwargs.items() if metadata_in_schema(k) if v} + _delete = {k:"" for k,v in kwargs.items() if metadata_in_schema(k) if not v} + if _prot != "irods": + raise ValueError("filesystem not irods filesystem. Please specify a path starting with irods:// and proper credentials") + + if dataset_root: + _collections = [c for c in self.session.collections.get(_p).subcollections if _fs.session.collections.exists(_p)] + else: + _collections = [self.session.collections.get(_p)] + + for c in _collections: + self.console.print("Estimating files and filling metadata.") + _files = [r[Collection.name]+"/"+r[DataObject.name] for r in self.session.query(Collection.name, DataObject.id, DataObject.name, DataObject.path, + DataObject.size, DataObject.checksum) + .filter(Criterion('=', Collection.id, c.id))] + #_files = [f for f in _fs.glob(os.path.join(_p,"**/*")) if not _fs.isdir(f)] + _file_details = fill_file_details(_fs, _files, _p) + _md= get_metadata_from_irods_collection(c) + _md = fill_metadata(_md, _file_details, True) + _md.update({k:v for k,v in _update.items() if metadata_in_schema(k)}) + self.show_details(_md) + if self.autoyes or ask_yes_no(self.console, f"Do you want to edit metadata for {path} (note that str.format will be done afterwards):"): + input_dict(_md, title="Modify Dataset Metadata") + _md = format_metadata(_md) + self.show_details(_md) + else: + _md = format_metadata(_md) + self.console.print("Saving metadata") + update_metadata_for_irods_collection(c, _md) + @AdminCommands.register