From 59acd1265f776befeea8b30de0b42128558c721c Mon Sep 17 00:00:00 2001 From: mgrani Date: Tue, 4 Aug 2026 09:29:10 +0200 Subject: [PATCH] fix(remote): --push-to must not select what to pull from `remote pull --push-to ` reported "0 datasets found" and exited 0 for every specifier whose data center is None. Two lines combined into a query that is empty by construction: spec.get("data_center") or (push_to_remote if push_to_remote else None), ... ignore_data_centers=[push_to_remote] if push_to_remote else None When the specifier carries no data center the source falls back to the push destination, which is then excluded -- "list from X while excluding X". That is not a corner case. `parse_specifier` leaves data_center None for `all`, and the it4i/lrz zone aliases *explicitly reset it to None* before setting a zone query, so the only scopes that ever worked were ones naming a repository directly (`lexis:...`). `remote_ls` was unaffected because it passes `spec.get("data_center")` alone, which is why `ls` and `pull` disagreed on the same specifier. Verified against a live OWI deployment: remote ls "all/collectionName=main" -> 584 datasets, 51 TiB remote pull "it4i:2026-08-01" --files "**/*.parquet" --yes -> 5 datasets, fetches ... the same plus --push-to owi-up -> 0 datasets found remote pull "lexis:latest" --files "**/*.parquet" --yes --push-to owi-up -> 5 datasets, fetches Ruled out by experiment first: specifier form, --files form (including omitting it), destination contents (a destination holding 5 collections also returned 0), auth, config and image staleness. Impact: an OWI mirror could not be filled through `--push-to` at all, and each attempt looked like a successful run that happened to move nothing. `ignore_data_centers` is kept -- not pulling from the remote you are pushing to is correct; only the source fallback was wrong. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019nPNpEG2QRnmjxxn5LeEb7 --- owilix/core/tasks/remote.py | 20 +++++++++- tests/owilix/core/tasks/test_remote.py | 54 ++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/owilix/core/tasks/remote.py b/owilix/core/tasks/remote.py index 3ec3e62..1d16564 100644 --- a/owilix/core/tasks/remote.py +++ b/owilix/core/tasks/remote.py @@ -228,8 +228,24 @@ def remote_pull( spec = manager.parse_specifier(specifier) access, query = split_query_access(spec.get("query")) + # Source is the specifier's data center, never the push destination. + # + # This used to fall back to `push_to_remote` when the specifier carried no + # data center -- and then passed that same remote as `ignore_data_centers`. + # The two combined into "list from X while excluding X", which is empty by + # construction, so `--push-to` returned "0 datasets found" and exited 0. + # + # It hit every specifier that matters: `parse_specifier` leaves + # `data_center` None for `all`, and the `it4i`/`lrz` zone aliases + # *explicitly reset it to None* before setting a zone query. So the only + # scopes that worked were ones naming a repository directly (`lexis:...`). + # `remote_ls` never had this because it passes `spec.get("data_center")` + # alone -- which is what a pull should do too. + # + # Keeping `ignore_data_centers` is right: when pushing to a remote, do not + # also treat that remote as a source to pull from. datasets = manager.remote_data.list( - spec.get("data_center") or (push_to_remote if push_to_remote else None), + spec.get("data_center"), access, day=spec.get("day"), duration=spec.get("duration") or 0, @@ -648,7 +664,7 @@ def remote_doctor( else: results["token_error"] = str(e) - return CommandResult(success=True, object=None) + return CommandResult(success=True, object=results if as_json else None) def remote_push( diff --git a/tests/owilix/core/tasks/test_remote.py b/tests/owilix/core/tasks/test_remote.py index 983a955..14f824f 100644 --- a/tests/owilix/core/tasks/test_remote.py +++ b/tests/owilix/core/tasks/test_remote.py @@ -691,6 +691,60 @@ class TestRemoteDoctor: # remote_pull # --------------------------------------------------------------------------- class TestRemotePull: + def test_push_to_does_not_become_the_source(self): + """`--push-to` must not select what to pull FROM. + + Regression for a self-cancelling query: the source argument fell back to + `push_to_remote` when the specifier carried no data center, while + `ignore_data_centers` excluded that same remote -- "list from X + excluding X", which is empty by construction. `remote pull --push-to` + therefore reported "0 datasets found" and exited 0 for every specifier + whose data center is None, which is `all` and (because the zone aliases + reset it) every `it4i:`/`lrz:` scope as well. Observed on both OWI + clusters: the mirror could never be filled, and each attempt recorded a + successful run that moved nothing. + """ + manager = _make_mock_manager(datasets=[]) + manager.parse_specifier.return_value = { + "data_center": None, + "query": {"access": "public"}, + "day": None, + "duration": 0, + } + + remote_pull( + manager, + specifier="all", + push_to_remote="owi-up", + auto_yes=True, + console=MagicMock(), + ) + + args, kwargs = manager.remote_data.list.call_args + assert args[0] is None, "source must stay the specifier's data center, not the destination" + # Still right to skip the destination as a source; only the fallback was wrong. + assert kwargs["ignore_data_centers"] == ["owi-up"] + + def test_an_explicit_data_center_is_still_passed_through(self): + manager = _make_mock_manager(datasets=[]) + manager.parse_specifier.return_value = { + "data_center": "lexis", + "query": {"access": "public"}, + "day": None, + "duration": 0, + } + + remote_pull( + manager, + specifier="lexis:latest", + push_to_remote="owi-up", + auto_yes=True, + console=MagicMock(), + ) + + args, _ = manager.remote_data.list.call_args + assert args[0] == "lexis" + def test_no_datasets_found(self): manager = _make_mock_manager(datasets=[]) result = remote_pull(manager, specifier="dc1/public", auto_yes=True, console=MagicMock()) -- 2.51.2