diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ee4734..febf489 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,25 @@ ## Unreleased +## v5.10.2 (2026-08-07) + +M3 of the summarize/integrity plan: `README.md` gains a machine-readable head. Additive — the prose below it is unchanged. + +### Features + +- **summarize**: `README.md` now opens with YAML front matter carrying the same fields as an index row — `id`, `title`, `collectionName`, `dataCenter`, `access`, `startDate`, `endDate`, `declared`, `observed`, `materialised`, `parquet_files_found`, `computed_at`, `owilix_version`. A program reads one shape whether it comes from the README, `stats.json`, or (later) the collection index, instead of parsing prose. +- **summarize**: A metadata value PyYAML cannot represent is rendered as its string form rather than raising. Dataset metadata is not a closed schema, and a README write is the wrong place to discover that. + +### Bug Fixes + +- **summarize**: The unchanged-write check understands front matter. `computed_at` moved into the README as well as `stats.json`, so a byte comparison would never match and **every README in the mirror would have been rewritten on every run** — silently restoring the write churn v5.10.1 removed, while looking like a feature. The comparison now ignores `computed_at` inside the front matter and compares the prose literally. A README with no front matter (an older mirror) is still compared bytewise. + +### Notes + +- Verified on the fixture: a second `--force` pass performs zero writes with the timestamp present, the front matter round-trips through `yaml.safe_load`, and the human section is byte-identical when only the timestamp moves. +- A malformed front-matter block is treated as "no front matter" rather than an error — a bad head must not cost the body. + + ## v5.10.1 (2026-08-07) M0–M2 of the summarize/integrity plan. All additive; `stats.json` keeps its existing shape and every current reader is unaffected. diff --git a/Readme.md b/Readme.md index 4f0b091..7a1a3ed 100644 --- a/Readme.md +++ b/Readme.md @@ -1,6 +1,6 @@ # OWILIX - Open Web Index CLI -[![Version](https://img.shields.io/badge/version-5.10.1-blue.svg)](https://openwebsearcheu-public.pages.it4i.eu/owi-cli/) +[![Version](https://img.shields.io/badge/version-5.10.2-blue.svg)](https://openwebsearcheu-public.pages.it4i.eu/owi-cli/) [![Python](https://img.shields.io/badge/python-3.11+-green.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-Apache_2.0-orange.svg)](http://www.apache.org/licenses/LICENSE-2.0) diff --git a/docs/changes.md b/docs/changes.md index 97e74ce..f214c27 100644 --- a/docs/changes.md +++ b/docs/changes.md @@ -24,6 +24,24 @@ Brief description of what was accomplished. ## Unreleased +### main @ v5.10.2 - 2026-08-07 + +#### README gets a machine-readable head, without restoring the churn + +**Summary**: M3 of the summarize/integrity plan. YAML front matter on `README.md`, prose untouched. + +**Changes**: +- `_readme_front_matter` emits the index-row fields as YAML; `split_front_matter` parses them back and is tolerant of a malformed or absent head. +- **The interaction that mattered**: `computed_at` now appears in the README too, so the M2 unchanged-write check would never have matched and every README would have been rewritten on every run — reinstating the exact churn that could invalidate a dircache ETag and, before v5.9.0, empty a listing. `_stats_computed_at_only_diff` now strips volatile keys from front matter as well as from `stats.json`, and compares the prose literally. Caught by running the fixture, not by reading the diff. +- `_yaml_safe` coerces unrepresentable metadata values rather than raising. + +**Verified**: front matter round-trips; a second `--force` pass writes nothing; the human section is byte-identical when only the timestamp moves. + +**Breaking Changes**: None. Additive head; existing readers of the prose are unaffected. + +**Still open from the plan**: M4 (`_index.parquet` + `remote reindex` — where the risk concentrates, and its own release), M5 (HTTP exposure), M6 (verify tiers, checksums, cost controls). + + ### main @ v5.10.1 - 2026-08-07 #### Observed vs declared, provenance, and an end to write churn diff --git a/owilix/_version.py b/owilix/_version.py index 72937b3..71b55a7 100644 --- a/owilix/_version.py +++ b/owilix/_version.py @@ -1,3 +1,3 @@ # Version is set here and imported elsewhere -__version__ = "5.10.1" -__version_tuple__ = (5, 10, 1) +__version__ = "5.10.2" +__version_tuple__ = (5, 10, 2) diff --git a/owilix/core/tasks/remote.py b/owilix/core/tasks/remote.py index c4dcbb0..d38318a 100644 --- a/owilix/core/tasks/remote.py +++ b/owilix/core/tasks/remote.py @@ -2440,12 +2440,22 @@ def _write_if_changed(dataset, name: str, content: str, console: Optional[Consol return True +def _without_volatile(document: Dict) -> Dict: + return {k: v for k, v in document.items() if k not in _VOLATILE_ARTIFACT_KEYS} + + def _stats_computed_at_only_diff(dataset, name: str, content: str) -> bool: """Whether the existing object is equivalent to `content`. - For `stats.json`, `computed_at` changes on every run by design, so a plain - byte comparison would never match and the churn fix would do nothing. The - comparison therefore ignores that one field -- and only that one. + `computed_at` changes on every run by design, so a plain byte comparison + would never match and the churn fix would do nothing. The comparison + therefore ignores that field -- and only that field -- in both artifacts + that carry it: + + - `stats.json`, as a top-level key; + - `README.md`, inside its YAML front matter, with the prose compared + literally. Adding the front matter without this would have silently + undone M2 for every README in the mirror. """ try: existing_lines = dataset.repository.readlines(dataset, name) @@ -2457,17 +2467,22 @@ def _stats_computed_at_only_diff(dataset, name: str, content: str) -> bool: if existing == content: return True - if not name.endswith(".json"): - return False - try: - old = json.loads(existing) - new = json.loads(content) - except (ValueError, TypeError): - return False - if not isinstance(old, dict) or not isinstance(new, dict): + + if name.endswith(".json"): + try: + old, new = json.loads(existing), json.loads(content) + except (ValueError, TypeError): + return False + if not isinstance(old, dict) or not isinstance(new, dict): + return False + return _without_volatile(old) == _without_volatile(new) + + old_head, old_body = split_front_matter(existing) + new_head, new_body = split_front_matter(content) + if old_head is None or new_head is None: + # No front matter to reason about; the byte comparison above stands. return False - return {k: v for k, v in old.items() if k != "computed_at"} == \ - {k: v for k, v in new.items() if k != "computed_at"} + return old_body == new_body and _without_volatile(old_head) == _without_volatile(new_head) def _declared_from_metadata(dataset) -> Dict[str, Any]: @@ -2738,6 +2753,81 @@ def _build_group_summary_table(group_by: str, grouped: Dict[str, Dict]): return table +#: Fields that change on every run by design. A comparison that did not ignore +#: them would rewrite every artifact every time, which is the churn M2 removed. +_VOLATILE_ARTIFACT_KEYS = ("computed_at",) + + +def _readme_front_matter(dataset, stats: Dict) -> List[str]: + """The machine-readable head of README.md, as YAML front matter lines. + + Carries the same fields as an index row, so a consumer reads one shape + whether it comes from the README, `stats.json` or (later) `_index.parquet`. + """ + import yaml + + md = dataset.metadata + document = { + "id": md.get("internalID") or md.get("id"), + "title": md.get("title"), + "collectionName": md.get("collectionName"), + "dataCenter": getattr(dataset, "dataCenter", None) or md.get("dataCenter"), + "access": getattr(dataset, "access", None) or md.get("access"), + "startDate": str(md.get("startDate"))[:10] if md.get("startDate") else None, + "endDate": str(md.get("endDate"))[:10] if md.get("endDate") else None, + # declared is the upstream claim; observed is what is in the store. Both, + # never one standing in for the other. + "declared": stats.get("declared"), + "observed": stats.get("observed"), + "materialised": stats.get("materialised"), + "parquet_files_found": stats.get("parquet_files_found"), + "computed_at": stats.get("computed_at"), + "owilix_version": stats.get("owilix_version"), + } + body = yaml.safe_dump( + _yaml_safe(document), sort_keys=True, default_flow_style=False, allow_unicode=True + ) + return ["---", body.rstrip("\n"), "---", ""] + + +def _yaml_safe(value): + """Coerce to something ``yaml.safe_dump`` can represent. + + Dataset metadata is not a closed schema -- a field can hold a type PyYAML + refuses, and a README write is the wrong place to discover that. Unknown + types become their string form rather than raising, so an odd value costs a + less useful field instead of the whole artifact. + """ + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, dict): + return {str(k): _yaml_safe(v) for k, v in value.items()} + if isinstance(value, (list, tuple, set)): + return [_yaml_safe(v) for v in value] + return str(value) + + +def split_front_matter(text: str): + """Split ``text`` into (front_matter_dict_or_None, body). + + Returns ``(None, text)`` when there is no front matter, or when it does not + parse -- a malformed head is not a reason to lose the document. + """ + if not text.startswith("---\n"): + return None, text + end = text.find("\n---\n", 3) + if end == -1: + return None, text + head, body = text[4:end + 1], text[end + 5:] + try: + import yaml + + parsed = yaml.safe_load(head) + except Exception: + return None, text + return (parsed, body) if isinstance(parsed, dict) else (None, text) + + def _generate_readme_markdown(dataset, stats: Dict) -> str: """ Generate README.md content from dataset metadata and statistics. @@ -2776,7 +2866,12 @@ def _generate_readme_markdown(dataset, stats: Dict) -> str: description = str(descriptions) # Build README content - lines = [] + # + # YAML front matter first, so a program does not have to parse prose. The + # human section below is unchanged -- the front matter is additive, and a + # reader that does not know about it sees a fenced block at the top rather + # than mangled text. + lines = _readme_front_matter(dataset, stats) lines.append(f"# OpenWebIndex: {title}") lines.append("") lines.append(f"**Dataset ID:** `{ds_id}`") diff --git a/pyproject.toml b/pyproject.toml index 5dfae2d..cf47ead 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "owilix" -version = "5.10.1" +version = "5.10.2" description = "OWILIX - the Command Line Interface for slicing and consuming the Open Web Index. " readme = "Readme.md" license = { text = "MIT" } diff --git a/tests/owilix/core/tasks/test_summarize_integrity.py b/tests/owilix/core/tasks/test_summarize_integrity.py index 902b05f..21fee94 100644 --- a/tests/owilix/core/tasks/test_summarize_integrity.py +++ b/tests/owilix/core/tasks/test_summarize_integrity.py @@ -34,6 +34,10 @@ def _dataset(declared=None, files=None): dataset = MagicMock() dataset.metadata.get.side_effect = lambda k, d=None: metadata.get(k, d) dataset.files_details.return_value = files if files is not None else [] + # Real values: a MagicMock attribute is not YAML-representable, and the + # front matter reads these directly off the dataset. + dataset.dataCenter = "owi-up" + dataset.access = "public" return dataset @@ -185,3 +189,132 @@ class TestBackwardCompatibility: "declared", "observed", "materialised"]) def test_new_keys_are_additive(self, key): assert key in _generate_stats_json(_dataset(files=[_file(1)]), hosts_topk=0) + + +class TestReadmeFrontMatter: + """M3 -- a machine-readable head, without touching the prose.""" + + def _readme(self, dataset=None, stats=None): + from owilix.core.tasks.remote import _generate_readme_markdown + + dataset = dataset or _dataset(files=[_file(100)]) + stats = stats if stats is not None else _generate_stats_json(dataset, hosts_topk=0) + return _generate_readme_markdown(dataset, stats) + + def test_it_starts_with_parseable_front_matter(self): + import yaml + + from owilix.core.tasks.remote import split_front_matter + + text = self._readme() + + assert text.startswith("---\n") + head, body = split_front_matter(text) + assert isinstance(head, dict) + assert yaml.safe_dump(head) # round-trips + + def test_it_carries_declared_observed_and_provenance(self): + from owilix.core.tasks.remote import split_front_matter + + head, _ = split_front_matter(self._readme(_dataset(files=[]))) + + assert head["declared"]["files"] == 567 + assert head["observed"] == {"files": 0, "bytes": 0} + assert head["materialised"] is False + assert head["computed_at"] and head["owilix_version"] + + def test_the_prose_still_follows(self): + from owilix.core.tasks.remote import split_front_matter + + _, body = split_front_matter(self._readme()) + + assert body.lstrip().startswith("# OpenWebIndex:") + assert "## Statistics" in body + + def test_the_human_section_is_unchanged_when_only_the_timestamp_moves(self): + """The prose is the part a person reads; it must not churn.""" + from owilix.core.tasks.remote import split_front_matter + + dataset = _dataset(files=[_file(100)]) + first = _generate_stats_json(dataset, hosts_topk=0) + second = dict(first, computed_at="2099-01-01T00:00:00+00:00") + + _, body_a = split_front_matter(self._readme(dataset, first)) + _, body_b = split_front_matter(self._readme(dataset, second)) + + assert body_a == body_b + + +class TestSplitFrontMatter: + def test_absent_front_matter_returns_the_whole_text(self): + from owilix.core.tasks.remote import split_front_matter + + assert split_front_matter("# Title\n\nbody") == (None, "# Title\n\nbody") + + def test_malformed_front_matter_does_not_lose_the_document(self): + from owilix.core.tasks.remote import split_front_matter + + text = "---\n: : not yaml : :\n---\nbody" + head, body = split_front_matter(text) + + assert head is None + assert body == text, "a bad head must not cost the body" + + def test_an_unterminated_head_is_not_front_matter(self): + from owilix.core.tasks.remote import split_front_matter + + assert split_front_matter("---\nkey: value\nno terminator")[0] is None + + +class TestFrontMatterDoesNotUndoM2: + """The interaction that would otherwise silently restore the churn. + + `computed_at` moved into the README as well as `stats.json`. A byte + comparison would then never match, and every README in the mirror would be + rewritten on every run -- reinstating exactly the write churn M2 removed, + while looking like a feature addition. + """ + + def _dataset_holding(self, content): + repo = MagicMock() + repo.readlines.return_value = [content] + return SimpleNamespace(repository=repo) + + def _readme_with(self, computed_at, files=2): + import yaml + + head = yaml.safe_dump( + {"id": "x", "observed": {"files": files}, "computed_at": computed_at}, + sort_keys=True, + ) + return f"---\n{head.rstrip()}\n---\n\n# OpenWebIndex: x\n\nbody\n" + + def test_a_readme_differing_only_by_computed_at_is_not_rewritten(self): + old = self._readme_with("2026-08-01T00:00:00+00:00") + new = self._readme_with("2026-08-07T00:00:00+00:00") + dataset = self._dataset_holding(old) + + assert _write_if_changed(dataset, "README.md", new) is False + dataset.repository.writelines.assert_not_called() + + def test_a_readme_whose_numbers_moved_is_rewritten(self): + old = self._readme_with("2026-08-01T00:00:00+00:00", files=0) + new = self._readme_with("2026-08-07T00:00:00+00:00", files=2) + dataset = self._dataset_holding(old) + + assert _write_if_changed(dataset, "README.md", new) is True + + def test_a_readme_whose_prose_moved_is_rewritten(self): + old = self._readme_with("2026-08-01T00:00:00+00:00") + new = old.replace("body", "different body").replace( + "2026-08-01T00:00:00+00:00", "2026-08-07T00:00:00+00:00" + ) + dataset = self._dataset_holding(old) + + assert _write_if_changed(dataset, "README.md", new) is True + + def test_a_readme_without_front_matter_is_still_compared_bytewise(self): + """An older mirror's README has no head; it must not be treated as equal.""" + dataset = self._dataset_holding("# Old README\n") + + assert _write_if_changed(dataset, "README.md", "# New README\n") is True diff --git a/uv.lock b/uv.lock index b6b5e9d..4da0965 100644 --- a/uv.lock +++ b/uv.lock @@ -1102,7 +1102,7 @@ wheels = [ [[package]] name = "owilix" -version = "5.10.1" +version = "5.10.2" source = { editable = "." } dependencies = [ { name = "ciff-toolkit" },