From 9676a0f109382d6534a705613f317b00737716b5 Mon Sep 17 00:00:00 2001 From: mgrani Date: Fri, 7 Aug 2026 20:08:31 +0200 Subject: [PATCH] feat(summarize): give README a machine-readable head M3 of the summarize/integrity plan. YAML front matter carrying the same fields as an index row -- id, title, collectionName, dataCenter, access, dates, declared, observed, materialised, parquet_files_found, computed_at, owilix_version -- so a consumer reads one shape whether it comes from the README, stats.json, or the collection index later. The prose below is unchanged. The part worth recording is the interaction, because it would have shipped looking like a feature. Putting computed_at in the README means the M2 unchanged-write comparison can never match, so every README in the mirror would be rewritten on every run -- restoring exactly the write churn 5.10.1 removed, and with it the thing that invalidates a dircache ETag and, before 5.9.0, could empty a listing. The comparison now strips volatile keys from front matter as well as from stats.json and compares the prose literally. Found by re-running the fixture rather than by reading the diff. Two smaller pieces of hardening, both from the same instinct that a describing artifact must not break the thing it describes: a metadata value PyYAML cannot represent becomes its string form instead of raising, and a malformed front matter block is read as "no front matter" so a bad head never costs the body. Verified on the fixture: front matter round-trips through yaml.safe_load, a second --force pass performs zero writes with the timestamp present, and the human section is byte-identical when only the timestamp moves. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017phbSd6D8u4iEQsCAPEw6s --- CHANGELOG.md | 19 +++ Readme.md | 2 +- docs/changes.md | 18 +++ owilix/_version.py | 4 +- owilix/core/tasks/remote.py | 123 ++++++++++++++-- pyproject.toml | 2 +- .../core/tasks/test_summarize_integrity.py | 133 ++++++++++++++++++ uv.lock | 2 +- 8 files changed, 284 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ee4734..febf489 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,25 @@ ## Unreleased +## v5.10.2 (2026-08-07) + +M3 of the summarize/integrity plan: `README.md` gains a machine-readable head. Additive — the prose below it is unchanged. + +### Features + +- **summarize**: `README.md` now opens with YAML front matter carrying the same fields as an index row — `id`, `title`, `collectionName`, `dataCenter`, `access`, `startDate`, `endDate`, `declared`, `observed`, `materialised`, `parquet_files_found`, `computed_at`, `owilix_version`. A program reads one shape whether it comes from the README, `stats.json`, or (later) the collection index, instead of parsing prose. +- **summarize**: A metadata value PyYAML cannot represent is rendered as its string form rather than raising. Dataset metadata is not a closed schema, and a README write is the wrong place to discover that. + +### Bug Fixes + +- **summarize**: The unchanged-write check understands front matter. `computed_at` moved into the README as well as `stats.json`, so a byte comparison would never match and **every README in the mirror would have been rewritten on every run** — silently restoring the write churn v5.10.1 removed, while looking like a feature. The comparison now ignores `computed_at` inside the front matter and compares the prose literally. A README with no front matter (an older mirror) is still compared bytewise. + +### Notes + +- Verified on the fixture: a second `--force` pass performs zero writes with the timestamp present, the front matter round-trips through `yaml.safe_load`, and the human section is byte-identical when only the timestamp moves. +- A malformed front-matter block is treated as "no front matter" rather than an error — a bad head must not cost the body. + + ## v5.10.1 (2026-08-07) M0–M2 of the summarize/integrity plan. All additive; `stats.json` keeps its existing shape and every current reader is unaffected. diff --git a/Readme.md b/Readme.md index 4f0b091..7a1a3ed 100644 --- a/Readme.md +++ b/Readme.md @@ -1,6 +1,6 @@ # OWILIX - Open Web Index CLI -[![Version](https://img.shields.io/badge/version-5.10.1-blue.svg)](https://openwebsearcheu-public.pages.it4i.eu/owi-cli/) +[![Version](https://img.shields.io/badge/version-5.10.2-blue.svg)](https://openwebsearcheu-public.pages.it4i.eu/owi-cli/) [![Python](https://img.shields.io/badge/python-3.11+-green.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-Apache_2.0-orange.svg)](http://www.apache.org/licenses/LICENSE-2.0) diff --git a/docs/changes.md b/docs/changes.md index 97e74ce..f214c27 100644 --- a/docs/changes.md +++ b/docs/changes.md @@ -24,6 +24,24 @@ Brief description of what was accomplished. ## Unreleased +### main @ v5.10.2 - 2026-08-07 + +#### README gets a machine-readable head, without restoring the churn + +**Summary**: M3 of the summarize/integrity plan. YAML front matter on `README.md`, prose untouched. + +**Changes**: +- `_readme_front_matter` emits the index-row fields as YAML; `split_front_matter` parses them back and is tolerant of a malformed or absent head. +- **The interaction that mattered**: `computed_at` now appears in the README too, so the M2 unchanged-write check would never have matched and every README would have been rewritten on every run — reinstating the exact churn that could invalidate a dircache ETag and, before v5.9.0, empty a listing. `_stats_computed_at_only_diff` now strips volatile keys from front matter as well as from `stats.json`, and compares the prose literally. Caught by running the fixture, not by reading the diff. +- `_yaml_safe` coerces unrepresentable metadata values rather than raising. + +**Verified**: front matter round-trips; a second `--force` pass writes nothing; the human section is byte-identical when only the timestamp moves. + +**Breaking Changes**: None. Additive head; existing readers of the prose are unaffected. + +**Still open from the plan**: M4 (`_index.parquet` + `remote reindex` — where the risk concentrates, and its own release), M5 (HTTP exposure), M6 (verify tiers, checksums, cost controls). + + ### main @ v5.10.1 - 2026-08-07 #### Observed vs declared, provenance, and an end to write churn diff --git a/owilix/_version.py b/owilix/_version.py index 72937b3..71b55a7 100644 --- a/owilix/_version.py +++ b/owilix/_version.py @@ -1,3 +1,3 @@ # Version is set here and imported elsewhere -__version__ = "5.10.1" -__version_tuple__ = (5, 10, 1) +__version__ = "5.10.2" +__version_tuple__ = (5, 10, 2) diff --git a/owilix/core/tasks/remote.py b/owilix/core/tasks/remote.py index c4dcbb0..d38318a 100644 --- a/owilix/core/tasks/remote.py +++ b/owilix/core/tasks/remote.py @@ -2440,12 +2440,22 @@ def _write_if_changed(dataset, name: str, content: str, console: Optional[Consol return True +def _without_volatile(document: Dict) -> Dict: + return {k: v for k, v in document.items() if k not in _VOLATILE_ARTIFACT_KEYS} + + def _stats_computed_at_only_diff(dataset, name: str, content: str) -> bool: """Whether the existing object is equivalent to `content`. - For `stats.json`, `computed_at` changes on every run by design, so a plain - byte comparison would never match and the churn fix would do nothing. The - comparison therefore ignores that one field -- and only that one. + `computed_at` changes on every run by design, so a plain byte comparison + would never match and the churn fix would do nothing. The comparison + therefore ignores that field -- and only that field -- in both artifacts + that carry it: + + - `stats.json`, as a top-level key; + - `README.md`, inside its YAML front matter, with the prose compared + literally. Adding the front matter without this would have silently + undone M2 for every README in the mirror. """ try: existing_lines = dataset.repository.readlines(dataset, name) @@ -2457,17 +2467,22 @@ def _stats_computed_at_only_diff(dataset, name: str, content: str) -> bool: if existing == content: return True - if not name.endswith(".json"): - return False - try: - old = json.loads(existing) - new = json.loads(content) - except (ValueError, TypeError): - return False - if not isinstance(old, dict) or not isinstance(new, dict): + + if name.endswith(".json"): + try: + old, new = json.loads(existing), json.loads(content) + except (ValueError, TypeError): + return False + if not isinstance(old, dict) or not isinstance(new, dict): + return False + return _without_volatile(old) == _without_volatile(new) + + old_head, old_body = split_front_matter(existing) + new_head, new_body = split_front_matter(content) + if old_head is None or new_head is None: + # No front matter to reason about; the byte comparison above stands. return False - return {k: v for k, v in old.items() if k != "computed_at"} == \ - {k: v for k, v in new.items() if k != "computed_at"} + return old_body == new_body and _without_volatile(old_head) == _without_volatile(new_head) def _declared_from_metadata(dataset) -> Dict[str, Any]: @@ -2738,6 +2753,81 @@ def _build_group_summary_table(group_by: str, grouped: Dict[str, Dict]): return table +#: Fields that change on every run by design. A comparison that did not ignore +#: them would rewrite every artifact every time, which is the churn M2 removed. +_VOLATILE_ARTIFACT_KEYS = ("computed_at",) + + +def _readme_front_matter(dataset, stats: Dict) -> List[str]: + """The machine-readable head of README.md, as YAML front matter lines. + + Carries the same fields as an index row, so a consumer reads one shape + whether it comes from the README, `stats.json` or (later) `_index.parquet`. + """ + import yaml + + md = dataset.metadata + document = { + "id": md.get("internalID") or md.get("id"), + "title": md.get("title"), + "collectionName": md.get("collectionName"), + "dataCenter": getattr(dataset, "dataCenter", None) or md.get("dataCenter"), + "access": getattr(dataset, "access", None) or md.get("access"), + "startDate": str(md.get("startDate"))[:10] if md.get("startDate") else None, + "endDate": str(md.get("endDate"))[:10] if md.get("endDate") else None, + # declared is the upstream claim; observed is what is in the store. Both, + # never one standing in for the other. + "declared": stats.get("declared"), + "observed": stats.get("observed"), + "materialised": stats.get("materialised"), + "parquet_files_found": stats.get("parquet_files_found"), + "computed_at": stats.get("computed_at"), + "owilix_version": stats.get("owilix_version"), + } + body = yaml.safe_dump( + _yaml_safe(document), sort_keys=True, default_flow_style=False, allow_unicode=True + ) + return ["---", body.rstrip("\n"), "---", ""] + + +def _yaml_safe(value): + """Coerce to something ``yaml.safe_dump`` can represent. + + Dataset metadata is not a closed schema -- a field can hold a type PyYAML + refuses, and a README write is the wrong place to discover that. Unknown + types become their string form rather than raising, so an odd value costs a + less useful field instead of the whole artifact. + """ + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, dict): + return {str(k): _yaml_safe(v) for k, v in value.items()} + if isinstance(value, (list, tuple, set)): + return [_yaml_safe(v) for v in value] + return str(value) + + +def split_front_matter(text: str): + """Split ``text`` into (front_matter_dict_or_None, body). + + Returns ``(None, text)`` when there is no front matter, or when it does not + parse -- a malformed head is not a reason to lose the document. + """ + if not text.startswith("---\n"): + return None, text + end = text.find("\n---\n", 3) + if end == -1: + return None, text + head, body = text[4:end + 1], text[end + 5:] + try: + import yaml + + parsed = yaml.safe_load(head) + except Exception: + return None, text + return (parsed, body) if isinstance(parsed, dict) else (None, text) + + def _generate_readme_markdown(dataset, stats: Dict) -> str: """ Generate README.md content from dataset metadata and statistics. @@ -2776,7 +2866,12 @@ def _generate_readme_markdown(dataset, stats: Dict) -> str: description = str(descriptions) # Build README content - lines = [] + # + # YAML front matter first, so a program does not have to parse prose. The + # human section below is unchanged -- the front matter is additive, and a + # reader that does not know about it sees a fenced block at the top rather + # than mangled text. + lines = _readme_front_matter(dataset, stats) lines.append(f"# OpenWebIndex: {title}") lines.append("") lines.append(f"**Dataset ID:** `{ds_id}`") diff --git a/pyproject.toml b/pyproject.toml index 5dfae2d..cf47ead 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "owilix" -version = "5.10.1" +version = "5.10.2" description = "OWILIX - the Command Line Interface for slicing and consuming the Open Web Index. " readme = "Readme.md" license = { text = "MIT" } diff --git a/tests/owilix/core/tasks/test_summarize_integrity.py b/tests/owilix/core/tasks/test_summarize_integrity.py index 902b05f..21fee94 100644 --- a/tests/owilix/core/tasks/test_summarize_integrity.py +++ b/tests/owilix/core/tasks/test_summarize_integrity.py @@ -34,6 +34,10 @@ def _dataset(declared=None, files=None): dataset = MagicMock() dataset.metadata.get.side_effect = lambda k, d=None: metadata.get(k, d) dataset.files_details.return_value = files if files is not None else [] + # Real values: a MagicMock attribute is not YAML-representable, and the + # front matter reads these directly off the dataset. + dataset.dataCenter = "owi-up" + dataset.access = "public" return dataset @@ -185,3 +189,132 @@ class TestBackwardCompatibility: "declared", "observed", "materialised"]) def test_new_keys_are_additive(self, key): assert key in _generate_stats_json(_dataset(files=[_file(1)]), hosts_topk=0) + + +class TestReadmeFrontMatter: + """M3 -- a machine-readable head, without touching the prose.""" + + def _readme(self, dataset=None, stats=None): + from owilix.core.tasks.remote import _generate_readme_markdown + + dataset = dataset or _dataset(files=[_file(100)]) + stats = stats if stats is not None else _generate_stats_json(dataset, hosts_topk=0) + return _generate_readme_markdown(dataset, stats) + + def test_it_starts_with_parseable_front_matter(self): + import yaml + + from owilix.core.tasks.remote import split_front_matter + + text = self._readme() + + assert text.startswith("---\n") + head, body = split_front_matter(text) + assert isinstance(head, dict) + assert yaml.safe_dump(head) # round-trips + + def test_it_carries_declared_observed_and_provenance(self): + from owilix.core.tasks.remote import split_front_matter + + head, _ = split_front_matter(self._readme(_dataset(files=[]))) + + assert head["declared"]["files"] == 567 + assert head["observed"] == {"files": 0, "bytes": 0} + assert head["materialised"] is False + assert head["computed_at"] and head["owilix_version"] + + def test_the_prose_still_follows(self): + from owilix.core.tasks.remote import split_front_matter + + _, body = split_front_matter(self._readme()) + + assert body.lstrip().startswith("# OpenWebIndex:") + assert "## Statistics" in body + + def test_the_human_section_is_unchanged_when_only_the_timestamp_moves(self): + """The prose is the part a person reads; it must not churn.""" + from owilix.core.tasks.remote import split_front_matter + + dataset = _dataset(files=[_file(100)]) + first = _generate_stats_json(dataset, hosts_topk=0) + second = dict(first, computed_at="2099-01-01T00:00:00+00:00") + + _, body_a = split_front_matter(self._readme(dataset, first)) + _, body_b = split_front_matter(self._readme(dataset, second)) + + assert body_a == body_b + + +class TestSplitFrontMatter: + def test_absent_front_matter_returns_the_whole_text(self): + from owilix.core.tasks.remote import split_front_matter + + assert split_front_matter("# Title\n\nbody") == (None, "# Title\n\nbody") + + def test_malformed_front_matter_does_not_lose_the_document(self): + from owilix.core.tasks.remote import split_front_matter + + text = "---\n: : not yaml : :\n---\nbody" + head, body = split_front_matter(text) + + assert head is None + assert body == text, "a bad head must not cost the body" + + def test_an_unterminated_head_is_not_front_matter(self): + from owilix.core.tasks.remote import split_front_matter + + assert split_front_matter("---\nkey: value\nno terminator")[0] is None + + +class TestFrontMatterDoesNotUndoM2: + """The interaction that would otherwise silently restore the churn. + + `computed_at` moved into the README as well as `stats.json`. A byte + comparison would then never match, and every README in the mirror would be + rewritten on every run -- reinstating exactly the write churn M2 removed, + while looking like a feature addition. + """ + + def _dataset_holding(self, content): + repo = MagicMock() + repo.readlines.return_value = [content] + return SimpleNamespace(repository=repo) + + def _readme_with(self, computed_at, files=2): + import yaml + + head = yaml.safe_dump( + {"id": "x", "observed": {"files": files}, "computed_at": computed_at}, + sort_keys=True, + ) + return f"---\n{head.rstrip()}\n---\n\n# OpenWebIndex: x\n\nbody\n" + + def test_a_readme_differing_only_by_computed_at_is_not_rewritten(self): + old = self._readme_with("2026-08-01T00:00:00+00:00") + new = self._readme_with("2026-08-07T00:00:00+00:00") + dataset = self._dataset_holding(old) + + assert _write_if_changed(dataset, "README.md", new) is False + dataset.repository.writelines.assert_not_called() + + def test_a_readme_whose_numbers_moved_is_rewritten(self): + old = self._readme_with("2026-08-01T00:00:00+00:00", files=0) + new = self._readme_with("2026-08-07T00:00:00+00:00", files=2) + dataset = self._dataset_holding(old) + + assert _write_if_changed(dataset, "README.md", new) is True + + def test_a_readme_whose_prose_moved_is_rewritten(self): + old = self._readme_with("2026-08-01T00:00:00+00:00") + new = old.replace("body", "different body").replace( + "2026-08-01T00:00:00+00:00", "2026-08-07T00:00:00+00:00" + ) + dataset = self._dataset_holding(old) + + assert _write_if_changed(dataset, "README.md", new) is True + + def test_a_readme_without_front_matter_is_still_compared_bytewise(self): + """An older mirror's README has no head; it must not be treated as equal.""" + dataset = self._dataset_holding("# Old README\n") + + assert _write_if_changed(dataset, "README.md", "# New README\n") is True diff --git a/uv.lock b/uv.lock index b6b5e9d..4da0965 100644 --- a/uv.lock +++ b/uv.lock @@ -1102,7 +1102,7 @@ wheels = [ [[package]] name = "owilix" -version = "5.10.1" +version = "5.10.2" source = { editable = "." } dependencies = [ { name = "ciff-toolkit" }, -- 2.51.2