From d2c7e3da336ecd41277417b9d875040a8338bc23 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Wed, 5 Aug 2026 02:26:47 -0600 Subject: [PATCH] fixture(segment): land the projection generator, and disambiguate two steps of its rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fixture claimed a generated rule while no generator existed beside it, so a reader of this repository could not regenerate or verify it. The generator now lives in scripts/, where the other contract gates already are, and it refuses to emit unless the legacy half reproduces the shipped derivation and every declared step has an observable probe. Two steps were ambiguous about which value they applied to, and the ambiguity produced a real disagreement: read one way, the rule yields iphone_device for label iPhone with source !!!, where the corpus says iphone. Stated properly now — the empty-result fallback applies to the LABEL only and never to the source, and the source is appended when the PROJECTED source is non-empty, so a source that filters away appends nothing rather than taking the fallback. Same defect class as the four vectors this sentence contradicted a day ago, in the same sentence: prose adjacent to a checked artifact drifts because nothing compares them. The step probes are what compare them now. make ci green. --- .../stream-name-projection-vectors.json | 2 +- scripts/gen_stream_projection_vectors.py | 387 ++++++++++++++++++ 2 files changed, 388 insertions(+), 1 deletion(-) create mode 100755 scripts/gen_stream_projection_vectors.py diff --git a/core/fixtures/stream-name-projection-vectors.json b/core/fixtures/stream-name-projection-vectors.json index 7c3032088..25f070fd1 100644 --- a/core/fixtures/stream-name-projection-vectors.json +++ b/core/fixtures/stream-name-projection-vectors.json @@ -3,7 +3,7 @@ "format_id": "journal-stream-name-projection", "schema_owner": "the segment-media/journal-segment boundary", "file_kind": "vectors", - "rule": "NFKD normalize, so accents separate from their base letters -> full Unicode casefold -- NOT a lowercase mapping -> drop combining marks (category Mn), so `café` becomes `cafe` -> replace runs of [^a-z0-9_-] with a single '_' -> strip leading and trailing '_-.' -> an empty result becomes 'device' -> a Windows reserved stem gains a trailing '_' -> append '_' + projected(source) when source is non-empty -> truncate to 64 bytes -- AFTER the source is appended. Collisions are resolved by the ordinal allocator, NOT by this function.", + "rule": "NFKD normalize, so accents separate from their base letters -> full Unicode casefold -- NOT a lowercase mapping -> drop combining marks (category Mn), so `café` becomes `cafe` -> replace runs of [^a-z0-9_-] with a single '_' -> strip leading and trailing '_-.' -> a LABEL that projects to empty becomes 'device' -- this fallback applies to the label only, never to the source -> a Windows reserved stem gains a trailing '_' -> append '_' + projected(source) when the PROJECTED source is non-empty -- a source that projects to empty appends nothing and does NOT take the fallback -> truncate to 64 bytes -- AFTER the source is appended. Collisions are resolved by the ordinal allocator, NOT by this function.", "rule_steps": [ "decompose", "casefold", diff --git a/scripts/gen_stream_projection_vectors.py b/scripts/gen_stream_projection_vectors.py new file mode 100755 index 000000000..851662357 --- /dev/null +++ b/scripts/gen_stream_projection_vectors.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +"""Author the stream-name projection vectors, and pin the legacy names beside them. + +Two halves, deliberately different in kind: + + legacy/ -- RECORDED from the live Python `stream_name()`. These are names + that already exist on owners' disks. A rebuild must keep reading + them, so they are captured from the implementation that wrote + them, not invented. + + project/ -- AUTHORED from the projection rule. No implementation produces + these yet, which is the point: a vector recorded from the code it + is meant to pin cannot catch the code being wrong. The rule is + stated once, here, and the vectors are derived from it. + +The generator refuses to emit unless the legacy half reproduces the live +implementation exactly. If `stream_name()` ever changes, this breaks rather +than silently re-baselining. + +Run from the solstone-journal checkout with its venv: + + .venv/bin/python +""" + +from __future__ import annotations + +import json +import re +import sys +import unicodedata +from pathlib import Path + +# --------------------------------------------------------------------------- +# The projection rule. This is the specification; the Rust implements it. +# --------------------------------------------------------------------------- + +WINDOWS_RESERVED = { + "CON", "PRN", "AUX", "NUL", + *(f"COM{i}" for i in range(1, 10)), + *(f"LPT{i}" for i in range(1, 10)), +} + +MAX_BYTES = 64 +EMPTY_FALLBACK = "device" + +# The rule is GENERATED from this list rather than hand-written beside it. +# It was hand-written once, the implementation gained a decomposition step, and +# the sentence silently went false while every vector stayed green -- prose +# adjacent to a checked artifact drifts precisely because nothing compares it. +# `refuse_unless_sound` binds each step below to an output only that step can +# produce, so a step that leaves the implementation also leaves the sentence. +PIPELINE_STEPS: list[tuple[str, str]] = [ + ("decompose", "NFKD normalize, so accents separate from their base letters"), + ("casefold", "full Unicode casefold -- NOT a lowercase mapping"), + ("strip-marks", "drop combining marks (category Mn), so `café` becomes `cafe`"), + ("filter", "replace runs of [^a-z0-9_-] with a single '_'"), + ("strip", "strip leading and trailing '_-.'"), + ("fallback", f"a LABEL that projects to empty becomes '{EMPTY_FALLBACK}' -- " + f"this fallback applies to the label only, never to the source"), + ("reserved", "a Windows reserved stem gains a trailing '_'"), + ("source", "append '_' + projected(source) when the PROJECTED source is " + "non-empty -- a source that projects to empty appends nothing " + "and does NOT take the fallback"), + ("truncate", f"truncate to {MAX_BYTES} bytes -- AFTER the source is appended"), +] + +# Each step name mapped to (input, output) that ONLY that step produces. If a +# step is dropped from the implementation, its probe stops matching and the +# generator refuses; if it is dropped from PIPELINE_STEPS, the rule sentence +# loses it and the same check fires. That is the bind the first version lacked. +STEP_PROBES: dict[str, tuple[str, str, str]] = { + "decompose": ("café", "", "cafe"), + "casefold": ("Straße", "", "strasse"), + "strip-marks": ("İstanbul", "", "istanbul"), + "filter": ("my.phone", "", "my_phone"), + "strip": (" spaced ", "", "spaced"), + "fallback": ("!!!", "", EMPTY_FALLBACK), + "reserved": ("CON", "", "con_"), + "source": ("iPhone", "watch", "iphone_watch"), + "truncate": ("A" * 200, "", "a" * MAX_BYTES), +} + + +def project(label: str, source: str = "") -> str: + """Project an owner-facing display label to a filesystem-safe stream name. + + Casefold happens BEFORE the charset filter, so two labels differing only in + case produce the same string and collide through the ordinal allocator + rather than through a comparison rule a later edit could drop. + """ + def one(part: str) -> str: + # NFKD rather than NFKC so accents separate from their base letters, and + # casefold before stripping them so `İ` -> `i` + combining dot -> `i`. + # Without this step `café` becomes `caf` and `Téléphone` becomes + # `t_l_phone`, which is not the human-friendly label the contract asks + # for. Scripts with no Latin decomposition (CJK, Cyrillic) still filter + # to `_`, which is correct -- the name is a label, not a transliteration. + part = unicodedata.normalize("NFKD", part).casefold() + part = "".join(c for c in part if unicodedata.category(c) != "Mn") + part = re.sub(r"[^a-z0-9_-]+", "_", part) + part = part.strip("_-.") + if not part: + return "" + # Windows refuses these basenames bare or with any extension. The dot is + # already gone by this point, so only the bare stem can match. + if part.upper() in WINDOWS_RESERVED: + part = f"{part}_" + return part + + name = one(label) or EMPTY_FALLBACK + projected_source = one(source) + if projected_source: + name = f"{name}_{projected_source}" + + # ASCII by construction, so bytes and chars coincide. + return name[:MAX_BYTES] + + +# --------------------------------------------------------------------------- +# The authored vectors. Each states WHY it exists; a vector whose reason is +# "coverage" is a vector nobody will maintain correctly. +# --------------------------------------------------------------------------- + +AUTHORED: list[dict] = [ + {"label": "iPhone", "source": "", "why": "the ordinary case"}, + {"label": "iPhone (2)", "source": "", + "why": "the worked example the contract names; the ordinal form the label allocator already emits"}, + {"label": "IPHONE (2)", "source": "", + "why": "🔴 case-collision pair with the row above -- the SAME output is the whole point. " + "On APFS and NTFS these two collide; on ext4 they would not. Casefolding first " + "makes them collide everywhere, so the ordinal allocator resolves it."}, + {"label": "iPhone (2)", "source": "watch", + "why": "one device, two streams -- the sub-stream discriminator, which is why `did` alone " + "cannot select a stream"}, + {"label": "my.phone", "source": "", + "why": "⛔ the dot is the legacy kind separator and `import.` is a live prefix; a projected " + "`my.phone` would read as a qualified name at seven call sites"}, + {"label": "workstation.browser", "source": "", + "why": "a legacy kind-carrying name arriving as a label -- the dot still goes"}, + {"label": " Ada's MacBook Pro ", "source": "", + "why": "apostrophe, spaces, and leading/trailing whitespace in one label"}, + {"label": "CON", "source": "", + "why": "Windows reserved basename, bare"}, + {"label": "com1", "source": "", + "why": "Windows reserved, already lowercase -- the check is case-insensitive on the stem"}, + {"label": "nul.txt", "source": "", + "why": "reserved stem with an extension; the dot becomes `_` first, so the stem is `nul_txt` " + "and is NOT reserved. Pins that the escape is not over-applied."}, + {"label": "!!!", "source": "", + "why": "everything filters out -- must not produce an empty directory name"}, + {"label": "...", "source": "", + "why": "dots only; strips to empty via a different path than the row above"}, + {"label": "-_-", "source": "", + "why": "only strip characters -- empty after strip, not after filter"}, + {"label": "Ｆｕｌｌｗｉｄｔｈ", "source": "", + "why": "NFKC folds fullwidth forms to ASCII before casefold; without normalization every " + "character would filter to `_`"}, + {"label": "Straße", "source": "", + "why": "🔴 casefold maps ß to ss where lower() does not -- the vector that catches " + "`to_lowercase()` used in place of a real casefold"}, + {"label": "İstanbul", "source": "", + "why": "dotted capital I -- casefold yields `i` plus a combining dot, which mark-stripping " + "removes. Pins the pipeline order: NFKD → casefold → strip marks → filter."}, + {"label": "café", "source": "", + "why": "a composed accent decomposes and its mark is stripped, so the owner gets `cafe` " + "rather than `caf`"}, + {"label": "Téléphone d'Amélie", "source": "", + "why": "🔴 the human-friendliness vector. Without mark-stripping this is `t_l_phone_d_am_lie` " + "-- unreadable, and the contract asks for a human-friendly label. Any owner whose " + "language uses accents hits this on their first device."}, + {"label": "Владимир", "source": "", + "why": "a script with no Latin decomposition correctly filters out entirely -- the name is a " + "label, NOT a transliteration, and nothing resolves by it"}, + {"label": "A" * 200, "source": "", + "why": "length cap; ASCII by construction so the byte cap cannot split a character"}, + {"label": "device", "source": "", + "why": "a label that is literally the empty-fallback value -- must be indistinguishable " + "from the fallback, which is fine because nothing resolves by the name"}, + {"label": "iPhone/../../etc", "source": "", + "why": "🔴 path traversal in a display label; separators and dots all filter to `_`"}, + {"label": "ẞ", "source": "", + "why": "🔴 capital sharp s. casefold expands it to `ss`; a `to_lowercase()` plus a " + "hand-rolled `ß -> ss` special case gets this WRONG, and without this vector " + "that implementation is corpus-clean"}, + {"label": "CON", "source": "watch", + "why": "🔴 a reserved stem WITH a source -- the escape and the suffix interact, and the " + "rule's order is what decides the result. No other vector exercises both"}, + {"label": "iPhone", "source": "!!!", + "why": "a source that filters to empty -- must not leave a trailing `_` or take the " + "device fallback in the source position"}, + {"label": "iPhone", "source": "WATCH.Series (9)", + "why": "the source is the one client-supplied string in the pipeline, so it gets the " + "same projection as the label -- uppercase, dot and parens all handled"}, + {"label": "com0", "source": "", + "why": "near-miss: NOT reserved (COM1-9 only), so it must NOT be escaped -- proves the " + "escape is not applied by prefix"}, + {"label": "console", "source": "", + "why": "near-miss: starts with `con` but is not reserved -- proves the check is on the " + "whole stem, not a prefix"}, +] + +# Every Windows reserved basename gets a vector. Two of twenty-two is not +# coverage: a regex built from `CON` and `COM1` alone passes a two-name corpus +# and then lets `PRN`, `AUX`, `NUL`, `COM2`-`COM9` and `LPT1`-`LPT9` through to +# a filesystem that refuses them. +AUTHORED += [ + {"label": name, "source": "", + "why": f"Windows reserved basename {name} -- the full set is pinned so an " + f"implementation cannot satisfy the corpus with a partial list"} + for name in sorted(WINDOWS_RESERVED) + if name not in {"CON", "COM1"} +] + +# Legacy names captured from the live implementation. These already exist on +# disk and the new reader must keep understanding them. +LEGACY_INPUTS: list[dict] = [ + {"kwargs": {"host": "archon"}, "why": "the plain local host"}, + {"kwargs": {"host": "ja1r.local"}, "why": "domain suffix stripped to the first label"}, + {"kwargs": {"host": "192.168.1.1"}, "why": "an IP address joins with dashes, not dots"}, + {"kwargs": {"host": "archon", "qualifier": "tmux"}, "why": "the dot qualifier form"}, + {"kwargs": {"host": "workstation", "qualifier": "browser"}, + "why": "the OTHER live dotted qualifier -- read at two talent sites, and absent from the " + "first version of this corpus, which left read-old with a hole exactly where the " + "kind-switching sites are"}, + {"kwargs": {"observer": "laptop.local"}, "why": "an observer name, domain stripped"}, + {"kwargs": {"import_source": "apple"}, "why": "the live `import.` prefix"}, + {"kwargs": {"import_source": "chatgpt"}, "why": "one of the hardcoded speaker-bootstrap set"}, +] + + +def build_legacy() -> list[dict]: + from solstone.think.streams import stream_name + + rows = [] + for entry in LEGACY_INPUTS: + rows.append({ + "input": entry["kwargs"], + "expect": stream_name(**entry["kwargs"]), + "why": entry["why"], + }) + return rows + + +def refuse_unless_sound(legacy: list[dict], authored: list[dict]) -> None: + """Refuse to emit unless each half carries what it exists to carry.""" + from solstone.think.streams import _STREAM_NAME_RE + + problems: list[str] = [] + + # The legacy half must be reproducible from the live implementation, and + # every legacy name must satisfy the live name rule. + for row in legacy: + if not _STREAM_NAME_RE.match(row["expect"]): + problems.append(f"legacy name {row['expect']!r} does not match the live name rule") + + # Independently-known literals. If these drift, the capture is not of the + # implementation this fixture claims to pin. + known = {"archon": {"host": "archon"}, "ja1r": {"host": "ja1r.local"}, + "192-168-1-1": {"host": "192.168.1.1"}, "archon.tmux": {"host": "archon", "qualifier": "tmux"}, + "import.apple": {"import_source": "apple"}} + got = {json.dumps(r["input"], sort_keys=True): r["expect"] for r in legacy} + for expected_name, kwargs in known.items(): + key = json.dumps(kwargs, sort_keys=True) + if got.get(key) != expected_name: + problems.append( + f"legacy {kwargs} reproduced {got.get(key)!r}, expected the published {expected_name!r}" + ) + + # The authored half must satisfy the projection's own invariants. + for row in authored: + name = row["expect"] + if not name: + problems.append(f"{row['label']!r} projected to an empty name") + continue + if "." in name: + problems.append(f"⛔ {row['label']!r} projected to {name!r}, which contains a dot") + if not re.fullmatch(r"[a-z0-9_-]+", name): + problems.append(f"{row['label']!r} projected to {name!r}, outside [a-z0-9_-]") + if len(name.encode()) > MAX_BYTES: + problems.append(f"{row['label']!r} projected to {len(name.encode())} bytes") + # A leading strip-character is always a defect. A TRAILING underscore is + # legal in exactly one case -- the deliberate Windows reserved-name + # escape -- so it is only a defect when removing it does not reveal a + # reserved stem. + if name[0] in "_-": + problems.append(f"{row['label']!r} projected to {name!r}, which is not left-stripped") + if name.endswith("-") or ( + name.endswith("_") and name[:-1].upper() not in WINDOWS_RESERVED + ): + problems.append(f"{row['label']!r} projected to {name!r}, which is not right-stripped") + # The name IS the whole path component the OS sees, so the reserved check + # is over the whole name. Splitting on '_' would re-flag the escape it is + # meant to accept, and would flag `nul_txt`, which is not reserved. + if name.upper() in WINDOWS_RESERVED: + problems.append(f"{row['label']!r} projected to the reserved name {name!r}") + + # The case-collision pair is the reason the vector set exists. If it stops + # colliding, the fixture has stopped testing its own headline property. + by_label = {(r["label"], r["source"]): r["expect"] for r in authored} + lower = by_label.get(("iPhone (2)", "")) + upper = by_label.get(("IPHONE (2)", "")) + if lower is None or upper is None or lower != upper: + problems.append( + f"🔴 the case-collision pair must project identically; got {lower!r} and {upper!r}" + ) + + # Every declared step must be observable in the implementation's output, and + # every step the implementation performs must be declared. This is the check + # whose absence let the rule sentence say NFKC while the code did NFKD. + declared = {name for name, _ in PIPELINE_STEPS} + probed = set(STEP_PROBES) + if declared != probed: + problems.append( + f"PIPELINE_STEPS and STEP_PROBES disagree: " + f"undeclared={probed - declared}, unprobed={declared - probed}" + ) + for step, (label, source, expected) in STEP_PROBES.items(): + actual = project(label, source) + if actual != expected: + problems.append( + f"step {step!r} is declared in the rule but its probe " + f"{label!r}+{source!r} produced {actual!r}, not {expected!r} -- " + f"the rule sentence and the implementation have diverged" + ) + + # And the casefold-vs-lowercase discriminator. + strasse = by_label.get(("Straße", "")) + if strasse != "strasse": + problems.append( + f"🔴 `Straße` must project to 'strasse' (casefold, not lower()); got {strasse!r}" + ) + + if problems: + for problem in problems: + print(f"REFUSED: {problem}", file=sys.stderr) + raise SystemExit(1) + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("usage: gen-stream-projection-vectors.py ") + + authored = [ + {"label": v["label"], "source": v["source"], + "expect": project(v["label"], v["source"]), "why": v["why"]} + for v in AUTHORED + ] + legacy = build_legacy() + refuse_unless_sound(legacy, authored) + + document = { + "x-journal-contract": { + "format_id": "journal-stream-name-projection", + "schema_owner": "the segment-media/journal-segment boundary", + "file_kind": "vectors", + "rule": " -> ".join(description for _, description in PIPELINE_STEPS) + + ". Collisions are resolved by the ordinal allocator, NOT by this function.", + "rule_steps": [name for name, _ in PIPELINE_STEPS], + "note": ( + "`project` vectors are AUTHORED from the rule above -- no implementation " + "produces them yet, so they can catch an implementation being wrong. `legacy` " + "vectors are RECORDED from the shipped name derivation; they already exist on " + "disk and must stay readable. The projected name is NEVER required to be " + "invertible, only unique." + ), + "known_limitation": ( + "The human-friendly property holds for Latin scripts and degrades to nothing " + "for scripts with no Latin decomposition: a device labelled in Cyrillic, Greek, " + "Hebrew, Arabic or CJK filters to empty and takes the fallback, so an owner " + "with three such devices sees `device`, `device_2`, `device_3`. This is safe -- " + "nothing resolves by the name and uniqueness still holds -- but it is a real " + "gap for those owners, and closing it means a per-script transliteration " + "commitment rather than a rule change." + ), + }, + "project": authored, + "legacy": legacy, + } + + output = Path(sys.argv[1]) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(document, indent=2, ensure_ascii=False) + "\n", "utf-8") + print(f"wrote {output} ({len(authored)} projection, {len(legacy)} legacy)") + + +if __name__ == "__main__": + main() -- 2.51.2