Something went wrong. Try again.
Hardware-in-the-loop development platform for the Commodore 64.
Something went wrong. Try again.
30 kB · 640 lines
Python
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641#!/usr/bin/env python3"""Check that the first-party documentation a release ships has no broken links.
Every Ultimate Tool Markdown document that travels inside a release archive linksto sibling documents by relative path, sometimes to a heading anchor. A link to adocument that is not in the archive, or to a heading that moved, is invisibleuntil a reader follows it. This checker resolves every relative link and everyanchor in the *first-party* shipped documentation against the members the archiveactually contains.
Scope is fixed and small. The archives also carry third-party correspondingsource and notices under ``vendor/``, ``runtime-sources/`` and``runtime-notices/``. Those upstream Markdown files are written for their ownrepositories, not for this archive, so they contain Rust intra-doc links andprose constructs that are not file targets — ``[ZeroSlice](ZeroSlice)``,``[the `crate`](crate)``, ``[&self](&self)``, crate example paths such as``crates/examples/src/bin/dump.rs``, and images. Validating them would reportupstream documentation as if it were Ultimate Tool documentation. Only thosethree fixed roots are excluded, always, with no manifest-controlled orrun-time-configurable exclusion: a first-party guide can never be dropped fromvalidation by a document that ships beside it. Their member names are stillindexed, so a first-party document that links into them resolves.
What this checker does not do: it does not validate third-party Markdown, it doesnot verify source completeness or licensing (separate gates), and it does notprove that a document's wording is correct or reviewed. It is a bounded linkchecker, not an authenticity or provenance check: it reads the mapping an archiverecords for itself and holds it to the recorded shape, and it makes no claim thatany archive is the one that was intended to be built. It needs no compiler, nonetwork, and no hardware.
Modes:
- **manifest mode** (the default) derives the archive membership from the packager's own selection functions in ``tools/package_release.py``, so the check follows exactly what ``--prepare-source`` and ``--pack`` would write.- **archive mode** (``--source-archive`` and/or ``--binary-archive``) reads real member names and the first-party Markdown out of an already-built ``.tar.gz`` through the packager's bounded tar reader."""
from __future__ import annotations
import argparseimport hashlibimport jsonimport posixpathimport reimport sysfrom pathlib import Path
TOOLS = Path(__file__).resolve().parentif str(TOOLS) not in sys.path: sys.path.insert(0, str(TOOLS))
import package_release as packaging # noqa: E402
# The fixed third-party trees, excluded from link validation always. See the# module docstring: these are upstream corresponding-source and notice trees,# not Ultimate Tool documentation.FOREIGN_SOURCE_PREFIXES = ("runtime-notices/", "runtime-sources/", "vendor/")
# The members an archive records its own sanitized-copy mapping in, in preference# order: the resolved record the packager wrote into the archive, then the static# manifest the archive also carries. Archive mode reads only from these, so the# caller's root never changes what archive mode resolves.BOUND_MAPPING_SOURCES = ( "SOURCE-PROVENANCE.json", "source-manifest.json", "tools/release_source_manifest.json",)
# The two roots a bound mapping may name: the reviewed copies are stored under# the first and shipped at the second. The mapping is validated against these,# so a record cannot point a copy at an arbitrary path.COPY_ROOT = "tools/release-docs/"DEST_ROOT = "docs/"
LIMITS_NOTE = ( "Only relative Markdown links and heading anchors in first-party documents are " "checked. The fixed third-party trees {roots} are excluded, where upstream links " "point at their own repositories. Source completeness and licensing are separate " "gates and are not decided here.")
# A Markdown link: [text](target) or [text](target "title"), with an optional# angle-bracketed target. Emphasis and code spans inside the text are ignored.LINK = re.compile(r"\[(?:[^\[\]]|\[[^\]]*\])*\]\(\s*<?([^)\s>]+)>?(?:\s+\"[^\"]*\")?\s*\)")FENCE = re.compile(r"^\s*(```|~~~)")HEADING = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$")# A link target with a URL scheme (http:, https:, mailto:, ...) is external.SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:")
def is_first_party(path: str) -> bool: """True when a member path is first-party documentation, not foreign source.""" return not any(path.startswith(prefix) for prefix in FOREIGN_SOURCE_PREFIXES)
def heading_id(text: str) -> str: """The GitHub-style anchor id for one heading's text.""" lowered = text.strip().lower() kept = "".join(character for character in lowered if character.isalnum() or character in " -_") return kept.replace(" ", "-")
def anchors(markdown: str) -> set[str]: """Every heading anchor id in a document, ignoring fenced code blocks.""" found: set[str] = set() fenced = False for line in markdown.splitlines(): if FENCE.match(line): fenced = not fenced continue if fenced: continue match = HEADING.match(line) if match: found.add(heading_id(match.group(2))) return found
def relative_links(markdown: str) -> list[tuple[str, str]]: """The ``(path, anchor)`` of every relative link in a document.""" found: list[tuple[str, str]] = [] for match in LINK.finditer(markdown): target = match.group(1) if SCHEME.match(target) or target.startswith(("#", "/")): continue path, _, anchor = target.partition("#") if not path: continue found.append((path, anchor)) return found
def check_documents( label: str, names: set[str], documents: dict[str, str], placements: dict[str, str] | None = None) -> list[str]: """Resolve every relative link and anchor in ``documents`` against ``names``.
``names`` are archive-relative member paths without the archive's top-level prefix directory, and hold *every* member, so a first-party link into a third-party tree resolves. ``documents`` holds the first-party Markdown only, so third-party links are never validated.
``placements`` maps a stored path to the location it renders from. A reviewed sanitized copy travels under ``tools/release-docs/`` as source but ships in the ``docs/`` directory, so its links are written for the placement, not for the source path. """ placements = placements or {} problems: list[str] = [] for path in sorted(documents): base = posixpath.dirname(placements.get(path, path)) for target, anchor in relative_links(documents[path]): resolved = posixpath.normpath(posixpath.join(base, target)) if resolved.startswith(".."): problems.append(f"[{label}] {path}: link '{target}' escapes the archive root") continue if resolved not in names: problems.append( f"[{label}] {path}: link '{target}' resolves to '{resolved}', " "which is not an archive member" ) continue if not anchor: continue if resolved not in documents: problems.append( f"[{label}] {path}: link '{target}' targets anchor '{anchor}' on a " "non-Markdown member" ) continue if anchor not in anchors(documents[resolved]): problems.append( f"[{label}] {path}: link '{target}' targets heading '{anchor}', " f"which does not exist in '{resolved}'" ) return problems
def copy_placements(manifest: dict) -> dict[str, str]: """Map each sanitized copy's source path to the archive path it renders at.""" section = (manifest.get("package_layout") or {}).get("distribution_docs") or {} placements: dict[str, str] = {} for entry in section.get("entries", []): copy = entry.get("copy") archive_path = entry.get("archive_path") if copy and archive_path and copy != archive_path: placements[copy] = archive_path return placements
def render_generated_docs(manifest: dict) -> dict[str, str]: """Render the three generated distribution documents from the manifest.
The packager writes these with its own ``render_*`` functions, so the same functions are called here with a minimal context; only the relative targets they contain are of interest. """ version = manifest["release"]["reserved_preview_version"] names = packaging.versioned_names(version, None) context = { "release": manifest["release"], "version": version, "development": False, "development_label": None, "names": names, "archive": {"name": names["archive"], "sha256": None}, "source_archive": {"name": names["source_archive"], "sha256": "0" * 64, "size": None}, "source": {"revision": None, "tag": None, "dirty": False, "in_git_worktree": False, "tree_state": "unrecorded"}, "source_manifest": {"file_count": 0, "vendor": {}, "runtime_bundle": {"present": False}}, "source_completeness": {"status": "unrecorded", "pending_inventory": []}, "inventory": manifest["inventory"], "privacy": {"findings": []}, "external_prerequisites": manifest["external_prerequisites"], } return { "README.md": packaging.render_readme(context).decode("utf-8"), "INSTALL.md": packaging.render_install(context).decode("utf-8"), "SOURCE-AND-LICENSES.md": packaging.render_source_and_licenses(context).decode("utf-8"), }
def manifest_membership(root: Path, manifest: dict) -> tuple[set[str], dict[str, str], set[str], dict[str, str]]: """Derive both archives' members and first-party Markdown texts from the manifest.""" limits = packaging.DEFAULT_LIMITS files, _report = packaging.collect_source_files(root, manifest, limits) distribution = packaging.distribution_doc_members(root, manifest, limits) by_archive_path = {entry["archive_path"]: entry for entry in distribution}
source_names = {entry["path"] for entry in files} source_names |= {entry["archive_path"] for entry in distribution} source_names |= {"SOURCE-PROVENANCE.json", "vendor-sources.toml"} source_docs = { entry["path"]: (root / entry["path"]).read_text(encoding="utf-8") for entry in files if entry["path"].endswith(".md") and is_first_party(entry["path"]) } source_docs.update( { entry["archive_path"]: entry["bytes"].decode("utf-8") for entry in distribution if is_first_party(entry["archive_path"]) } )
layout = manifest["package_layout"] binary_names: set[str] = set(layout["license_files"]) binary_docs: dict[str, str] = {} for relative in layout["shipped_docs"]: binary_names.add(relative) entry = by_archive_path.get(relative) if entry is not None: if is_first_party(relative): binary_docs[relative] = entry["bytes"].decode("utf-8") elif relative.endswith(".md") and is_first_party(relative): binary_docs[relative] = (root / relative).read_text(encoding="utf-8") for example in layout["shipped_examples"]: for relative in packaging.select_shipped_files(root, example, manifest, limits): binary_names.add(relative) if relative.endswith(".md") and is_first_party(relative): binary_docs[relative] = (root / relative).read_text(encoding="utf-8") generated = render_generated_docs(manifest) binary_names |= set(generated) | {"release-manifest.json", "source-manifest.json"} binary_docs.update(generated)
return source_names, source_docs, binary_names, binary_docs
def _member_bytes(tar, member, limits) -> bytes: handle = tar.extractfile(member) if handle is None: return b"" payload = handle.read(limits.max_member_bytes + 1) if len(payload) > limits.max_member_bytes: raise packaging.CheckFailure(f"{member.name} is above the per-member byte bound") return payload
def _entries_from(document, source_name: str) -> list: """The distribution document entries a bound record carries.
``SOURCE-PROVENANCE.json`` and ``source-manifest.json`` carry a ``distribution_docs`` list; ``tools/release_source_manifest.json`` carries the same entries under ``package_layout.distribution_docs.entries``. A record that carries no such key maps nothing and is accepted; a record whose key has the wrong shape is refused rather than silently ignored, so a malformed record cannot be skipped in favour of another one that hides the contradiction. """ if not isinstance(document, dict): raise packaging.CheckFailure(f"the bound record {source_name} is not a JSON object") value = document.get("distribution_docs") if value is None: layout = document.get("package_layout") if isinstance(layout, dict): section = layout.get("distribution_docs") if isinstance(section, dict): value = section.get("entries") if value is None: return [] if not isinstance(value, list): raise packaging.CheckFailure(f"{source_name} records a distribution_docs that is not a list") return value
def _canonical_path(value, label: str) -> str: """Require a canonical repo-relative path with no empty, '.' or '..' component.""" if not isinstance(value, str) or not value: raise packaging.CheckFailure(f"{label} must be a non-empty string") if any(ord(character) < 32 or ord(character) == 127 for character in value): raise packaging.CheckFailure(f"{label} {value!r} must not contain control characters") if any(part in ("", ".", "..") for part in value.split("/")): raise packaging.CheckFailure( f"{label} {value!r} must be canonical, with no empty, '.' or '..' component" ) return value
def _valid_digest(value) -> bool: """A lowercase 64-character hexadecimal SHA-256.""" return isinstance(value, str) and len(value) == 64 and all( character in "0123456789abcdef" for character in value )
def bound_mapping(tar, members: dict, limits) -> tuple[dict[str, str], dict[str, tuple, ], str | None]: """Read the sanitized-copy mapping the archive itself records.
Returns the ``{copy: archive_path}`` placements, the recorded ``{archive_path: (size, sha256)}`` copies — either element may be ``None`` when the record states only the other — and the member the mapping was read from (``None`` when the archive records none). Only the archive's own bound members are consulted, so a caller's root can never change what archive mode resolves.
A bound record that is present but malformed is refused, never skipped: a checker that fell through to another record could hide a contradiction between them. Each entry must name a canonical ``tools/release-docs/*.md`` copy, a canonical ``docs/*.md`` archive path, and a matching origin; duplicate copies or destinations are refused. The resolved records state each copy's size and SHA-256 — a positive integer and a lowercase 64-character digest — and both are mandatory there. The static manifest may state only the digest, and whatever it states is verified. """ for name in BOUND_MAPPING_SOURCES: member = members.get(name) if member is None: continue try: document = json.loads(_member_bytes(tar, member, limits).decode("utf-8")) except (ValueError, UnicodeDecodeError) as error: raise packaging.CheckFailure(f"the bound record {name} is not readable JSON: {error}") from error require_digest = name in ("SOURCE-PROVENANCE.json", "source-manifest.json") placements: dict[str, str] = {} recorded: dict[str, tuple] = {} destinations: set[str] = set() for entry in _entries_from(document, name): if not isinstance(entry, dict): raise packaging.CheckFailure(f"{name} records a distribution document entry that is not an object") copy = _canonical_path(entry.get("copy"), f"{name} copy") archive_path = _canonical_path(entry.get("archive_path"), f"{name} archive_path") if not copy.startswith(COPY_ROOT) or not copy.endswith(".md"): raise packaging.CheckFailure(f"{name} names a copy outside {COPY_ROOT}: {copy!r}") if not archive_path.startswith(DEST_ROOT) or not archive_path.endswith(".md"): raise packaging.CheckFailure(f"{name} names an archive path outside {DEST_ROOT}: {archive_path!r}") if entry.get("origin", archive_path) != archive_path: raise packaging.CheckFailure( f"{name} records an origin that is not its archive path for {archive_path!r}" ) if copy in placements: raise packaging.CheckFailure(f"{name} maps the copy {copy!r} more than once") if archive_path in destinations: raise packaging.CheckFailure(f"{name} maps more than one copy to {archive_path!r}") placements[copy] = archive_path destinations.add(archive_path) size = entry.get("size") digest = entry.get("sha256") if require_digest: if not _valid_digest(digest): raise packaging.CheckFailure( f"{name} does not record a lowercase 64-character SHA-256 for {archive_path!r}" ) if isinstance(size, bool) or not isinstance(size, int) or size <= 0: raise packaging.CheckFailure( f"{name} does not record a positive integer size for {archive_path!r}" ) recorded[archive_path] = (size, digest) else: # The static manifest may state the digest without a size, or the # size without a digest; either way what it states is kept, so a # stated hash is verified rather than ignored. if size is not None and (isinstance(size, bool) or not isinstance(size, int) or size < 0): raise packaging.CheckFailure(f"{name} records a negative or non-integer size for {archive_path!r}") if digest is not None and not _valid_digest(digest): raise packaging.CheckFailure( f"{name} records a SHA-256 that is not lowercase 64-character hexadecimal " f"for {archive_path!r}" ) if size is not None or digest is not None: recorded[archive_path] = (size if size is not None else None, digest if digest is not None else None) return placements, recorded, name return {}, {}, None
def recorded_copy_problems(label: str, membership: dict) -> list[str]: """Verify every recorded copy digest against the bytes the archive carries.
Both members of a mapping are checked against the size and SHA-256 the record states: the input copy under ``tools/release-docs/`` and the alias the documentation actually links, at its ``docs/`` path. A binary archive carries the alias without the input copy, which is allowed — the members that are present are the ones verified. """ problems: list[str] = [] digests = membership["digests"] for copy, archive_path in sorted(membership["placements"].items()): expected = membership["recorded"].get(archive_path) if expected is None: continue size, digest = expected for member in (copy, archive_path): observed = digests.get(member) if observed is None: continue if size is not None and observed[0] != size: problems.append( f"[{label}] {member} has size {observed[0]}, but {membership['bound_manifest']} " f"records {size} for {archive_path}" ) continue if digest is not None and observed[1] != digest: problems.append( f"[{label}] {member} has SHA-256 {observed[1][:12]}…, but {membership['bound_manifest']} " f"records {digest[:12]}… for {archive_path}" ) return problems
def archive_membership(path: Path) -> dict: """Read one built archive: every member name, the first-party Markdown, and the mapping the archive itself records.
Third-party Markdown is never read, so upstream intra-doc links cannot be mistaken for broken first-party links; its member names are still returned so a first-party link into it resolves.
When the archive carries reviewed copies under ``tools/release-docs/``, the bound mapping is mandatory and must account for every one of them, and every destination it names must be a member. A stored copy whose placement is unknown would otherwise be validated against the wrong directory and pass. """ limits = packaging.DEFAULT_LIMITS names: set[str] = set() members: dict[str, object] = {} with packaging.open_bounded_tar(path, limits) as tar: for member in tar.getmembers(): if not member.isreg(): continue relative = member.name.split("/", 1)[1] if "/" in member.name else member.name names.add(relative) members[relative] = member placements, recorded, bound = bound_mapping(tar, members, limits) copies = sorted(name for name in names if name.startswith(COPY_ROOT) and name.endswith(".md")) if copies and not placements: raise packaging.CheckFailure( f"the archive carries copies under {COPY_ROOT} but records no mapping for them" ) unmapped = [name for name in copies if name not in placements] if unmapped: raise packaging.CheckFailure( "the bound record does not map these archived copies: " + ", ".join(unmapped[:5]) ) for archive_path in sorted(set(placements.values())): if archive_path not in names: raise packaging.CheckFailure( f"the bound record maps a copy to {archive_path}, which is not in the archive" ) documents: dict[str, str] = {} digests: dict[str, tuple[int, str]] = {} skipped = 0 for relative in sorted(names): if not relative.endswith(".md"): continue if not is_first_party(relative): skipped += 1 continue payload = _member_bytes(tar, members[relative], limits) documents[relative] = payload.decode("utf-8", errors="replace") digests[relative] = (len(payload), hashlib.sha256(payload).hexdigest()) return { "names": names, "documents": documents, "digests": digests, "skipped": skipped, "placements": placements, "recorded": recorded, "bound_manifest": bound, }
def audit(archive: str | None, root: Path, manifest: dict | None) -> dict: """Check one archive, or both archives derived from a manifest.
Returns the problems and, per archive, the explicit scope it applied: how many first-party documents were validated, how many third-party documents were excluded under the fixed roots, and which bound member supplied the mapping. In archive mode nothing outside the archive is consulted; in manifest mode the mapping comes from the release source manifest. """ reports: list[dict] = [] problems: list[str] = [] if archive is not None: membership = archive_membership(Path(archive)) label = Path(archive).name found = check_documents(label, membership["names"], membership["documents"], membership["placements"]) # The bound record's recorded copy digests are verified against the bytes # the archive carries — both the input copy under tools/release-docs/ and # the alias under docs/ — so a mapping whose copy and alias disagree with # the record is reported rather than silently trusted. found.extend(recorded_copy_problems(label, membership)) reports.append( { "archive": label, "bound_manifest": membership["bound_manifest"], "validated": len(membership["documents"]), "skipped": membership["skipped"], "problems": found, } ) problems.extend(found) elif manifest is not None: source_names, source_docs, binary_names, binary_docs = manifest_membership(root, manifest) placements = copy_placements(manifest) for label, names, documents in ( ("source archive", source_names, source_docs), ("binary archive", binary_names, binary_docs), ): found = check_documents(label, names, documents, placements) reports.append( { "archive": label, "bound_manifest": None, "validated": len(documents), "skipped": sum( 1 for name in names if name.endswith(".md") and not is_first_party(name) ), "problems": found, } ) problems.extend(found)
return { "ok": not problems, "problems": problems, "reports": reports, "limits": LIMITS_NOTE.format(roots=", ".join(FOREIGN_SOURCE_PREFIXES)), }
def check(archive: str | None, root: Path, manifest: dict | None) -> list[str]: """Return every first-party link problem in one archive, or both, from a manifest.""" return audit(archive, root, manifest)["problems"]
def summarise(report: dict) -> str: """One line per archive stating exactly what was validated and what was not.""" return "\n".join( f"{entry['archive']}: {entry['validated']} first-party Markdown document(s) validated, " f"{entry['skipped']} third-party document(s) under {', '.join(FOREIGN_SOURCE_PREFIXES)} excluded" for entry in report["reports"] )
def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--root", type=Path, default=TOOLS.parent, help="repository root (manifest mode)") parser.add_argument( "--manifest", type=Path, default=None, help=f"release source manifest (default: {packaging.SOURCE_MANIFEST_RELATIVE})", ) parser.add_argument("--source-archive", type=Path, default=None, help="checked source archive") parser.add_argument("--binary-archive", type=Path, default=None, help="checked binary archive") parser.add_argument("--json", action="store_true", help="emit a JSON report") return parser
def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) root = args.root.resolve() manifest_path = args.manifest or (root / packaging.SOURCE_MANIFEST_RELATIVE) manifest = None if manifest_path.is_file(): manifest = packaging.load_source_manifest(manifest_path) elif args.manifest is not None or (args.source_archive is None and args.binary_archive is None): print(f"error: {manifest_path} is missing", file=sys.stderr) return 2
reports: list[dict] = [] problems: list[str] = [] for archive in (args.source_archive, args.binary_archive): if archive is None: continue try: found = audit(str(archive), root, None) reports.extend(found["reports"]) problems.extend(found["problems"]) except packaging.CheckFailure as error: message = f"[{archive}] could not be read: {error}" reports.append({"archive": str(archive), "bound_manifest": None, "validated": 0, "skipped": 0, "problems": [message]}) problems.append(message) if manifest is not None: try: found = audit(None, root, manifest) reports.extend(found["reports"]) problems.extend(found["problems"]) except packaging.CheckFailure as error: message = f"manifest mode failed: {error}" reports.append({"archive": "manifest", "bound_manifest": None, "validated": 0, "skipped": 0, "problems": [message]}) problems.append(message)
note = LIMITS_NOTE.format(roots=", ".join(FOREIGN_SOURCE_PREFIXES)) if args.json: print(json.dumps({"ok": not problems, "reports": reports, "problems": problems, "limits": note}, indent=2)) else: print(summarise({"reports": reports})) if problems: print(f"{len(problems)} first-party documentation link problem(s):") for problem in problems: print(f" {problem}") else: print("documentation links resolve in every validated first-party document") print(f"limits: {note}") return 1 if problems else 0
if __name__ == "__main__": raise SystemExit(main())