Something went wrong. Try again.
Identities for entities did.bot
agent llm did
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328#!/usr/bin/env python3"""Check that the repository paths docs/, plan/ and README.md name exist.
The prose names code in code spans, such as `crates/didbot-pds/src/estop.rs:40`.A span is checked when its first word starts with a directory at the top ofthe tree, as `git ls-files` lists it, or with the directory name of a memberof the workspace in Cargo.toml, such as `didbot-pds/src/estop.rs`, which isread from that member's directory. That word is read as a path:
- A trailing `:line` or `:line-line` is stripped, and the file has to have at least that many lines. - One `{a,b}` group is expanded, and each member is checked. - A trailing `/` names a directory. Without one, a file or a directory passes.
Any other span is skipped, and so is a code fence. A bare file name such as`estop.rs` could be in any crate, so it is skipped too, as is a word holdinga glob, a placeholder such as `<id>`, or any other character these pathsnever contain. Missing a real path is better than failing on prose.
A finding is keyed by its page and that first word, not by its line, so anedit elsewhere on the page leaves it alone. scripts/doc-paths-baseline.txtlists the findings this check lets through. It fails on any other finding,and on a baseline line that no longer occurs, so the baseline only shrinks.
No third-party modules, for the reason check-plan.py has none.
Run by prek, or by hand: scripts/check-doc-paths.pyTest the checker itself: scripts/check-doc-paths.py --self-test"""
import fnmatchimport osimport reimport subprocessimport sysimport tomllib
SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))REPO_ROOT = os.path.dirname(SCRIPTS_DIR)BASELINE = os.path.join(SCRIPTS_DIR, "doc-paths-baseline.txt")FIXTURE = os.path.join(SCRIPTS_DIR, "fixtures", "doc-paths")
FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})")# A run of backticks, then the shortest text up to a run of the same length.SPAN_RE = re.compile(r"(?<!`)(`+)(?!`)(.+?)(?<!`)\1(?!`)", re.S)LINE_RE = re.compile(r"^(.*?):([0-9]+)(?:-([0-9]+))?$")PATH_RE = re.compile(r"[A-Za-z0-9._/+@-]+")BRACE_RE = re.compile(r"^([^{}]*)\{([^{}]*)\}([^{}]*)$")
def tracked_files(root): """Every path git tracks under root, relative to it.""" out = subprocess.run( ["git", "ls-files", "-z"], cwd=root, capture_output=True, check=True ).stdout return {p for p in out.decode("utf-8").split("\0") if p}
def walk(root): """Every file under root, relative to it. The self-test's tracked_files.""" files = set() for directory, _, names in os.walk(root): for name in names: files.add(os.path.relpath(os.path.join(directory, name), root)) return files
def pages(files): """README.md, and the markdown under docs/ and plan/.""" return sorted( p for p in files if p == "README.md" or (p.endswith(".md") and p.split("/", 1)[0] in ("docs", "plan")) )
def members(root, files): """Each workspace member's directory name, mapped to that directory.
The members are the globs in Cargo.toml's [workspace] table, matched one path segment at a time against the directories holding a Cargo.toml. A name two members share is left out, since a span could mean either. """ try: with open(os.path.join(root, "Cargo.toml"), "rb") as fh: patterns = tomllib.load(fh).get("workspace", {}).get("members", []) except FileNotFoundError: return {} found = {} for path in files: directory, name = os.path.split(path) if name != "Cargo.toml" or not directory: continue parts = directory.split("/") for pattern in patterns: globs = pattern.split("/") if len(globs) == len(parts) and all( fnmatch.fnmatchcase(part, glob) for part, glob in zip(parts, globs) ): found.setdefault(parts[-1], set()).add(directory) return {name: dirs.pop() for name, dirs in found.items() if len(dirs) == 1}
def directories(files): """Every directory holding a file, without a trailing slash.""" dirs = set() for path in files: parts = path.split("/")[:-1] for i in range(1, len(parts) + 1): dirs.add("/".join(parts[:i])) return dirs
def count_lines(path): """Lines in a file as an editor numbers them, or None if unreadable.""" try: with open(path, "rb") as fh: data = fh.read() except OSError: return None return data.count(b"\n") + (1 if data and not data.endswith(b"\n") else 0)
def code_spans(text): """Yield (line number, text) for each inline code span.
Fenced blocks are blanked first; a fence closes on a fence of the same character, at least as long. A span can wrap onto the next line but never past a blank one, so each paragraph is searched on its own. """ lines = text.split("\n") fence = None for i, line in enumerate(lines): m = FENCE_RE.match(line) if fence is None and m: fence = m.group(1) lines[i] = "" elif fence is not None: closer = m.group(1) if m else "" if closer[:1] == fence[0] and len(closer) >= len(fence): fence = None lines[i] = ""
start = 0 for i in range(len(lines) + 1): if i < len(lines) and lines[i].strip(): continue paragraph = "\n".join(lines[start:i]) for m in SPAN_RE.finditer(paragraph): yield start + 1 + paragraph.count("\n", 0, m.start()), m.group(2) start = i + 1
def named_paths(word, roots): """The paths a span's first word names and the line it names, or None.
roots maps a first path segment to the directory it is read from: each top-level directory to itself, and each member's name to its directory. """ line = None m = LINE_RE.match(word) if m: word = m.group(1) line = max(int(n) for n in m.groups()[1:] if n) paths = [word] if "{" in word or "}" in word: b = BRACE_RE.match(word) if not b: return None head, parts, tail = b.groups() paths = [head + part + tail for part in parts.split(",")] resolved = [] for path in paths: first, slash, rest = path.partition("/") if not slash or first not in roots or not PATH_RE.fullmatch(path): return None resolved.append(f"{roots[first]}/{rest}") return resolved, line
def resolve(root, files, dirs, path, line): """Why a path does not resolve, or None when it does.""" if path.rstrip("/") in dirs: if line is None: return None return f"{path} is a directory, not a file with lines" if path.endswith("/"): return f"{path} is not a directory" have = count_lines(os.path.join(root, path)) if path in files else None if have is None: return f"{path} does not exist" if line is not None and have < line: return f"{path} ends at line {have}" return None
def find_problems(root, files): """(page, line number, word, reason) for each path that does not resolve.""" top_dirs = {p.split("/", 1)[0] for p in files if "/" in p} roots = members(root, files) roots.update((top, top) for top in top_dirs) dirs = directories(files) problems = [] for page in pages(files): with open(os.path.join(root, page), encoding="utf-8") as fh: text = fh.read() for lineno, span in code_spans(text): words = span.split() named = named_paths(words[0], roots) if words else None if named is None: continue paths, line = named for path in paths: reason = resolve(root, files, dirs, path, line) if reason is not None: problems.append((page, lineno, words[0], reason)) return problems
def finding(page, word): """A finding as the baseline spells it.""" return f"{page}: {word}"
def read_baseline(path): """Each baseline line, mapped to its line number in the file.""" entries = {} with open(path, encoding="utf-8") as fh: for n, line in enumerate(fh, 1): line = line.strip() if line and not line.startswith("#"): entries.setdefault(line, n) return entries
def compare(problems, baseline): """The problems the baseline does not list, and its lines that are stale.""" found = {finding(page, word) for page, _, word, _ in problems} new = [p for p in problems if finding(p[0], p[2]) not in baseline] stale = sorted((n, key) for key, n in baseline.items() if key not in found) return new, stale
def report(new, stale, baseline_name): """Say on stderr what failed, and how to fix each kind.""" print(f"doc paths check failed ({len(new) + len(stale)} problem(s)):\n", file=sys.stderr) for page, lineno, word, reason in new: print(f"{page}:{lineno}: `{word}`: {reason}.", file=sys.stderr) if new: print( "\n Point each span at where the file is now, or reword the " "sentence to say what\n" " the code does now. `git log -1 -- <path>` finds the commit " "that moved or\n" " removed a file, and `git show --stat -M <commit>` says where " "it went.\n", file=sys.stderr, ) for n, key in stale: print(f"{baseline_name}:{n}: `{key}` no longer occurs.", file=sys.stderr) if stale: print( "\n Delete each of those lines. The baseline only shrinks.\n", file=sys.stderr, )
def self_test(): """Run the checker over scripts/fixtures/doc-paths/ and compare.""" baseline = read_baseline(os.path.join(FIXTURE, "baseline.txt")) new, stale = compare(find_problems(FIXTURE, walk(FIXTURE)), baseline) lib = "crates/demo/src/lib.txt" want_new = [ ("README.md", 3, "crates/demo/src/gone.txt", "crates/demo/src/gone.txt does not exist"), ("docs/page.md", 12, "crates/demo/src/gone.txt", "crates/demo/src/gone.txt does not exist"), ("docs/page.md", 14, f"{lib}:9", f"{lib} ends at line 3"), ("docs/page.md", 15, f"{lib}:2-9", f"{lib} ends at line 3"), ("docs/page.md", 16, "crates/demo/src/{lib,gone}.txt", "crates/demo/src/gone.txt does not exist"), ("docs/page.md", 17, f"{lib}/", f"{lib}/ is not a directory"), ("docs/page.md", 18, "scripts/gone", "scripts/gone does not exist"), ("docs/page.md", 33, "demo/src/gone.txt", "crates/demo/src/gone.txt does not exist"), ("plan/epic.md", 3, "crates/demo/src/wire.txt:9", "crates/demo/src/wire.txt ends at line 1"), ] want_stale = [(3, "plan/epic.md: crates/demo/src/fixed.txt")] if new == want_new and stale == want_stale: print("check-doc-paths: self-test ok") return 0 print("check-doc-paths self-test failed.", file=sys.stderr) for name, got, want in ( ("new findings", new, want_new), ("stale baseline lines", stale, want_stale), ): if got != want: print(f"\n {name}, got:", file=sys.stderr) for item in got: print(f" {item}", file=sys.stderr) print(" want:", file=sys.stderr) for item in want: print(f" {item}", file=sys.stderr) return 1
def main(argv): if argv == ["--self-test"]: return self_test() if argv: print("usage: check-doc-paths.py [--self-test]", file=sys.stderr) return 2 problems = find_problems(REPO_ROOT, tracked_files(REPO_ROOT)) new, stale = compare(problems, read_baseline(BASELINE)) if not new and not stale: return 0 report(new, stale, os.path.relpath(BASELINE, REPO_ROOT)) return 1
if __name__ == "__main__": sys.exit(main(sys.argv[1:]))