diff --git a/AGENTS.md b/AGENTS.md index dc358e78..d6a48794 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,7 @@ make bench # Run performance benchmarks with -O; append ab make measure-cpu # Steady-state CPU + per-symbol attribution of the running Prowl Debug app make capture-spike # Sample the running Prowl Debug app when CPU crosses a threshold make measure-titles # Black-box check that animated tab titles stay coalesced (~1 change/s) +make agent-versions # Compare installed tier-A agent CLI versions with the managed-hook attestation (docs-ai 064) make log-stream # Stream app logs (subsystem: com.onevcat.prowl) make build-cli # Build CLI (prowl) via SwiftPM make test-cli-smoke # Run CLI executable smoke tests diff --git a/Makefile b/Makefile index 6acb02bd..d148cdf5 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ TEST_SIGNING_ARGS := CODE_SIGNING_ALLOWED=NO CODE_SIGNING_REQUIRED=NO CODE_SIGN_ endif .DEFAULT_GOAL := help -.PHONY: build-ghostty-xcframework ensure-ghostty sync-ghostty _record-ghostty-hash build-app build-cli build-cli-release embed-cli-debug embed-cli embed-docs embed-skills run-app install-dev-build install-release archive export-archive format format-changed format-lint lint check test test-app test-scripts test-cli-smoke test-cli-unit test-cli-integration benchmark-build bump-version log-stream +.PHONY: build-ghostty-xcframework ensure-ghostty sync-ghostty _record-ghostty-hash build-app build-cli build-cli-release embed-cli-debug embed-cli embed-docs embed-skills run-app install-dev-build install-release archive export-archive format format-changed format-lint lint check test test-app test-scripts test-cli-smoke test-cli-unit test-cli-integration benchmark-build bump-version log-stream agent-versions help: # Display this help. @-+echo "Run make with one of the following targets:" @@ -469,6 +469,10 @@ capture-spike: # Sample the running Prowl Debug app the moment CPU crosses a thr measure-titles: # Black-box check that animated tab titles stay coalesced to ~1 change/s (works on Release builds) @bash scripts/measure-title-coalescing.sh +AGENT_VERSIONS_ARGS ?= +agent-versions: # Compare installed tier-A agent CLI versions with the managed-hook attestation (AGENT_VERSIONS_ARGS="--json" / "--strict" / "--check-matrix") + @python3 "$(CURRENT_MAKEFILE_DIR)/scripts/agent_versions.py" $(AGENT_VERSIONS_ARGS) + format: # Format all Swift code with swift-format (full-tree cleanup) swift-format -p --in-place --recursive --configuration ./.swift-format.json supacode supacodeTests diff --git a/docs-ai/064-agent-completion-signals/agent-attestation.json b/docs-ai/064-agent-completion-signals/agent-attestation.json new file mode 100644 index 00000000..dd64f0db --- /dev/null +++ b/docs-ai/064-agent-completion-signals/agent-attestation.json @@ -0,0 +1,78 @@ +{ + "schema": 1, + "description": "Managed-hook contract attestation for the tier-A runtimes (docs-ai 064, #726 T0). One entry per runtime: the version its launch-scoped hook contract last passed a live sweep against, the date, the record that attested it, and the command that prints the version. Checked by `make agent-versions`; the research matrix's generated `Tier-A attestation` line is derived from this file (`make test-scripts` fails on drift). Schema and update rules: 015-t0-version-attestation.md.", + "runtimes": [ + { + "runtime": "claude", + "name": "Claude Code", + "binary": "claude", + "version_command": ["claude", "--version"], + "attested_version": "2.1.245", + "attested_on": "2026-08-26", + "record": "011-s3c-action.md" + }, + { + "runtime": "codex", + "name": "Codex", + "binary": "codex", + "version_command": ["codex", "--version"], + "attested_version": "0.149.1", + "attested_on": "2026-08-26", + "record": "011-s3c-action.md" + }, + { + "runtime": "copilot", + "name": "GitHub Copilot CLI", + "binary": "copilot", + "version_command": ["copilot", "--version"], + "attested_version": "1.0.80", + "attested_on": "2026-08-26", + "record": "011-s3c-action.md" + }, + { + "runtime": "droid", + "name": "Factory Droid", + "binary": "droid", + "version_command": ["droid", "--version"], + "attested_version": "0.203.0", + "attested_on": "2026-08-26", + "record": "011-s3c-action.md" + }, + { + "runtime": "qodercli", + "name": "Qoder CLI", + "binary": "qodercli", + "version_command": ["qodercli", "--version"], + "attested_version": "1.1.29", + "attested_on": "2026-08-26", + "record": "011-s3c-action.md" + }, + { + "runtime": "pi", + "name": "Pi", + "binary": "pi", + "version_command": ["pi", "--version"], + "attested_version": "0.84.3", + "attested_on": "2026-08-26", + "record": "011-s3c-action.md" + }, + { + "runtime": "omp", + "name": "Oh My Pi", + "binary": "omp", + "version_command": ["omp", "--version"], + "attested_version": "18.0.6", + "attested_on": "2026-08-26", + "record": "011-s3c-action.md" + }, + { + "runtime": "opencode", + "name": "OpenCode", + "binary": "opencode", + "version_command": ["opencode", "--version"], + "attested_version": "1.18.23", + "attested_on": "2026-08-26", + "record": "011-s3c-action.md" + } + ] +} diff --git a/docs-ai/064-agent-completion-signals/research-agent-completion-signals.md b/docs-ai/064-agent-completion-signals/research-agent-completion-signals.md index 819a9033..82eceea1 100644 --- a/docs-ai/064-agent-completion-signals/research-agent-completion-signals.md +++ b/docs-ai/064-agent-completion-signals/research-agent-completion-signals.md @@ -3,7 +3,12 @@ > Living document for [064](000-plan.md): which deterministic "turn complete / needs > input / session start-end" channels each recognized agent CLI offers, how they can be > enabled per launch, what the payload carries, and what is not achievable. Update rows in -> place when a CLI changes; record the version and verification method per row. +> place when a CLI changes; record the version and verification method per row. The tier-A +> versions the managed-hook contract is attested against live in +> [agent-attestation.json](agent-attestation.json), which generates the line below; the dated +> re-attestation paragraphs are history. + +**Tier-A attestation** (generated from [agent-attestation.json](agent-attestation.json) by `scripts/agent_versions.py --write-matrix`; `make test-scripts` fails when this line drifts): claude 2.1.245 · codex 0.149.1 · copilot 1.0.80 · droid 0.203.0 · qodercli 1.1.29 · pi 0.84.3 · omp 18.0.6 · opencode 1.18.23 — last live sweep 2026-08-26 ([011-s3c-action.md](011-s3c-action.md)). **S3c re-attestation (2026-08-26):** Pi 0.84.3 · Oh My Pi 18.0.6 · OpenCode 1.18.23 (all upgraded first; see [010-s3c-plan.md](010-s3c-plan.md) for the measured lifecycles). diff --git a/scripts/agent_versions.py b/scripts/agent_versions.py new file mode 100644 index 00000000..4f8e7992 --- /dev/null +++ b/scripts/agent_versions.py @@ -0,0 +1,530 @@ +#!/usr/bin/env python3 +"""Compare the installed tier-A agent CLIs with the managed-hook version attestation. + +Prowl's launch-scoped hooks (docs-ai 064, S3 wave 1) are a contract with eight external +binaries, and that contract moves with them. The version each runtime last passed a live +sweep against is recorded in `docs-ai/064-agent-completion-signals/agent-attestation.json`; +this script is the cheap half of #726 (T0). For every attested runtime it locates the binary, +runs its version command with a timeout, parses the semantic version out of whatever banner +the CLI prints, and reports one of: + + attested the installed build is the one the contract was verified against + newer the contract has not been verified against this build (warning) + older an older build than the attested one is installed (warning) + missing no binary on PATH, nor on the login shell's PATH (warning, not a failure) + unparseable the binary ran but printed no recognizable version, failed, or timed out + +Binaries are looked up on this process's PATH first. When one is missing there, the user's +shell (`$SHELL -lic`, login and interactive, because Homebrew and `mise activate` usually live in +`.zshrc`) is asked for its PATH once and the lookup is retried on it, so a `mise`-managed tool or +a PATH extended only in shell rc files is still found when the script runs from `make` under an +editor or agent harness. A version command runs with the PATH it was found on, so shims can +resolve their own toolchain. `--no-login-shell` disables the fallback. + +The research matrix keeps a generated "Tier-A attestation" line whose versions must equal the +record; `--check-matrix` verifies that (and `make test-scripts` runs it), `--write-matrix` +regenerates the line. + +Usage: + + scripts/agent_versions.py [--json] [--strict] [--timeout SECONDS] [--no-login-shell] + scripts/agent_versions.py --check-matrix + scripts/agent_versions.py --write-matrix + +The exit code is 0 unless `--strict` is given, in which case any status other than +`attested` exits 1. `--check-matrix` exits 1 on drift. +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass +from typing import Callable, Optional, Sequence + +ROOT = pathlib.Path(__file__).resolve().parents[1] +ENTRY_DIR = ROOT / "docs-ai" / "064-agent-completion-signals" +ATTESTATION_PATH = ENTRY_DIR / "agent-attestation.json" +MATRIX_PATH = ENTRY_DIR / "research-agent-completion-signals.md" + +ATTESTATION_SCHEMA = 1 +TIER_A_RUNTIMES = ("claude", "codex", "copilot", "droid", "qodercli", "pi", "omp", "opencode") +ENTRY_KEYS = frozenset( + {"runtime", "name", "binary", "version_command", "attested_version", "attested_on", "record"} +) +STATUSES = ("attested", "newer", "older", "missing", "unparseable") +DEFAULT_TIMEOUT = 20.0 + +MATRIX_LINE_PREFIX = "**Tier-A attestation**" +MATRIX_LINE_INTRO = ( + " (generated from [agent-attestation.json](agent-attestation.json) by" + " `scripts/agent_versions.py --write-matrix`; `make test-scripts` fails when this line drifts): " +) + +ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;?]*[ -/]*[@-~]") +# Three dotted integers not preceded by a digit or dot ("v1.2.3", "omp/18.0.6", "codex-cli 0.149.1") +# and not followed by a word character or hyphen ("1.0.80." keeps its sentence period out; a +# fourth component such as "2026.05.09.1" is not a semantic version and does not match at all). +VERSION_PATTERN = re.compile(r"(? "AttestedRuntime": + if not isinstance(fields, dict): + raise AttestationError("each runtime entry must be an object") + keys = set(fields) + if keys != ENTRY_KEYS: + unexpected = sorted(keys - ENTRY_KEYS) + missing = sorted(ENTRY_KEYS - keys) + raise AttestationError(f"runtime entry keys: unexpected {unexpected}, missing {missing}") + command = fields["version_command"] + if not isinstance(command, list) or not command or not all(isinstance(part, str) for part in command): + raise AttestationError(f"{fields['runtime']}: version_command must be a non-empty list of strings") + for key in ("runtime", "name", "binary", "attested_version", "attested_on", "record"): + if not isinstance(fields[key], str) or not fields[key]: + raise AttestationError(f"{fields['runtime']}: {key} must be a non-empty string") + if parse_version(fields["attested_version"]) is None: + raise AttestationError(f"{fields['runtime']}: attested_version {fields['attested_version']!r} is not a version") + try: + datetime.date.fromisoformat(fields["attested_on"]) + except ValueError as error: + raise AttestationError(f"{fields['runtime']}: attested_on must be an ISO date: {error}") from None + return cls( + runtime=fields["runtime"], + name=fields["name"], + binary=fields["binary"], + version_command=tuple(command), + attested_version=fields["attested_version"], + attested_on=fields["attested_on"], + record=fields["record"], + ) + + +@dataclass(frozen=True) +class ResolvedBinary: + path: str + resolution: str # "path" or "login-shell" + search_path: str + + +@dataclass(frozen=True) +class CommandResult: + stdout: str + stderr: str + returncode: Optional[int] + timed_out: bool + + +@dataclass +class RuntimeReport: + entry: AttestedRuntime + status: str + detail: str = "" + path: Optional[str] = None + resolution: Optional[str] = None + installed: Optional[Version] = None + raw_output: str = "" + + +def load_attestation(path: pathlib.Path) -> list: + try: + document = json.loads(path.read_text()) + except (OSError, ValueError) as error: + raise AttestationError(f"cannot read {path}: {error}") from None + if not isinstance(document, dict) or document.get("schema") != ATTESTATION_SCHEMA: + raise AttestationError(f"{path}: expected an object with schema {ATTESTATION_SCHEMA}") + runtimes = document.get("runtimes") + if not isinstance(runtimes, list) or not runtimes: + raise AttestationError(f"{path}: runtimes must be a non-empty list") + entries = [AttestedRuntime.from_json(fields) for fields in runtimes] + seen = set() + for item in entries: + if item.runtime in seen: + raise AttestationError(f"{path}: runtime {item.runtime!r} is listed twice") + seen.add(item.runtime) + if not (path.parent / item.record).is_file(): + raise AttestationError(f"{path}: {item.runtime}: record {item.record!r} does not exist next to the attestation") + return entries + + +def parse_version(text: str) -> Optional[Version]: + match = VERSION_PATTERN.search(ANSI_ESCAPE.sub("", text)) + if match is None: + return None + core = (int(match.group(1)), int(match.group(2)), int(match.group(3))) + prerelease = match.group(4) + if prerelease: + prerelease = prerelease.rstrip(".") + rendered = ".".join(str(part) for part in core) + if prerelease: + rendered += f"-{prerelease}" + return Version(core=core, prerelease=prerelease or None, text=rendered) + + +def compare_versions(installed: Version, attested: Version) -> str: + if installed.sort_key() == attested.sort_key(): + return "attested" + return "newer" if installed.sort_key() > attested.sort_key() else "older" + + +def login_shell_path(shell: Optional[str] = None, timeout: float = 10.0) -> Optional[str]: + """The PATH of a fresh login + interactive shell, or None when it cannot be obtained. + + Interactive matters: on a stock macOS setup Homebrew's `shellenv` and `mise activate` are + sourced from `.zshrc`, which a login-only shell skips. The PATH is fenced with markers so + banners printed by the rc files do not leak into it. + """ + shell = shell or os.environ.get("SHELL") or "/bin/zsh" + marker = "__PROWL_PATH__" + try: + result = subprocess.run( + [shell, "-lic", f'printf "\\n{marker}%s{marker}\\n" "$PATH"'], + capture_output=True, + text=True, + errors="replace", + stdin=subprocess.DEVNULL, + timeout=timeout, + ) + except (OSError, subprocess.SubprocessError): + return None + match = re.search(f"{marker}(.*?){marker}", result.stdout, re.DOTALL) + if match is None or not match.group(1).strip(): + return None + return match.group(1).strip() + + +class BinaryResolver: + """Looks a binary up on this process's PATH, then on the login shell's PATH (once, lazily).""" + + def __init__(self, use_login_shell: bool = True): + self.use_login_shell = use_login_shell + self._login_path: Optional[str] = None + self._login_path_queried = False + + def _login_shell_path(self) -> Optional[str]: + if not self._login_path_queried: + self._login_path_queried = True + self._login_path = login_shell_path() + return self._login_path + + def __call__(self, binary: str) -> Optional[ResolvedBinary]: + current = os.environ.get("PATH", "") + found = shutil.which(binary, path=current) + if found: + return ResolvedBinary(path=found, resolution="path", search_path=current) + if not self.use_login_shell: + return None + login = self._login_shell_path() + if not login: + return None + found = shutil.which(binary, path=login) + if found: + return ResolvedBinary(path=found, resolution="login-shell", search_path=login) + return None + + +def run_version_command(command: Sequence[str], search_path: str, timeout: float) -> CommandResult: + env = {**os.environ, "PATH": search_path, "NO_COLOR": "1"} + try: + result = subprocess.run( + list(command), + capture_output=True, + text=True, + errors="replace", + stdin=subprocess.DEVNULL, + env=env, + timeout=timeout, + ) + except subprocess.TimeoutExpired as expired: + return CommandResult( + stdout=_decode(expired.stdout), + stderr=_decode(expired.stderr), + returncode=None, + timed_out=True, + ) + except OSError as error: + return CommandResult(stdout="", stderr=str(error), returncode=None, timed_out=False) + return CommandResult(stdout=result.stdout, stderr=result.stderr, returncode=result.returncode, timed_out=False) + + +def _decode(data) -> str: + if data is None: + return "" + if isinstance(data, bytes): + return data.decode("utf-8", errors="replace") + return str(data) + + +def assess( + entries: Sequence[AttestedRuntime], + resolve: Callable[[str], Optional[ResolvedBinary]], + run: Callable[[Sequence[str], str, float], CommandResult], + timeout: float, +) -> list: + reports = [] + for item in entries: + resolved = resolve(item.binary) + if resolved is None: + reports.append( + RuntimeReport( + entry=item, + status="missing", + detail=f"{item.binary} not found on PATH (nor on the login shell PATH)", + ) + ) + continue + command = (resolved.path, *item.version_command[1:]) + result = run(command, resolved.search_path, timeout) + raw_output = result.stdout if result.stdout.strip() else result.stderr + report = RuntimeReport( + entry=item, + status="unparseable", + path=resolved.path, + resolution=resolved.resolution, + raw_output=result.stdout + result.stderr, + ) + shown = " ".join(item.version_command) + if result.timed_out: + report.detail = f"`{shown}` timed out after {timeout:g}s" + reports.append(report) + continue + installed = parse_version(raw_output) + if installed is None: + first_line = next((line for line in (result.stdout + result.stderr).splitlines() if line.strip()), "") + exit_text = "did not start" if result.returncode is None else f"exited {result.returncode}" + report.detail = f"`{shown}` {exit_text} without a recognizable version: {first_line.strip()!r}" + reports.append(report) + continue + report.installed = installed + report.status = compare_versions(installed, parse_version(item.attested_version)) + reports.append(report) + return reports + + +def warnings_for(reports: Sequence[RuntimeReport]) -> list: + warnings = [] + for report in reports: + item = report.entry + provenance = f"{item.record}, {item.attested_on}" + if report.status == "newer": + warnings.append( + f"warning: {item.binary} {report.installed.text} is newer than the attested" + f" {item.attested_version} ({provenance}); the managed-hook contract is unverified" + f" against it — run `make test-agent-contracts` (#726 T1) and update" + f" {ATTESTATION_PATH.relative_to(ROOT)} on a pass" + ) + elif report.status == "older": + warnings.append( + f"warning: {item.binary} {report.installed.text} is older than the attested" + f" {item.attested_version} ({provenance}); upgrade, or re-run" + f" `make test-agent-contracts` (#726 T1) against this build" + ) + elif report.status == "missing": + warnings.append(f"warning: {report.detail}; its managed-hook contract cannot be checked on this machine") + elif report.status == "unparseable": + warnings.append(f"warning: {item.binary}: {report.detail}") + return warnings + + +def render_table(reports: Sequence[RuntimeReport]) -> str: + rows = [("runtime", "attested", "installed", "status")] + for report in reports: + installed = report.installed.text if report.installed else "-" + rows.append((report.entry.binary, report.entry.attested_version, installed, report.status)) + widths = [max(len(row[column]) for row in rows) for column in range(4)] + lines = [] + for row in rows: + cells = [cell.ljust(width) for cell, width in zip(row, widths)] + lines.append(" ".join(cells).rstrip()) + return "\n".join(lines) + + +def json_document(reports: Sequence[RuntimeReport], attestation_path: pathlib.Path) -> dict: + runtimes = [] + for report in reports: + item = report.entry + runtimes.append( + { + "runtime": item.runtime, + "name": item.name, + "binary": item.binary, + "version_command": list(item.version_command), + "attested_version": item.attested_version, + "attested_on": item.attested_on, + "record": item.record, + "path": report.path, + "resolution": report.resolution, + "installed_version": report.installed.text if report.installed else None, + "raw_output": report.raw_output, + "status": report.status, + "detail": report.detail, + } + ) + summary = {status: sum(1 for report in reports if report.status == status) for status in STATUSES} + return {"attestation": str(attestation_path), "runtimes": runtimes, "summary": summary} + + +def exit_code(reports: Sequence[RuntimeReport], strict: bool) -> int: + if strict and any(report.status != "attested" for report in reports): + return 1 + return 0 + + +def render_matrix_line(entries: Sequence[AttestedRuntime]) -> str: + versions = " · ".join(f"{item.binary} {item.attested_version}" for item in entries) + sweeps = {} + for item in entries: + sweeps.setdefault((item.attested_on, item.record), []).append(item.binary) + if len(sweeps) == 1: + (date, record), _ = next(iter(sweeps.items())) + provenance = f"last live sweep {date} ([{record}]({record}))" + else: + parts = [ + f"{date} ([{record}]({record})): {', '.join(binaries)}" + for (date, record), binaries in sorted(sweeps.items()) + ] + provenance = "last live sweeps: " + "; ".join(parts) + return f"{MATRIX_LINE_PREFIX}{MATRIX_LINE_INTRO}{versions} — {provenance}." + + +def _matrix_versions(line: str) -> dict: + body = line.split("): ", 1)[1] if "): " in line else line + body = body.split(" — ", 1)[0] + versions = {} + for token in body.split(" · "): + parts = token.strip().split(" ") + if len(parts) == 2: + versions[parts[0]] = parts[1] + return versions + + +def check_matrix(entries: Sequence[AttestedRuntime], matrix_text: str) -> list: + """Problems with the matrix's generated line; empty when it matches the attestation.""" + lines = [line for line in matrix_text.splitlines() if line.startswith(MATRIX_LINE_PREFIX)] + if not lines: + return [f"no line starting with {MATRIX_LINE_PREFIX!r} in the research matrix"] + if len(lines) > 1: + return [f"{len(lines)} lines start with {MATRIX_LINE_PREFIX!r}; expected exactly one"] + expected = render_matrix_line(entries) + actual = lines[0] + if actual == expected: + return [] + problems = [] + found = _matrix_versions(actual) + for item in entries: + version = found.get(item.binary) + if version is None: + problems.append(f"{item.binary}: matrix lists no version, attestation {item.attested_version}") + elif version != item.attested_version: + problems.append(f"{item.binary}: matrix {version}, attestation {item.attested_version}") + for binary in found: + if binary not in {item.binary for item in entries}: + problems.append(f"{binary}: listed in the matrix but not in the attestation") + if not problems: + problems.append("the generated line's wording or provenance differs from the attestation") + problems.append(f"expected line:\n{expected}\nactual line:\n{actual}") + return problems + + +def write_matrix(entries: Sequence[AttestedRuntime], matrix_path: pathlib.Path) -> bool: + """Regenerate the matrix line in place; returns whether the file changed.""" + text = matrix_path.read_text() + expected = render_matrix_line(entries) + lines = text.split("\n") + indexes = [index for index, line in enumerate(lines) if line.startswith(MATRIX_LINE_PREFIX)] + if len(indexes) != 1: + raise SystemExit( + f"{matrix_path}: expected exactly one line starting with {MATRIX_LINE_PREFIX!r}, found {len(indexes)}" + ) + if lines[indexes[0]] == expected: + return False + lines[indexes[0]] = expected + matrix_path.write_text("\n".join(lines)) + return True + + +def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--attestation", type=pathlib.Path, default=ATTESTATION_PATH, help="attestation record to check against") + parser.add_argument("--matrix", type=pathlib.Path, default=MATRIX_PATH, help="research matrix for --check-matrix / --write-matrix") + parser.add_argument("--json", action="store_true", help="print the report as JSON instead of a table") + parser.add_argument("--strict", action="store_true", help="exit 1 unless every runtime is attested") + parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT, help=f"seconds per version command (default {DEFAULT_TIMEOUT:g})") + parser.add_argument("--no-login-shell", action="store_true", help="do not consult the login shell's PATH for missing binaries") + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--check-matrix", action="store_true", help="verify the research matrix's generated line matches the attestation") + mode.add_argument("--write-matrix", action="store_true", help="regenerate the research matrix's generated line from the attestation") + return parser.parse_args(argv) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = parse_args(argv) + try: + entries = load_attestation(args.attestation) + except AttestationError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + + if args.check_matrix: + problems = check_matrix(entries, args.matrix.read_text()) + if problems: + print(f"error: {args.matrix.relative_to(ROOT) if args.matrix.is_relative_to(ROOT) else args.matrix} drifts from {args.attestation.name}:", file=sys.stderr) + for problem in problems: + print(f" {problem}", file=sys.stderr) + print(" run `scripts/agent_versions.py --write-matrix` to regenerate the line", file=sys.stderr) + return 1 + print(f"{args.matrix.name} matches {args.attestation.name}") + return 0 + + if args.write_matrix: + changed = write_matrix(entries, args.matrix) + print(f"{args.matrix.name}: {'updated' if changed else 'already current'}") + return 0 + + resolver = BinaryResolver(use_login_shell=not args.no_login_shell) + reports = assess(entries, resolve=resolver, run=run_version_command, timeout=args.timeout) + if args.json: + print(json.dumps(json_document(reports, args.attestation), indent=2)) + else: + print(render_table(reports)) + # Keep the report ahead of the warnings when stdout is a pipe. + sys.stdout.flush() + for warning in warnings_for(reports): + print(warning, file=sys.stderr) + return exit_code(reports, strict=args.strict) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_agent_versions.py b/scripts/test_agent_versions.py new file mode 100644 index 00000000..472a21b4 --- /dev/null +++ b/scripts/test_agent_versions.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python3 +"""Tests for the tier-A agent version attestation check. + +The version parsers are pinned to the `--version` output each tier-A CLI actually printed on the +attesting Mac, so a runtime that changes its banner fails here before it confuses the report. The +comparison, the missing/unparseable handling, the JSON shape, and the research-matrix drift check +run against fakes, and the drift check additionally runs against the committed record and matrix +so `make test-scripts` notices when one is edited without the other. + +Run with `make test-scripts`, or directly: + + python3 -m unittest discover -s scripts -p 'test_*.py' +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import pathlib +import stat +import subprocess +import sys +import tempfile +import unittest +import unittest.mock + +SCRIPT = pathlib.Path(__file__).resolve().parent / "agent_versions.py" + +_spec = importlib.util.spec_from_file_location("agent_versions", SCRIPT) +agent_versions = importlib.util.module_from_spec(_spec) +# dataclasses resolve deferred annotations through sys.modules, so register before executing. +sys.modules[_spec.name] = agent_versions +_spec.loader.exec_module(agent_versions) + +# Verbatim `--version` output captured on 2026-08-29 (stdout; every binary printed nothing on stderr). +REAL_OUTPUTS = { + "claude": "2.1.251 (Claude Code)\n", + "codex": "codex-cli 0.149.1\n", + "copilot": "GitHub Copilot CLI 1.0.80.\nRun 'copilot update' to check for updates.\n", + "droid": "0.204.0\n", + "qodercli": "1.1.31\n", + "pi": "0.84.3\n", + "omp": "omp/18.0.6\n", + "opencode": "1.18.23\n", +} + +REAL_VERSIONS = { + "claude": "2.1.251", + "codex": "0.149.1", + "copilot": "1.0.80", + "droid": "0.204.0", + "qodercli": "1.1.31", + "pi": "0.84.3", + "omp": "18.0.6", + "opencode": "1.18.23", +} + + +def entry(binary, version="1.0.0", **overrides): + fields = { + "runtime": binary, + "name": binary.title(), + "binary": binary, + "version_command": [binary, "--version"], + "attested_version": version, + "attested_on": "2026-08-26", + "record": "011-s3c-action.md", + } + fields.update(overrides) + return agent_versions.AttestedRuntime.from_json(fields) + + +class VersionParsing(unittest.TestCase): + def test_every_real_banner_yields_its_version(self): + for binary, output in REAL_OUTPUTS.items(): + with self.subTest(binary=binary): + version = agent_versions.parse_version(output) + self.assertIsNotNone(version) + self.assertEqual(version.text, REAL_VERSIONS[binary]) + + def test_copilot_trailing_period_is_not_part_of_the_version(self): + # "GitHub Copilot CLI 1.0.80." — the sentence ends in a period right after the patch. + self.assertEqual(agent_versions.parse_version(REAL_OUTPUTS["copilot"]).core, (1, 0, 80)) + + def test_prefixes_and_prerelease_tags(self): + self.assertEqual(agent_versions.parse_version("v1.2.3\n").text, "1.2.3") + version = agent_versions.parse_version("tool 2.0.0-beta.1\n") + self.assertEqual(version.core, (2, 0, 0)) + self.assertEqual(version.prerelease, "beta.1") + self.assertEqual(version.text, "2.0.0-beta.1") + + def test_ansi_escapes_are_stripped_before_parsing(self): + self.assertEqual(agent_versions.parse_version("\x1b[1m3.4.5\x1b[0m (thing)\n").text, "3.4.5") + + def test_output_without_a_version_is_none(self): + self.assertIsNone(agent_versions.parse_version("")) + self.assertIsNone(agent_versions.parse_version("Run 'copilot update' to check for updates.\n")) + # Four dotted components are a build stamp, not a semantic version. + self.assertIsNone(agent_versions.parse_version("build 1.2.3.4\n")) + + def test_first_version_wins_when_a_banner_mentions_several(self): + self.assertEqual(agent_versions.parse_version("cli 1.2.3 (node 22.21.1)\n").text, "1.2.3") + + +class VersionComparison(unittest.TestCase): + def compare(self, installed, attested): + return agent_versions.compare_versions( + agent_versions.parse_version(installed), agent_versions.parse_version(attested) + ) + + def test_equal_is_attested(self): + self.assertEqual(self.compare("0.149.1", "0.149.1"), "attested") + + def test_numeric_segments_compare_as_integers(self): + self.assertEqual(self.compare("0.204.0", "0.203.0"), "newer") + self.assertEqual(self.compare("1.1.31", "1.1.29"), "newer") + self.assertEqual(self.compare("2.1.251", "2.1.245"), "newer") + self.assertEqual(self.compare("0.84.2", "0.84.3"), "older") + self.assertEqual(self.compare("18.0.6", "17.2.7"), "newer") + self.assertEqual(self.compare("1.18.9", "1.18.23"), "older") + + def test_prerelease_sorts_below_its_release(self): + self.assertEqual(self.compare("2.0.0-beta.1", "2.0.0"), "older") + self.assertEqual(self.compare("2.0.0", "2.0.0-beta.1"), "newer") + self.assertEqual(self.compare("2.0.0-beta.2", "2.0.0-beta.1"), "newer") + + +class AttestationRecord(unittest.TestCase): + def test_committed_record_covers_the_tier_a_runtimes(self): + entries = agent_versions.load_attestation(agent_versions.ATTESTATION_PATH) + self.assertEqual(tuple(item.runtime for item in entries), agent_versions.TIER_A_RUNTIMES) + for item in entries: + with self.subTest(runtime=item.runtime): + self.assertEqual(item.version_command[0], item.binary) + self.assertIsNotNone(agent_versions.parse_version(item.attested_version)) + self.assertTrue((agent_versions.ATTESTATION_PATH.parent / item.record).is_file()) + + def test_loader_rejects_a_malformed_record(self): + good = json.loads(agent_versions.ATTESTATION_PATH.read_text()) + cases = { + "unknown schema": {**good, "schema": 99}, + "duplicate runtime": {**good, "runtimes": good["runtimes"] + [good["runtimes"][0]]}, + "unparseable version": { + **good, + "runtimes": [{**good["runtimes"][0], "attested_version": "latest"}], + }, + "bad date": {**good, "runtimes": [{**good["runtimes"][0], "attested_on": "26/08/2026"}]}, + "missing record": {**good, "runtimes": [{**good["runtimes"][0], "record": "nope.md"}]}, + "extra key": {**good, "runtimes": [{**good["runtimes"][0], "notes": "x"}]}, + } + with tempfile.TemporaryDirectory() as tmp: + for label, document in cases.items(): + with self.subTest(case=label): + path = pathlib.Path(tmp) / "agent-attestation.json" + path.write_text(json.dumps(document)) + (pathlib.Path(tmp) / "011-s3c-action.md").write_text("# record\n") + with self.assertRaises(agent_versions.AttestationError): + agent_versions.load_attestation(path) + + +class ShellPathLookup(unittest.TestCase): + """The shell fallback reads PATH through a stub shell that behaves like a chatty rc file.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + root = pathlib.Path(self.tmp.name) + self.bin = root / "shell-bin" + self.bin.mkdir() + self.shell = root / "stub-shell" + # Prints a banner first, then evaluates the requested command with an extended PATH. + self.shell.write_text( + "#!/bin/sh\necho 'Welcome to the stub shell'\nexport PATH=\"" + str(self.bin) + ":$PATH\"\neval \"$2\"\n" + ) + self.shell.chmod(self.shell.stat().st_mode | stat.S_IEXEC) + + def tearDown(self): + self.tmp.cleanup() + + def stub_pi(self): + stub = self.bin / "pi" + stub.write_text("#!/bin/sh\necho 0.84.3\n") + stub.chmod(stub.stat().st_mode | stat.S_IEXEC) + return stub + + def test_path_is_read_from_the_marker_and_ignores_banners(self): + path = agent_versions.login_shell_path(shell=str(self.shell)) + self.assertTrue(path.startswith(str(self.bin) + ":"), path) + self.assertNotIn("Welcome", path) + + def test_failing_shell_yields_none(self): + self.assertIsNone(agent_versions.login_shell_path(shell=str(self.bin / "does-not-exist"))) + + def test_resolver_falls_back_to_the_shell_path_only_when_allowed(self): + stub = self.stub_pi() + with unittest.mock.patch.dict(os.environ, {"SHELL": str(self.shell), "PATH": "/usr/bin:/bin"}): + self.assertIsNone(agent_versions.BinaryResolver(use_login_shell=False)("pi")) + resolved = agent_versions.BinaryResolver(use_login_shell=True)("pi") + self.assertEqual(resolved.path, str(stub)) + self.assertEqual(resolved.resolution, "login-shell") + self.assertTrue(resolved.search_path.startswith(str(self.bin) + ":")) + + def test_resolver_prefers_the_process_path(self): + self.stub_pi() + with unittest.mock.patch.dict(os.environ, {"SHELL": str(self.shell), "PATH": str(self.bin)}): + resolved = agent_versions.BinaryResolver(use_login_shell=True)("pi") + self.assertEqual(resolved.resolution, "path") + + +class FakeRunner: + """Stands in for the binary resolver and the version command.""" + + def __init__(self, paths, outputs): + self.paths = paths + self.outputs = outputs + self.commands = [] + + def resolve(self, binary): + return self.paths.get(binary) + + def run(self, command, search_path, timeout): + self.commands.append((tuple(command), timeout)) + result = self.outputs[command[0]] + if result == "timeout": + return agent_versions.CommandResult(stdout="", stderr="", returncode=None, timed_out=True) + return result + + +def report_with(entries, paths, outputs): + runner = FakeRunner(paths, outputs) + return agent_versions.assess(entries, resolve=runner.resolve, run=runner.run, timeout=5.0) + + +class Assessment(unittest.TestCase): + def test_statuses_cover_every_outcome(self): + entries = [ + entry("claude", "2.1.245"), + entry("codex", "0.149.1"), + entry("pi", "0.84.3"), + entry("droid", "0.203.0"), + entry("omp", "18.0.6"), + entry("qodercli", "1.1.29"), + ] + paths = { + "claude": agent_versions.ResolvedBinary("/x/claude", "path", "/x"), + "codex": agent_versions.ResolvedBinary("/x/codex", "login-shell", "/x"), + "pi": agent_versions.ResolvedBinary("/x/pi", "path", "/x"), + "omp": agent_versions.ResolvedBinary("/x/omp", "path", "/x"), + "qodercli": agent_versions.ResolvedBinary("/x/qodercli", "path", "/x"), + } + outputs = { + "/x/claude": agent_versions.CommandResult(REAL_OUTPUTS["claude"], "", 0, False), + "/x/codex": agent_versions.CommandResult(REAL_OUTPUTS["codex"], "", 0, False), + "/x/pi": agent_versions.CommandResult("0.84.2\n", "", 0, False), + "/x/omp": agent_versions.CommandResult("", "cannot start: no display\n", 1, False), + "/x/qodercli": "timeout", + } + reports = report_with(entries, paths, outputs) + statuses = {report.entry.runtime: report.status for report in reports} + self.assertEqual( + statuses, + { + "claude": "newer", + "codex": "attested", + "pi": "older", + "droid": "missing", + "omp": "unparseable", + "qodercli": "unparseable", + }, + ) + by_runtime = {report.entry.runtime: report for report in reports} + self.assertEqual(by_runtime["claude"].installed.text, "2.1.251") + self.assertEqual(by_runtime["codex"].resolution, "login-shell") + self.assertIsNone(by_runtime["droid"].installed) + self.assertIn("not found", by_runtime["droid"].detail) + self.assertIn("exited 1", by_runtime["omp"].detail) + self.assertIn("no display", by_runtime["omp"].detail) + self.assertIn("timed out after 5s", by_runtime["qodercli"].detail) + + def test_version_is_taken_from_stderr_when_stdout_is_empty(self): + reports = report_with( + [entry("pi", "0.84.3")], + {"pi": agent_versions.ResolvedBinary("/x/pi", "path", "/x")}, + {"/x/pi": agent_versions.CommandResult("", "0.84.3\n", 0, False)}, + ) + self.assertEqual(reports[0].status, "attested") + + def test_version_command_runs_with_the_path_it_was_found_on(self): + runner = FakeRunner( + {"pi": agent_versions.ResolvedBinary("/shims/pi", "login-shell", "/shims:/usr/bin")}, + {"/shims/pi": agent_versions.CommandResult("0.84.3\n", "", 0, False)}, + ) + agent_versions.assess([entry("pi", "0.84.3")], resolve=runner.resolve, run=runner.run, timeout=7.0) + self.assertEqual(runner.commands, [(("/shims/pi", "--version"), 7.0)]) + + def test_warnings_point_newer_builds_at_the_contract_gate(self): + reports = report_with( + [entry("claude", "2.1.245"), entry("droid", "0.203.0"), entry("pi", "0.84.3")], + { + "claude": agent_versions.ResolvedBinary("/x/claude", "path", "/x"), + "pi": agent_versions.ResolvedBinary("/x/pi", "path", "/x"), + }, + { + "/x/claude": agent_versions.CommandResult(REAL_OUTPUTS["claude"], "", 0, False), + "/x/pi": agent_versions.CommandResult("0.84.3\n", "", 0, False), + }, + ) + warnings = agent_versions.warnings_for(reports) + self.assertEqual(len(warnings), 2) + self.assertIn("claude 2.1.251 is newer than the attested 2.1.245", warnings[0]) + self.assertIn("make test-agent-contracts", warnings[0]) + self.assertIn("011-s3c-action.md", warnings[0]) + self.assertIn("droid", warnings[1]) + self.assertIn("not found", warnings[1]) + + def test_table_lists_every_runtime_with_aligned_columns(self): + reports = report_with( + [entry("claude", "2.1.245"), entry("droid", "0.203.0")], + {"claude": agent_versions.ResolvedBinary("/x/claude", "path", "/x")}, + {"/x/claude": agent_versions.CommandResult(REAL_OUTPUTS["claude"], "", 0, False)}, + ) + table = agent_versions.render_table(reports).splitlines() + self.assertEqual(table[0].split(), ["runtime", "attested", "installed", "status"]) + self.assertEqual(table[1].split(), ["claude", "2.1.245", "2.1.251", "newer"]) + self.assertEqual(table[2].split(), ["droid", "0.203.0", "-", "missing"]) + self.assertEqual(table[1].index("2.1.245"), table[2].index("0.203.0")) + + def test_json_document_carries_the_evidence_per_runtime(self): + reports = report_with( + [entry("claude", "2.1.245"), entry("droid", "0.203.0")], + {"claude": agent_versions.ResolvedBinary("/x/claude", "path", "/x")}, + {"/x/claude": agent_versions.CommandResult(REAL_OUTPUTS["claude"], "", 0, False)}, + ) + document = agent_versions.json_document(reports, pathlib.Path("/repo/agent-attestation.json")) + self.assertEqual(document["attestation"], "/repo/agent-attestation.json") + self.assertEqual(document["summary"], {"attested": 0, "newer": 1, "older": 0, "missing": 1, "unparseable": 0}) + claude, droid = document["runtimes"] + self.assertEqual( + {key: claude[key] for key in ("runtime", "attested_version", "installed_version", "status", "path")}, + { + "runtime": "claude", + "attested_version": "2.1.245", + "installed_version": "2.1.251", + "status": "newer", + "path": "/x/claude", + }, + ) + self.assertEqual(claude["raw_output"], REAL_OUTPUTS["claude"]) + self.assertEqual(claude["version_command"], ["claude", "--version"]) + self.assertEqual(claude["record"], "011-s3c-action.md") + self.assertIsNone(droid["installed_version"]) + self.assertIsNone(droid["path"]) + self.assertEqual(droid["status"], "missing") + json.dumps(document) + + def test_strict_fails_on_anything_but_attested(self): + attested = report_with( + [entry("pi", "0.84.3")], + {"pi": agent_versions.ResolvedBinary("/x/pi", "path", "/x")}, + {"/x/pi": agent_versions.CommandResult("0.84.3\n", "", 0, False)}, + ) + missing = report_with([entry("pi", "0.84.3")], {}, {}) + self.assertEqual(agent_versions.exit_code(attested, strict=False), 0) + self.assertEqual(agent_versions.exit_code(attested, strict=True), 0) + self.assertEqual(agent_versions.exit_code(missing, strict=False), 0) + self.assertEqual(agent_versions.exit_code(missing, strict=True), 1) + + +class MatrixCheck(unittest.TestCase): + def setUp(self): + self.entries = agent_versions.load_attestation(agent_versions.ATTESTATION_PATH) + self.matrix = agent_versions.MATRIX_PATH.read_text() + + def test_committed_matrix_matches_the_committed_record(self): + self.assertEqual(agent_versions.check_matrix(self.entries, self.matrix), []) + + def test_rendered_line_lists_every_runtime_and_its_provenance(self): + line = agent_versions.render_matrix_line(self.entries) + self.assertTrue(line.startswith(agent_versions.MATRIX_LINE_PREFIX)) + self.assertIn("[agent-attestation.json](agent-attestation.json)", line) + for item in self.entries: + self.assertIn(f"{item.binary} {item.attested_version}", line) + self.assertIn(f"[{item.record}]({item.record})", line) + self.assertIn(item.attested_on, line) + + def test_provenance_groups_by_sweep(self): + entries = [ + entry("claude", "2.1.245", attested_on="2026-08-25", record="009-s3b-action.md"), + entry("pi", "0.84.3"), + entry("omp", "18.0.6"), + ] + line = agent_versions.render_matrix_line(entries) + self.assertIn("2026-08-25 ([009-s3b-action.md](009-s3b-action.md)): claude", line) + self.assertIn("2026-08-26 ([011-s3c-action.md](011-s3c-action.md)): pi, omp", line) + + def test_drift_is_reported_per_runtime(self): + first = self.entries[0] + drifted = self.matrix.replace(f"{first.binary} {first.attested_version}", f"{first.binary} 0.0.1", 1) + problems = agent_versions.check_matrix(self.entries, drifted) + self.assertEqual(len(problems), 2) + self.assertIn(first.binary, problems[0]) + self.assertIn("matrix 0.0.1", problems[0]) + self.assertIn(f"attestation {first.attested_version}", problems[0]) + self.assertTrue(problems[1].startswith("expected line:\n")) + self.assertIn("actual line:\n", problems[1]) + + def test_missing_and_duplicated_lines_are_reported(self): + without = "\n".join( + line for line in self.matrix.splitlines() if not line.startswith(agent_versions.MATRIX_LINE_PREFIX) + ) + self.assertTrue(any("no line starting with" in problem for problem in agent_versions.check_matrix(self.entries, without))) + rendered = agent_versions.render_matrix_line(self.entries) + doubled = self.matrix.replace(rendered, rendered + "\n\n" + rendered, 1) + self.assertTrue(any("2 lines" in problem for problem in agent_versions.check_matrix(self.entries, doubled))) + + def test_write_matrix_replaces_only_the_generated_line(self): + first = self.entries[0] + drifted = self.matrix.replace(f"{first.binary} {first.attested_version}", f"{first.binary} 0.0.1", 1) + with tempfile.TemporaryDirectory() as tmp: + path = pathlib.Path(tmp) / "matrix.md" + path.write_text(drifted) + self.assertTrue(agent_versions.write_matrix(self.entries, path)) + self.assertEqual(path.read_text(), self.matrix) + self.assertFalse(agent_versions.write_matrix(self.entries, path)) + + +class CommandLine(unittest.TestCase): + """Runs the script as `make agent-versions` does, against stub binaries on a private PATH.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + root = pathlib.Path(self.tmp.name) + self.bin = root / "bin" + self.bin.mkdir() + self.record_dir = root / "record" + self.record_dir.mkdir() + (self.record_dir / "011-s3c-action.md").write_text("# record\n") + self.attestation = self.record_dir / "agent-attestation.json" + self.attestation.write_text( + json.dumps( + { + "schema": 1, + "description": "test", + "runtimes": [ + entry_json("claude", "2.1.245"), + entry_json("copilot", "1.0.80"), + entry_json("droid", "0.203.0"), + ], + } + ) + ) + self.stub("claude", REAL_OUTPUTS["claude"]) + self.stub("copilot", REAL_OUTPUTS["copilot"]) + + def tearDown(self): + self.tmp.cleanup() + + def stub(self, name, output): + path = self.bin / name + path.write_text("#!/bin/sh\nprintf '%s' " + shell_quote(output) + "\n") + path.chmod(path.stat().st_mode | stat.S_IEXEC) + + def run_script(self, *args): + env = {**os.environ, "PATH": f"{self.bin}:/usr/bin:/bin"} + return subprocess.run( + [sys.executable, str(SCRIPT), "--attestation", str(self.attestation), "--no-login-shell", *args], + capture_output=True, + text=True, + env=env, + ) + + def test_table_and_warnings(self): + result = self.run_script() + self.assertEqual(result.returncode, 0, result.stderr) + rows = [line.split() for line in result.stdout.splitlines()] + self.assertEqual(rows[0], ["runtime", "attested", "installed", "status"]) + self.assertEqual(rows[1], ["claude", "2.1.245", "2.1.251", "newer"]) + self.assertEqual(rows[2], ["copilot", "1.0.80", "1.0.80", "attested"]) + self.assertEqual(rows[3], ["droid", "0.203.0", "-", "missing"]) + self.assertIn("make test-agent-contracts", result.stderr) + self.assertIn("droid", result.stderr) + + def test_strict_exit_code(self): + self.assertEqual(self.run_script("--strict").returncode, 1) + + def test_json_output(self): + result = self.run_script("--json") + self.assertEqual(result.returncode, 0, result.stderr) + document = json.loads(result.stdout) + self.assertEqual([item["status"] for item in document["runtimes"]], ["newer", "attested", "missing"]) + self.assertEqual(document["runtimes"][0]["path"], str(self.bin / "claude")) + self.assertEqual(document["runtimes"][0]["resolution"], "path") + + def test_check_matrix_reports_drift_and_exit_code(self): + entries = agent_versions.load_attestation(self.attestation) + matrix = self.record_dir / "matrix.md" + matrix.write_text("# Matrix\n\n" + agent_versions.render_matrix_line(entries) + "\n") + ok = self.run_script("--check-matrix", "--matrix", str(matrix)) + self.assertEqual(ok.returncode, 0, ok.stderr) + matrix.write_text(matrix.read_text().replace("droid 0.203.0", "droid 0.202.0")) + drifted = self.run_script("--check-matrix", "--matrix", str(matrix)) + self.assertEqual(drifted.returncode, 1) + self.assertIn("droid: matrix 0.202.0, attestation 0.203.0", drifted.stderr) + fixed = self.run_script("--write-matrix", "--matrix", str(matrix)) + self.assertEqual(fixed.returncode, 0, fixed.stderr) + self.assertEqual(self.run_script("--check-matrix", "--matrix", str(matrix)).returncode, 0) + + +def entry_json(binary, version): + return { + "runtime": binary, + "name": binary.title(), + "binary": binary, + "version_command": [binary, "--version"], + "attested_version": version, + "attested_on": "2026-08-26", + "record": "011-s3c-action.md", + } + + +def shell_quote(text): + return "'" + text.replace("'", "'\\''") + "'" + + +if __name__ == "__main__": + unittest.main() -- 2.51.2 From 906f3a0fe6d80e4e0c9980aba9e7fb06065ba282 Mon Sep 17 00:00:00 2001 From: onevcat Date: Sat, 29 Aug 2026 11:03:54 +0900 Subject: [PATCH 2/2] Record the T0 version attestation slice in docs-ai 064 Document the attestation schema, where each attested version comes from (all eight from the S3c live acceptance, with the date and Claude ambiguities spelled out), the decisions behind one generic parser and the interactive-shell PATH fallback, and the actual `make agent-versions` output from this Mac; amend the 064 plan accordingly. T1 (headless contract tests) stays in R2b. Claude-Session: https://claude.ai/code/session_01YSXSCVNoycSbCmwPMvcCnw --- .../064-agent-completion-signals/000-plan.md | 6 + .../015-t0-version-attestation.md | 144 ++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 docs-ai/064-agent-completion-signals/015-t0-version-attestation.md diff --git a/docs-ai/064-agent-completion-signals/000-plan.md b/docs-ai/064-agent-completion-signals/000-plan.md index dc488be2..55f61b9e 100644 --- a/docs-ai/064-agent-completion-signals/000-plan.md +++ b/docs-ai/064-agent-completion-signals/000-plan.md @@ -246,6 +246,12 @@ opencode; partial for qodercli/qwen/amp; docs/bundle for the rest). Key conclusi ## Amendments +- Updated 2026-08-29 (#726 T0): [agent-attestation.json](agent-attestation.json) now records the + version each tier-A runtime last passed a live sweep against (all eight from + [011-s3c-action.md](011-s3c-action.md)); `make agent-versions` compares the installed binaries + against it and warns on newer builds, and the research matrix's tier-A line is generated from + the record with `make test-scripts` guarding drift. See + [015-t0-version-attestation.md](015-t0-version-attestation.md). T1 stays in R2b. - Updated 2026-08-29 (063 B1 kickoff): 063's `expect` activations are records in this entry's dispatch store (`launch` via the S2 prompted-launch path, `message` via #733's re-dispatch), and S5's watchdog part ships with 063-B2 instead of D2. #733 therefore lands before 063-B3. diff --git a/docs-ai/064-agent-completion-signals/015-t0-version-attestation.md b/docs-ai/064-agent-completion-signals/015-t0-version-attestation.md new file mode 100644 index 00000000..1ff3bb00 --- /dev/null +++ b/docs-ai/064-agent-completion-signals/015-t0-version-attestation.md @@ -0,0 +1,144 @@ +# 064.015 — T0 Version Attestation: Plan and Action + +## Status + +Implemented from `feat/agent-version-attestation` as the T0 half of +[#726](https://github.com/onevcat/Prowl/issues/726) (R2a in the shared +[release plan](../063-agent-workflows/release-plan.md)). T1 — headless contract tests against +the real binaries, `make test-agent-contracts` — is a later slice and is not started here. + +## Scope + +The managed hooks shipped by S3 wave 1 ([007](007-s3a-action.md), [009](009-s3b-action.md), +[011](011-s3c-action.md)) are a contract with eight external binaries, and three drifts found +while closing S3b (Droid's `droid exec` engine child, Codex 0.149.1's app-server EOF behavior, +Qoder's folder-trust gate on flag hooks) were invisible to the unit suite. T0 makes the +*version* side of that contract explicit and cheap to check: + +- an attestation record that says, per tier-A runtime, which version the contract last passed a + live sweep against, when, and where that is recorded; +- `make agent-versions`, which compares the binaries installed on this Mac with the record; +- the research matrix's tier-A baseline derived from the record instead of hand-edited. + +Non-goals: running any agent (T1), the optional scheduled npm/brew "latest" check from the issue +(nothing in the repo runs on a schedule yet; revisit with T1), and touching the interactive E2E +sweep. + +## The record + +`agent-attestation.json` (next to the research matrix), schema 1: + +| Key | Meaning | +| --- | --- | +| `schema` | `1`; the loader rejects anything else | +| `description` | free text for readers of the file | +| `runtimes[]` | one object per tier-A runtime, in the S3 order claude, codex, copilot, droid, qodercli, pi, omp, opencode | +| `runtimes[].runtime` | `AgentProfileRuntime` raw value (`qodercli`, not `qoder`) | +| `runtimes[].name` | display name | +| `runtimes[].binary` | executable looked up on PATH | +| `runtimes[].version_command` | argv that prints the version, e.g. `["claude", "--version"]`; the first element must be the binary | +| `runtimes[].attested_version` | the version the live sweep passed against; must parse as `MAJOR.MINOR.PATCH[-pre]` | +| `runtimes[].attested_on` | ISO date of that sweep | +| `runtimes[].record` | the `docs-ai/064` file that documents the sweep; must exist next to the JSON | + +Every key is required and no other key is allowed, so a typo fails `make test-scripts` rather +than silently going unread. Update rule until T1 exists: after a live sweep that passes for a +runtime, set its `attested_version` / `attested_on` / `record` by hand and run +`scripts/agent_versions.py --write-matrix`. T1's `make test-agent-contracts` is meant to do the +same on a passing run. + +### Attested versions and their provenance + +All eight entries point at the S3c live acceptance in [011-s3c-action.md](011-s3c-action.md), +the last sweep that exercised every tier-A runtime through a Prowl-launched Profile: Pi 0.84.3, +Oh My Pi 18.0.6, OpenCode 1.18.23, plus the "regression on the same build" row for Claude +2.1.245, Codex 0.149.1, Copilot 1.0.80, Droid 0.203.0, and Qoder 1.1.29 (PASS ×5). Two +ambiguities, resolved as follows: + +- 011 carries no explicit date. `attested_on` is 2026-08-26, the day PR #728 was opened with that + record (merged 2026-08-27); the same five versions had already passed 009's upgrade + re-verification on 2026-08-25, so the date is bounded either way. +- [013](013-idle-evidence-fallback.md) exercised Claude live again on 2026-08-28/29 but never + states the binary version, so Claude stays at the last explicitly verified 2.1.245 rather than + the 2.1.251 installed when this record was written. + +## Decisions + +| Decision | Rejected alternatives | +| --- | --- | +| One generic semantic-version parser (first `MAJOR.MINOR.PATCH[-pre]` token in stdout, else stderr, ANSI stripped) pinned by tests to the verbatim banner each of the eight CLIs printed on 2026-08-29. | Per-CLI parsers keyed by runtime: more code to maintain for no measured gain — every banner (`2.1.251 (Claude Code)`, `codex-cli 0.149.1`, `GitHub Copilot CLI 1.0.80.` + an update hint, `omp/18.0.6`, bare versions) yields the right token with one pattern, and the tests fail loudly if a banner changes. | +| Binary lookup on the process PATH first, then once on the PATH of `$SHELL -lic` for anything still missing; a version command runs with the PATH it was found on. | Process PATH only: `make` run from an editor or agent harness misses Homebrew and `mise` tools (measured on this Mac: a bare PATH plus `zsh -lc` sees 9 entries and no `/opt/homebrew/bin`; `zsh -lic` sees 34 with Homebrew and mise). `mise which` per binary: only covers mise, and none of the eight is mise-managed here. Login-only (`-lc`): skips `.zshrc`, which is where Homebrew's `shellenv` and `mise activate` live on a stock macOS setup. | +| A missing binary is a warning; nothing fails without `--strict`. `--strict` fails on any status but `attested`. | Failing on missing by default: a machine without every agent installed is normal, and the pre-release use is "look at the table", not a gate. | +| The matrix keeps its dated re-attestation paragraphs as history and gains one generated `**Tier-A attestation**` line; `--check-matrix` compares that line with the record and prints a per-runtime diff, `--write-matrix` regenerates it, and `make test-scripts` runs the check. | Rewriting the research paragraphs from the record: they are evidence about specific dates and would lose meaning. Deleting the line and pointing at the JSON only: readers of the matrix would have to open a second file for the one number they ask most often. | +| No `docs/` change: `make agent-versions` is a maintainer tool, so it is listed in `CLAUDE.md`'s build commands and here, not in the agent-facing manual. | — | + +## Delivered behavior + +- `docs-ai/064-agent-completion-signals/agent-attestation.json` — the record above. +- `scripts/agent_versions.py` + `make agent-versions` (`AGENT_VERSIONS_ARGS="--json"`, + `"--strict"`, `"--check-matrix"`, `"--write-matrix"`, `"--timeout N"`, `"--no-login-shell"`): + prints `runtime / attested / installed / status` with status one of `attested`, `newer`, + `older`, `missing`, `unparseable`; `newer` warns with a hint to run `make test-agent-contracts` + (T1) and update the record on a pass; `missing` and `unparseable` warn with the resolved path + or the first output line; the exit code is 0 unless `--strict`. `--json` emits, per runtime, + the attestation fields plus `path`, `resolution` (`path` / `login-shell`), `installed_version`, + `raw_output`, `status`, `detail`, and a `summary` count per status. +- `research-agent-completion-signals.md` — intro sentence naming the record as the source of + the tier-A versions, and the generated line. +- `scripts/test_agent_versions.py` (32 tests, run by `make test-scripts` and therefore `make + check`): parsers against the captured banners, comparison including prerelease ordering, the + record's shape and tier-A coverage, every status through a fake resolver/runner, the shell + PATH fallback through a stub shell, JSON and table shapes, strict exit codes, the matrix check + against the committed files and against tampered copies, and the script end to end with stub + binaries on a private PATH. + +## Verification + +- `make test-scripts`: 76 tests, OK (44 before this slice plus the 32 in `test_agent_versions.py`). +- `make agent-versions` on this Mac, 2026-08-29: + + ``` + runtime attested installed status + claude 2.1.245 2.1.251 newer + codex 0.149.1 0.149.1 attested + copilot 1.0.80 1.0.80 attested + droid 0.203.0 0.204.0 newer + qodercli 1.1.29 1.1.31 newer + pi 0.84.3 0.84.3 attested + omp 18.0.6 18.0.6 attested + opencode 1.18.23 1.18.23 attested + warning: claude 2.1.251 is newer than the attested 2.1.245 (011-s3c-action.md, 2026-08-26); the managed-hook contract is unverified against it — run `make test-agent-contracts` (#726 T1) and update docs-ai/064-agent-completion-signals/agent-attestation.json on a pass + warning: droid 0.204.0 is newer than the attested 0.203.0 (…) + warning: qodercli 1.1.31 is newer than the attested 1.1.29 (…) + ``` + + Three runtimes have moved past their attestation since the S3c sweep; that is exactly the + signal T0 exists to surface, and T1 is what clears it. +- `make agent-versions AGENT_VERSIONS_ARGS=--json`: summary `attested 5, newer 3, older 0, + missing 0, unparseable 0`; every binary resolved on the process PATH. +- `make agent-versions AGENT_VERSIONS_ARGS=--check-matrix`: `research-agent-completion-signals.md + matches agent-attestation.json`, exit 0. `--strict` exits 1 from the script (`make` reports 2). +- Shell fallback, with PATH reduced to `/usr/bin:/bin`: all eight resolved as `login-shell` + (`~/.local/bin` and `/opt/homebrew/bin`) with the same statuses; with `--no-login-shell` all + eight are `missing`. A login-only shell had found just the three under `~/.local/bin`, which is + what settled `-lic`. +- `make check`: format-changed (no Swift changes), format-lint, lint, test-scripts all pass. + +## What T1 builds on + +- The record is the place a passing `make test-agent-contracts` writes to: per runtime, set + `attested_version` to the version it just passed against, `attested_on` to today, and `record` + to the T1 record, then regenerate the matrix line. +- `scripts/agent_versions.py` exposes `load_attestation`, `parse_version`, `compare_versions`, + and `BinaryResolver`, so T1 can reuse the same binary lookup and version parsing to name the + exact build each contract ran against, and `--strict` gives a release gate once every runtime is + attested. +- The `newer` warning text already names T1's target so nothing has to change when it lands. + +## Observed but not changed + +- `/opt/homebrew/bin/droid` is a stale cask symlink to 0.134.0, shadowed by `~/.local/bin/droid` + 0.204.0 from Factory's installer. First-on-PATH wins here, as it does in a shell, so the report + shows 0.204.0; the cask is left alone. +- The release runbook has no pre-release checklist section to hang `make agent-versions` on; the + release skill and runbook are unchanged.