Something went wrong. Try again.
Hardware-in-the-loop development platform for the Commodore 64.
Something went wrong. Try again.
52 kB · 1249 lines
Python
at main
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250#!/usr/bin/env python3"""Build the accepted Rust-MOS fixtures from a frozen, checksummed build input.
The fixtures link against the C64 SDK crate that lives *inside* the repositoryworkspace, at ``crates/ultimate-tool-c64``. That crate inherits its version from``[workspace.package] version``, so bumping the workspace version changes thefixture artifacts even though no fixture source changed. The accepted PRGidentity therefore has to be rebuilt from an explicit frozen input rather thanfrom the moving host workspace.
This script composes a temporary build root that reproduces the accepted buildinputs exactly:
* ``<root>/Cargo.toml`` -- a minimal workspace pinned to the frozen version;* ``<root>/crates/ultimate-tool-c64`` -- the frozen SDK manifest and source copied from ``fixtures/accepted-sdk`` (the original relative path, so the fixtures resolve their dependency unchanged);* ``<root>/fixtures/<fixture>`` -- a copy of the fixture's own sources, linker script, build script, manifest, lock file and cargo config.
It then runs the pinned rust-mos container image over that root with the exactcompiler invocation the fixtures have always used, stages the PRG/ELF/map,refuses to publish unless the PRG hash matches the recorded accepted hash, andonly then publishes atomically into the fixture's ``build/`` directory.
The script invokes ``cargo`` directly inside the container. It must never call``make build``: the fixture Makefiles are thin wrappers around this script, soshelling back out to them would recurse.
Only the Python standard library and the ``docker`` CLI are used. The scriptnever follows a symlink, never writes outside its temporary root plus thefixture's ``build/`` directory, never mounts the host home directory or anycredential store, and never changes the host workspace version.
Usage::
python3 tools/build_accepted_fixture.py --fixture rust-mos-smoke python3 tools/build_accepted_fixture.py --fixture rust-mos-smoke --mode verify-debug python3 tools/build_accepted_fixture.py --fixture rust-mos-timeline --mode verify-reproducible python3 tools/build_accepted_fixture.py --fixture rust-mos-smoke --json
Exit codes: 0 success, 1 build/verification failure, 2 refused input."""
from __future__ import annotations
import argparseimport hashlibimport jsonimport osimport reimport shutilimport statimport subprocessimport sysimport tempfileimport timefrom pathlib import Pathfrom typing import Any, Iterable, Mapping, Sequence
SCHEMA = 1INPUT_RECORD_KIND = "accepted_fixture_build_input"REPORT_RECORD_KIND = "accepted_fixture_build"
REPOSITORY_ROOT = Path(__file__).resolve().parent.parentDEFAULT_CONTRACT = REPOSITORY_ROOT / "fixtures/accepted-sdk/accepted-sdk.json"DEFAULT_WORK_PARENT = REPOSITORY_ROOT / "target/accepted-fixture-build"DEFAULT_CACHE_DIR = REPOSITORY_ROOT / "target/accepted-fixture-cargo-home"DEFAULT_DOCKER = "docker"DEFAULT_TIMEOUT_SECONDS = 900.0TEARDOWN_TIMEOUT_SECONDS = 30.0
CONTAINER_ROOT = "/work"CONTAINER_CARGO_HOME = "/tmp/ut-cargo"CONTAINER_FROZEN_VERSION = "0.1.0"
# Names that mark a directory as a host cargo home or credential store. The# tool must never mount one of these into the container, and must never treat# one as its own build cache.CREDENTIAL_OR_HOST_CARGO_MARKERS = ( "config.toml", "config", "credentials", "credentials.toml", "credentials.json", ".credentials", ".netrc", "auth.toml",)
# Only the cargo registry cache is kept when the project-local cache is pruned.CACHE_KEEP_NAMES = ("registry",)
MODES = ("build", "verify-debug", "verify-reproducible")
# Fixture inputs copied into the composed root, in a fixed order. Anything not# listed here (a stray target directory, an evidence directory, an editor# backup) is deliberately not copied.FIXTURE_INPUT_DIRS = ("src", ".cargo")FIXTURE_INPUT_FILES = ("Cargo.toml", "Cargo.lock", "build.rs", "link.ld")
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
class Refused(Exception): """An input was rejected before any build was attempted."""
class BuildFailed(Exception): """The build ran but did not produce the accepted artifact."""
def sha256_file(path: Path) -> str: """Hash one regular file, refusing to follow a symlink.""" digest = hashlib.sha256() with open_file_nofollow(path, "rb") as handle: for chunk in iter(lambda: handle.read(1 << 20), b""): digest.update(chunk) return digest.hexdigest()
def open_file_nofollow(path: Path, mode: str): """Open ``path`` refusing symlinks and non-regular files.""" flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) try: descriptor = os.open(path, flags) except OSError as exc: raise Refused(f"cannot open {path} without following links: {exc}") from exc try: info = os.fstat(descriptor) if not stat.S_ISREG(info.st_mode): raise Refused(f"{path} is not a regular file") except BaseException: os.close(descriptor) raise raw = os.fdopen(descriptor, mode) return raw
def ensure_within(root: Path, candidate: Path) -> Path: """Refuse a path that escapes ``root`` after normalisation.""" root = root.resolve() resolved = Path(os.path.normpath(candidate)) if resolved != root and root not in resolved.parents: raise Refused(f"{candidate} is outside the approved root {root}") return resolved
def reject_symlink_chain(root: Path, target: Path, what: str) -> Path: """Refuse a target outside ``root`` or reachable only through a symlink.
Guards the directories the tool creates, mounts and replaces files in, so a symlinked cache, build directory or destination artifact can never redirect a write (or a container mount) outside the repository. """ root = Path(os.path.abspath(root)) target = Path(os.path.abspath(target)) try: relative = target.relative_to(root) except ValueError: raise Refused(f"{what} {target} is outside the repository root {root}") from None current = root for part in relative.parts: current = current / part if current.is_symlink(): raise Refused(f"{what} {target} traverses the symlink {current}") return target
def reject_host_cargo_or_credentials(directory: Path, what: str) -> None: """Refuse a directory that looks like a host cargo home or credential store.""" lowered = directory.name.lower() for marker in (".cargo", ".docker", ".aws", ".config", "credentials"): if lowered == marker: raise Refused(f"{what} {directory} must not be a host {marker} directory") for marker in CREDENTIAL_OR_HOST_CARGO_MARKERS: candidate = directory / marker if candidate.exists() or candidate.is_symlink(): raise Refused( f"{what} {directory} contains {marker}; refusing to mount a cargo " "configuration or credential store into the container" )
def prepare_project_directory(path: Path, repository_root: Path, what: str) -> Path: """Validate, then create, a project-local directory used by the build.""" reject_symlink_chain(repository_root, path, what) reject_host_cargo_or_credentials(path, what) if path.is_symlink(): raise Refused(f"{what} {path} must not be a symlink") path.mkdir(parents=True, exist_ok=True) if path.is_symlink(): raise Refused(f"{what} {path} must not be a symlink") return path
def prune_cache(cache_dir: Path) -> list[str]: """Remove everything in the project-local cache except the cargo registry.""" removed = [] if not cache_dir.is_dir() or cache_dir.is_symlink(): return removed for entry in sorted(os.scandir(cache_dir), key=lambda item: item.name): if entry.name in CACHE_KEEP_NAMES: continue if entry.is_dir(follow_symlinks=False): shutil.rmtree(entry.path, ignore_errors=True) else: try: os.unlink(entry.path) except OSError: continue removed.append(entry.name) return removed
def read_json(path: Path) -> Any: try: with open_file_nofollow(path, "rb") as handle: text = handle.read().decode("utf-8") except OSError as exc: raise Refused(f"cannot read {path}: {exc}") from exc try: return json.loads(text) except json.JSONDecodeError as exc: raise Refused(f"{path} is not valid JSON: {exc}") from exc
def write_json_atomic(path: Path, document: Any) -> None: """Replace ``path`` with ``document`` using a rename in the same directory.""" payload = json.dumps(document, indent=2, sort_keys=False) + "\n" temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}") with open(temporary, "w", encoding="utf-8") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path)
# ---------------------------------------------------------------------------# Contract
def load_contract(path: Path) -> dict: contract = read_json(path) if not isinstance(contract, dict): raise Refused(f"{path} must contain a JSON object") if contract.get("schema") != SCHEMA: raise Refused(f"{path} must declare schema {SCHEMA}") if contract.get("record_kind") != INPUT_RECORD_KIND: raise Refused(f"{path} must declare record_kind {INPUT_RECORD_KIND!r}") for key in ("sdk", "workspace", "toolchain", "fixtures"): if not isinstance(contract.get(key), dict): raise Refused(f"{path} is missing the {key!r} object") sdk = contract["sdk"] for key in ("name", "version", "source_dir", "workspace_path", "files"): if key not in sdk: raise Refused(f"{path} sdk is missing {key!r}") if sdk["version"] != CONTAINER_FROZEN_VERSION: raise Refused( f"the frozen SDK version must stay {CONTAINER_FROZEN_VERSION}, " f"found {sdk['version']!r}" ) for name, entry in sdk["files"].items(): if not SHA256_RE.match(str(entry)): raise Refused(f"{path} sdk file {name!r} is not a sha256 digest") workspace = contract["workspace"] if workspace.get("version") != CONTAINER_FROZEN_VERSION: raise Refused( "the frozen workspace version must stay " f"{CONTAINER_FROZEN_VERSION}, found {workspace.get('version')!r}" ) if list(workspace.get("members", [])) != [sdk["workspace_path"]]: raise Refused( f"the frozen workspace must list only {sdk['workspace_path']!r} as a member" ) for name, fixture in contract["fixtures"].items(): for key in ( "package", "working_directory", "binary", "artifacts", "artifact_sha256", "artifact_size", "sources", ): if key not in fixture: raise Refused(f"{path} fixture {name!r} is missing {key!r}") if not SHA256_RE.match(str(fixture["artifact_sha256"])): raise Refused(f"{path} fixture {name!r} artifact_sha256 is not a digest") for source, digest in fixture["sources"].items(): if not SHA256_RE.match(str(digest)): raise Refused( f"{path} fixture {name!r} source {source!r} is not a digest" ) toolchain = contract["toolchain"] for key in ("image", "image_id", "target", "cargo_args"): if key not in toolchain: raise Refused(f"{path} toolchain is missing {key!r}") return contract
def host_workspace_version(path: Path) -> str | None: """Read ``[workspace.package] version`` from the host manifest, if present.
The value is recorded so the artifact identity can be explained later; it is deliberately not an input to the fixture build. """ try: with open_file_nofollow(path, "rb") as handle: text = handle.read().decode("utf-8") except (OSError, Refused): return None section = None for line in text.splitlines(): stripped = line.strip() if stripped.startswith("["): section = stripped.strip("[]") continue if section == "workspace.package": match = re.match(r'^version\s*=\s*"([^"]+)"', stripped) if match: return match.group(1) return None
# ---------------------------------------------------------------------------# Input verification
def verify_frozen_sdk(contract: Mapping[str, Any], repository_root: Path = REPOSITORY_ROOT) -> list[dict]: source_dir = repository_root / contract["sdk"]["source_dir"] checks = [] for relative, expected in sorted(contract["sdk"]["files"].items()): path = source_dir / relative if path.is_symlink(): raise Refused(f"the frozen SDK input {path} must not be a symlink") if not path.is_file(): raise Refused(f"the frozen SDK input {path} is missing") observed = sha256_file(path) matches = observed == expected checks.append( {"check": "frozen_sdk_file", "path": str(path), "expected": expected, "observed": observed, "ok": matches} ) if not matches: raise Refused( f"the frozen SDK input {path} changed: expected {expected}, " f"found {observed}. The accepted SDK bytes are the frozen input; " "restore them or re-accept deliberately instead of editing the pins." ) return checks
def verify_fixture_sources(fixture_name: str, fixture: Mapping[str, Any], repository_root: Path = REPOSITORY_ROOT) -> list[dict]: directory = repository_root / fixture["working_directory"] checks = [] drifted = [] for relative, expected in sorted(fixture["sources"].items()): path = directory / relative if path.is_symlink(): raise Refused(f"fixture input {path} must not be a symlink") if not path.is_file(): raise Refused(f"fixture input {path} is missing") observed = sha256_file(path) matches = observed == expected checks.append( {"check": "fixture_source", "fixture": fixture_name, "path": str(path), "expected": expected, "observed": observed, "ok": matches} ) if not matches: drifted.append((relative, expected, observed)) if drifted: detail = ", ".join(f"{name} {old} -> {new}" for name, old, new in drifted) raise Refused( f"fixture {fixture_name} source drift detected: {detail}. " "The recorded accepted artifact belongs to the pinned sources; a real " "fixture change invalidates it. Re-run with --update to record the new " "pins, which requires re-acceptance of the artifact and does not " "authorise this build as accepted." ) return checks
# ---------------------------------------------------------------------------# Composed root
def render_workspace_manifest(contract: Mapping[str, Any]) -> str: workspace = contract["workspace"] members = ", ".join(json.dumps(item) for item in workspace["members"]) excluded = ", ".join(json.dumps(item) for item in workspace["exclude"]) lints = workspace["lints"] rust_lints = [] for name, value in lints["rust"].items(): if isinstance(value, dict): level = json.dumps(value["level"]) check_cfg = ", ".join(json.dumps(item) for item in value.get("check-cfg", [])) rust_lints.append( f'{name} = {{ level = {level}, check-cfg = [{check_cfg}] }}' ) else: rust_lints.append(f"{name} = {json.dumps(value)}") clippy_lints = [f"{name} = {json.dumps(value)}" for name, value in lints["clippy"].items()] lines = [ "# Generated by tools/build_accepted_fixture.py from the frozen build input.", "# This workspace exists only inside the temporary build root; it pins the", "# SDK version so the fixtures link against their accepted identity.", "", "[workspace]", f"members = [{members}]", f"exclude = [{excluded}]", 'resolver = "2"', "", "[workspace.package]", f'edition = {json.dumps(workspace["edition"])}', f'version = {json.dumps(workspace["version"])}', f'license = {json.dumps(workspace["license"])}', "", "[workspace.lints.rust]", *rust_lints, "", "[workspace.lints.clippy]", *clippy_lints, "", ] return "\n".join(lines)
def copy_file_nofollow(source: Path, destination: Path) -> None: if source.is_symlink(): raise Refused(f"{source} must not be a symlink") info = os.lstat(source) if not stat.S_ISREG(info.st_mode): raise Refused(f"{source} is not a regular file") destination.parent.mkdir(parents=True, exist_ok=True) with open_file_nofollow(source, "rb") as reader, open(destination, "wb") as writer: shutil.copyfileobj(reader, writer, 1 << 20)
def copy_tree_nofollow(source: Path, destination: Path, copied: list[Path]) -> None: if source.is_symlink(): raise Refused(f"{source} must not be a symlink") for entry in sorted(os.scandir(source), key=lambda item: item.name): target = destination / entry.name if entry.is_symlink(): raise Refused(f"{entry.path} must not be a symlink") if entry.is_dir(follow_symlinks=False): target.mkdir(parents=True, exist_ok=True) copy_tree_nofollow(Path(entry.path), target, copied) elif entry.is_file(follow_symlinks=False): copy_file_nofollow(Path(entry.path), target) copied.append(target) else: raise Refused(f"{entry.path} is neither a regular file nor a directory")
def compose_root(root: Path, contract: Mapping[str, Any], fixture_name: str, repository_root: Path = REPOSITORY_ROOT) -> dict: """Populate the temporary build root with frozen inputs only.""" fixture = contract["fixtures"][fixture_name] sdk_source = repository_root / contract["sdk"]["source_dir"] sdk_target = root / contract["sdk"]["workspace_path"] copied: list[Path] = []
manifest = root / "Cargo.toml" manifest.write_text(render_workspace_manifest(contract), encoding="utf-8") copied.append(manifest)
for relative in sorted(contract["sdk"]["files"]): copy_file_nofollow(sdk_source / relative, sdk_target / relative) copied.append(sdk_target / relative)
fixture_source = repository_root / fixture["working_directory"] copied.append(fixture_source / "Cargo.toml") for name in FIXTURE_INPUT_FILES: candidate = fixture_source / name if candidate.exists(): copy_file_nofollow(candidate, root / fixture["working_directory"] / name) copied.append(root / fixture["working_directory"] / name) for name in FIXTURE_INPUT_DIRS: candidate = fixture_source / name if candidate.is_dir(): copy_tree_nofollow( candidate, root / fixture["working_directory"] / name, copied )
working_directory = root / fixture["working_directory"] (working_directory / "build").mkdir(parents=True, exist_ok=True) return { "root": root, "working_directory": working_directory, "files": copied, "workspace_manifest": manifest, }
# ---------------------------------------------------------------------------# Container execution
def run_docker(argv: Sequence[str], echo: bool, timeout: float = TEARDOWN_TIMEOUT_SECONDS) -> subprocess.CompletedProcess: """Run a short, bounded docker command such as ``image inspect``.""" if echo: print("+ " + " ".join(argv), file=sys.stderr) try: return subprocess.run(list(argv), capture_output=True, text=True, timeout=timeout) except subprocess.TimeoutExpired as exc: raise BuildFailed( f"{' '.join(argv[:3])} exceeded its {timeout:.0f}s deadline" ) from exc
class Unresolved(BuildFailed): """A container could not be confirmed gone, so the build root was retained.
Raised instead of claiming a clean teardown. The composed root is kept on disk and named in the message so the operator can inspect, stop or remove the container by hand. """
def __init__(self, message: str, retained_root: Path | None = None) -> None: super().__init__(message) self.retained_root = retained_root
class ContainerRunner: """Builds and runs the pinned container with a name, a deadline and teardown.
Every execution allocates exactly one container name and one argv, and both are recorded once. On a timeout, an interrupt or a spawn error the container is stopped and removed and its absence is *checked*; a container that cannot be confirmed gone keeps the composed root and fails as ``Unresolved`` rather than reporting a clean stop. """
def __init__(self, contract: Mapping[str, Any], docker: str, root: Path, fixture: Mapping[str, Any], cache_dir: Path, timeout: float = DEFAULT_TIMEOUT_SECONDS, echo: bool = False) -> None: if timeout <= 0: raise Refused("the container deadline must be a positive number of seconds") self.contract = contract self.docker = docker self.root = root self.fixture = fixture self.cache_dir = cache_dir self.timeout = timeout self.echo = echo self.executions: list[dict] = [] self._index = 0
def _next_name(self) -> str: self._index += 1 return f"ultimate-tool-accepted-{self.fixture['package']}-{os.getpid()}-{self._index}"
def container_command(self, command: Sequence[str]) -> tuple[str, list[str]]: """The exact container invocation. Only two volumes are ever mounted.""" name = self._next_name() argv = [ self.docker, "run", "--rm", "--name", name, "--user", f"{os.getuid()}:{os.getgid()}", "--env", f"CARGO_HOME={CONTAINER_CARGO_HOME}", "--volume", f"{self.cache_dir}:{CONTAINER_CARGO_HOME}", "--volume", f"{self.root}:{CONTAINER_ROOT}", "--workdir", f"{CONTAINER_ROOT}/{self.fixture['working_directory']}", self.contract["toolchain"]["image"], *command, ] return name, argv
def plan(self, command: Sequence[str]) -> list[str]: """Build the argv for a planned run without executing or recording it.""" return self.container_command(command)[1]
def _spawn(self, argv: Sequence[str], timeout: float) -> subprocess.CompletedProcess: return subprocess.run(list(argv), capture_output=True, text=True, timeout=timeout)
def container_present(self, name: str) -> bool | None: """True/False when the runtime answered, None when it could not be asked.""" try: result = self._spawn( [self.docker, "ps", "-aq", "--filter", f"name=^{name}$"], TEARDOWN_TIMEOUT_SECONDS, ) except (subprocess.TimeoutExpired, OSError, KeyboardInterrupt): return None if result.returncode != 0: return None return bool(result.stdout.strip())
def teardown(self, name: str) -> dict: """Stop and remove a container, then confirm it is really gone.
An interruption during stop, remove or the confirmation query never escapes: the outcome is recorded and an unconfirmed container is reported as present, so the caller retains the build root instead of deleting it under a container that may still be running. """ report: dict[str, Any] = { "name": name, "stop_ok": False, "remove_ok": False, "absent": False, "container_state_known": False, "interrupted": False, "errors": [], } for key, arguments in (("stop", ["stop", "--time", "5", name]), ("remove", ["rm", "-f", name])): try: result = self._spawn([self.docker, *arguments], TEARDOWN_TIMEOUT_SECONDS) except KeyboardInterrupt: report["interrupted"] = True report["errors"].append(f"{key}: interrupted") continue except (subprocess.TimeoutExpired, OSError) as error: report["errors"].append(f"{key}: {type(error).__name__}") continue if result.returncode == 0: report[f"{key}_ok"] = True else: report["errors"].append( f"{key}: exit {result.returncode} {result.stderr.strip()[:160]}" ) present = self.container_present(name) report["container_state_known"] = present is not None report["absent"] = present is False return report
def _record(self, name: str, argv: Sequence[str], state: str) -> dict: entry = {"name": name, "argv": list(argv), "state": state} self.executions.append(entry) return entry
def execute(self, command: Sequence[str], what: str) -> subprocess.CompletedProcess: """Run one container command, allocating and recording its name exactly once.""" name, argv = self.container_command(command) entry = self._record(name, argv, "running") if self.echo: print("+ " + " ".join(argv), file=sys.stderr) try: result = self._spawn(argv, self.timeout) except (subprocess.TimeoutExpired, KeyboardInterrupt, OSError) as error: entry["state"] = "failed" self._fail_after_teardown(entry, name, what, error) raise # unreachable: _fail_after_teardown always raises entry["state"] = "exited" entry["exit_code"] = result.returncode return result
def _fail_after_teardown(self, entry: dict, name: str, what: str, error: BaseException): report = self.teardown(name) entry["teardown"] = report detail = "; ".join(report["errors"]) or "the absence check did not answer" if isinstance(error, subprocess.TimeoutExpired): description = f"exceeded the {self.timeout:.0f}s deadline" else: description = type(error).__name__ if not report["absent"]: raise Unresolved( f"{what} failed ({description}) and container {name} could not be " f"confirmed gone after stop and remove ({detail}); the build root " f"{self.root} was retained and no artifact was published", self.root, ) from error if isinstance(error, KeyboardInterrupt): raise error raise BuildFailed( f"{what} failed ({description}); container {name} was stopped and " "removed (absence confirmed) and nothing was published" ) from error
def executed_commands(self) -> list[list[str]]: return [list(item["argv"]) for item in self.executions]
def executed_names(self) -> list[str]: return [item["name"] for item in self.executions]
def image_identity(docker: str, image: str, contract: Mapping[str, Any]) -> str: result = run_docker( [docker, "image", "inspect", "--format", "{{.Id}}", image], echo=False ) if result.returncode != 0: raise Refused( f"cannot inspect the pinned image {image}: " f"{result.stderr.strip() or result.stdout.strip()}" ) observed = result.stdout.strip() expected = contract["toolchain"]["image_id"] if observed != expected: raise Refused( f"the pinned image {image} is {observed}, expected {expected}" ) return observed
def cargo_args(contract: Mapping[str, Any], fixture: Mapping[str, Any], extra: Sequence[str] = (), target_directory: str | None = None) -> list[str]: base = list(contract["toolchain"]["cargo_args"]) if target_directory is not None: index = base.index("--target-dir") + 1 base[index] = target_directory return base + list(extra)
# ---------------------------------------------------------------------------# Output handling
def stage_outputs(composed: Mapping[str, Any], fixture: Mapping[str, Any]) -> dict: """Collect PRG/ELF/map out of the composed root into a staging directory.""" working_directory = composed["working_directory"] stage = composed["root"] / "stage" stage.mkdir(parents=True, exist_ok=True)
binary = working_directory / fixture["binary"] if not binary.is_file(): raise BuildFailed(f"the build did not produce {binary}")
deps = binary.parent / "deps" pattern = fixture.get("deps_elf_glob") elves = sorted(deps.glob(pattern)) if pattern else [] elves = [item for item in elves if item.is_file() and not item.is_symlink()] if len(elves) != 1: raise BuildFailed( f"expected exactly one freshly linked fixture ELF matching {pattern}, " f"found {len(elves)}" )
outputs = {} for key, relative in fixture["artifacts"].items(): name = Path(relative).name destination = stage / name if key == "prg": copy_file_nofollow(binary, destination) elif key == "elf": copy_file_nofollow(elves[0], destination) else: map_path = working_directory / relative if not map_path.is_file(): raise BuildFailed(f"the build did not produce the linker map {map_path}") copy_file_nofollow(map_path, destination) outputs[key] = destination
prg = outputs["prg"] size = os.lstat(prg).st_size limit = int(fixture.get("max_artifact_bytes", 0) or 0) if limit and size > limit: raise BuildFailed(f"{prg} is {size} bytes, above the {limit} byte bound") return { "stage": stage, "outputs": outputs, "prg_sha256": sha256_file(prg), "prg_size": size, "elf": outputs["elf"], }
def validate_artifact(staged: Mapping[str, Any], fixture: Mapping[str, Any]) -> dict: expected = fixture["artifact_sha256"] observed = staged["prg_sha256"] size_expected = int(fixture["artifact_size"]) size_observed = staged["prg_size"] checks = [ {"check": "artifact_sha256", "expected": expected, "observed": observed, "ok": observed == expected}, {"check": "artifact_size", "expected": size_expected, "observed": size_observed, "ok": size_observed == size_expected}, ] if observed != expected or size_observed != size_expected: raise BuildFailed( "the rebuilt artifact is not the accepted artifact: " f"sha256 {observed} (expected {expected}), size {size_observed} " f"(expected {size_expected}). Nothing was published; the previously " "accepted build outputs, if any, were left untouched." ) return {"checks": checks}
def preflight_publish_destinations(fixture: Mapping[str, Any], repository_root: Path = REPOSITORY_ROOT ) -> tuple[Path, list[tuple[str, Path]]]: """Validate every destination before anything is replaced.
The fixture directory, the ``build`` directory and each destination artifact are checked for symlinks (and for being outside the repository) up front, so a single bad entry cannot leave a half-replaced output directory behind. """ fixture_dir = repository_root / fixture["working_directory"] reject_symlink_chain(repository_root, fixture_dir, "fixture directory") destination_dir = fixture_dir / "build" reject_symlink_chain(repository_root, destination_dir, "fixture build directory") if destination_dir.is_symlink(): raise Refused(f"the fixture build directory {destination_dir} must not be a symlink") destinations = [] for key, relative in sorted(fixture["artifacts"].items()): destination = destination_dir / Path(relative).name try: info = os.lstat(destination) except FileNotFoundError: info = None if info is not None: if stat.S_ISLNK(info.st_mode): raise Refused( f"refusing to replace {destination}: it is a symlink, which would " "redirect the published artifact outside the fixture" ) if not stat.S_ISREG(info.st_mode): raise Refused(f"refusing to replace {destination}: not a regular file") destinations.append((key, destination)) return destination_dir, destinations
def publish(staged: Mapping[str, Any], fixture: Mapping[str, Any], repository_root: Path = REPOSITORY_ROOT) -> list[dict]: """Atomically replace the fixture's published build outputs.""" destination_dir, destinations = preflight_publish_destinations(fixture, repository_root) destination_dir.mkdir(parents=True, exist_ok=True) published = [] for key, destination in destinations: source = staged["outputs"][key] temporary = destination.with_name(f".{destination.name}.tmp-{os.getpid()}") with open_file_nofollow(source, "rb") as reader, open(temporary, "wb") as writer: shutil.copyfileobj(reader, writer, 1 << 20) writer.flush() os.fsync(writer.fileno()) os.replace(temporary, destination) published.append( {"key": key, "path": str(destination), "sha256": sha256_file(destination), "size": os.lstat(destination).st_size} ) return published
# ---------------------------------------------------------------------------# Modes
def elf_section_check(runner: "ContainerRunner", elf: Path, sections: Iterable[str]) -> dict: """Assert the ELF carries the given sections, read inside the container.""" container_elf = f"{CONTAINER_ROOT}/{elf.relative_to(runner.root)}" listing = runner.execute(["llvm-readelf", "--sections", container_elf], f"llvm-readelf on {elf.name}") if listing.returncode != 0: raise BuildFailed( f"cannot read sections of {elf.name}: {listing.stderr.strip()}" ) missing = [name for name in sections if name not in listing.stdout] if missing: raise BuildFailed(f"{elf.name} is missing the sections {', '.join(missing)}") return {"check": "elf_sections", "elf": elf.name, "present": list(sections)}
def run_build(runner: "ContainerRunner", contract: Mapping[str, Any], fixture: Mapping[str, Any]) -> None: command = cargo_args(contract, fixture) result = runner.execute(command, "the fixture build") if result.returncode != 0: raise BuildFailed( f"cargo build failed for {fixture['package']} (exit {result.returncode}):\n" f"{(result.stderr or result.stdout).strip()[-2000:]}" )
def _stage_and_validate(contract: Mapping[str, Any], fixture: Mapping[str, Any], composed: Mapping[str, Any]) -> dict: staged = stage_outputs(composed, fixture) staged["stage_checks"] = validate_artifact(staged, fixture)["checks"] return staged
def mode_build(runner, contract, fixture, composed): run_build(runner, contract, fixture) staged = _stage_and_validate(contract, fixture, composed) return {"staged": staged, "extra_checks": []}
def mode_verify_debug(runner, contract, fixture, composed): run_build(runner, contract, fixture) staged = _stage_and_validate(contract, fixture, composed)
working = composed["working_directory"] # The crash-site expectation is per fixture: the smoke fixture carries exactly # one crash_here! invocation, the timeline fixture carries none. expected_sites = int(fixture.get("crash_here_count", 0)) with open_file_nofollow(working / "src/main.rs", "rb") as handle: text = handle.read().decode("utf-8", "replace") observed_sites = text.count("crash_here!") if observed_sites != expected_sites: raise BuildFailed( f"expected exactly {expected_sites} crash_here! invocation(s) in the " f"fixture source, found {observed_sites}" )
extra: list[dict] = [] if fixture.get("debug_build_compare", False): debug_off = cargo_args( contract, fixture, extra=["--config", "profile.release.debug=0", "--config", "profile.release.strip=true"], target_directory="target/verify-release", ) trim = cargo_args( contract, fixture, extra=["--config", 'profile.release.debug="line-tables-only"', "--config", "profile.release.strip=false"], target_directory="target/verify-lines", ) for command in (debug_off, trim): result = runner.execute(command, "a debug verification build") if result.returncode != 0: raise BuildFailed( f"cargo build failed for {fixture['package']}: " f"{(result.stderr or result.stdout).strip()[-2000:]}" ) plain = working / fixture["binary"].replace("target/line-tables", "target/verify-release", 1) trimmed = working / fixture["binary"].replace("target/line-tables", "target/verify-lines", 1) if sha256_file(plain) != sha256_file(trimmed): raise BuildFailed( "the debug-off and line-table builds produced different binaries" ) extra.append({"check": "debug_builds_match", "ok": True}) elf_dir = trimmed.parent / "deps" else: elf_dir = (working / fixture["binary"]).parent / "deps"
elves = sorted(item for item in elf_dir.glob(fixture["deps_elf_glob"]) if item.is_file() and not item.is_symlink()) if len(elves) != 1: raise BuildFailed( f"expected exactly one freshly linked fixture ELF, found {len(elves)}" ) extra.append(elf_section_check(runner, elves[0], fixture.get("debug_sections", [".debug_line"]))) return {"staged": staged, "extra_checks": extra}
def mode_verify_reproducible(runner, contract, fixture, composed): run_build(runner, contract, fixture) staged = _stage_and_validate(contract, fixture, composed)
second = cargo_args(contract, fixture, target_directory="target/reproduce") result = runner.execute(second, "the reproducibility rebuild") if result.returncode != 0: raise BuildFailed( f"the reproducibility rebuild failed: " f"{(result.stderr or result.stdout).strip()[-2000:]}" ) rebuilt = composed["working_directory"] / fixture["binary"].replace( "target/line-tables", "target/reproduce", 1 ) if sha256_file(rebuilt) != staged["prg_sha256"]: raise BuildFailed( "the two independent builds produced different artifacts; the fixture " "build is not reproducible in this environment" ) return {"staged": staged, "extra_checks": [{"check": "reproducible_rebuild", "ok": True}]}
MODE_HANDLERS = { "build": mode_build, "verify-debug": mode_verify_debug, "verify-reproducible": mode_verify_reproducible,}
# ---------------------------------------------------------------------------# Driver
def build_fixture( *, fixture_name: str, mode: str, contract_path: Path = DEFAULT_CONTRACT, work_parent: Path = DEFAULT_WORK_PARENT, cache_dir: Path = DEFAULT_CACHE_DIR, docker: str = DEFAULT_DOCKER, repository_root: Path = REPOSITORY_ROOT, timeout: float = DEFAULT_TIMEOUT_SECONDS, dry_run: bool = False, keep_work: bool = False, keep_cache: bool = False, echo: bool = False,) -> dict: contract = load_contract(contract_path) if fixture_name not in contract["fixtures"]: raise Refused( f"unknown fixture {fixture_name!r}; known fixtures are " f"{', '.join(sorted(contract['fixtures']))}" ) if mode not in MODES: raise Refused(f"unknown mode {mode!r}; expected one of {', '.join(MODES)}") fixture = contract["fixtures"][fixture_name]
checks = verify_frozen_sdk(contract, repository_root) checks.extend(verify_fixture_sources(fixture_name, fixture, repository_root))
prepare_project_directory(work_parent, repository_root, "the temporary work parent") prepare_project_directory(cache_dir, repository_root, "the project-local cargo cache")
root = Path(tempfile.mkdtemp(prefix=f"{fixture_name}-", dir=work_parent)) os.chmod(root, 0o700) composed = compose_root(root, contract, fixture_name, repository_root) runner = ContainerRunner(contract, docker, root, fixture, cache_dir, timeout=timeout, echo=echo)
record: dict[str, Any] = { "schema": SCHEMA, "record_kind": REPORT_RECORD_KIND, "fixture": fixture_name, "mode": mode, "status": "planned" if dry_run else "built", "composed_root": str(root), "sdk": { "name": contract["sdk"]["name"], "version": contract["sdk"]["version"], "workspace_path": contract["sdk"]["workspace_path"], "source_dir": contract["sdk"]["source_dir"], }, "frozen_workspace_version": contract["workspace"]["version"], "host_workspace_version": host_workspace_version(repository_root / "Cargo.toml"), "toolchain": { "image": contract["toolchain"]["image"], "image_id": contract["toolchain"]["image_id"], "target": contract["toolchain"]["target"], "timeout_seconds": timeout, }, "containers": [], "checks": checks, "commands": [], "artifacts": {}, "published": [], "problems": [], }
if dry_run: record["commands"] = [runner.plan(cargo_args(contract, fixture))] record["planned_removal"] = str(root) shutil.rmtree(root, ignore_errors=True) return record
retain_root = False try: image_identity(docker, contract["toolchain"]["image"], contract) outcome = MODE_HANDLERS[mode](runner, contract, fixture, composed) record["commands"] = runner.executed_commands() record["containers"] = runner.executed_names() record["checks"].extend(outcome["staged"]["stage_checks"]) record["checks"].extend(outcome["extra_checks"]) staged = outcome["staged"] staged_elf = staged["elf"] record["checks"].append( {"check": "staged_elf_sha256", "observed": sha256_file(staged_elf), "size": os.lstat(staged_elf).st_size, "ok": True} ) record["artifacts"] = { "prg": { "path": str(staged["outputs"]["prg"]), "sha256": staged["prg_sha256"], "size": staged["prg_size"], "expected_sha256": fixture["artifact_sha256"], "matches_expected": staged["prg_sha256"] == fixture["artifact_sha256"], }, "elf": {"path": str(staged["outputs"]["elf"]), "sha256": sha256_file(staged["outputs"]["elf"]), "size": os.lstat(staged["outputs"]["elf"]).st_size}, "map": {"path": str(staged["outputs"]["map"]), "sha256": sha256_file(staged["outputs"]["map"]), "size": os.lstat(staged["outputs"]["map"]).st_size}, } record["published"] = publish(staged, fixture, repository_root) record["status"] = mode except Unresolved as error: retain_root = True record["containers"] = runner.executed_names() record["executions"] = runner.executions record["status"] = "unresolved" record["retained_root"] = str(error.retained_root or root) record["problems"] = [str(error)] setattr(error, "record", record) raise except (Refused, BuildFailed) as error: record["containers"] = runner.executed_names() record["executions"] = runner.executions record["status"] = "refused" if isinstance(error, Refused) else "failed" record["problems"] = [str(error)] setattr(error, "record", record) raise finally: if retain_root: print( f"unresolved: the composed root {root} was retained; no artifact was " "published and no container was confirmed removed", file=sys.stderr, ) elif not keep_work: shutil.rmtree(root, ignore_errors=True) if not keep_cache: record["cache_pruned"] = prune_cache(cache_dir) return record
def update_fixture_pins(fixture_name: str, contract_path: Path = DEFAULT_CONTRACT) -> dict: """Record the current fixture source hashes, explicitly not accepting them.""" contract = load_contract(contract_path) if fixture_name not in contract["fixtures"]: raise Refused(f"unknown fixture {fixture_name!r}") fixture = contract["fixtures"][fixture_name] directory = REPOSITORY_ROOT / fixture["working_directory"] previous = dict(fixture["sources"]) updated = {} for relative in previous: candidate = directory / relative if candidate.is_symlink() or not candidate.is_file(): raise Refused(f"cannot pin {candidate}: not a regular file") updated[relative] = sha256_file(candidate) fixture["sources"] = updated write_json_atomic(contract_path, contract) changed = {name: (previous[name], updated[name]) for name in updated if previous[name] != updated[name]} return { "schema": SCHEMA, "record_kind": "accepted_fixture_build_pin_update", "fixture": fixture_name, "contract": str(contract_path), "changed": changed, "requires_reacceptance": bool(changed), "note": ( "The accepted artifact hash was NOT changed. Any fixture source change " "requires re-acceptance of the fixture artifact before this build can be " "reported as accepted." ), }
def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser( description="Build an accepted Rust-MOS fixture from frozen, checked inputs." ) parser.add_argument("--fixture", required=True, help="fixture directory name, for example rust-mos-smoke") parser.add_argument("--mode", default="build", choices=MODES) parser.add_argument("--contract", default=str(DEFAULT_CONTRACT)) parser.add_argument("--work-parent", default=str(DEFAULT_WORK_PARENT), help="approved directory that holds temporary build roots") parser.add_argument("--cache-dir", default=str(DEFAULT_CACHE_DIR), help="project-local cargo cache mounted as CARGO_HOME") parser.add_argument("--docker", default=DEFAULT_DOCKER) parser.add_argument( "--timeout", type=float, default=DEFAULT_TIMEOUT_SECONDS, help="wall-clock deadline in seconds for each container command", ) parser.add_argument("--json", action="store_true", help="emit one JSON record") parser.add_argument("--dry-run", action="store_true", help="compose the root and print the container command only") parser.add_argument("--keep-work", action="store_true", help="keep the temporary build root for inspection") parser.add_argument("--keep-cache", action="store_true", help="keep the whole project-local cargo cache instead of " "pruning everything except the registry") parser.add_argument("--echo-commands", action="store_true", help="print container commands to stderr while running") parser.add_argument("--update", action="store_true", help="record current fixture source pins; requires re-acceptance") arguments = parser.parse_args(argv)
contract_path = Path(arguments.contract).resolve() try: if arguments.update: record = update_fixture_pins(arguments.fixture, contract_path) else: record = build_fixture( fixture_name=arguments.fixture, mode=arguments.mode, contract_path=contract_path, work_parent=Path(os.path.abspath(arguments.work_parent)), cache_dir=Path(os.path.abspath(arguments.cache_dir)), docker=arguments.docker, timeout=arguments.timeout, dry_run=arguments.dry_run, keep_work=arguments.keep_work, keep_cache=arguments.keep_cache, echo=arguments.echo_commands, ) except Refused as error: return _fail("refused", str(error), arguments.json, getattr(error, "record", None)) except BuildFailed as error: return _fail("failed", str(error), arguments.json, getattr(error, "record", None))
if arguments.json: print(json.dumps(record, indent=2)) else: _print_human(record) return 0
def _fail(kind: str, message: str, as_json: bool, record: dict | None = None) -> int: if as_json: payload = dict(record) if record else { "schema": SCHEMA, "record_kind": REPORT_RECORD_KIND, } payload["status"] = kind payload["problems"] = [message] print(json.dumps(payload, indent=2)) else: print(f"{kind}: {message}", file=sys.stderr) if record and record.get("retained_root"): print(f"retained root: {record['retained_root']}", file=sys.stderr) return 2 if kind == "refused" else 1
def _print_human(record: Mapping[str, Any]) -> None: print(f"{record['record_kind']} schema {record['schema']}") print(f" fixture: {record.get('fixture')}") if "sdk" not in record: for name, change in (record.get("changed") or {}).items(): print(f" pin change: {name} {change[0]} -> {change[1]}") print(f" requires re-accept: {record.get('requires_reacceptance')}") return print(f" mode/status: {record.get('mode')} / {record.get('status')}") print(f" frozen SDK: {record['sdk']['name']} {record['sdk']['version']} " f"at {record['sdk']['workspace_path']}") print(f" frozen version: {record['frozen_workspace_version']}") print(f" host version: {record.get('host_workspace_version')}") artifacts = record.get("artifacts") or {} if artifacts: prg = artifacts["prg"] print(f" PRG sha256: {prg['sha256']}") print(f" PRG size: {prg['size']}") print(f" matches expected: {prg['matches_expected']}") for item in record.get("published", []): print(f" published: {item['key']} -> {item['path']}")
if __name__ == "__main__": raise SystemExit(main())