jetstream v2 in zig stream.waow.tech
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365#!/usr/bin/env bash# Exact-artifact admission for Stream.## The previous whole-network experiment deployed image e1926f3, which was not# the artifact its receipt covered. Nothing detected that. This binds one# receipt to one image digest and gives deployment a way to refuse anything# else.## scripts/admit build build the exact linux/amd64 image, no publish# scripts/admit run clean tree -> suites -> image -> receipt# scripts/admit publish [sha] push the admitted image and pin the receipt# to the registry digest (default: HEAD)# scripts/admit verify <digest> exit non-zero unless <digest> is admitted## Design rules, learned from the failure this replaces:# - the receipt records the artifact's own digest, never a tag; tags move# - a skipped suite is written into the receipt as "skipped", never omitted,# so a partial run cannot read as a full one# - any suite failure aborts before a receipt exists# - the tree must be clean, so the receipt's revisions describe the source# that actually produced the binaryset -euo pipefailcd "$(dirname "${BASH_SOURCE[0]}")/.."ROOT="$PWD"RECEIPTS="${STREAM_RECEIPTS_DIR:-$ROOT/receipts}"IMAGE_REPO="${STREAM_IMAGE_REPO:-atcr.io/zat.dev/stream}"# Registry auth is the caller's problem: `publish` needs a docker login for# atcr.io in the ambient DOCKER_CONFIG (CI's, or one you set explicitly).# No keychain fallback — workstation keychains must never authenticate this.die() { printf 'admit: %s\n' "$*" >&2; exit 1; }# stderr, not stdout: build_image returns the image id on stdout, and progress# lines printed there end up captured inside the receipt's recorded identity.note() { printf '\033[1m==> %s\033[0m\n' "$*" >&2; }# Receipts are outputs, not source. A run writes its own receipt into the repo,# so counting it as drift would make every completed run look dirty.tree_drift() { git status --porcelain -- ':!receipts'; }require_clean_tree() { [ -z "$(tree_drift)" ] || die "worktree is dirty; the receipt must describe the source that built the artifact$(tree_drift)"}# Every pinned input that can change behavior. A receipt that does not name# these cannot be checked later.revisions_json() { python3 - "$ROOT" <<'PY'import json, os, re, subprocess, sys, pathlibroot = pathlib.Path(sys.argv[1])zon = (root / "build.zig.zon").read_text()deps = dict(re.findall(r'\.(\w[\w-]*) = \.\{\s*\.url = "([^"]+)"', zon))def git(*a): return subprocess.check_output(["git", *a], cwd=root, text=True).strip()# Respect STREAM_UPSTREAM_REPO exactly as the justfile does. Hardcoding the# workstation layout meant a runner that clones upstream elsewhere recorded# "upstream_jetstream": null — the receipt silently stops naming a pinned# input it exists to pin, and nothing fails.upstream = pathlib.Path( os.environ.get("STREAM_UPSTREAM_REPO") or (pathlib.Path.home() / "github.com/bluesky-social/jetstream"))try: upstream_rev = subprocess.check_output( ["git", "rev-parse", "HEAD"], cwd=upstream, text=True).strip() upstream_dirty = bool(subprocess.check_output( ["git", "status", "--porcelain", "--untracked-files=no"], cwd=upstream, text=True).strip())except Exception: upstream_rev, upstream_dirty = None, Noneprint(json.dumps({ "stream": git("rev-parse", "HEAD"), "stream_describe": git("describe", "--always", "--dirty"), "zig": subprocess.check_output(["zig", "version"], text=True).strip(), "dependencies": deps, "upstream_jetstream": upstream_rev, "upstream_jetstream_dirty": upstream_dirty, "dashboard_sha256": __import__("hashlib").sha256( (root / "deploy/grafana/jetstream-upstream.json").read_bytes()).hexdigest(),}, indent=2, sort_keys=True))PY}# name|command. Negative suites come first: they are the ones that decide# whether a green positive run means anything.# The two simulator-backed oracles run FIRST. The pinned simulator accumulates# world state for as long as it is up, and a full admission run is long enough# to grow it by orders of magnitude; running them late meant they faced a world# that made bootstrap outlast the harness's fixed serving timeout. Everything# after them is simulator-independent.SUITES=( "lifecycle-oracle|just oracle" "powerloss-oracle|just powerloss-oracle" "unit-debug|zig build test" "unit-releasesafe|zig build test -Doptimize=ReleaseSafe" "differential-oracle|just differential-oracle" "listener-contract|just listener-contract" "shutdown-contract|just shutdown-contract" "logging-contract|just logging-contract" "environment-contract|just environment-contract" "dashboard-test|just dashboard-test" "process-metrics-contract|just process-metrics-contract" "http-metrics-contract|just http-metrics-contract" "archive-contract|just archive-contract" "plan-config-contract|just plan-config-contract" "cursor-lookback-contract|just cursor-lookback-contract" "compaction-config-contract|just compaction-config-contract" "retry-config-contract|just retry-config-contract" "subscribe-config-contract|just subscribe-config-contract" "subscribe-read-batch-contract|just subscribe-read-batch-contract" "status-contract|just status-contract" "seam-handoff-contract|just seam-handoff-contract")run_suites() { local out="$1" log_dir="$2" first=1 mkdir -p "$log_dir" printf '{\n' >"$out" for entry in "${SUITES[@]}"; do local name="${entry%%|*}" cmd="${entry#*|}" status started elapsed # STREAM_ADMIT_SKIP is an escape hatch for an environment that genuinely # cannot run a suite. It is recorded, not hidden. if [[ ",${STREAM_ADMIT_SKIP:-}," == *",$name,"* ]]; then status="skipped" note "SKIP $name (STREAM_ADMIT_SKIP)" else note "$name" started=$SECONDS if (cd "$ROOT" && eval "$cmd") >"$log_dir/$name.log" 2>&1; then status="pass" else status="fail" fi elapsed=$((SECONDS - started)) if [ "$status" = fail ]; then tail -30 "$log_dir/$name.log" >&2 die "$name failed after ${elapsed}s; no receipt written" fi printf ' %s (%ss)\n' "$status" "$elapsed" fi [ $first -eq 1 ] || printf ',\n' >>"$out" first=0 printf ' "%s": "%s"' "$name" "$status" >>"$out" done printf '\n}\n' >>"$out"}# Build the deploy artifact. STREAM_BUILD_HOST points at a native linux/amd64# docker daemon (ssh://root@host); emulating amd64 on an arm64 workstation# turns an 8-minute compile into 40-plus and produces a cross-emulated# approximation of the thing that actually ships. When a remote is used the# image is transferred back so the push uses local registry credentials and the# build host never receives them.build_image() { local tag="$1" if [ -n "${STREAM_BUILD_HOST:-}" ]; then # Deliberately not DOCKER_HOST=ssh://. That keeps one SSH-carried Docker API # stream open for the whole compile, and a multi-minute build drops it # ("broken pipe"), losing the run. Ship the context once, build in its own # session with keepalives, then stream the image back. The push still runs # locally, so the build host never receives registry credentials. local host key host="${STREAM_BUILD_HOST#ssh://}" key="${STREAM_SSH_KEY:-$HOME/.ssh/waow_ed25519}" local ssh_opts=(-i "$key" -o IdentitiesOnly=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=10 -o ConnectTimeout=10) note "shipping build context to $host" # .git is part of the context: the Dockerfile copies it so the binary can # report the commit it was built from. tar -C "$ROOT" -czf - --exclude=.zig-cache --exclude=zig-out --exclude=receipts . | ssh "${ssh_opts[@]}" "$host" 'rm -rf /opt/streambuild && mkdir -p /opt/streambuild && tar -C /opt/streambuild -xzf -' note "building linux/amd64 image $tag natively on $host (detached)" # Keepalives do not save a build from a dropped session: ssh SIGHUPs the # remote docker build and the whole compile is lost (a publish on # 2026-08-08 and a full gate on 2026-08-09, both to transient network # blips). Detach the build from the session and poll for its exit code, so # connectivity to the box is only required at poll instants. # --network=host: the box's buildkit build network cannot resolve DNS # (NameServerFailure on every dependency fetch); the host stack can. ssh "${ssh_opts[@]}" "$host" "cd /opt/streambuild && rm -f build.log build.status && setsid sh -c 'docker build --network=host --platform linux/amd64 -t \"$tag\" . >build.log 2>&1; echo \$? >build.status' </dev/null >/dev/null 2>&1 &" || die "could not start remote build on $host" local status="" waited=0 step="" while [ -z "$status" ]; do sleep 15 waited=$((waited + 15)) [ "$waited" -lt 2700 ] || die "remote build exceeded 45m; see $host:/opt/streambuild/build.log" # a poll that cannot reach the box is a blip, not a build failure status=$(ssh "${ssh_opts[@]}" "$host" "cat /opt/streambuild/build.status 2>/dev/null" 2>/dev/null || true) if [ -z "$status" ] && [ $((waited % 60)) -eq 0 ]; then local now now=$(ssh "${ssh_opts[@]}" "$host" "grep -E '^Step ' /opt/streambuild/build.log | tail -1" 2>/dev/null || true) [ -z "$now" ] || [ "$now" = "$step" ] || { step="$now"; note " $step"; } fi done if [ "$status" != "0" ]; then ssh "${ssh_opts[@]}" "$host" "tail -40 /opt/streambuild/build.log" >&2 || true die "remote build failed (exit $status)" fi ssh "${ssh_opts[@]}" "$host" "docker image inspect '$tag' >/dev/null" || die "remote build did not produce $tag" note "transferring image to the local daemon (checksummed)" # Never stream docker save through the ssh pipe unverified: a dropped # stream yields a stub archive whose load "succeeds" via content-store # dedupe while layers are missing, and the failure surfaces later as # push NotFound (2026-08-08, the fd9fe7f 6.6KB archive). ssh "${ssh_opts[@]}" "$host" "docker save '$tag' > /opt/streambuild/image.tar && sha256sum /opt/streambuild/image.tar" > /tmp/stream-image.sum scp -q "${ssh_opts[@]}" "$host:/opt/streambuild/image.tar" /tmp/stream-image.tar local want got want=$(awk '{print $1}' /tmp/stream-image.sum) got=$(shasum -a 256 /tmp/stream-image.tar | awk '{print $1}') [ "$want" = "$got" ] || die "image transfer corrupted: box sha $want != local sha $got" docker load < /tmp/stream-image.tar >&2 ssh "${ssh_opts[@]}" "$host" "rm -f /opt/streambuild/image.tar" rm -f /tmp/stream-image.tar /tmp/stream-image.sum docker image inspect --format '{{.Id}}' "$tag" return fi note "building linux/amd64 image $tag" docker build --platform linux/amd64 -t "$tag" "$ROOT" >&2 # The local image ID is the artifact's identity before it is pushed. The # registry digest replaces it at publish time. docker image inspect --format '{{.Id}}' "$tag"}cmd_build() { require_clean_tree local sha; sha=$(git rev-parse --short HEAD) build_image "$IMAGE_REPO:$sha"}cmd_run() { require_clean_tree local sha sha_full stamp receipt log_dir suites_json revisions image_id tag sha=$(git rev-parse --short HEAD) sha_full=$(git rev-parse HEAD) stamp=$(date -u +%Y%m%dT%H%M%SZ) log_dir="$RECEIPTS/logs-$sha-$stamp" suites_json=$(mktemp) # The trap fires after this function returns, when the local is already out # of scope; without the default it aborts under `set -u`. trap 'rm -f "${suites_json:-}"' EXIT run_suites "$suites_json" "$log_dir" # STREAM_ADMIT_NO_BUILD exists so a runner can exercise this path cheaply # (e.g. one suite, no 8-minute native image build) while proving the real # code runs. It deliberately writes NO receipt: a run that never built an # image has nothing to bind suites to, and `verify` must never see one. if [ -n "${STREAM_ADMIT_NO_BUILD:-}" ]; then note "STREAM_ADMIT_NO_BUILD set: suites ran, no image built, no receipt written" return 0 fi tag="$IMAGE_REPO:$sha" image_id=$(build_image "$tag") revisions=$(revisions_json) # Re-check: a suite run can take long enough for someone to touch the tree. require_clean_tree # A commit leaves the tree clean, so the check above cannot see it -- but the # tag was fixed at entry while revisions_json() re-reads HEAD, so committing # mid-run yields a receipt whose recorded revision is not the one the image # was tagged for. That happened, and it is the same shape as deploying an # image no receipt covers. Pin the revision to where the run started. [ "$(git rev-parse HEAD)" = "$sha_full" ] || die "HEAD moved during the run: started at $sha_full now at $(git rev-parse HEAD)the receipt would describe a different revision than the image it names.re-run from a stable HEAD" mkdir -p "$RECEIPTS" receipt="$RECEIPTS/$sha.json" # Revisions travel through a file, never interpolated into the program text: # JSON's `false`/`null` are not Python literals, which turned a completed run # into a NameError after every suite had already passed. local revisions_file revisions_file=$(mktemp) printf '%s\n' "$revisions" >"$revisions_file" python3 - "$receipt" "$image_id" "$tag" "$stamp" "$suites_json" "$revisions_file" <<'PY'import json, sysreceipt, image_id, tag, stamp, suites_path, revisions_path = sys.argv[1:7]json.dump({ "schema": "stream-admission/1", "created_utc": stamp, "image": {"tag": tag, "local_id": image_id, "registry_digest": None}, "revisions": json.load(open(revisions_path)), "suites": json.load(open(suites_path)),}, open(receipt, "w"), indent=2, sort_keys=True)open(receipt, "a").write("\n")PY rm -f "$revisions_file" note "receipt $receipt" note "image not published; run 'scripts/admit publish' to pin its registry digest"}cmd_publish() { # Deliberately no clean-tree check: the receipt already records the revisions # its artifact was built from, so source state now is irrelevant. The real # guard is that the local image still matches the identity the receipt covers. local sha receipt tag digest sha="${1:-$(git rev-parse --short HEAD)}" receipt="$RECEIPTS/$sha.json" [ -f "$receipt" ] || die "no receipt for $sha; run 'scripts/admit run' first" tag=$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1]))["image"]["tag"])' "$receipt") # The image must still be the one the receipt covers. local recorded current recorded=$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1]))["image"]["local_id"])' "$receipt") current=$(docker image inspect --format '{{.Id}}' "$tag") [ "$recorded" = "$current" ] || die "local image $tag is $current but the receipt covers $recorded; rerun 'admit run'" note "pushing $tag" ATCR_AUTO_AUTH=1 docker push "$tag" >&2 digest=$(docker image inspect --format '{{index .RepoDigests 0}}' "$tag" | cut -d@ -f2) [ -n "$digest" ] || die "could not read the pushed digest" python3 - "$receipt" "$digest" <<'PY'import json, sysreceipt, digest = sys.argv[1], sys.argv[2]data = json.load(open(receipt))if data["image"]["registry_digest"] not in (None, digest): raise SystemExit(f"admit: receipt already pinned to {data['image']['registry_digest']}")data["image"]["registry_digest"] = digestjson.dump(data, open(receipt, "w"), indent=2, sort_keys=True)open(receipt, "a").write("\n")PY note "admitted $digest"}# The deployment-side gate. Anything not carrying an admitted digest is# refused, which is exactly what was missing when e1926f3 shipped.cmd_verify() { local digest="${1:-}" [ -n "$digest" ] || die "usage: scripts/admit verify <sha256:...>" python3 - "$RECEIPTS" "$digest" <<'PY'import json, pathlib, sysreceipts, digest = pathlib.Path(sys.argv[1]), sys.argv[2]for path in sorted(receipts.glob("*.json")): data = json.load(open(path)) if data.get("image", {}).get("registry_digest") != digest: continue suites = data.get("suites", {}) bad = {k: v for k, v in suites.items() if v != "pass"} if bad: print(f"REJECT {digest}: {path.name} has non-passing suites: {bad}") raise SystemExit(1) print(f"ADMITTED {digest} ({path.name}, stream {data['revisions']['stream'][:12]})") raise SystemExit(0)print(f"REJECT {digest}: no receipt admits this digest")raise SystemExit(1)PY}case "${1:-}" in build) shift; cmd_build "$@" ;; run) shift; cmd_run "$@" ;; publish) shift; cmd_publish "$@" ;; verify) shift; cmd_verify "$@" ;; *) die "usage: scripts/admit {build|run|publish [sha]|verify <digest>}" ;;esac