diff --git a/toolchain/fleet/deploy-worker.sh b/toolchain/fleet/deploy-worker.sh index 0b28de3d5c..437784c518 100755 --- a/toolchain/fleet/deploy-worker.sh +++ b/toolchain/fleet/deploy-worker.sh @@ -29,17 +29,18 @@ REMOTE_TMP="$(ssh -o BatchMode=yes "$HOST" 'mktemp -d /tmp/ac-fleet-worker.XXXXX cleanup() { ssh -o BatchMode=yes -o ConnectTimeout=5 "$HOST" \ - "rm -f '$REMOTE_TMP/worker.mjs' '$REMOTE_TMP/install-worker.sh' '$REMOTE_TMP/performance-guard.sh' '$REMOTE_TMP/token'; rmdir '$REMOTE_TMP' 2>/dev/null || true" \ + "rm -f '$REMOTE_TMP/worker.mjs' '$REMOTE_TMP/install-worker.sh' '$REMOTE_TMP/performance-guard.sh' '$REMOTE_TMP/performance_guard.py' '$REMOTE_TMP/git-guard.sh' '$REMOTE_TMP/swift-guard.sh' '$REMOTE_TMP/build-lock.sh' '$REMOTE_TMP/token'; rmdir '$REMOTE_TMP' 2>/dev/null || true" \ >/dev/null 2>&1 || true } trap cleanup EXIT HUP INT TERM scp -q "$HERE/worker.mjs" "$HERE/install-worker.sh" \ - "$REPO/toolchain/macos/performance-guard.sh" "$TOKEN" "$HOST:$REMOTE_TMP/" + "$REPO/toolchain/macos/performance-guard.sh" "$REPO/toolchain/macos/performance_guard.py" \ + "$REPO/toolchain/macos/git-guard.sh" "$REPO/toolchain/macos/swift-guard.sh" \ + "$REPO/slab/bin/build-lock.sh" "$TOKEN" "$HOST:$REMOTE_TMP/" ssh -o BatchMode=yes "$HOST" \ "mkdir -p '$REMOTE_HOME/.local/lib/ac-fleet-worker'; \ - install -m 755 '$REMOTE_TMP/performance-guard.sh' '$REMOTE_HOME/.local/lib/ac-fleet-worker/performance-guard.sh'; \ - env AC_REPO='$REMOTE_REPO' bash '$REMOTE_HOME/.local/lib/ac-fleet-worker/performance-guard.sh' --install; \ + env AC_REPO='$REMOTE_REPO' bash '$REMOTE_TMP/performance-guard.sh' --install && \ bash '$REMOTE_TMP/install-worker.sh' --name '$NAME' --role '$ROLE' --token-file '$REMOTE_TMP/token'" echo "deployed fleet worker to $NAME ($HOST, $ROLE)" diff --git a/toolchain/macos/PERFORMANCE-GUARD.md b/toolchain/macos/PERFORMANCE-GUARD.md index 10841239ee..6b32cf6265 100644 --- a/toolchain/macos/PERFORMANCE-GUARD.md +++ b/toolchain/macos/PERFORMANCE-GUARD.md @@ -1,29 +1,68 @@ # AC performance guard -`performance-guard.sh` samples the prompt host every 30 seconds. It records a -full snapshot when load, memory pressure, display rendering, swapping, session -count, or duplicate AC Caddy servers exceed their budgets. Three consecutive -pressure samples produce a rate-limited macOS notification. +The guard samples each Mac every 30 seconds and publishes +`~/.local/share/slab/performance/pressure-active`. It watches CPU load, memory +availability, swap activity, display CPU, session/build counts, and **20 GiB of +free disk headroom**. Fleet workers defer new missions while pressure is active. -While a threshold is active, the guard exposes -`~/.local/share/slab/performance/pressure-active` as a stable eco-governor -signal for Slab renderers and other local tools. +The installed PATH shims check live disk, memory, and load before `git worktree +add` or `swift build`, including builds outside the AC checkout. The check covers +both the home volume and the destination volume. Recent sampler signals such as +swap activity also defer launches. Refusal exits **75** before starting the job; +callers must check that result rather than continuing a recovery sequence. -The fleet worker uses the same flag as an admission gate. An active render is -allowed to finish, but that host accepts no additional missions until pressure -clears. This turns the guard into a decentralized capacity signal without -giving it authority to migrate or kill arbitrary processes. - -The guard never terminates arbitrary hot processes. Its one repair is narrowly -scoped: when multiple processes have both the exact `caddy run --config -Caddyfile` command and `system/` working directory, it keeps the newest and -removes the duplicates. +Swift builds retain per-package serialization and the existing low-priority, +two-job default on hosts with at most 16 GiB. Admission is checked again after +waiting for the package lock. Other Git and Swift commands pass through. ```bash toolchain/macos/performance-guard.sh --install -npm run perf:status -npm run perf:audit +ac-performance-guard --status +ac-performance-guard --admit 'my expensive job' /path/to/output +python3 toolchain/macos/deploy-performance-guard.py --local neo panda chicken frisbee poorslice ``` -State and threshold-crossing history live in -`~/.local/share/slab/performance/`. The log rotates at 5 MB. +The installer owns `~/.local/lib/ac-performance-guard/` and links `git`, `swift`, +and `ac-performance-guard` into `~/.local/bin`. That directory must precede the +real commands on PATH; fleet deployment verifies the user's login-shell routing. +It refuses to overwrite unrelated command wrappers. The deployment tool preserves +the fleet’s existing portable-Git launcher as `git.before-ac-guard` and invokes +it underneath the new gate; uninstall restores it. Other wrappers require +explicit inspection and `--install --preserve-git-wrapper`. The guard requires Python +3.9 or later and macOS command-line tools; it does not build or install packages. + +Deployment packages the exact current **committed revision**, verifies installed +file hashes, launchd registration, a new sample, and command routing. It preserves +remote repository edits. Failed hosts are retried every five minutes for up to +24 hours using the same pinned bundle. The private receipt and pending list are +in `~/.local/share/slab/performance-rollout/receipt.json`. Re-run the deployment +command to begin a new rollout after that window. + +`latest.json` and the compatible `latest.txt` contain current readings. Pressure +history is stored in `performance-guard.log` and `incidents.jsonl`, each bounded +to about 5 MiB. Incident records include top processes, parent chains, and up to +eight Git processes with start time, sanitized command, and working directory. +They stay local with owner-only permissions. Shell/agent command arguments and +process environments are not collected; Git config values, messages, and URLs +are redacted. Sampling locks release automatically when a process dies. Atomic +writes preserve the last complete sample if the filesystem fills. + +Three consecutive pressure samples trigger a rate-limited notification. Critical +memory or less than 5 GiB disk headroom can notify immediately. The guard never +kills arbitrary workloads. Its optional `--repair` action sends SIGTERM only to +validated duplicate `caddy run --config Caddyfile` processes in this repo's +`system/` directory. + +This is admission control, not a memory or disk quota. It cannot stop a job that +was already running, predict the size of an entire checkout, intercept absolute +Git/Swift paths, or govern unrelated build tools such as Xcode and npm. Use sparse +worktrees from the start and keep large builds on compute hosts. An explicit +`AC_PERFORMANCE_ALLOW_PRESSURE=1` bypass is available for deliberate recovery; +normal automation must not set it. Failed live measurements defer work. Stale +sampler files do not keep blocking after live headroom recovers. + +Validation (no compiler or real workload runs): + +```bash +python3 -B toolchain/macos/test_performance_guard.py +``` diff --git a/toolchain/macos/SCORE.md b/toolchain/macos/SCORE.md index c6e2edc8e4..8cc1ed3d32 100644 --- a/toolchain/macos/SCORE.md +++ b/toolchain/macos/SCORE.md @@ -169,6 +169,10 @@ passes, and it is never included in the weekly LaunchAgent. Use [Neo cleanup](NEO-CLEANUP.md) for the verified worktree and local Photos procedures, recovery records, and the host's daily schedule. +The [AC performance guard](PERFORMANCE-GUARD.md) checks disk headroom every +30 seconds and gates new shell-launched Git worktrees and Swift builds. Install +it on every fleet Mac; deployment verifies the PATH shims as well as launchd. + Keep at least 20 GiB free on Neo for swap and system updates. Its hourly `disk-space-watch` checks only filesystem free space and warns at most once per day; it never deletes data. Install it with diff --git a/toolchain/macos/deploy-performance-guard.py b/toolchain/macos/deploy-performance-guard.py new file mode 100644 index 0000000000..b9af55e2db --- /dev/null +++ b/toolchain/macos/deploy-performance-guard.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Deploy a committed, small guard bundle without pulling a host's dirty repo. + +Usage: deploy-performance-guard.py --local neo panda chicken frisbee poorslice +Unreachable targets remain in a private receipt for --retry-pending. +""" +import concurrent.futures +import hashlib +import json +import os +from pathlib import Path +import plistlib +import shlex +import subprocess +import sys +import tempfile +import time + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent.parent +STATE = Path.home() / ".local/share/slab/performance-rollout" +LABEL = "computer.aesthetic.performance-guard-rollout" +FILES = ["performance-guard.sh", "performance_guard.py", "git-guard.sh", "swift-guard.sh"] + +# Only this bounded installer is executed remotely; hostnames are SSH argv, +# never shell interpolation. The bundle has an exact filename allowlist. +INSTALL = r''' +import hashlib,json,os,pathlib,subprocess,sys,tempfile +bundle=json.load(sys.stdin) +allowed={'performance-guard.sh','performance_guard.py','git-guard.sh','swift-guard.sh','build-lock.sh','revision'} +if set(bundle['files'])!=allowed:raise ValueError('unexpected bundle files') +home=pathlib.Path.home() +env={**os.environ,'PATH':str(home/'.local/bin')+':/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin'} +with tempfile.TemporaryDirectory(prefix='ac-guard-') as temp: + for name,content in bundle['files'].items(): + p=pathlib.Path(temp)/name;p.write_text(content);p.chmod(0o700) + flags=[] + previous=home/'.local/bin/git' + # Preserve the inspected portable-Git launcher used by older fleet setups. + if previous.is_file() and not previous.is_symlink(): + text=previous.read_text() + if 'export GIT_EXEC_PATH=' in text and 'export GIT_TEMPLATE_DIR=' in text and 'exec "$HOME/' in text: + flags=['--preserve-git-wrapper'] + r=subprocess.run(['/bin/bash',temp+'/performance-guard.sh','--install',*flags],env=env,capture_output=True,text=True,timeout=25) + if r.returncode:raise RuntimeError(r.stderr or r.stdout) +target=home/'.local/lib/ac-performance-guard' +hashes={name:hashlib.sha256((target/name).read_bytes()).hexdigest() for name in bundle['files']} +expected={name:hashlib.sha256(content.encode()).hexdigest() for name,content in bundle['files'].items()} +if hashes!=expected:raise RuntimeError('installed content mismatch') +guard=str(target/'performance-guard.sh') +r=subprocess.run(['/bin/bash',guard,'--once'],env=env,capture_output=True,text=True,timeout=20) +if r.returncode:raise RuntimeError(r.stderr or 'sampler failed') +state=home/'.local/share/slab/performance/latest.json' +import time +for attempt in range(20): + try: + sample=json.loads(state.read_text()) + if sample.get('revision')==bundle['revision']:break + except (OSError,ValueError):pass + time.sleep(0.25) +else:raise RuntimeError('no sample from deployed revision') +loaded=subprocess.run(['launchctl','print',f'gui/{os.getuid()}/computer.aesthetic.performance-guard'],capture_output=True).returncode==0 +if not loaded:raise RuntimeError('launch agent not loaded') +# Check resolution in the user's actual login shell, not just installer PATH. +shell=os.environ.get('SHELL','/bin/zsh') +paths=subprocess.run([shell,'-lc','command -v git; command -v swift'],capture_output=True,text=True,timeout=8) +resolved=paths.stdout.strip().splitlines() +expected_paths=[str(home/'.local/bin/git'),str(home/'.local/bin/swift')] +if resolved!=expected_paths:raise RuntimeError('shell PATH bypasses guards: '+repr(resolved)) +probe=subprocess.run(['/bin/bash',guard,'--admit','deployment verification'],env=env,capture_output=True,text=True,timeout=8) +if probe.returncode not in (0,75):raise RuntimeError('admission probe failed') +print(json.dumps({'revision':bundle['revision'],'loaded':loaded,'paths':resolved,'sample':{k:sample[k] for k in ['timestamp','version','revision','reason','disk_free_bytes']},'admission_exit':probe.returncode,'admission_message':probe.stderr.strip(),'verified_files':len(hashes)})) +''' + + +def save(path, data): + temp = path.with_suffix(".next") + temp.write_text(json.dumps(data, indent=2) + "\n") + temp.chmod(0o600) + os.replace(temp, path) + + +def deploy(host, bundle): + command = [sys.executable, "-c", INSTALL] if host == "--local" else [ + "ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", host, + "python3 -c " + shlex.quote(INSTALL)] + try: + result = subprocess.run(command, input=json.dumps(bundle), capture_output=True, text=True, timeout=65) + if result.returncode: + return {"host": host, "ok": False, "error": result.stderr[-1800:] or result.stdout[-1800:]} + return {"host": host, "ok": True, **json.loads(result.stdout)} + except (OSError, ValueError, subprocess.TimeoutExpired) as error: + return {"host": host, "ok": False, "error": str(error)} + + +def schedule_retry(): + # Pin both the bundle and retry script. A future repository edit cannot + # silently change the already-authorized rollout. + script = STATE / "deploy-performance-guard.py" + if Path(__file__).resolve() != script: + script.write_text(Path(__file__).read_text()) + script.chmod(0o700) + plist = Path.home() / "Library/LaunchAgents" / (LABEL + ".plist") + plist.parent.mkdir(parents=True, exist_ok=True) + config = {"Label": LABEL, "ProgramArguments": [sys.executable, str(script), "--retry-pending"], + "StartInterval": 300, "ProcessType": "Background", "LowPriorityIO": True, + "EnvironmentVariables": {"PATH": f"{Path.home()}/.local/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"}, + "StandardOutPath": str(STATE / "retry.out"), "StandardErrorPath": str(STATE / "retry.err")} + plist.write_bytes(plistlib.dumps(config)) + subprocess.run(["launchctl", "bootout", f"gui/{os.getuid()}/{LABEL}"], capture_output=True) + subprocess.run(["launchctl", "bootstrap", f"gui/{os.getuid()}", str(plist)], check=True) + + +def main(args): + STATE.mkdir(parents=True, exist_ok=True) + os.chmod(STATE, 0o700) + receipt_path = STATE / "receipt.json" + retry = args == ["--retry-pending"] + if retry: + receipt = json.loads(receipt_path.read_text()) + hosts = receipt["pending"] + if not hosts or time.time() > receipt["retry_until"]: + (Path.home() / "Library/LaunchAgents" / (LABEL + ".plist")).unlink(missing_ok=True) + subprocess.run(["launchctl", "bootout", f"gui/{os.getuid()}/{LABEL}"], capture_output=True) + return 0 + bundle = json.loads((STATE / "bundle.json").read_text()) + else: + hosts = args + if not hosts or any(h != "--local" and (h.startswith("-") or not all(c.isalnum() or c in "@._-" for c in h)) for h in hosts): + raise SystemExit("usage: deploy-performance-guard.py [--local] HOST ... | --retry-pending") + revision = subprocess.check_output(["git", "-C", str(ROOT), "rev-parse", "HEAD"], text=True).strip() + paths = ["toolchain/macos/" + name for name in FILES] + ["slab/bin/build-lock.sh"] + # git show selects the exact committed release, even in a dirty checkout. + contents = {Path(path).name: subprocess.check_output(["git", "-C", str(ROOT), "show", revision + ":" + path], text=True) for path in paths} + contents["revision"] = revision + "\n" + bundle = {"revision": revision, "files": contents} + save(STATE / "bundle.json", bundle) + receipt = {"revision": revision, "results": {}, "pending": hosts, "retry_until": time.time() + 86400} + # Small script transfers only, at most three hosts concurrently. + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: + results = list(pool.map(lambda host: deploy(host, bundle), hosts)) + for result in results: + receipt["results"][result["host"]] = result + print(json.dumps(result), flush=True) + receipt["pending"] = [h for h, r in receipt["results"].items() if not r["ok"]] + receipt["updated_at"] = time.time() + save(receipt_path, receipt) + if receipt["pending"] and not retry: + schedule_retry() + return 0 if not receipt["pending"] else 75 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/toolchain/macos/git-guard.sh b/toolchain/macos/git-guard.sh new file mode 100644 index 0000000000..9daef821e2 --- /dev/null +++ b/toolchain/macos/git-guard.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# AC performance guard Git shim. All commands except worktree add pass through. +set -euo pipefail +source_path="${BASH_SOURCE[0]}" +while [[ -L "$source_path" ]]; do + source_dir="$(cd "$(dirname "$source_path")" && pwd)" + source_path="$(readlink "$source_path")" + [[ "$source_path" == /* ]] || source_path="$source_dir/$source_path" +done +source_dir="$(cd "$(dirname "$source_path")" && pwd)" +real_git=/usr/bin/git +[[ ! -f "$source_dir/real-git" ]] || IFS= read -r real_git < "$source_dir/real-git" +for arg in "$@"; do + if [[ "$arg" == worktree ]]; then + exec /bin/bash "$source_dir/performance-guard.sh" --git "$@" + fi +done +exec "$real_git" "$@" diff --git a/toolchain/macos/performance-guard.sh b/toolchain/macos/performance-guard.sh index 8208e99112..e7eeeb0a73 100755 --- a/toolchain/macos/performance-guard.sh +++ b/toolchain/macos/performance-guard.sh @@ -1,201 +1,11 @@ #!/usr/bin/env bash -# Aggressive, low-overhead host pressure monitor for the AC prompt machine. -# Arbitrary hot processes are reported, never killed. The sole self-heal is -# duplicate Caddy instances validated by exact command and repo working dir. - -set -u - -SCRIPT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" -REPO="${AC_REPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" -SYSTEM_DIR="${REPO}/system" -STATE_DIR="${HOME}/.local/share/slab/performance" -LOG_PATH="${STATE_DIR}/performance-guard.log" -LATEST_PATH="${STATE_DIR}/latest.txt" -BREACH_PATH="${STATE_DIR}/breach-count" -ALERT_PATH="${STATE_DIR}/last-alert-epoch" -SWAP_PATH="${STATE_DIR}/last-swapouts" -PRESSURE_FLAG="${STATE_DIR}/pressure-active" -LOCK_DIR="${TMPDIR:-/tmp}/computer.aesthetic.performance-guard-$(id -u).lock" -PLIST="${HOME}/Library/LaunchAgents/computer.aesthetic.performance-guard.plist" -LABEL="computer.aesthetic.performance-guard" -INTERVAL=30 -REPAIR=0 - -usage() { - echo "usage: performance-guard.sh [--once [--repair] | --watch [--repair] | --status | --install | --uninstall]" -} - -number_or_zero() { - case "${1:-}" in ''|*[!0-9.]*) echo 0 ;; *) echo "$1" ;; esac -} - -validated_caddy_pids() { - local pid command cwd - pgrep -x caddy 2>/dev/null | while IFS= read -r pid; do - command="$(ps -p "$pid" -o command= 2>/dev/null || true)" - case "$command" in - *"caddy run --config Caddyfile"*) ;; - *) continue ;; - esac - cwd="$(lsof -a -p "$pid" -d cwd -Fn 2>/dev/null | sed -n 's/^n//p' | head -1)" - [[ "$cwd" == "$SYSTEM_DIR" ]] && echo "$pid" - done -} - -repair_duplicate_caddy() { - local pids="$1" keep pid repaired=0 - keep="$(printf '%s\n' "$pids" | awk 'NF' | sort -n | tail -1)" - [[ -n "$keep" ]] || { echo 0; return; } - while IFS= read -r pid; do - [[ -n "$pid" && "$pid" != "$keep" ]] || continue - kill -TERM "$pid" 2>/dev/null && repaired=$((repaired + 1)) - done <<<"$pids" - sleep 0.25 - while IFS= read -r pid; do - [[ -n "$pid" && "$pid" != "$keep" ]] || continue - kill -0 "$pid" 2>/dev/null && kill -KILL "$pid" 2>/dev/null || true - done <<<"$pids" - echo "$repaired" -} - -notify_pressure() { - local message="$1" - /usr/bin/osascript -e "display notification \"${message//\"/}\" with title \"AC performance guard\"" >/dev/null 2>&1 || true -} - -rotate_log() { - [[ -f "$LOG_PATH" ]] || return - local bytes - bytes="$(stat -f %z "$LOG_PATH" 2>/dev/null || echo 0)" - if (( bytes > 5242880 )); then - tail -1000 "$LOG_PATH" > "${LOG_PATH}.next" - mv "${LOG_PATH}.next" "$LOG_PATH" - fi -} - -sample_once() { - mkdir -p "$STATE_DIR" - if ! mkdir "$LOCK_DIR" 2>/dev/null; then return 0; fi - trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT HUP INT TERM - - local timestamp cores load1 free_pct processes threads codex_count node_count swift_build_count - local caddy_pids caddy_count repaired terminal_cpu window_cpu slab_cpu menuband_cpu - local swapouts previous_swap swap_delta severe reason top_cpu top_mem breaches last_alert now - timestamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - cores="$(number_or_zero "$(sysctl -n hw.logicalcpu 2>/dev/null)")" - load1="$(number_or_zero "$(sysctl -n vm.loadavg 2>/dev/null | awk '{print $2}')")" - free_pct="$(number_or_zero "$(memory_pressure 2>/dev/null | awk '/System-wide memory free percentage/{gsub(/%/,"",$5); print $5; exit}')")" - processes="$(ps -A -o pid= 2>/dev/null | wc -l | tr -d ' ')" - threads="$(ps -M -A -o pid= 2>/dev/null | wc -l | tr -d ' ')" - codex_count="$(pgrep -x codex 2>/dev/null | wc -l | tr -d ' ')" - node_count="$(pgrep -x node 2>/dev/null | wc -l | tr -d ' ')" - swift_build_count="$(pgrep -x swift-build 2>/dev/null | wc -l | tr -d ' ')" - caddy_pids="$(validated_caddy_pids)" - caddy_count="$(printf '%s\n' "$caddy_pids" | awk 'NF{n++} END{print n+0}')" - repaired=0 - if (( REPAIR == 1 && caddy_count > 1 )); then - repaired="$(repair_duplicate_caddy "$caddy_pids")" - caddy_count=$((caddy_count - repaired)) - fi - - terminal_cpu="$(ps -A -o %cpu=,comm= | awk '$2 ~ /\/Terminal$/ {s+=$1} END{printf "%.1f",s+0}')" - window_cpu="$(ps -A -o %cpu=,comm= | awk '$2 ~ /\/WindowServer$/ {s+=$1} END{printf "%.1f",s+0}')" - slab_cpu="$(ps -A -o %cpu=,comm= | awk '$2 ~ /\/slab-menubar$/ {s+=$1} END{printf "%.1f",s+0}')" - menuband_cpu="$(ps -A -o %cpu=,comm= | awk '$2 ~ /\/MenuBand$/ {s+=$1} END{printf "%.1f",s+0}')" - swapouts="$(number_or_zero "$(memory_pressure 2>/dev/null | awk '/Swapouts:/{print $2; exit}')")" - previous_swap="$(number_or_zero "$(cat "$SWAP_PATH" 2>/dev/null || true)")" - swap_delta=0 - if (( previous_swap > 0 && swapouts >= previous_swap )); then swap_delta=$((swapouts - previous_swap)); fi - printf '%s\n' "$swapouts" > "$SWAP_PATH" - - severe=0 - reason="" - awk -v l="$load1" -v c="$cores" 'BEGIN{exit !(l > c*1.5)}' && { severe=1; reason="load"; } - awk -v f="$free_pct" 'BEGIN{exit !(f < 15)}' && { severe=1; reason="${reason:+$reason+}memory"; } - awk -v t="$terminal_cpu" -v w="$window_cpu" 'BEGIN{exit !((t+w) > 125)}' && { severe=1; reason="${reason:+$reason+}display"; } - (( swap_delta > 4096 )) && { severe=1; reason="${reason:+$reason+}swap"; } - (( codex_count > 8 )) && { severe=1; reason="${reason:+$reason+}sessions"; } - (( swift_build_count > 1 )) && { severe=1; reason="${reason:+$reason+}builds"; } - (( caddy_count > 1 || repaired > 0 )) && { severe=1; reason="${reason:+$reason+}caddy"; } - - top_cpu="$(ps -A -o pid=,%cpu=,rss=,comm= | sort -k2 -nr | head -5 | tr '\n' ';')" - top_mem="$(ps -A -o pid=,%cpu=,rss=,comm= | sort -k3 -nr | head -5 | tr '\n' ';')" - { - echo "timestamp=$timestamp" - echo "load1=$load1 cores=$cores free_pct=$free_pct processes=$processes threads=$threads" - echo "codex=$codex_count node=$node_count swift_builds=$swift_build_count caddy=$caddy_count caddy_repaired=$repaired swapout_pages_delta=$swap_delta" - echo "terminal_cpu=$terminal_cpu windowserver_cpu=$window_cpu slab_cpu=$slab_cpu menuband_cpu=$menuband_cpu" - echo "top_cpu=$top_cpu" - echo "top_mem=$top_mem" - echo "pressure=$severe reason=${reason:-none}" - } > "${LATEST_PATH}.next" - mv "${LATEST_PATH}.next" "$LATEST_PATH" - - if (( severe == 1 )); then - : > "$PRESSURE_FLAG" - rotate_log - tr '\n' ' ' < "$LATEST_PATH" >> "$LOG_PATH" - echo >> "$LOG_PATH" - breaches="$(number_or_zero "$(cat "$BREACH_PATH" 2>/dev/null || true)")" - breaches=$((breaches + 1)) - echo "$breaches" > "$BREACH_PATH" - now="$(date +%s)" - last_alert="$(number_or_zero "$(cat "$ALERT_PATH" 2>/dev/null || true)")" - if (( repaired > 0 )); then - notify_pressure "Removed ${repaired} duplicate AC Caddy processes." - echo "$now" > "$ALERT_PATH" - elif (( breaches >= 3 && now - last_alert >= 600 )); then - notify_pressure "Sustained ${reason} pressure: load ${load1}, free memory ${free_pct}%." - echo "$now" > "$ALERT_PATH" - fi - else - rm -f "$PRESSURE_FLAG" - echo 0 > "$BREACH_PATH" - fi - - rmdir "$LOCK_DIR" 2>/dev/null || true - trap - EXIT HUP INT TERM -} - -install_guard() { - mkdir -p "$HOME/Library/LaunchAgents" "$STATE_DIR" - cat > "$PLIST" < - - - Label$LABEL - ProgramArguments - /bin/bash$SCRIPT_PATH--once--repair - - RunAtLoad - StartInterval$INTERVAL - ProcessTypeBackground - LowPriorityIO - EnvironmentVariables - AC_REPO$REPO - - StandardOutPath$STATE_DIR/launchd.out - StandardErrorPath$STATE_DIR/launchd.err - -EOF - plutil -lint "$PLIST" - launchctl bootout "gui/$(id -u)/$LABEL" 2>/dev/null || true - launchctl bootstrap "gui/$(id -u)" "$PLIST" - echo "installed $LABEL (every ${INTERVAL}s)" -} - -uninstall_guard() { - launchctl bootout "gui/$(id -u)/$LABEL" 2>/dev/null || true - rm -f "$PLIST" - echo "uninstalled $LABEL; retained logs in $STATE_DIR" -} - -case "${1:---once}" in - --once) [[ "${2:-}" == "--repair" ]] && REPAIR=1; sample_once ;; - --watch) [[ "${2:-}" == "--repair" ]] && REPAIR=1; while true; do sample_once; sleep "$INTERVAL"; done ;; - --status) if [[ -f "$LATEST_PATH" ]]; then cat "$LATEST_PATH"; else echo "no performance sample yet"; fi ;; - --install) install_guard ;; - --uninstall) uninstall_guard ;; - -h|--help) usage ;; - *) usage >&2; exit 2 ;; -esac +# AC performance guard entry point; keep this beside performance_guard.py. +set -euo pipefail +source_path="${BASH_SOURCE[0]}" +while [[ -L "$source_path" ]]; do + source_dir="$(cd "$(dirname "$source_path")" && pwd)" + source_path="$(readlink "$source_path")" + [[ "$source_path" == /* ]] || source_path="$source_dir/$source_path" +done +source_dir="$(cd "$(dirname "$source_path")" && pwd)" +exec python3 "$source_dir/performance_guard.py" "$@" diff --git a/toolchain/macos/performance_guard.py b/toolchain/macos/performance_guard.py new file mode 100644 index 0000000000..bed8b11f39 --- /dev/null +++ b/toolchain/macos/performance_guard.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +"""Local macOS pressure sampling and admission. Standard library only.""" +import datetime +import fcntl +import json +import math +import os +from pathlib import Path +import plistlib +import re +import shlex +import shutil +import subprocess +import sys +import time + +LABEL = "computer.aesthetic.performance-guard" +HERE = Path(__file__).resolve().parent +STATE = Path.home() / ".local/share/slab/performance" +FLOOR = 20 * 1024**3 +INTERVAL = 30 +MAX_LOG = 5 * 1024**2 +VERSION = "2026-10-01.1" + + +def run(args, timeout=2): + return subprocess.run(args, capture_output=True, text=True, timeout=timeout).stdout.strip() + + +def atomic(path, data): + """Never replace valid state with an incomplete ENOSPC write.""" + temp = path.with_name(path.name + ".next") + try: + with temp.open("w", encoding="utf-8") as stream: + os.chmod(temp, 0o600) + stream.write(data) + os.replace(temp, path) + finally: + temp.unlink(missing_ok=True) + + +def read_json(path): + try: + value = json.loads(path.read_text()) + return value if isinstance(value, dict) else {} + except (OSError, ValueError): + return {} + + +def previous_sample(): + value = read_json(STATE / "latest.json") + for key in ("epoch", "last_alert", "breaches", "swapouts"): + number = value.get(key, 0) + if not isinstance(number, (int, float)) or not math.isfinite(number) or number < 0: + return {} + if not isinstance(value.get("reasons", []), list) or not all(isinstance(r, str) for r in value.get("reasons", [])): + return {} + return value + + +def existing_parent(path): + path = Path(path).absolute() + while not path.exists() and path != path.parent: + path = path.parent + return path + + +def metrics(destination=None): + raw = run(["/usr/sbin/sysctl", "hw.logicalcpu", "kern.memorystatus_level", + "vm.loadavg", "vm.swapusage", "kern.boottime"]) + values = dict(line.split(": ", 1) for line in raw.splitlines() if ": " in line) + memory = values.get("kern.memorystatus_level", "") + if not memory.isdigit(): + found = re.search(r"free percentage:\s*(\d+)%", run(["/usr/bin/memory_pressure", "-Q"])) + memory = found.group(1) if found else "" + cores = int(values["hw.logicalcpu"]) + free_pct = int(memory) + load1 = float(values["vm.loadavg"].strip("{} ").split()[0]) + if cores < 1 or not 0 <= free_pct <= 100: + raise ValueError("invalid host pressure measurements") + disks = [shutil.disk_usage(Path.home()).free] + if destination: + disks.append(shutil.disk_usage(existing_parent(destination)).free) + return {"cores": cores, "free_pct": free_pct, "load1": load1, + "disk_free_bytes": min(disks), "disk_floor_bytes": FLOOR, + "swapusage": values.get("vm.swapusage", "unknown"), + "boot": values.get("kern.boottime", "unknown")} + + +def reasons_for(m): + reasons = [] + if m["load1"] > m["cores"] * 1.5: + reasons.append("load") + if m["free_pct"] < 15: + reasons.append("memory") + if m["disk_free_bytes"] < FLOOR: + reasons.append("disk") + return reasons + + +def admit(operation, destination=None): + if os.environ.get("AC_PERFORMANCE_ALLOW_PRESSURE") == "1": + print("AC performance guard: explicit pressure override", file=sys.stderr) + return 0 + try: + m = metrics(destination) + reasons = reasons_for(m) + previous = previous_sample() + # Keep the sampler's recent swap/display/session signals; stale files + # never deny work indefinitely after a reboot or a disabled sampler. + age = time.time() - previous.get("epoch", 0) + if 0 <= age <= 90 and previous.get("boot") == m["boot"]: + reasons += [r for r in previous.get("reasons", []) + if r not in ("disk", "load", "memory") and r not in reasons] + if not reasons: + return 0 + detail = (f"{'+'.join(reasons)} pressure; {m['disk_free_bytes']/1024**3:.1f} GiB free " + f"(20 GiB reserve), memory available {m['free_pct']}%, load {m['load1']:.2f}") + except (OSError, ValueError, KeyError, TypeError, subprocess.TimeoutExpired) as error: + detail = f"cannot verify host headroom ({error})" + print(f"AC performance guard: deferred {operation}: {detail}. " + "Wait for recovery or use a compute host. Exit 75; nothing launched.", file=sys.stderr) + return 75 + + +def worktree_destination(args, cwd): + """Interpret Git global options, preserving the original argv for execution.""" + i = 0 + while i < len(args): + arg = args[i] + if arg in ("-C", "-c", "--git-dir", "--work-tree", "--namespace", "--config-env"): + if i + 1 >= len(args): + return None + if arg == "-C" and args[i + 1]: + cwd = os.path.abspath(os.path.join(cwd, args[i + 1])) + i += 2 + elif arg.startswith("-C") and arg != "-C": + cwd = os.path.abspath(os.path.join(cwd, arg[2:])) + i += 1 + elif arg.startswith("-"): + i += 1 + else: + break + if args[i:i + 2] != ["worktree", "add"]: + return None + if any(arg in ("-h", "--help") for arg in args[i + 2:]): + return None + i += 2 + while i < len(args): + arg = args[i] + if arg == "--": + i += 1 + break + if arg in ("-b", "-B", "--reason"): + i += 2 + elif arg.startswith("-"): + i += 1 + else: + break + return os.path.abspath(os.path.join(cwd, args[i])) if i < len(args) else None + + +def git_main(args): + real_path = HERE / "real-git" + real = real_path.read_text().strip() if real_path.exists() else "/usr/bin/git" + destination = worktree_destination(args, os.getcwd()) + if destination: + status = admit("git worktree add", destination) + if status: + return status + os.execv(real, [real, *args]) + + +def processes(): + rows = [] + for line in run(["/bin/ps", "-A", "-o", "pid=,ppid=,%cpu=,rss=,comm="]).splitlines(): + bits = line.split(None, 4) + if len(bits) == 5: + try: + rows.append({"pid": int(bits[0]), "ppid": int(bits[1]), + "cpu": float(bits[2]), "rss_kib": int(bits[3]), "executable": bits[4]}) + except ValueError: + pass + return rows + + +def safe_git_command(command): + """Keep command identity, not config values, messages, URLs, or inline code.""" + tokens = shlex.split(command) + safe = [] + redact_next = False + for token in tokens: + if redact_next: + safe.append("[redacted]") + redact_next = False + elif token in ("-c", "--config-env", "-m", "--message"): + safe.append(token) + redact_next = True + elif token == "config": + safe.extend(["config", "[arguments omitted]"]) + break + elif "://" in token or "@" in token or re.search(r"(?i)(token|password|secret|authorization|credential|prompt)", token): + safe.append("[redacted]") + elif token.startswith(("-c", "--config-env=", "--message=", "-m")): + safe.append("[redacted]") + else: + safe.append(token[:256]) + return shlex.join(safe)[:2048] + + +def incident_processes(rows): + by_pid = {p["pid"]: p for p in rows} + selected = {p["pid"]: dict(p) for p in sorted(rows, key=lambda p: p["rss_kib"], reverse=True)[:8]} + for p in sorted(rows, key=lambda p: p["cpu"], reverse=True)[:8]: + selected[p["pid"]] = dict(p) + git = sorted([p for p in rows if Path(p["executable"]).name == "git"], + key=lambda p: p["rss_kib"], reverse=True)[:8] + deadline = time.monotonic() + 4 + for p in git: + item = selected.setdefault(p["pid"], dict(p)) + if time.monotonic() >= deadline: + break + try: + # Check start time before and after to avoid attributing a reused PID. + start = run(["/bin/ps", "-p", str(p["pid"]), "-o", "lstart="]) + command = run(["/bin/ps", "-ww", "-p", str(p["pid"]), "-o", "command="]) + cwd = run(["/usr/sbin/lsof", "-a", "-p", str(p["pid"]), "-d", "cwd", "-Fn"], timeout=0.4) + if start and start == run(["/bin/ps", "-p", str(p["pid"]), "-o", "lstart="]): + item.update(started=start, command=safe_git_command(command), + cwd=next((s[1:] for s in cwd.splitlines() if s.startswith("n")), None)) + except (OSError, ValueError, subprocess.TimeoutExpired): + pass + for p in list(selected.values()): + chain, parent = [], p["ppid"] + while parent in by_pid and len(chain) < 6: + ancestor = by_pid[parent] + chain.append({k: ancestor[k] for k in ("pid", "ppid", "executable")}) + parent = ancestor["ppid"] + p["parents"] = chain + return list(selected.values()) + + +def notify(message): + # argv-based AppleScript avoids interpolating commands into source. + script = 'on run argv\ndisplay notification (item 1 of argv) with title "AC performance guard"\nend run' + try: + run(["/usr/bin/osascript", "-e", script, message], timeout=3) + except (OSError, subprocess.TimeoutExpired): + pass + + +def append_log(path, line): + if path.exists() and path.stat().st_size > MAX_LOG: + with path.open("rb") as stream: + stream.seek(-1024**2, os.SEEK_END) + tail = stream.read().split(b"\n", 1)[-1].decode(errors="replace") + atomic(path, tail) + with path.open("a") as stream: + os.chmod(path, 0o600) + stream.write(line + "\n") + + +def repair_caddy(rows): + repo = os.environ.get("AC_REPO") + if not repo: + return 0 + matches = [] + for p in rows: + if Path(p["executable"]).name != "caddy": + continue + try: + argv = shlex.split(run(["/bin/ps", "-ww", "-p", str(p["pid"]), "-o", "command="])) + cwd = run(["/usr/sbin/lsof", "-a", "-p", str(p["pid"]), "-d", "cwd", "-Fn"]) + if argv[1:] == ["run", "--config", "Caddyfile"] and "n" + repo + "/system" in cwd.splitlines(): + matches.append(p["pid"]) + except (OSError, ValueError, subprocess.TimeoutExpired): + pass + # SIGTERM only; never escalate against a potentially reused PID. + for pid in sorted(matches)[:-1]: + try: + os.kill(pid, 15) + except ProcessLookupError: + pass + return max(0, len(matches) - 1) + + +def sample(repair=False): + STATE.mkdir(parents=True, exist_ok=True) + with (STATE / "sample.lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return 0 + previous = previous_sample() + m = metrics() + revision_file = HERE / "revision" + m.update(version=VERSION, revision=revision_file.read_text().strip() if revision_file.exists() else "source", + epoch=time.time(), timestamp=datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")) + rows = processes() + names = [Path(p["executable"]).name for p in rows] + m.update(processes=len(rows), codex=names.count("codex"), node=names.count("node"), + swift_builds=names.count("swift-build"), caddy=names.count("caddy")) + vm = run(["/usr/bin/vm_stat"]) + swaps = re.search(r"Swapouts:\s*(\d+)", vm) + m["swapouts"] = int(swaps.group(1)) if swaps else 0 + elapsed = m["epoch"] - previous.get("epoch", 0) + delta = max(0, m["swapouts"] - previous.get("swapouts", m["swapouts"])) + if previous.get("boot") != m["boot"] or not 0 < elapsed <= 300: + delta = 0 + m["swapout_pages_delta"] = delta + reasons = reasons_for(m) + if delta > 4096 * max(elapsed, 1) / INTERVAL: + reasons.append("swap") + if m["codex"] > 8: + reasons.append("sessions") + if m["swift_builds"] > 1: + reasons.append("builds") + display = sum(p["cpu"] for p in rows if Path(p["executable"]).name in ("Terminal", "WindowServer")) + if display > 125: + reasons.append("display") + m["caddy_repaired"] = repair_caddy(rows) if repair else 0 + if m["caddy_repaired"]: + reasons.append("caddy") + m.update(pressure=int(bool(reasons)), reasons=reasons, reason="+".join(reasons) or "none") + recent = 0 < elapsed <= 90 and previous.get("boot") == m["boot"] + m["breaches"] = (previous.get("breaches", 0) + 1 if recent else 1) if reasons else 0 + m["last_alert"] = previous.get("last_alert", 0) + if reasons: + # Publish the admission signal before optional incident enrichment. + atomic(STATE / "pressure-active", m["reason"] + "\n") + critical = m["disk_free_bytes"] < 5 * 1024**3 or m["free_pct"] < 5 + if (critical or m["breaches"] >= 3) and m["epoch"] - m["last_alert"] >= 600: + notify(f"{m['reason']} pressure: {m['disk_free_bytes']/1024**3:.1f} GiB disk free, memory {m['free_pct']}%.") + m["last_alert"] = m["epoch"] + else: + (STATE / "pressure-active").unlink(missing_ok=True) + atomic(STATE / "latest.json", json.dumps(m) + "\n") + keys = ("version", "revision", "timestamp", "load1", "cores", "free_pct", "disk_free_bytes", "processes", "codex", "node", + "swift_builds", "caddy", "caddy_repaired", "swapout_pages_delta", "pressure", "reason") + text = "\n".join(f"{k}={m[k]}" for k in keys) + atomic(STATE / "latest.txt", text + "\n") + if reasons: + append_log(STATE / "performance-guard.log", text.replace("\n", " ")) + append_log(STATE / "incidents.jsonl", json.dumps({**m, "process_details": incident_processes(rows)})) + return 0 + + +def install(preserve_git=False): + home = Path.home() + target = home / ".local/lib/ac-performance-guard" + bindir = home / ".local/bin" + shims = {"git": "git-guard.sh", "swift": "swift-guard.sh", "ac-performance-guard": "performance-guard.sh"} + existing_git = None + for name, source in shims.items(): + link = bindir / name + if link.exists() or link.is_symlink(): + resolved = link.resolve() + if resolved != target / source and not (link.is_symlink() and resolved.name == source): + if name == "git" and preserve_git and link.is_file() and not link.is_symlink(): + existing_git = link + else: + raise RuntimeError(f"Refusing to replace unrelated command: {link}") + real_git_file = target / "real-git" + real_git = real_git_file.read_text().strip() if real_git_file.exists() else shutil.which("git") + if not real_git or Path(real_git).resolve() == (target / "git-guard.sh").resolve(): + raise RuntimeError("Cannot resolve underlying Git") + for directory in (target, bindir, STATE, home / "Library/LaunchAgents"): + directory.mkdir(parents=True, exist_ok=True) + if existing_git: + backup = bindir / "git.before-ac-guard" + if backup.exists(): + raise RuntimeError(f"Git wrapper backup already exists: {backup}") + shutil.copy2(existing_git, backup) + real_git = str(backup) + for name in ("performance-guard.sh", "performance_guard.py", "git-guard.sh", "swift-guard.sh", "build-lock.sh"): + source = HERE / name + if name == "build-lock.sh" and not source.exists(): + source = HERE.parent.parent / "slab/bin/build-lock.sh" + if source.resolve() != (target / name).resolve(): + atomic(target / name, source.read_text()) + (target / name).chmod(0o755) + atomic(real_git_file, real_git + "\n") + if (HERE / "revision").exists() and HERE != target: + atomic(target / "revision", (HERE / "revision").read_text()) + for name, source in shims.items(): + link = bindir / name + replacement = bindir / (name + ".guard-next") + replacement.unlink(missing_ok=True) + replacement.symlink_to(target / source) + os.replace(replacement, link) + repo = os.environ.get("AC_REPO", str(home / "aesthetic-computer")) + atomic(target / "repo-path", repo + "\n") + config = {"Label": LABEL, "ProgramArguments": ["/bin/bash", str(target / "performance-guard.sh"), "--once", "--repair"], + "RunAtLoad": True, "StartInterval": INTERVAL, "ProcessType": "Background", "LowPriorityIO": True, + "EnvironmentVariables": {"AC_REPO": repo, "PATH": f"{bindir}:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"}, + "StandardOutPath": str(STATE / "launchd.out"), "StandardErrorPath": str(STATE / "launchd.err")} + plist = home / "Library/LaunchAgents" / (LABEL + ".plist") + atomic(plist, plistlib.dumps(config).decode()) + subprocess.run(["launchctl", "bootout", f"gui/{os.getuid()}/{LABEL}"], capture_output=True) + subprocess.run(["launchctl", "bootstrap", f"gui/{os.getuid()}", str(plist)], check=True) + print(f"Installed {LABEL}, Git and Swift guards. Put {bindir} first on PATH.") + + +def main(args): + action = args[0] if args else "--once" + if action == "--git": + return git_main(args[1:]) + if action == "--admit": + return admit(args[1] if len(args) > 1 else "work", args[2] if len(args) > 2 else None) + if action == "--install": + install("--preserve-git-wrapper" in args) + elif action == "--status": + print((STATE / "latest.txt").read_text() if (STATE / "latest.txt").exists() else "no performance sample yet") + elif action in ("--once", "--watch"): + while True: + try: + sample("--repair" in args) + except (OSError, ValueError, KeyError, TypeError, subprocess.TimeoutExpired) as error: + print(f"AC performance guard sample failed: {error}", file=sys.stderr) + try: + atomic(STATE / "pressure-active", "measurement-unavailable\n") + except OSError: + pass + if action == "--once": + return 1 + if action == "--once": + break + time.sleep(INTERVAL) + elif action == "--uninstall": + subprocess.run(["launchctl", "bootout", f"gui/{os.getuid()}/{LABEL}"], capture_output=True) + (Path.home() / "Library/LaunchAgents" / (LABEL + ".plist")).unlink(missing_ok=True) + for name in ("git", "swift", "ac-performance-guard"): + link = Path.home() / ".local/bin" / name + if link.is_symlink() and link.resolve().parent == HERE: + link.unlink() + backup = link.with_name("git.before-ac-guard") + if name == "git" and backup.exists(): + os.replace(backup, link) + (STATE / "pressure-active").unlink(missing_ok=True) + else: + print("usage: performance-guard.sh [--once [--repair] | --watch | --status | --install | --uninstall | --admit OPERATION [PATH]]") + return 0 if action in ("-h", "--help") else 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/toolchain/macos/swift-guard.sh b/toolchain/macos/swift-guard.sh index c6916b3e39..d7a4fe1b00 100755 --- a/toolchain/macos/swift-guard.sh +++ b/toolchain/macos/swift-guard.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# PATH shim for `swift`: serialize SwiftPM builds per AC package while leaving +# PATH shim for `swift`: check headroom and serialize SwiftPM builds while leaving # every other Swift command untouched. Xcode's internal absolute tool paths are # unaffected; this covers humans and agents invoking `swift build` in shells. @@ -12,11 +12,14 @@ while [[ -L "$SOURCE" ]]; do [[ "$SOURCE" == /* ]] || SOURCE="$SOURCE_DIR/$SOURCE" done REPO="$(cd "$(dirname "$SOURCE")/../.." && pwd)" +GUARD_DIR="$(cd "$(dirname "$SOURCE")" && pwd)" +[[ ! -f "$GUARD_DIR/repo-path" ]] || IFS= read -r REPO < "$GUARD_DIR/repo-path" [[ "${1:-}" == "build" ]] || exec "$REAL_SWIFT" "$@" run_guarded_build() { local have_jobs=0 arg memory_bytes jobs=2 + /bin/bash "$GUARD_DIR/performance-guard.sh" --admit 'swift build' "$build_path" || return $? for arg in "$@"; do case "$arg" in -j|--jobs|--jobs=*) have_jobs=1 ;; esac done @@ -30,6 +33,7 @@ run_guarded_build() { } package_path="$PWD" +build_path="" args=("$@") for ((i = 0; i < ${#args[@]}; i++)); do case "${args[$i]}" in @@ -37,6 +41,10 @@ for ((i = 0; i < ${#args[@]}; i++)); do ((i + 1 < ${#args[@]})) && package_path="${args[$((i + 1))]}" ;; --package-path=*) package_path="${args[$i]#--package-path=}" ;; + --scratch-path) + ((i + 1 < ${#args[@]})) && build_path="${args[$((i + 1))]}" + ;; + --scratch-path=*) build_path="${args[$i]#--scratch-path=}" ;; esac done @@ -48,15 +56,15 @@ while [[ "$probe" == "$REPO"/* && ! -f "$probe/Package.swift" ]]; do probe="$(dirname "$probe")" done [[ -f "$probe/Package.swift" ]] && package_path="$probe" +[[ -n "$build_path" ]] || build_path="$package_path/.build" case "$package_path" in "$REPO/slab/menubar-swift"*) lock_name="slab-menubar" ;; "$REPO/slab/menuband"*) lock_name="menuband" ;; - "$REPO"/*) + *) lock_id="$(printf '%s' "$package_path" | cksum | awk '{print $1}')" lock_name="swift-${lock_id}" ;; - *) exec "$REAL_SWIFT" "$@" ;; esac # An installer that already owns this exact lock must be allowed to invoke its @@ -66,6 +74,12 @@ if [[ "${AC_BUILD_LOCK_HELD:-}" == "$lock_name" ]]; then exit $? fi -source "$REPO/slab/bin/build-lock.sh" +if [[ -f "$GUARD_DIR/build-lock.sh" ]]; then + source "$GUARD_DIR/build-lock.sh" +else + source "$REPO/slab/bin/build-lock.sh" +fi +# Check before waiting for a lock, then check again immediately before launch. +/bin/bash "$GUARD_DIR/performance-guard.sh" --admit 'swift build' "$build_path" acquire_build_lock "$lock_name" run_guarded_build "$@" diff --git a/toolchain/macos/test_performance_guard.py b/toolchain/macos/test_performance_guard.py new file mode 100644 index 0000000000..f606fe044e --- /dev/null +++ b/toolchain/macos/test_performance_guard.py @@ -0,0 +1,141 @@ +import contextlib +import importlib.util +import io +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("guard", Path(__file__).with_name("performance_guard.py")) +guard = importlib.util.module_from_spec(spec) +spec.loader.exec_module(guard) + + +class GuardTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.state = Path(self.temp.name) + self.patcher = patch.object(guard, "STATE", self.state) + self.patcher.start() + self.addCleanup(self.patcher.stop) + self.healthy = dict(cores=6, free_pct=60, load1=2, disk_free_bytes=40 * 1024**3, + disk_floor_bytes=guard.FLOOR, boot="boot-1", swapusage="0") + + def check(self, metrics=None): + with patch.object(guard, "metrics", return_value=metrics or self.healthy), contextlib.redirect_stderr(io.StringIO()): + return guard.admit("test") + + def test_low_disk_blocks_without_sampler(self): + self.assertEqual(self.check({**self.healthy, "disk_free_bytes": 4 * 1024**3}), 75) + + def test_healthy_host_admitted_without_sampler(self): + self.assertEqual(self.check(), 0) + + def test_memory_and_cpu_budgets(self): + self.assertEqual(self.check({**self.healthy, "free_pct": 3}), 75) + self.assertEqual(self.check({**self.healthy, "load1": 20}), 75) + + def test_stale_pressure_does_not_block_recovery(self): + guard.atomic(self.state / "pressure-active", "swap") + guard.atomic(self.state / "latest.json", json.dumps(dict(epoch=time.time()-300, boot="boot-1", reasons=["swap"]))) + self.assertEqual(self.check(), 0) + + def test_recent_swap_pressure_blocks_and_reboot_clears_it(self): + data = dict(epoch=time.time(), boot="boot-1", reasons=["swap"]) + guard.atomic(self.state / "latest.json", json.dumps(data)) + self.assertEqual(self.check(), 75) + self.assertEqual(self.check({**self.healthy, "boot": "boot-2"}), 0) + + def test_failed_measurement_defers_work(self): + with patch.object(guard, "metrics", side_effect=OSError("unavailable")), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(guard.admit("test"), 75) + + def test_valid_json_with_corrupt_counters_recovers(self): + guard.atomic(self.state / "latest.json", '{"epoch":"timestamp=broken","breaches":"bad"}') + self.assertEqual(self.check(), 0) + + def test_global_git_options_and_destination(self): + cases = [ + (["-C", "/repo", "-C", "sub", "-c", "x=y", "worktree", "add", "-b", "topic", "../new", "HEAD"], "/repo/new"), + (["-C/repo", "worktree", "add", "--orphan", "new"], "/repo/new"), + (["worktree", "add", "--lock", "--reason", "a reason", "--", "-new"], "/repo/-new"), + (["worktree", "add", "--no-checkout", "/other/new", "HEAD"], "/other/new"), + (["worktree", "list"], None), + (["show", "worktree", "add"], None), + (["worktree", "add", "--help"], None), + ] + for args, expected in cases: + with self.subTest(args=args): + self.assertEqual(guard.worktree_destination(args, "/repo"), expected) + + def test_git_denial_never_executes(self): + with patch.object(guard, "admit", return_value=75), patch.object(guard.os, "execv") as execute: + self.assertEqual(guard.git_main(["worktree", "add", "/tmp/example"]), 75) + execute.assert_not_called() + + def test_git_read_only_preserves_argv_without_admission(self): + args = ["-C", "/some path", "worktree", "list", "--porcelain"] + with patch.object(guard, "admit") as admission, patch.object(guard.os, "execv") as execute: + guard.git_main(args) + admission.assert_not_called() + self.assertEqual(execute.call_args.args[1][1:], args) + + def test_redacts_git_credentials_and_messages(self): + text = guard.safe_git_command('git -c "http.extraHeader=Authorization: Bearer SECRET" fetch https://user:SECRET@example.com/repo') + self.assertNotIn("SECRET", text) + self.assertIn("fetch", text) + self.assertNotIn("private message", guard.safe_git_command('git commit -m "private message"')) + self.assertIn("/tmp/work", guard.safe_git_command('git -C /tmp/work merge origin/main --no-edit')) + + def test_failed_atomic_replace_preserves_previous_sample(self): + file = self.state / "latest.json" + file.write_text('{"healthy":true}') + with patch.object(guard.os, "replace", side_effect=OSError("disk full")): + with self.assertRaises(OSError): + guard.atomic(file, "broken") + self.assertEqual(file.read_text(), '{"healthy":true}') + self.assertFalse((self.state / "latest.json.next").exists()) + + def test_corrupt_state_recovers_and_records_disk_pressure(self): + (self.state / "latest.json").write_text("timestamp=corrupted") + with patch.object(guard, "metrics", return_value={**self.healthy, "disk_free_bytes": 10 * 1024**3}), \ + patch.object(guard, "processes", return_value=[]), \ + patch.object(guard, "run", return_value="Swapouts: 123."), patch.object(guard, "notify"): + self.assertEqual(guard.sample(), 0) + data = guard.read_json(self.state / "latest.json") + self.assertEqual(data["reasons"], ["disk"]) + self.assertEqual(data["swapout_pages_delta"], 0) + self.assertEqual((self.state / "pressure-active").read_text(), "disk\n") + + def test_swap_rate_and_boot_reset(self): + previous = dict(epoch=time.time()-30, boot="boot-1", swapouts=100, breaches=1) + guard.atomic(self.state / "latest.json", json.dumps(previous)) + with patch.object(guard, "metrics", return_value=self.healthy), \ + patch.object(guard, "processes", return_value=[]), \ + patch.object(guard, "run", return_value="Swapouts: 10100."), patch.object(guard, "notify"): + guard.sample() + self.assertIn("swap", guard.read_json(self.state / "latest.json")["reasons"]) + with patch.object(guard, "metrics", return_value={**self.healthy, "boot": "boot-2"}), \ + patch.object(guard, "processes", return_value=[]), \ + patch.object(guard, "run", return_value="Swapouts: 10100."): + guard.sample() + self.assertFalse((self.state / "pressure-active").exists()) + + def test_swift_refusal_never_invokes_compiler(self): + # Exercise the shell entry point outside the AC checkout, with a stub + # gate. A stub lock fails too, proving refusal happens before locking. + source = Path(__file__).with_name("swift-guard.sh") + (self.state / "swift-guard.sh").write_text(source.read_text()) + (self.state / "performance-guard.sh").write_text('#!/bin/bash\nexit 75\n') + (self.state / "build-lock.sh").write_text('acquire_build_lock() { exit 99; }\n') + result = subprocess.run(["/bin/bash", str(self.state / "swift-guard.sh"), "build", "--package-path", str(self.state)], capture_output=True) + self.assertEqual(result.returncode, 75) + + +if __name__ == "__main__": + unittest.main()