From 940cbf2fcb74a2cb4e72c8295bcc3df4bcfa1ee2 Mon Sep 17 00:00:00 2001
From: dietrich ayala <me@burrito.space>
Date: Thu, 24 Sep 2026 17:50:13 +0200
Subject: [PATCH] Model variety for generation runs: qwen3.8-27b-fast-2 and
 deepseek-v4.1-flash

Two more model sources behind the gateway, because candidates from a different model
fail differently where more replicas of the same model do not:

- qwen3.8-27b-fast-2: qwen3.8-27b on a new four-card GPU VM (bench/gpu-vm/), 16 in
  flight; absolute-seam batch after the prompt fixes 8/32 compiled, best 82/108.
- deepseek-v4.1-flash: DeepSeek-V4.1-Flash EXL3 on a two-node ARM pair
  (bench/sparks/dsv41/), its own proxy frontend; first batch 4/8 compiled, best
  82/108. qwen3.8-flash-next drops to two replicas and 16 in flight.
- bench/rig-snapshot.sh snapshots and restores each rig role's serving configs;
  bench/sparks/hf-to-nas.sh fetches weights to the NAS.
- docs/infrastructure.md gains the rig change log; measurements in
  docs/model-first-cost.md and docs/rig-capacity.md.

Hosts are named by role only.
---
 bench/gpu-vm/create-vm.sh                     |  88 +++++
 bench/gpu-vm/folknet-qwen38-vllm-worker       |  54 +++
 .../folknet-qwen38-vllm-worker@.service       |  23 ++
 bench/gpu-vm/provision.sh                     |  65 ++++
 bench/rig-snapshot.sh                         | 312 ++++++++++++++++++
 bench/sparks/dsv41/env.template               |  80 +++++
 .../dsv41/folknet-dsv41-exl3-pair.service     |  32 ++
 bench/sparks/dsv41/ssh-config-worker          |   9 +
 bench/sparks/hf-to-nas.sh                     | 112 +++++++
 docs/infrastructure.md                        | 217 ++++++++++--
 docs/model-first-cost.md                      |  58 +++-
 docs/rig-capacity.md                          |  24 ++
 12 files changed, 1051 insertions(+), 23 deletions(-)
 create mode 100644 bench/gpu-vm/create-vm.sh
 create mode 100644 bench/gpu-vm/folknet-qwen38-vllm-worker
 create mode 100644 bench/gpu-vm/folknet-qwen38-vllm-worker@.service
 create mode 100644 bench/gpu-vm/provision.sh
 create mode 100755 bench/rig-snapshot.sh
 create mode 100644 bench/sparks/dsv41/env.template
 create mode 100644 bench/sparks/dsv41/folknet-dsv41-exl3-pair.service
 create mode 100644 bench/sparks/dsv41/ssh-config-worker
 create mode 100644 bench/sparks/hf-to-nas.sh

diff --git a/bench/gpu-vm/create-vm.sh b/bench/gpu-vm/create-vm.sh
new file mode 100644
index 00000000..b94e06c1
--- /dev/null
+++ b/bench/gpu-vm/create-vm.sh
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# Creates the four-card serving VM on a Proxmox node. Runs ON the node as root:
+#
+#   ssh root@<proxmox-node> "VM_SSH_PUBKEY='$(cat ~/.ssh/<key>.pub)' bash -s -- <vmid> <name> <source-vmid>" \
+#     < bench/gpu-vm/create-vm.sh
+#
+# The GPUs are taken from an existing, stopped VM (<source-vmid>) that already has the
+# cards passed through: its hostpci entries and its QEMU args (the 64-bit PCI hole that
+# four 24 GB cards need) are copied, nothing else of it is touched. The two VMs share
+# those cards, so only one of them can run at a time; the source VM must keep
+# onboot 0, and the new VM's description says which cards it borrows.
+#
+# Ubuntu 24.04 cloud image, q35 + OVMF, DHCP on the node's bridge (the address it got
+# is printed at the end). The OS disk is kept small: checkpoints live on the NAS, so
+# the node's thin pool (shared with other tenants' disks) is not asked for model space.
+set -euo pipefail
+main() {
+
+VMID="${1:?usage: create-vm.sh <vmid> <name> <source-vmid>}"
+NAME="${2:?usage: create-vm.sh <vmid> <name> <source-vmid>}"
+SRC="${3:?usage: create-vm.sh <vmid> <name> <source-vmid>}"
+: "${VM_SSH_PUBKEY:?set VM_SSH_PUBKEY to the public key line that may log in}"
+
+CORES="${VM_CORES:-32}"
+MEMORY_MB="${VM_MEMORY_MB:-163840}"
+DISK_GB="${VM_DISK_GB:-120}"
+STORAGE="${VM_STORAGE:-local-lvm}"
+BRIDGE="${VM_BRIDGE:-vmbr0}"
+IMAGE="${VM_IMAGE:-/var/lib/vz/template/iso/noble-server-cloudimg-amd64.img}"
+USER_NAME="${VM_USER:-ubuntu}"
+
+if qm status "$VMID" >/dev/null 2>&1; then
+  echo "vmid $VMID already exists on $(hostname); pick another" >&2
+  exit 1
+fi
+if [ "$(qm status "$SRC" | awk '{print $2}')" != stopped ]; then
+  echo "source vm $SRC must be stopped: it holds the same cards" >&2
+  exit 1
+fi
+test -f "$IMAGE" || { echo "no cloud image at $IMAGE" >&2; exit 1; }
+
+src_conf="$(qm config "$SRC")"
+hostpci="$(printf '%s\n' "$src_conf" | grep '^hostpci[0-9]*:')"
+args="$(printf '%s\n' "$src_conf" | sed -n 's/^args: //p')"
+[ -n "$hostpci" ] || { echo "source vm $SRC has no hostpci entries" >&2; exit 1; }
+cards="$(printf '%s\n' "$hostpci" | awk '{print $2}' | cut -d, -f1 | paste -sd' ')"
+
+keyfile="$(mktemp)"
+trap 'rm -f "$keyfile"' EXIT
+printf '%s\n' "$VM_SSH_PUBKEY" > "$keyfile"
+
+qm create "$VMID" --name "$NAME" \
+  --machine q35 --bios ovmf --cpu host --sockets 1 --cores "$CORES" --numa 1 \
+  --memory "$MEMORY_MB" --balloon 0 \
+  --efidisk0 "$STORAGE:1,efitype=4m,pre-enrolled-keys=0" \
+  --scsihw virtio-scsi-single \
+  --net0 "virtio,bridge=$BRIDGE" \
+  --serial0 socket --vga none --agent enabled=1 --ostype l26 \
+  --ide2 "$STORAGE:cloudinit" \
+  --ciuser "$USER_NAME" --sshkeys "$keyfile" --ipconfig0 ip=dhcp \
+  --onboot 1 \
+  --description "Four-card model-serving replica. Borrows the GPUs of VM $SRC ($cards); only one of the two may run at a time, and VM $SRC stays onboot 0."
+qm importdisk "$VMID" "$IMAGE" "$STORAGE" --format raw
+qm set "$VMID" --scsi0 "$STORAGE:vm-$VMID-disk-1,discard=on,iothread=1,ssd=1" --boot order=scsi0
+qm resize "$VMID" scsi0 "${DISK_GB}G"
+[ -z "$args" ] || qm set "$VMID" --args "$args"
+while read -r key val; do
+  qm set "$VMID" "--${key%:}" "$val"
+done <<< "$hostpci"
+
+qm start "$VMID"
+# The stock cloud image has no guest agent, so the DHCP address is found by the VM's MAC
+# in the node's neighbour table after a ping sweep of the bridge's /24.
+mac="$(qm config "$VMID" | sed -n 's/^net0: virtio=\([^,]*\),.*/\1/p' | tr 'A-F' 'a-f')"
+net="$(ip -4 -o addr show "$BRIDGE" | awk '{print $4}' | head -1)"
+prefix="${net%.*}"
+echo "vm $VMID mac $mac; waiting for it to take an address on ${prefix}.0/24"
+for _ in $(seq 1 60); do
+  for i in $(seq 1 254); do ping -c1 -W1 "$prefix.$i" >/dev/null 2>&1 & done
+  wait
+  addr="$(ip neigh show dev "$BRIDGE" | awk -v m="$mac" 'tolower($3)==m {print $1; exit}')"
+  [ -n "$addr" ] && { echo "$addr"; return 0; }
+  sleep 5
+done
+echo "no address seen for $mac yet; check the serial console (qm terminal $VMID)" >&2
+
+}
+main "$@"
diff --git a/bench/gpu-vm/folknet-qwen38-vllm-worker b/bench/gpu-vm/folknet-qwen38-vllm-worker
new file mode 100644
index 00000000..f6820665
--- /dev/null
+++ b/bench/gpu-vm/folknet-qwen38-vllm-worker
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Starts one qwen3.8-27b worker (vLLM, W4A16 AutoRound + DFlash2 speculative decoding)
+# on one RTX 3090 of the four-card VM. Installed as /usr/local/libexec/folknet-qwen38-vllm-worker
+# and run by folknet-qwen38-vllm-worker@<profile>.service.
+#
+# Copied from the recipe the rig's eight-card sampling host runs for its "fast" tier:
+# same image (folknet/qwen38-27b-3090:e5554bb, moved over with docker save | docker load),
+# same prepared checkpoint directory, same environment. The only differences are the
+# paths and that each worker publishes on all interfaces, because the load balancer in
+# front of these workers runs on another host.
+set -euo pipefail
+
+profile="${1:?worker profile is required}"
+
+image="folknet/qwen38-27b-3090:e5554bb"
+models="/srv/models/qwen38-27b"
+cache_root="/srv/cache/qwen38-27b"
+
+case "${profile}" in
+  fast-0) gpu_ids="0"; port="8101" ;;
+  fast-1) gpu_ids="1"; port="8102" ;;
+  fast-2) gpu_ids="2"; port="8103" ;;
+  fast-3) gpu_ids="3"; port="8104" ;;
+  *) echo "unsupported worker profile: ${profile}" >&2; exit 64 ;;
+esac
+context="long"; max_len="131072"; max_seqs="4"
+
+test -d "${models}/Qwen3.8-27B-W4A16-AutoRound-fast"
+test -f "${models}/Qwen3.8-27B-DFlash2-W4A16/model.safetensors"
+mkdir -p "${cache_root}/${profile}"
+
+container="folknet-qwen38-${profile}"
+/usr/bin/docker rm -f "${container}" >/dev/null 2>&1 || true
+
+exec /usr/bin/docker run --rm \
+  --name "${container}" \
+  --gpus "device=${gpu_ids}" \
+  --ipc host \
+  --publish "0.0.0.0:${port}:${port}" \
+  --volume "${models}:/app/models:ro" \
+  --volume "${cache_root}/${profile}:/cache" \
+  --env "PORT=${port}" \
+  --env SPEC=dflash2 \
+  --env DFLASH_TOKENS=7 \
+  --env PREFIX_CACHE=1 \
+  --env "CTX=${context}" \
+  --env "DFLASH_MAX_LEN=${max_len}" \
+  --env "MAX_SEQS=${max_seqs}" \
+  --env GPU_UTIL=0.93 \
+  --env VLLM_OFFLOAD_KEEP_SHM=1 \
+  --env HF_HUB_OFFLINE=1 \
+  --env TRANSFORMERS_OFFLINE=1 \
+  --env "EXTRA_ARGS=" \
+  "${image}" single
diff --git a/bench/gpu-vm/folknet-qwen38-vllm-worker@.service b/bench/gpu-vm/folknet-qwen38-vllm-worker@.service
new file mode 100644
index 00000000..b18138ff
--- /dev/null
+++ b/bench/gpu-vm/folknet-qwen38-vllm-worker@.service
@@ -0,0 +1,23 @@
+[Unit]
+Description=FolkNet Qwen3.8-27B W4A16 DFlash2 worker %i (four-card VM)
+After=docker.service network-online.target
+Wants=network-online.target
+Requires=docker.service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/libexec/folknet-qwen38-vllm-worker %i
+ExecStop=-/usr/bin/docker stop --time 30 folknet-qwen38-%i
+Restart=on-failure
+RestartSec=10s
+TimeoutStartSec=20min
+TimeoutStopSec=2min
+KillSignal=SIGTERM
+LimitNOFILE=1048576
+NoNewPrivileges=true
+ProtectHome=true
+ProtectSystem=strict
+ReadWritePaths=/srv/cache/qwen38-27b
+
+[Install]
+WantedBy=multi-user.target
diff --git a/bench/gpu-vm/provision.sh b/bench/gpu-vm/provision.sh
new file mode 100644
index 00000000..0acf316b
--- /dev/null
+++ b/bench/gpu-vm/provision.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Provisions the four-card serving VM made by bench/gpu-vm/create-vm.sh. Runs INSIDE the
+# guest as root (re-runnable; every step checks before it changes anything):
+#
+#   ssh <user>@<vm> "sudo APT_PROXY=http://<apt-cache>:3142 NAS_EXPORT=<nas>:/<export> bash -s" \
+#     < bench/gpu-vm/provision.sh
+#
+# Installs the NVIDIA server driver (headless, open kernel modules), docker with the
+# NVIDIA container toolkit, the guest agent, and mounts the NAS export that holds the
+# checkpoints read-only at /mnt/models-nas. A reboot is needed after the first run
+# so the driver loads; the script says so.
+set -euo pipefail
+main() {
+
+# 595 is the branch the rig's bare-metal eight-card host runs (CUDA 13.2), new enough
+# for the cu130 torch in the serving image.
+DRIVER="${NVIDIA_DRIVER:-595}"
+NAS_EXPORT="${NAS_EXPORT:-}"
+NAS_MOUNT="${NAS_MOUNT:-/mnt/models-nas}"
+
+if [ -n "${APT_PROXY:-}" ]; then
+  # The rig's apt cache refuses HTTPS CONNECT, so HTTPS repositories go direct.
+  printf 'Acquire::http::Proxy "%s";\nAcquire::https::Proxy "DIRECT";\nAcquire::Retries "5";\n' \
+    "$APT_PROXY" > /etc/apt/apt.conf.d/01proxy
+fi
+export DEBIAN_FRONTEND=noninteractive
+# A fresh cloud image is still running cloud-init and unattended-upgrades; wait for the
+# dpkg lock rather than failing on it.
+cloud-init status --wait >/dev/null 2>&1 || true
+apt() { apt-get -o DPkg::Lock::Timeout=900 "$@"; }
+apt update
+apt install -y qemu-guest-agent nfs-common ca-certificates curl gnupg jq rsync \
+  "nvidia-headless-${DRIVER}-server-open" "nvidia-utils-${DRIVER}-server"
+systemctl enable --now qemu-guest-agent || true
+
+# Docker from the Ubuntu archive (docker.io), because it comes through the apt cache;
+# only the NVIDIA toolkit repository needs a direct HTTPS fetch.
+rm -f /etc/apt/sources.list.d/docker.list
+if ! command -v nvidia-ctk >/dev/null; then
+  curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
+    | gpg --dearmor --yes -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
+  curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
+    | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
+    > /etc/apt/sources.list.d/nvidia-container-toolkit.list
+fi
+apt update
+apt install -y docker.io docker-buildx nvidia-container-toolkit
+nvidia-ctk runtime configure --runtime=docker
+systemctl restart docker
+
+if [ -n "$NAS_EXPORT" ]; then
+  mkdir -p "$NAS_MOUNT"
+  line="$NAS_EXPORT $NAS_MOUNT nfs4 ro,vers=4.2,nconnect=8,hard,_netdev,nofail 0 0"
+  grep -qF " $NAS_MOUNT " /etc/fstab || echo "$line" >> /etc/fstab
+  mountpoint -q "$NAS_MOUNT" || mount "$NAS_MOUNT"
+fi
+
+if nvidia-smi -L >/dev/null 2>&1; then
+  nvidia-smi -L
+else
+  echo "driver installed but not loaded: reboot the guest, then check nvidia-smi"
+fi
+
+}
+main "$@"
diff --git a/bench/rig-snapshot.sh b/bench/rig-snapshot.sh
new file mode 100755
index 00000000..3142bcca
--- /dev/null
+++ b/bench/rig-snapshot.sh
@@ -0,0 +1,312 @@
+#!/usr/bin/env bash
+# Snapshots and restores the serving configuration of every rig role, from the dev box.
+#
+#   bench/rig-snapshot.sh snapshot <label> [role...]
+#   bench/rig-snapshot.sh restore  <label> [role...] [--with-keys]
+#   bench/rig-snapshot.sh list
+#
+# Roles (default: all):
+#   haproxy  haproxy.cfg on the proxy host. Restore: config check, then reload.
+#   gateway  the LiteLLM gateway container on the proxy host: its config directory,
+#            env file, unit, key helper, and a pg_dump of its key database. Restore puts
+#            the files back and restarts the gateway; the key database is only restored
+#            with --with-keys, since other people's keys change independently of ours.
+#   sparks   each GB10 serving host: folknet-* units, TabbyAPI configs and sampler
+#            files, and which folknet-* units are enabled and active. Restore puts the
+#            files back, re-applies enabled/disabled, and restarts what was active.
+#   gpu-vm   the four-card serving VM: its Proxmox VM config on the node (restored only
+#            while the VM is stopped) and, inside the guest, folknet-* units and
+#            /srv/serving, with their enabled and active state.
+#
+# Each snapshot lands on the host it came from, under /root/rig-snapshots/<UTC>-<label>/
+# (the login account's ~/rig-snapshots on the Sparks, which have no root login, and in
+# the four-card VM's guest), with a
+# MANIFEST of the files and their sha256, and a copy is pulled to
+# $HOME/rig-notes/snapshots/<UTC>-<label>/<role>[-<host name>]/ on the dev box. Restore
+# uses the newest on-host snapshot with that label.
+#
+# Host addresses, accounts and keys come from $RIG_ROLES (default
+# $HOME/rig-notes/rig-roles.json), never from this repository: `.jump`, `.proxy_host`
+# (user, host, key, gateway_ct), `.sparks[]` (name, user, host, key), `.gpu_node` (the
+# Proxmox node holding the four-card VM: user, host, key, vmid) and `.gpu_vm` (that VM's
+# guest: user, host, key). Snapshots contain the
+# gateway master key and upstream key: they must never go into git or onto the NAS
+# unencrypted.
+set -euo pipefail
+
+ROLES_FILE="${RIG_ROLES:-$HOME/rig-notes/rig-roles.json}"
+LOCAL_ROOT="${RIG_SNAPSHOT_DIR:-$HOME/rig-notes/snapshots}"
+ALL_ROLES=(haproxy gateway sparks gpu-vm)
+
+die() { echo "rig-snapshot: $*" >&2; exit 1; }
+[ -f "$ROLES_FILE" ] || die "no roles file at $ROLES_FILE"
+command -v jq >/dev/null || die "jq is required"
+
+cfg() { jq -r "$1" "$ROLES_FILE"; }
+JUMP="$(cfg '.jump // empty')"
+
+# rssh <user> <host> <key> <command...>: ssh through the jump host, batch mode.
+rssh() {
+  local user="$1" host="$2" key="${3/#\~/$HOME}"; shift 3
+  LC_ALL=C LANG=C ssh -i "$key" -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=20 \
+    ${JUMP:+-J "$JUMP"} "$user@$host" "$@"
+}
+
+# ---- remote payloads (shipped with declare -f; each runs on the target host) ----
+
+remote_find() { # <root> <label>: newest snapshot dir with that label, or nothing
+  ls -1d "$1"/*-"$2" 2>/dev/null | sort | tail -1
+}
+
+remote_manifest() { # <dir>
+  (cd "$1" && find . -type f ! -name MANIFEST -print0 | sort -z | xargs -0 -r sha256sum) > "$1/MANIFEST"
+}
+
+haproxy_snapshot() { # <dir>
+  set -e
+  mkdir -p "$1/haproxy"
+  cp -a /etc/haproxy/haproxy.cfg "$1/haproxy/"
+  echo "show stat" | socat stdio /run/haproxy/admin.sock > "$1/haproxy/show-stat.csv" 2>/dev/null || true
+}
+
+haproxy_restore() { # <dir> <stamp>
+  set -e
+  local src="$1/haproxy/haproxy.cfg"
+  [ -f "$src" ] || { echo "no haproxy.cfg in $1"; return 1; }
+  haproxy -c -f "$src" >/dev/null || { echo "snapshot haproxy.cfg fails the config check"; return 1; }
+  if cmp -s "$src" /etc/haproxy/haproxy.cfg; then
+    echo "haproxy.cfg already matches the snapshot"
+  else
+    cp -a /etc/haproxy/haproxy.cfg "/etc/haproxy/haproxy.cfg.pre-restore-$2"
+    cp "$src" /etc/haproxy/haproxy.cfg
+  fi
+  haproxy -c -f /etc/haproxy/haproxy.cfg
+  systemctl reload haproxy
+  systemctl is-active haproxy
+}
+
+GATEWAY_FILES="/etc/folknet-model-gateway /etc/folknet-model-gateway.env /etc/systemd/system/folknet-model-gateway.service /root/gateway-keys.py"
+
+gateway_snapshot() { # <dir> <ct>
+  set -e
+  mkdir -p "$1/gateway"
+  # shellcheck disable=SC2086
+  pct exec "$2" -- tar cf - $GATEWAY_FILES 2>/dev/null > "$1/gateway/files.tar"
+  pct exec "$2" -- bash -c 'set -a; . /etc/folknet-model-gateway.env; pg_dump -Fc "$FOLKNET_LITELLM_DATABASE_URL"' \
+    > "$1/gateway/keys.pgdump"
+  chmod 600 "$1/gateway/files.tar" "$1/gateway/keys.pgdump"
+}
+
+gateway_restore() { # <dir> <ct> <with-keys:0|1> <stamp>
+  set -e
+  [ -f "$1/gateway/files.tar" ] || { echo "no gateway files in $1"; return 1; }
+  # shellcheck disable=SC2086
+  pct exec "$2" -- tar cf - $GATEWAY_FILES 2>/dev/null > "$1/../gateway-pre-restore-$4.tar" || true
+  pct exec "$2" -- tar xpf - -C / < "$1/gateway/files.tar"
+  if [ "$3" = 1 ]; then
+    pct exec "$2" -- bash -c 'set -a; . /etc/folknet-model-gateway.env; pg_restore --clean --if-exists -d "$FOLKNET_LITELLM_DATABASE_URL"' \
+      < "$1/gateway/keys.pgdump"
+  fi
+  pct exec "$2" -- systemctl daemon-reload
+  pct exec "$2" -- systemctl restart folknet-model-gateway.service
+  for _ in $(seq 1 60); do
+    pct exec "$2" -- curl -fsS -o /dev/null http://127.0.0.1:4000/health/liveliness 2>/dev/null && { echo "gateway up"; return 0; }
+    sleep 2
+  done
+  echo "gateway did not come back within 120 s"; return 1
+}
+
+# Files a serving host keeps; globs that match nothing are skipped.
+SERVING_GLOBS="/etc/systemd/system/folknet-*.service /usr/local/libexec/folknet-* /home/admin/qwen-trial/tabby-config.yml /home/admin/qwen-trial/sampler-genera.yml /home/admin/qwen-trial/tabby/config.yml /home/admin/qwen-trial/tabby/sampler_overrides/*.yml /home/admin/dsv41/.env /home/admin/.ssh/config /srv/serving/*"
+
+serving_snapshot() { # <dir>
+  set -e
+  mkdir -p "$1/serving"
+  local f files=()
+  for f in $SERVING_GLOBS; do [ -f "$f" ] && files+=("$f"); done
+  [ "${#files[@]}" -eq 0 ] || tar cf "$1/serving/files.tar" "${files[@]}" 2>/dev/null
+  # Templates (name@.service) are enabled per instance, so instances come from the
+  # wants directory instead of the template's own line.
+  {
+    systemctl list-unit-files 'folknet-*.service' --no-legend --no-pager | awk '$1 !~ /@\.service$/ {print $1, $2}'
+    for f in /etc/systemd/system/*.wants/folknet-*@*.service; do
+      [ -e "$f" ] && echo "$(basename "$f") enabled"
+    done
+  } | sort -u > "$1/serving/unit-files.txt"
+  systemctl list-units 'folknet-*.service' --state=active --no-legend --no-pager | awk '{print $1}' > "$1/serving/active.txt"
+}
+
+# <dir> <stamp> <root-prefix>: root-prefix runs a shell command as root ("sudo" in the
+# four-card VM's guest; a privileged container's chroot on the Sparks, whose admin account has no
+# sudo password but is in the docker group).
+serving_restore() {
+  set -e
+  local dir="$1" stamp="$2" asroot="$3"
+  [ -f "$dir/serving/unit-files.txt" ] || { echo "no serving state in $dir"; return 1; }
+  root() {
+    if [ "$asroot" = chroot ]; then
+      docker run --rm --privileged --pid=host -v /:/host alpine chroot /host sh -c "$1"
+    else
+      sudo sh -c "$1"
+    fi
+  }
+  local before="$dir/../serving-pre-restore-$stamp.tar" f files=()
+  for f in $SERVING_GLOBS; do [ -f "$f" ] && files+=("$f"); done
+  [ "${#files[@]}" -eq 0 ] || tar cf "$before" "${files[@]}" 2>/dev/null || true
+  if [ -f "$dir/serving/files.tar" ]; then
+    root "tar xpf '$dir/serving/files.tar' -C /"
+  fi
+  root "systemctl daemon-reload"
+  local unit state
+  while read -r unit state; do
+    case "$state" in
+      enabled) root "systemctl enable '$unit'" ;;
+      disabled) root "systemctl disable '$unit'" ;;
+    esac
+  done < "$dir/serving/unit-files.txt"
+  for unit in $(systemctl list-units 'folknet-*.service' --state=active --no-legend --no-pager | awk '{print $1}'); do
+    grep -qx "$unit" "$dir/serving/active.txt" || { echo "stopping $unit"; root "systemctl stop '$unit'"; }
+  done
+  while read -r unit; do
+    [ -n "$unit" ] || continue
+    echo "restarting $unit"; root "systemctl restart --no-block '$unit'"
+  done < "$dir/serving/active.txt"
+}
+
+vmconf_snapshot() { # <dir> <vmid>
+  set -e
+  mkdir -p "$1/gpu-vm"
+  cp "/etc/pve/qemu-server/$2.conf" "$1/gpu-vm/qemu-$2.conf"
+  qm status "$2" > "$1/gpu-vm/status.txt"
+}
+
+vmconf_restore() { # <dir> <vmid>
+  set -e
+  # Snapshots taken before the role was renamed keep the config in a subdirectory under
+  # the role's old name, so take it from whichever subdirectory holds it.
+  local src
+  src="$(ls -1 "$1"/*/"qemu-$2.conf" 2>/dev/null | head -1)"
+  [ -n "$src" ] && [ -f "$src" ] || { echo "no vm config in $1"; return 1; }
+  if cmp -s "$src" "/etc/pve/qemu-server/$2.conf"; then echo "vm $2 config already matches"; return 0; fi
+  if [ "$(qm status "$2" | awk '{print $2}')" != stopped ]; then
+    echo "vm $2 is running; config differs from the snapshot but is only restored while stopped:"
+    diff "$src" "/etc/pve/qemu-server/$2.conf" || true
+    return 0
+  fi
+  cp "$src" "/etc/pve/qemu-server/$2.conf"
+  echo "vm $2 config restored"
+}
+
+payload() { # function names... -> a script that defines them
+  declare -f remote_find remote_manifest "$@"
+  printf 'GATEWAY_FILES=%q\nSERVING_GLOBS=%q\n' "$GATEWAY_FILES" "$SERVING_GLOBS"
+}
+
+# pull <user> <host> <key> <remote-dir> <local-dir>
+pull() {
+  mkdir -p "$5"
+  set -o pipefail
+  rssh "$1" "$2" "$3" "tar cf - -C '$4' ." | tar xf - -C "$5"
+}
+
+# ---- per-role drivers ----
+
+do_role() { # <mode> <role> <label> <stamp> <with-keys>
+  local mode="$1" role="$2" label="$3" stamp="$4" keys="$5"
+  local local_dir="$LOCAL_ROOT/$stamp-$label"
+  case "$role" in
+    haproxy|gateway)
+      local u h k ct root=/root/rig-snapshots
+      u="$(cfg .proxy_host.user)"; h="$(cfg .proxy_host.host)"; k="$(cfg .proxy_host.key)"; ct="$(cfg .proxy_host.gateway_ct)"
+      if [ "$mode" = snapshot ]; then
+        local d="$root/$stamp-$label"
+        if [ "$role" = haproxy ]; then
+          rssh "$u" "$h" "$k" bash -s <<< "$(payload haproxy_snapshot); umask 077; haproxy_snapshot '$d'; remote_manifest '$d'"
+        else
+          rssh "$u" "$h" "$k" bash -s <<< "$(payload gateway_snapshot); umask 077; gateway_snapshot '$d' '$ct'; remote_manifest '$d'"
+        fi
+        pull "$u" "$h" "$k" "$d/$role" "$local_dir/$role"
+      else
+        if [ "$role" = haproxy ]; then
+          rssh "$u" "$h" "$k" bash -s <<< "$(payload haproxy_restore); d=\$(remote_find $root '$label'); [ -n \"\$d\" ] || { echo 'no snapshot labelled $label'; exit 1; }; echo \"from \$d\"; haproxy_restore \"\$d\" '$stamp'"
+        else
+          rssh "$u" "$h" "$k" bash -s <<< "$(payload gateway_restore); d=\$(remote_find $root '$label'); [ -n \"\$d\" ] || { echo 'no snapshot labelled $label'; exit 1; }; echo \"from \$d\"; gateway_restore \"\$d\" '$ct' '$keys' '$stamp'"
+        fi
+      fi
+      ;;
+    sparks)
+      local n i failed=0
+      n="$(cfg '.sparks | length')"
+      for ((i = 0; i < n; i++)); do
+        local u h k name root='$HOME/rig-snapshots'
+        u="$(cfg ".sparks[$i].user")"; h="$(cfg ".sparks[$i].host")"; k="$(cfg ".sparks[$i].key")"; name="$(cfg ".sparks[$i].name")"
+        echo "== spark $name"
+        if [ "$mode" = snapshot ]; then
+          { rssh "$u" "$h" "$k" bash -s <<< "$(payload serving_snapshot); umask 077; d=$root/$stamp-$label; serving_snapshot \"\$d\"; remote_manifest \"\$d\"; echo \"\$d\"" \
+            && pull "$u" "$h" "$k" "rig-snapshots/$stamp-$label/serving" "$local_dir/spark-$name"; } \
+            || { echo "!! spark $name snapshot failed"; failed=1; }
+        else
+          rssh "$u" "$h" "$k" bash -s <<< "$(payload serving_restore); d=\$(remote_find $root '$label'); [ -n \"\$d\" ] || { echo 'no snapshot labelled $label'; exit 1; }; echo \"from \$d\"; serving_restore \"\$d\" '$stamp' chroot" \
+            || { echo "!! spark $name restore failed"; failed=1; }
+        fi
+      done
+      return "$failed"
+      ;;
+    gpu-vm)
+      local u h k vmid gu gh gk root=/root/rig-snapshots groot='$HOME/rig-snapshots'
+      u="$(cfg .gpu_node.user)"; h="$(cfg .gpu_node.host)"; k="$(cfg .gpu_node.key)"; vmid="$(cfg .gpu_node.vmid)"
+      gu="$(cfg .gpu_vm.user)"; gh="$(cfg .gpu_vm.host)"; gk="$(cfg .gpu_vm.key)"
+      if [ "$mode" = snapshot ]; then
+        local d="$root/$stamp-$label"
+        rssh "$u" "$h" "$k" bash -s <<< "$(payload vmconf_snapshot); umask 077; vmconf_snapshot '$d' '$vmid'; remote_manifest '$d'"
+        pull "$u" "$h" "$k" "$d/gpu-vm" "$local_dir/gpu-vm-node"
+        if [ "$(rssh "$u" "$h" "$k" qm status "$vmid" | awk '{print $2}')" = running ]; then
+          rssh "$gu" "$gh" "$gk" bash -s <<< "$(payload serving_snapshot); umask 077; d=$groot/$stamp-$label; serving_snapshot \"\$d\"; remote_manifest \"\$d\""
+          pull "$gu" "$gh" "$gk" "rig-snapshots/$stamp-$label/serving" "$local_dir/gpu-vm-guest"
+        else
+          echo "four-card vm stopped: guest serving files not captured"
+        fi
+      else
+        rssh "$u" "$h" "$k" bash -s <<< "$(payload vmconf_restore); d=\$(remote_find $root '$label'); [ -n \"\$d\" ] || { echo 'no vm snapshot labelled $label'; exit 1; }; vmconf_restore \"\$d\" '$vmid'"
+        if [ "$(rssh "$u" "$h" "$k" qm status "$vmid" | awk '{print $2}')" = running ]; then
+          rssh "$gu" "$gh" "$gk" bash -s <<< "$(payload serving_restore); d=\$(remote_find $groot '$label'); [ -n \"\$d\" ] || { echo 'no guest snapshot labelled $label'; exit 0; }; echo \"from \$d\"; serving_restore \"\$d\" '$stamp' sudo"
+        fi
+      fi
+      ;;
+    *) die "unknown role $role (roles: ${ALL_ROLES[*]})" ;;
+  esac
+}
+
+mode="${1:-}"; shift || true
+case "$mode" in
+  list)
+    ls -1 "$LOCAL_ROOT" 2>/dev/null || true
+    exit 0 ;;
+  snapshot|restore) ;;
+  *) sed -n '2,6p' "$0"; exit 2 ;;
+esac
+label="${1:?label required}"; shift
+[[ "$label" =~ ^[A-Za-z0-9._-]+$ ]] || die "label may only hold letters, digits, dot, dash, underscore"
+roles=(); keys=0
+for a in "$@"; do
+  case "$a" in
+    --with-keys) keys=1 ;;
+    *) roles+=("$a") ;;
+  esac
+done
+[ "${#roles[@]}" -gt 0 ] || roles=("${ALL_ROLES[@]}")
+stamp="$(date -u +%Y%m%dT%H%M%SZ)"
+
+if [ "$mode" = snapshot ]; then
+  mkdir -p "$LOCAL_ROOT"; chmod 700 "$LOCAL_ROOT"
+fi
+rc=0
+for r in "${roles[@]}"; do
+  echo "== $mode $r ($label)"
+  do_role "$mode" "$r" "$label" "$stamp" "$keys" || { echo "!! $r failed"; rc=1; }
+done
+if [ "$mode" = snapshot ]; then
+  { echo "label: $label"; echo "utc: $stamp"; echo "roles: ${roles[*]}"; } > "$LOCAL_ROOT/$stamp-$label/SNAPSHOT"
+  echo "local copy: $LOCAL_ROOT/$stamp-$label"
+fi
+exit "$rc"
diff --git a/bench/sparks/dsv41/env.template b/bench/sparks/dsv41/env.template
new file mode 100644
index 00000000..362a1bf0
--- /dev/null
+++ b/bench/sparks/dsv41/env.template
@@ -0,0 +1,80 @@
+# .env for the MiaAI two-Spark DeepSeek-V4.1-Flash EXL3 recipe
+# (github MiaAI-Lab/DeepSeek-v4.1-Flash-EXL3-2x-DGX-Sparks, pinned commit
+# 6f7d1590ad49a2b8995188e45d7b9db31e677452), as the rig runs it. install.sh fills the
+# @...@ placeholders from its arguments; everything else is the recipe's own
+# .env.example value unless a comment says why it differs.
+
+# Head and worker talk over the 200G fabric port the retired GLM pair used (the
+# enP2p1s0f0np0 half of the first CX7 port, RoCE device roceP2p1s0f0, GID index 5),
+# not the recipe's own direct-cable addresses, which are not configured here.
+HEAD_IP=@HEAD_IP@
+WORKER_IP=@WORKER_IP@
+WORKER_USER=admin
+WORKER_SSH=dsv41-worker
+HEAD_CX7_IF=enP2p1s0f0np0
+WORKER_CX7_IF=enP2p1s0f0np0
+HEAD_CX7_IB=roceP2p1s0f0
+WORKER_CX7_IB=roceP2p1s0f0
+NCCL_IB_GID_INDEX=5
+
+# Weights come from the NAS (fetched there by bench/sparks/hf-to-nas.sh, pinned
+# revisions) and are copied to local disk on both nodes; never from Hugging Face.
+AUTO_DOWNLOAD=0
+MODEL_HOST=/srv/dsv41/model
+ENGRAM_DIR=/srv/dsv41/engram-src
+WORKER_MODEL_DIR=/srv/dsv41/model
+WORKER_ENGRAM_DIR=/srv/dsv41/engram-worker
+EXPECTED_SHARDS=39
+WEIGHT_SYNC=rsync
+NFS_SHARE=0
+ZFS_POOL=models
+
+IMAGE=ghcr.io/miaai-lab/deepseek-v4.1-flash-exl3-2x-dgx-sparks:2.9bpw
+IMAGE_SHIP=rsync
+PORT=8888
+TP=2
+NNODES=2
+# 29521/29522 were the GLM pairs' rendezvous ports.
+MASTER_PORT=29523
+QUANTIZATION=exl3
+ENFORCE_EAGER=0
+EXL3_FUSED_MOE=1
+EXL3_FAT_KERNEL=0
+EXL3_FAT_GROUPED=1
+EXL3_TEMP_ROWS_FUSED=16
+SERVED_MODEL_NAME=DeepSeek-v4.1-Flash-EXL3
+SPEC_METHOD=dspark
+DSPARK_TOKENS=3
+MAX_MODEL_LEN=600000
+MAX_NUM_SEQS=2
+MAX_NUM_BATCHED_TOKENS=1536
+GPU_MEM_UTIL=0.88
+KV_CACHE_MEMORY_BYTES=2684354560
+KV_BLOCK_SIZE=64
+PREFIX_CACHE_RETENTION_INTERVAL=4096
+DSV41_IO_THREADS=96
+DSV41_MEM_GUARD=0
+LANGUAGE_MODEL_ONLY=0
+SKIP_MM_PROFILING=1
+LIMIT_MM='{"image":100}'
+DSV41_BOOT_SHAPE_WARMUP=1
+GLM53_SUPPRESS_STOPS_IN_REASONING=1
+GLM53_MIXED_PREFILL_CHUNK=0
+GLM53_SPINWAIT_MS=stock
+USE_HOST_NCCL=0
+CG_ESTIMATE=1
+NCCL_DEBUG=WARN
+NCCL_BUFFSIZE=1048576
+NCCL_LL128_BUFFSIZE=262144
+NCCL_PROTO=^LL128
+NCCL_MAX_NCHANNELS=8
+PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+DSV41_DROP_PAGE_CACHE=1
+DSV41_SKIP_MIXED_WARMUP=0
+VLLM_DISABLE_SHARED_EXPERTS_STREAM=1
+DSV41_EXL3_SERIAL_STREAMS=1
+DSV41_INDEXER_PREFILL_FACTOR=1
+VLLM_SPARSE_INDEXER_MAX_LOGITS_MB=256
+DSV41_PREFILL_EMPTY_CACHE_TOKENS=8192
+DSV41_PREFILL_EMPTY_CACHE_MEMAVAIL_GIB=2.5
+DSV41_PREFILL_END_EMPTY_CACHE=0
diff --git a/bench/sparks/dsv41/folknet-dsv41-exl3-pair.service b/bench/sparks/dsv41/folknet-dsv41-exl3-pair.service
new file mode 100644
index 00000000..3d431c3b
--- /dev/null
+++ b/bench/sparks/dsv41/folknet-dsv41-exl3-pair.service
@@ -0,0 +1,32 @@
+[Unit]
+Description=FolkNet DeepSeek-V4.1-Flash EXL3 two-Spark pair (head; drives the worker over ssh)
+After=docker.service network-online.target
+Requires=docker.service
+Wants=network-online.target
+
+[Service]
+# The recipe's start.sh preflights, syncs weights to the worker, launches the worker
+# and head containers detached, waits for /health and warms up, then exits; the
+# containers keep serving. `start.sh stop` removes both.
+Type=oneshot
+RemainAfterExit=yes
+User=admin
+Group=admin
+Environment=HOME=/home/admin
+# The image is already on both nodes (pulled once, by digest in the rig notes); a boot
+# must not depend on the registry.
+Environment=SKIP_PULL=1
+# The published :2.9bpw image carries recipe stamp 8c01a854 (the recipe as of commit
+# f083d7e); the pinned checkout hashes to 9c03768d, which would make start.sh compile
+# the image locally. The only image-side change between the two is
+# overlay/patch_responses_content_types.py, which the pinned start.sh bind-mounts into
+# both containers at launch, so the published image is used as-is.
+Environment=SKIP_BUILD=1
+WorkingDirectory=/home/admin/dsv41
+ExecStart=/home/admin/dsv41/start.sh start
+ExecStop=/home/admin/dsv41/start.sh stop
+TimeoutStartSec=3600
+TimeoutStopSec=300
+
+[Install]
+WantedBy=multi-user.target
diff --git a/bench/sparks/dsv41/ssh-config-worker b/bench/sparks/dsv41/ssh-config-worker
new file mode 100644
index 00000000..0bcc7780
--- /dev/null
+++ b/bench/sparks/dsv41/ssh-config-worker
@@ -0,0 +1,9 @@
+# Appended to the head's ~admin/.ssh/config: the recipe's start.sh reaches the worker
+# as WORKER_SSH=dsv41-worker. The key is the Spark-to-Spark key already authorized on
+# the other Sparks; the address is the worker's 200G fabric address (rig notes).
+Host dsv41-worker
+  HostName <worker fabric address>
+  User admin
+  IdentityFile ~/.ssh/spark_sync_ed25519
+  IdentitiesOnly yes
+  StrictHostKeyChecking accept-new
diff --git a/bench/sparks/hf-to-nas.sh b/bench/sparks/hf-to-nas.sh
new file mode 100644
index 00000000..a8fca695
--- /dev/null
+++ b/bench/sparks/hf-to-nas.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+# Fetches one pinned Hugging Face checkpoint onto the rig's NAS through the multi-WAN
+# downloader on the downloader host. Runs ON that host as root:
+#
+#   ssh root@<downloader-host> "NAS_MOUNT=<nas-mount> bash -s -- <owner/repo> <revision-sha> <bundle-name>" < bench/sparks/hf-to-nas.sh
+#
+# NAS_MOUNT is required: the directory the NAS is mounted at on the downloader host.
+# Checkpoints go under $NAS_MOUNT/models unless NAS_MODELS says otherwise.
+#
+# It builds a runtime bundle under /root/folknet-runtime/<bundle-name>/ the way the
+# host's existing bundles are laid out: the large (LFS) files go through
+# huggingface-multiwan-shards.sh over all WANs with SHA-256 checks, small files
+# (configs, tokenizer, code) through plain curl, and a finalize step moves the staging
+# directory $NAS_MODELS/<owner>/<repo>/.multiwan-<revision> to <revision>
+# and points `current` at it. The downloader scripts and the WAN table are copied from
+# an existing bundle on the host ($TEMPLATE_BUNDLE), so no address lives here.
+#
+# INCLUDE='<regex>' (matched against each whole repository path) limits the fetch to a
+# subset, e.g. two shards plus the index and config of a checkpoint whose other shards
+# are never read.
+#
+# The download runs as the oneshot unit folknet-<bundle-name>-multiwan-download.service
+# (restarted on failure; the shard driver resumes and re-verifies what is already there).
+# Check it with:
+#   systemctl status folknet-<bundle-name>-multiwan-download
+#   journalctl -u folknet-<bundle-name>-multiwan-download -f
+#   du -sh $NAS_MODELS/<owner>/<repo>/.multiwan-<revision>
+# Done when $NAS_MODELS/<owner>/<repo>/current points at <revision>.
+set -euo pipefail
+main() {
+
+REPO="${1:?usage: hf-to-nas.sh <owner/repo> <revision-sha> <bundle-name>}"
+REV="${2:?usage: hf-to-nas.sh <owner/repo> <revision-sha> <bundle-name>}"
+NAME="${3:?usage: hf-to-nas.sh <owner/repo> <revision-sha> <bundle-name>}"
+TEMPLATE_BUNDLE="${TEMPLATE_BUNDLE:-/root/folknet-runtime/glm53-exl3}"
+: "${NAS_MOUNT:?set NAS_MOUNT to the directory the NAS is mounted at}"
+NAS_MOUNT="${NAS_MOUNT%/}"
+NAS_MODELS="${NAS_MODELS:-$NAS_MOUNT/models}"
+MOUNT_UNIT="$(systemd-escape --path --suffix=mount "$NAS_MOUNT")"
+
+B="/root/folknet-runtime/$NAME"
+DEST="$NAS_MODELS/$REPO"
+STAGE="$DEST/.multiwan-$REV"
+UNIT="folknet-$NAME-multiwan-download"
+
+mountpoint -q "$NAS_MOUNT" || { echo "NAS not mounted at $NAS_MOUNT" >&2; exit 1; }
+mkdir -p "$B" "$STAGE"
+for f in multiwan-download.sh huggingface-multiwan-shards.sh multiwan-download.tsv; do
+  cp -n "$TEMPLATE_BUNDLE/$f" "$B/$f"
+done
+
+# Manifests from the Hugging Face API: LFS files with size and SHA-256, and the rest.
+curl -fsSL "https://huggingface.co/api/models/$REPO/revision/$REV?blobs=true" -o "$B/revision.json"
+python3 - "$B" "${INCLUDE:-}" <<'PY'
+import json, re, sys
+b, include = sys.argv[1], sys.argv[2]
+d = json.load(open(f"{b}/revision.json"))
+with open(f"{b}/model-lfs.tsv", "w") as lfs, open(f"{b}/small-files.txt", "w") as small:
+    lfs.write("filename\tsize\tsha256\n")
+    for s in d["siblings"]:
+        if include and not re.fullmatch(include, s["rfilename"]):
+            continue
+        if s.get("lfs"):
+            assert "/" not in s["rfilename"], s["rfilename"]
+            lfs.write(f'{s["rfilename"]}\t{s["lfs"]["size"]}\t{s["lfs"]["sha256"]}\n')
+        else:
+            small.write(s["rfilename"] + "\n")
+PY
+
+cat > "$B/fetch-small.sh" <<EOF
+#!/usr/bin/env bash
+set -euo pipefail
+while read -r f; do
+  [ -n "\$f" ] || continue
+  mkdir -p "$STAGE/\$(dirname "\$f")"
+  [ -s "$STAGE/\$f" ] || curl -fsSL --retry 5 -o "$STAGE/\$f" "https://huggingface.co/$REPO/resolve/$REV/\$f"
+done < "$B/small-files.txt"
+EOF
+cat > "$B/finalize.sh" <<EOF
+#!/usr/bin/env bash
+set -euo pipefail
+test -f "$STAGE/.multiwan-shards-complete-$REV"
+test -f "$STAGE/config.json"
+if [ -d "$DEST/$REV" ]; then echo "$DEST/$REV already exists"; else mv "$STAGE" "$DEST/$REV"; fi
+ln -sfn "$REV" "$DEST/current"
+echo "published $DEST/current -> $REV"
+EOF
+chmod +x "$B/fetch-small.sh" "$B/finalize.sh"
+
+cat > "/etc/systemd/system/$UNIT.service" <<EOF
+[Unit]
+Description=Download pinned $REPO ($REV) to the NAS over four WANs
+Requires=$MOUNT_UNIT
+After=$MOUNT_UNIT network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=$B
+ExecStart=$B/fetch-small.sh
+ExecStart=$B/huggingface-multiwan-shards.sh --manifest $B/model-lfs.tsv --output-dir $STAGE --repo $REPO --revision $REV --downloader $B/multiwan-download.sh --config $B/multiwan-download.tsv --file-concurrency 4 --chunk-size-mib 64 --connections-per-wan 4 --curl-retries 5
+ExecStart=$B/finalize.sh
+Restart=on-failure
+RestartSec=30s
+TimeoutStartSec=infinity
+EOF
+systemctl daemon-reload
+systemctl start --no-block "$UNIT.service"
+echo "started $UNIT; staging at $STAGE"
+awk -F'\t' 'NR>1 {s+=$2; n++} END {printf "%d large files, %.1f GB\n", n, s/1e9}' "$B/model-lfs.tsv"
+
+}
+main "$@"
diff --git a/docs/infrastructure.md b/docs/infrastructure.md
index 0919e9e7..6c7f5436 100644
--- a/docs/infrastructure.md
+++ b/docs/infrastructure.md
@@ -227,11 +227,13 @@ HTTP inventory (CPU model and core count, memory, disks, virtual-machine guests)
 collector aggregates the lot. The plan above was written from partial information; this
 is what the announcements say, described by role and class only.
 
-- **The model gateway** is a LiteLLM instance fronting six aliases: the measured dense
+- **The model gateway** is a LiteLLM instance fronting eight aliases: the measured dense
   27B Qwen (`qwen3.8-27b`), the same model on an eight-card host as `qwen3.8-27b-fast`
-  and `qwen3.8-27b-long`, GLM-5.3 Flash as `glm-5.3-flash` and
-  `glm-5.3-flash-exl3` (no backends since 2026-09-22, see below), and
-  Qwen3.8-Flash-Next as `qwen3.8-flash-next` (four single-Spark replicas). Its virtual
+  and `qwen3.8-27b-long` and on a four-card VM as `qwen3.8-27b-fast-2`, GLM-5.3 Flash as
+  `glm-5.3-flash` and `glm-5.3-flash-exl3` (no backends since 2026-09-22, see below),
+  Qwen3.8-Flash-Next as `qwen3.8-flash-next` (two single-Spark replicas since
+  2026-09-24, four before), and DeepSeek-V4.1-Flash as `deepseek-v4.1-flash` (one
+  two-Spark tensor-parallel pair). Its virtual
   keys carry per-key model allowlists, so a new alias also has to be granted to the key
   a caller uses. Each backend pool sits behind its own HAProxy and reports
   liveness but not readiness, so a `/v1/models` answer is not proof a completion will
@@ -383,26 +385,52 @@ stay in the local rig notes and `bench/*.local.json`.
 - **Eight-card sampling host.** Bare Ubuntu, not Proxmox, so no guest layer: docker
   containers, each holding cards, behind the `-fast` and `-long` aliases. No card is
   free. Stopped and started as docker containers on the host.
-- **Four ARM unified-memory nodes.** Each runs one systemd unit around one docker
-  container serving one replica on port 5000 (`bench/sparks/`). In front of them: the
-  proxy host's HAProxy (least connections, health-checked) and the LiteLLM gateway, which
-  is an LXC on the same proxy host and is reached with `pct exec` from that host's root.
-  The unit is stopped and started through a privileged container that chroots into the
-  host, because the admin account has no sudo (a bare `docker stop` races the unit's
-  restart policy). The replicas serve independently, so any one can be stopped alone; the
-  rule to stop the two nodes of a pair together applies only to the two-node
-  tensor-parallel GLM pool, whose units are stopped and disabled.
-- **Four-card Qwen server.** A virtual machine with four consumer cards passed through,
-  on one Proxmox node; started and stopped as a VM.
+- **Four ARM unified-memory nodes, split two ways since 2026-09-24.**
+  - *Two nodes serve `qwen3.8-flash-next`*, each one systemd unit around one docker
+    container serving one replica on port 5000 (`bench/sparks/`). In front of them: the
+    proxy host's HAProxy (least connections, health-checked, eight in flight per
+    replica) and the LiteLLM gateway, which is an LXC on the same proxy host and is
+    reached with `pct exec` from that host's root. With two replicas the alias takes
+    **16 requests in flight, not 32**; a batch drawn at `GENERA_CONCURRENCY=32` (as
+    `specs/overflow-scrollbars.md` Stage 2 sets for it) queues the rest in HAProxy. The
+    replicas serve independently, so either can be stopped alone.
+  - *The other two, a pair joined by a direct cable, serve DeepSeek-V4.1-Flash* as one
+    two-node tensor-parallel group (the MiaAI two-Spark EXL3 recipe, pinned commit, 2.9
+    bpw checkpoint plus the model's Engram tables on local disk), alias
+    `deepseek-v4.1-flash`. One oneshot unit on the head node,
+    `bench/sparks/dsv41/folknet-dsv41-exl3-pair.service`, runs the recipe's `start.sh`,
+    which starts the worker's container over ssh, then the head's, and returns once the
+    API is healthy (about 7.5 minutes from cold); `ExecStop` removes both containers.
+    Stop and start the pair only through that unit on the head, never one node alone. Its
+    own HAProxy frontend and backend on the proxy host, `maxconn 2`, since the recipe
+    runs two sequences at once. The flash-next unit and config on these two nodes are
+    disabled, not deleted.
+
+  Units on these nodes are stopped and started through a privileged container that
+  chroots into the host, because the admin account has no sudo (a bare `docker stop`
+  races a unit's restart policy). The GLM pool's units are stopped and disabled on all
+  four.
+- **Four-card qwen3.8-27b VM (alias `qwen3.8-27b-fast-2`).** A VM on a Proxmox node with
+  four 3090s passed through, borrowed from a stopped tenant test VM on the same node
+  (`bench/gpu-vm/create-vm.sh`; only one of the two VMs may run at a time, and the tenant
+  VM stays off at boot). Provisioned by `bench/gpu-vm/provision.sh`. Four systemd units
+  `folknet-qwen38-vllm-worker@fast-{0..3}` (`bench/gpu-vm/`), each a docker container on
+  one card running the eight-card host's own `-fast` recipe (same image, same prepared
+  checkpoint, four sequences per card), published on ports 8101–8104. In front: a
+  frontend and backend of its own in the proxy host's HAProxy (least connections,
+  `maxconn 4` per worker, so 16 in flight), then the gateway alias
+  `qwen3.8-27b-fast-2`. Started and stopped as a VM on the node, or per worker with
+  `systemctl` inside it (the login account has sudo). It is a separate alias rather than
+  a fifth member of `qwen3.8-27b-fast` because that alias's load balancer runs on the
+  eight-card host, which this project does not edit.
 
 **Open.** Each line names what would settle it.
 
-- Which node carries the four-card Qwen VM, how its cards are attached, and how it is
-  started and stopped are not recorded. The gateway's backend address for `qwen3.8-27b`
-  names the VM; `qm list` and `qm config` on its node give the rest.
-- How the eight-card host's containers are supervised (restart policy, whether a unit
-  owns them) is not recorded; `docker inspect` and `systemctl list-units` on that host
-  would show it.
+- The eight-card host's containers are owned by a templated unit,
+  `folknet-qwen38-vllm-worker@<profile>` (restart on failure), each running one docker
+  container: four one-card `fast` workers and two two-card `long` workers behind that
+  host's own HAProxy on port 4000. Whether anything restarts that HAProxy or re-pulls the
+  image is not recorded.
 - What keeps the development box's local port forwards to the gateway up after a drop or
   reboot is not recorded. The process list on the development box, and whatever starts
   those processes, would show whether a unit or a login script owns them; none is in this
@@ -911,7 +939,13 @@ update invalidates only the versions it touches, never silently.
 ## Rig change log
 
 Every change tried on the rig's hardware, worked or not, newest last. Hosts by role and
-class only.
+class only. Before a change, and again after each change that works,
+`bench/rig-snapshot.sh snapshot <label>` records the serving configs of every role (proxy
+host HAProxy, gateway container config and key database, each ARM node's units and
+TabbyAPI configs, the four-card VM's Proxmox config and units) on the host itself and in
+the local rig notes; `restore <label>` puts them back. The label named in an entry holds
+the working state after it. Snapshots contain the gateway's master key and never go into
+git or onto the NAS.
 
 **2026-09-24**
 
@@ -922,3 +956,142 @@ class only.
   `FOLKNET_WRITER_API_KEY` in `$HOME/.config/folknet-writer/env` (mode 600) on the
   development box. First used by stage 0 of `docs/folknet-writer-design.md`
   (`docs/writer-harness-spike.md` → "Rerun through Pi"). Worked.
+- *Snapshot and restore.* `bench/rig-snapshot.sh` written; `known-good-2026-09-24`
+  taken before any serving change. `restore known-good-2026-09-24 haproxy` run once as the
+  round-trip test: config check passed, HAProxy reloaded, all four ARM replicas still up.
+  Worked.
+- *Four-card VM.* A new VM on a Proxmox node, given the four 3090s of a stopped tenant
+  test VM (same `hostpci` entries, q35, OVMF, 256 GB 64-bit PCI hole), 32 cores, 160 GB
+  RAM, a 120 GB OS disk, DHCP on the main LAN (`bench/gpu-vm/create-vm.sh`). The tenant
+  VM's disks were not touched. Worked; four 3090s visible.
+- *Provisioning, first two attempts failed.* First: the fresh cloud image was still
+  running cloud-init's package upgrade, which held the dpkg lock and then rebooted the
+  guest. Second: the rig's apt cache answers `403 CONNECT denied` for HTTPS repositories,
+  so docker's and NVIDIA's apt repositories could not be reached through it. Fixed in
+  `bench/gpu-vm/provision.sh`: wait for cloud-init and the dpkg lock, send HTTPS
+  repositories direct, take docker from the Ubuntu archive (`docker.io`, which the cache
+  serves). Third run worked: driver 595 (open kernel modules) loaded without a reboot,
+  `nvidia-smi` shows four 3090s, docker has the NVIDIA runtime.
+- *Model choice for the four cards, by size (no download attempted).*
+  - `qwen3.8-flash-next`, EXL3 4.05 bpw (the ARM nodes' checkpoint): 101 GB on disk, of
+    which 39 GB is the hashed n-gram table that exllamav3 keeps in host RAM or on disk,
+    so about 62 GB of weights would sit on the cards. That the checkpoint does not fit
+    96 GB was assumed, never tested. A 3.05 bpw branch is also published (85 GB on disk,
+    32.6 GB of it n-gram table). Not pursued: the target moved to a different model family
+    for variety.
+  - DeepSeek-V4-Flash (the July release, 284B parameters, 13B active): 167 GB native
+    (MXFP4 experts). Mainline llama.cpp with CUDA serves it on sm86 from GGUF. The
+    smallest quant its quantizer recommends (3-bit, 104 GB) does not fit 96 GB of VRAM;
+    2-bit quants (87–91 GB) fit with only a few GB left for cache and compute buffers,
+    and re-quantizing the native 4-bit experts to 2 bits costs a lot of accuracy
+    (published KL divergence around 0.4 at 2 bits against 0.01 at 4 bits). Download
+    straight from Hugging Face inside the VM measured about 250 KB/s. Not tried.
+  - DeepSeek-V4.1-Flash is a different, larger model, not a revision of the one above: a
+    new architecture (`deepseek_v41`, 384 experts over 40 layers, n-gram memory tables),
+    510 GB native. It does not fit the four cards. The first reading, that it fits no
+    pair of ARM nodes either, was wrong: it looked only at GGUF quants (smallest about
+    260 GB) and missed a published EXL3 recipe for exactly two GB10s — a 2.9 bpw EXL3
+    checkpoint (~197 GiB, about 99.5 GiB of weights per node at tensor-parallel 2) plus
+    the model's unquantized n-gram ("Engram") tables, which stay on disk and are read
+    per lookup rather than held in memory (two native shards, ~190 GiB), served by a
+    vLLM build that carries the `deepseek_v41` architecture. That recipe is what the
+    ARM pair uses (entries below).
+  - qwen3.8-27b, W4A16 AutoRound with DFlash2 speculative decoding, the eight-card
+    host's `-fast` recipe: about 22 GB per card, one worker per card. Chosen.
+- *qwen3.8-27b on the four cards.* The eight-card host's image and prepared checkpoint
+  (24 GB) were streamed to the VM read-only (`tar` and `docker save` piped over ssh,
+  nothing on that host changed). One worker per card came up in about 6.5 minutes from
+  cold; all four report healthy and answer chat completions. Worked; snapshot
+  `vm-qwen27b-workers-up`.
+- *Proxy and gateway for the VM.* A new HAProxy frontend and backend on the proxy host
+  (the existing backends untouched; backup `haproxy.cfg.pre-vm-qwen27b-20260924` beside
+  the config) and a gateway alias `qwen3.8-27b-fast-2` (backup
+  `config.yaml.pre-qwen27b-vm-20260924`), granted to the project's key only. The first
+  gateway attempt failed with "Missing credentials": an OpenAI-style entry needs some
+  `api_key` even when the upstream checks none; `sk-none`, as the flash-next entry uses,
+  fixed it. Worked: the alias is listed and returns code through the gateway; snapshot
+  `vm-qwen27b-fast-2-alias`.
+- *Measurement batch at `qwen3.8-27b-fast-2`* on a build lane: one 32-candidate
+  absolute-seam batch, 2/32 compiled, best 74/108 (`docs/model-first-cost.md`, row
+  `vm-batch 1`). Stopping the loop after batch 1 with `pkill -f` on the shared lane also
+  killed another track's run of the same binary in its own lane worktree (a mixed-model
+  batch at 32 of 64 draws). On a shared lane, stop a run by the pid whose
+  `/proc/<pid>/cwd` is this worktree's lane copy, never by name.
+- *Checkpoint fetch for an ARM pair.* `bench/sparks/hf-to-nas.sh` (a runtime bundle and
+  oneshot unit on the downloader host, reusing that host's multi-WAN scripts) started
+  DeepSeek-V4-Flash (July release, 167 GB); stopped after 14 GB when the ARM target
+  changed to DeepSeek-V4.1-Flash. The partial staging directory is left on the NAS. Then
+  both parts of the V4.1 recipe, pinned by revision: the EXL3 2.9 bpw checkpoint
+  (42 files, 210.7 GB) and the two Engram shards plus index and config of the native
+  checkpoint (203.1 GB), both through the multi-WAN downloader at about 130 MB/s
+  combined. Snapshot before: `pre-deepseek-sparks`.
+- *Which two ARM nodes.* LLDP shows each node's first 200G port on the switch and its
+  second port cabled straight to one other node, in two cabled pairs, each crossing
+  the old GLM pairs (which ran over the switch). The DeepSeek pair is one of the cabled
+  pairs; the other cabled pair keeps flash-next. Its tensor-parallel traffic is set to
+  run over the switch port with the RoCE settings the GLM pairs used, because the direct
+  cable carries no addresses today; moving it onto the cable means addressing that
+  port on both nodes (not done).
+- *Staging the pair (nothing serving yet).* The recipe is cloned at a pinned commit on
+  the head node, its `.env` written from `bench/sparks/dsv41/env.template` (weights from
+  local disk, no Hugging Face download, `AUTO_DOWNLOAD=0`), an ssh alias for the worker
+  added to the head's admin account (`bench/sparks/dsv41/ssh-config-worker`; previous
+  config kept as `config.pre-dsv41`), `/srv/dsv41` created on both nodes, and the
+  recipe image pulled on both (the `2.9bpw` tag, 22.5 GB; digest in the local rig
+  notes). Both nodes still serve flash-next. Snapshot `dsv41-staged`.
+- *Throughput sweep of `qwen3.8-27b-fast-2`* (`bench/rig_throughput.py`) on a build lane:
+  failed twice before sending a request — the lane's Python has no `requests` module,
+  and `python3 -m venv` fails there without the `python3.12-venv` package. Not run.
+- *Four-card VM reboot.* All four workers healthy three minutes after `reboot`, same
+  DHCP address, HAProxy backends back UP without intervention. Worked.
+- *Second measurement batch at `qwen3.8-27b-fast-2`*, after the prompt fixes, run with
+  `bench/absolute-mixed-batch.sh` on a build lane: 8/32 compiled, best 82/108, 233 s for
+  the batch (`docs/model-first-cost.md`, row `vm-batch 2`). The run ended by itself
+  (`GENERA_MAX_BATCHES=1`); nothing needed stopping. No serving change.
+- *Throughput sweep of `qwen3.8-27b-fast-2`, second attempt*, run from the dev box
+  (which has `requests`) through the gateway, with the prompt generated on a build lane:
+  worked, 819 output tok/s at 16 in flight (`docs/rig-capacity.md` → "Measured — the
+  four-card VM"). No serving change.
+- *Abandoned DeepSeek-V4-Flash download removed* from the downloader host: the 14 GB
+  partial staging directory on the NAS, its oneshot unit and runtime bundle. Worked; the
+  two V4.1 units and their published trees are untouched.
+- *DeepSeek-V4.1-Flash weights to local disk on the ARM pair.* Both NAS downloads had
+  finished and published. `rsync -a` from the NAS mount to local NVMe on both nodes, the
+  EXL3 checkpoint (39 shards, 197 GB) to each, the two Engram shards with index and
+  config (190 GB) to the head's engram source directory and to the worker's engram
+  directory. The worker gets no second engram tree: the recipe's `start.sh` builds the
+  slim engram set on the head and rsyncs it to the worker, and with the NAS mtimes kept,
+  that rsync finds the shards already in place. About 100 MB/s per node, both done in
+  about 55 minutes. Both nodes kept serving flash-next throughout, all backends UP
+  after. Worked; no serving change, so no new snapshot (`dsv41-staged` still holds the
+  serving state).
+- *Two ARM nodes out of the flash-next pool.* Snapshot before: `pre-dsv41-pair-move`.
+  Their two server lines were removed from the flash-next HAProxy backend (backup
+  `haproxy.cfg.pre-dsv41-pair-20260924` beside the config; config check, reload; the
+  other two replicas UP), then their flash-next units disabled and stopped; unit,
+  config and checkpoint stay on disk. Nothing was drawing from the alias at the time.
+  The alias now takes 16 requests in flight (two replicas × 8), not 32. Worked; snapshot
+  `flashnext-two-replicas`.
+- *DeepSeek-V4.1-Flash on the pair.* The published recipe image carries the recipe's
+  own build stamp from four commits before the pinned one, so the pinned `start.sh`
+  would have compiled the image on the node. The only image-side change in between is
+  one Python patch, which that `start.sh` bind-mounts into both containers anyway, so the
+  unit runs the published image as-is (`SKIP_BUILD=1`, reason in the unit file). Unit
+  installed on the head and started: preflight passed (memory headroom 117.6 GiB
+  available per node against 111.5 needed, RoCE GID accepted), the weight sync to the
+  worker found everything already in place and took 3 s, weights loaded and warmed up,
+  API healthy about 7.5 minutes after the start; the README's `17*19` request returned
+  `323`. The recipe's own warmup reports one sampler-kernel variant not pre-compiled
+  ("may JIT mid-serve"); harmless so far. Worked; snapshot `dsv41-pair-serving`.
+- *Proxy and gateway for the pair.* A new HAProxy frontend and backend on the proxy
+  host, one server, `maxconn 2` (backup `haproxy.cfg.pre-dsv41-backend-20260924`), and
+  a gateway alias `deepseek-v4.1-flash` → the recipe's served model name, `api_key:
+  sk-none` (backup `config.yaml.pre-dsv41-20260924`), granted to the project's key only.
+  Worked: the alias is listed through the gateway and a genera-shaped chat completion
+  (thinking off) returned a one-function Rust code block in 2.7 s with no reasoning in
+  `content`. Snapshot `dsv41-alias-serving`.
+- *First measuring batch at `deepseek-v4.1-flash`* on a build lane, 8 candidates at 2 in
+  flight: 4/8 compiled, best 82/108, 1,731 s (`docs/model-first-cost.md`, row
+  `dsv-batch 1`). The first launch failed before drawing anything: the lane's shared git
+  mirror refused a ref update (`cannot lock ref 'refs/heads/main'`) because another run
+  had moved `main` at the same moment; a plain retry worked. No serving change.
diff --git a/docs/model-first-cost.md b/docs/model-first-cost.md
index f041acf1..630f5ba6 100644
--- a/docs/model-first-cost.md
+++ b/docs/model-first-cost.md
@@ -65,6 +65,9 @@ per-category tally is the admitted (best) candidate's.
 | diagnosis batch (mixed, 16 `qwen3.8-flash-next` + 16 `qwen3.8-27b-fast`, before the fixes below) | 11/32: flash-next 10/16 (6 scored, 4 panicked), 27b-fast 1/16 | `model_2` (flash-next) | 74/108 | flash-next best 74/108, flash-batch 1's tally; 27b-fast best 63/108, rig-batch 1's tally | 27b-fast only, not summed | not summed; 27b-fast replies 37–104 s, flash-next 332–484 s |
 | mixed-batch 1 (32 flash-next + 32 27b-fast, both at 32 in flight) | 23/64: flash-next 13/19 (13 of its 32 requests dropped, see below; 11 scored), 27b-fast 10/32 (10 scored) | `model_40` (27b-fast) | 82/108 | 27b-fast best 82/108, flash-batch 3's tally; flash-next best 74/108, flash-batch 1's tally | 27b-fast only, not summed | not summed; wall-clock 737.9 s |
 | mixed-batch 2 (same shape, three-import contract) | 29/64: flash-next 25/32 (22 scored), 27b-fast 4/32 (3 scored) | `model_58` (27b-fast) | 82/108 | as mixed-batch 1: 27b-fast best 82/108 with `OverConstrainedRtl` 8/8, flash-next best 74/108 | 6,624 per 27b-fast request | not summed; wall-clock 849.8 s, slowest flash-next reply 760.2 s |
+| vm-batch 1 (`qwen3.8-27b-fast-2`, 32 candidates, concurrency 16, before the prompt fixes) | 2/32 | `model_13` (tied with `model_18` at 74/108) | 74/108 | identical to flash-batch 1 above | 212,160 (0/32 null-content) | 197,661 (12 of 32 replies stopped at the 8,000-token cap); slowest single reply 106.5 s |
+| vm-batch 2 (`qwen3.8-27b-fast-2`, same shape, after the prompt fixes) | 8/32 (8 scored) | `model_4` (tied with `model_5` at 82/108) | 82/108 | flash-batch 3's tally: `OverConstrainedRtl` 8/8, `MarginAutoCentering` 11/11 | 211,968 (6,624 per request) | 175,004 (0 of 32 at the 12,000-token cap, longest 11,788); wall-clock 233.1 s, slowest single reply 107.9 s |
+| dsv-batch 1 (`deepseek-v4.1-flash`, 8 candidates, concurrency 2) | 4/8 (4 scored) | `model_7` | 82/108 | flash-batch 3's tally: `OverConstrainedRtl` 8/8, `MarginAutoCentering` 11/11 | 50,496 (6,312 per request) | 76,115 (5 of 8 at the 12,000-token cap); wall-clock 1,731.3 s, replies 143–512 s |
 
 The three `rig-batch N` rows are one unattended run of `absolute_real_model_run` against the
 rig's free `qwen3.8-27b-fast` alias (`NEV_MODEL=qwen3.8-27b-fast NEV_CANDIDATES=32
@@ -93,7 +96,60 @@ pipe the way the `rig-batch N` rows above did. Five batches, 160 draws, 5,345.3
 total; the loop stopped itself after batch 5 on two consecutive batches with no per-category
 improvement, short of the six-batch cap.
 
-Three things separate this alias from `qwen3.8-27b-fast`. Its compile rate is about five
+The `vm-batch 1` row is one batch of the same binary at `qwen3.8-27b-fast-2`
+(`GENERA_MODEL=qwen3.8-27b-fast-2 GENERA_CANDIDATES=32 GENERA_CONCURRENCY=16`), the same
+model and serving recipe as `qwen3.8-27b-fast` on four more cards of the same class, one
+worker per card, four requests each (`docs/infrastructure.md` → "Where each piece
+runs"). It started from the same state as `flash-batch 1` (the 6-case seed regression
+set, the same 22,959-byte prompt) and was run to measure the new alias against the
+recorded ones, so it was stopped after batch 1 rather than left to loop. Its compile rate
+matches the older alias (2 of 32, against 10 of 96 in the three `rig-batch` rows) and is
+far below `qwen3.8-flash-next`'s (14 of 32 from the same starting state); its best, 74/108
+with `MarginAutoCentering` 11/11, is above every `rig-batch` candidate (63/108) and equal
+to `flash-batch 1`'s, short of the committed 82/108, so nothing was committed. Draws were
+fast: all 32 replies back within 106.5 s of their own start, against 584–600 s for the
+slowest flash-next reply, but 12 of 32 ran into the 8,000-token output cap.
+
+The `vm-batch 2` row reruns that batch after the prompt fixes in "Why candidates fail to
+compile" below (seam files shown with `layout::` paths, three import lines, a 12,000-token
+cap): `GENERA_MODELS=qwen3.8-27b-fast-2:32:16 bash bench/absolute-mixed-batch.sh` on a
+build lane, the same 32 candidates at 16 in flight, from the same reseeded 6-case regression
+set (a 22,944-byte prompt). 8 of 32 compiled, four times vm-batch 1's 2, and none
+panicked while scoring. Two candidates reached 82/108 with `OverConstrainedRtl` 8/8 and
+`MarginAutoCentering` 11/11, tying the committed candidate, so nothing was committed; the
+other six scored 74, 74, 74, 73, 63 and 51. No reply reached the new cap (longest 11,788
+tokens, every one `finish_reason: "stop"`) and the whole batch took 233.1 s. The 24
+failures are model-side, the same kinds as mixed-batch 2's 27b-fast failures: 11
+candidates have a type mismatch (`E0308`), 6 declare `fn resolve` without `pub` (`E0603`;
+for one of them that is the only error), 5 read a name out of scope (`E0425`), 4 call a
+method that does not exist (`E0599`), 3 assign twice to an immutable binding (`E0384`),
+and one has a syntax error. None imports `crate::width` or leaves out `Direction`. This
+matches the older alias after the same fixes (mixed-batch 1 and 2: 10/32 and 4/32
+compiled, best 82/108 both times), as expected of the same model and recipe on the same
+card class.
+
+The `dsv-batch 1` row is the first batch from a different model family:
+DeepSeek-V4.1-Flash (EXL3 2.9 bpw, two ARM nodes at tensor-parallel 2, alias
+`deepseek-v4.1-flash`), `GENERA_MODELS=deepseek-v4.1-flash:8:2 bash
+bench/absolute-mixed-batch.sh`, 8 candidates at 2 in flight (the pair runs two sequences
+at once), same reseeded 6-case regression set. 4 of 8 compiled and none panicked; three
+scored 74/108 and `model_7` 82/108 with `OverConstrainedRtl` 8/8, tying the committed
+candidate, so nothing was committed. It is slow: replies took 143–512 s each at about 26
+output tokens/s per request, 1,731 s for the batch including the build, and the model's
+built-in speculative decoding accepted almost no drafted tokens on this prompt (mean
+acceptance length 1.02). The request's `enable_thinking: false` is honoured (the
+template reads it, and a smoke request with it reported 0 reasoning tokens), but with thinking off this
+model still reasons at length in prose inside `content`, writing and revising code
+blocks as it goes: six replies did that, and five of them hit the 12,000-token cap
+mid-thought. `extract_rust_source` still found a compiling block in three of those five.
+The two that answered with one code block and nothing else (3,049 and 3,909 tokens) both
+failed to compile, one on a missing `mut`, one on possibly-uninitialized bindings. Its
+failures differ from the Qwen models': no path or import faults and no private `fn
+resolve`; the other two failures are replies cut off at the cap, whose extracted block
+was a fragment. At 2 in flight a
+32-candidate batch would take about two hours.
+
+Three things separate `qwen3.8-flash-next` from `qwen3.8-27b-fast`. Its compile rate is about five
 times higher — 75 of 160 candidates compiled (14, 12, 10, 20, 19 per batch) against 10 of 96
 at the old alias. Its ceiling is higher: batch 3's `model_31` scored 82/108 and was committed
 over the paid `claude-opus-5` candidate's 74/108 through `admit::beats_committed`, the first
diff --git a/docs/rig-capacity.md b/docs/rig-capacity.md
index 40e73832..fae5c215 100644
--- a/docs/rig-capacity.md
+++ b/docs/rig-capacity.md
@@ -261,6 +261,30 @@ twice the cards giving close to proportional decode bandwidth, though the two po
 not swept to the same top concurrency (this sweep stops at 16, the four-card sweep also
 tested 32).
 
+### Measured — the four-card VM (`qwen3.8-27b-fast-2`)
+
+The same model and serving recipe as the eight-card pool, one worker per card on four
+3090s in a VM (`docs/infrastructure.md` → "Where each piece runs"), four requests per
+worker at the proxy. `bench/rig_throughput.py 1 2 4 8 16` through the gateway from the dev
+box, same request shape, the width-equation prompt `bin/real_model_run.rs` assembles
+(15,678 bytes, 5,040 prompt tokens per request), one round per level, 2026-09-24:
+
+| concurrency | completions/s | output tok/s | time to first token, mean (s) | latency, mean (s) | latency, max (s) |
+|---|---|---|---|---|---|
+| 1  | 0.070 | 106.8 | 5.9 | 14.3 | 14.3 |
+| 2  | 0.087 | 254.1 | 5.7 | 21.5 | 22.9 |
+| 4  | 0.174 | 317.7 | 5.1 | 15.8 | 22.9 |
+| 8  | 0.322 | 685.1 | 4.7 | 18.2 | 24.8 |
+| 16 | 0.398 | 819.3 | 8.8 | 27.6 | 40.2 |
+
+At 16 in flight, its designed ceiling (four workers × four), it delivered more output
+tokens per second than the eight-card pool's recorded 523 at the same level. The two
+sweeps are not a controlled comparison: they ran on different days, and while this one
+ran, another writer was drawing overflow-seam batches from `qwen3.8-27b-fast` (the
+eight-card pool) and `qwen3.8-flash-next`, but nothing else was on this VM. What it does
+show is that the VM's alias is not the slower of the two, and that 16 in flight is where
+its time to first token starts to climb.
+
 ### Measured — GLM-5.3 Flash (2-way tensor parallel, 4-bit EXL3)
 
 A second model on the rig, served by vLLM directly at `127.0.0.1:18001` (model id
-- 
2.54.0