From acbe89a1f555f213f389099c5e76a109606bf2c7 Mon Sep 17 00:00:00 2001 From: File Magic Date: Sat, 25 Jul 2026 22:33:20 -0400 Subject: [PATCH] harden/debug/: automated VM testing and documentation - run-test.sh: boot VM, SSH in, run security checks with pass/fail reporting - README.md: debugging reference, QEMU flags, process detachment, known issues --- harden/debug/README.md | 93 ++++++++++++++++++++++++ harden/debug/run-test.sh | 150 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 harden/debug/README.md create mode 100755 harden/debug/run-test.sh diff --git a/harden/debug/README.md b/harden/debug/README.md new file mode 100644 index 0000000..5b7c6b7 --- /dev/null +++ b/harden/debug/README.md @@ -0,0 +1,93 @@ +# Hardened NixOS VM Debugging + +## Prerequisites + +- NixOS VM built via `nixos-rebuild build-vm --flake .#vm` (from `harden/`) +- Serial console enabled: `boot.kernelParams = ["console=ttyS0"]` +- SSH enabled with key auth in `harden/configuration.nix` +- SSH key pair at `harden/debug/ssh-key` (generated per-project, gitignored) + +## Quick Reference + +### Build & Boot + +```bash +cd harden +nixos-rebuild build-vm --flake .#vm +``` + +The run script is in the nix store (printed by the build). Disk image is created at `./harden-vm.qcow2` on first boot. + +### Automated Test (recommended) + +```bash +cd harden +setsid bash debug/run-test.sh > debug/test-output.log 2>&1 & +# Wait ~2min for boot + checks, then: +cat debug/test-output.log +``` + +The test script boots the VM, waits for SSH, runs all security checks, and outputs results. + +### Manual SSH into VM + +```bash +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -i harden/debug/ssh-key -p 2222 nixos@localhost +``` + +### Run Security Checks Inside VM + +```bash +SSH="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -i harden/debug/ssh-key -p 2222 nixos@localhost" + +$SSH "cat /proc/cmdline" +$SSH "cat /proc/sys/kernel/kptr_restrict" # expect 2 +$SSH "cat /proc/sys/kernel/dmesg_restrict" # expect 1 +$SSH "cat /proc/sys/kernel/sysrq" # expect 0 +$SSH "aa-status 2>&1 | head -5" +$SSH "systemctl --failed --no-pager" +``` + +### Cleanup + +```bash +kill $(cat harden/debug/vm.pid) 2>/dev/null +rm -f harden/harden-vm.qcow2 harden/debug/serial.log harden/debug/test-output.log +``` + +## Key QEMU Flags + +| Flag | Purpose | +|------|---------| +| `-nographic` | Serial to terminal (requires `console=ttyS0`) | +| `QEMU_NET_OPTS="hostfwd=tcp::2222-:22"` | SSH port forwarding | +| `-cpu host` | Use host CPU (kvm: needed for `lockdown=confidentiality`) | + +## Process Detachment + +QEMU must run in a detached session to survive tool timeouts. Use `setsid`: + +```bash +setsid bash debug/run-test.sh > debug/test-output.log 2>&1 & +``` + +Without `setsid`, opencode's bash tool kills child processes on timeout. + +## Known Issues + +1. **OVMF doesn't output to serial** — firmware messages only on VGA +2. **QEMU user networking opens ports immediately** — open port != booted guest +3. **`generate-sb-keys.service` and `prepare-sb-auto-enroll.service` fail in VM** — these are Lanzaboote services that try to write Secure Boot keys to UEFI firmware. In QEMU without real OVMF enrollment support, they will always fail. This is expected and harmless for testing. + +## Files + +| Path | Purpose | +|------|---------| +| `harden/debug/ssh-key` | SSH private key (gitignored) | +| `harden/debug/ssh-key.pub` | SSH public key (git-tracked, read by config) | +| `harden/debug/run-test.sh` | Automated test script | +| `harden/debug/test-output.log` | Test output capture | +| `harden/debug/serial.log` | Serial console capture | +| `harden/harden-vm.qcow2` | VM disk image (auto-created) | diff --git a/harden/debug/run-test.sh b/harden/debug/run-test.sh new file mode 100755 index 0000000..6405f75 --- /dev/null +++ b/harden/debug/run-test.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +HARDEN_DIR="$(dirname "$SCRIPT_DIR")" +SERIAL_LOG="$SCRIPT_DIR/serial.log" +PID_FILE="$SCRIPT_DIR/vm.pid" +SSH_KEY="$SCRIPT_DIR/ssh-key" + +# Find the run-harden-vm-vm via nix build (always gets the current flake output) +cd "$HARDEN_DIR" +RUN_SCRIPT_DIR=$(nix build --no-link --print-out-paths '.#nixosConfigurations.vm.config.system.build.vm' 2>/dev/null) +RUN_SCRIPT="$RUN_SCRIPT_DIR/bin/run-harden-vm-vm" +if [ ! -x "$RUN_SCRIPT" ]; then + echo "ERROR: run-harden-vm-vm not found. Run 'nixos-rebuild build-vm --flake .#vm' first." + exit 1 +fi +echo "Using: $RUN_SCRIPT" + +cleanup() { + if [ -f "$PID_FILE" ]; then + kill "$(cat "$PID_FILE")" 2>/dev/null || true + rm -f "$PID_FILE" + fi +} +trap cleanup EXIT + +# Clean old state +rm -f "$SERIAL_LOG" "$HARDEN_DIR/harden-vm.qcow2" + +# Start VM fully detached +cd "$HARDEN_DIR" +setsid env QEMU_NET_OPTS="hostfwd=tcp::2222-:22" \ + "$RUN_SCRIPT" -nographic > "$SERIAL_LOG" 2>&1 & +echo $! > "$PID_FILE" +echo "VM started, PID=$(cat "$PID_FILE")" + +# Wait for SSH (up to 90s) +echo "Waiting for SSH..." +for i in $(seq 1 90); do + if ssh -o StrictHostKeyChecking=no \ + -o ConnectTimeout=2 \ + -o UserKnownHostsFile=/dev/null \ + -i "$SSH_KEY" \ + -p 2222 nixos@localhost true 2>/dev/null; then + echo "SSH ready after ${i}s" + break + fi + if [ "$i" -eq 90 ]; then + echo "SSH timeout after 90s" + echo "=== Serial log (last 50 lines) ===" + tail -50 "$SERIAL_LOG" + exit 1 + fi + sleep 1 +done + +# Run checks +echo "" +echo "=== Running checks ===" +FAILURES=0 + +ssh_cmd() { + ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -i "$SSH_KEY" \ + -p 2222 nixos@localhost "$@" +} + +check() { + local name="$1" + local expected="$2" + local actual="$3" + if echo "$actual" | grep -q "$expected"; then + echo " PASS: $name (got '$actual')" + else + echo " FAIL: $name (expected '$expected', got '$actual')" + FAILURES=$((FAILURES + 1)) + fi +} + +echo "--- Kernel command line ---" +CMDLINE=$(ssh_cmd "cat /proc/cmdline") +echo " $CMDLINE" +check "lockdown=confidentiality" "lockdown=confidentiality" "$CMDLINE" +check "module.sig_enforce=1" "module.sig_enforce=1" "$CMDLINE" +check "slab_nomerge" "slab_nomerge" "$CMDLINE" +check "init_on_alloc=1" "init_on_alloc=1" "$CMDLINE" +check "init_on_free=1" "init_on_free=1" "$CMDLINE" +check "apparmor=1" "apparmor=1" "$CMDLINE" + +echo "--- Kernel hardening ---" +KPTR=$(ssh_cmd "cat /proc/sys/kernel/kptr_restrict") +check "kptr_restrict=2" "^2$" "$KPTR" +DMESG=$(ssh_cmd "cat /proc/sys/kernel/dmesg_restrict") +check "dmesg_restrict=1" "^1$" "$DMESG" +SYSRQ=$(ssh_cmd "cat /proc/sys/kernel/sysrq") +check "sysrq=0" "^0$" "$SYSRQ" +MODS=$(ssh_cmd "cat /proc/sys/kernel/modules_disabled") +check "modules_disabled (soft fail)" "^0$" "$MODS" + +echo "--- AppArmor ---" +AA=$(ssh_cmd "aa-status 2>&1 | head -5") +echo "$AA" +if echo "$AA" | grep -q "apparmor module is loaded"; then + echo " PASS: AppArmor module loaded" +else + echo " FAIL: AppArmor module not loaded" + FAILURES=$((FAILURES + 1)) +fi + +echo "--- Failed services ---" +FAILED=$(ssh_cmd "systemctl --failed --no-pager --no-legend") +echo "$FAILED" +if [ -z "$FAILED" ]; then + echo " PASS: No failed services" +else + # Lanzaboote SB key services are expected to fail in QEMU VM + UNEXPECTED=$(echo "$FAILED" | grep -v "sb-keys\|sb-auto-enroll" || true) + if [ -z "$UNEXPECTED" ]; then + echo " PASS: Only Lanzaboote SB services failed (expected in VM)" + else + echo " FAIL: Unexpected failed services:" + echo "$UNEXPECTED" + FAILURES=$((FAILURES + 1)) + fi +fi + +echo "--- Loaded modules ---" +ssh_cmd "lsmod | head -15" + +echo "--- User config ---" +ID_OUT=$(ssh_cmd "id nixos 2>/dev/null || echo 'NOT_FOUND'") +echo " $ID_OUT" +check "user nixos exists" "nixos" "$ID_OUT" +check "user in wheel group" "wheel" "$ID_OUT" + +SSHD_CONF=$(ssh_cmd "grep -E '^(PermitRoot|PasswordAuth)' /etc/ssh/sshd_config 2>/dev/null || echo 'NOT_FOUND'") +echo " $SSHD_CONF" +check "PasswordAuthentication no" "PasswordAuthentication no" "$SSHD_CONF" +check "PermitRootLogin prohibit-password" "PermitRootLogin prohibit-password" "$SSHD_CONF" + +echo "" +echo "=== Results ===" +if [ "$FAILURES" -eq 0 ]; then + echo "All checks PASSED" +else + echo "$FAILURES check(s) FAILED" +fi +echo "VM still running at PID $(cat "$PID_FILE"). Use 'kill \$(cat debug/vm.pid)' to stop." -- 2.51.2