#!/usr/bin/env bash # backup.sh — daily restic backup (§6.2). # # A zero exit from `restic backup` is NOT proof of a successful backup: it # exits 0 having archived an empty or wrong directory. This script therefore # asserts CONTENT. Without that, a misconfigured path would report green for # up to 90 days, until the quarterly restore drill finally noticed. set -uo pipefail : "${FLIT_ETC:=/etc/flit}" # so a hand-run works from the default; a unit # run overrides it via EnvironmentFile=. # systemd supplies these via EnvironmentFile= (read as root, then privileges # dropped to the operator account). Sourcing them here would fail when run by # hand as the operator account, because secrets.env is 0600 root:root — so # source only what is readable, and rely on the unit otherwise. # shellcheck disable=SC1091 [[ -r "$FLIT_ETC/env" ]] && { set -a; source "$FLIT_ETC/env"; set +a; } [[ -r "$FLIT_ETC/secrets.env" ]] && { set -a; source "$FLIT_ETC/secrets.env"; set +a; } : "${RESTIC_REPOSITORY:?not set}" : "${RESTIC_PASSWORD:?not set}" : "${FLIT_NAME:=flit}" : "${SENTINEL_PATH:=$HOME/workspace/.${FLIT_NAME}-sentinel}" : "${BASELINE_FILE:=/var/lib/${FLIT_NAME}/backup-baseline}" : "${SIZE_FLOOR_PCT:=50}" # a new snapshot below this % of baseline fails export RESTIC_REPOSITORY RESTIC_PASSWORD # restic reads extended options only from -o on the command line — there is no # environment variable it consults, so RESTIC_OPTIONS from $FLIT_ETC/env # reaches restic only if passed explicitly, at every call site below. restic_opts=() [[ -n "${RESTIC_OPTIONS:-}" ]] && restic_opts=(-o "$RESTIC_OPTIONS") fail() { echo "BACKUP ASSERTION FAILED: $*" >&2; exit 1; } # The sentinel proves we are backing up the real workspace. Created if absent # so a fresh machine is not permanently unverifiable. mkdir -p "$(dirname "$SENTINEL_PATH")" [[ -f "$SENTINEL_PATH" ]] || date -Is >"$SENTINEL_PATH" # The crontab lives in /var/spool, outside anything restic reads, so it is # copied under ~/.config first. No crontab means no stale copy either. cron_copy="${HOME}/.config/crontab.backup" crontab -l >"$cron_copy" 2>/dev/null || rm -f "$cron_copy" # Outside ~/workspace: small state the box cannot regenerate — credentials # (.ssh, .gnupg, .config, .railway, .cloudflared, .claude.json), the scripts and # state the cron jobs use (.local/bin, .local/state), and working directories # with no remote (rig-notes, peek-backups, lane-shared, repos, claude-config, # forge, tap.db, folknet, .pi, .openhands), and ~/flit, whose gitignored # deployment.md and .env exist nowhere else, and ~/box-backup, a daily copy # (its own cron job) of the crontab and the dagu config and scripts. Deliberately absent: ~/lanes (clones) # and ~/dagu (27 GB, contents not yet reviewed), and build output. restic "${restic_opts[@]}" backup \ --exclude-caches \ --exclude='**/node_modules' \ --exclude='**/.pnpm-store' \ --exclude='**/target' \ --exclude="$HOME/.cache" \ "$HOME/workspace" "$HOME/.claude" "$HOME/.bash_history" "$HOME/.vim/undo" \ "$HOME/.ssh" "$HOME/.gnupg" "$HOME/.config" "$HOME/.railway" "$HOME/.cloudflared" \ "$HOME/.claude.json" "$HOME/.local/bin" "$HOME/.local/state" "$HOME/rig-notes" \ "$HOME/peek-backups" "$HOME/lane-shared" "$HOME/repos" "$HOME/claude-config" \ "$HOME/forge" "$HOME/tap.db" "$HOME/folknet" "$HOME/.pi" "$HOME/.openhands" \ "$HOME/flit" "$HOME/box-backup" \ || fail "restic backup exited nonzero" restic "${restic_opts[@]}" forget --prune --keep-daily 7 --keep-weekly 4 --keep-monthly 6 \ || fail "restic forget/prune exited nonzero" # ---- assertion 1: the newest snapshot actually contains the sentinel latest=$(restic "${restic_opts[@]}" snapshots --json --latest 1 | jq -r '.[0].short_id // empty') [[ -n "$latest" ]] || fail "no snapshot exists after a successful backup" # Not `restic ls | grep -qF`: grep -q exits at the first match, the still-writing # restic takes SIGPIPE, and pipefail reports 141 for the pipeline — so a PASSING # assertion fails as soon as the listing outgrows the pipe buffer. It read as a # missing sentinel, which is the alarm for backing up the wrong path. Never put # an early-exit consumer on the end of an unbounded producer under pipefail. # Materialising the listing also separates a restic failure from a real miss, # which the discarded stderr had conflated. ls_out=$(mktemp -t "${FLIT_NAME}-ls.XXXXXX") trap 'rm -f "$ls_out"' EXIT restic "${restic_opts[@]}" ls "$latest" >"$ls_out" 2>/dev/null \ || fail "restic ls of snapshot $latest exited nonzero" grep -qF "$SENTINEL_PATH" "$ls_out" \ || fail "snapshot $latest does not contain the sentinel $SENTINEL_PATH — wrong path backed up?" # ---- assertion 2: size is plausible against the recorded baseline size=$(restic "${restic_opts[@]}" stats "$latest" --mode raw-data --json 2>/dev/null | jq -r '.total_size // 0') [[ "$size" -gt 0 ]] || fail "snapshot $latest reports zero bytes" mkdir -p "$(dirname "$BASELINE_FILE")" if [[ -f "$BASELINE_FILE" ]]; then baseline=$(cat "$BASELINE_FILE") floor=$(( baseline * SIZE_FLOOR_PCT / 100 )) [[ "$size" -ge "$floor" ]] \ || fail "snapshot is ${size}B against a ${baseline}B baseline (floor ${floor}B) — sudden collapse" else # First backup: no previous snapshot. Assert the sentinel only (done above) # and record the baseline. "No baseline" must never silently satisfy this # check, or the very first backup would be the least verified one. echo "no baseline yet; recording ${size}B from snapshot ${latest}" >&2 fi echo "$size" >"$BASELINE_FILE" echo "backup ok: snapshot ${latest}, ${size} bytes, sentinel present"