From 34d67d31b9220ccf11a112d19563cb315398e456 Mon Sep 17 00:00:00 2001 From: Patrick Singletary Date: Sun, 16 Aug 2026 13:34:34 -0400 Subject: [PATCH] =?UTF-8?q?backup3:=20remove=20rsync=20--max-size=20?= =?UTF-8?q?=E2=80=94=20silent=20data-loss=20bug=20(4=20.mov=20files,=20~3.?= =?UTF-8?q?3GB,=202026-08-16)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --max-size=N makes rsync SKIP files over N (never transferred), but --remove-source-files still deletes them from the source, exit 0, no error. Proven via sandbox repro. --checksum now applies to every file unconditionally. --- bin/backup3.zsh | 10 +++++++--- config/backup3/backup3.conf | 6 ++++-- docs/BACKUP3-PLAN.md | 3 ++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/bin/backup3.zsh b/bin/backup3.zsh index ddea3f1..3d12543 100755 --- a/bin/backup3.zsh +++ b/bin/backup3.zsh @@ -80,8 +80,12 @@ move_entry() { [[ -e "$src" ]] || return 0 (( APPLY )) && mkdir -p "$dst" # never touch the filesystem in dry-run local opts; opts=("${RSYNC_OPTS[@]}") - local max=$((CHECKSUM_MAX_MB * 1024 * 1024)) - opts+=("--checksum" "--max-size=${max}") + # CRITICAL: never pass --max-size to rsync here. rsync's --max-size SKIPS + # files larger than the limit (never transferred), but --remove-source-files + # still deletes them from the source, exit 0, no error. This silently lost + # 4 .mov files (~3.3GB) on 2026-08-16. --checksum alone gives verify-then- + # delete for every file (checksum all; size+mtime is NOT a fallback here). + opts+=(--checksum) (( APPLY )) || opts+=(--dry-run) local verb; (( APPLY )) && verb="MOVE" || verb="WOULD-MOVE" if [[ -d "$src" ]]; then @@ -232,7 +236,7 @@ status() { echo "config: $CONFIG_DIR" echo "X10 mounted: $(x10_mounted && echo yes || echo no) ($X10_ROOT, uuid $X10_UUID)" echo "OneDrive root: $ONEDRIVE_ROOT $([[ -d "$ONEDRIVE_ROOT" ]] && echo ok || echo MISSING)" - echo "size threshold: ${SIZE_THRESHOLD_MB}MB checksum-max: ${CHECKSUM_MAX_MB}MB ledger-hash-max: ${LEDGER_HASH_MAX_MB}MB" + echo "size threshold: ${SIZE_THRESHOLD_MB}MB checksum: always ledger-hash-max: ${LEDGER_HASH_MAX_MB}MB" echo "categories:" local line cat src od xd always for line in "${CATEGORIES[@]}"; do diff --git a/config/backup3/backup3.conf b/config/backup3/backup3.conf index 01321cd..4724303 100644 --- a/config/backup3/backup3.conf +++ b/config/backup3/backup3.conf @@ -26,8 +26,10 @@ RSYNC_OPTS=( --exclude=.DS_Store --exclude='._*' --exclude=.localized --exclude=node_modules ) -# Files up to this size are checksum-verified by rsync; larger use size+mtime. -CHECKSUM_MAX_MB=200 +# NOTE: CHECKSUM_MAX_MB was REMOVED 2026-08-16. It was passed to rsync as +# --max-size, which SKIPS (never transfers) files over the limit while +# --remove-source-files still deletes them from the source — silent data loss +# (4 .mov files, ~3.3GB). --checksum now applies to every file unconditionally. # --- Ledger --- # sha256 recorded for moved files up to this size; larger files record size only. diff --git a/docs/BACKUP3-PLAN.md b/docs/BACKUP3-PLAN.md index 4c935f3..36bfaf7 100644 --- a/docs/BACKUP3-PLAN.md +++ b/docs/BACKUP3-PLAN.md @@ -116,7 +116,8 @@ Excluded entirely: ~/Library, ~/dev (git/Tangled), ~/.hermes, ~/.npm, ~/.cache, `7A471DF2-9F4E-474B-B236-068C7EABC006` (prevents acting on a wrong drive at that path) - Machine: hostname-derived (ole-blu / mac-studio), loads `config/backup3/machines/.conf` - Verify-then-delete: rsync `-rt --no-perms --no-owner --no-group --modify-window=2 - --checksum --max-size=200MB --remove-source-files` (checksum for <=200MB, size+mtime + --checksum --remove-source-files` (checksum ALL files — see the max-size + pitfall below; size+mtime was never a fallback) above); never `--delete`; empty source dirs pruned; `.DS_Store`/`._*`/node_modules excluded - Ledger: `~/.local/state/backup3/ledger.csv` (ts|machine|category|source|dest|size|sha256, sha for files <=100MB); mirrored to `X10/archive/_ledger/` on apply — enables restore + dedupe -- 2.51.2