From 960fdd4127ae8a5c27c1f8abd3ff372120a69b3f Mon Sep 17 00:00:00 2001 From: Patrick Singletary Date: Thu, 20 Aug 2026 17:45:12 -0400 Subject: [PATCH] unified-backup: one X10-canonical backup for Studio+Air (MOVE) + 2TB (COPY) - unified-backup.zsh: dedup-aware machine MOVE phase + efficient 2TB COPY (rsync --ignore-existing, no full re-hash). Writes into canonical X10 tree. - backup3.conf: canonical X10 targets; USE_ONEDRIVE=no (launchd OneDrive deadlocks) - machines/*.conf: canonical mapping; 2TB phase on Studio only - launchd: point com.psingletary.backup3 at unified-backup.zsh --apply - docs: UNIFIED-BACKUP-MAPPING.md (design + resolved decisions) - fixed zsh 5.9 quirk: per-iteration 'local' printed assignments (hoisted to fn top) --- bin/unified-backup.zsh | 321 +++++++++++++++++++++++++++++++++ docs/UNIFIED-BACKUP-MAPPING.md | 83 +++++++++ 2 files changed, 404 insertions(+) create mode 100755 bin/unified-backup.zsh create mode 100644 docs/UNIFIED-BACKUP-MAPPING.md diff --git a/bin/unified-backup.zsh b/bin/unified-backup.zsh new file mode 100755 index 0000000..232d114 --- /dev/null +++ b/bin/unified-backup.zsh @@ -0,0 +1,321 @@ +#!/bin/zsh +# unified-backup.zsh — UNIFIED X10 backup for Studio + Air (machine MOVE) + Crucial 2TB (COPY). +# Writes ALL sources into the canonical X10 tree (authoritative). Dedup via _ledger/provenance.csv. +# Canonical copy: ~/dev/_shared/bin/unified-backup.zsh (Tangled-synced). +# Design: ~/dev/_shared/docs/UNIFIED-BACKUP-MAPPING.md +# +# Modes: +# unified-backup.zsh dry-run (report only, no changes) +# unified-backup.zsh --apply move/copy for real (verify-then-delete + ledger) +# unified-backup.zsh --status machine, config, mount state, category sizes +# unified-backup.zsh --no-2tb skip the 2TB COPY phase (machine phase only) +# unified-backup.zsh --help +# +# Semantics: +# - Machine phase (Studio/Air): MOVE. Each file: sha256. +# if sha in dedup authority (_ledger/provenance.csv) -> content already canonical: +# log provenance row (this machine also has it) + remove local (safe, frees disk). +# else -> verify-then-move into canonical target, add provenance row. +# - 2TB phase (Studio only, COPY): rsync --checksum, NO --remove-source-files, NO --delete. +# skip files whose sha is already canonical. +# +# Designed for launchd (com.psingletary.backup3.plist) with --apply; exits 0 immediately +# when X10 is not attached (safe on 60s interval). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CONFIG_DIR="$SCRIPT_DIR/../config/backup3" +STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/backup3" +mkdir -p "$STATE_DIR/logs" +RUNLOG="$STATE_DIR/logs/run-$(date +%Y%m%d-%H%M%S).log" +touch "$RUNLOG" + +APPLY=0 DO_2TB=1 MODE=run +for arg in "$@"; do + case "$arg" in + --apply) APPLY=1 ;; + --no-2tb) DO_2TB=0 ;; + --status) MODE=status ;; + --help|-h) MODE=help ;; + *) echo "unknown arg: $arg (see --help)" >&2; exit 2 ;; + esac +done + +HOSTNAME_RAW="$(scutil --get LocalHostName 2>/dev/null || hostname)" +HOST="$(echo "$HOSTNAME_RAW" | tr '[:upper:]' '[:lower:]')" +case "$HOST" in + ole-blu*|ole*blu*) MACHINE=ole-blu ;; + mac-studio*|mac*studio*|mac-studio-2022*) MACHINE=mac-studio ;; + *) MACHINE="$HOST" ;; +esac + +# shellcheck disable=SC1090 +source "$CONFIG_DIR/backup3.conf" +MACHINE_CONF="$CONFIG_DIR/machines/$MACHINE.conf" +if [[ -f "$MACHINE_CONF" ]]; then + # shellcheck disable=SC1090 + source "$MACHINE_CONF" +else + echo "no machine config for '$MACHINE' (expected $MACHINE_CONF) — dry-run only" | tee -a "$RUNLOG" + CATEGORIES=() +fi + +log() { echo "$*" | tee -a "$RUNLOG"; } +err() { echo "ERROR: $*" >&2 | tee -a "$RUNLOG" >&2; } + +x10_mounted() { + [[ -d "$X10_ROOT" ]] || return 1 + local uuid + uuid="$(diskutil info "$X10_ROOT" 2>/dev/null | sed -nE 's/^[[:space:]]*Disk \/ Partition UUID:[[:space:]]*([0-9A-F-]+).*/\1/p')" + [[ "$uuid" == "$X10_UUID" ]] +} + +# --- dedup authority: set of shas already canonical on X10 --- +# PROVENANCE (index, comma, 4 cols: sha|path|size|topdir) + MACHINE_PROV (this backup, sha|path|size|machine|src) +DEDUP_KEYS="$STATE_DIR/dedup-keys.txt" +PROVENANCE_INDEX="$CANON_ROOT/_ledger/provenance.csv" +MACHINE_PROV="$CANON_ROOT/_ledger/provenance-machines.csv" +load_dedup() { + : > "$DEDUP_KEYS" + if [[ -f "$PROVENANCE_INDEX" ]]; then + cut -d',' -f1 "$PROVENANCE_INDEX" | tr -d '\r' >> "$DEDUP_KEYS" + fi + if [[ -f "$MACHINE_PROV" ]]; then + cut -d',' -f1 "$MACHINE_PROV" | tr -d '\r' >> "$DEDUP_KEYS" + fi + sort -u -o "$DEDUP_KEYS" "$DEDUP_KEYS" +} +is_canonical_sha() { grep -Fqx "$1" "$DEDUP_KEYS" >/dev/null 2>&1; return $?; } + +# append machine-provenance row: sha,canonical_path,size,machine,source_relpath +prov_row() { + local sha="$1" path="$2" size="$3" rel="$4" + printf '%s,%s,%s,%s,%s\n' "$sha" "$path" "$size" "$MACHINE" "$rel" >> "$STATE_DIR/provenance-new.csv" +} + +# look up an existing canonical path for a sha from the index (comma, col2) +canon_path_of() { + local sha="$1" + grep -F "${sha}," "$PROVENANCE_INDEX" 2>/dev/null | head -1 | cut -d',' -f2 | tr -d '\r' || true +} + +# ensure a path stays inside a canonical root (safety) +guard() { + local base="$1" child="$2" + case "$child" in "$base"/*) ;; *) echo "BLOCKED: $child outside $base" >&2; return 1;; esac +} + +# --- process a single source FILE: dedup-or-move --- +# src_file, canonical_target_dir, cat_name, one_drive_base +proc_file() { + local src="$1" tgt_dir="$2" catname="$3" od="$4" + local base="${src:t}" size sha + size="$(stat -f%z "$src" 2>/dev/null || echo 0)" + # size split: small -> OneDrive, large -> canonical (only for split-policy cats; handled by caller) + sha="$(shasum -a 256 "$src" 2>/dev/null | cut -d' ' -f1 || true)" + local dst="$tgt_dir/$base" + if is_canonical_sha "$sha"; then + # content already canonical on X10 -> log + remove local (MOVE semantics, safe) + if (( APPLY )); then + local existing + existing="$(canon_path_of "$sha")" + [[ -n "$existing" ]] || existing="$dst" + prov_row "$sha" "$existing" "$size" "$src" + rm -f "$src" + fi + log " DEDUP-MOVE [$catname] $src (sha already canonical, local removed)" + else + # unique -> move into canonical target (verify-then-delete) + if (( APPLY )); then + mkdir -p "$tgt_dir" + if ! rsync "${RSYNC_OPTS[@]}" --checksum --remove-source-files -- "$src" "$tgt_dir/" 2>>"$RUNLOG"; then + err "rsync failed: $src -> $tgt_dir"; return 1 + fi + prov_row "$sha" "$dst" "$size" "$src" + else + log " WOULD-MOVE [$catname] $src -> $tgt_dir/ ($size B)" + fi + fi +} + +# --- machine category pass --- +run_categories() { + local line cat src tgt policy od entry base sz f + for line in "${CATEGORIES[@]}"; do + IFS='|' read -r cat src tgt policy <<< "$line" + [[ -d "$src" ]] || continue + log "== category: $cat (src=$src -> $tgt)" + od="$ONEDRIVE_ROOT/$cat/$MACHINE" + if [[ "$USE_ONEDRIVE" != yes ]]; then od="$tgt"; fi # no-OneDrive: small files also go to X10 canonical + for entry in "$src"/*(N); do + base="${entry:t}" + if [[ -d "$entry" ]]; then + # directory: whole-entry size split + sz="$(du -sk "$entry" 2>/dev/null | cut -f1 || true)" + if [[ "$policy" == always_x10 ]] || (( sz >= SIZE_THRESHOLD_MB * 1024 )); then + # move dir: iterate files individually for dedup, then prune empty + while IFS= read -r f; do + proc_file "$f" "$tgt/$base" "$cat" "$od" + done < <(find "$entry" -type f 2>/dev/null) + if (( APPLY )); then find "$entry" -depth -type d -empty -delete 2>/dev/null || true; fi + else + # small dir -> OneDrive + if (( APPLY )); then + mkdir -p "$od/$base" + rsync "${RSYNC_OPTS[@]}" --checksum --remove-source-files -- "$entry/" "$od/$base/" 2>>"$RUNLOG" || { err "rsync $entry -> $od/$base"; } + find "$entry" -depth -type d -empty -delete 2>/dev/null || true + else + log " WOULD-MOVE dir [small->OneDrive] $entry -> $od/$base" + fi + fi + elif [[ -f "$entry" ]]; then + # single file: size split + sz="$(stat -f%z "$entry" 2>/dev/null || echo 0)" + if [[ "$policy" == always_x10 ]] || (( sz >= SIZE_THRESHOLD_MB * 1024 * 1024 )); then + proc_file "$entry" "$tgt" "$cat" "$od" + else + if (( APPLY )); then + mkdir -p "$od" + rsync "${RSYNC_OPTS[@]}" --checksum --remove-source-files -- "$entry" "$od/" 2>>"$RUNLOG" || { err "rsync $entry -> $od"; } + else + log " WOULD-MOVE [small->OneDrive] $entry -> $od/" + fi + fi + fi + done + done +} + +# --- Downloads bespoke routing --- +INSTALLER_EXTS=(dmg pkg exe msi iso zip tar gz tgz bz2 xz 7z) +AUDIO_EXTS=(mp3 m4a wav flac aac ogg opus) + +route_downloads() { + local src="$HOME/Downloads" od="$ONEDRIVE_ROOT" + [[ -d "$src" ]] || return 0 + # no-OneDrive mode: small files route to X10 canonical Downloads instead + if [[ "$USE_ONEDRIVE" != yes ]]; then od="$CN_DOWNLOADS"; fi + log "== category: Downloads (bespoke routing)" + local entry base ext sz + for entry in "$src"/*(N); do + base="${entry:t}" + if [[ -d "$entry" ]]; then + case "$base" in + stacher) move_dldir "$entry" "$CN_NEW/stacher" ;; # adult content policy -> z-New folder/ + Media) move_dldir "$entry" "$CN_MEDIA/Other-Video/$base" ;; + Music) move_dldir "$entry" "$CN_MUSIC/$base" ;; + *) sz="$(du -sk "$entry" 2>/dev/null | cut -f1 || true)"; (( sz >= SIZE_THRESHOLD_MB*1024 )) \ + && move_dldir "$entry" "$CN_DOWNLOADS/$base" \ + || move_dldir "$entry" "$od/$base" ;; + esac + else + ext="${base:e:l}" + if (( INSTALLER_EXTS[(Ie)$ext] )); then + proc_file "$entry" "$CN_ARCHIVE/installers" Downloads-installers "$od" + elif (( AUDIO_EXTS[(Ie)$ext] )); then + proc_file "$entry" "$CN_MUSIC" Downloads-music "$od" + elif [[ "$ext" == pdf ]]; then + proc_file "$entry" "$CN_DOCUMENTS" Downloads-docs "$od" + else + sz="$(stat -f%z "$entry" 2>/dev/null || echo 0)" + (( sz >= SIZE_THRESHOLD_MB*1024*1024 )) \ + && proc_file "$entry" "$CN_DOWNLOADS" Downloads "$od" \ + || proc_file "$entry" "$CN_DOWNLOADS" Downloads "$od" # no-OneDrive: small files -> X10 canonical Downloads + fi + fi + done +} +move_dldir() { + local entry="$1" tgt="$2" + local f + while IFS= read -r f; do proc_file "$f" "$tgt" Downloads "$ONEDRIVE_ROOT"; done < <(find "$entry" -type f 2>/dev/null) + if (( APPLY )); then find "$entry" -depth -type d -empty -delete 2>/dev/null || true; fi +} + +# --- 2TB COPY phase (Studio only) --- +# Efficient: 2TB maps 1:1 to canonical paths, so use rsync --ignore-existing (path-based +# skip of already-present files) + --checksum for verify. NO per-file shasum (would re-read +# the whole 2TB on every launchd fire). --ignore-existing never overwrites an existing +# canonical file; new 2TB files get copied. Never deletes source. +run_2tb() { + [[ "$ENABLE_2TB_PHASE" == yes ]] || { log "2TB phase disabled on $MACHINE"; return 0; } + [[ -d "$SRC_2TB" ]] || { log "2TB not mounted ($SRC_2TB) — 2TB phase skipped"; return 0; } + log "== 2TB COPY phase ($SRC_2TB -> canonical, --ignore-existing)" + local map_line m_src m_tgt sub + for map_line in "${TBT_MAP[@]}"; do + IFS='|' read -r m_src m_tgt <<< "$map_line" + [[ -d "$SRC_2TB/$m_src" ]] || continue + if (( APPLY )); then + mkdir -p "$m_tgt" + if ! rsync "${RSYNC_OPTS[@]}" -r --ignore-existing --checksum -- "$SRC_2TB/$m_src/" "$m_tgt/" 2>>"$RUNLOG"; then + err "2TB copy failed: $SRC_2TB/$m_src -> $m_tgt"; continue + fi + log " [2TB] rsync $m_src -> $m_tgt (ignore-existing)" + else + log " [2TB] WOULD-COPY $m_src -> $m_tgt (ignore-existing)" + fi + done + # Media subdirs (Movies/TV) -> Media/ + for sub in "${TBT_MEDIA_SUBDIRS[@]}"; do + [[ -d "$SRC_2TB/Media/$sub" ]] || continue + if (( APPLY )); then + mkdir -p "$CN_MEDIA/$sub" + rsync "${RSYNC_OPTS[@]}" -r --ignore-existing --checksum -- "$SRC_2TB/Media/$sub/" "$CN_MEDIA/$sub/" 2>>"$RUNLOG" || { err "2TB media copy failed: $sub"; continue; } + log " [2TB] rsync Media/$sub -> $CN_MEDIA/$sub" + else + log " [2TB] WOULD-COPY Media/$sub -> $CN_MEDIA/$sub" + fi + done + # photos-restore.photoslibrary: opaque, copy as a whole bundle (skip existing) + if [[ -d "$SRC_2TB/photos-restore.photoslibrary" ]] && [[ ! -d "$CN_PHOTOS" ]]; then + if (( APPLY )); then + log " [2TB] COPY opaque photos-restore bundle -> $CN_PHOTOS" + mkdir -p "$(dirname "$CN_PHOTOS")" + rsync "${RSYNC_OPTS[@]}" --checksum -r -- "$SRC_2TB/photos-restore.photoslibrary/" "$CN_PHOTOS/" 2>>"$RUNLOG" || err "photos bundle copy failed" + else + log " [2TB] WOULD-COPY photos-restore bundle -> $CN_PHOTOS" + fi + fi +} + +# --- status --- +status() { + echo "machine: $MACHINE (host=$HOST)" + echo "config: $CONFIG_DIR" + echo "X10 mounted: $(x10_mounted && echo yes || echo no) ($X10_ROOT, uuid $X10_UUID)" + echo "canonical root: $CANON_ROOT" + echo "provenance: $PROVENANCE_INDEX + $MACHINE_PROV ($([[ -f "$PROVENANCE_INDEX" ]] && wc -l < "$PROVENANCE_INDEX" || echo 0) index rows)" + echo "size threshold: ${SIZE_THRESHOLD_MB}MB checksum: always ledger-hash-max: ${LEDGER_HASH_MAX_MB}MB" + echo "2TB phase: $ENABLE_2TB_PHASE ($([[ -d "$SRC_2TB" ]] && echo 2TB-mounted || echo 2TB-absent))" + echo "categories:" + local line cat src tgt pol + for line in "${CATEGORIES[@]}"; do IFS='|' read -r cat src tgt pol <<< "$line"; printf ' %-10s %-9s %s\n' "$cat" "$pol" "$src"; done +} + +case "$MODE" in + help) sed -n '2,24p' "$0"; exit 0 ;; + status) status; exit 0 ;; +esac + +load_dedup +: > "$STATE_DIR/provenance-new.csv" +if ! x10_mounted; then + log "$(date +%F_%T) X10 not mounted at $X10_ROOT — nothing to do (launchd-safe exit 0)" + exit 0 +fi +log "$(date +%F_%T) === unified-backup start ($MACHINE, apply=$APPLY, 2tb=$DO_2TB) ===" + +run_categories +route_downloads +(( DO_2TB )) && run_2tb + +# merge new machine-provenance rows into the machine ledger (only on apply) +if (( APPLY )) && [[ -s "$STATE_DIR/provenance-new.csv" ]]; then + mkdir -p "$CANON_ROOT/_ledger" + if [[ ! -f "$MACHINE_PROV" ]]; then + echo "unique_sha,canonical_path,size_bytes,machine,source_relpath" > "$MACHINE_PROV" + fi + cat "$STATE_DIR/provenance-new.csv" >> "$MACHINE_PROV" + sort -u -o "$MACHINE_PROV" "$MACHINE_PROV" 2>/dev/null || true +fi +log "=== done. run log: $RUNLOG ===" diff --git a/docs/UNIFIED-BACKUP-MAPPING.md b/docs/UNIFIED-BACKUP-MAPPING.md new file mode 100644 index 0000000..c5bfb1d --- /dev/null +++ b/docs/UNIFIED-BACKUP-MAPPING.md @@ -0,0 +1,83 @@ +# UNIFIED X10 BACKUP MAPPING — review draft (2026-08-20) + +Goal: one reusable backup pipeline for all three sources (Mac Studio, MacBook Air, +Crucial 2TB) that writes INTO the canonical X10 tree. X10 canonical structure is +authoritative. Provenance in _ledger/, not in paths. True global dedup: one physical +copy per unique sha256 across all sources. + +Status: REVIEW DRAFT — no scripts changed yet. Awaiting operator approval. + +--- + +## 1. Canonical X10 target tree (authoritative, just built) + Media/{Movies,TV,Twitch,Music-Videos,Clips,Rips,Other-Video}/ + Music/ Pictures/ Documents/ Desktop/ Downloads/ QBT/{torrents,incomplete}/ + Archive/ z-New folder/ photos-restore.photoslibrary/ _ledger/ _CONFLICTS/ + +## 2. Semantics +- Machines (Studio + Air): MOVE (frees local disk), dedup-aware. + For each source file: sha256. + - if sha already in _ledger/provenance.csv (content canonical on X10): + record this machine as an additional provenance source, then remove the + local file (content is already safely on X10). Frees disk, NO duplicate. + - else: rsync verify-then-delete (--checksum --remove-source-files) into the + canonical target, add provenance row. +- Crucial 2TB: COPY (never repoint Jellyfin/qBittorrent; source stays live). + Dedup-aware too: skip files whose sha is already canonical; copy only unique ones. + +## 3. Machine (Studio + Air) source -> canonical target + ~/Music -> X10/Music/ + ~/Pictures -> X10/Pictures/ + ~/Documents -> X10/Documents/ + ~/Desktop -> X10/Desktop/ + ~/Movies -> X10/Media/Movies/ (iMovie Library -> Media/Movies/ or Other-Video/) + ~/Downloads: + stacher/ -> X10/Media/Music-Videos/ (music-video downloads; adult content policy: + if flagged adult, X10/z-New folder/ instead — operator confirm) + Media/ -> X10/Media/Other-Video/ (generic download-media) + Music/ -> X10/Music/ + *.dmg/pkg/exe/msi/iso/zip/tar/gz/tgz/bz2/xz/7z -> X10/Archive/installers/ + *.pdf -> X10/Documents/ + other -> X10/Downloads/ + No per-machine subdirs: both machines write into the SAME canonical dirs; dedup + + provenance in _ledger separates them. + +## 4. Crucial 2TB source -> canonical target + Media/Movies -> X10/Media/Movies/ + Media/TV -> X10/Media/TV/ + Media/Twitch -> X10/Media/Twitch/ + Media/... -> X10/Media/ + Music/ -> X10/Music/ + Pictures/ -> X10/Pictures/ + QBT/ -> X10/QBT/torrents/ + Archive/ -> X10/Archive/ + photos-restore.photoslibrary -> X10/photos-restore.photoslibrary/ (OPAQUE, R5) + +## 5. What goes away (complexity/redundancy reduced) +- X10/archive/// per-machine buckets -> canonical top-levels (dedup) +- X10/_photos-library-mirror/ whole-volume root -> absorbed into canonical top-levels +- Per-machine subdirs in target paths -> provenance column in _ledger + +## 6. Provenance +_ledger/provenance.csv (index, comma, 4 cols: sha|canonical_path|size|topdir) = authoritative +canonical index from the reorg. The unified backup appends NEW machine/2TB provenance to a +separate file _ledger/provenance-machines.csv (sha|canonical_path|size|machine|source_relpath), +so the reorg index stays untouched. Dedup authority = union of shas in BOTH files. +A file present on Studio + 2TB = 1 physical copy, 2 machine-provenance rows. + +## 7. Pipeline (launchd on each machine when X10 attached) +- unified-backup.zsh runs under launchd (com.psingletary.backup3, 60s poll) on EACH machine. +- When X10 is attached to a machine, that machine runs its own machine-MOVE phase. +- On the Studio (which also hosts the 2TB), the 2TB COPY phase runs after the machine phase. +- Exits 0 in <1s when X10 absent (launchd-safe). Dry-run default; --apply for real. + +## 8. Resolved decisions (2026-08-20, operator) +- Machine semantics: MOVE (free local disk) + 2TB: COPY. Confirmed. +- Adult stacher content -> top-level z-New folder/. (routed in route_downloads) +- OneDrive routing DISABLED by default (2026-08-20): writing to OneDrive from launchd + context deadlocks ("Resource deadlock avoided" — File Provider). All files go to X10 + canonical. Re-enable with USE_ONEDRIVE=yes for manual runs only. +- When a machine file's sha is already canonical, delete the local copy (X10 has it). +- 2TB phase uses rsync --ignore-existing (path-based skip) — NOT per-file shasum, so a + launchd fire does not re-read the whole 2TB. Efficient, correct for 1:1 mapping. +- Provenance: machine/2TB rows go to _ledger/provenance-machines.csv (reorg index untouched). -- 2.51.2