Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111#!/usr/bin/env zshemulate -L zshsetopt PIPE_FAILexport LC_ALL=Ctypeset -F SECONDSSRC="/Volumes/MicroSD-512"DST="/Volumes/photos library iCloud"AUDIT="$HOME/msd512-audit"LEDGER="$AUDIT/ledger.tsv"HN=$(cat "$AUDIT/hash.algo")case "$HN" in blake3) HASH=(b3sum) ;; xxh128) HASH=(xxh128sum) ;; *) HASH=(shasum -a 256) ;;esachash_of() { "${HASH[@]}" < "$1" | awk '{print $1}'; }
[[ -d "$SRC" ]] || { print "FATAL: SRC not mounted"; exit 1; }[[ -d "$DST" ]] || { print "FATAL: DST not mounted"; exit 1; }[[ -s "$AUDIT/worklist.tsv" ]] || { print "FATAL: run phase1 first"; exit 1; }mkdir -p "$AUDIT/tmp"touch "$LEDGER"
# already-done set (resume) and dedup oracle in memorytypeset -A DONE HASHIDX INBUNDLEwhile IFS=$'\t' read -r st h sz sp dp nt; do [[ "$st" == "ERROR" ]] && continue [[ -n "$sp" ]] && DONE[$sp]=1 [[ -n "$h" && -n "$dp" ]] && HASHIDX[$h]="$dp"done < "$LEDGER"while IFS=$'\t' read -r h ib p; do [[ -n "$h" ]] && : ${HASHIDX[$h]::="$p"} [[ "$ib" == "1" ]] && INBUNDLE[$h]=1done < "$AUDIT/dst.hashes.tsv"
t0=$SECONDS; nmig=0; ndup=0; nerr=0; ncon=0; bytes=0total=$(wc -l < "$AUDIT/worklist.tsv" | tr -d ' ')idx=0
while IFS=$'\t' read -r s t sz; do idx=$(( idx + 1 )) [[ -n "${DONE[$s]}" ]] && continue [[ -d "$SRC" ]] || { print "ABORT: SRC vanished at item $idx. Re-run to resume."; break; } [[ -r "$s" ]] || { printf 'ERROR\t\t%d\t%s\t\tunreadable\n' "$sz" "$s" >> "$LEDGER"; nerr=$(( nerr + 1 )); continue; }
mkdir -p "${t:h}" 2>/dev/null part="$t.part" # SINGLE READ of SRC: tee writes DST, hash consumes the same stream h_src=$( cat -- "$s" | tee -- "$part" | "${HASH[@]}" | awk '{print $1}' ) rc=$? if [[ $rc -ne 0 || -z "$h_src" ]]; then rm -f -- "$part" printf 'ERROR\t\t%d\t%s\t\tstream rc=%d\n' "$sz" "$s" "$rc" >> "$LEDGER" nerr=$(( nerr + 1 )); continue fi
wsz=$(stat -f %z -- "$part" 2>/dev/null) h_dst=$(hash_of "$part") if [[ "$h_src" != "$h_dst" || "$wsz" != "$sz" ]]; then mkdir -p "$DST/_QUARANTINE/${${t#$DST/}:h}" 2>/dev/null mv -f -- "$part" "$DST/_QUARANTINE/${t#$DST/}" 2>/dev/null || rm -f -- "$part" printf 'ERROR\t%s\t%d\t%s\t\tverify-mismatch\n' "$h_src" "$sz" "$s" >> "$LEDGER" nerr=$(( nerr + 1 )); continue fi
# zero-byte files never dedup by hash alone if [[ -n "${HASHIDX[$h_src]}" && "$sz" != "0" ]]; then ex="${HASHIDX[$h_src]}" rm -f -- "$part" if [[ -n "${INBUNDLE[$h_src]}" ]]; then printf 'DEDUP_BUNDLE\t%s\t%d\t%s\t%s\tbundle-only\n' "$h_src" "$sz" "$s" "$ex" >> "$LEDGER" printf '%s\t%s\n' "$s" "$ex" >> "$AUDIT/in-bundle-only.tsv" else printf 'DEDUP\t%s\t%d\t%s\t%s\talready-present\n' "$h_src" "$sz" "$s" "$ex" >> "$LEDGER" fi ndup=$(( ndup + 1 )); continue fi
if [[ -e "$t" ]]; then if [[ "$(hash_of "$t")" == "$h_src" ]]; then rm -f -- "$part" printf 'DEDUP\t%s\t%d\t%s\t%s\tidentical-at-target\n' "$h_src" "$sz" "$s" "$t" >> "$LEDGER" ndup=$(( ndup + 1 )); continue fi c="$DST/_CONFLICTS/${t#$DST/}" mkdir -p "${c:h}" 2>/dev/null mv -f -- "$part" "$c" touch -r "$s" -- "$c" 2>/dev/null printf 'CONFLICT\t%s\t%d\t%s\t%s\tdifferent-content-same-path\n' "$h_src" "$sz" "$s" "$c" >> "$LEDGER" ncon=$(( ncon + 1 )); continue fi
mv -f -- "$part" "$t" touch -r "$s" -- "$t" 2>/dev/null HASHIDX[$h_src]="$t" printf 'MIGRATED\t%s\t%d\t%s\t%s\t\n' "$h_src" "$sz" "$s" "$t" >> "$LEDGER" nmig=$(( nmig + 1 )); bytes=$(( bytes + sz ))
if (( idx % 25 == 0 )); then el=$(( SECONDS - t0 + 0.001 )) printf '[%d/%d] mig=%d dup=%d err=%d %.1f GiB @ %.2f MiB/s\n' $idx $total $nmig $ndup $nerr $(( bytes/1073741824.0 )) $(( bytes/1048576.0/el )) >> "$AUDIT/phase3.progress" fidone < "$AUDIT/worklist.tsv"
el=$(( SECONDS - t0 + 0.001 ))print "===== PHASE3 ====="printf 'processed=%d/%d migrated=%d dedup=%d conflict=%d error=%d\n' $idx $total $nmig $ndup $ncon $nerrprintf 'copied %.1f GiB in %.2f h => %.2f MiB/s\n' $(( bytes/1073741824.0 )) $(( el/3600 )) $(( bytes/1048576.0/el ))awk -F'\t' '$1=="ERROR"{c[$6]++} END{for(k in c) printf "ERR %d\t%s\n", c[k], k}' "$LEDGER"tail -3 "$AUDIT/phase3.progress" 2>/dev/null