diff --git a/drain/drain.sh b/drain/drain.sh index 23ef9d6..71f53bf 100755 --- a/drain/drain.sh +++ b/drain/drain.sh @@ -16,12 +16,17 @@ touch done.txt idx_of() { python3 -c 'import sys;print(int(sys.argv[1][4:14],36))' "$(basename "$1" .jss)"; } -put() { # key file - local code - code=$(curl -sS -m 600 -o /tmp/drain.put.json -w "%{http_code}" -X PUT "$STRATA_URL/archive/$1" \ - -H "Authorization: Bearer $STRATA_INGEST_TOKEN" -H "Content-Type: application/octet-stream" \ - -A "strata-drain/1" --data-binary "@$2") - [ "$code" = "200" ] || { echo "PUT $1 -> $code $(cat /tmp/drain.put.json)" >&2; return 1; } +put() { # key file — retries a 5xx (R2 hiccup, worker exception) three times with backoff + local code attempt + for attempt in 1 2 3 4; do + code=$(curl -sS -m 600 -o /tmp/drain.put.json -w "%{http_code}" -X PUT "$STRATA_URL/archive/$1" \ + -H "Authorization: Bearer $STRATA_INGEST_TOKEN" -H "Content-Type: application/octet-stream" \ + -A "strata-drain/1" --data-binary "@$2" || echo 000) + [ "$code" = "200" ] && return 0 + echo "PUT $1 -> $code $(head -c 200 /tmp/drain.put.json) (attempt $attempt)" >&2 + case "$code" in 5*|000) sleep $((attempt * attempt * 5));; *) return 1;; esac + done + return 1 } mapfile -t all < <(ls "$SEGMENTS"/seg_*.jss | sort)