diff --git a/jobs/scripts/poll-and-land.sh b/jobs/scripts/poll-and-land.sh index 44eee0d3..058aa1af 100755 --- a/jobs/scripts/poll-and-land.sh +++ b/jobs/scripts/poll-and-land.sh @@ -32,7 +32,8 @@ # 1 a stage failed: `failed`, with the failing stage read from the forge's result when # it can be; not resent. A result fetched here (exit 1, 11, and 9's fallback) is # also pushed to $JOBS_TAG_REMOTE as the tag result//, as a landing's -# is; one that cannot be adds "result not kept on " to the message. +# is; one that cannot be adds "result not kept on yet; each pass retries +# it" to the message, and is queued in $STATE_DIR/results-pending for later passes. # 11 the job's failed rework rounds are used up and it waits for a person # (specs/job-review-rework.md): `needs-person`, with the failing stage read from the # forge's result as for exit 1, and the line `resend with jobs/scripts/submit.sh @@ -408,11 +409,54 @@ save_result() { # result//, so the run's result.json outlives the forge's result/, # which a landing deletes. Called only right after a fetch that succeeded, since # result.json names no run and an older ref would be kept under the wrong run. Never under -# --check, and never moves a tag: a push that would is a failure. +# --check, and never moves a tag: a push that would is a failure. A push that still fails +# after its tries is queued in $STATE_DIR/results-pending, which every later pass retries +# (keep_pending), and the call fails so the message says the result is not kept yet. keep_result() { [ "${#CHECK[@]}" -eq 0 ] || return 0 - git -C "$REPO" push --quiet "$CURRENT_TAG_REMOTE" "refs/jobs/$1/result:refs/tags/result/$1/$2" >/dev/null 2>&1 && + local commit + commit="$(git -C "$REPO" rev-parse -q --verify "refs/jobs/$1/result^{commit}")" || return 1 + if push_tag "$CURRENT_TAG_REMOTE" "$commit" "result/$1/$2" "${JOBS_KEEP_TRIES:-3}"; then : > "$TMP/kept-$2" + return 0 + fi + printf '%s\t%s\t%s\n' "$CURRENT_TAG_REMOTE" "$commit" "result/$1/$2" >> "$STATE_DIR/results-pending" + return 1 +} + +# : pushes to as refs/tags/, trying +# up to times JOBS_KEEP_PAUSE seconds apart (default 10), since a push over the +# network can fail for a moment. git's output for each failed try goes to stderr, which the +# schedule's log keeps; it names hosts and paths, so no record message carries it. +push_tag() { + local try=1 err + while :; do + err="$(git -C "$REPO" push --quiet "$1" "$2:refs/tags/$3" 2>&1 >/dev/null)" && return 0 + echo "poll-and-land: push of tag $3 to $1 failed (try $try of $4): ${err//$'\n'/ | }" >&2 + [ "$try" -lt "$4" ] || return 1 + try=$((try + 1)) + sleep "${JOBS_KEEP_PAUSE:-10}" + done +} + +# Retries each result queued in $STATE_DIR/results-pending by an earlier pass, one try each, +# and keeps only the lines whose push still fails. Never under --check. +keep_pending() { + local file="$STATE_DIR/results-pending" remote commit tag left=() + [ "${#CHECK[@]}" -eq 0 ] && [ -s "$file" ] || return 0 + while IFS=$'\t' read -r remote commit tag; do + [ -n "$tag" ] || continue + if push_tag "$remote" "$commit" "$tag" 1; then + echo "poll-and-land: kept $tag on $remote, queued by an earlier pass" + else + left+=("$(printf '%s\t%s\t%s' "$remote" "$commit" "$tag")") + fi + done < "$file" + if [ "${#left[@]}" -eq 0 ]; then + rm -f "$file" + else + printf '%s\n' "${left[@]}" > "$file.tmp" && mv "$file.tmp" "$file" + fi } # : fetches and keeps the run's result, then prints "stage : " @@ -420,7 +464,7 @@ keep_result() { failed_stage() { local name="$1" id="$2" stage notes="" fetch_result "$name" || return 0 - keep_result "$name" "$id" || notes="result not kept on $CURRENT_TAG_REMOTE" + keep_result "$name" "$id" || notes="result not kept on $CURRENT_TAG_REMOTE yet; each pass retries it" stage="$(git -C "$REPO" show "refs/jobs/$name/result:result.json" 2>/dev/null | python3 -c ' import json, sys try: @@ -679,7 +723,7 @@ sys.exit(0 if any(s.get("name") == "store" and s.get("status") == "passed" for s # The result is kept on the tag remote before the forge's copy is deleted; when it # cannot be, job/ and result/ stay on the forge, so nothing is lost. if ! keep_result "$name" "$CURRENT_RUN"; then - notes="$notes; result not kept on $CURRENT_TAG_REMOTE as result/$name/$CURRENT_RUN, so job/$name and result/$name stay on the forge" + notes="$notes; result not kept on $CURRENT_TAG_REMOTE as result/$name/$CURRENT_RUN yet, so job/$name and result/$name stay on the forge; each pass retries it" elif git -C "$REPO" push --quiet "$CURRENT_FORGE" --delete "refs/heads/job/$name" "refs/heads/result/$name" \ >/dev/null 2>&1; then git -C "$REPO" update-ref -d "refs/jobs/$name/branch" 2>/dev/null || true @@ -1052,6 +1096,7 @@ poll() { return 0 } +keep_pending STOP=0 mapfile -t OPEN < <("${RECORDS[@]}" "$STATE_DIR" open) for entry in "${OPEN[@]}"; do diff --git a/jobs/tests/poll_and_land.rs b/jobs/tests/poll_and_land.rs index 4da92344..63d08570 100644 --- a/jobs/tests/poll_and_land.rs +++ b/jobs/tests/poll_and_land.rs @@ -270,6 +270,7 @@ impl Setup { .env("JOBS_TAG_REMOTE", "tags") .env("JOBS_CHECK", env!("CARGO_BIN_EXE_job-check")) .env("JOBS_WATCH_RETRY_SECONDS", "0") + .env("JOBS_KEEP_PAUSE", "0") .env("STUB_DIR", self.stub_dir()); cmd } @@ -1476,7 +1477,7 @@ fn a_failed_tag_push_still_lands_and_keeps_the_forge_refs() { rec["message"], format!( "stub land line for {NAME}; tag spec/{NAME}/{} not pushed to tags; result not kept on tags as \ - result/{NAME}/run-1, so job/{NAME} and result/{NAME} stay on the forge", + result/{NAME}/run-1 yet, so job/{NAME} and result/{NAME} stay on the forge; each pass retries it", s.spec_commit() ) ); @@ -1535,10 +1536,35 @@ fn a_failed_stage_whose_result_cannot_be_kept_says_so() { s.poll(); assert_eq!( s.record()["message"], - "run run-1: a stage failed: stage review: defect; result not kept on tags" + "run run-1: a stage failed: stage review: defect; result not kept on tags yet; each pass retries it" ); } +#[test] +fn a_result_that_could_not_be_kept_is_kept_by_a_later_pass() { + let s = Setup::new("keep-later"); + s.tag_remote(&s.tmp.path().join("tags.git")); + s.publish(|r| { + r["status"] = json!("failed"); + r["failed_stage"] = json!("review"); + r["reason"] = json!("defect"); + }); + let result = s.forge_result(); + s.open_job(0, "failed 1 run:failed"); + let out = s.poll(); + assert!(text(&out.stderr).contains(&format!("push of tag result/{NAME}/run-1 to tags failed (try 3 of 3)"))); + let pending = std::fs::read_to_string(s.state().join("results-pending")).unwrap(); + assert_eq!(pending, format!("tags\t{result}\tresult/{NAME}/run-1\n")); + + // The tag remote comes back; the next pass keeps the queued result and empties the queue. + sh(s.tmp.path(), "git init -q --bare -b main tags.git"); + let out = s.poll(); + assert!(text(&out.stdout).contains(&format!("kept result/{NAME}/run-1 on tags")), "{}", text(&out.stdout)); + let tags = sh(&s.tmp.path().join("tags.git"), "git for-each-ref --format='%(refname) %(objectname)' refs/tags"); + assert_eq!(tags, format!("refs/tags/result/{NAME}/run-1 {result}")); + assert!(!s.state().join("results-pending").exists()); +} + #[test] fn a_failed_stage_with_no_result_on_the_forge_keeps_nothing() { let s = Setup::new("keep-none");