diff --git a/tests/bench/collection_interning_ab.sh b/tests/bench/collection_interning_ab.sh --- a/tests/bench/collection_interning_ab.sh +++ b/tests/bench/collection_interning_ab.sh @@ -37,6 +37,10 @@ # identical in both arms, so the ratio stays valid even though the absolute # numbers are not prod-tuned. MEMTABLE_MB=${MEMTABLE_MB:-8} +# consecutive samples with an empty queue AND a static record count before an +# arm counts as drained +SETTLE_SAMPLES=${SETTLE_SAMPLES:-12} +DRAIN_TIMEOUT=${DRAIN_TIMEOUT:-10800} # plc.klbr.net is dawn's unthrottled mirror; falls back if unreachable PLC_URL=${PLC_URL:-https://plc.klbr.net} @@ -140,19 +144,35 @@ --data-binary "@$SAMPLE" -o "$dir/put-repos.json" \ -w 'put /repos -> %{http_code}\n' - # drain: pending must hit zero and stay there, since a repo can be requeued + # drain. pending is only the QUEUE depth: it reads zero while in-flight + # backfills are still writing records, so waiting on it alone stops the arm + # mid-flight and the arms end up having indexed different repo sets. require + # the record count to have stopped growing as well. echo "draining..." - local zero=0 elapsed=0 - while [ "$zero" -lt 4 ]; do + local settled=0 elapsed=0 last_records=-1 + while [ "$settled" -lt "$SETTLE_SAMPLES" ]; do sleep "$INTERVAL" elapsed=$((elapsed + INTERVAL)) - local pend - pend=$(curl -s --max-time 5 "http://127.0.0.1:$API_PORT/stats" \ - | jq -r '.counts.pending // 0' 2>/dev/null || echo 1) - if [ "$pend" = "0" ]; then zero=$((zero + 1)); else zero=0; fi - if [ $((elapsed % 60)) -lt "$INTERVAL" ]; then echo " ${elapsed}s pending=$pend"; fi - if [ "$elapsed" -gt 7200 ]; then echo " giving up after 2h with pending=$pend" >&2; break; fi + local st pend recs + st=$(curl -s --max-time 10 "http://127.0.0.1:$API_PORT/stats" 2>/dev/null || echo '{}') + pend=$(echo "$st" | jq -r '.counts.pending // 1' 2>/dev/null || echo 1) + recs=$(echo "$st" | jq -r '.counts.records // -1' 2>/dev/null || echo -1) + if [ "$pend" = "0" ] && [ "$recs" = "$last_records" ]; then + settled=$((settled + 1)) + else + settled=0 + fi + last_records=$recs + if [ $((elapsed % 60)) -lt "$INTERVAL" ]; then + echo " ${elapsed}s pending=$pend records=$recs settled=$settled" + fi + if [ "$elapsed" -gt "$DRAIN_TIMEOUT" ]; then + echo " giving up after ${DRAIN_TIMEOUT}s with pending=$pend records=$recs" >&2 + echo "INCOMPLETE" > "$dir/INCOMPLETE" + break + fi done + echo " drained at records=$last_records after ${elapsed}s" date +%s > "$dir/end_epoch" curl -s --max-time 60 "http://127.0.0.1:$API_PORT/stats" > "$dir/stats-predrain.json" @@ -261,7 +281,12 @@ printf " counts_b_per_rec %.2f -> %.2f (%+.2f%%)\n", acb, bcb, (acb>0?(bcb-acb)/acb*100:0) printf " total_b_per_rec %.2f -> %.2f (%+.2f%%)\n", atb, btb, (btb-atb)/atb*100 if (arss>0) printf " peak_rss_mb %.1f -> %.1f (%+.2f%%)\n", arss, brss, (brss-arss)/arss*100 - if (ar != br) printf " !! record counts differ (%d vs %d): bytes/record still\n comparable but the arms did not index the same set\n", ar, br + if (ar != br) { + d = (br-ar)/ar*100; if (d<0) d=-d + printf " !! RECORD COUNTS DIFFER: %d vs %d (%.1f%%)\n", ar, br, d + printf " !! the arms did not index the same set, so this is NOT a\n" + printf " !! valid paired comparison -- rerun with a longer drain\n" + } } } ' "$RUN_DIR/summary.tsv"