From b61bf522c4d4f7acd69625407a6b3d69b7c3ba2e Mon Sep 17 00:00:00 2001
From: dawn <90008@klbr.net>
Date: Sat, 1 Aug 2026 01:45:37 +0300
Subject: [PATCH] [test] fix A/B drain to wait for in-flight backfills

pending is only the backfill QUEUE depth, so it reads zero while in-flight
backfills are still writing records. waiting on it alone stopped each arm
mid-flight: the first 10k-repo run had baseline at 7,456,623 records and
interned at 4,511,412, a 40% gap, which makes the comparison meaningless
because the arms indexed different repo sets.

the drain now also requires the record count to have stopped growing, for 12
consecutive samples, and marks an arm INCOMPLETE if it hits the timeout rather
than reporting a number that looks finished. the summary shouts when record
counts differ instead of noting it in passing.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 tests/bench/collection_interning_ab.sh | 45 ++++++++++++++++++++------
 1 file changed, 35 insertions(+), 10 deletions(-)

diff --git a/tests/bench/collection_interning_ab.sh b/tests/bench/collection_interning_ab.sh
index 5f7e9b8..ab3bdce 100755
--- a/tests/bench/collection_interning_ab.sh
+++ b/tests/bench/collection_interning_ab.sh
@@ -37,6 +37,10 @@ INTERVAL=${INTERVAL:-5}
 # identical in both arms, so the ratio stays valid even though the absolute
 # numbers are not prod-tuned.
 MEMTABLE_MB=${MEMTABLE_MB:-8}
+# consecutive samples with an empty queue AND a static record count before an
+# arm counts as drained
+SETTLE_SAMPLES=${SETTLE_SAMPLES:-12}
+DRAIN_TIMEOUT=${DRAIN_TIMEOUT:-10800}
 # plc.klbr.net is dawn's unthrottled mirror; falls back if unreachable
 PLC_URL=${PLC_URL:-https://plc.klbr.net}
 
@@ -140,19 +144,35 @@ run_arm() {
         --data-binary "@$SAMPLE" -o "$dir/put-repos.json" \
         -w 'put /repos -> %{http_code}\n'
 
-    # drain: pending must hit zero and stay there, since a repo can be requeued
+    # drain. pending is only the QUEUE depth: it reads zero while in-flight
+    # backfills are still writing records, so waiting on it alone stops the arm
+    # mid-flight and the arms end up having indexed different repo sets. require
+    # the record count to have stopped growing as well.
     echo "draining..."
-    local zero=0 elapsed=0
-    while [ "$zero" -lt 4 ]; do
+    local settled=0 elapsed=0 last_records=-1
+    while [ "$settled" -lt "$SETTLE_SAMPLES" ]; do
         sleep "$INTERVAL"
         elapsed=$((elapsed + INTERVAL))
-        local pend
-        pend=$(curl -s --max-time 5 "http://127.0.0.1:$API_PORT/stats" \
-            | jq -r '.counts.pending // 0' 2>/dev/null || echo 1)
-        if [ "$pend" = "0" ]; then zero=$((zero + 1)); else zero=0; fi
-        if [ $((elapsed % 60)) -lt "$INTERVAL" ]; then echo "  ${elapsed}s pending=$pend"; fi
-        if [ "$elapsed" -gt 7200 ]; then echo "  giving up after 2h with pending=$pend" >&2; break; fi
+        local st pend recs
+        st=$(curl -s --max-time 10 "http://127.0.0.1:$API_PORT/stats" 2>/dev/null || echo '{}')
+        pend=$(echo "$st" | jq -r '.counts.pending // 1' 2>/dev/null || echo 1)
+        recs=$(echo "$st" | jq -r '.counts.records // -1' 2>/dev/null || echo -1)
+        if [ "$pend" = "0" ] && [ "$recs" = "$last_records" ]; then
+            settled=$((settled + 1))
+        else
+            settled=0
+        fi
+        last_records=$recs
+        if [ $((elapsed % 60)) -lt "$INTERVAL" ]; then
+            echo "  ${elapsed}s pending=$pend records=$recs settled=$settled"
+        fi
+        if [ "$elapsed" -gt "$DRAIN_TIMEOUT" ]; then
+            echo "  giving up after ${DRAIN_TIMEOUT}s with pending=$pend records=$recs" >&2
+            echo "INCOMPLETE" > "$dir/INCOMPLETE"
+            break
+        fi
     done
+    echo "  drained at records=$last_records after ${elapsed}s"
     date +%s > "$dir/end_epoch"
 
     curl -s --max-time 60 "http://127.0.0.1:$API_PORT/stats" > "$dir/stats-predrain.json"
@@ -261,7 +281,12 @@ if [ "$(wc -l < "$RUN_DIR/summary.tsv" 2>/dev/null || echo 0)" -eq 2 ]; then
                 printf "  counts_b_per_rec     %.2f -> %.2f  (%+.2f%%)\n", acb, bcb, (acb>0?(bcb-acb)/acb*100:0)
                 printf "  total_b_per_rec      %.2f -> %.2f  (%+.2f%%)\n", atb, btb, (btb-atb)/atb*100
                 if (arss>0) printf "  peak_rss_mb          %.1f -> %.1f  (%+.2f%%)\n", arss, brss, (brss-arss)/arss*100
-                if (ar != br) printf "  !! record counts differ (%d vs %d): bytes/record still\n     comparable but the arms did not index the same set\n", ar, br
+                if (ar != br) {
+                    d = (br-ar)/ar*100; if (d<0) d=-d
+                    printf "  !! RECORD COUNTS DIFFER: %d vs %d (%.1f%%)\n", ar, br, d
+                    printf "  !! the arms did not index the same set, so this is NOT a\n"
+                    printf "  !! valid paired comparison -- rerun with a longer drain\n"
+                }
             }
         }
     ' "$RUN_DIR/summary.tsv"
-- 
2.51.2