diff --git a/deploy/ops/check6.sh b/deploy/ops/check6.sh index 30a8c34..28b2a4a 100755 --- a/deploy/ops/check6.sh +++ b/deploy/ops/check6.sh @@ -63,21 +63,43 @@ PYD fi echo "$now $comp $failed $disk" > "$STATE" -# Nested windows. A trend requires these to move together. -printf 'rate windows:' +# The backfill-era numbers below (crawl rate, per-repo cost, batch troughs) are +# meaningless once bootstrap is over. Report merge/serving state instead, or a +# reader sees "1.3 repos/s" in phase 2 and thinks the thing has died. +if [ "$phase" != "1" ]; then + echo "--- phase $phase: bootstrap is over, backfill rates below do not apply ---" + for m in merge_events_kept_total merge_events_dropped_total merge_segments_consumed_total; do + printf ' %s = %s\n' "$m" "$(Q "jetstream_orchestrator_$m" | scalar)" + done + printf ' compaction_watermark_seq = %s (0 = compaction has not started)\n' \ + "$(Q stream_compaction_watermark_seq | scalar)" + printf ' pending (repos left in the pending pass) = %s\n' \ + "$(Q 'stream_backfill_repos_durable%7Bstatus%3D%22pending%22%7D' | scalar)" + code=$(S 'cd /opt/stream-experiment; docker compose exec -T prometheus wget -S -qO- http://172.18.0.2:8080/xrpc/jetstream.listSegments 2>&1 | grep -m1 -oE "HTTP/1.1 [0-9]+"') + printf ' listSegments -> %s (503 until steady_state; 200 is the win)\n' "${code:-unreadable}" + printf ' backfill tree = %s (GONE means merge cleanup finished)\n' \ + "$(S 'du -sh /data/stream/backfill 2>/dev/null | cut -f1 || echo GONE')" + echo " replay window: run 'ssh root@37.27.43.39 /usr/local/bin/wsprobe 32408688748'" + echo " 101 + a binary frame = still replayable" +fi + +# Nested windows. A trend requires these to move together. BOOTSTRAP ONLY. +[ "$phase" = "1" ] && printf 'rate windows:' +if [ "$phase" = "1" ]; then for w in 1h 6h 12h; do r=$(Q "rate(stream_backfill_repos_durable%7Bstatus%3D%22complete%22%7D%5B$w%5D)" | scalar) printf ' %s=%s' "$w" "$(python3 -c "print(round(float('${r:-0}'),1))")" done; echo +fi # Queue state qualifies the 1h number: near-zero not_started == batch trough. -if [ -n "$notstarted" ] && [ "${notstarted%.*}" -lt 1000 ]; then +if [ "$phase" = "1" ] && [ -n "$notstarted" ] && [ "${notstarted%.*}" -lt 1000 ]; then echo " ^ NOTE: not_started=$notstarted (batch trough) — a low 1h rate here is EXPECTED, not a fault" fi # Per-repo cost: rising means big repos, falling means the tail is thinning. pr=$(Q 'rate(jetstream_backfill_handle_repo_duration_seconds_sum%5B2h%5D)/rate(jetstream_backfill_handle_repo_duration_seconds_count%5B2h%5D)' | scalar) -[ -n "$pr" ] && printf 'per-repo: %.1f ms (falling => tail thinning, NOT degradation; bytes/s falls with it)\n' \ +[ "$phase" = "1" ] && [ -n "$pr" ] && printf 'per-repo: %.1f ms (falling => tail thinning, NOT degradation; bytes/s falls with it)\n' \ "$(python3 -c "print(float('$pr')*1000)")" # Per-process counter: it RESETS on every restart, so only increase() is