#!/bin/sh
# capture a wedged/deaf zlay's state BEFORE restarting it.
# usage (in-pod):  kubectl exec -n <ns> <pod> -- sh -c 'curl -s <raw-url> | sh' > forensics.txt
#        or copy this file in and run it.  takes ~10 seconds, read-only.
#
# every section answers a question the 2026-08-04 canary could not:
#   what is the scheduler doing / who is listening / who is connected /
#   is ingest advancing / where is every thread stuck.
set -x
date -u +%FT%TZ
P=$(pidof zlay) || { echo "NO ZLAY PROCESS"; exit 1; }
echo "pid=$P uptime=$(ps -o etime= -p "$P")"

# --- is the process serving at all (both ports, short timeouts) ---
curl -sm5 -o /dev/null -w "healthz=%{http_code} t=%{time_total}\n" http://127.0.0.1:3001/_healthz
curl -sm5 -o /dev/null -w "ws_port=%{http_code} t=%{time_total}\n" http://127.0.0.1:3000/xrpc/_health

# --- ingest liveness: two samples, 5s apart. flat relay_seq = dead ingest ---
curl -sm10 http://127.0.0.1:3001/metrics > /tmp/m1 2>/dev/null
sleep 5
curl -sm10 http://127.0.0.1:3001/metrics > /tmp/m2 2>/dev/null
grep -hE '^relay_seq |^relay_db_queue|^relay_broadcast_queue|^zio_' /tmp/m1 /tmp/m2

# --- sockets: established count, listener backlog (accept queue depth
#     discriminates "not accepting" from "accepting but not serving") ---
echo "established=$(grep -c ' 01 ' /proc/$P/net/tcp)"
echo "fds=$(ls /proc/$P/fd | wc -l) limit=$(grep 'Max open files' /proc/$P/limits)"
grep -E ':0BB8|:0BB9' /proc/$P/net/tcp | head -5   # 3000/3001 listeners

# --- epoll registrations: how many fds the runtime is actually watching ---
for f in /proc/$P/fd/*; do
  case "$(readlink "$f" 2>/dev/null)" in *eventpoll*) E=$(basename "$f");; esac
done
[ -n "$E" ] && { echo "epfd=$E registrations=$(grep -c tfd: /proc/$P/fdinfo/$E)"; head -8 /proc/$P/fdinfo/$E; }

# --- threads: where is everyone stuck ---
echo "threads=$(ls /proc/$P/task | wc -l)"
for t in /proc/$P/task/*/; do printf '%s ' "$(cat "$t/wchan" 2>/dev/null)"; done; echo
# the loop thread is usually the 2nd task; its kernel stack names the wedge
L=$(ls /proc/$P/task | sort -n | sed -n 2p)
echo "loop_thread=$L"; head -6 /proc/$P/task/$L/stack 2>/dev/null

# --- userspace stacks (best evidence; needs gdb, skip if absent) ---
command -v gdb >/dev/null && timeout 60 gdb -p "$P" -batch -ex 'set pagination off' \
  -ex 'thread apply all bt 8' 2>/dev/null | head -200

echo "FORENSICS-COMPLETE"