atproto relay in zig zlay.waow.tech
relay zig atproto
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748#!/bin/sh# capture a wedged/deaf zlay's state BEFORE restarting it.# usage (in-pod): kubectl exec -n <ns> <pod> -- sh -c 'curl -s <raw-url> | sh' > forensics.txt# or copy this file in and run it. takes ~10 seconds, read-only.## every section answers a question the 2026-08-04 canary could not:# what is the scheduler doing / who is listening / who is connected /# is ingest advancing / where is every thread stuck.set -xdate -u +%FT%TZP=$(pidof zlay) || { echo "NO ZLAY PROCESS"; exit 1; }echo "pid=$P uptime=$(ps -o etime= -p "$P")"
# --- is the process serving at all (both ports, short timeouts) ---curl -sm5 -o /dev/null -w "healthz=%{http_code} t=%{time_total}\n" http://127.0.0.1:3001/_healthzcurl -sm5 -o /dev/null -w "ws_port=%{http_code} t=%{time_total}\n" http://127.0.0.1:3000/xrpc/_health
# --- ingest liveness: two samples, 5s apart. flat relay_seq = dead ingest ---curl -sm10 http://127.0.0.1:3001/metrics > /tmp/m1 2>/dev/nullsleep 5curl -sm10 http://127.0.0.1:3001/metrics > /tmp/m2 2>/dev/nullgrep -hE '^relay_seq |^relay_db_queue|^relay_broadcast_queue|^zio_' /tmp/m1 /tmp/m2
# --- sockets: established count, listener backlog (accept queue depth# discriminates "not accepting" from "accepting but not serving") ---echo "established=$(grep -c ' 01 ' /proc/$P/net/tcp)"echo "fds=$(ls /proc/$P/fd | wc -l) limit=$(grep 'Max open files' /proc/$P/limits)"grep -E ':0BB8|:0BB9' /proc/$P/net/tcp | head -5 # 3000/3001 listeners
# --- epoll registrations: how many fds the runtime is actually watching ---for f in /proc/$P/fd/*; do case "$(readlink "$f" 2>/dev/null)" in *eventpoll*) E=$(basename "$f");; esacdone[ -n "$E" ] && { echo "epfd=$E registrations=$(grep -c tfd: /proc/$P/fdinfo/$E)"; head -8 /proc/$P/fdinfo/$E; }
# --- threads: where is everyone stuck ---echo "threads=$(ls /proc/$P/task | wc -l)"for t in /proc/$P/task/*/; do printf '%s ' "$(cat "$t/wchan" 2>/dev/null)"; done; echo# the loop thread is usually the 2nd task; its kernel stack names the wedgeL=$(ls /proc/$P/task | sort -n | sed -n 2p)echo "loop_thread=$L"; head -6 /proc/$P/task/$L/stack 2>/dev/null
# --- userspace stacks (best evidence; needs gdb, skip if absent) ---command -v gdb >/dev/null && timeout 60 gdb -p "$P" -batch -ex 'set pagination off' \ -ex 'thread apply all bt 8' 2>/dev/null | head -200
echo "FORENSICS-COMPLETE"