Something went wrong. Try again.
Monorepo for Aesthetic.Computer aesthetic.computer
Something went wrong. Try again.
core monitor-emacs.sh
8.2 kB · 264 lines
Shell
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264#!/usr/bin/env bash# DEPRECATED: Use ac-emacs-crash-monitor (fish) instead.# The fish monitor has startup-lock awareness, correct config on restart,# load-aware timeouts, and CPU monitoring. Run: ac-emacs-start-monitor## Emacs Watchdog - monitors emacs daemon health and auto-recovers from hangs# Run: ./monitor-emacs.sh (foreground) or ./monitor-emacs.sh & (background)# Stop: kill $(cat /tmp/emacs-watchdog.pid) or ac-watchdog-stop
CHECK_INTERVAL="${WATCHDOG_INTERVAL:-10}" # Check every N secondsCPU_THRESHOLD="${WATCHDOG_CPU:-85}" # CPU % threshold to consider "stuck"CPU_SAMPLES="${WATCHDOG_SAMPLES:-3}" # Consecutive high-CPU samples before actionTIMEOUT_THRESHOLD=5 # Seconds to wait for emacsclient response
LOG_DIR="/workspaces/aesthetic-computer/.emacs-logs"LOG_FILE="$LOG_DIR/watchdog.log"PID_FILE="/tmp/emacs-watchdog.pid"WARNING_FILE="/tmp/emacs-watchdog-warning"
mkdir -p "$LOG_DIR"
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"}
# Track high CPU sampleshigh_cpu_count=0
check_emacs_health() { local daemon_pid daemon_pid=$(pgrep -f "emacs.*daemon" 2>/dev/null | head -1) if [ -z "$daemon_pid" ]; then echo "not_running" return fi # Check if responsive (quick eval) if ! timeout "$TIMEOUT_THRESHOLD" emacsclient -e "t" >/dev/null 2>&1; then echo "unresponsive:$daemon_pid" return fi # Check CPU usage local cpu cpu=$(ps -p "$daemon_pid" -o %cpu= 2>/dev/null | tr -d ' ' | cut -d. -f1) if [ -n "$cpu" ] && [ "$cpu" -gt "$CPU_THRESHOLD" ]; then echo "high_cpu:$cpu:$daemon_pid" return fi echo "healthy:$daemon_pid"}
restart_emacs() { local reason="$1" log "🔄 WATCHDOG: Restarting emacs daemon (reason: $reason)" # Kill all emacs processes pkill -9 -f "emacs.*daemon" 2>/dev/null pkill -9 emacs 2>/dev/null pkill -9 emacsclient 2>/dev/null sleep 2 # Start fresh daemon log "🚀 WATCHDOG: Starting fresh emacs daemon..." emacs --daemon 2>&1 | head -10 >> "$LOG_FILE" sleep 2 # Verify if timeout 5 emacsclient -e "t" >/dev/null 2>&1; then log "✅ WATCHDOG: Emacs daemon restarted successfully" # Trigger aesthetic-backend so tabs/terminals come back after crash if timeout 15 emacsclient -e "(aesthetic-backend \"artery\")" >/dev/null 2>&1; then log "🧭 WATCHDOG: aesthetic-backend triggered after restart" else log "⚠️ WATCHDOG: Failed to trigger aesthetic-backend after restart" fi show_warning "$reason" return 0 else log "❌ WATCHDOG: Emacs daemon failed to restart" return 1 fi}
show_warning() { local reason="$1" local timestamp timestamp=$(date '+%Y-%m-%d %H:%M:%S') # Write warning file for artery TUI to detect cat > "$WARNING_FILE" << EOF{ "timestamp": "$timestamp", "reason": "$reason", "message": "Emacs daemon was auto-restarted. Restart the '💻 Aesthetic' task to reconnect.", "acknowledged": false}EOF # Try ac-notify if available if [ -x "/workspaces/aesthetic-computer/ac-notify" ]; then /workspaces/aesthetic-computer/ac-notify "⚠️ Emacs Recovered" "Watchdog: $reason. Restart the Aesthetic task." 2>/dev/null & fi # Ring terminal bell printf '\a' 2>/dev/null log "⚠️ WARNING: $reason - user notification sent"}
show_status() { local status status=$(check_emacs_health) echo "=== Emacs Watchdog Status ===" echo "PID File: $PID_FILE" echo "Log File: $LOG_FILE" echo "Check Interval: ${CHECK_INTERVAL}s" echo "CPU Threshold: ${CPU_THRESHOLD}%" echo "CPU Samples: ${CPU_SAMPLES}" echo "" case "$status" in not_running) echo "Emacs: 🔴 Not running" ;; unresponsive:*) echo "Emacs: ⚠️ Unresponsive (PID: ${status#unresponsive:})" ;; high_cpu:*:*) local cpu="${status#high_cpu:}" cpu="${cpu%%:*}" local pid="${status##*:}" echo "Emacs: ⚠️ High CPU ${cpu}% (PID: $pid)" ;; healthy:*) echo "Emacs: ✅ Healthy (PID: ${status#healthy:})" ;; esac if [ -f "$PID_FILE" ]; then local wpid wpid=$(cat "$PID_FILE") if ps -p "$wpid" >/dev/null 2>&1; then echo "Watchdog: 🟢 Running (PID: $wpid)" else echo "Watchdog: 🔴 Dead (stale PID file)" fi else echo "Watchdog: 🔴 Not running" fi}
main_loop() { # Write PID file echo $$ > "$PID_FILE" log "🐕 WATCHDOG: Starting emacs health monitor (PID: $$)" log " Check interval: ${CHECK_INTERVAL}s, CPU threshold: ${CPU_THRESHOLD}%, Samples needed: ${CPU_SAMPLES}" while true; do status=$(check_emacs_health) case "$status" in "not_running") # Daemon not running - probably intentional, don't log spam high_cpu_count=0 ;; unresponsive:*) local pid="${status#unresponsive:}" log "⚠️ WATCHDOG: Emacs daemon (PID: $pid) UNRESPONSIVE - restarting" high_cpu_count=0 restart_emacs "unresponsive (timeout after ${TIMEOUT_THRESHOLD}s)" ;; high_cpu:*:*) local cpu="${status#high_cpu:}" cpu="${cpu%%:*}" local pid="${status##*:}" high_cpu_count=$((high_cpu_count + 1)) log "⚠️ WATCHDOG: High CPU detected (${cpu}%) on PID $pid - sample $high_cpu_count/$CPU_SAMPLES" if [ "$high_cpu_count" -ge "$CPU_SAMPLES" ]; then log "🔥 WATCHDOG: Sustained high CPU for ${CPU_SAMPLES} checks - restarting" restart_emacs "sustained high CPU (${cpu}% for ${CPU_SAMPLES} samples)" high_cpu_count=0 fi ;; healthy:*) if [ "$high_cpu_count" -gt 0 ]; then log "✅ WATCHDOG: CPU normalized after $high_cpu_count samples" fi high_cpu_count=0 ;; esac sleep "$CHECK_INTERVAL" done}
cleanup() { log "🛑 WATCHDOG: Shutting down (signal received)" rm -f "$PID_FILE" exit 0}
# Handle signals gracefullytrap cleanup SIGTERM SIGINT SIGHUP
# Parse argumentscase "${1:-}" in status|--status|-s) show_status exit 0 ;; stop|--stop) if [ -f "$PID_FILE" ]; then wpid=$(cat "$PID_FILE") if kill -0 "$wpid" 2>/dev/null; then kill "$wpid" echo "Watchdog (PID: $wpid) stopped" rm -f "$PID_FILE" else echo "Watchdog not running (stale PID file)" rm -f "$PID_FILE" fi else echo "Watchdog not running (no PID file)" fi exit 0 ;; help|--help|-h) echo "Usage: $0 [command]" echo "" echo "Commands:" echo " (none) Start watchdog in foreground" echo " status Show emacs and watchdog status" echo " stop Stop running watchdog" echo " help Show this help" echo "" echo "Environment variables:" echo " WATCHDOG_INTERVAL Check interval in seconds (default: 10)" echo " WATCHDOG_CPU CPU threshold percentage (default: 85)" echo " WATCHDOG_SAMPLES High-CPU samples before restart (default: 3)" exit 0 ;;esac
# Check if already runningif [ -f "$PID_FILE" ]; then existing_pid=$(cat "$PID_FILE") if ps -p "$existing_pid" >/dev/null 2>&1; then echo "Watchdog already running (PID: $existing_pid)" echo "Use '$0 stop' to stop it first, or '$0 status' to check status" exit 1 else rm -f "$PID_FILE" fifi
main_loop