diff --git a/cmd/spindle-microvm-run/main_linux.go b/cmd/spindle-microvm-run/main_linux.go --- a/cmd/spindle-microvm-run/main_linux.go +++ b/cmd/spindle-microvm-run/main_linux.go @@ -304,6 +304,11 @@ Stderr: os.Stderr, }) if err != nil { + if detail := microvm.VMCrashLog(vm); detail != "" { + if parsedErr, ok := microvm.ParseCrashLog(detail); ok { + return fmt.Errorf("%w: %w", parsedErr, err) + } + } return err } diff --git a/shuttle/src/command.rs b/shuttle/src/command.rs --- a/shuttle/src/command.rs +++ b/shuttle/src/command.rs @@ -153,7 +153,26 @@ }) } +fn read_oom_kill_count() -> u32 { + let content = match std::fs::read_to_string("/proc/vmstat") { + Ok(c) => c, + Err(_) => return 0, + }; + for line in content.lines() { + let mut parts = line.split_whitespace(); + if parts.next() == Some("oom_kill") { + if let Some(count_str) = parts.next() { + if let Ok(count) = count_str.parse::() { + return count; + } + } + } + } + 0 +} + pub fn spawn_streaming(mut spec: Spec) -> Result { + let oom_kill_before = read_oom_kill_count(); let mut child = spawn(&mut spec)?; let stdout = child.stdout.take().context("stdout pipe missing")?; let stderr = child.stderr.take().context("stderr pipe missing")?; @@ -164,7 +183,7 @@ drop(events_tx); let exit = tokio::spawn(async move { - let exit = wait_child(&mut child, spec.timeout).await; + let exit = wait_child(&mut child, spec.timeout, oom_kill_before).await; // ensure all output is observed before exiting // this assumes children dont daemonize and hold onto the stdout/err @@ -226,7 +245,11 @@ .with_context(|| format!("spawn {}", display_os(&spec.program))) } -async fn wait_child(child: &mut Child, timeout: Option) -> ExitResult { +async fn wait_child( + child: &mut Child, + timeout: Option, + oom_kill_before: u32, +) -> ExitResult { let wait = child.wait(); let status = match timeout { Some(timeout) => match tokio::time::timeout(timeout, wait).await { @@ -249,14 +272,25 @@ }; match status { - Ok(status) => ExitResult { - exit_code: status - .code() - .or_else(|| status.signal().map(|signal| 128 + signal)) - .unwrap_or(1), - error: None, - timed_out: false, - }, + Ok(status) => { + let code = status.code(); + let signal = status.signal(); + let exit_code = code.or_else(|| signal.map(|sig| 128 + sig)).unwrap_or(1); + + let mut error = None; + if signal == Some(9) { + let oom_kill_after = read_oom_kill_count(); + if oom_kill_after > oom_kill_before { + error = Some("guest process killed by guest kernel OOM".to_owned()); + } + } + + ExitResult { + exit_code, + error, + timed_out: false, + } + } Err(error) => ExitResult { exit_code: 1, error: Some(error.to_string()), diff --git a/spindle/engines/microvm/agent.go b/spindle/engines/microvm/agent.go --- a/spindle/engines/microvm/agent.go +++ b/spindle/engines/microvm/agent.go @@ -186,13 +186,15 @@ } else if p := msg.BuiltPaths; p != nil { // s.l.Debug("guest built paths", "reason", p.Reason, "count", len(p.Paths)) } else if p := msg.ExecExit; p != nil { + var err error if p.Error != "" { s.l.Warn("guest exec error", "id", msg.Id, "error", p.Error) + err = fmt.Errorf("guest exec error: %s", p.Error) } if p.TimedOut { return int(p.ExitCode), errGuestTimedOut } - return int(p.ExitCode), nil + return int(p.ExitCode), err } } } diff --git a/spindle/engines/microvm/engine.go b/spindle/engines/microvm/engine.go --- a/spindle/engines/microvm/engine.go +++ b/spindle/engines/microvm/engine.go @@ -11,6 +11,7 @@ "os" "path/filepath" "slices" + "strings" "sync" "sync/atomic" "time" @@ -248,7 +249,7 @@ if setupDone { return } - if detail := vmCrashLog(state.VM); detail != "" { + if detail := VMCrashLog(state.VM); detail != "" { l.Error("microVM setup failed", "detail", detail) } if err := e.cleanupState(context.Background(), wid, state); err != nil { @@ -416,7 +417,7 @@ if oom { reason = "microVM killed by OOM (cgroup memory limit exceeded)" } - if detail := vmCrashLog(state.VM); detail != "" { + if detail := VMCrashLog(state.VM); detail != "" { fmt.Fprintf(stderr, "%s:\n%s\n", reason, detail) l.Error(reason, "oom", oom, "detail", detail) } else { @@ -434,13 +435,24 @@ // the agent connection dropped while qemu stayed up (eg. the guest kernel // OOM-killed the agent or a guest panic), so surface serial logs, those // will be more helpful. - if detail := vmCrashLog(state.VM); detail != "" { + var crashErr error + if detail := VMCrashLog(state.VM); detail != "" { fmt.Fprintf(stderr, "step failed (%v):\n%s\n", err, detail) l.Error("step failed", "error", err, "detail", detail) + if parsedErr, ok := ParseCrashLog(detail); ok { + crashErr = parsedErr + } else { + if strings.Contains(err.Error(), "guest exec error:") { + crashErr = err + } else { + crashErr = fmt.Errorf("guest agent connection lost: %w", err) + } + } } else { l.Error("step failed", "error", err) + crashErr = err } - return fmt.Errorf("%s:\n%w", category, err) + return fmt.Errorf("%s:\n%w", category, crashErr) } func (e *Engine) activateConfig(ctx context.Context, wid models.WorkflowId, state *workflowState, step Step, out io.Writer) error { diff --git a/spindle/engines/microvm/qemu.go b/spindle/engines/microvm/qemu.go --- a/spindle/engines/microvm/qemu.go +++ b/spindle/engines/microvm/qemu.go @@ -137,7 +137,7 @@ var ok bool defer func() { if !ok { - if detail := vmCrashLog(handle); detail != "" { + if detail := VMCrashLog(handle); detail != "" { logger.Error("microVM failed to start", "cid", handle.cid, "detail", detail) } _ = handle.Close() diff --git a/spindle/engines/microvm/test-spindle-microvm.sh b/spindle/engines/microvm/test-spindle-microvm.sh --- a/spindle/engines/microvm/test-spindle-microvm.sh +++ b/spindle/engines/microvm/test-spindle-microvm.sh @@ -971,6 +971,47 @@ echo "success: alpine guest substituted, queried the store db, built via both CLIs, and uploaded both outputs" } +test_oom_detection() { + local label spec + for label in "alpine" "nixos"; do + echo "testing oom on $label..." + local work_dir="$TEMP_DIR/work-oom-test-$label" + mkdir -p "$work_dir" + + local -a cmd_args + if [ "$label" = "alpine" ]; then + spec="$ALPINE_IMAGE_SPEC_JSON" + cmd_args=(awk 'BEGIN { while(1) a[i++]=1 }') + else + spec="$IMAGE_SPEC_JSON" + cmd_args=(/run/current-system/sw/bin/jq -n '[repeat(1)]') + fi + + local mem_mib=128 + if [ "$label" = "nixos" ]; then + mem_mib=512 + fi + + local args=( + --image-spec "$spec" + --work-dir "$work_dir" + --exec-timeout "45s" + --port "$SPINDLE_TEST_VSOCK_PORT" + --memory-mib "$mem_mib" + ) + + local out + if out=$(./spindle/spindle-microvm-run "${args[@]}" -- "${cmd_args[@]}" 2>&1); then + echo "error: expected spindle-microvm-run to fail on $label, but it exited 0" >&2 + echo "output: $out" >&2 + return 1 + fi + + check_needles "$out" "killed by guest kernel OOM" || return 1 + done + echo "success: guest process OOM detected successfully on both alpine and nixos" +} + TESTS=( test_alpine test_alpine_nix @@ -986,6 +1027,7 @@ test_activation_cache_substitution test_activation_docker test_activation_cached_realize + test_oom_detection ) log "running ${#TESTS[@]} tests" diff --git a/spindle/engines/microvm/vm.go b/spindle/engines/microvm/vm.go --- a/spindle/engines/microvm/vm.go +++ b/spindle/engines/microvm/vm.go @@ -14,6 +14,7 @@ "os" "os/exec" "path/filepath" + "regexp" "slices" "strings" "sync/atomic" @@ -285,7 +286,7 @@ return watchCtx, exited, cancel } -func vmCrashLog(vm VMHandle) string { +func VMCrashLog(vm VMHandle) string { if vm == nil { return "" } @@ -372,4 +373,22 @@ } return runner.Start(ctx, cfg, volumePaths, logger) +} + +// checks serial log for ooms or kernel panic +// this is very linux specific! but these strings are stable in linux itself, see mm/oom_kill.c and kernel/panic.c +func ParseCrashLog(detail string) (error, bool) { + if strings.Contains(detail, "Out of memory:") { + // we can show process name where possible + re := regexp.MustCompile(`Out of memory: Killed process \d+ \(([^)]+)\)`) + matches := re.FindStringSubmatch(detail) + if len(matches) > 1 { + return fmt.Errorf("guest out of memory (process '%s' killed by guest kernel OOM)", matches[1]), true + } + return errors.New("guest out of memory (OOM killer invoked)"), true + } + if strings.Contains(detail, "Kernel panic") { + return errors.New("guest kernel panic"), true + } + return nil, false }