From 6033716cbd6bce524f4ec276d7d41af0e1f0d64c Mon Sep 17 00:00:00 2001 From: dawn Date: Tue, 25 Aug 2026 00:21:24 +0900 Subject: [PATCH] spindle/engines/microvm: test idle debug ssh shells Signed-off-by: dawn --- spindle/engines/microvm/debug_e2e_test.go | 267 ++++++++++++++++++ .../engines/microvm/test-spindle-microvm.sh | 8 + 2 files changed, 275 insertions(+) create mode 100644 spindle/engines/microvm/debug_e2e_test.go diff --git a/spindle/engines/microvm/debug_e2e_test.go b/spindle/engines/microvm/debug_e2e_test.go new file mode 100644 index 00000000..2a5717fc --- /dev/null +++ b/spindle/engines/microvm/debug_e2e_test.go @@ -0,0 +1,267 @@ +//go:build linux + +package microvm + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/pem" + "fmt" + "io" + "log/slog" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + gliderssh "github.com/gliderlabs/ssh" + gossh "golang.org/x/crypto/ssh" + agentv1 "tangled.org/core/spindle/agentproto/gen" + "tangled.org/core/spindle/models" +) + +func TestDebugShellStaysOpenWhileIdleE2E(t *testing.T) { + imageSpecPath := os.Getenv("SPINDLE_E2E_IMAGE_SPEC") + if imageSpecPath == "" { + t.Skip("SPINDLE_E2E_IMAGE_SPEC is not set") + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + imageSpec, err := LoadImageSpec(imageSpecPath) + if err != nil { + t.Fatal(err) + } + + agentListener, agentPort, err := listenRandomVsockPort(ctx) + if err != nil { + t.Fatal(err) + } + defer agentListener.Close() + imageSpec.BootArgs = fmt.Sprintf("%s shuttle.vsock_port=%d", imageSpec.BootArgs, agentPort) + + workDir, err := os.MkdirTemp("/tmp", "spindle-debug-e2e-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(workDir) + + logger := slog.Default() + vm, err := StartVM(ctx, VMConfig{ + Image: imageSpec, + BootTimeout: 30 * time.Second, + EnableKVM: true, + WorkDir: workDir, + Dev: true, + }, logger) + if err != nil { + t.Fatal(err) + } + defer vm.Close() + + conn, err := acceptWithTimeout(ctx, &cidFilteredVsockListener{ + Listener: agentListener, + cid: vm.CID(), + logger: logger, + }, 60*time.Second) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + agent := NewAgentSession(conn, logger) + if err := agent.Init(ctx, &agentv1.Init{JobId: "debug-idle-e2e"}); err != nil { + t.Fatal(err) + } + + failedStep := &agentv1.ExecStart{ + Argv: []string{ + "/run/current-system/sw/bin/bash", + "-lc", + "exit 37", + }, + Env: []string{ + "HOME=/workspace", + "LOGNAME=spindle-workflow", + "PATH=/run/current-system/sw/bin:/usr/bin:/bin", + "USER=spindle-workflow", + "CI=true", + "WF_ONLY=workflow", + "STEP_ONLY=step", + "OVERRIDE=step", + }, + Cwd: "/workspace/repo", + User: "spindle-workflow", + } + if exitCode, err := agent.Exec(ctx, AgentExec{ + ID: "failed-step", + ExecStart: failedStep, + Stdout: io.Discard, + Stderr: io.Discard, + }); err != nil || exitCode != 37 { + t.Fatalf("failed step: exit code %d, error %v", exitCode, err) + } + if failedStep.TimeoutSeconds == 0 { + t.Fatal("AgentSession.Exec did not retain the effective step timeout") + } + + wid := models.WorkflowId{ + PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "pipeline"}, + Name: "debug-idle-e2e.yml", + } + engine := &Engine{l: logger, debug: make(map[string]debugTarget)} + engine.registerDebugTarget(wid, debugTarget{ + cid: vm.CID(), + agent: agent, + failedStep: failedStep, + connected: make(chan struct{}), + released: make(chan struct{}), + }) + handle := newDebugHandle(wid) + + sshListener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + sshServer := &gliderssh.Server{ + Handler: engine.debugHandle, + PublicKeyHandler: func(gliderssh.Context, gliderssh.PublicKey) bool { + return true + }, + } + hostKeyPath := filepath.Join(workDir, "host-key") + if err := ensureDebugHostKey(hostKeyPath); err != nil { + t.Fatal(err) + } + if err := sshServer.SetOption(gliderssh.HostKeyFile(hostKeyPath)); err != nil { + t.Fatal(err) + } + serverDone := make(chan error, 1) + go func() { serverDone <- sshServer.Serve(sshListener) }() + defer func() { + _ = sshServer.Close() + if err := <-serverDone; err != nil && err != gliderssh.ErrServerClosed { + t.Errorf("debug SSH server: %v", err) + } + }() + + _, privateKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + privateKeyBlock, err := gossh.MarshalPrivateKey(privateKey, "") + if err != nil { + t.Fatal(err) + } + clientKeyPath := filepath.Join(workDir, "client-key") + if err := os.WriteFile(clientKeyPath, pem.EncodeToMemory(privateKeyBlock), 0o600); err != nil { + t.Fatal(err) + } + + host, port, err := net.SplitHostPort(sshListener.Addr().String()) + if err != nil { + t.Fatal(err) + } + target := handle + "@" + host + controlPath := filepath.Join(workDir, "control") + commonArgs := []string{ + "-o", "BatchMode=yes", + "-o", "IdentitiesOnly=yes", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-i", clientKeyPath, + "-p", port, + } + masterArgs := append([]string{"-M", "-N", "-S", controlPath}, commonArgs...) + masterArgs = append(masterArgs, target) + master := exec.CommandContext(ctx, "ssh", masterArgs...) + var masterOutput bytes.Buffer + master.Stdout = &masterOutput + master.Stderr = &masterOutput + if err := master.Start(); err != nil { + t.Fatal(err) + } + masterDone := make(chan error, 1) + go func() { masterDone <- master.Wait() }() + masterStopped := false + defer func() { + if !masterStopped { + _ = master.Process.Kill() + <-masterDone + } + }() + + controlDeadline := time.Now().Add(10 * time.Second) + for { + if _, err := os.Stat(controlPath); err == nil { + break + } + select { + case err := <-masterDone: + masterStopped = true + t.Fatalf("SSH control master exited before creating its socket: %v\noutput:\n%s", err, masterOutput.String()) + case <-time.After(10 * time.Millisecond): + } + if time.Now().After(controlDeadline) { + t.Fatalf("SSH control master did not create %s\noutput:\n%s", controlPath, masterOutput.String()) + } + } + + sessionArgs := append([]string{"-S", controlPath, "-tt"}, commonArgs...) + sessionArgs = append(sessionArgs, target) + session := exec.CommandContext(ctx, "ssh", sessionArgs...) + stdin, err := session.StdinPipe() + if err != nil { + t.Fatal(err) + } + var output bytes.Buffer + session.Stdout = &output + session.Stderr = &output + session.Env = append(os.Environ(), "TERM=xterm-idle-e2e") + if err := session.Start(); err != nil { + t.Fatal(err) + } + waitDone := make(chan error, 1) + go func() { waitDone <- session.Wait() }() + + const idleTime = 5 * time.Second + select { + case err := <-waitDone: + t.Fatalf("debug shell exited while idle before %s: %v\noutput:\n%s", idleTime, err, output.String()) + case <-time.After(idleTime): + } + + probe := `printf '__DEBUG_IDLE_E2E__|CI=%s|WF=%s|STEP=%s|OVERRIDE=%s|XDG=%s|PWD=%s|USER=%s|LOGNAME=%s|HOME=%s|TERM=%s|\n' "$CI" "$WF_ONLY" "$STEP_ONLY" "$OVERRIDE" "$XDG_CACHE_HOME" "$PWD" "$USER" "$LOGNAME" "$HOME" "$TERM"; exit` + "\n" + if _, err := io.WriteString(stdin, probe); err != nil { + t.Fatal(err) + } + + select { + case err := <-waitDone: + if err != nil { + t.Fatalf("debug shell exit: %v\noutput:\n%s", err, output.String()) + } + case <-time.After(15 * time.Second): + t.Fatal("debug shell did not exit after probe") + } + + if err := master.Process.Kill(); err != nil { + t.Fatal(err) + } + <-masterDone + masterStopped = true + + want := "__DEBUG_IDLE_E2E__|CI=true|WF=workflow|STEP=step|OVERRIDE=step|XDG=/workspace/.cache|PWD=/workspace/repo|USER=spindle-workflow|LOGNAME=spindle-workflow|HOME=/workspace|TERM=xterm-idle-e2e|" + if !strings.Contains(output.String(), want) { + t.Fatalf("debug shell did not preserve the failed step context\nwant: %s\noutput:\n%s", want, output.String()) + } + if _, ok := engine.lookupDebugTarget(handle); ok { + t.Fatal("debug target remained registered after shell exit") + } +} diff --git a/spindle/engines/microvm/test-spindle-microvm.sh b/spindle/engines/microvm/test-spindle-microvm.sh index 98226f32..36484313 100755 --- a/spindle/engines/microvm/test-spindle-microvm.sh +++ b/spindle/engines/microvm/test-spindle-microvm.sh @@ -1050,6 +1050,13 @@ test_almalinux10_nix() { test_generic_distro_nix almalinux10 $AL10_IMAGE_SPEC_JSON } +test_debug_shell_idle() { + nix develop --command env SPINDLE_E2E_IMAGE_SPEC="$IMAGE_SPEC_JSON" \ + go test -mod=mod ./spindle/engines/microvm \ + -run '^TestDebugShellStaysOpenWhileIdleE2E$' -count=1 -v || return 1 + echo "success: debug SSH shell stayed alive while idle, preserved the failed step context, and exited cleanly" +} + test_oom_detection() { local label spec for label in "almalinux10" "alpine" "nixos"; do @@ -1113,6 +1120,7 @@ TESTS=( test_activation_cache_substitution test_activation_docker test_activation_cached_realize + test_debug_shell_idle test_oom_detection ) -- 2.51.2