diff --git a/nix/vm.nix b/nix/vm.nix index a54e282f..937d72ff 100644 --- a/nix/vm.nix +++ b/nix/vm.nix @@ -43,11 +43,11 @@ in guest.port = 6000; } # spindle - { - from = "host"; - host.port = 6555; - guest.port = 6555; - } + # { + # from = "host"; + # host.port = 6555; + # guest.port = 6555; + # } ]; sharedDirectories = { # We can't use the 9p mounts directly for most of these @@ -83,7 +83,7 @@ in }; }; services.tangled-spindle = { - enable = true; + enable = false; server = { owner = envVar "TANGLED_VM_SPINDLE_OWNER"; hostname = "localhost:6555"; diff --git a/spindle/engines/nixery/engine.go b/spindle/engines/nixery/engine.go index 73e26065..4ac4672b 100644 --- a/spindle/engines/nixery/engine.go +++ b/spindle/engines/nixery/engine.go @@ -173,16 +173,6 @@ func New(ctx context.Context, cfg *config.Config) (*Engine, error) { func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) error { e.l.Info("setting up workflow", "workflow", wid) - _, err := e.docker.NetworkCreate(ctx, networkName(wid), network.CreateOptions{ - Driver: "bridge", - }) - if err != nil { - return err - } - e.registerCleanup(wid, func(ctx context.Context) error { - return e.docker.NetworkRemove(ctx, networkName(wid)) - }) - addl := wf.Data.(addlFields) reader, err := e.docker.ImagePull(ctx, addl.image, image.PullOptions{}) @@ -194,6 +184,16 @@ func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *m defer reader.Close() io.Copy(os.Stdout, reader) + _, err = e.docker.NetworkCreate(ctx, networkName(wid), network.CreateOptions{ + Driver: "bridge", + }) + if err != nil { + return err + } + e.registerCleanup(wid, func(ctx context.Context) error { + return e.docker.NetworkRemove(ctx, networkName(wid)) + }) + resp, err := e.docker.ContainerCreate(ctx, &container.Config{ Image: addl.image, Cmd: []string{"cat"}, @@ -294,21 +294,25 @@ func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.W for _, s := range secrets { workflowEnvs.AddEnv(s.Key, s.Value) } - step := w.Steps[idx].(Step) - select { case <-ctx.Done(): return ctx.Err() default: } - envs := append(EnvVars(nil), workflowEnvs...) for k, v := range step.environment { envs.AddEnv(k, v) } envs.AddEnv("HOME", homeDir) + e.l.Info("executing step", + "workflow_id", wid.String(), + "step_index", idx, + "step_name", step.Name, + "command", step.command, + ) + mkExecResp, err := e.docker.ContainerExecCreate(ctx, addl.container, container.ExecOptions{ Cmd: []string{"bash", "-c", step.command}, AttachStdout: true, @@ -327,16 +331,13 @@ func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.W select { case <-tailDone: - case <-ctx.Done(): // cleanup will be handled by DestroyWorkflow, since // Docker doesn't provide an API to kill an exec run // (sure, we could grab the PID and kill it ourselves, // but that's wasted effort) e.l.Warn("step timed out", "step", step.Name) - <-tailDone - return engine.ErrTimedOut } @@ -346,26 +347,89 @@ func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.W default: } - execInspectResp, err := e.docker.ContainerExecInspect(ctx, mkExecResp.ID) + if err = e.handleStepFailure(ctx, wid, w, idx, mkExecResp.ID); err != nil { + return err + } + + e.l.Info("step completed successfully", + "workflow_id", wid.String(), + "step_index", idx, + "step_name", step.Name, + ) + + return nil +} + +// logStepFailure logs detailed information about a failed workflow step +func (e *Engine) handleStepFailure( + ctx context.Context, + wid models.WorkflowId, + w *models.Workflow, + idx int, + execID string, +) error { + addl := w.Data.(addlFields) + step := w.Steps[idx].(Step) + + inspectResp, err := e.docker.ContainerInspect(ctx, addl.container) if err != nil { return err } - if execInspectResp.ExitCode != 0 { - inspectResp, err := e.docker.ContainerInspect(ctx, addl.container) - if err != nil { - return err + execInspectResp, err := e.docker.ContainerExecInspect(ctx, execID) + if err != nil { + return err + } + + // no error + if execInspectResp.ExitCode == 0 { + return nil + } + + logFields := []any{ + "workflow_id", wid.String(), + "step_index", idx, + "step_name", step.Name, + "command", step.command, + "container_exit_code", inspectResp.State.ExitCode, + "container_oom_killed", inspectResp.State.OOMKilled, + "exec_exit_code", execInspectResp.ExitCode, + } + + // Add container state information + if inspectResp.State != nil { + logFields = append(logFields, + "container_status", inspectResp.State.Status, + "container_running", inspectResp.State.Running, + "container_paused", inspectResp.State.Paused, + "container_restarting", inspectResp.State.Restarting, + "container_dead", inspectResp.State.Dead, + ) + + if inspectResp.State.Error != "" { + logFields = append(logFields, "container_error", inspectResp.State.Error) } - e.l.Error("workflow failed!", "workflow_id", wid.String(), "exit_code", execInspectResp.ExitCode, "oom_killed", inspectResp.State.OOMKilled) + if inspectResp.State.StartedAt != "" { + logFields = append(logFields, "container_started_at", inspectResp.State.StartedAt) + } - if inspectResp.State.OOMKilled { - return ErrOOMKilled + if inspectResp.State.FinishedAt != "" { + logFields = append(logFields, "container_finished_at", inspectResp.State.FinishedAt) } - return engine.ErrWorkflowFailed } - return nil + // Add resource usage if available + if inspectResp.HostConfig != nil && inspectResp.HostConfig.Memory > 0 { + logFields = append(logFields, "memory_limit", inspectResp.HostConfig.Memory) + } + + e.l.Error("workflow step failed!", logFields...) + + if inspectResp.State.OOMKilled { + return ErrOOMKilled + } + return engine.ErrWorkflowFailed } func (e *Engine) tailStep(ctx context.Context, wfLogger *models.WorkflowLogger, execID string, wid models.WorkflowId, stepIdx int, step models.Step) error {