From c4f864e6229282ed749b76a3b781796cb370be0f Mon Sep 17 00:00:00 2001 From: dawn <90008@gaze.systems> Date: Fri, 3 Apr 2026 21:05:46 +0300 Subject: [PATCH] spindle/qemu: use vms for building the images Signed-off-by: dawn <90008@gaze.systems> --- spindle/engines/qemu/bakers/alpine.go | 163 +++++++++++++++----------- spindle/engines/qemu/bakers/baker.go | 32 ++--- spindle/engines/qemu/bakers/nbd.go | 133 --------------------- spindle/engines/qemu/bakers/ubuntu.go | 146 +++++++++++++---------- spindle/engines/qemu/engine.go | 46 +++----- spindle/engines/qemu/virt/models.go | 13 +- spindle/engines/qemu/virt/vm.go | 132 ++++++++++++++++++++- 7 files changed, 343 insertions(+), 322 deletions(-) delete mode 100644 spindle/engines/qemu/bakers/nbd.go diff --git a/spindle/engines/qemu/bakers/alpine.go b/spindle/engines/qemu/bakers/alpine.go index 46a8bf4f..e42b92dc 100644 --- a/spindle/engines/qemu/bakers/alpine.go +++ b/spindle/engines/qemu/bakers/alpine.go @@ -1,39 +1,18 @@ package bakers import ( + "bytes" "context" "encoding/json" "fmt" - "io" "os" - "os/exec" - "path/filepath" + "time" "tangled.org/core/log" + "tangled.org/core/spindle/engines/qemu/virt" ) -type AlpineBaker struct { - nbd *NBDManager -} - -func copyFile(src, dst string) error { - in, err := os.Open(src) - if err != nil { - return err - } - defer in.Close() - - out, err := os.Create(dst) - if err != nil { - return err - } - defer out.Close() - - if _, err := io.Copy(out, in); err != nil { - return err - } - return out.Sync() -} +type AlpineBaker struct{} func (p *AlpineBaker) Prepare(ctx context.Context, dir string) error { l := log.FromContext(ctx) @@ -44,61 +23,110 @@ func (p *AlpineBaker) Prepare(ctx context.Context, dir string) error { return fmt.Errorf("missing '%s' in %s", DiskName, dir) } - nbd, err := p.nbd.Connect(ctx, l, diskPath) + // use a temporary user-data to install the guest agent + userData := ` +packages: + - qemu-guest-agent +runcmd: + - rc-update add qemu-guest-agent default + - rc-service qemu-guest-agent start +` + + l.Info("starting vm for alpine preparation", "disk", diskPath) + img := virt.ResolvedImage{ + Disk: diskPath, + Shell: "/bin/ash", + UserData: userData, + } + conf := virt.VMConfig{ + Memory: "1G", + SMP: 1, + EnableKVM: true, + EnableDiskCow: false, // writable disk! + BootTimeout: 2 * time.Minute, + } + state, err := virt.StartVM(ctx, conf, img, l) if err != nil { - return err + return fmt.Errorf("starting preparation vm: %w", err) + } + defer state.Close() + + l.Info("vm is up, updating packages and ensuring guest agent is enabled") + + alpineInterfaces := `auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet static + address 10.0.2.15 + netmask 255.255.255.0 + gateway 10.0.2.2 + hostname alpine +` + + cmds := []string{ + "apk update", + "apk add --no-cache git curl qemu-guest-agent alpine-conf", + fmt.Sprintf("cat > /etc/network/interfaces < /etc/systemd/network/25-ens3.network < /etc/resolv.conf < 0 { - _, _ = wfLogger.DataWriter(idx, "stdout").Write(status.OutData) - } - if len(status.ErrData) > 0 { - _, _ = wfLogger.DataWriter(idx, "stderr").Write(status.ErrData) - } - - if status.Exited { - if status.ExitCode != 0 { - return engine.ErrWorkflowFailed - } - break - } - - select { - case <-ctx.Done(): - return engine.ErrTimedOut - case <-time.After(100 * time.Millisecond): - } + if exitCode != 0 { + return engine.ErrWorkflowFailed } return nil } diff --git a/spindle/engines/qemu/virt/models.go b/spindle/engines/qemu/virt/models.go index 523122bf..7e4d6306 100644 --- a/spindle/engines/qemu/virt/models.go +++ b/spindle/engines/qemu/virt/models.go @@ -2,16 +2,19 @@ package virt import ( "os" + "time" "github.com/digitalocean/go-qemu/qmp" ) type VMConfig struct { - Memory string - SMP int - EnableKVM bool - Dev bool - OverlayDir string + Memory string + SMP int + EnableKVM bool + EnableDiskCow bool + Dev bool + OverlayDir string + BootTimeout time.Duration } type ResolvedImage struct { diff --git a/spindle/engines/qemu/virt/vm.go b/spindle/engines/qemu/virt/vm.go index 6c13a4f0..73cae471 100644 --- a/spindle/engines/qemu/virt/vm.go +++ b/spindle/engines/qemu/virt/vm.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "io" "log/slog" "net" "os" @@ -17,6 +18,10 @@ import ( ) func StartVM(ctx context.Context, cfg VMConfig, img ResolvedImage, l *slog.Logger) (*VMState, error) { + if cfg.BootTimeout == 0 { + cfg.BootTimeout = time.Minute + } + // some systems have tmpfs at /tmp which is not ideal for large workloads targetTempDir := cfg.OverlayDir if targetTempDir == "" { @@ -62,7 +67,12 @@ func StartVM(ctx context.Context, cfg VMConfig, img ResolvedImage, l *slog.Logge "-m", cfg.Memory, "-smp", fmt.Sprintf("%d", cfg.SMP), "-display", "none", "-monitor", "none", "-nodefaults", "-no-user-config", // use snapshot=on to do copy-on-write without having us manage qcow overlays manually - "-drive", fmt.Sprintf("file=%s,media=disk,snapshot=on,if=virtio", img.Disk), + "-drive", fmt.Sprintf("file=%s,media=disk,snapshot=%s,if=virtio", img.Disk, func() string { + if cfg.EnableDiskCow { + return "on" + } + return "off" + }()), "-drive", fmt.Sprintf("file=%s,media=cdrom", SeedISOPath(tempDir)), "-netdev", "user,id=net0", "-device", "virtio-net-pci,netdev=net0", @@ -75,7 +85,7 @@ func StartVM(ctx context.Context, cfg VMConfig, img ResolvedImage, l *slog.Logge if cfg.Dev { argv = append(argv, "-serial", "stdio") } else { - argv = append(argv, "-serial", "none") + argv = append(argv, "-serial", "null") } // support booting using qemu bios still, but otherwise we make it faster! @@ -150,20 +160,20 @@ func StartVM(ctx context.Context, cfg VMConfig, img ResolvedImage, l *slog.Logge return nil, fmt.Errorf("qemu guest not running (status: %s)", status) } - qgaCtx, cancelQga := context.WithTimeout(ctx, time.Minute) + qgaCtx, cancelQga := context.WithTimeout(ctx, cfg.BootTimeout) defer cancelQga() // wait for guest agent to be ready (aka boot) for { if _, err := os.Stat(qgaSock); err == nil { - pingCtx, cancelPing := context.WithTimeout(qgaCtx, 5*time.Second) + pingCtx, cancelPing := context.WithTimeout(qgaCtx, 250*time.Millisecond) err = state.QGAGuestPing(pingCtx) cancelPing() if err == nil { l.Info("vm booted and guest-agent ready", "elapsed", time.Since(startedBootAt).Round(time.Millisecond)) break } - l.Debug("qga guest-ping failed", "error", err) + // l.Debug("qga guest-ping failed", "error", err) } select { @@ -318,3 +328,115 @@ func (s *VMState) QGAGuestExecStatus(ctx context.Context, pid int) (GuestExecSta return status, nil } + +// qgaexec executes a command on the guest and waits for it to finish, writing its output to stdout/stderr +func (s *VMState) QGAExec(ctx context.Context, command string, env []string, stdout, stderr io.Writer) (int, error) { + pid, err := s.QGAGuestExec(ctx, command, env) + if err != nil { + return 0, err + } + + for { + status, err := s.QGAGuestExecStatus(ctx, pid) + if err != nil { + return 0, err + } + + if len(status.OutData) > 0 && stdout != nil { + _, _ = stdout.Write(status.OutData) + } + if len(status.ErrData) > 0 && stderr != nil { + _, _ = stderr.Write(status.ErrData) + } + + if status.Exited { + return status.ExitCode, nil + } + + select { + case <-ctx.Done(): + return 0, ctx.Err() + case <-time.After(100 * time.Millisecond): + } + } +} + +// qgaguestfileopen opens a file on the guest and returns a handle +func (s *VMState) QGAGuestFileOpen(ctx context.Context, path string, mode string) (int, error) { + raw, err := s.QGARun(ctx, qmp.Command{ + Execute: "guest-file-open", + Args: map[string]any{"path": path, "mode": mode}, + }) + if err != nil { + return 0, fmt.Errorf("qga guest-file-open: %w", err) + } + + var resp struct { + Return int `json:"return"` + } + if err := json.Unmarshal(raw, &resp); err != nil { + return 0, fmt.Errorf("qga guest-file-open parse: %w", err) + } + return resp.Return, nil +} + +// qgaguestfileread reads data from an open file handle +func (s *VMState) QGAGuestFileRead(ctx context.Context, handle int, count int) ([]byte, bool, error) { + raw, err := s.QGARun(ctx, qmp.Command{ + Execute: "guest-file-read", + Args: map[string]any{"handle": handle, "count": count}, + }) + if err != nil { + return nil, false, fmt.Errorf("qga guest-file-read: %w", err) + } + + var resp struct { + Return struct { + BufB64 string `json:"buf-b64"` + EOF bool `json:"eof"` + } `json:"return"` + } + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, false, fmt.Errorf("qga guest-file-read parse: %w", err) + } + + data, err := base64.StdEncoding.DecodeString(resp.Return.BufB64) + if err != nil { + return nil, false, fmt.Errorf("qga guest-file-read decode: %w", err) + } + + return data, resp.Return.EOF, nil +} + +// qgaguestfileclose closes an open file handle +func (s *VMState) QGAGuestFileClose(ctx context.Context, handle int) error { + _, err := s.QGARun(ctx, qmp.Command{ + Execute: "guest-file-close", + Args: map[string]any{"handle": handle}, + }) + return err +} + +// qgaguestfilereadfull reads an entire file from the guest and returns its content +func (s *VMState) QGAGuestFileReadFull(ctx context.Context, path string) ([]byte, error) { + handle, err := s.QGAGuestFileOpen(ctx, path, "r") + if err != nil { + return nil, err + } + defer func() { _ = s.QGAGuestFileClose(ctx, handle) }() + + var fullData []byte + const chunkSize = 1024 * 1024 // 1MB chunks + for { + data, eof, err := s.QGAGuestFileRead(ctx, handle, chunkSize) + if err != nil { + return nil, err + } + fullData = append(fullData, data...) + if eof { + break + } + } + + return fullData, nil +} -- 2.54.0