diff --git a/flake.nix b/flake.nix index 40261a0e..510c2857 100644 --- a/flake.nix +++ b/flake.nix @@ -189,6 +189,12 @@ zoekt-webserver = self.callPackage ./nix/pkgs/zoekt-webserver.nix {}; zoekt-tngl-indexserver = self.callPackage ./nix/pkgs/zoekt-tngl-indexserver.nix {}; }); + + spindleNixosFor = system: + nixpkgs.lib.nixosSystem { + inherit system; + modules = [self.nixosModules.spindle-nixos]; + }; in { overlays.default = final: prev: { inherit @@ -216,6 +222,8 @@ packages = forAllSystems (system: let pkgs = nixpkgsFor.${system}; linuxPkgs = nixpkgsFor."x86_64-linux"; + imageSystem = "${nixpkgs.lib.head (nixpkgs.lib.splitString "-" system)}-linux"; + imagePkgs = nixpkgsFor.${imageSystem}; packages = mkPackageSet pkgs; staticPackages = mkPackageSet pkgs.pkgsStatic; crossPackages = mkPackageSet pkgs.pkgsCross.gnu64.pkgsStatic; @@ -298,10 +306,10 @@ }; }; - spindle-nixos-image = linuxPkgs.callPackage ./nix/pkgs/spindle-nixos-image.nix { - nixosSystem = self.nixosConfigurations.spindle-nixos; + spindle-nixos-image = imagePkgs.callPackage ./nix/pkgs/spindle-nixos-image.nix { + nixosSystem = spindleNixosFor imageSystem; }; - spindle-nixos-image-tarball = linuxPkgs.runCommand "spindle-nixos-image-tarball.tar.gz" {} '' + spindle-nixos-image-tarball = imagePkgs.runCommand "spindle-nixos-image-tarball.tar.gz" {} '' tar -S -C ${self.packages.${system}.spindle-nixos-image} -h -czf $out . ''; @@ -695,13 +703,9 @@ formatter = forAllSystems (system: self.packages.${system}.treefmt-wrapper); - nixosConfigurations = let - spindleNixosBase = nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [self.nixosModules.spindle-nixos]; - }; - in { - spindle-nixos = spindleNixosBase; + nixosConfigurations = { + spindle-nixos = spindleNixosFor "x86_64-linux"; + spindle-nixos-aarch64 = spindleNixosFor "aarch64-linux"; }; }; } diff --git a/nix/microvm/qemu.nix b/nix/microvm/qemu.nix index e72ec720..0d014e37 100644 --- a/nix/microvm/qemu.nix +++ b/nix/microvm/qemu.nix @@ -1,7 +1,8 @@ {...}: { microvm = { hypervisor = "qemu"; - qemu.machine = "microvm"; + # qemu.machine is not set because microvm.nix already defaults it per arch + # (microvm on x86_64, virt on aarch64) optimize.enable = true; diff --git a/nix/pkgs/spindle-almalinux-image.nix b/nix/pkgs/spindle-almalinux-image.nix index db810779..7ef9f115 100644 --- a/nix/pkgs/spindle-almalinux-image.nix +++ b/nix/pkgs/spindle-almalinux-image.nix @@ -1,6 +1,7 @@ { doas-sudo-shim-src, pkgsStatic, + lib, runCommand, writeText, python3, @@ -152,14 +153,11 @@ kernel = "kernel"; initrd = "initrd"; runnerType = "qemu"; - runnerConfig = { - cpu = "host,+x2apic,-sgx"; - machine = "microvm,accel=kvm:tcg,acpi=on,mem-merge=on,pcie=on,pic=off,pit=off,rtc=on,usb=off"; - console = "hvc0"; - extraArgs = []; - # RHEL/AlmaLinux kernels are built with CONFIG_VIRTIO_MMIO, so use PCI - # instead (this is why pcie=on in the `machine` line above, instead of - # `pcie=off` like other VM images) + # RHEL/AlmaLinux kernels are built with CONFIG_VIRTIO_MMIO, so use PCI + # instead (this is why pcie=on, instead of `pcie=off` like other VM images) + runnerConfig = (import ./spindle-qemu-runner.nix {inherit lib;}).mkQemuRunner { + inherit arch; + pcie = true; virtioTransport = "pci"; }; memoryMiB = 4096; diff --git a/nix/pkgs/spindle-alpine-image.nix b/nix/pkgs/spindle-alpine-image.nix index b33e61e0..ba85d128 100644 --- a/nix/pkgs/spindle-alpine-image.nix +++ b/nix/pkgs/spindle-alpine-image.nix @@ -1,5 +1,6 @@ { pkgsStatic, + lib, runCommand, writeText, squashfsTools, @@ -82,12 +83,7 @@ kernel = "kernel"; initrd = "initrd"; runnerType = "qemu"; - runnerConfig = { - cpu = "host,+x2apic,-sgx"; - machine = "microvm,accel=kvm:tcg,acpi=on,mem-merge=on,pcie=off,pic=off,pit=off,rtc=on,usb=off"; - console = "hvc0"; - extraArgs = []; - }; + runnerConfig = (import ./spindle-qemu-runner.nix {inherit lib;}).mkQemuRunner {inherit arch;}; memoryMiB = 4096; storeDisk = "store-disk"; storeDiskType = "squashfs"; diff --git a/nix/pkgs/spindle-nixos-image.nix b/nix/pkgs/spindle-nixos-image.nix index af04f8f0..f8b0d94b 100644 --- a/nix/pkgs/spindle-nixos-image.nix +++ b/nix/pkgs/spindle-nixos-image.nix @@ -3,8 +3,10 @@ lib, nixosSystem, }: let - system = nixosSystem.pkgs.stdenv.hostPlatform.qemuArch; + guest = nixosSystem.pkgs.stdenv.hostPlatform; + system = guest.qemuArch; microvm = nixosSystem.config.microvm; + inherit (import ./spindle-qemu-runner.nix {inherit lib;}) mkQemuRunner; baseConfigHash = lib.pipe nixosSystem.config.system.build.toplevel.outPath [ (lib.strings.removePrefix "/nix/store/") (lib.strings.splitString "-") @@ -13,15 +15,15 @@ imageSpecJSON = pkgs.writeText "spec.json" ( builtins.toJSON { arch = system; - bootArgs = "earlyprintk=ttyS0 console=hvc0 reboot=t panic=-1 ${lib.concatStringsSep " " microvm.kernelParams}"; + # earlyprintk is x86-only + bootArgs = "${lib.optionalString guest.isx86_64 "earlyprintk=ttyS0 "}console=hvc0 reboot=t panic=-1 ${lib.concatStringsSep " " microvm.kernelParams}"; kernel = "kernel"; initrd = "initrd"; runnerType = "qemu"; - runnerConfig = { - cpu = "host,+x2apic,-sgx"; - machine = "microvm,accel=kvm:tcg,acpi=on,mem-merge=on,pcie=off,pic=off,pit=off,rtc=on,usb=off"; - console = "hvc0"; - extraArgs = []; + # the runner has to boot the machine the guest was built for + runnerConfig = mkQemuRunner { + arch = system; + machine = microvm.qemu.machine; }; memoryMiB = microvm.mem; storeDisk = "store-disk"; @@ -52,7 +54,7 @@ in pkgs.runCommand "spindle-nixos-image-${system}" {} '' mkdir -p "$out" cp ${imageSpecJSON} "$out/spec.json" - ln -s ${microvm.kernel}/bzImage "$out/kernel" + ln -s ${microvm.kernel}/${guest.linux-kernel.target} "$out/kernel" ln -s ${microvm.initrdPath} "$out/initrd" ln -s ${microvm.storeDisk} "$out/store-disk" '' diff --git a/nix/pkgs/spindle-qemu-runner.nix b/nix/pkgs/spindle-qemu-runner.nix new file mode 100644 index 00000000..61bf5153 --- /dev/null +++ b/nix/pkgs/spindle-qemu-runner.nix @@ -0,0 +1,32 @@ +{lib}: { + # runner args for the arch and machine the guest was built for + mkQemuRunner = { + arch ? "x86_64", + machine ? "microvm", + pcie ? false, + virtioTransport ? null, + }: let + isX86 = arch == "x86_64"; + machineOpts = { + microvm = "acpi=on,mem-merge=on,pcie=${ + if pcie + then "on" + else "off" + },pic=off,pit=off,rtc=on,usb=off"; + virt = "gic-version=max,its=off,msi=off,mem-merge=on"; + }; + in + { + # qemu < 10.0 crashes when a microvm guest reads the sgx cpuid leaf + # https://gitlab.com/qemu-project/qemu/-/issues/2142 + cpu = + if isX86 + then "host,+x2apic,-sgx" + else "host"; + machine = "${machine},accel=kvm:tcg,${machineOpts.${machine}}"; + console = "hvc0"; + # i8042 only exists on x86, it is the kernel's reset path there + extraArgs = lib.optionals isX86 ["-device" "i8042"]; + } + // lib.optionalAttrs (virtioTransport != null) {inherit virtioTransport;}; +} diff --git a/spindle/engines/microvm/qemu.go b/spindle/engines/microvm/qemu.go index da62d4ab..99f32547 100644 --- a/spindle/engines/microvm/qemu.go +++ b/spindle/engines/microvm/qemu.go @@ -707,7 +707,6 @@ func addQEMUKVMArgs(b *argBuilder, image ImageSpec) { if image.RunnerConfig.CPU != "" { b.Opt("-cpu", image.RunnerConfig.CPU) } - b.Opt("-device", "i8042") } func addQEMUVolumeArgs(b *argBuilder, cfg qemuArgsConfig) error {