diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ .env *.rdb .envrc # Created if following hacking.md -genjwks.out \ No newline at end of file +genjwks.out +/nix/vm-data diff --git a/docs/hacking.md b/docs/hacking.md --- a/docs/hacking.md +++ b/docs/hacking.md @@ -64,16 +64,14 @@ You will also need to set the `$TANGLED_VM_SPINDLE_OWNER` variable to some value. If you don't want to [set up a spindle](#running-a-spindle), you can use any placeholder -value. +value. -You can now start a lightweight NixOS VM using -`nixos-shell` like so: +You can now start a lightweight NixOS VM like so: ```bash -nix run .#vm -# or nixos-shell --flake .#vm +nix run --impure .#vm -# hit Ctrl-a + c + q to exit the VM +# type `poweroff` at the shell to exit the VM ``` This starts a knot on port 6000, a spindle on port 6555 diff --git a/flake.nix b/flake.nix --- a/flake.nix +++ b/flake.nix @@ -175,28 +175,31 @@ type = "app"; program = ''${tailwind-watcher}/bin/run''; }; vm = let - system = + guestSystem = if pkgs.stdenv.hostPlatform.isAarch64 - then "aarch64" - else "x86_64"; - - nixos-shell = pkgs.nixos-shell.overrideAttrs (old: { - patches = - (old.patches or []) - ++ [ - # https://github.com/Mic92/nixos-shell/pull/94 - (pkgs.fetchpatch { - name = "fix-foreign-vm.patch"; - url = "https://github.com/Mic92/nixos-shell/commit/113e4cc55ae236b5b0b1fbd8b321e9b67c77580e.patch"; - hash = "sha256-eauetBK0wXAOcd9PYbExokNCiwz2QyFnZ4FnwGi9VCo="; - }) - ]; - }); + then "aarch64-linux" + else "x86_64-linux"; in { type = "app"; - program = toString (pkgs.writeShellScript "vm" '' - ${nixos-shell}/bin/nixos-shell --flake .#vm-${system} --guest-system ${system}-linux - ''); + program = + (pkgs.writeShellApplication { + name = "launch-vm"; + text = '' + rootDir=$(jj --ignore-working-copy root || git rev-parse --show-toplevel) || (echo "error: can't find repo root?"; exit 1) + cd "$rootDir" + + mkdir -p nix/vm-data/{knot,repos,spindle,spindle-logs} + + export TANGLED_VM_DATA_DIR="$rootDir/nix/vm-data" + exec ${pkgs.lib.getExe + (import ./nix/vm.nix { + inherit nixpkgs self; + system = guestSystem; + hostSystem = system; + }).config.system.build.vm} + ''; + }) + + /bin/launch-vm; }; gomod2nix = { type = "app"; @@ -257,14 +260,6 @@ }: { imports = [./nix/modules/spindle.nix]; services.tangled-spindle.package = lib.mkDefault self.packages.${pkgs.system}.spindle; - }; - nixosConfigurations.vm-x86_64 = import ./nix/vm.nix { - inherit self nixpkgs; - system = "x86_64-linux"; - }; - nixosConfigurations.vm-aarch64 = import ./nix/vm.nix { - inherit self nixpkgs; - system = "aarch64-linux"; }; }; } diff --git a/nix/vm.nix b/nix/vm.nix --- a/nix/vm.nix +++ b/nix/vm.nix @@ -1,6 +1,7 @@ { nixpkgs, system, + hostSystem, self, }: let envVar = name: let @@ -16,18 +17,15 @@ modules = [ self.nixosModules.knot self.nixosModules.spindle ({ + lib, config, pkgs, ... }: { - nixos-shell = { - inheritPath = false; - mounts = { - mountHome = false; - mountNixProfile = false; - }; - }; - virtualisation = { + virtualisation.vmVariant.virtualisation = { + host.pkgs = import nixpkgs {system = hostSystem;}; + + graphics = false; memorySize = 2048; diskSize = 10 * 1024; cores = 2; @@ -51,21 +49,34 @@ host.port = 6555; guest.port = 6555; } ]; + sharedDirectories = { + # We can't use the 9p mounts directly for most of these + # as SQLite is incompatible with them. So instead we + # mount the shared directories to a different location + # and copy the contents around on service start/stop. + knotData = { + source = "$TANGLED_VM_DATA_DIR/knot"; + target = "/mnt/knot-data"; + }; + spindleData = { + source = "$TANGLED_VM_DATA_DIR/spindle"; + target = "/mnt/spindle-data"; + }; + spindleLogs = { + source = "$TANGLED_VM_DATA_DIR/spindle-logs"; + target = "/var/log/spindle"; + }; + }; }; + # This is fine because any and all ports that are forwarded to host are explicitly marked above, we don't need a separate guest firewall + networking.firewall.enable = false; services.getty.autologinUser = "root"; environment.systemPackages = with pkgs; [curl vim git sqlite litecli]; - systemd.tmpfiles.rules = let - u = config.services.tangled-knot.gitUser; - g = config.services.tangled-knot.gitUser; - in [ - "d /var/lib/knot 0770 ${u} ${g} - -" # Create the directory first - "f+ /var/lib/knot/secret 0660 ${u} ${g} - KNOT_SERVER_SECRET=${envVar "TANGLED_VM_KNOT_SECRET"}" - ]; services.tangled-knot = { enable = true; motd = "Welcome to the development knot!\n"; server = { - secretFile = "/var/lib/knot/secret"; + secretFile = builtins.toFile "knot-secret" ("KNOT_SERVER_SECRET=" + (envVar "TANGLED_VM_KNOT_SECRET")); hostname = "localhost:6000"; listenAddr = "0.0.0.0:6000"; }; @@ -81,6 +92,33 @@ secrets = { provider = "sqlite"; }; }; + }; + users = { + # So we don't have to deal with permission clashing between + # blank disk VMs and existing state + users.${config.services.tangled-knot.gitUser}.uid = 666; + groups.${config.services.tangled-knot.gitUser}.gid = 666; + + # TODO: separate spindle user + }; + systemd.services = let + mkDataSyncScripts = source: target: { + enableStrictShellChecks = true; + + preStart = lib.mkBefore '' + mkdir -p ${target} + ${lib.getExe pkgs.rsync} -a ${source}/ ${target} + ''; + + postStop = lib.mkAfter '' + ${lib.getExe pkgs.rsync} -a ${target}/ ${source} + ''; + + serviceConfig.PermissionsStartOnly = true; + }; + in { + knot = mkDataSyncScripts "/mnt/knot-data" config.services.tangled-knot.stateDir; + spindle = mkDataSyncScripts "/mnt/spindle-data" (builtins.dirOf config.services.tangled-spindle.server.dbPath); }; }) ];