From 29871c3554f035c5180e490730ba1db3afdebf91 Mon Sep 17 00:00:00 2001 From: Peter Rice Date: Thu, 9 Jul 2026 16:23:18 -0400 Subject: [PATCH] migrate incus containers to microvms --- modules/flake/containers.nix | 43 --------------------------------- modules/incus/containers.nix | 41 ------------------------------- modules/incus/guest.nix | 20 --------------- modules/incus/incus.nix | 27 --------------------- modules/nixos/tests.nix | 2 +- modules/nixos/yggdrasil/dns.nix | 35 +++++---------------------- modules/services/navidrome.nix | 28 ++++++++++++++------- modules/services/tangled.nix | 19 ++++++++++++--- 8 files changed, 41 insertions(+), 174 deletions(-) delete mode 100644 modules/flake/containers.nix delete mode 100644 modules/incus/containers.nix delete mode 100644 modules/incus/guest.nix delete mode 100644 modules/incus/incus.nix diff --git a/modules/flake/containers.nix b/modules/flake/containers.nix deleted file mode 100644 index 4e696f1..0000000 --- a/modules/flake/containers.nix +++ /dev/null @@ -1,43 +0,0 @@ -{ inputs, config, ... }: -let - inherit (inputs.nixpkgs) lib; - inherit (inputs.self.modules.nixos) container hostContainer; - cfg = config.local; - mkHostname = - containerName: - let - parts = lib.splitString "." containerName; - host = config.flake.nixosConfigurations.${lib.last parts}; - in - { - networking.hostName = lib.head parts; - networking.domain = host.config.networking.fqdn; - }; - mkHost = sharedModule: name: hostModule: { - imports = [ - hostModule - sharedModule - (mkHostname name) - ]; - }; - mkHosts = sharedModule: builtins.mapAttrs (mkHost sharedModule); - hostContainers = mkHosts hostContainer ( - lib.mapAttrs' (name: lib.nameValuePair "${name}.grancel") cfg.hosts - ); -in -{ - options.local = { - containers = lib.mkOption { - type = lib.types.attrsOf lib.types.deferredModule; - default = { }; - }; - resolvedContainers = lib.mkOption { - readOnly = true; - default = mkHosts container cfg.containers; - }; - }; - - config.flake.nixosConfigurations = builtins.mapAttrs ( - _: module: lib.nixosSystem { modules = [ module ]; } - ) (cfg.resolvedContainers // hostContainers); -} diff --git a/modules/incus/containers.nix b/modules/incus/containers.nix deleted file mode 100644 index 482d7d1..0000000 --- a/modules/incus/containers.nix +++ /dev/null @@ -1,41 +0,0 @@ -{ inputs, lib, ... }: -let - lxcKey = ./.; -in -{ - flake.modules.nixos.lxc = - { pkgs, modulesPath, ... }: - { - key = lxcKey; - - imports = [ "${modulesPath}/virtualisation/lxc-container.nix" ]; - - nixpkgs.hostPlatform = "x86_64-linux"; - - networking.useHostResolvConf = false; - networking.firewall.allowedTCPPorts = [ 22 ]; - }; - - flake.modules.nixos.test.disabledModules = [ lxcKey ]; - - flake.modules.nixos.container = { - imports = [ inputs.self.modules.nixos.lxc ]; - - system.stateVersion = "26.05"; - - security.sudo.wheelNeedsPassword = false; - boot.kernel.sysctl."net.ipv4.ip_unprivileged_port_start" = 80; - }; - - flake.modules.nixos.hostContainer = { - imports = [ inputs.self.modules.nixos.lxc ]; - - disabledModules = [ inputs.srvos.nixosModules.hardware-vultr-vm ]; - - hardware.facter.report = { }; - fileSystems = lib.mkForce { }; - local.persistence.enable = lib.mkForce false; - boot.loader.systemd-boot.enable = lib.mkForce false; - services.displayManager.ly.enable = lib.mkForce false; - }; -} diff --git a/modules/incus/guest.nix b/modules/incus/guest.nix deleted file mode 100644 index 84d9b63..0000000 --- a/modules/incus/guest.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ self, ... }: -let - host = self.nixosConfigurations.ruan.config.networking.fqdn; - guest = "griff.${host}"; -in -{ - local.servers.crossbell = - { pkgs, ... }: - { - local.caddy.reverseProxies."griffin.pvsr.dev" = guest; - systemd.services.forward-guest-ssh = { - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - ExecStart = "${pkgs.socat}/bin/socat TCP-LISTEN:12262,fork,reuseaddr TCP6:${guest}:22"; - Restart = "always"; - }; - }; - networking.firewall.allowedTCPPorts = [ 12262 ]; - }; -} diff --git a/modules/incus/incus.nix b/modules/incus/incus.nix deleted file mode 100644 index a07de27..0000000 --- a/modules/incus/incus.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ self, ... }: -{ - local.desktops.grancel.imports = [ self.modules.nixos.incus ]; - local.desktops.ruan.imports = [ self.modules.nixos.incus ]; - flake.modules.nixos.incus = - { config, pkgs, ... }: - { - virtualisation.incus.enable = true; - virtualisation.incus.package = pkgs.incus; - - # TODO declarative setup for incus should include: - # ipv6.address = "${config.local.prefix}/64"; - # dns.domain = config.networking.hostName; - # raw.dnsmasq = "domain=${config.networking.fqdn}\nlocal=/${config.networking.fqdn}/"; - - local.user.extraGroups = [ "incus-admin" ]; - - networking.firewall.interfaces.incusbr0.allowedTCPPorts = [ 53 ]; - networking.firewall.interfaces.incusbr0.allowedUDPPorts = [ - 53 - 67 - 547 - ]; - - environment.persistence.nixos.directories = [ "/var/lib/incus" ]; - }; -} diff --git a/modules/nixos/tests.nix b/modules/nixos/tests.nix index 3da6bbf..7878e1a 100644 --- a/modules/nixos/tests.nix +++ b/modules/nixos/tests.nix @@ -26,6 +26,6 @@ }; inherit (self.nixosConfigurations.${name}.config.local) testScript; } - ) (config.local.hosts // config.local.resolvedContainers); + ) config.local.hosts; }; } diff --git a/modules/nixos/yggdrasil/dns.nix b/modules/nixos/yggdrasil/dns.nix index 8d336f7..2c6e7a0 100644 --- a/modules/nixos/yggdrasil/dns.nix +++ b/modules/nixos/yggdrasil/dns.nix @@ -1,8 +1,10 @@ { self, lib, ... }: let domain = "ygg.pvsr.dev"; - hosts = lib.filterAttrs (_: host: !host.config.boot.isContainer) self.nixosConfigurations; - incusHosts = lib.filterAttrs (_: host: host.config.virtualisation.incus.enable) hosts; + hosts = lib.filterAttrs ( + _: host: !(host.config ? microvm && host.config.microvm ? guest) + ) self.nixosConfigurations; + microvmHosts = lib.filterAttrs (_: host: host.config.microvm.host.enable) hosts; in { flake.modules.nixos.core.networking = { @@ -29,7 +31,7 @@ in services.dnsmasq.resolveLocalQueries = false; services.dnsmasq.settings = { interface = "ygg0"; - bind-interfaces = true; + bind-dynamic = true; domain-needed = true; cache-size = 10000; local = [ "/${domain}/" ]; @@ -39,7 +41,7 @@ in ] ++ (lib.mapAttrsToList ( _: host: "/*.${host.config.networking.fqdn}/${host.config.local.prefix}::1" - ) incusHosts); + ) microvmHosts); }; }; @@ -54,29 +56,4 @@ in machine.succeed(f"dig @{address} ${hosts.grancel.config.networking.fqdn} AAAA") ''; }; - - flake.modules.nixos.incus = - { config, pkgs, ... }: - { - networking.firewall.interfaces.ygg0 = { - allowedTCPPorts = [ 53 ]; - allowedUDPPorts = [ 53 ]; - }; - - systemd.services.incus-dns-incusbr0 = { - description = "Configure DNS for incusbr0"; - wantedBy = [ "sys-subsystem-net-devices-incusbr0.device" ]; - after = [ "sys-subsystem-net-devices-incusbr0.device" ]; - bindsTo = [ "sys-subsystem-net-devices-incusbr0.device" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - }; - script = '' - ${pkgs.systemd}/bin/resolvectl dns incusbr0 ${config.local.prefix}::1 - ${pkgs.systemd}/bin/resolvectl domain incusbr0 '~${config.networking.fqdn}' - ''; - postStop = "${pkgs.systemd}/bin/resolvectl revert incusbr0"; - }; - }; } diff --git a/modules/services/navidrome.nix b/modules/services/navidrome.nix index 770162f..4f0fea4 100644 --- a/modules/services/navidrome.nix +++ b/modules/services/navidrome.nix @@ -1,9 +1,12 @@ -{ inputs, ... }: +{ self, ... }: +let + internalDomain = + self.nixosConfigurations.grancel.config.microvm.vms.navidrome.config.config.networking.fqdn; +in { - local.servers.crossbell.local.caddy.reverseProxies."music.pvsr.dev" = - "music.${inputs.self.nixosConfigurations.ruan.config.networking.fqdn}"; + local.servers.crossbell.local.caddy.reverseProxies."music.pvsr.dev" = internalDomain; - local.containers."music.ruan" = + local.desktops.grancel.vms.navidrome = { config, pkgs, ... }: { services.navidrome = { @@ -12,11 +15,18 @@ settings.Address = "[::]"; settings.MusicFolder = "/var/lib/navidrome/annex/music"; }; + + boot.kernel.sysctl."net.ipv4.ip_unprivileged_port_start" = 80; networking.firewall.allowedTCPPorts = [ 80 ]; - local.testScript = '' - machine.succeed("mkdir -p /var/lib/navidrome/annex/music") - machine.wait_for_unit("navidrome.service") - machine.wait_for_open_port(80) - ''; + + microvm.mem = 768; + microvm.shares = [ + { + source = "/home/peter/annex"; + mountPoint = "/var/lib/navidrome/annex"; + tag = "music"; + proto = "virtiofs"; + } + ]; }; } diff --git a/modules/services/tangled.nix b/modules/services/tangled.nix index c993d75..589b951 100644 --- a/modules/services/tangled.nix +++ b/modules/services/tangled.nix @@ -1,7 +1,7 @@ { inputs, ... }: let hosts = inputs.self.nixosConfigurations; - internal = "code.${hosts.ruan.config.networking.fqdn}"; + internal = hosts.grancel.config.microvm.vms.tangled-knot.config.config.networking.fqdn; external = "knot.pvsr.dev"; in { @@ -19,18 +19,29 @@ in }; }; - local.containers."code.ruan" = + local.desktops.grancel.vms.tangled-knot = { pkgs, ... }: { imports = [ inputs.tangled.nixosModules.knot ]; environment.systemPackages = [ - inputs.tangled.packages.${pkgs.hostPlatform.system}.knot + inputs.tangled.packages.${pkgs.stdenv.hostPlatform.system}.knot ]; + microvm.shares = [ + { + source = "/var/lib/microvms/tangled/git"; + mountPoint = "/home/git"; + tag = "tangled-git-user"; + proto = "virtiofs"; + } + ]; + + services.openssh.openFirewall = true; networking.firewall.allowedTCPPorts = [ 5555 ]; services.tangled.knot = { - enable = true; + # TODO needs internet in VM + enable = false; server = { owner = "did:plc:l7ruokyumokt2tduqqvu33j6"; hostname = external; -- 2.51.2