diff --git a/infra/_modules/hetzner-vm/outputs.tf b/infra/_modules/hetzner-vm/outputs.tf index 3ed81d6..49b7081 100644 --- a/infra/_modules/hetzner-vm/outputs.tf +++ b/infra/_modules/hetzner-vm/outputs.tf @@ -1,3 +1,7 @@ -output "ipv6_addresses" { - value = { for node in hcloud_server.nodes : node.name => node.ipv6_address } +output "hosts" { + value = { + for node in hcloud_server.nodes : node.name => { + ipv6_address = node.ipv6_address + } + } } diff --git a/infra/_modules/nixos/configuration.nix b/infra/_modules/nixos/configuration.nix index 35bb552..3e90819 100644 --- a/infra/_modules/nixos/configuration.nix +++ b/infra/_modules/nixos/configuration.nix @@ -17,7 +17,11 @@ }; networking = { - networkmanager = { + tempAddresses = "disabled"; + }; + + systemd = { + network = { enable = true; }; }; diff --git a/infra/_modules/nixos/flake.nix b/infra/_modules/nixos/flake.nix index 1d28f91..56c4a30 100644 --- a/infra/_modules/nixos/flake.nix +++ b/infra/_modules/nixos/flake.nix @@ -32,7 +32,22 @@ ./profiles/k3s.nix { networking.hostName = "kube-1"; - services.k3s.clusterInit = true; + systemd.network.networks."30-wan" = { + matchConfig.Name = "ens18"; + networkConfig.DHCP = "ipv4"; + address = [ + hosts.kube-1.ipv6_address + ]; + routes = [ + { Gateway = "fe80::1"; } + ]; + }; + services.k3s = { + clusterInit = true; + extraFlags = nixpkgs.lib.mkAfter [ + "--node-external-ip=${hosts.kube-1.ipv6_address}" + ]; + }; } ]; }; @@ -46,7 +61,22 @@ ./profiles/k3s.nix { networking.hostName = "kube-2"; - services.k3s.serverAddr = "https://[${hosts.kube-1.ipv6_address}]:6443"; + systemd.network.networks."30-wan" = { + matchConfig.Name = "ens18"; + networkConfig.DHCP = "ipv4"; + address = [ + hosts.kube-2.ipv6_address + ]; + routes = [ + { Gateway = "fe80::1"; } + ]; + }; + services.k3s = { + serverAddr = "https://[${hosts.kube-1.ipv6_address}]:6443"; + extraFlags = nixpkgs.lib.mkAfter [ + "--node-external-ip=${hosts.kube-2.ipv6_address}" + ]; + }; } ]; }; @@ -60,7 +90,51 @@ ./profiles/k3s.nix { networking.hostName = "kube-3"; - services.k3s.serverAddr = "https://[${hosts.kube-1.ipv6_address}]:6443"; + systemd.network.networks."30-wan" = { + matchConfig.Name = "ens18"; + networkConfig.DHCP = "ipv4"; + address = [ + hosts.kube-3.ipv6_address + ]; + routes = [ + { Gateway = "fe80::1"; } + ]; + }; + services.k3s = { + serverAddr = "https://[${hosts.kube-1.ipv6_address}]:6443"; + extraFlags = nixpkgs.lib.mkAfter [ + "--node-external-ip=${hosts.kube-3.ipv6_address}" + ]; + }; + } + ]; + }; + kube-4 = nixpkgs.lib.nixosSystem { + system = "aarch64-linux"; + modules = [ + disko.nixosModules.disko + sops-nix.nixosModules.sops + ./configuration.nix + ./disks.nix + ./profiles/k3s-agent.nix + { + networking.hostName = "kube-4"; + systemd.network.networks."30-wan" = { + matchConfig.Name = "enp1s0"; + networkConfig.DHCP = "ipv4"; + address = [ + hosts.kube-4.ipv6_address + ]; + routes = [ + { Gateway = "fe80::1"; } + ]; + }; + services.k3s = { + serverAddr = "https://[${hosts.kube-1.ipv6_address}]:6443"; + extraFlags = nixpkgs.lib.mkAfter [ + "--node-external-ip=${hosts.kube-4.ipv6_address}" + ]; + }; } ]; }; diff --git a/infra/_modules/nixos/hosts.json b/infra/_modules/nixos/hosts.json index 613ce06..254c5cc 100644 --- a/infra/_modules/nixos/hosts.json +++ b/infra/_modules/nixos/hosts.json @@ -1 +1 @@ -{"kube-1":{"ipv6_address":"2402:800:63e2:5af5:1b0:f4a:e854:5355"},"kube-2":{"ipv6_address":"2402:800:63e2:5af5:cb46:f8e4:131b:93c5"},"kube-3":{"ipv6_address":"2402:800:63e2:5af5:1ced:4595:b4bf:3634"}} \ No newline at end of file +{"kube-1":{"ipv6_address":"2402:800:63e2:5af5:be24:11ff:fe24:6daf"},"kube-2":{"ipv6_address":"2402:800:63e2:5af5:be24:11ff:fe34:24df"},"kube-3":{"ipv6_address":"2402:800:63e2:5af5:be24:11ff:fe50:9cba"},"kube-4":{"ipv6_address":"2a01:4f9:c012:3cad::1"}} \ No newline at end of file diff --git a/infra/_modules/nixos/main.tf b/infra/_modules/nixos/main.tf index d23e38e..4a216f7 100644 --- a/infra/_modules/nixos/main.tf +++ b/infra/_modules/nixos/main.tf @@ -21,4 +21,8 @@ module "nixos" { extra_environment = { SOPS_FILE = var.sops_file } + + depends_on = [ + local_file.hosts + ] } diff --git a/infra/_modules/nixos/profiles/installer.nix b/infra/_modules/nixos/profiles/installer.nix index d460a32..a4adb46 100644 --- a/infra/_modules/nixos/profiles/installer.nix +++ b/infra/_modules/nixos/profiles/installer.nix @@ -5,6 +5,10 @@ (modulesPath + "/installer/cd-dvd/installation-cd-minimal.nix") ]; + networking = { + tempAddresses = "disabled"; + }; + services = { openssh = { enable = true; diff --git a/infra/_modules/nixos/profiles/k3s-agent.nix b/infra/_modules/nixos/profiles/k3s-agent.nix new file mode 100644 index 0000000..88d64ac --- /dev/null +++ b/infra/_modules/nixos/profiles/k3s-agent.nix @@ -0,0 +1,24 @@ +{ config, ... }: + +{ + networking = { + firewall = { + # https://docs.k3s.io/installation/requirements#inbound-rules-for-k3s-nodes + allowedTCPPorts = [ + 10250 # Kubelet metrics + ]; + allowedUDPPorts = [ + 51820 # Flannel Wireguard with IPv4 + 51821 # Flannel Wireguard with IPv6 + ]; + }; + }; + + services = { + k3s = { + enable = true; + role = "agent"; + tokenFile = config.sops.secrets.k3s_token.path; + }; + }; +} diff --git a/infra/_modules/nixos/profiles/k3s.nix b/infra/_modules/nixos/profiles/k3s.nix index f33e935..5eab3a1 100644 --- a/infra/_modules/nixos/profiles/k3s.nix +++ b/infra/_modules/nixos/profiles/k3s.nix @@ -3,14 +3,19 @@ { networking = { firewall = { - # https://docs.k3s.io/installation/requirements#inbound-rules-for-k3s-server-nodes + # https://docs.k3s.io/installation/requirements#inbound-rules-for-k3s-nodes allowedTCPPorts = [ - 6443 - 10250 + 6443 # K3s supervisor and Kubernetes API Server + 10250 # Kubelet metrics ]; allowedTCPPortRanges = [ + # Required only for HA with embedded etcd { from = 2379; to = 2380; } ]; + allowedUDPPorts = [ + 51820 # Flannel Wireguard with IPv4 + 51821 # Flannel Wireguard with IPv6 + ]; }; }; @@ -19,10 +24,16 @@ enable = true; role = "server"; tokenFile = config.sops.secrets.k3s_token.path; - extraFlags = toString [ + extraFlags = [ "--disable-helm-controller" "--disable-network-policy" "--disable=traefik" + "--cluster-cidr=fd6a:7c7b:3e12:0::/56" # TODO proper ULA planning + "--service-cidr=fd6a:7c7b:3e12:100::/112" # TODO proper ULA planning + "--flannel-backend=wireguard-native" + "--flannel-external-ip" + "--flannel-ipv6-masq" # Enable IPv6 NAT, as per default pods use their pod IPv6 address for outgoing traffic + # TODO net.ipv6.conf.all.accept_ra=2 ]; }; }; diff --git a/infra/_modules/proxmox-vm/outputs.tf b/infra/_modules/proxmox-vm/outputs.tf index ec59036..87f2c6b 100644 --- a/infra/_modules/proxmox-vm/outputs.tf +++ b/infra/_modules/proxmox-vm/outputs.tf @@ -1,7 +1,79 @@ +# TODO wtf +# Build MAC-derived EUI-64 suffix per VM, no bitwise ops used +locals { + eui64_suffix_by_node = { + for node in proxmox_virtual_environment_vm.main : + node.id => ( + # Parse MAC -> 6 bytes + # Example: BC:24:11:EB:01:83 + # Flip the U/L bit (bit 1) of the first byte: if it’s 0 add 2, else subtract 2 + # Then insert ff:fe and format into 4x16-bit hex groups. + # Result suffix for example above: be24:11ff:feeb:183 + ( + # lowercase, split, parse + # keep these short to avoid paren hell + # b = bytes[0..5], b0f = flipped first byte + # words w1..w4 form the IID + # join as hex groups + # (all Terraform/HCL functions; no bitwise) + # -- + # Precompute the bytes list + # NOTE: locals inside expressions are not supported, so repeat split/parse where needed + # + # first byte (original) + # b0 = parseint(split(":", lower(node.network_device[0].mac_address))[0], 16) + # bit1 = floor(b0/2) % 2 -> 0 or 1 + # b0f = bit1 == 0 ? b0 + 2 : b0 - 2 + # + join(":", [ + format( + "%x", + ( + ( + ( (floor(parseint(split(":", lower(node.network_device[0].mac_address))[0], 16) / 2)) % 2 ) == 0 + ) + ? parseint(split(":", lower(node.network_device[0].mac_address))[0], 16) + 2 + : parseint(split(":", lower(node.network_device[0].mac_address))[0], 16) - 2 + ) * 256 + + parseint(split(":", lower(node.network_device[0].mac_address))[1], 16) + ), + format( + "%x", + parseint(split(":", lower(node.network_device[0].mac_address))[2], 16) * 256 + 255 + ), + format( + "%x", + 254 * 256 + parseint(split(":", lower(node.network_device[0].mac_address))[3], 16) + ), + format( + "%x", + parseint(split(":", lower(node.network_device[0].mac_address))[4], 16) * 256 + + parseint(split(":", lower(node.network_device[0].mac_address))[5], 16) + ) + ]) + ) + ) + } +} + +# Pick the IPv6 whose IID matches the MAC-derived EUI-64; fallback to first global (non-loopback, non-link-local) output "hosts" { value = { for node in proxmox_virtual_environment_vm.main : node.name => { - ipv6_address = node.ipv6_addresses[1][0] + ipv6_address = ( + length([ + for ip in flatten(node.ipv6_addresses) : + ip if endswith(lower(ip), local.eui64_suffix_by_node[node.id]) + ]) > 0 + ? [ + for ip in flatten(node.ipv6_addresses) : + ip if endswith(lower(ip), local.eui64_suffix_by_node[node.id]) + ][0] + : [ + for ip in flatten(node.ipv6_addresses) : + ip if ip != "::1" && !startswith(lower(ip), "fe80:") + ][0] + ) } } } diff --git a/infra/production/hetzner/compute/terragrunt.hcl b/infra/production/hetzner/compute/terragrunt.hcl index d0bd38d..20489e8 100644 --- a/infra/production/hetzner/compute/terragrunt.hcl +++ b/infra/production/hetzner/compute/terragrunt.hcl @@ -1,7 +1,3 @@ -# TODO temporarily disable Hetzner until I fix the IPv6 issue -# https://wiki.nixos.org/wiki/Install_NixOS_on_Hetzner_Cloud -skip = true - include "root" { path = find_in_parent_folders("root.hcl") expose = true @@ -13,14 +9,11 @@ terraform { inputs = { nodes = { - "master-1" = { + "kube-4" = { location = "hel1" } - # "worker-1" = { + # "kube-5" = { # location = "nbg1" # } - # "worker-2" = { - # location = "fsn1" - # } } } diff --git a/infra/production/nixos/terragrunt.hcl b/infra/production/nixos/terragrunt.hcl index 7257f3a..6ba7d02 100644 --- a/infra/production/nixos/terragrunt.hcl +++ b/infra/production/nixos/terragrunt.hcl @@ -11,8 +11,15 @@ dependency "proxmox" { config_path = "../proxmox/compute" } +dependency "hetzner" { + config_path = "../hetzner/compute" +} + inputs = { flake = "${find_in_parent_folders("_modules")}//nixos" - hosts = dependency.proxmox.outputs.hosts + hosts = merge( + dependency.proxmox.outputs.hosts, + dependency.hetzner.outputs.hosts, + ) sops_file = find_in_parent_folders("secrets.yaml") }