diff --git a/infra/_modules/nixos/configuration.nix b/infra/_modules/nixos/configuration.nix index 3e90819..57cd531 100644 --- a/infra/_modules/nixos/configuration.nix +++ b/infra/_modules/nixos/configuration.nix @@ -23,6 +23,7 @@ systemd = { network = { enable = true; + wait-online.enable = false; }; }; diff --git a/infra/_modules/nixos/flake.nix b/infra/_modules/nixos/flake.nix index c2ee074..a50fea3 100644 --- a/infra/_modules/nixos/flake.nix +++ b/infra/_modules/nixos/flake.nix @@ -19,10 +19,14 @@ installer = nixpkgs.lib.nixosSystem { system = "x86_64-linux"; modules = [ + disko.nixosModules.disko + sops-nix.nixosModules.sops ./profiles/installer.nix ]; }; - kube-1 = nixpkgs.lib.nixosSystem { + + # Production + production-master-1 = nixpkgs.lib.nixosSystem { system = "x86_64-linux"; modules = [ disko.nixosModules.disko @@ -32,85 +36,87 @@ ./profiles/k3s-server.nix ./profiles/k3s-addons.nix { - networking.hostName = "kube-1"; + networking.hostName = "production-master-1"; systemd.network.networks."30-wan" = { matchConfig.Name = "ens18"; networkConfig.DHCP = "ipv4"; address = [ - hosts.kube-1.ipv6_address + hosts.production-master-1.ipv6_address ]; routes = [ { Gateway = "fe80::1"; } ]; }; services.k3s = { - clusterInit = true; + # TODO may need HA later + # clusterInit = true; + disableAgent = true; extraFlags = nixpkgs.lib.mkAfter [ - "--node-external-ip=${hosts.kube-1.ipv6_address}" + "--node-external-ip=${hosts.production-master-1.ipv6_address}" ]; }; } ]; }; - kube-2 = nixpkgs.lib.nixosSystem { + production-aGVsbG8K = nixpkgs.lib.nixosSystem { system = "x86_64-linux"; modules = [ disko.nixosModules.disko sops-nix.nixosModules.sops ./configuration.nix ./disks.nix - ./profiles/k3s-server.nix + ./profiles/k3s-agent.nix { - networking.hostName = "kube-2"; + networking.hostName = "production-aGVsbG8K"; systemd.network.networks."30-wan" = { - matchConfig.Name = "ens18"; + matchConfig.Name = "enp1s0"; networkConfig.DHCP = "ipv4"; address = [ - hosts.kube-2.ipv6_address + hosts.production-aGVsbG8K.ipv6_address ]; routes = [ { Gateway = "fe80::1"; } ]; }; services.k3s = { - serverAddr = "https://[${hosts.kube-1.ipv6_address}]:6443"; + serverAddr = "https://[${hosts.production-master-1.ipv6_address}]:6443"; extraFlags = nixpkgs.lib.mkAfter [ - "--node-external-ip=${hosts.kube-2.ipv6_address}" + "--node-external-ip=${hosts.production-aGVsbG8K.ipv6_address}" ]; }; } ]; }; - kube-3 = nixpkgs.lib.nixosSystem { + production-d29ybGQK = nixpkgs.lib.nixosSystem { system = "x86_64-linux"; modules = [ disko.nixosModules.disko sops-nix.nixosModules.sops ./configuration.nix ./disks.nix - ./profiles/k3s-server.nix + ./profiles/k3s-agent.nix { - networking.hostName = "kube-3"; + networking.hostName = "production-d29ybGQK"; systemd.network.networks."30-wan" = { - matchConfig.Name = "ens18"; + matchConfig.Name = "enp1s0"; networkConfig.DHCP = "ipv4"; address = [ - hosts.kube-3.ipv6_address + hosts.production-d29ybGQK.ipv6_address ]; routes = [ { Gateway = "fe80::1"; } ]; }; services.k3s = { - serverAddr = "https://[${hosts.kube-1.ipv6_address}]:6443"; + serverAddr = "https://[${hosts.production-master-1.ipv6_address}]:6443"; extraFlags = nixpkgs.lib.mkAfter [ - "--node-external-ip=${hosts.kube-3.ipv6_address}" + "--node-external-ip=${hosts.production-d29ybGQK.ipv6_address}" ]; }; } ]; }; - kube-4 = nixpkgs.lib.nixosSystem { + production-YnJ1aGgK = nixpkgs.lib.nixosSystem { system = "aarch64-linux"; modules = [ disko.nixosModules.disko @@ -119,21 +125,21 @@ ./disks.nix ./profiles/k3s-agent.nix { - networking.hostName = "kube-4"; + networking.hostName = "production-YnJ1aGgK"; systemd.network.networks."30-wan" = { matchConfig.Name = "enp1s0"; networkConfig.DHCP = "ipv4"; address = [ - hosts.kube-4.ipv6_address + hosts.production-YnJ1aGgK.ipv6_address ]; routes = [ { Gateway = "fe80::1"; } ]; }; services.k3s = { - serverAddr = "https://[${hosts.kube-1.ipv6_address}]:6443"; + serverAddr = "https://[${hosts.production-master-1.ipv6_address}]:6443"; extraFlags = nixpkgs.lib.mkAfter [ - "--node-external-ip=${hosts.kube-4.ipv6_address}" + "--node-external-ip=${hosts.production-YnJ1aGgK.ipv6_address}" ]; }; } diff --git a/infra/_modules/nixos/hosts.json b/infra/_modules/nixos/hosts.json index 254c5cc..c988eff 100644 --- a/infra/_modules/nixos/hosts.json +++ b/infra/_modules/nixos/hosts.json @@ -1 +1 @@ -{"kube-1":{"ipv6_address":"2402:800:63e2:5af5:be24:11ff:fe24:6daf"},"kube-2":{"ipv6_address":"2402:800:63e2:5af5:be24:11ff:fe34:24df"},"kube-3":{"ipv6_address":"2402:800:63e2:5af5:be24:11ff:fe50:9cba"},"kube-4":{"ipv6_address":"2a01:4f9:c012:3cad::1"}} \ No newline at end of file +{"production-YnJ1aGgK":{"ipv6_address":"2a01:4f9:c012:3cad::1"},"production-aGVsbG8K":{"ipv6_address":"2402:800:63e2:5af5:be24:11ff:fe0e:80d3"},"production-d29ybGQK":{"ipv6_address":"2402:800:63e2:5af5:be24:11ff:fe30:1671"},"production-master-1":{"ipv6_address":"2402:800:63e2:5af5:be24:11ff:fe3e:c73c"}} \ No newline at end of file diff --git a/infra/_modules/nixos/main.tf b/infra/_modules/nixos/main.tf index 1ba1353..6b9a6a5 100644 --- a/infra/_modules/nixos/main.tf +++ b/infra/_modules/nixos/main.tf @@ -17,6 +17,7 @@ module "nixos" { nixos_partitioner_attr = "${var.flake}#nixosConfigurations.${each.key}.config.system.build.diskoScript" target_host = each.value.ipv6_address instance_id = each.key + build_on_remote = true extra_files_script = "${path.module}/decrypt-age-keys.sh" extra_environment = { SOPS_FILE = var.sops_file @@ -32,7 +33,7 @@ data "external" "kubeconfig" { query = { user = "root" - host = var.hosts["kube-1"].ipv6_address # TODO better way to get this + host = var.kube_api_host } depends_on = [ diff --git a/infra/_modules/nixos/variables.tf b/infra/_modules/nixos/variables.tf index 935147b..1bea160 100644 --- a/infra/_modules/nixos/variables.tf +++ b/infra/_modules/nixos/variables.tf @@ -8,6 +8,10 @@ variable "hosts" { })) } +variable "kube_api_host" { + type = string +} + variable "sops_file" { type = string } diff --git a/infra/_modules/proxmox-vm/outputs.tf b/infra/_modules/proxmox-vm/outputs.tf index 87f2c6b..944bcfb 100644 --- a/infra/_modules/proxmox-vm/outputs.tf +++ b/infra/_modules/proxmox-vm/outputs.tf @@ -30,7 +30,7 @@ locals { "%x", ( ( - ( (floor(parseint(split(":", lower(node.network_device[0].mac_address))[0], 16) / 2)) % 2 ) == 0 + ((floor(parseint(split(":", lower(node.network_device[0].mac_address))[0], 16) / 2)) % 2) == 0 ) ? parseint(split(":", lower(node.network_device[0].mac_address))[0], 16) + 2 : parseint(split(":", lower(node.network_device[0].mac_address))[0], 16) - 2 @@ -66,13 +66,13 @@ output "hosts" { ip if endswith(lower(ip), local.eui64_suffix_by_node[node.id]) ]) > 0 ? [ - for ip in flatten(node.ipv6_addresses) : - ip if endswith(lower(ip), local.eui64_suffix_by_node[node.id]) - ][0] + for ip in flatten(node.ipv6_addresses) : + ip if endswith(lower(ip), local.eui64_suffix_by_node[node.id]) + ][0] : [ - for ip in flatten(node.ipv6_addresses) : - ip if ip != "::1" && !startswith(lower(ip), "fe80:") - ][0] + for ip in flatten(node.ipv6_addresses) : + ip if ip != "::1" && !startswith(lower(ip), "fe80:") + ][0] ) } } diff --git a/infra/production/hetzner/compute/terragrunt.hcl b/infra/production/hetzner/compute/terragrunt.hcl index 20489e8..5db9b24 100644 --- a/infra/production/hetzner/compute/terragrunt.hcl +++ b/infra/production/hetzner/compute/terragrunt.hcl @@ -9,11 +9,9 @@ terraform { inputs = { nodes = { - "kube-4" = { + # Masters are pets, workers are cattle, hence worker names are random + "production-YnJ1aGgK" = { location = "hel1" } - # "kube-5" = { - # location = "nbg1" - # } } } diff --git a/infra/production/nixos/terragrunt.hcl b/infra/production/nixos/terragrunt.hcl index 6ba7d02..2b5bef9 100644 --- a/infra/production/nixos/terragrunt.hcl +++ b/infra/production/nixos/terragrunt.hcl @@ -16,10 +16,11 @@ dependency "hetzner" { } inputs = { - flake = "${find_in_parent_folders("_modules")}//nixos" - hosts = merge( + flake = "${find_in_parent_folders("_modules")}//nixos" + hosts = merge( dependency.proxmox.outputs.hosts, dependency.hetzner.outputs.hosts, ) - sops_file = find_in_parent_folders("secrets.yaml") + kube_api_host = dependency.proxmox.outputs.hosts["production-master-1"].ipv6_address + sops_file = find_in_parent_folders("secrets.yaml") } diff --git a/infra/production/proxmox/compute/terragrunt.hcl b/infra/production/proxmox/compute/terragrunt.hcl index a7a51e0..47103ee 100644 --- a/infra/production/proxmox/compute/terragrunt.hcl +++ b/infra/production/proxmox/compute/terragrunt.hcl @@ -9,12 +9,13 @@ terraform { inputs = { hosts = { - "kube-1" = { cpu = 4, memory = 12, disk = 128 } - "kube-2" = { cpu = 4, memory = 12, disk = 128 } - "kube-3" = { cpu = 4, memory = 12, disk = 128 } + # Masters are pets, workers are cattle, hence worker names are random + "production-master-1" = { cpu = 4, memory = 12, disk = 128 } + "production-aGVsbG8K" = { cpu = 4, memory = 12, disk = 128 } + "production-d29ybGQK" = { cpu = 4, memory = 12, disk = 128 } } tags = [ - "production" + "kube-production" ] } diff --git a/infra/staging/nixos/terragrunt.hcl b/infra/staging/nixos/terragrunt.hcl index 14070a3..317b213 100644 --- a/infra/staging/nixos/terragrunt.hcl +++ b/infra/staging/nixos/terragrunt.hcl @@ -12,8 +12,8 @@ dependency "proxmox" { } inputs = { - flake = "${find_in_parent_folders("_modules")}//nixos" - hosts = merge( + flake = "${find_in_parent_folders("_modules")}//nixos" + hosts = merge( dependency.proxmox.outputs.hosts, ) sops_file = find_in_parent_folders("secrets.yaml") diff --git a/infra/staging/proxmox/compute/terragrunt.hcl b/infra/staging/proxmox/compute/terragrunt.hcl index 94b2f4d..953d047 100644 --- a/infra/staging/proxmox/compute/terragrunt.hcl +++ b/infra/staging/proxmox/compute/terragrunt.hcl @@ -13,6 +13,6 @@ inputs = { } tags = [ - "staging" + "kube-staging" ] }