From b70b40e60321bb49af834873d1ab69acec2a960d Mon Sep 17 00:00:00 2001 From: File Magic Date: Thu, 16 Jul 2026 00:34:25 -0400 Subject: [PATCH] docs/superpowers/specs/: add configurable cluster topology design --- ...15-configurable-cluster-topology-design.md | 263 ++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-15-configurable-cluster-topology-design.md diff --git a/docs/superpowers/specs/2026-07-15-configurable-cluster-topology-design.md b/docs/superpowers/specs/2026-07-15-configurable-cluster-topology-design.md new file mode 100644 index 0000000..380ad78 --- /dev/null +++ b/docs/superpowers/specs/2026-07-15-configurable-cluster-topology-design.md @@ -0,0 +1,263 @@ +# Design: Configurable Cluster Topology + +## Overview + +Parameterize the number of master and worker nodes so each deployment can choose its cluster size. Combined with `cluster_index` from the subnet isolation spec, this enables lightweight dev clusters (1 master + 1 worker) alongside production-like test clusters (1 master + 5 workers). + +## Problem + +Currently hardcoded to 1 master + 3 workers in: +- `tofu/variables.tf` (static `k8s_nodes` map) +- `flake.nix` (static `nodeConfigs` map) +- `kubernetes/common.nix` (static `extraHosts`) +- `kubernetes/worker.nix` (default `nodeIp` for worker-0) +- `Makefile` (no topology control) + +## Solution + +Add `master_count` and `worker_count` variables. Derive all node definitions dynamically. + +| Variable | Default | Range | Description | +|----------|---------|-------|-------------| +| `master_count` | 1 | 1 | Number of master nodes (always 1 for now — etcd quorum requires odd count) | +| `worker_count` | 3 | 0-10 | Number of worker nodes | + +**Naming convention:** `k8s-master-0`, `k8s-worker-0`, `k8s-worker-1`, ... + +**IP assignment:** Master gets `.10`, workers get `.11` through `.10 + worker_count`. + +## Files Modified + +| File | Change | +|------|--------| +| `tofu/variables.tf` | Add `master_count`, `worker_count`; generate `k8s_nodes` dynamically | +| `tofu/main.tf` | Use generated `k8s_nodes` (no change to resource logic — it already iterates) | +| `tofu/outputs.tf` | Export dynamic node list | +| `flake.nix` | Generate `nodeConfigs` dynamically from count variables | +| `kubernetes/common.nix` | Generate `extraHosts` dynamically | +| `kubernetes/master.nix` | No change (master is always index 0) | +| `kubernetes/worker.nix` | No change (receives `nodeIp` via `extraConfig`) | +| `Makefile` | Expose `MASTER_COUNT`, `WORKER_COUNT` env vars | + +## Detailed Changes + +### 1. `tofu/variables.tf` + +Replace static `k8s_nodes` with count variables: +```hcl +variable "master_count" { + description = "Number of master nodes" + type = number + default = 1 + + validation { + condition = var.master_count >= 1 && var.master_count <= 1 + error_message = "master_count must be 1 (etcd quorum requires odd count)." + } +} + +variable "worker_count" { + description = "Number of worker nodes" + type = number + default = 3 + + validation { + condition = var.worker_count >= 0 && var.worker_count <= 10 + error_message = "worker_count must be between 0 and 10." + } +} +``` + +Generate `k8s_nodes` dynamically: +```hcl +locals { + # Generate node definitions from counts + master_nodes = { + for i in range(var.master_count) : + "k8s-master-${i}" => { + role = "master" + ip_suffix = 10 + i + mac = "52:54:00:00:00:${format("%02x", 10 + i)}" + vcpus = 2 + memory = 4096 + } + } + + worker_nodes = { + for i in range(var.worker_count) : + "k8s-worker-${i}" => { + role = "worker" + ip_suffix = 11 + i + mac = "52:54:00:00:00:${format("%02x", 11 + i)}" + vcpus = 2 + memory = 4096 + } + } + + k8s_nodes = merge(local.master_nodes, local.worker_nodes) +} +``` + +Remove the static `k8s_nodes` variable definition (the `variable "k8s_nodes"` block). + +### 2. `tofu/main.tf` + +Update `libvirt_volume` resources to use `local.k8s_nodes` instead of `var.k8s_nodes`: +```hcl +resource "libvirt_volume" "master" { + for_each = { for name, node in local.k8s_nodes : name => node if node.role == "master" } + # ... rest unchanged +} + +resource "libvirt_volume" "worker" { + for_each = { for name, node in local.k8s_nodes : name => node if node.role == "worker" } + # ... rest unchanged +} + +resource "libvirt_domain" "k8s" { + for_each = local.k8s_nodes + # ... rest unchanged +} +``` + +Update `local_file.ansible_inventory` to use `local.k8s_nodes`. + +### 3. `tofu/outputs.tf` + +```hcl +output "vm_ips" { + description = "IP addresses of all K8s nodes" + value = { + for name, node in local.k8s_nodes : + name => "192.168.${122 + var.cluster_index}.${node.ip_suffix}" + } +} + +output "master_ip" { + description = "IP address of the first master node" + value = "192.168.${122 + var.cluster_index}.10" +} + +output "worker_count" { + description = "Number of worker nodes" + value = var.worker_count +} + +output "node_names" { + description = "All node names" + value = keys(local.k8s_nodes) +} +``` + +### 4. `flake.nix` + +Generate `nodeConfigs` dynamically: +```nix +let + clusterIndex = let + envVal = builtins.getEnv "CLUSTER_INDEX"; + in if envVal == "" then 0 else builtins.fromJSON envVal; + + subnetThirdOctet = 122 + clusterIndex; + + workerCount = let + envVal = builtins.getEnv "WORKER_COUNT"; + in if envVal == "" then 3 else builtins.fromJSON envVal; + + masterIp = "192.168.${toString subnetThirdOctet}.10"; + + # Generate worker configs dynamically + workerConfigs = lib.genAttrs (lib.genList (i: "k8s-worker-${toString i}") workerCount) + (name: let + idx = lib.last (lib.splitString "-" name); + ipSuffix = 11 + (builtins.fromJSON idx); + in { + ip = "192.168.${toString subnetThirdOctet}.${toString ipSuffix}"; + modules = [ ./worker.nix ]; + extraConfig = { + services.kubernetes.kubelet.nodeIp = "192.168.${toString subnetThirdOctet}.${toString ipSuffix}"; + }; + }); + + nodeConfigs = { + k8s-master-0 = { + ip = masterIp; + modules = [ ./master.nix ]; + }; + } // workerConfigs; +in +``` + +### 5. `kubernetes/common.nix` + +Generate `extraHosts` dynamically. This requires passing the full node map from flake.nix: +```nix +{ + config, + pkgs, + lib, + sshPublicKey, + k8sApiToken, + subnetThirdOctet ? 122, + nodeHosts ? "", # generated string of "IP hostname" lines + ... +}: +{ + networking.extraHosts = nodeHosts; +} +``` + +In `flake.nix`, generate the hosts string: +```nix +nodeHosts = lib.concatStringsSep "\n" ( + lib.mapAttrsToList (name: node: "${node.ip} ${name}") computedNodes +); +``` + +Where `computedNodes` is the fully-resolved map with IPs. + +### 6. `Makefile` + +```makefile +CLUSTER_INDEX ?= 0 +MASTER_COUNT ?= 1 +WORKER_COUNT ?= 3 +export CLUSTER_INDEX MASTER_COUNT WORKER_COUNT + +deploy: + cd $(TOFU) && tofu destroy -auto-approve + cd $(TOFU) && tofu apply -auto-approve \ + -var "image_dir=../result" \ + -var "cluster_index=$(CLUSTER_INDEX)" \ + -var "master_count=$(MASTER_COUNT)" \ + -var "worker_count=$(WORKER_COUNT)" +``` + +## Usage + +```bash +# Default: 1 master + 3 workers +make up + +# Lightweight dev cluster +CLUSTER_INDEX=1 WORKER_COUNT=1 make up + +# Large test cluster +CLUSTER_INDEX=2 WORKER_COUNT=5 make up + +# Master-only (no workers, for etcd testing) +WORKER_COUNT=0 make up +``` + +## Backward Compatibility + +- Default `worker_count=3` preserves current behavior +- `master_count` locked to 1 (etcd quorum constraint) +- Existing `tofu destroy` required before applying (variable structure changed) + +## Constraints + +- `master_count` must be 1 (etcd requires odd-count quorum; multi-master is a future enhancement) +- `worker_count` 0-10 (practical limit — more nodes need bigger subnet) +- Max total nodes: 11 (1 master + 10 workers) +- Each node needs ~4GB RAM, so 11 nodes = ~44GB host memory minimum -- 2.51.2