From 6bc0cf020e0913ab91e122270c5ffbf2e9b2e440 Mon Sep 17 00:00:00 2001 From: File Magic Date: Wed, 19 Aug 2026 19:45:25 -0400 Subject: [PATCH] offline-images.nix [kubernetes]: derive offline images and Helm discovery --- kubernetes/README.md | 49 ++++++-- .../ansible/playbooks/import-images.yml | 2 +- kubernetes/common.nix | 5 + kubernetes/docs/resume-state.md | 4 +- kubernetes/flake.nix | 4 +- kubernetes/justfile | 16 ++- kubernetes/offline-images.list | 26 ---- kubernetes/offline-images.nix | 116 ++++++++++++++++++ kubernetes/scripts/list-helm-images.sh | 26 ++++ kubernetes/scripts/pull-offline-images.sh | 48 -------- 10 files changed, 201 insertions(+), 95 deletions(-) delete mode 100644 kubernetes/offline-images.list create mode 100644 kubernetes/offline-images.nix create mode 100755 kubernetes/scripts/list-helm-images.sh delete mode 100755 kubernetes/scripts/pull-offline-images.sh diff --git a/kubernetes/README.md b/kubernetes/README.md index 339ea0b..30b8b1d 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -9,7 +9,7 @@ Cilium CNI + Rook-Ceph storage. - `libvirtd` running (`systemctl status libvirtd`) - OpenTofu 1.12+ - [just](https://github.com/casey/just) -- Podman with internet access on the control machine (used to pull offline images) +- Network access while building the Nix offline-image derivation ## Quick Start @@ -35,29 +35,52 @@ See `docs/resume-state.md` for all justfile targets and options. ## Offline Images The nodes have no external registry access. `just up` runs `just offline-images` after -SSH is available. That target runs `scripts/pull-offline-images.sh`, then runs the -Ansible import playbook. +SSH is available. That target builds the fixed-output `.#offline-images` derivation, +then runs the Ansible import playbook against its Nix store output. To build and import the bundle manually, after `just deploy` and `just wait-ssh`: ```bash -scripts/pull-offline-images.sh +bundle=$(nix build --no-link --print-out-paths .#offline-images) cd ansible -ansible-playbook playbooks/import-images.yml +ansible-playbook playbooks/import-images.yml \ + -e offline_image_bundle="$bundle" ``` -The script reads the pinned image references from `offline-images.list`, pulls them -with Podman on the control machine, and writes one archive per image under -`.cache/offline-images/`. The list hash is cached, so rerunning the script is quick -when the list is unchanged. To force a refresh after changing tags or digests: +The image definitions are in `offline-images.nix`. Each image is pinned by registry +manifest digest and fixed archive hash, and the derivation produces one Docker archive +per image. The first build needs registry access; later builds use the Nix store or a +binary cache. The build runs through Nix's unprivileged `skopeo` fetcher and does not +need a Podman daemon or `/etc/containers/policy.json`. + +The Kubernetes pod sandbox image is pinned to `registry.k8s.io/pause:3.10` in +`common.nix` and is included in the same bundle. CoreDNS is built and seeded locally +by the NixOS Kubernetes module; kube-proxy is disabled because Cilium replaces it. + +The generated archives can require several gigabytes of local storage. They live in +the Nix store and are generated input for the Ansible import step. + +The bundle can be built before any cluster exists: + +```bash +just offline-images-build +``` + +This only needs registry access during the first build. Run `just offline-images` +after the VMs are available to copy the already-built archives to the nodes. + +## Discover Helm Images + +List the images a chart will render before deploying a cluster: ```bash -rm -rf .cache/offline-images .cache/offline-images.list.sha256 -scripts/pull-offline-images.sh +just helm-images ./path/to/chart --values ./path/to/values.yaml ``` -The generated archives can require several gigabytes of local storage. Do not commit -`.cache/`; it is generated input for the Ansible import step. +Additional Helm template flags can be passed after the chart, for example +`--set image.tag=...` or `--kube-version 1.32.0`. The command renders the chart +locally and extracts images from regular, init, and ephemeral containers. The output +can then be pinned and added to `offline-images.nix`. ## SSH into Master diff --git a/kubernetes/ansible/playbooks/import-images.yml b/kubernetes/ansible/playbooks/import-images.yml index d3529ce..bbd1d34 100644 --- a/kubernetes/ansible/playbooks/import-images.yml +++ b/kubernetes/ansible/playbooks/import-images.yml @@ -10,7 +10,7 @@ - name: Copy offline image archives to node ansible.builtin.copy: - src: "../../.cache/offline-images/" + src: "{{ offline_image_bundle }}/offline-images/" dest: /tmp/offline-images/ mode: "0755" diff --git a/kubernetes/common.nix b/kubernetes/common.nix index 951569a..6d633d1 100644 --- a/kubernetes/common.nix +++ b/kubernetes/common.nix @@ -102,6 +102,11 @@ in { # Container runtime virtualisation.containerd.enable = true; + # Pin the pod sandbox image so offline nodes never try to pull the mutable + # containerd default (docker.io/library/pause:latest). + virtualisation.containerd.settings = { + plugins."io.containerd.grpc.v1.cri".sandbox_image = "registry.k8s.io/pause:3.10"; + }; fileSystems."/var/lib/containerd" = { device = containerdDevice; diff --git a/kubernetes/docs/resume-state.md b/kubernetes/docs/resume-state.md index 0aa34c7..bb65820 100644 --- a/kubernetes/docs/resume-state.md +++ b/kubernetes/docs/resume-state.md @@ -184,7 +184,9 @@ just fmt # Auto-fix formatting issues - **`just build`** — Runs `nix build .#images` to create QCOW2 images - **`just genkey`** — Generates the SSH keypair if missing (no-op if `ssh-key` exists) - **`just deploy`** — Runs `tofu destroy` then `tofu apply` with the built images -- **`just offline-images`** — Runs the offline image pull script and imports its per-image archives on every node; see `README.md#offline-images` +- **`just offline-images-build`** — Builds the fixed-output Nix image bundle without requiring a cluster; see `README.md#offline-images` +- **`just offline-images`** — Builds the fixed-output Nix image bundle and imports its per-image archives on every node; see `README.md#offline-images` +- **`just helm-images`** — Renders a chart locally and lists its container images; see `README.md#discover-helm-images` - **`just ansible`** — Runs the full Ansible playbook which: 1. Waits for SSH on all nodes 2. Waits for the Kubernetes apiserver readiness (covers cfssl PKI generation race) diff --git a/kubernetes/flake.nix b/kubernetes/flake.nix index bfc877e..564b2b0 100644 --- a/kubernetes/flake.nix +++ b/kubernetes/flake.nix @@ -143,7 +143,7 @@ pkgs.ansible-lint pkgs.cfssl pkgs.cilium-cli - pkgs.helm + pkgs.kubernetes-helm pkgs.just pkgs.kubectl pkgs.OVMF.fd @@ -152,6 +152,7 @@ pkgs.python3Packages.pyyaml pkgs.tflint pkgs.yamllint + pkgs.yq-go ]; }; @@ -170,6 +171,7 @@ configs // { ovmf = pkgs.OVMF.fd; + offline-images = import ./offline-images.nix {inherit lib pkgs;}; images = pkgs.runCommand "k8s-images" {} '' mkdir -p $out ${lib.concatStringsSep "\n" ( diff --git a/kubernetes/justfile b/kubernetes/justfile index 3ceddea..3d3db77 100755 --- a/kubernetes/justfile +++ b/kubernetes/justfile @@ -6,6 +6,7 @@ export WORKER_COUNT := env_var_or_default("WORKER_COUNT", "3") # OVMF must be built (not just evaluated) so FV/OVMF_CODE.fd actually exists OVMF_CODE_PATH := `nix build --no-link --print-out-paths .#ovmf` +OFFLINE_IMAGES := `nix build --no-link --print-out-paths .#offline-images` KUBE_DIR := justfile_directory() TOFU := KUBE_DIR / "tofu" ANSIBLE := KUBE_DIR / "ansible" @@ -69,13 +70,18 @@ genkey: # Full lifecycle: genkey → build → deploy → wait → offline images → install → verify up: genkey build deploy wait-ssh offline-images ansible +# Build the offline image bundle without requiring a cluster +offline-images-build: + nix build .#offline-images + # Build the offline image bundle and import it on every IPv6 node -offline-images: - #!/usr/bin/env bash - set -euo pipefail - {{KUBE_DIR}}/scripts/pull-offline-images.sh +offline-images: offline-images-build cd {{ANSIBLE}} - ansible-playbook playbooks/import-images.yml + ansible-playbook playbooks/import-images.yml -e offline_image_bundle={{OFFLINE_IMAGES}} + +# Render a Helm chart and list its container images without a cluster +helm-images *ARGS: + PATH="{{DEV_TOOLS}}/bin:$PATH" {{KUBE_DIR}}/scripts/list-helm-images.sh {{ARGS}} # Run all linters lint: diff --git a/kubernetes/offline-images.list b/kubernetes/offline-images.list deleted file mode 100644 index 0d02be2..0000000 --- a/kubernetes/offline-images.list +++ /dev/null @@ -1,26 +0,0 @@ -# Container images pre-loaded into node containerd at deploy time. -# Nodes have no internet (IPv6-only cluster), so every runtime image must be -# pulled on the control machine and imported via `ctr images import`. -# Keep in sync with the pinned Cilium chart (cilium_version) and Rook -# manifests (rook_version) in ansible/group_vars/all.yml. - -# Cilium 1.20.0 (digest-pinned from installed chart) -quay.io/cilium/cilium:v1.20.0@sha256:383968cd5e8873f7976fa76aa6196045643558f4cc9518a207b9335cb24a0e93 -quay.io/cilium/cilium-envoy:v1.37.5-1782911245-7cffc778c923f68a77954a53b1a98d6b5353f004@sha256:583057dd4f7d54cd41efff3c413aa0b148ac201f522e2c3336851fa89c78b039 -quay.io/cilium/operator-generic:v1.20.0@sha256:80744a8cc7c91c2f9e6347629406844eb35d79b30a732c6d41c15b17232a74f3 - -# Rook v1.20.2 operator + ceph-csi-operator -docker.io/rook/ceph:v1.20.2 -quay.io/cephcsi/ceph-csi-operator:v1.0.4 - -# Ceph v19.2.0 (Squid) daemon image -quay.io/ceph/ceph:v19.2.0 - -# ceph-csi-operator image set (from rook csi-operator.yaml) -quay.io/cephcsi/cephcsi:v3.17.0 -quay.io/csiaddons/k8s-sidecar:v0.14.0 -registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.17.0 -registry.k8s.io/sig-storage/csi-provisioner:v6.2.0 -registry.k8s.io/sig-storage/csi-attacher:v4.12.0 -registry.k8s.io/sig-storage/csi-resizer:v2.1.0 -registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0 diff --git a/kubernetes/offline-images.nix b/kubernetes/offline-images.nix new file mode 100644 index 0000000..2ea731c --- /dev/null +++ b/kubernetes/offline-images.nix @@ -0,0 +1,116 @@ +{ + lib, + pkgs, +}: let + containerImageDefinitions = [ + { + imageName = "registry.k8s.io/pause"; + imageDigest = "sha256:ee6521f290b2168b6e0935a181d4cff9be1ac3f505666ef0e3c98fae8199917a"; + outputHash = "sha256-yy9gxg8AVujnaRvzZDa5bvKoVks8OyLTGwBjidT0zmc="; + finalImageTag = "3.10"; + } + { + imageName = "quay.io/cilium/cilium"; + imageDigest = "sha256:383968cd5e8873f7976fa76aa6196045643558f4cc9518a207b9335cb24a0e93"; + outputHash = "sha256-09CBSw1Njy74nd8qOaYIIxpfMdlpW2QcMhslxUou6Wk="; + finalImageTag = "v1.20.0"; + } + { + imageName = "quay.io/cilium/cilium-envoy"; + imageDigest = "sha256:583057dd4f7d54cd41efff3c413aa0b148ac201f522e2c3336851fa89c78b039"; + outputHash = "sha256-XXSykMxRfsi4H4TuzyBbdFQF4VK+b/dcoNOjAVKqnjs="; + finalImageTag = "v1.37.5-1782911245-7cffc778c923f68a77954a53b1a98d6b5353f004"; + } + { + imageName = "quay.io/cilium/operator-generic"; + imageDigest = "sha256:80744a8cc7c91c2f9e6347629406844eb35d79b30a732c6d41c15b17232a74f3"; + outputHash = "sha256-lNPPNCrg/ZKIJtCf3r9YqKM6COff7WkuQX8NCg/7JPQ="; + finalImageTag = "v1.20.0"; + } + { + imageName = "docker.io/rook/ceph"; + imageDigest = "sha256:a642e07d4d0b4d02214e435337c7f5d3f5154369270ea5180de9d5d5d9d965c2"; + outputHash = "sha256-7+Qy9zfEMOxmgVGorzMgxp8MiWQWfUnLC1nQsPDqNKQ="; + finalImageTag = "v1.20.2"; + } + { + imageName = "quay.io/cephcsi/ceph-csi-operator"; + imageDigest = "sha256:c62933fd4083635f969f8a61932af45dba9902d48867e2b5d98a69d8e4344eb6"; + outputHash = "sha256-JMDdDKGYNITxPKPX4kFTYuWC3BXDQMy+0nG3dYuWHwU="; + finalImageTag = "v1.0.4"; + } + { + imageName = "quay.io/ceph/ceph"; + imageDigest = "sha256:200087c35811bf28e8a8073b15fa86c07cce85c575f1ccd62d1d6ddbfdc6770a"; + outputHash = "sha256-LmId8STKafnYYKJjbtZ1af0RHX8CPNgFIWQRr4pclcU="; + finalImageTag = "v19.2.0"; + } + { + imageName = "quay.io/cephcsi/cephcsi"; + imageDigest = "sha256:886e6d2416d62dd7c8fbe659b6306b6c9451d6918e35ad5d1ac774520e11ef87"; + outputHash = "sha256-v9dM94N4N2IJ7d5us5cqSAmg5u1oRJ67XRp8sWlHM/U="; + finalImageTag = "v3.17.0"; + } + { + imageName = "quay.io/csiaddons/k8s-sidecar"; + imageDigest = "sha256:69092d026323307933fba2bed9c57d864f2a92cfd385d644b478442ded0d2886"; + outputHash = "sha256-e4EHleDHSq5QJaCWYRLBCv+5MUXL02nPMmeYzaL1AuM="; + finalImageTag = "v0.14.0"; + } + { + imageName = "registry.k8s.io/sig-storage/csi-node-driver-registrar"; + imageDigest = "sha256:f9de845b170155199f2a2a3f9531cf13d78e31235e9db6b6582a8b0db0a50dad"; + outputHash = "sha256-HR8+yCYNZqAlG53Olg6O0KU0B9QnTtoZYIP6dNPugNo="; + finalImageTag = "v2.17.0"; + } + { + imageName = "registry.k8s.io/sig-storage/csi-provisioner"; + imageDigest = "sha256:6be9f63ca4caa6c46aae55aa372500949d8a21473d72f819da1f746076b32d4e"; + outputHash = "sha256-9QbESOtAOUtY+eo2GyRo3TOcU4dZPS7F/CCu1CcaWJY="; + finalImageTag = "v6.2.0"; + } + { + imageName = "registry.k8s.io/sig-storage/csi-attacher"; + imageDigest = "sha256:b9dc9a714a484ccdeeb6f86d88d4db9b7a5ecfc5a55da6db3a60bb3fa33c278a"; + outputHash = "sha256-OxWMH/1jqP8M5y4QuyOGcswFkmrVLp+GZX6GPDtSKJE="; + finalImageTag = "v4.12.0"; + } + { + imageName = "registry.k8s.io/sig-storage/csi-resizer"; + imageDigest = "sha256:589e525cddef6d768e68da1f0bc9ffd0a24bf3add3dd010648eb7189976fde79"; + outputHash = "sha256-wTVyZD6/rSihWJDR71qLcgl7Rh3yE/uykCJEahxpDmQ="; + finalImageTag = "v2.1.0"; + } + { + imageName = "registry.k8s.io/sig-storage/csi-snapshotter"; + imageDigest = "sha256:da081c27e8a6d91f36042c1942362d0515ced8d06e18c11b8f893e58c4d6d797"; + outputHash = "sha256-r5xBrwkt02j4tJev15Tu+KkPUDwSqJNw3AqvncB9xn4="; + finalImageTag = "v8.5.0"; + } + ]; + + containers = + map ( + definition: + definition + // { + archive = pkgs.dockerTools.pullImage { + inherit (definition) imageName imageDigest outputHash finalImageTag; + outputHashAlgo = "sha256"; + }; + } + ) + containerImageDefinitions; +in + pkgs.runCommand "kubernetes-offline-images" {} '' + mkdir -p "$out/offline-images" + printf '%s\n' "# image archive" > "$out/offline-images/manifest" + ${lib.concatStringsSep "\n" (lib.imap0 ( + index: container: '' + archive="$(printf '%02d.tar' ${toString index})" + cp ${container.archive} "$out/offline-images/$archive" + printf '%s %s\n' '${container.imageName}:${container.finalImageTag}' "$archive" >> "$out/offline-images/manifest" + '' + ) + containers)} + '' diff --git a/kubernetes/scripts/list-helm-images.sh b/kubernetes/scripts/list-helm-images.sh new file mode 100755 index 0000000..a38b0e1 --- /dev/null +++ b/kubernetes/scripts/list-helm-images.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Render a Helm chart and list images used by Pod templates. +set -euo pipefail + +if [[ $# -eq 0 ]]; then + printf 'Usage: %s CHART [HELM_TEMPLATE_FLAGS...]\n' "$0" >&2 + exit 2 +fi + +CHART=$1 +shift + +helm template image-discovery "$CHART" --include-crds "$@" | + yq -r ' + [ + .. | select(type == "!!map") | + ( + .containers[]?.image, + .initContainers[]?.image, + .ephemeralContainers[]?.image + ) + ] + | .[] + | select(tag == "!!str" and . != "") + ' | + sort -u diff --git a/kubernetes/scripts/pull-offline-images.sh b/kubernetes/scripts/pull-offline-images.sh deleted file mode 100755 index 368ca08..0000000 --- a/kubernetes/scripts/pull-offline-images.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# Pull all offline cluster images on the control machine and bundle them into a -# per-image docker archives for `ctr images import` on the nodes. -# Idempotent: skips if the bundle exists for the current image list. -set -euo pipefail - -KUBE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -LIST="$KUBE_DIR/offline-images.list" -OUT_DIR="$KUBE_DIR/.cache" -BUNDLE_DIR="$OUT_DIR/offline-images" -STAMP="$OUT_DIR/offline-images.list.sha256" -MANIFEST="$BUNDLE_DIR/manifest" - -LIST_HASH="$(sha256sum "$LIST" | cut -d ' ' -f1)" - -if [[ -f "$MANIFEST" && -f "$STAMP" && "$(<"$STAMP")" == "$LIST_HASH" ]]; then - echo "Offline image bundle already exists: $BUNDLE_DIR" - du -sh "$BUNDLE_DIR" - exit 0 -fi - -mapfile -t IMAGES < <(grep -vE '^\s*(#|$)' "$LIST") -if [[ ${#IMAGES[@]} -eq 0 ]]; then - echo "ERROR: no images listed in $LIST" >&2 - exit 1 -fi - -mkdir -p "$OUT_DIR" -rm -rf "$BUNDLE_DIR" "$OUT_DIR/offline-images.tar" -mkdir -p "$BUNDLE_DIR" -echo "Pulling ${#IMAGES[@]} offline images (control machine needs internet)..." -SAVE_REFS=() -printf '%s\n' "# archive_ref archive_path" > "$MANIFEST" -for image in "${IMAGES[@]}"; do - index=${#SAVE_REFS[@]} - image_id=$(podman pull --quiet "$image") - archive_ref="${image%@*}" - podman tag "$image_id" "$archive_ref" - archive_path="$BUNDLE_DIR/$(printf '%02d.tar' "$index")" - podman save -o "$archive_path" "$archive_ref" - printf '%s %s\n' "$archive_ref" "$archive_path" >> "$MANIFEST" - SAVE_REFS+=("$archive_ref") -done - -printf '%s\n' "$LIST_HASH" > "$STAMP" - -du -sh "$BUNDLE_DIR" -echo "Done." -- 2.51.2