diff --git a/.gitignore b/.gitignore index e2fdb96..06ff767 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,14 @@ kubernetes/pki/out/ # SSH private keys kubernetes/ssh-key harden/debug/ssh-key +13-inch-thin-cannon/tests/ssh-key + +# ISO test artifacts +13-inch-thin-cannon/tests/*.qcow2 +13-inch-thin-cannon/tests/*.pid +13-inch-thin-cannon/tests/*.log +13-inch-thin-cannon/tests/ovmf/ +13-inch-thin-cannon/tests/ovmf-vars.fd # Generated Ansible inventory kubernetes/ansible/inventory.ini diff --git a/13-inch-thin-cannon/configuration.nix b/13-inch-thin-cannon/configuration.nix index 8ed1263..b807f9f 100644 --- a/13-inch-thin-cannon/configuration.nix +++ b/13-inch-thin-cannon/configuration.nix @@ -1,45 +1,41 @@ -# Edit this configuration file to define what should be installed on -# your system. Help is available in the configuration.nix(5) man page -# and in the NixOS manual (accessible by running ‘nixos-help’). +# 13-inch-thin-cannon: hardened NixOS host. +# Security: Secure Boot (Lanzaboote), LUKS full-disk encryption, +# tmpfs-style impermanence (root rolls back each boot). +# Disk layout is owned by ./disko.nix (see docs/disko-notes.md). { config, pkgs, lib, + inputs, ... }: { imports = [ ./hardware-configuration.nix + ../modules/hardened.nix ../modules/secrets.nix + "${inputs.impermanence}/nixos.nix" + inputs.disko.nixosModules.disko + inputs.lanzaboote.nixosModules.lanzaboote + ./disko.nix ] ++ lib.optional (builtins.pathExists ./wireguard.nix) ./wireguard.nix; + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + services = { fwupd.enable = true; + openssh = { + enable = true; + settings.PermitRootLogin = "prohibit-password"; + }; }; - # Bootloader. - boot.loader.systemd-boot.enable = true; - boot.loader.efi.canTouchEfiVariables = true; - - networking.hostName = "13-inch-thin-cannon"; # Define your hostname. - #networking.wireless.enable = true; # Enables wireless support via wpa_supplicant. - #networking.wireless.networks = { - # "Fios-BKP8q" = { # SSID with spaces and/or special characters - # psk = "haul66jewel33row"; - # }; - - #}; - # Configure network proxy if necessary - # networking.proxy.default = "http://user:password@proxy:port/"; - # networking.proxy.noProxy = "127.0.0.1,localhost,internal.domain"; - - # Enable networking + networking.hostName = "13-inch-thin-cannon"; networking.networkmanager.enable = true; - # Set your time zone. time.timeZone = "America/New_York"; # Select internationalisation properties. @@ -71,18 +67,8 @@ alsa.enable = true; alsa.support32Bit = true; pulse.enable = true; - # If you want to use JACK applications, uncomment this - #jack.enable = true; - - # use the example session manager (no others are packaged yet so this is enabled by default, - # no need to redefine it in your config for now) - #media-session.enable = true; }; - # Enable touchpad support (enabled default in most desktopManager). - # services.xserver.libinput.enable = true; - - # Define a user account. Don't forget to set a password with ‘passwd’. users.users.file_magic = { isNormalUser = true; description = "file_magic"; @@ -90,50 +76,77 @@ "networkmanager" "wheel" ]; + openssh.authorizedKeys.keys = [ + (builtins.readFile ./tests/ssh-key.pub) + ]; }; - # Some programs need SUID wrappers, can be configured further or are - # started in user sessions. - # programs.mtr.enable = true; - # programs.gnupg.agent = { - # enable = true; - # enableSSHSupport = true; - # }; + users.users.root.openssh.authorizedKeys.keys = [ + (builtins.readFile ./tests/ssh-key.pub) + ]; - # List services that you want to enable: + # Security hardening (secure boot, LUKS, impermanence). + hardened = { + secureboot.enable = true; + luks.enable = true; + impermanence.enable = true; + # apparmor + kernel.hardened can be enabled later (one-liners). + }; - # Enable the OpenSSH daemon. - # services.openssh.enable = true; + # Impermanence additions beyond ../modules/impermanence.nix defaults. + environment.persistence."/persistent" = { + directories = [ + "/home/file_magic" + ]; + files = [ + "/etc/passwd" + "/etc/group" + "/etc/shadow" + "/etc/gshadow" + ]; + }; - # Open ports in the firewall. - # networking.firewall.allowedTCPPorts = [ ... ]; - # networking.firewall.allowedUDPPorts = [ ... ]; - # Or disable the firewall altogether. - # networking.firewall.enable = false; + # Swapfile on the encrypted /swap subvolume (created on first boot). + swapDevices = [ + {device = "/swap/swapfile";} + ]; - # Auto update - system.autoUpgrade = { - enable = true; - flake = "/home/file_magic/Projects/tangled/nix"; - operation = "boot"; - upgrade = true; + systemd.services.create-swapfile = { + description = "Create encrypted btrfs swapfile"; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + unitConfig.RequiresMountsFor = ["/swap"]; + before = ["swap.target"]; + wantedBy = ["swap.target"]; + script = '' + if [[ ! -e /swap/swapfile ]]; then + truncate -s 0 /swap/swapfile + chattr +C /swap/swapfile + btrfs property set /swap/swapfile compression none + dd if=/dev/zero of=/swap/swapfile bs=1M count=16384 status=none + chmod 600 /swap/swapfile + mkswap /swap/swapfile + fi + ''; }; programs.hyprland.enable = true; powerManagement.cpuFreqGovernor = "performance"; - # Proactive overheating prevention (especially for Intel CPUs) services.thermald.enable = true; - #services.tlp.enable = true; - #services.auto-cpu-freq.enable = true; powerManagement.powertop.enable = true; boot.kernelPackages = pkgs.linuxPackages_latest; + boot.kernelParams = ["console=ttyS0" "console=tty0"]; + + # Auto update + system.autoUpgrade = { + enable = true; + flake = "/home/file_magic/Projects/tangled/nix"; + operation = "boot"; + upgrade = true; + }; - # This value determines the NixOS release from which the default - # settings for stateful data, like file locations and database versions - # on your system were taken. It‘s perfectly fine and recommended to leave - # this value at the release version of the first install of this system. - # Before changing this value read the documentation for this option - # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html). - system.stateVersion = "25.11"; # Did you read the comment? + system.stateVersion = "25.11"; } diff --git a/13-inch-thin-cannon/disko.nix b/13-inch-thin-cannon/disko.nix new file mode 100644 index 0000000..9b7c787 --- /dev/null +++ b/13-inch-thin-cannon/disko.nix @@ -0,0 +1,59 @@ +{lib, ...}: { + disko.devices = { + disk = { + main = { + type = "disk"; + content = { + type = "gpt"; + partitions = { + ESP = { + size = "512M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + mountOptions = [ + "fmask=0077" + "dmask=0077" + ]; + }; + }; + crypt = { + size = "100%"; + content = { + type = "luks"; + name = "cryptroot"; + settings = { + allowDiscards = true; + }; + content = { + type = "btrfs"; + extraArgs = ["-f"]; + subvolumes = { + "/root" = { + mountpoint = "/"; + mountOptions = ["compress=zstd" "noatime"]; + }; + "/root-blank" = { + mountpoint = "/root-blank"; + mountOptions = []; + }; + "/persistent" = { + mountpoint = "/persistent"; + mountOptions = ["compress=zstd" "noatime"]; + }; + "/swap" = { + mountpoint = "/swap"; + mountOptions = []; + }; + }; + }; + }; + }; + }; + }; + }; + }; + }; +} diff --git a/13-inch-thin-cannon/docs/iso-build.md b/13-inch-thin-cannon/docs/iso-build.md new file mode 100644 index 0000000..85bd91e --- /dev/null +++ b/13-inch-thin-cannon/docs/iso-build.md @@ -0,0 +1,293 @@ +# Building & testing the 13-inch-thin-cannon install ISO + +Single source of truth for how the install ISO for `13-inch-thin-cannon` is +built and tested. **Keep this updated** — every new build/test learning belongs +here. Reference this file instead of copying facts into other docs +(see AGENTS.md → DRY). + +## Purpose + +`13-inch-thin-cannon` is a hardened NixOS host (Secure Boot via Lanzaboote, +LUKS full-disk encryption, tmpfs-as-root impermanence). The ISO is a +**disko-based live installer** that lets you install that host onto bare metal: + +```bash +disko-install --flake /iso/repo#13-inch-thin-cannon \ + --disk main /dev/vda --write-efi-boot-entries +``` + +## Architecture (the moving parts) + +| Piece | File | Role | +|-------|------|------| +| ISO config | `13-inch-thin-cannon/installer.nix` | Live installer environment (SSH, hardened posture, baked flake **+ baked install closure**, EFI boot) | +| Host config | `13-inch-thin-cannon/configuration.nix` | What gets *installed*: hardening, impermanence, swapfile, ssh, etc. | +| Disk layout | `13-inch-thin-cannon/disko.nix` | GPT: 512M ESP (`/boot`, vfat, fmask/dmask 0077) + LUKS `cryptroot` with btrfs subvolumes `/`, `/root-blank`, `/persistent`, `/swap` | +| Hardened module | `modules/hardened.nix` | Option flags toggled per machine: secureboot, luks, impermanence, apparmor, kernel.hardened, agenix, user | +| Flake wiring | `flake.nix` | `nixosConfigurations."13-inch-thin-cannon-iso"` imports `installer.nix` only; host configs get `specialArgs = { inherit inputs self; }`; `packages.${system}.ovmf = OVMFFull.fd` | +| Test harness | `13-inch-thin-cannon/tests/iso-test.sh` | QEMU end-to-end test: live → install → boot | +| just targets | `justfile` | `iso`, `ovmf`, `test-iso`, `up` | + +### ISO vs installed system posture (intentional differences) + +`installer.nix` enables `hardened.luks`, `hardened.apparmor`, and +`hardened.kernel.hardened`, but leaves **secureboot and impermanence OFF**: + +- Lanzaboote owns the *installed* bootloader, not the ISO's GRUB. +- impermanence asserts a btrfs root, which a live ISO does not have. + +### Key ISO facts (hard-won) + +- **EFI El Torito is opt-in in nixpkgs.** `isoImage.makeEfiBootable = true` + must be set or the ISO has no UEFI boot path. The ISO is GRUB-based and + self-contained; with it set, the second EFI image is present in the + `efi.img` and OVMF boots it. `makeUsbBootable = true` is also set. +- **The flake is baked in, not fetched.** `isoImage.contents` copies + `self.outPath` (the repo) to `/repo`; the live system mounts its CD at + `/iso`, so the flake lives at `/iso/repo`. This is how `disko-install` + works with no network. +- **The machine's install closure is baked in too** (see "Self-contained + store" below) — `disko-install`'s internal `nix-build` is a no-op against + the ISO's own store. No host-store sharing, no network, at install time. +- **Serial console.** `boot.kernelParams = ["console=ttyS0"]` on the ISO so + QEMU `-serial file:` captures boot output. The installed host uses + `console=ttyS0 console=tty0` (see the LUKS gotcha below). +- **SSH access for testing.** The ISO enables openssh with + `PermitRootLogin = "prohibit-password"` and root authorized key + `tests/ssh-key.pub` (`PasswordAuthentication = false`). The installed + host authorizes the same key for both `root` and `file_magic`. + +## Building the ISO + +```bash +just iso # nix build .#nixosConfigurations."13-inch-thin-cannon-iso".config.system.build.isoImage +``` + +Output: `result/iso/nixos--x86_64-linux.iso`. `just up` builds and +tests in one go. The ISO is `nix`-buildable directly on the host (tested on +Nix 2.34.8); no network needed at runtime because everything is baked in. + +Note: `result/iso` and the OVMF `result/FV` cannot coexist in `result/`, which +is why `just ovmf` copies OVMF into `tests/ovmf/FV/` (gitignored). + +## Installing the host + +The ISO boots a live env; on real hardware you run `disko-install` yourself +(the `install.service` prints the command). `disko-install`: + +1. runs the disko script (partitions, LUKS, btrfs subvolumes, mounts at + `/mnt/disko-install-root`), +2. copies the NixOS system closure into the fresh store, +3. loads the store registration DB, +4. runs `nixos-install --no-channel-copy --no-root-password --system …`, +5. writes EFI boot entries (`--write-efi-boot-entries`). + +### Blank-store copy path (what disko-install actually does) + +From `disko_src/install-cli.nix` (disko input at +`github:nix-community/disko/de57087…`): + +```bash +xargs xcp --recursive --target-directory "$mountPoint/nix/store" \ + < closure_info/store-paths +NIX_STATE_DIR="$mountPoint/nix/var/nix" nix-store --load-db \ + < closure_info/registration +nixos-install --no-channel-copy --no-root-password \ + --system "$nixos_system" --root "$mountPoint" +``` + +It chooses this path when `$mountPoint/nix/var/nix/db/db.sqlite` does **not** +exist; otherwise it falls back to plain `nixos-install` (which nix-builds into +the mounted store). Copy tool is **xcp** (`gebner/xcp`, Rust). + +### Self-contained store (current design — no host store sharing) + +`installer.nix` bakes the machine's install closure into the ISO's own store: + +```nix +isoImage.storeContents = [ + config.system.build.toplevel # the installer system itself + installToplevel # machine toplevel as disko-install builds it + installClosureInfo # closureInfo of the above + installDiskoScript # disko script with disk/rootMountPoint overrides +]; +``` + +`installer.nix` replicates disko's `install-cli.nix` inline via +`extendModules` (pure — no `getFlake`, which fails on unlocked/store-path +flake refs). Two overridden systems, mirroring `install-cli.nix` exactly: + +- `diskoSystem`: `disko.rootMountPoint = "/mnt/disko-install-root"` + disk + devices overridden to `/dev/vda` → source of `installDiskoScript`. +- `installSystem`: `boot.loader.efi.canTouchEfiVariables = true` + + `boot.loader.grub.devices = ["/dev/vda"]` → source of `installToplevel` + and `installClosureInfo`. + +Verified byte-identical to disko-install's own `--argstr` build +(same store paths). With the closure in the ISO store, `disko-install`'s +internal `nix-build` finds everything already present → no-op. `xcp` then +copies from the ISO's local squashfs-backed store (fast, no 9p). + +Also: `isoImage.squashfsCompression = "zstd -Xcompression-level 6"` — +default level 19 is far too slow on the ~14 GiB store. The ISO is ~9.35 GiB. + +**History (superseded):** an earlier design shared the host `/nix` into the +VM via 9p + overlayfs. That failed with `xcp: Cannot allocate memory +(os error 12)` at ~85% of the ~14 GiB copy (not nix-daemon OOM; RAM=10240 +still failed). The self-contained design removed the 9p source entirely and +fixed it — **install phase now passes** (`disko-install succeeded`, exit 0). + +## Testing the image + +**`just test-iso`** (depends on `ovmf`) runs `tests/iso-test.sh` end-to-end: +live → install → boot. Or run phases individually: + +```bash +./13-inch-thin-cannon/tests/iso-test.sh # all phases +./13-inch-thin-cannon/tests/iso-test.sh live # boot ISO, verify live env +./13-inch-thin-cannon/tests/iso-test.sh install# disko-install onto /dev/vda +./13-inch-thin-cannon/tests/iso-test.sh boot # boot installed system +``` + +The full pipeline **must pass before committing** any ISO/host change +(AGENTS.md → Infrastructure changes testing). The script is the source of +truth for what "healthy" means; `verify.yml`-style checks live inside it. + +### Environment variables + +| Var | Default | Meaning | +|-----|---------|---------| +| `RAM` | `6144` | VM memory in MiB. Use `RAM=10240` for the install phase | +| `SSH_PORT` | `2223` | Host port forwarded to VM :22 | +| `SERIAL_PORT` | `5557` | QEMU serial TCP port (boot phase LUKS unlock + log tee) | +| `MONITOR_PORT` | `5555` | QEMU monitor TCP port (diagnostics: screendump, info registers) | +| `PASS` | `test-passphrase-13-inch-thin-cannon` | LUKS passphrase (install create + boot unlock) | +| `INTERACTIVE` | `0` | `INTERACTIVE=1`: ask the user to type the LUKS passphrase instead of using `$PASS` | + +Requires: `qemu-system-x86_64`, `qemu-img`, `ssh`, `nix`, host `/dev/kvm`. +OVMF must exist: `just ovmf` (source: `OVMFFull.fd` from the flake's `ovmf` +package). The test disk is `tests/disk.qcow2` (40G, recreated each run). + +### Phase: live + +Boots the ISO under OVMF and verifies: +- kernel cmdline hardening: `lockdown=confidentiality`, + `module.sig_enforce=1`, `slab_nomerge`, `init_on_alloc=1`, + `init_on_free=1`, `apparmor=1` +- flake baked at `/iso/repo/flake.nix`; `disko-install` on PATH; `/dev/vda` + present; AppArmor loaded; no failed services + +### Phase: install + +Boots the ISO again (self-contained — no host store share) and runs +`disko-install`. The LUKS passphrase is piped over SSH into a pty +(`ssh -tt`) and answered twice (create + confirm). On failure the script dumps +the full disko-install output to `tests/disko-install.out.log` and prints +target-state diagnostics via SSH (profiles, `system` readlink, store count, +db dir, `df`/`du` for the root mount, `dmesg` OOM grep). + +Clean-up trick for the ANSI-flooded log +(`/tmp/clean.log`): + +```bash +perl -pe 's/\x1b\[[0-9;]*[A-Za-z]//g; s/\x1b\][^\x07\x1b]*(\x07|\x1b\\)//g; s/\r//g' +``` + +### Phase: boot — CURRENTLY BLOCKED + +Boots the **installed** disk (no CD) under OVMF. LUKS unlock goes over the +**serial console**: the prompt is read from `/dev/console`, whose input is +`ttyS0` (the *first* `console=` parameter wins for input), so VGA `sendkey` +can never unlock it. `phase_boot` opens the QEMU serial TCP socket +(`SERIAL_PORT`, default 5557) directly with bash `/dev/tcp` (fd 5); a +`dd bs=1` tee copies reads into `boot-serial.log` (byte-flushed so the +prompt is detected promptly) and `unlock_luks()` writes the passphrase out on +the same fd. `INTERACTIVE=1` makes `unlock_luks` ask the user to type the +passphrase instead of using `$PASS`. (bash `coproc`+`nc` must NOT be used +inside functions — fds come out `Bad file descriptor`.) + +Once SSH is up it verifies: Secure Boot enabled (sbctl), `cryptroot` is LUKS, +`/` is btrfs with `/persistent` and `/root-blank`, `/etc/nixos` persisted, +kernel cmdline, hostname, no failed services. + +**Status:** unlock works (passphrase accepted — see Known issues), but the +system never reaches SSH. This is the active blocker. + +## Known issues & gotchas + +- **BOOT PHASE BLOCKED: passphrase accepted, then nothing.** The installed + system's LUKS unlock provably works over serial: + - bare `\n` → re-prompt (Enter is delivered and processed) + - wrong passphrase → re-prompt (submission works, rejected) + - correct passphrase → **no re-prompt** (accepted), but zero boot output + afterward; serial log frozen at ~3.3 KB (prompt + 35 masked bullets); + VGA screendump static near-black (6 unique colors); SSH never comes up + (watched to 120–240 s). + - Guest liveness via QEMU monitor: `info registers` identical across 3 s + samples with `HLT=1` → the **kernel is up and in its idle loop** (not a + spin/panic; no panic output on serial). Whether it is stuck in the initrd + or fully booted with dead networking is still unresolved. + - Ruled out: no `networking.firewall` anywhere in the host/modules; + `modules/luks.nix` is a stub; nothing in the hardened module set blocks + sshd. The UKI stub prints + `[ WARN]: stub/src/thin.rs@071: Secure Boot is not active!` on boot. + - Next experiment queued by the user: **disable Secure Boot** + (`hardened.secureboot.enable = false`) → rebuild ISO → reinstall → boot. + Not yet applied. A SysRq task-dump via monitor `sendkey + ctrl-alt-sysrq-t` was attempted to identify the boot state but the + monitor connection was refused (wrong port in the diag script — + `MONITOR_PORT` default is 5555). +- **LUKS prompt input is serial (`ttyS0`), not VGA.** cryptsetup reads + `/dev/console`, whose input is the first `console=` parameter (`ttyS0`). + The prompt text *does* appear in the serial log; VGA `sendkey` can only + echo dots, never unlock. Unlock via the serial TCP socket (see Phase: boot). +- **OVMF PXE delay (~65 s).** PXE/HTTP boot entries created during the + install phase persist in `ovmf-vars.fd` NVRAM and are attempted on every + boot even without a NIC. Use a fresh copy of `OVMF_VARS.fd` for + boot-from-disk-only checks. +- **`pkill` self-match.** `pkill -f "qemu-system-x86_64.*2223"` matches and + kills the invoking shell; use `pkill -f "[h]ostfwd=tcp:127.0.0.1:2223"`. +- **disko-install blank-store decision is a `db.sqlite` existence check.** + `[[ ! -f "$mountPoint/nix/var/nix/db/db.sqlite" ]]` picks the xcp-copy path + vs plain `nixos-install`. Anything that pre-creates that file skips the copy. +- **Host `sudo` needs a password** — `iso-test.sh` never uses sudo; QEMU runs + as the current user. Don't add sudo into the test path. +- **`result/iso` vs `result/FV` collision** — keep OVMF in `tests/ovmf/FV/` + (gitignored), refreshed by `just ovmf`. +- **ISO rebuild is a full `nix build`** (minutes; ~9.35 GiB ISO at zstd-6); + `iso-test.sh` always uses the newest `result/iso/*.iso`. Rebuild after any + `installer.nix`/machine-config change or the test will exercise a stale ISO. +- **Resolved (history):** the 9p host-store share + xcp blank-store copy + failed with `xcp: Cannot allocate memory (os error 12)` at ~85% of the + ~14 GiB copy. Fixed by baking the install closure into the ISO + (see "Self-contained store"). The 9p/overlay machinery was removed from + `iso-test.sh`. + +## Current status & missing work + +- **Green:** `live` phase, `install` phase (`disko-install succeeded`, + exit 0, `RAM=10240`). +- **Red:** `boot` phase — the only remaining blocker to a verified working + machine (see Known issues). +- **Queued:** secure-boot-off experiment (user-requested); if boot comes up, + finish boot-phase verifications and run `just up` end-to-end. +- **Committed, not yet green.** All host/ISO/test files are committed, but + the full pipeline (`just up`) only passes through the install phase. +- **Deferred hardening:** `hardened.apparmor.enable` and + `hardened.kernel.hardened` are one-liners on the host config (enabled on + the ISO, off on the machine "for now"). + +## Useful links + +- [disko](https://github.com/nix-community/disko) — declarative disk layout + + `disko-install`; `install-cli.nix` is the source of truth for the + blank-store copy path. +- [xcp (gebner/xcp)](https://github.com/gebner/xcp) — Rust `cp` used by + disko-install's blank-store copy; ENOMEM under the old 9p/overlay setup + (resolved by the self-contained ISO). +- nixpkgs `nixos/modules/installer/cd-dvd/iso-image.nix` — `makeEfiBootable` + controls the UEFI El Torito image; EFI is *not* enabled by default. +- [lanzaboote](https://github.com/nix-community/lanzaboote) — the installed + system's Secure Boot bootloader. +- [impermanence](https://github.com/nix-community/impermanence) — tmpfs-as-root + module used by the host (off on the ISO). diff --git a/13-inch-thin-cannon/hardware-configuration.nix b/13-inch-thin-cannon/hardware-configuration.nix index 66a48b1..f41b63f 100644 --- a/13-inch-thin-cannon/hardware-configuration.nix +++ b/13-inch-thin-cannon/hardware-configuration.nix @@ -1,6 +1,5 @@ -# Do not modify this file! It was generated by ‘nixos-generate-config’ -# and may be overwritten by future invocations. Please make changes -# to /etc/nixos/configuration.nix instead. +# Disk layout (fileSystems, swapDevices) is now owned by ./disko.nix. +# This file only carries hardware-specific module and microcode settings. { config, lib, @@ -12,26 +11,20 @@ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = ["xhci_pci" "nvme" "usb_storage" "usbhid" "sd_mod"]; + boot.initrd.availableKernelModules = [ + "xhci_pci" + "nvme" + "usb_storage" + "usbhid" + "sd_mod" + "virtio_pci" + "virtio_blk" + "virtio_scsi" + ]; boot.initrd.kernelModules = []; boot.kernelModules = ["kvm-intel"]; boot.extraModulePackages = []; - fileSystems."/" = { - device = "/dev/disk/by-uuid/76cffb2a-7482-4116-a423-c7ede185ca7d"; - fsType = "ext4"; - }; - - fileSystems."/boot" = { - device = "/dev/disk/by-uuid/209F-2960"; - fsType = "vfat"; - options = ["fmask=0077" "dmask=0077"]; - }; - - swapDevices = [ - {device = "/dev/disk/by-uuid/5bece5cb-4baf-4d7b-87bd-400f997c401e";} - ]; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/13-inch-thin-cannon/installer.nix b/13-inch-thin-cannon/installer.nix new file mode 100644 index 0000000..3c018bd --- /dev/null +++ b/13-inch-thin-cannon/installer.nix @@ -0,0 +1,163 @@ +# 13-inch-thin-cannon live install ISO. +# +# Boots a minimal live environment (same hardening posture as the installed +# system) with the repo baked in (the live ISO mounts its CD contents at /iso) +# so you can run: +# +# disko-install --flake /iso/repo#13-inch-thin-cannon \ +# --disk main /dev/vda --write-efi-boot-entries +# +# Build: `just iso` → Test: `just test-iso` +{ + config, + pkgs, + lib, + inputs, + self, + ... +}: let + # Replicate what `disko-install` builds internally (install-cli.nix) so the + # machine's install closure is baked into the ISO's store. In the live VM, + # disko-install's internal `nix-build` then finds every path already in the + # ISO's own store and is a no-op — the installer is fully self-contained + # (no host Nix store share, no network). Done without getFlake so the ISO + # build stays pure. + installDiskMappings = {main = "/dev/vda";}; + + originalSystem = self.nixosConfigurations."13-inch-thin-cannon"; + + modifiedDisks = lib.mapAttrs ( + name: value: + let + dev = installDiskMappings.${name}; + in + value + // { + device = dev; + content = value.content // {device = dev;}; + } + ) originalSystem.config.disko.devices.disk; + # filter all nixos module internal attributes + cleanedDisks = lib.filterAttrsRecursive (n: _: !lib.hasPrefix "_" n) modifiedDisks; + + diskoSystem = originalSystem.extendModules { + modules = [ + { + disko.rootMountPoint = "/mnt/disko-install-root"; + disko.devices.disk = lib.mkVMOverride cleanedDisks; + } + ]; + }; + + installSystem = originalSystem.extendModules { + modules = [ + ({ + lib, + ... + }: { + boot.loader.efi.canTouchEfiVariables = lib.mkVMOverride true; + boot.loader.grub.devices = lib.mkVMOverride (lib.attrValues installDiskMappings); + imports = [ + ({_file = "disko-install --system-config";} // (builtins.fromJSON "{}")) + ]; + }) + ]; + }; + + installToplevel = installSystem.config.system.build.toplevel; + installClosureInfo = installSystem.pkgs.closureInfo { + rootPaths = [installToplevel]; + }; + installDiskoScript = diskoSystem.config.system.build.diskoScript; +in { + imports = [ + "${inputs.impermanence}/nixos.nix" + inputs.disko.nixosModules.disko + inputs.agenix.nixosModules.age + inputs.lanzaboote.nixosModules.lanzaboote + ../modules/hardened.nix + "${inputs.nixpkgs}/nixos/modules/installer/cd-dvd/iso-image.nix" + ]; + + boot.kernelParams = ["console=ttyS0"]; + + networking.hostName = "13-inch-thin-cannon-installer"; + networking.networkmanager.enable = true; + + services.openssh = { + enable = true; + settings = { + PermitRootLogin = "prohibit-password"; + PasswordAuthentication = false; + }; + }; + + users.users.root.openssh.authorizedKeys.keys = [ + (builtins.readFile ./tests/ssh-key.pub) + ]; + + hardened = { + luks.enable = true; + apparmor.enable = true; + kernel.hardened = true; + # secureboot + impermanence are intentionally OFF on the live ISO: + # - lanzaboote owns the *installed* bootloader, not the ISO's + # - impermanence asserts a btrfs root, which a live ISO does not have + }; + + nix.settings.experimental-features = [ + "nix-command" + "flakes" + ]; + + environment.systemPackages = with pkgs; [ + vim + git + inputs.disko.packages.${pkgs.system}.disko-install + ]; + + isoImage = { + makeEfiBootable = true; + makeUsbBootable = true; + # Level 19 on a ~14 GiB store is very slow; level 6 is nearly as compact + # for binaries and much faster to iterate on. + squashfsCompression = "zstd -Xcompression-level 6"; + }; + + # Bake the machine's install closure (toplevel + closureInfo + diskoScript) + # into the ISO's /nix/store so the installer needs nothing but the ISO itself. + isoImage.storeContents = [ + config.system.build.toplevel + installToplevel + installClosureInfo + installDiskoScript + ]; + + # Bake the flake into the ISO so disko-install works without network access. + # The live system mounts its CD at /iso, so the flake is at /iso/repo. + isoImage.contents = [ + { + source = self.outPath; + target = "/repo"; + } + ]; + + systemd.services.install = { + description = "Install 13-inch-thin-cannon"; + wantedBy = ["multi-user.target"]; + after = ["network.target"]; + script = '' + echo "================================================================" + echo " 13-inch-thin-cannon installer" + echo "" + echo " The flake is baked in at /iso/repo." + echo "" + echo " To install (WIPES the target disk):" + echo " disko-install --flake /iso/repo#13-inch-thin-cannon \\" + echo " --disk main /dev/vda --write-efi-boot-entries" + echo "================================================================" + ''; + }; + + system.stateVersion = "25.11"; +} diff --git a/13-inch-thin-cannon/tests/ssh-key.pub b/13-inch-thin-cannon/tests/ssh-key.pub new file mode 100644 index 0000000..aa38364 --- /dev/null +++ b/13-inch-thin-cannon/tests/ssh-key.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGNbot/sq8c6YE51nNwxgcsAHhSALAKftg/1SeBiQ4ti 13-inch-thin-cannon-iso-test diff --git a/AGENTS.md b/AGENTS.md index b252369..27da223 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,6 +61,7 @@ # Debugging docs - **Before debugging an issue, check `kubernetes/docs/security-hardening.md` first.** This contains known issues, root cause analyses, and debugging techniques from previous sessions. Do not waste time re-discovering solutions that are already documented. +- **Before debugging the 13-inch-thin-cannon ISO build or install, check `13-inch-thin-cannon/docs/iso-build.md` first** (Known issues & gotchas section). Keep that file updated with every build/test learning — it is the single source of truth for that workflow. - **When you find the real solution to a problem (tested, verified, end-to-end passing), document it.** Add the root cause, the fix, and how you verified it to `kubernetes/docs/security-hardening.md`. If the fix involved a specific debugging technique or command that was useful, add it to the debugging techniques section. - **After finishing a feature or debugging session, audit your debugging techniques.** Did you use any new commands, tools, or workflows that aren't documented yet? If so, add them to the debugging techniques section in `kubernetes/docs/security-hardening.md`. The goal is that the next person (or future you) can reproduce the debugging process without re-deriving everything from scratch. - **Do NOT put debugging documentation into `resume-state.md`.** That file is a cluster state snapshot only (what's running, commits, commands). All debugging knowledge goes into `security-hardening.md`. @@ -87,6 +88,7 @@ | `debugging.md` | Debugging notes | Root cause analysis, investigation techniques, fix details, infrastructure commands (OVMF, virsh, rebuild cycles) | State snapshots, security posture | | `session-state-YYYY-MM-DD.md` | Historical session log | What was changed in this session, design decisions, commits made | Full cluster state, duplicated debugging fixes (reference `debugging.md`) | | `custom-ca.md` | Feature-specific guide | CA setup, cert generation procedures | — | +| `13-inch-thin-cannon/docs/iso-build.md` | ISO build + test guide | How the install ISO for `13-inch-thin-cannon` is built, the disko-install/9p-overlay architecture, how to test the image (`iso-test.sh` phases), known issues/gotchas | Component state snapshots (use `resume-state.md`), debugging narratives that belong in `debugging.md` | | `AGENTS.md` | Agent behavior rules | How to work in this repo (commit style, DRY, testing, docs) | Project-specific technical details | ## When to add more documentation diff --git a/flake.lock b/flake.lock index 6be7120..7be0fd7 100644 --- a/flake.lock +++ b/flake.lock @@ -23,6 +23,21 @@ "type": "github" } }, + "crane": { + "locked": { + "lastModified": 1784407669, + "narHash": "sha256-gcFMcRjw0ZSn380Rx2QLlU1goUQeSrKX/DF12omI6+o=", + "owner": "ipetkov", + "repo": "crane", + "rev": "1316b7d278ad77a16aec024b71d971366e123bec", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "darwin": { "inputs": { "nixpkgs": [ @@ -45,6 +60,43 @@ "type": "github" } }, + "disko": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1768920986, + "narHash": "sha256-CNzzBsRhq7gg4BMBuTDObiWDH/rFYHEuDRVOwCcwXw4=", + "owner": "nix-community", + "repo": "disko", + "rev": "de5708739256238fb912c62f03988815db89ec9a", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "disko", + "rev": "de5708739256238fb912c62f03988815db89ec9a", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, "flake-parts": { "inputs": { "nixpkgs-lib": [ @@ -107,6 +159,73 @@ "type": "github" } }, + "home-manager_3": { + "inputs": { + "nixpkgs": [ + "impermanence", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1768598210, + "narHash": "sha256-kkgA32s/f4jaa4UG+2f8C225Qvclxnqs76mf8zvTVPg=", + "owner": "nix-community", + "repo": "home-manager", + "rev": "c47b2cc64a629f8e075de52e4742de688f930dc6", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "home-manager", + "type": "github" + } + }, + "impermanence": { + "inputs": { + "home-manager": "home-manager_3", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1769548169, + "narHash": "sha256-03+JxvzmfwRu+5JafM0DLbxgHttOQZkUtDWBmeUkN8Y=", + "owner": "nix-community", + "repo": "impermanence", + "rev": "7b1d382faf603b6d264f58627330f9faa5cba149", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "impermanence", + "rev": "7b1d382faf603b6d264f58627330f9faa5cba149", + "type": "github" + } + }, + "lanzaboote": { + "inputs": { + "crane": "crane", + "nixpkgs": [ + "nixpkgs" + ], + "pre-commit": "pre-commit", + "rust-overlay": "rust-overlay" + }, + "locked": { + "lastModified": 1784568171, + "narHash": "sha256-t17AqLEhPG6m27ipkp8mJd8Ug0XkdANFDVsgyIFBZcQ=", + "owner": "nix-community", + "repo": "lanzaboote", + "rev": "f4b0aef3dba28677a5ca4b3416827aade60b5a0b", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "lanzaboote", + "rev": "f4b0aef3dba28677a5ca4b3416827aade60b5a0b", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1785318670, @@ -144,14 +263,60 @@ "type": "github" } }, + "pre-commit": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": [ + "lanzaboote", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1784288435, + "narHash": "sha256-ReRHaLgr/uVqdD8afFSn+myXIfpHeOhP0yYe0TJqAA8=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "43b3c1ab9d40fb1dbb008f451988a91e375825e9", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "root": { "inputs": { "agenix": "agenix", + "disko": "disko", "home-manager": "home-manager_2", + "impermanence": "impermanence", + "lanzaboote": "lanzaboote", "nixpkgs": "nixpkgs", "nur": "nur" } }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "lanzaboote", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1784438913, + "narHash": "sha256-NYF7ZM5ip0u+w1pBFDpIGEbrbgN/wpnLFAmBkWkYMXw=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "afacd6819d3765a05814ee8e3de74c77d42ac799", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, "systems": { "locked": { "lastModified": 1681028828, diff --git a/flake.nix b/flake.nix index cd03abb..3a71629 100644 --- a/flake.nix +++ b/flake.nix @@ -15,6 +15,18 @@ url = "github:ryantm/agenix"; inputs.nixpkgs.follows = "nixpkgs"; }; + impermanence = { + url = "github:nix-community/impermanence/7b1d382faf603b6d264f58627330f9faa5cba149"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + disko = { + url = "github:nix-community/disko/de5708739256238fb912c62f03988815db89ec9a"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + lanzaboote = { + url = "github:nix-community/lanzaboote/f4b0aef3dba28677a5ca4b3416827aade60b5a0b"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = { @@ -24,7 +36,7 @@ nur, agenix, ... - }: let + } @ inputs: let lib = nixpkgs.lib; system = "x86_64-linux"; nurOverlay = final: prev: { @@ -43,6 +55,11 @@ packages = [agenix.packages."${system}".agenix]; }; + packages."${system}" = { + # OVMF firmware with Secure Boot enabled (for ISO / boot testing) + ovmf = nixpkgs.legacyPackages.${system}.OVMFFull.fd; + }; + nixosConfigurations = { thick-black-cannon = lib.nixosSystem { inherit system; @@ -72,6 +89,9 @@ "13-inch-thin-cannon" = lib.nixosSystem { inherit system; + specialArgs = { + inherit inputs self; + }; modules = [ ./13-inch-thin-cannon/configuration.nix agenix.nixosModules.age @@ -95,6 +115,16 @@ } ]; }; + + "13-inch-thin-cannon-iso" = lib.nixosSystem { + inherit system; + specialArgs = { + inherit inputs self; + }; + modules = [ + ./13-inch-thin-cannon/installer.nix + ]; + }; }; }; } diff --git a/justfile b/justfile new file mode 100644 index 0000000..76c871e --- /dev/null +++ b/justfile @@ -0,0 +1,21 @@ +default: help + +# Build the 13-inch-thin-cannon install ISO (disko-backed, hardened) +iso: + nix build .#nixosConfigurations."13-inch-thin-cannon-iso".config.system.build.isoImage + +# Copy OVMF firmware into the tests dir (result/iso and result/FV cannot coexist) +ovmf: + mkdir -p 13-inch-thin-cannon/tests/ovmf + cp -a $$(nix build .#ovmf --no-link --print-out-paths)/FV 13-inch-thin-cannon/tests/ovmf/ + +# Boot the generated ISO in QEMU (OVMF UEFI) and verify it +test-iso: ovmf + ./13-inch-thin-cannon/tests/iso-test.sh + +# Full lifecycle: build the ISO, then test it +up: iso test-iso + +# Show available targets +help: + @just --list