diff --git a/README.md b/README.md index caa2368..4526a49 100644 --- a/README.md +++ b/README.md @@ -173,10 +173,10 @@ sudo nixos-rebuild switch --flake .#odin sudo nixos-rebuild switch --flake .#othinus ``` -Trust builder host keys on the client: +Builder host keys are declared alongside their addresses in `modules/nix-remote-builders.nix` and installed in the system SSH known-hosts file. When a builder's SSH host key changes, verify it and update its `publicKey` before rebuilding clients: ```bash -sudo ssh-keyscan -H shizuri odin othinus | sudo tee -a /root/.ssh/known_hosts >/dev/null +ssh-keyscan -t ed25519 ``` #### NixOS clients @@ -227,16 +227,40 @@ Import `modules/nix-builder.nix` from the new host configuration: } ``` -Add a matching entry to the `builders` list in `modules/nix-remote-builders.nix`. +Add a matching entry, including its verified SSH `publicKey`, to the `builders` list in `modules/nix-remote-builders.nix`. Rebuild the new builder first, then rebuild each client that should use it: ```bash sudo nixos-rebuild switch --flake .#new-builder -sudo ssh-keyscan -H new-builder | sudo tee -a /root/.ssh/known_hosts >/dev/null sudo nixos-rebuild switch --flake .# ``` +### Automated cache warming + +Misaki runs `nix-impatient`, which updates all updateable flake inputs every four hours and builds `packages.x86_64-linux.nix-impatient` using the remote builder pool. The aggregate package contains every Linux NixOS system closure in this flake and the standalone Linux Home Manager activation package. Build results return to Misaki's Nix store, where `cache.ngp.computer` serves and signs them. + +`nix-impatient` commits candidates as `nix-impatient `, runs the flake checks, builds the aggregate closure, and fast-forwards `master` in `/srv/src/nixos.git` only after everything succeeds. The bare repository's post-receive hook then mirrors the commit to its remotes. A concurrent update to `master` causes publication to fail instead of overwriting the newer commit. + +The latest successful aggregate remains rooted. Previous successful roots are retained for 14 days. Configure the schedule, input selection, retention, and Git identity through `services.nix-impatient` in Misaki's configuration. + +Trigger a complete update and build manually, then follow its logs: + +```bash +sudo systemctl start nix-impatient.service +sudo journalctl -fu nix-impatient.service +``` + +The equivalent `mkfile` targets are `mk impatient` and `mk impatient-log`. + +Build the current checkout without updating or publishing its lock file: + +```bash +mk warm +# Equivalent to: +nix build .#nix-impatient --max-jobs 0 -L +``` + ## Maintenance Regular maintenance tasks: diff --git a/common.nix b/common.nix index 95c7f47..8df3baa 100644 --- a/common.nix +++ b/common.nix @@ -21,13 +21,6 @@ # Automatic doc cache generation documentation.man.cache.enable = true; - # Automatic system upgrades - system.autoUpgrade = { - enable = true; - dates = "09:00"; - randomizedDelaySec = "45min"; - }; - # Automatic Garbage Collection nix.gc.automatic = true; nix.gc.options = "--delete-older-than 8d"; diff --git a/flake.nix b/flake.nix index 293de28..d8ecd31 100644 --- a/flake.nix +++ b/flake.nix @@ -271,6 +271,40 @@ inputs = inputs; }; }; + packages.x86_64-linux.nix-impatient = + let + systems = self.nixosConfigurations; + in + nixpkgs.legacyPackages.x86_64-linux.linkFarm "nix-impatient" [ + { + name = "odin"; + path = systems.odin.config.system.build.toplevel; + } + { + name = "othinus"; + path = systems.othinus.config.system.build.toplevel; + } + { + name = "shizuri"; + path = systems.shizuri.config.system.build.toplevel; + } + { + name = "misaki"; + path = systems.misaki.config.system.build.toplevel; + } + { + name = "edge"; + path = systems.edge.config.system.build.toplevel; + } + { + name = "touma-wsl"; + path = systems.touma-wsl.config.system.build.toplevel; + } + { + name = "home-noah"; + path = self.homeConfigurations.noah.activationPackage; + } + ]; checks = forAllSystems (system: { pre-commit-check = pre-commit-hooks.lib.${system}.run { src = ./.; diff --git a/host-specific/misaki/configuration.nix b/host-specific/misaki/configuration.nix index 839701c..8646051 100644 --- a/host-specific/misaki/configuration.nix +++ b/host-specific/misaki/configuration.nix @@ -7,8 +7,14 @@ ./networking.nix ./packages.nix ./services.nix + ../../modules/nix-impatient.nix ../../modules/nixery.nix ]; + services.nix-impatient = { + enable = true; + commitName = "nix-impatient"; + commitEmail = "misaki@ngp.computer"; + }; nixpkgs.config.allowUnfree = true; system.stateVersion = "23.11"; # Did you read the comment? } diff --git a/mkfile b/mkfile index 13b7f76..2f98b42 100644 --- a/mkfile +++ b/mkfile @@ -24,5 +24,14 @@ hmswitch:V: copyall:V: exec $gitroot/scripts/copy-nix-closure-to-misaki --all +impatient:V: + exec sudo systemctl start nix-impatient.service + +impatient-log:V: + exec sudo journalctl -fu nix-impatient.service + +warm:V: + exec nix build $gitroot'#nix-impatient' --max-jobs 0 -L + clean:V: exec sudo nix-collect-garbage --delete-old diff --git a/modules/copy-nix-store-to-misaki-home.nix b/modules/copy-nix-store-to-misaki-home.nix index 00d80eb..b9d1c0c 100644 --- a/modules/copy-nix-store-to-misaki-home.nix +++ b/modules/copy-nix-store-to-misaki-home.nix @@ -11,7 +11,12 @@ in { home.activation.copyNixStoreToMisaki = lib.mkIf (osConfig == null) ( lib.hm.dag.entryAfter [ "linkGeneration" ] '' - export PATH=${lib.makeBinPath [ config.nix.package pkgs.openssh ]}:$PATH + export PATH=${ + lib.makeBinPath [ + config.nix.package + pkgs.openssh + ] + }:$PATH if ! $DRY_RUN_CMD "${copyScript}" "$newGenPath"; then echo "warning: failed to copy local Nix store objects to Misaki" >&2 fi diff --git a/modules/nix-binary-cache.nix b/modules/nix-binary-cache.nix index 31883cc..5e41f00 100644 --- a/modules/nix-binary-cache.nix +++ b/modules/nix-binary-cache.nix @@ -18,6 +18,8 @@ let HOME=${lib.escapeShellArg cfg.home} "NIX_SSHOPTS=-F ${lib.escapeShellArg cfg.sshConfig} -o BatchMode=yes" ${pkgs.nix}/bin/nix + --extra-experimental-features + "nix-command flakes" copy --to ${lib.escapeShellArg cfg.target} diff --git a/modules/nix-impatient.nix b/modules/nix-impatient.nix new file mode 100644 index 0000000..8980795 --- /dev/null +++ b/modules/nix-impatient.nix @@ -0,0 +1,209 @@ +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.services.nix-impatient; + updateInputs = lib.concatMapStringsSep " " lib.escapeShellArg cfg.updateInputs; + nixImpatient = pkgs.writeShellApplication { + name = "nix-impatient"; + runtimeInputs = with pkgs; [ + coreutils + findutils + git + gnugrep + nix + openssh + ]; + text = '' + repository=${lib.escapeShellArg cfg.repository} + branch=${lib.escapeShellArg cfg.branch} + flake_output=${lib.escapeShellArg cfg.flakeOutput} + retention_days=${lib.escapeShellArg (builtins.toString cfg.retentionDays)} + state_dir="''${STATE_DIRECTORY:-/var/lib/nix-impatient}" + checkout="$state_dir/checkout" + roots="$state_dir/roots" + last_success="$state_dir/last-successful-revision" + update_inputs=( ${updateInputs} ) + nix_command=(nix --extra-experimental-features "nix-command flakes") + + rm -rf "$checkout" + mkdir -p "$roots" + trap 'rm -rf "$checkout"' EXIT + + echo "Cloning $branch from $repository" + git clone --quiet --branch "$branch" --single-branch "$repository" "$checkout" + git -C "$checkout" config user.name ${lib.escapeShellArg cfg.commitName} + git -C "$checkout" config user.email ${lib.escapeShellArg cfg.commitEmail} + git -C "$checkout" config commit.gpgSign false + + base_revision="$(git -C "$checkout" rev-parse HEAD)" + "''${nix_command[@]}" flake update "''${update_inputs[@]}" --flake "$checkout" + + unexpected_changes="$(git -C "$checkout" diff --name-only | grep -vx 'flake.lock' || true)" + if [ -n "$unexpected_changes" ]; then + echo "nix flake update unexpectedly changed files other than flake.lock:" >&2 + echo "$unexpected_changes" >&2 + exit 1 + fi + + if ! git -C "$checkout" diff --quiet -- flake.lock; then + git -C "$checkout" add flake.lock + git -C "$checkout" commit --no-verify --message ${lib.escapeShellArg cfg.commitMessage} + fi + + candidate_revision="$(git -C "$checkout" rev-parse HEAD)" + candidate_root="$roots/$candidate_revision" + find "$roots" -mindepth 1 -maxdepth 1 -type l \ + -mtime "+$retention_days" ! -name "$candidate_revision" -delete + + if [ -e "$candidate_root" ] && [ -f "$last_success" ] \ + && [ "$(cat "$last_success")" = "$candidate_revision" ]; then + echo "$candidate_revision is already the latest successful build" + exit 0 + fi + + echo "Checking candidate $candidate_revision" + "''${nix_command[@]}" flake check "$checkout" \ + --keep-going \ + --max-jobs 0 \ + --no-update-lock-file \ + -L + + echo "Building $flake_output on the remote builder pool" + "''${nix_command[@]}" build "$checkout#$flake_output" \ + --keep-going \ + --max-jobs 0 \ + --no-update-lock-file \ + --out-link "$candidate_root" \ + -L + + if [ "$candidate_revision" != "$base_revision" ]; then + remote_revision="$(git ls-remote --exit-code "$repository" "refs/heads/$branch" | cut -f1)" + if [ "$remote_revision" != "$base_revision" ]; then + echo "$branch changed from $base_revision to $remote_revision during the build; refusing to publish" >&2 + rm -f "$candidate_root" + exit 1 + fi + + echo "Publishing $candidate_revision to $branch" + git -C "$checkout" push origin "$candidate_revision:refs/heads/$branch" + fi + + printf '%s\n' "$candidate_revision" > "$last_success.tmp" + mv "$last_success.tmp" "$last_success" + + echo "Successfully warmed and retained $candidate_revision" + ''; + }; +in +{ + options.services.nix-impatient = { + enable = lib.mkEnableOption "periodic flake input updates and distributed cache-warming builds"; + + repository = lib.mkOption { + type = lib.types.str; + default = "/srv/src/nixos.git"; + description = "Bare Git repository to update after a successful build."; + }; + + branch = lib.mkOption { + type = lib.types.str; + default = "master"; + description = "Branch from which candidates are created and to which they are published."; + }; + + flakeOutput = lib.mkOption { + type = lib.types.str; + default = "nix-impatient"; + description = "Flake package output containing the closures to cache."; + }; + + updateInputs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Inputs to update. An empty list updates every updateable input."; + }; + + onCalendar = lib.mkOption { + type = lib.types.str; + default = "*-*-* 00/4:00:00"; + description = "systemd calendar expression controlling periodic runs."; + }; + + randomizedDelaySec = lib.mkOption { + type = lib.types.str; + default = "15m"; + description = "Maximum random delay added to each scheduled run."; + }; + + retentionDays = lib.mkOption { + type = lib.types.ints.positive; + default = 14; + description = "Days to retain successful build roots other than the latest one."; + }; + + commitName = lib.mkOption { + type = lib.types.str; + default = "nix-impatient"; + description = "Git author and committer name for lock file updates."; + }; + + commitEmail = lib.mkOption { + type = lib.types.str; + default = "misaki@ngp.computer"; + description = "Git author and committer email for lock file updates."; + }; + + commitMessage = lib.mkOption { + type = lib.types.str; + default = "Update flake inputs"; + description = "Commit message for successful lock file updates."; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.nix-impatient = { + description = "Update flake inputs and warm cache.ngp.computer"; + after = [ + "network-online.target" + "nix-daemon.service" + ]; + wants = [ "network-online.target" ]; + environment = { + HOME = "/home/noah"; + GIT_SSH_COMMAND = "${pkgs.openssh}/bin/ssh -o BatchMode=yes"; + GIT_AUTHOR_NAME = cfg.commitName; + GIT_AUTHOR_EMAIL = cfg.commitEmail; + GIT_COMMITTER_NAME = cfg.commitName; + GIT_COMMITTER_EMAIL = cfg.commitEmail; + }; + serviceConfig = { + Type = "oneshot"; + User = "noah"; + Group = "users"; + StateDirectory = "nix-impatient"; + StateDirectoryMode = "0750"; + WorkingDirectory = "/var/lib/nix-impatient"; + ExecStart = "${nixImpatient}/bin/nix-impatient"; + Nice = 10; + IOSchedulingClass = "best-effort"; + IOSchedulingPriority = 7; + TimeoutStartSec = "infinity"; + }; + }; + + systemd.timers.nix-impatient = { + description = "Periodically update flake inputs and warm cache.ngp.computer"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = cfg.onCalendar; + RandomizedDelaySec = cfg.randomizedDelaySec; + Persistent = true; + Unit = "nix-impatient.service"; + }; + }; + }; +} diff --git a/modules/nix-remote-builders.nix b/modules/nix-remote-builders.nix index ec69fc7..a159b53 100644 --- a/modules/nix-remote-builders.nix +++ b/modules/nix-remote-builders.nix @@ -4,6 +4,7 @@ let { name = "shizuri"; hostName = "192.168.1.15"; + publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIaMdVtl8UlDa9kI/PO62Glu/PeJXfgXNsVg92b+BibE"; sshUser = "nixremote"; sshKey = "/root/.ssh/nix-remote-builder"; system = "x86_64-linux"; @@ -20,6 +21,7 @@ let { name = "odin"; hostName = "192.168.1.6"; + publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJIuvOXEK7M2i/Q8FeableBS+L20zwQpLetOuFGUhba2"; sshUser = "nixremote"; sshKey = "/root/.ssh/nix-remote-builder"; system = "x86_64-linux"; @@ -36,6 +38,7 @@ let { name = "othinus"; hostName = "192.168.1.4"; + publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICed8x4g/8BA5J1DnkVZWcjyR8RO8QbvQJJiCjglfDbB"; sshUser = "nixremote"; sshKey = "/root/.ssh/nix-remote-builder"; system = "x86_64-linux"; @@ -55,7 +58,24 @@ in nix.distributedBuilds = true; nix.settings.builders-use-substitutes = true; - nix.buildMachines = map (builder: builtins.removeAttrs builder [ "name" ]) ( - lib.filter (builder: builder.name != config.networking.hostName) builders + programs.ssh.knownHosts = builtins.listToAttrs ( + map (builder: { + name = builder.name; + value = { + hostNames = [ + builder.name + builder.hostName + ]; + inherit (builder) publicKey; + }; + }) builders ); + + nix.buildMachines = map ( + builder: + builtins.removeAttrs builder [ + "name" + "publicKey" + ] + ) (lib.filter (builder: builder.name != config.networking.hostName) builders); }