diff --git a/common/base.nix b/common/base.nix index d26a08c..2ffe648 100644 --- a/common/base.nix +++ b/common/base.nix @@ -8,6 +8,8 @@ imports = [ (modulesPath + "/installer/scan/not-detected.nix") (modulesPath + "/profiles/qemu-guest.nix") + ./nix-gc.nix + ./logs.nix ]; boot.loader.grub = { @@ -20,6 +22,7 @@ services.prometheus.exporters.node = { enable = true; openFirewall = false; + enabledCollectors = ["systemd"]; }; networking.firewall.interfaces."tailscale0".allowedTCPPorts = [ 9100 ]; diff --git a/common/logs.nix b/common/logs.nix new file mode 100644 index 0000000..05f73ed --- /dev/null +++ b/common/logs.nix @@ -0,0 +1,17 @@ +{ + config, + lib, + ... +}: { + services.journald.extraConfig = '' + SystemMaxUse=2G + MaxRetentionSec=30d + ''; + + services.logrotate.settings = lib.mkIf config.services.nginx.enable { + nginx = { + frequency = "daily"; + rotate = 30; + }; + }; +} diff --git a/common/nix-gc.nix b/common/nix-gc.nix new file mode 100644 index 0000000..51f6c96 --- /dev/null +++ b/common/nix-gc.nix @@ -0,0 +1,220 @@ +{ + config, + lib, + pkgs, + ... +}: let + cfg = config.tangled.nixStoreGc; + systemProfile = "/nix/var/nix/profiles/system"; + + collectorPriority = { + Nice = 19; + IOSchedulingClass = "idle"; + }; + + bootloaderRefreshAttempts = 3; + bootloaderRefreshDelaySec = 60; + + stateDirectory = "nix-store-gc"; + refreshPending = "/var/lib/${stateDirectory}/bootloader-refresh-pending"; + + nix-env = lib.getExe' config.nix.package "nix-env"; + wc = lib.getExe' pkgs.coreutils "wc"; + sleep = lib.getExe' pkgs.coreutils "sleep"; + rm = lib.getExe' pkgs.coreutils "rm"; + + trimScript = lib.mapNullable (retention: + pkgs.writeShellScript "nix-store-gc-trim-generations" '' + set -euo pipefail + + count_generations() { + ${nix-env} --profile ${systemProfile} --list-generations | ${wc} -l + } + + refresh_bootloader() { + local attempt=1 + until ${systemProfile}/bin/switch-to-configuration boot; do + if [ "$attempt" -ge ${toString bootloaderRefreshAttempts} ]; then + return 1 + fi + attempt=$(( attempt + 1 )) + ${sleep} ${toString bootloaderRefreshDelaySec} + done + ${rm} -f ${refreshPending} + } + + generations_before="$(count_generations)" + ${nix-env} --profile ${systemProfile} --delete-generations ${retention} || true + generations_after="$(count_generations)" + + if [ "$generations_after" -lt "$generations_before" ]; then + : > ${refreshPending} + fi + + if [ -e ${refreshPending} ] && ! refresh_bootloader; then + echo "The bootloader menu still lists deleted generations, so we won't collect their closures. Set tangled.nixStoreGc.generationRetention = null on this host to collect anyway." + exit 1 + fi + '') + cfg.generationRetention; + + trimCommand = lib.optionalString (trimScript != null) "${trimScript}"; +in { + options.tangled.nixStoreGc = { + generationRetention = lib.mkOption { + type = lib.types.nullOr (lib.types.strMatching "[1-9][0-9]*d|\\+[1-9][0-9]*|old"); + default = "+20"; + description = "Argument for nix-env --delete-generations, which unroots older system generations so a collection can reclaim their closures. It accepts a count of recent generations to retain such as \"+20\", an age such as \"30d\", or \"old\" for everything but the current one. A run that deletes a generation rewrites the bootloader menu before it collects. We abort the run when that rewrite fails, so the menu never lists a generation whose closure a collection removed. With null we retain every generation, and a collection then frees only paths that are already dead."; + }; + + scheduled.enable = + lib.mkEnableOption "a weekly collection of every dead store path" + // {default = true;}; + + pressure = { + enable = lib.mkEnableOption "a quarter-hourly check that collects once usage reaches startAtPercent"; + + startAtPercent = lib.mkOption { + type = lib.types.ints.between 1 100; + default = 90; + description = "Usage of the filesystem containing /nix/store at which a run starts collecting, computed as used / (used + available) and rounded up. We exclude the blocks a filesystem reserves for root from both terms, so this matches what df prints under Use%."; + }; + + stopAtPercent = lib.mkOption { + type = lib.types.ints.between 1 100; + default = 80; + description = "Usage a run frees down to before it stops, on the same scale as startAtPercent."; + }; + + maxFreedGib = lib.mkOption { + type = lib.types.ints.positive; + default = 20; + description = "Upper bound in gibibytes on what one run frees. A run that reaches this bound stops above stopAtPercent, and the next run continues."; + }; + + retryAfterNoProgressSec = lib.mkOption { + type = lib.types.ints.positive; + default = 6 * 60 * 60; + description = "Seconds before we collect again after a run that reached startAtPercent and deleted no store path. A collection frees only store paths, so data elsewhere on the filesystem can keep usage above that threshold. The delay stops such a host from walking the whole store every quarter hour with nothing to reclaim. We schedule no delay after a run that deleted any path. We also collect before the delay expires once usage exceeds what the last such run measured."; + }; + }; + }; + + config = lib.mkMerge [ + { + assertions = lib.optional (cfg.scheduled.enable || cfg.pressure.enable) { + assertion = (cfg.generationRetention != null) -> config.system.switch.enable; + message = "tangled.nixStoreGc.generationRetention deletes system generations. Refreshing the bootloader menu afterwards needs system.switch.enable, so set generationRetention = null on a host without it."; + }; + } + + (lib.mkIf cfg.scheduled.enable { + nix.gc = { + automatic = true; + dates = "weekly"; + randomizedDelaySec = "45min"; + }; + + systemd.services.nix-gc.serviceConfig = + collectorPriority + // { + ExecStartPre = lib.optional (trimScript != null) trimCommand; + StateDirectory = stateDirectory; + }; + }) + + (lib.mkIf cfg.pressure.enable { + assertions = [ + { + assertion = cfg.pressure.stopAtPercent < cfg.pressure.startAtPercent; + message = "tangled.nixStoreGc.pressure.stopAtPercent (${toString cfg.pressure.stopAtPercent}) must be below startAtPercent (${toString cfg.pressure.startAtPercent}), otherwise a run computes a negative number of bytes to free."; + } + ]; + + systemd.services.nix-store-pressure-gc = { + description = "collect nix store garbage under disk pressure"; + + path = [ + pkgs.coreutils + config.nix.package + ]; + + serviceConfig = + collectorPriority + // { + Type = "oneshot"; + StateDirectory = stateDirectory; + }; + + script = '' + set -euo pipefail + + read_store_usage() { + local columns used avail + columns="$(df -B1 --output=used,avail /nix/store | tail -n 1)" + read -r used avail <<<"$columns" + + used_bytes="$used" + writable_bytes=$(( used + avail )) + usage=$(( (used * 100 + writable_bytes - 1) / writable_bytes )) + } + + stalled="$STATE_DIRECTORY/stalled-at" + read_store_usage + + if [ "$usage" -lt "${toString cfg.pressure.startAtPercent}" ]; then + rm -f "$stalled" + exit 0 + fi + + stalled_at="$(stat -c %Y "$stalled" 2>/dev/null || echo 0)" + stalled_usage="$(cat "$stalled" 2>/dev/null || true)" + : "''${stalled_usage:=0}" + + if [ $(( $(date +%s) - stalled_at )) -lt ${toString cfg.pressure.retryAfterNoProgressSec} ] && [ "$usage" -le "$stalled_usage" ]; then + echo "/nix/store is ''${usage}% full and the last run found no dead paths at ''${stalled_usage}%, so this run exits without collecting" + exit 0 + fi + + target_used=$(( writable_bytes * ${toString cfg.pressure.stopAtPercent} / 100 )) + # only free up to our target when possible + bytes_to_free=$(( used_bytes - target_used )) + + max_freed=$(( ${toString cfg.pressure.maxFreedGib} * 1024 * 1024 * 1024 )) + if [ "$bytes_to_free" -gt "$max_freed" ]; then + bytes_to_free="$max_freed" + fi + + ${trimCommand} + + echo "/nix/store is ''${usage}% full, so this run frees up to $bytes_to_free bytes" + report="$(nix-collect-garbage --max-freed "$bytes_to_free" | tail -n 1)" + deleted="''${report%% *}" + + read_store_usage + + if [ "$usage" -lt "${toString cfg.pressure.startAtPercent}" ]; then + echo "/nix/store is down to ''${usage}% full after this run deleted $deleted store paths" + rm -f "$stalled" + elif [ "$deleted" -gt 0 ]; then + echo "/nix/store is still ''${usage}% full after this run deleted $deleted store paths, so the next run continues" + rm -f "$stalled" + else + echo "/nix/store is still ''${usage}% full and this run found no dead paths, so we wait ${toString cfg.pressure.retryAfterNoProgressSec}s before collecting again, or until usage exceeds ''${usage}%" + printf '%s\n' "$usage" > "$stalled" + fi + ''; + }; + + systemd.timers.nix-store-pressure-gc = { + wantedBy = ["timers.target"]; + + timerConfig = { + OnBootSec = "15m"; + OnUnitActiveSec = "15m"; + RandomizedDelaySec = "5m"; + }; + }; + }) + ]; +} diff --git a/hosts/spindle-hel/services/cache.nix b/hosts/spindle-hel/services/cache.nix index f74e022..9ec921c 100644 --- a/hosts/spindle-hel/services/cache.nix +++ b/hosts/spindle-hel/services/cache.nix @@ -1,10 +1,5 @@ -{pkgs, ...}: +{...}: let - oneGb = 1024 * 1024 * 1024; - maxFreedPerRun = 50 * oneGb; - startGcAt = 94; - stopGcAt = 88; - port = 5000; in { @@ -29,59 +24,13 @@ in # have retention based cleanup instead of this, because nix doesn't cleanup # retention based (this would also allow us to limit cache per-user and so on # though so its useful anyway) - systemd.services.nix-store-pressure-gc = { - description = "garbage collect nix store under disk pressure"; - - path = [ - pkgs.bash - pkgs.coreutils - pkgs.nix - ]; - - serviceConfig = { - Type = "oneshot"; - Nice = 19; - IOSchedulingClass = "idle"; - }; - - script = '' - set -euo pipefail - - read -r size used percent < <( - df -B1 --output=size,used,pcent /nix/store | tail -n 1 - ) - - usage="''${percent%\%}" - usage="''${usage//[^0-9]/}" - - if [ "$usage" -lt "${toString startGcAt}" ]; then - exit 0 - fi - - target_used=$(( size * ${toString stopGcAt} / 100 )) - # only free up to our target when possible - bytes_to_free=$(( used - target_used )) - if [ "$bytes_to_free" -le 0 ]; then - exit 0 - fi - max_freed="${toString maxFreedPerRun}" - if [ "$bytes_to_free" -gt "$max_freed" ]; then - bytes_to_free="$max_freed" - fi - - echo "/nix/store is ''${usage}% full, so freeing up to $bytes_to_free bytes" - nix-collect-garbage --max-freed "$bytes_to_free" - ''; - }; - - systemd.timers.nix-store-pressure-gc = { - wantedBy = ["timers.target"]; - - timerConfig = { - OnBootSec = "15m"; - OnUnitActiveSec = "15m"; - RandomizedDelaySec = "5m"; - Persistent = true; + tangled.nixStoreGc = { + scheduled.enable = false; + pressure = { + enable = true; + startAtPercent = 94; + stopAtPercent = 88; + maxFreedGib = 50; }; }; }