From 35b4ec3fd06fce6cae21dbb5070c94cc1c54edb8 Mon Sep 17 00:00:00 2001 From: Kieran Klukas Date: Sun, 23 Aug 2026 14:13:45 -0400 Subject: [PATCH] nix: patch daemon abort when a substituter is unreachable prattle's attic going down took every host's ability to build anything with it. nix 2.34 does not fall back: a failing narinfo worker sets the thread pool's quit flag, the next worker logs through TunnelLogger, and that write throws Interrupted from inside a catch block. NixOS/nix#3768. --- flake.nix | 15 +++++ modules/nixos/nix-cache.nix | 15 ++++- ...-daemon-no-interrupt-on-client-write.patch | 59 +++++++++++++++++++ 3 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 patches/nix-2.34-daemon-no-interrupt-on-client-write.patch diff --git a/flake.nix b/flake.nix index 229b1c1..cd38587 100644 --- a/flake.nix +++ b/flake.nix @@ -172,6 +172,21 @@ config.allowUnfree = true; }; + # nix 2.34 aborts the daemon whenever a substituter is unreachable. + # A failing narinfo worker sets the thread pool's quit flag, the + # next worker logs its own error through TunnelLogger, and that + # write throws Interrupted from inside a catch block, unwinding out + # of the worker thread. The client just sees "Nix daemon + # disconnected unexpectedly". NixOS/nix#3768 (open since 2020) and + # NixOS/nix#12871. Drop this once upstream lands a fix. + nixVersions = prev.nixVersions.extend ( + _finalNix: prevNix: { + nixComponents_2_34 = prevNix.nixComponents_2_34.appendPatches [ + ./patches/nix-2.34-daemon-no-interrupt-on-client-write.patch + ]; + } + ); + zmx-binary = prev.callPackage ./packages/zmx.nix { }; bore-auth = prev.callPackage ./packages/bore-auth.nix { }; pear = inputs.pear.packages.${prev.stdenv.hostPlatform.system}.default; diff --git a/modules/nixos/nix-cache.nix b/modules/nixos/nix-cache.nix index a68ff50..2931a58 100644 --- a/modules/nixos/nix-cache.nix +++ b/modules/nixos/nix-cache.nix @@ -3,8 +3,19 @@ # Pull custom-built packages (knot, herald, the tangled bits) from prattle's # tailnet-only attic cache instead of recompiling them on every deploy. The # cache is public, so no token is needed to pull — the tailscale0 firewall is - # the gate. These are extra-* keys, so cache.nixos.org is still tried first and - # an unreachable prattle just falls back to it. + # the gate. + # + # An unreachable prattle does NOT quietly fall back to cache.nixos.org, whatever + # the extra-* keys suggest. Stock nix 2.34 aborts the daemon instead: a failing + # narinfo worker sets the thread pool's quit flag, the next worker logs its own + # error through TunnelLogger, and that write throws Interrupted from inside a + # catch block and unwinds out of the thread. The client sees "Nix daemon + # disconnected unexpectedly" and every host loses the ability to build anything. + # download-attempts, connect-timeout and --fallback were all tried; none help. + # NixOS/nix#3768 (open since 2020) and NixOS/nix#12871. + # + # The overlay in flake.nix patches that crash out, which is what makes this line + # safe to keep. Removing one without the other brings the outage back. nix.settings = { extra-substituters = [ "http://prattle:8091/dots" ]; extra-trusted-public-keys = [ "dots:Mgol9jjaoUcN6pfgLetO3fe/JAm/fVpKXYBZaQ1MhFM=" ]; diff --git a/patches/nix-2.34-daemon-no-interrupt-on-client-write.patch b/patches/nix-2.34-daemon-no-interrupt-on-client-write.patch new file mode 100644 index 0000000..a669a0f --- /dev/null +++ b/patches/nix-2.34-daemon-no-interrupt-on-client-write.patch @@ -0,0 +1,59 @@ +diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc +index 0e6668bc0..e9b61d785 100644 +--- a/src/libstore/daemon.cc ++++ b/src/libstore/daemon.cc +@@ -1012,6 +1012,14 @@ static void performOp( + + void processConnection(ref store, FdSource && from, FdSink && to, TrustedFlag trusted, RecursiveFlag recursive) + { ++ /* Never abandon a write to the client half-way. `TunnelLogger` writes ++ through this sink, and it is called from contexts that cannot cope with ++ an exception, such as `ignoreExceptionExceptInterrupt()` inside a ++ `ThreadPool` worker: the resulting `Interrupted` escapes the thread and ++ terminates the daemon. Logging must not check for interrupts, and a ++ partially written worker protocol message is unrecoverable anyway. */ ++ to.allowInterrupts = false; ++ + #ifndef _WIN32 // TODO need graceful async exit support on Windows? + auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; + (void) monitor; // suppress warning +diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh +index f8b545f49..5537f625d 100644 +--- a/src/libutil/include/nix/util/serialise.hh ++++ b/src/libutil/include/nix/util/serialise.hh +@@ -151,6 +151,14 @@ struct FdSink : BufferedSink + { + Descriptor fd; + size_t written = 0; ++ /** ++ * Whether a blocking write may be aborted by `checkInterrupt()`. ++ * ++ * Interrupting a write leaves a partially written message behind, which ++ * is unrecoverable for framed protocols and for logging. Sinks carrying ++ * either should set this to `false`. ++ */ ++ bool allowInterrupts = true; + + FdSink() + : fd(INVALID_DESCRIPTOR) +@@ -172,6 +180,7 @@ struct FdSink : BufferedSink + fd = s.fd; + s.fd = INVALID_DESCRIPTOR; + written = s.written; ++ allowInterrupts = s.allowInterrupts; + return *this; + } + +diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc +index 4cf09c298..6c92dcfaa 100644 +--- a/src/libutil/serialise.cc ++++ b/src/libutil/serialise.cc +@@ -66,7 +66,7 @@ void FdSink::writeUnbuffered(std::string_view data) + { + written += data.size(); + try { +- writeFull(fd, data); ++ writeFull(fd, data, allowInterrupts); + } catch (SystemError & e) { + _good = false; + throw; -- 2.51.2