From c9ea89a3ee98ed82de3dcf4a62c480d64f20d7ea Mon Sep 17 00:00:00 2001
From: Isaac Corbrey <isaac@isaaccorbrey.com>
Date: Sat, 3 Oct 2026 08:31:56 -0400
Subject: [PATCH] nvidia: Limit the free buffer pool for all GL-heavy clients

Scoping the profile to niri alone kept the compositor flat, but Xwayland,
quickshell, and Electron GPU processes still exhausted NVKMS allocations
and froze or crashed.
---
 modules/nixos/system/nvidia.nix | 50 ++++++++++++++++++++++++---------
 1 file changed, 36 insertions(+), 14 deletions(-)

diff --git a/modules/nixos/system/nvidia.nix b/modules/nixos/system/nvidia.nix
index b86cf69..ae5e36c 100644
--- a/modules/nixos/system/nvidia.nix
+++ b/modules/nixos/system/nvidia.nix
@@ -24,25 +24,47 @@
         powerManagement.enable = false;
       };
 
-      # Cap NVIDIA's free-buffer heap reuse for the niri process so the
-      # compositor doesn't drift toward ~1 GiB of VRAM over time. The driver
-      # otherwise fails to return freed buffers to the pool on Wayland
-      # compositors; niri should sit around ~100 MiB.
+      # Cap NVIDIA's free-buffer heap reuse for every GL-heavy Wayland (and
+      # Xwayland) client so none of them drift toward ~1 GiB+ of VRAM over
+      # time. The driver otherwise fails to return freed buffers to the pool
+      # on Wayland compositors and comparably-behaved EGL/GL clients; niri
+      # should sit around ~100 MiB.
+      #
+      # This was originally scoped to just "niri" (the wiki's documented
+      # fix), which does keep the compositor itself flat. But the same
+      # driver heuristic starves *every* other GL client independently, and
+      # an unpatched client eventually can't get a NVKMS GEM allocation at
+      # all ("Failed to allocate NVKMS memory for GEM object" in dmesg),
+      # which presents as that client freezing or crashing outright:
+      #   - Xwayland: hosts Steam (X11-only client) via xwayland-satellite;
+      #     resizing/rendering X11 windows was the original upstream repro.
+      #   - quickshell (DankMaterialShell): repaints layer-shell surfaces
+      #     across all 3 monitors continuously, same pattern as kwin_wayland
+      #     in the upstream thread.
+      #   - Electron/Chromium GPU processes (Discord, Fastmail, and any
+      #     other Flatpak Electron app): their GPU process hitting this
+      #     starves/crashes silently, which can leave the whole window
+      #     unresponsive or (for Fastmail specifically) stuck mid-launch
+      #     since the window never gets to paint.
       #
       # Upstream: https://github.com/NVIDIA/egl-wayland/issues/126#issuecomment-2379945259
       # Niri wiki: https://github.com/niri-wm/niri/wiki/Nvidia#high-vram-usage-fix
       #
-      # Picked up by the driver at niri startup; restart the session after
-      # any change.
+      # Picked up by the driver at process startup; restart the affected
+      # app (or log out/in for niri/Xwayland) after any change here.
       environment.etc."nvidia/nvidia-application-profiles-rc.d/50-niri-vram.json".text = builtins.toJSON {
-        rules = [
-          {
-            pattern = {
-              feature = "procname";
-              matches = "niri";
-            };
-            profile = "Limit Free Buffer Pool On Wayland Compositors";
-          }
+        rules = map (procname: {
+          pattern = {
+            feature = "procname";
+            matches = procname;
+          };
+          profile = "Limit Free Buffer Pool On Wayland Compositors";
+        }) [
+          "niri"
+          "Xwayland"
+          "quickshell"
+          "Discord"
+          "fastmail"
         ];
         profiles = [
           {
-- 
2.51.2