From 13953dffe9359a528d10114f371f47172170be32 Mon Sep 17 00:00:00 2001
From: dawn <90008@gaze.systems>
Date: Tue, 05 May 2026 02:13:06 +0000
Subject: [PATCH] [backfill] dont hold store / mst tree over db commit, drop early when we can

---
 AGENTS.md | 44 ++++++++++++++++++++++++++++++++++++++++++++
 flake.nix | 1 +
 src/backfill/mod.rs | 17 +++++++++++++++--
 src/control/mod.rs | 2 +-
 4 file(s) changed, 61 insertion(s)(+), 3 deletion(s)(-)

diff --git a/AGENTS.md b/AGENTS.md
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -139,3 +139,47 @@
 - `[ingest] implement backfill buffer replay`
 - `[api] add accurate count parameter to stats`
 - `[db] migrate block storage to msgpack`
+
+
+<!-- headroom:rtk-instructions -->
+# RTK (Rust Token Killer) - Token-Optimized Commands
+
+When running shell commands, **always prefix with `rtk`**. This reduces context
+usage by 60-90% with zero behavior change. If rtk has no filter for a command,
+it passes through unchanged — so it is always safe to use.
+
+## Key Commands
+```bash
+# Git (59-80% savings)
+rtk git status          rtk git diff            rtk git log
+
+# Files & Search (60-75% savings)
+rtk ls <path>           rtk read <file>         rtk grep <pattern>
+rtk find <pattern>      rtk diff <file>
+
+# Test (90-99% savings) — shows failures only
+rtk pytest tests/       rtk cargo test          rtk test <cmd>
+
+# Build & Lint (80-90% savings) — shows errors only
+rtk tsc                 rtk lint                rtk cargo build
+rtk prettier --check    rtk mypy                rtk ruff check
+
+# Analysis (70-90% savings)
+rtk err <cmd>           rtk log <file>          rtk json <file>
+rtk summary <cmd>       rtk deps                rtk env
+
+# GitHub (26-87% savings)
+rtk gh pr view <n>      rtk gh run list         rtk gh issue list
+
+# Infrastructure (85% savings)
+rtk docker ps           rtk kubectl get         rtk docker logs <c>
+
+# Package managers (70-90% savings)
+rtk pip list            rtk pnpm install        rtk npm run <script>
+```
+
+## Rules
+- In command chains, prefix each segment: `rtk git add . && rtk git commit -m "msg"`
+- For debugging, use raw command without rtk prefix
+- `rtk proxy <cmd>` runs command without filtering but tracks usage
+<!-- /headroom:rtk-instructions -->
diff --git a/flake.nix b/flake.nix
--- a/flake.nix
+++ b/flake.nix
@@ -69,6 +69,7 @@
               clang
               wild
               psmisc
+              rtk
               (pkgs.writeShellApplication {
                 name = "verbiage";
                 runtimeInputs = [ inputs'.verbiage.packages.default ];
diff --git a/src/backfill/mod.rs b/src/backfill/mod.rs
--- a/src/backfill/mod.rs
+++ b/src/backfill/mod.rs
@@ -520,7 +520,12 @@
     trace!(elapsed = %start.elapsed().as_secs_f32(), "parsed car");
 
     let start = Instant::now();
+    let root_cid = parsed.root;
     let store = Arc::new(MemoryBlockStore::new_from_blocks(parsed.blocks));
+    // parsed.blocks was moved into the store; parsed.root is now root_cid. drop the raw
+    // CAR bytes here so we don't hold two copies of the block data (raw + parsed) for
+    // the entire duration of the spawn_blocking call below.
+    drop(car_bytes);
     trace!(
         blocks = store.len(),
         elapsed = ?start.elapsed(),
@@ -529,7 +534,7 @@
 
     // 4. parse root commit to get mst root
     let root_bytes = store
-        .get(&parsed.root)
+        .get(&root_cid)
         .await
         .into_diagnostic()?
         .ok_or_else(|| miette::miette!("root block missing from CAR"))?;
@@ -575,7 +580,11 @@
             let mut added_blocks = 0;
             let mut collection_counts: HashMap<SmolStr, u64> = HashMap::new();
             let mut batch = app_state.db.inner.batch();
-            let store = mst.storage();
+            // clone the Arc so we hold an independent reference to the block store,
+            // allowing mst (and its entire loaded node tree) to be freed immediately
+            // rather than surviving until the end of spawn_blocking.
+            let store = mst.storage().clone();
+            drop(mst);
 
             let prefix = keys::record_prefix_did(&did);
             let mut existing_cids: HashMap<(SmolStr, DbRkey), SmolStr> = HashMap::new();
@@ -698,6 +707,10 @@
                     count += 1;
                 }
             }
+
+            // all blocks have been read; free the MemoryBlockStore now so it does not
+            // survive through the final batch commit or any fjall backpressure stall.
+            drop(store);
 
             // remove any remaining existing records (they weren't in the new MST)
             for ((collection, rkey), cid) in existing_cids {
diff --git a/src/control/mod.rs b/src/control/mod.rs
--- a/src/control/mod.rs
+++ b/src/control/mod.rs
@@ -126,7 +126,7 @@
     pub fn resolver(&self) -> &crate::resolver::Resolver {
         &self.state.resolver
     }
-    
+
     /// open the database and configure hydrant from `config`.
     ///
     /// this sets up the database, applies any filter configuration from `config`, and

--
tangled.sh