From a8720a193c0172f82ada7af88d4e5a366830372d Mon Sep 17 00:00:00 2001
From: dawn <90008@klbr.net>
Date: Sun, 28 Jun 2026 01:16:03 +0300
Subject: [PATCH] [crawler] exclude 404 from throttling and limit describeRepo
 response size

---
 src/crawler/list_repos/checker.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/crawler/list_repos/checker.rs b/src/crawler/list_repos/checker.rs
index 31f9832..ff1b465 100644
--- a/src/crawler/list_repos/checker.rs
+++ b/src/crawler/list_repos/checker.rs
@@ -115,7 +115,7 @@ fn is_throttle_worthy(e: &reqwest::Error) -> bool {
     e.status().is_some_and(|s| {
         // lets not consider internal server errors for throttling
         // a server might be having a hard time on one request, but not on the rest
-        s.as_u16() != 500 && is_status_their_fault(s.as_u16())
+        s.as_u16() != 500 && s.as_u16() != 404 && is_status_their_fault(s.as_u16())
     })
 }
 
@@ -251,7 +251,7 @@ impl SignalChecker {
                 }
             };
 
-            let bytes = match resp.bytes().await {
+            let bytes = match crate::util::read_limited_bytes(resp, 2 * 1024 * 1024).await {
                 Ok(b) => b,
                 Err(e) => {
                     error!(err = %e, "failed to read describeRepo response");
-- 
2.51.2