From d9c4f9001754fce701eaa332867b7ddc93fe6561 Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Fri, 31 Jul 2026 03:56:39 +0000 Subject: [PATCH] backfill: repin zat 0.3.23, fix the gzip transfer buffer The gzip fix in c8c8c5d passed a zero-length transfer buffer to readerDecompressing. That buffer feeds the decompressor its compressed input, and an empty one aborts the process on the first gzip response instead of returning an error — so the fix would have crashed the crawl the moment a PDS honoured the request. Caught by a loopback gzip test in zat, where the same shape failed. zat 0.3.23 replays a connection closed before it answers, so the listRepos retry added in 425ec73 moves down to the transport where it belongs and comes back out of here. Co-Authored-By: Claude Opus 5 (1M context) --- build.zig.zon | 4 ++-- src/internal/ingest/backfill/engine.zig | 37 +++++++++---------------------------- src/internal/ingest/backfill/fetched_car.zig | 12 ++++++++++-- 3 file(s) changed, 21 insertion(s)(+), 32 deletion(s)(-) diff --git a/build.zig.zon b/build.zig.zon --- a/build.zig.zon +++ b/build.zig.zon @@ -5,8 +5,8 @@ .minimum_zig_version = "0.16.0", .dependencies = .{ .zat = .{ - .url = "git+https://tangled.org/zat.dev/zat#955e9ca9afb41d66d41e998803f6fcfc8fc4330f", - .hash = "zat-0.3.22-5PuC7j54CwAB6eBKHagm0rbfuG0jOg068iGufg1Jz6Wz", + .url = "git+https://tangled.org/zat.dev/zat#409711c0ff5465211f3ff3eb8a5332332da09b3b", + .hash = "zat-0.3.23-5PuC7lHLCwBaCfw70KN-79Hgz2_icC83h7HGHfp789G-", }, .websocket = .{ .url = "https://github.com/zzstoatzz/websocket.zig/archive/73429df.tar.gz", diff --git a/src/internal/ingest/backfill/engine.zig b/src/internal/ingest/backfill/engine.zig --- a/src/internal/ingest/backfill/engine.zig +++ b/src/internal/ingest/backfill/engine.zig @@ -1164,36 +1164,17 @@ /// A batch takes ~40 minutes to drain, during which `listRepos` is never /// called, so the client's pooled keep-alive connection goes idle and the -/// relay closes it. The next page reuses that dead socket and `receiveHead` -/// reads zero bytes -> `HttpConnectionClosing`. Left unhandled it unwound to -/// `main` and ended the process at every batch boundary; Docker restarted it -/// and the crawl resumed from the durable cursor, which made a fatal bug look -/// like a scheduling quirk. +/// relay closes it. The next page reused that dead socket and `receiveHead` +/// read zero bytes, which unwound to `main` and ended the process at every +/// batch boundary; Docker restarted it and the crawl resumed from the durable +/// cursor, which made a fatal bug look like a scheduling quirk. /// -/// Zero bytes received means the request was never answered, so retrying is -/// unambiguously safe — nothing was applied. `ade3d5d` fixed the same race on -/// the getRepo path by disabling keep-alive; this path kept it. -const list_page_attempts = 4; - +/// zat 0.3.23 replays that case in the transport, where it belongs, so the +/// retry that used to live here is gone. `ade3d5d` had worked around the same +/// race on the getRepo path by disabling keep-alive; that is now redundant but +/// left alone until it is worth re-measuring. pub fn listReposPage(arena: Allocator, client: *zat.XrpcClient, cursor: ?[]const u8) !repos.ListReposPage { - var attempt: usize = 0; - while (true) { - attempt += 1; - return listReposPageOnce(arena, client, cursor) catch |err| { - // Matched by name rather than by `switch`: the transport's error set - // is inferred through zat, so naming a member it does not currently - // have is a compile error, and a set change would silently drop a - // case. These four all mean "the connection died before an answer", - // which is exactly the safe-to-retry condition. - const name = @errorName(err); - const retryable = std.mem.eql(u8, name, "HttpConnectionClosing") or - std.mem.eql(u8, name, "ConnectionResetByPeer") or - std.mem.eql(u8, name, "BrokenPipe") or - std.mem.eql(u8, name, "EndOfStream"); - if (retryable and attempt < list_page_attempts) continue; - return err; - }; - } + return listReposPageOnce(arena, client, cursor); } fn listReposPageOnce(arena: Allocator, client: *zat.XrpcClient, cursor: ?[]const u8) !repos.ListReposPage { diff --git a/src/internal/ingest/backfill/fetched_car.zig b/src/internal/ingest/backfill/fetched_car.zig --- a/src/internal/ingest/backfill/fetched_car.zig +++ b/src/internal/ingest/backfill/fetched_car.zig @@ -145,7 +145,11 @@ // gzip bytes. Upstream gets this for free from Go's net/http. var decompress: std.http.Decompress = undefined; var decompress_buf: [std.compress.flate.max_window_len]u8 = undefined; - const reader = response.readerDecompressing(&.{}, &decompress, &decompress_buf); + // The transfer buffer feeds the decompressor its compressed input; a + // zero-length one aborts the process on the first gzip response rather + // than returning an error. + var transfer_buf: [8 * 1024]u8 = undefined; + const reader = response.readerDecompressing(&transfer_buf, &decompress, &decompress_buf); if (progress) |p| p.body_started.store(true, .release); var read_buf: [64 * 1024]u8 = undefined; while (true) { @@ -355,7 +359,11 @@ // misclassify a known error as unknown. var decompress: std.http.Decompress = undefined; var decompress_buf: [std.compress.flate.max_window_len]u8 = undefined; - const reader = response.readerDecompressing(&.{}, &decompress, &decompress_buf); + // The transfer buffer feeds the decompressor its compressed input; a + // zero-length one aborts the process on the first gzip response rather + // than returning an error. + var transfer_buf: [8 * 1024]u8 = undefined; + const reader = response.readerDecompressing(&transfer_buf, &decompress, &decompress_buf); _ = reader.streamRemaining(&writer) catch {}; const body = writer.buffered(); -- tangled.sh