From ed91c3204cce772e10dfb70ee8550c9f6bb79280 Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Tue, 14 Jul 2026 01:03:04 -0500 Subject: [PATCH] contain oversized bootstrap repos --- README.md | 6 ++++-- docs/upstream-bootstrap-spec.md | 4 +++- src/internal/bootstrap/engine.zig | 28 ++++++++++++++++++++++------ 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 3f1f6e8..3842f86 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,10 @@ Worker count is a concurrency control, not a memory promise. The byte flag is the memory promise: all backfill-worker allocations block behind that global budget, and an oversized individual repo fails before it can monopolize the budget. Defaults request up to 16 workers and provide 8 GiB; effective -concurrency is derived from a 1.25 GiB worst-case worker allowance (six workers -at those defaults), so peer-held allocations cannot deadlock. Backfill progress, active workers, +concurrency is derived from a 2 GiB worst-case worker allowance (four workers +at those defaults), so peer-held allocations cannot deadlock. A repository +that exceeds its bounded conversion envelope is recorded as failed for retry +instead of terminating the bootstrap. Backfill progress, active workers, queue depth, current transient bytes, and peak transient bytes are exported on `/metrics`. diff --git a/docs/upstream-bootstrap-spec.md b/docs/upstream-bootstrap-spec.md index 69be1d6..004ad41 100644 --- a/docs/upstream-bootstrap-spec.md +++ b/docs/upstream-bootstrap-spec.md @@ -58,7 +58,9 @@ until steady_state. failed-repo retry loop: 4h interval, 16 workers, 4/host, CAR index, and conversion arena draws from one configurable global byte budget (8 GiB by default); worker count alone is never presented as a memory bound. Effective concurrency is capped so each active worker can reach its - 1.25 GiB worst-case allowance without allocator deadlock (six at the defaults). + 2 GiB worst-case allowance without allocator deadlock (four at the defaults). + Exhausting one worker's bounded conversion envelope sends that repository + to the durable retry path instead of terminating bootstrap. getRepo uses the upstream-shaped three-attempt transient/429 retry envelope (1s base, 30s ceiling, Retry-After honored). - Stream uses one RocksDB metadata database at `meta.rocksdb`, preserving the diff --git a/src/internal/bootstrap/engine.zig b/src/internal/bootstrap/engine.zig index f99302e..01a818a 100644 --- a/src/internal/bootstrap/engine.zig +++ b/src/internal/bootstrap/engine.zig @@ -29,10 +29,12 @@ const log = std.log.scoped(.stream); const list_page_limit = 1000; const max_repo_car_bytes = 512 * 1024 * 1024; -// Body + CAR index/conversion allowance. Effective concurrency is chosen so -// every worker can reach this ceiling simultaneously without blocking the -// shared allocator behind allocations held by its peers. -const max_worker_bytes = 2 * max_repo_car_bytes + 256 * 1024 * 1024; +// Body + CAR index/MST conversion allowance. Real network repos demonstrated +// that 1.25 GiB is not enough headroom for a sub-512 MiB CAR once the arena's +// growth and CID index coexist with the response body. Effective concurrency +// is chosen so every worker can reach this ceiling simultaneously without +// blocking the shared allocator behind allocations held by its peers. +const max_worker_bytes = 2 * 1024 * 1024 * 1024; pub const default_workers: usize = 16; pub const default_max_inflight_bytes: usize = 8 * 1024 * 1024 * 1024; @@ -233,7 +235,12 @@ const Work = struct { var local: Stats = .{}; const rev = downloadOne(self.allocator, allocator, self.io, &client, job.did, self.archive, &local) catch |err| { if (self.config.metrics) |m| _ = m.backfill_workers_active.fetchSub(1, .monotonic); - if (err == error.OutOfMemory or err == error.ArchiveAppendFailed) { + // This allocator is deliberately bounded. Exhausting a + // worker's slice means this repo exceeded the conversion + // envelope, not that the process or node is out of memory. + // Persist it as failed for the retry path instead of + // crash-looping the whole bootstrap on the same DID. + if (isFatalDownloadError(err)) { self.finishFatal(err); } else { log.warn("backfill {s} failed: {s}", .{ job.did, @errorName(err) }); @@ -284,6 +291,10 @@ const Work = struct { const WorkerLimits = struct { count: usize, bytes: usize }; +fn isFatalDownloadError(err: anyerror) bool { + return err == error.ArchiveAppendFailed; +} + fn effectiveWorkerLimits(requested: usize, budget: usize) WorkerLimits { const count = @min(requested, @max(budget / max_worker_bytes, 1)); return .{ .count = count, .bytes = @min(max_worker_bytes, budget / count) }; @@ -384,7 +395,7 @@ pub fn run( test "worker limits cannot deadlock the shared byte budget" { const limits = effectiveWorkerLimits(16, default_max_inflight_bytes); - try std.testing.expectEqual(@as(usize, 6), limits.count); + try std.testing.expectEqual(@as(usize, 4), limits.count); try std.testing.expectEqual(max_worker_bytes, limits.bytes); try std.testing.expect(limits.count * limits.bytes <= default_max_inflight_bytes); @@ -393,6 +404,11 @@ test "worker limits cannot deadlock the shared byte budget" { try std.testing.expectEqual(@as(usize, 512 * 1024 * 1024), small.bytes); } +test "bounded worker exhaustion fails one repo without killing bootstrap" { + try std.testing.expect(!isFatalDownloadError(error.OutOfMemory)); + try std.testing.expect(isFatalDownloadError(error.ArchiveAppendFailed)); +} + const list_repos_cursor_key = "relay/list_repos_cursor"; const bootstrap_cursor_key = "bootstrap/last_listrepos_cursor"; -- 2.54.0