diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 7d58307..82d5efd 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,19 +1,19 @@ {"_type":"issue","id":"hydrant-ct8","title":"Default IPv6 bind exposes management API","description":"Finding: Default IPv6 bind exposes management API\nSeverity: HIGH\nOriginal Finding ID: 35ca6b23efd88191b6b48f02dafd0cd9\nIntroducing Commit: 82e340fdce9249e45f46d34150b8c0acea1c62c7\nRelevant Paths: src/main.rs | src/api/mod.rs | docs/getting-started.md\n\nDescription:\nHydrant's main API router contains both public and unauthenticated management endpoints. Before this commit, the binary read HYDRANT_API_PORT and bound only 0.0.0.0:\u003cport\u003e. The new default constructs two wildcard listeners: 0.0.0.0:3000 and [::]:3000. On IPv6-enabled hosts, this opens the full API surface on all IPv6 interfaces by default. Deployments that protected the old IPv4 listener with an IPv4 firewall, NAT rule, or reverse proxy can unintentionally expose the management API on a globally routed IPv6 address. Because those management endpoints include operations such as changing firehose sources, repository tracking, ingestion state, and DB maintenance, a remote attacker reaching the IPv6 listener can alter service behavior or cause denial of service without authentication. The change also ignores the old HYDRANT_API_PORT variable, increasing the chance that upgraded deployments unexpectedly fall back to the new wildcard default.","status":"closed","priority":1,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:31:58Z","created_by":"dawn","updated_at":"2026-06-27T20:34:46Z","closed_at":"2026-06-27T20:34:46Z","close_reason":"not an issue","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"hydrant-jsk","title":"Unvalidated crawler sources enable SSRF and crawler DoS","description":"Finding: Unvalidated crawler sources enable SSRF and crawler DoS\nSeverity: MEDIUM\nOriginal Finding ID: 9f4f41a0383c8191a7bbf595fe317283\nIntroducing Commit: a3658e247c0692bcd7d38fe35f02efe92de56032\nRelevant Paths: src/api/mod.rs | src/api/crawler.rs | src/control.rs | src/crawler/mod.rs | src/crawler/relay.rs\n\nDescription:\nThe commit adds POST /crawler/sources to the main API router. The handler deserializes a url::Url and mode, but does not restrict the scheme, host, IP range, source count, or validate reachability before calling CrawlerHandle::add_source. add_source writes the URL to the database and spawns a producer task. The crawler base_url helper only rewrites ws/wss to http/https and otherwise leaves arbitrary schemes and hosts unchanged. In relay mode, request failures from the crawler source are logged and the loop immediately continues, so a source such as file:///x or an unreachable localhost/internal address can cause a tight error loop; an http(s) URL can also make the service repeatedly contact attacker-selected internal hosts with fixed XRPC paths. This is especially risky if management routes are reachable outside a trusted admin network; even though management APIs are intended to be private, this commit introduces a new persistent outbound-request primitive and amplifies a pre-existing no-backoff crawler error path.","status":"open","priority":2,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:06Z","created_by":"dawn","updated_at":"2026-06-27T20:32:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"hydrant-jsk","title":"Unvalidated crawler sources enable SSRF and crawler DoS","description":"Finding: Unvalidated crawler sources enable SSRF and crawler DoS\nSeverity: MEDIUM\nOriginal Finding ID: 9f4f41a0383c8191a7bbf595fe317283\nIntroducing Commit: a3658e247c0692bcd7d38fe35f02efe92de56032\nRelevant Paths: src/api/mod.rs | src/api/crawler.rs | src/control.rs | src/crawler/mod.rs | src/crawler/relay.rs\n\nDescription:\nThe commit adds POST /crawler/sources to the main API router. The handler deserializes a url::Url and mode, but does not restrict the scheme, host, IP range, source count, or validate reachability before calling CrawlerHandle::add_source. add_source writes the URL to the database and spawns a producer task. The crawler base_url helper only rewrites ws/wss to http/https and otherwise leaves arbitrary schemes and hosts unchanged. In relay mode, request failures from the crawler source are logged and the loop immediately continues, so a source such as file:///x or an unreachable localhost/internal address can cause a tight error loop; an http(s) URL can also make the service repeatedly contact attacker-selected internal hosts with fixed XRPC paths. This is especially risky if management routes are reachable outside a trusted admin network; even though management APIs are intended to be private, this commit introduces a new persistent outbound-request primitive and amplifies a pre-existing no-backoff crawler error path.","status":"closed","priority":2,"issue_type":"bug","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:06Z","created_by":"dawn","updated_at":"2026-06-27T22:04:07Z","started_at":"2026-06-27T21:53:31Z","closed_at":"2026-06-27T22:04:07Z","close_reason":"fixed all 12 reported bugs","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"hydrant-r6q","title":"Unthrottled filtered Jetstream replay can exhaust DB I/O","description":"Finding: Unthrottled filtered Jetstream replay can exhaust DB I/O\nSeverity: MEDIUM\nOriginal Finding ID: b20c3936a0d88191bf76a5a88121a67a\nIntroducing Commit: e36574744e2bd5df2c602de4450ae0c4c101c517\nRelevant Paths: src/api/mod.rs | src/api/jetstream.rs | src/control/jetstream.rs | src/config.rs | src/control/stream.rs\n\nDescription:\nThe public `/subscribe` Jetstream endpoint accepts attacker-controlled cursors and filters. When a cursor is supplied, `jetstream_stream_thread` replays historical Jetstream events up to the current head. In the replay loop, the new pacing logic calls `replay_chunk_size_for()` and only blocks when the outbound mpsc channel is full. However, filters such as `wantedEventTypes=live` reject all historical replay events in `JetstreamSubscriberOptions::wants()`. In that case `jetstream_event_to_bytes()` returns `None`, no messages are sent, the outbound channel stays empty, and the channel-capacity backpressure never activates. Because this commit also changed the default `stream_replay_chunk_pause` from 2ms to zero, the replay loop immediately reads chunk after chunk from the DB until the replay window is exhausted. A remote unauthenticated client can open `/subscribe?cursor=0\u0026wantedEventTypes=live` and force a full-speed historical DB scan while receiving little or no data; multiple such connections can cause substantial disk I/O and CPU exhaustion. This should be throttled independently of output-channel backpressure, or historical replay should be skipped when filters can never match historical events.","status":"closed","priority":2,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:05Z","created_by":"dawn","updated_at":"2026-06-27T20:34:47Z","closed_at":"2026-06-27T20:34:47Z","close_reason":"not an issue","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"hydrant-3h9","title":"Unbounded listHosts seeding can exhaust relay resources","description":"Finding: Unbounded listHosts seeding can exhaust relay resources\nSeverity: MEDIUM\nOriginal Finding ID: 398f22beaf9081918de6e7c4f0d5dce5\nIntroducing Commit: 3ccb5332094f4b7e8215d6054e4532352b9be3a0\nRelevant Paths: src/config.rs | src/control/mod.rs | src/control/seed.rs | src/control/firehose.rs\n\nDescription:\nThis commit adds startup seeding from configured relay listHosts endpoints, enabled by default in relay builds. The seeding code fetches each listHosts page with resp.bytes(), which buffers the entire HTTP body without an explicit size limit, then consumes pagination until the remote endpoint stops returning a cursor. For every non-banned host in every response it constructs a WSS URL and calls firehose.add_source(), which persists the source and spawns a firehose ingestor task. A malicious or compromised configured seed endpoint, or poisoned listHosts data from an upstream relay, can therefore return oversized responses, endless cursors, or very large numbers of unique hostnames to cause memory/disk growth, unbounded persistent source creation, many Tokio tasks, and continuous outbound reconnect attempts. Host strings are also accepted directly as URL authority material without policy checks such as private-address filtering, but the clearest security impact is availability/resource exhaustion.","status":"open","priority":2,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:04Z","created_by":"dawn","updated_at":"2026-06-27T20:32:04Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"hydrant-byv","title":"Unbounded describeRepo collection scan can OOM crawler","description":"Finding: Unbounded describeRepo collection scan can OOM crawler\nSeverity: MEDIUM\nOriginal Finding ID: 8804ed4fc0f481918a556492b330e29a\nIntroducing Commit: e263dacf6fb34fea58347054a8b3b22e22f10618\nRelevant Paths: src/crawler/mod.rs\n\nDescription:\nIn filtered crawler mode, unknown DIDs from the relay are passed to check_repo_signals. This commit changed the signal probe from per-signal com.atproto.repo.listRecords requests with limit=1 to a single com.atproto.repo.describeRepo request. describeRepo returns the repository's collection list, which is attacker-influenced and has no limit parameter. The code then calls resp.bytes() and serde_json::from_slice, buffering and deserializing the entire response before checking whether any configured signal is present. A malicious account/PDS discovered through normal crawling can return, or cause an honest PDS to return, an extremely large collections array, causing excessive memory/CPU use and potentially crashing the Hydrant process. This is distinct from the previous bounded query behavior because non-signal collections now have to be downloaded and parsed before the repo can be rejected.","status":"open","priority":2,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:04Z","created_by":"dawn","updated_at":"2026-06-27T20:32:04Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"hydrant-3h9","title":"Unbounded listHosts seeding can exhaust relay resources","description":"Finding: Unbounded listHosts seeding can exhaust relay resources\nSeverity: MEDIUM\nOriginal Finding ID: 398f22beaf9081918de6e7c4f0d5dce5\nIntroducing Commit: 3ccb5332094f4b7e8215d6054e4532352b9be3a0\nRelevant Paths: src/config.rs | src/control/mod.rs | src/control/seed.rs | src/control/firehose.rs\n\nDescription:\nThis commit adds startup seeding from configured relay listHosts endpoints, enabled by default in relay builds. The seeding code fetches each listHosts page with resp.bytes(), which buffers the entire HTTP body without an explicit size limit, then consumes pagination until the remote endpoint stops returning a cursor. For every non-banned host in every response it constructs a WSS URL and calls firehose.add_source(), which persists the source and spawns a firehose ingestor task. A malicious or compromised configured seed endpoint, or poisoned listHosts data from an upstream relay, can therefore return oversized responses, endless cursors, or very large numbers of unique hostnames to cause memory/disk growth, unbounded persistent source creation, many Tokio tasks, and continuous outbound reconnect attempts. Host strings are also accepted directly as URL authority material without policy checks such as private-address filtering, but the clearest security impact is availability/resource exhaustion.","status":"closed","priority":2,"issue_type":"bug","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:04Z","created_by":"dawn","updated_at":"2026-06-27T22:04:07Z","started_at":"2026-06-27T21:53:36Z","closed_at":"2026-06-27T22:04:07Z","close_reason":"fixed all 12 reported bugs","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"hydrant-byv","title":"Unbounded describeRepo collection scan can OOM crawler","description":"Finding: Unbounded describeRepo collection scan can OOM crawler\nSeverity: MEDIUM\nOriginal Finding ID: 8804ed4fc0f481918a556492b330e29a\nIntroducing Commit: e263dacf6fb34fea58347054a8b3b22e22f10618\nRelevant Paths: src/crawler/mod.rs\n\nDescription:\nIn filtered crawler mode, unknown DIDs from the relay are passed to check_repo_signals. This commit changed the signal probe from per-signal com.atproto.repo.listRecords requests with limit=1 to a single com.atproto.repo.describeRepo request. describeRepo returns the repository's collection list, which is attacker-influenced and has no limit parameter. The code then calls resp.bytes() and serde_json::from_slice, buffering and deserializing the entire response before checking whether any configured signal is present. A malicious account/PDS discovered through normal crawling can return, or cause an honest PDS to return, an extremely large collections array, causing excessive memory/CPU use and potentially crashing the Hydrant process. This is distinct from the previous bounded query behavior because non-signal collections now have to be downloaded and parsed before the repo can be rejected.","status":"closed","priority":2,"issue_type":"bug","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:04Z","created_by":"dawn","updated_at":"2026-06-27T22:04:07Z","started_at":"2026-06-27T21:53:40Z","closed_at":"2026-06-27T22:04:07Z","close_reason":"fixed all 12 reported bugs","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"hydrant-28z","title":"Unbounded crawler PDS ban map enables memory DoS","description":"Finding: Unbounded crawler PDS ban map enables memory DoS\nSeverity: MEDIUM\nOriginal Finding ID: c2eefaff8ac08191afe7275e94231542\nIntroducing Commit: 106ab40683271705bf2096914ebe83075a01040f\nRelevant Paths: src/crawler/ban.rs | src/crawler/mod.rs | src/resolver.rs\n\nDescription:\nThis commit introduces BanTracker, a process-lifetime HashMap keyed by the full resolved PDS Url. check_repo_signals resolves each unknown DID to a PDS endpoint, then unconditionally calls tracker.get_handle(\u0026pds_url), which inserts a new State if that URL has not been seen before. Entries are never removed when a request succeeds, when a ban expires, or when the related DID is no longer relevant. Because crawler listRepos/DID documents are untrusted inputs in the threat model, an attacker controlling many DIDs or a malicious configured relay can provide many distinct PDS serviceEndpoint URLs and force unbounded in-memory state growth. Keying on the whole Url also means variants of the same origin, if accepted by DID document parsing, can create separate ban states while actual XRPC requests are joined to a fixed /xrpc path.","status":"closed","priority":2,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:03Z","created_by":"dawn","updated_at":"2026-06-27T20:39:01Z","closed_at":"2026-06-27T20:39:01Z","close_reason":"resolved (ban.rs and BanTracker completely removed from codebase back in March)","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"hydrant-7o8","title":"Unbounded collection enumeration in public describeRepo","description":"Finding: Unbounded collection enumeration in public describeRepo\nSeverity: MEDIUM\nOriginal Finding ID: b4a352096900819184744d6128f0583c\nIntroducing Commit: 6624ccf71b0e47458bbb555539d78c61f866eb02\nRelevant Paths: src/api/mod.rs | src/api/xrpc/mod.rs | src/api/xrpc/describe_repo.rs | src/control/repos.rs\n\nDescription:\nThe commit adds /xrpc/systems.gaze.hydrant.describeRepo to the public XRPC router. Handling a request resolves the identifier, obtains a repo handle, then concurrently calls repo.collections(). The collections() helper scans the entire counts keyspace prefix for that DID and inserts every matching collection into a HashMap. The handler then converts that into a HashSet for the JSON response, discarding the count values. There is no cap on scanned entries, response size, execution time, or memory use. Because collection-count entries are derived from indexed repository data, a malicious indexed account can create many distinct collection NSIDs and then any unauthenticated client can repeatedly call this public endpoint to force large DB prefix scans, allocations, and JSON serialization. This is a resource-exhaustion issue introduced by the new public endpoint.","status":"open","priority":2,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:03Z","created_by":"dawn","updated_at":"2026-06-27T20:32:03Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"hydrant-0hs","title":"Stream event format change can trigger replay DoS","description":"Finding: Stream event format change can trigger replay DoS\nSeverity: MEDIUM\nOriginal Finding ID: 809b9f6d9b2c8191ad81c8da842aee68\nIntroducing Commit: a022ce698b1f3ddad6f88f35a4042f16063c3a33\nRelevant Paths: src/types.rs | src/api/stream.rs | src/api/mod.rs | src/ops.rs | src/backfill/mod.rs\n\nDescription:\n`StoredEvent` is stored in the database with `rmp_serde::to_vec` and replayed from `/stream` with `rmp_serde::from_slice`. `rmp-serde`'s default struct encoding is positional, so adding `live` at the start changes the on-disk field order. Older rows were encoded as `[did, rev, collection, rkey, action, cid]`; the new decoder expects `[live, did, rev, collection, rkey, action, cid]`. The `#[serde(default)]` on `live` does not protect this case because the old first element is present and is decoded as a boolean, which fails. In the stream handler, the cursor is advanced before deserialization, and deserialization failures are logged and skipped. Because `/stream` is publicly exposed, an attacker can connect with an old cursor such as `cursor=0` on an upgraded instance with many old-format events and repeatedly make the server scan the events keyspace and emit one error per legacy event, bypassing normal send-side backpressure for those skipped events. This also breaks historical replay integrity for legitimate clients.","status":"open","priority":2,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:02Z","created_by":"dawn","updated_at":"2026-06-27T20:32:02Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"hydrant-7o8","title":"Unbounded collection enumeration in public describeRepo","description":"Finding: Unbounded collection enumeration in public describeRepo\nSeverity: MEDIUM\nOriginal Finding ID: b4a352096900819184744d6128f0583c\nIntroducing Commit: 6624ccf71b0e47458bbb555539d78c61f866eb02\nRelevant Paths: src/api/mod.rs | src/api/xrpc/mod.rs | src/api/xrpc/describe_repo.rs | src/control/repos.rs\n\nDescription:\nThe commit adds /xrpc/systems.gaze.hydrant.describeRepo to the public XRPC router. Handling a request resolves the identifier, obtains a repo handle, then concurrently calls repo.collections(). The collections() helper scans the entire counts keyspace prefix for that DID and inserts every matching collection into a HashMap. The handler then converts that into a HashSet for the JSON response, discarding the count values. There is no cap on scanned entries, response size, execution time, or memory use. Because collection-count entries are derived from indexed repository data, a malicious indexed account can create many distinct collection NSIDs and then any unauthenticated client can repeatedly call this public endpoint to force large DB prefix scans, allocations, and JSON serialization. This is a resource-exhaustion issue introduced by the new public endpoint.","status":"closed","priority":2,"issue_type":"bug","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:03Z","created_by":"dawn","updated_at":"2026-06-27T22:04:07Z","started_at":"2026-06-27T21:53:45Z","closed_at":"2026-06-27T22:04:07Z","close_reason":"fixed all 12 reported bugs","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"hydrant-0hs","title":"Stream event format change can trigger replay DoS","description":"Finding: Stream event format change can trigger replay DoS\nSeverity: MEDIUM\nOriginal Finding ID: 809b9f6d9b2c8191ad81c8da842aee68\nIntroducing Commit: a022ce698b1f3ddad6f88f35a4042f16063c3a33\nRelevant Paths: src/types.rs | src/api/stream.rs | src/api/mod.rs | src/ops.rs | src/backfill/mod.rs\n\nDescription:\n`StoredEvent` is stored in the database with `rmp_serde::to_vec` and replayed from `/stream` with `rmp_serde::from_slice`. `rmp-serde`'s default struct encoding is positional, so adding `live` at the start changes the on-disk field order. Older rows were encoded as `[did, rev, collection, rkey, action, cid]`; the new decoder expects `[live, did, rev, collection, rkey, action, cid]`. The `#[serde(default)]` on `live` does not protect this case because the old first element is present and is decoded as a boolean, which fails. In the stream handler, the cursor is advanced before deserialization, and deserialization failures are logged and skipped. Because `/stream` is publicly exposed, an attacker can connect with an old cursor such as `cursor=0` on an upgraded instance with many old-format events and repeatedly make the server scan the events keyspace and emit one error per legacy event, bypassing normal send-side backpressure for those skipped events. This also breaks historical replay integrity for legitimate clients.","status":"closed","priority":2,"issue_type":"bug","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:02Z","created_by":"dawn","updated_at":"2026-06-27T22:04:07Z","started_at":"2026-06-27T21:53:49Z","closed_at":"2026-06-27T22:04:07Z","close_reason":"fixed all 12 reported bugs","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"hydrant-8b8","title":"Public getRecord misses can cause disk-heavy DB reads","description":"Finding: Public getRecord misses can cause disk-heavy DB reads\nSeverity: MEDIUM\nOriginal Finding ID: 5a0782626f448191a4d162e1f6ca8438\nIntroducing Commit: a0a9359a895131783a843a7dd5327f2b8b7da0f5\nRelevant Paths: src/db/mod.rs | src/api/xrpc.rs\n\nDescription:\nThe records keyspace was changed from default Fjall options to `expect_point_read_hits(true)`. The added comment explicitly treats this as turning off Bloom filters because getRecord is assumed to be uncommon. However, `com.atproto.repo.getRecord` is publicly routed, accepts attacker-controlled repo/collection/rkey inputs, and directly performs a point lookup in `db.records` without first checking whether the repo or record exists. An unauthenticated client can therefore generate large numbers of guaranteed-missing record keys. On a large LSM keyspace, disabling miss filters makes those misses significantly more expensive because the database must do point-lookup work across on-disk state instead of cheaply rejecting absent keys. This can amplify public HTTP traffic into disk I/O and blocking-thread exhaustion.","status":"closed","priority":2,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:01Z","created_by":"dawn","updated_at":"2026-06-27T20:53:30Z","closed_at":"2026-06-27T20:53:30Z","close_reason":"resolved (added HYDRANT_DB_RECORDS_BLOOM_FILTERS configuration option to toggle bloom filters, defaulting to true to protect against public getRecord miss attacks)","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"hydrant-5ci","title":"Legacy DID excludes ignored after key-format change","description":"Finding: Legacy DID excludes ignored after key-format change\nSeverity: MEDIUM\nOriginal Finding ID: 8a58834a31e88191a776fe5ff8ae9c72\nIntroducing Commit: 53ac9dddc4e289bb6eaf5c9fabbf105306e328d1\nRelevant Paths: src/db/filter.rs | src/ingest/firehose.rs | src/crawler/mod.rs\n\nDescription:\nBefore this commit, exclude entries were stored as `x|{raw_did}`. The new `exclude_key` stores excludes as `x|{TrimmedDid}`. Existing databases are not migrated, and the firehose and crawler now only check the new key format. As a result, any DID that was excluded before the upgrade will no longer be skipped, allowing that repo's firehose events to be processed and crawler-discovered repos to be queued despite the operator's denylist. Patch removal also computes only the new key, so legacy entries are not removed unless the operator replaces the entire exclude set.","status":"open","priority":2,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:00Z","created_by":"dawn","updated_at":"2026-06-27T20:32:00Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"hydrant-m6v","title":"Crawler retry wake can busy-loop on jittered retries","description":"Finding: Crawler retry wake can busy-loop on jittered retries\nSeverity: MEDIUM\nOriginal Finding ID: a84c29ca490881918a976bc3b0231ae7\nIntroducing Commit: e558361eb571e7e019a4a7967bb4ae7e666f3f25\nRelevant Paths: src/crawler/mod.rs\n\nDescription:\nprocess_retry_queue() adds a randomized backoff to each retry entry and correctly treats the entry as not ready when state.after + backoff is still in the future. However, the newly introduced Duration-based wake calculation uses only state.after - now. If now is after state.after but before state.after + backoff, to_std() fails and unwrap_or(Duration::ZERO) is used. The retry thread then calls sleep(Duration::ZERO), immediately re-enters process_retry_queue(), scans the retry keyspace again, and repeats until at least the minimum jitter window has elapsed. A malicious PDS or relay-listed DID that causes retry entries, especially with large Retry-After/throttle durations, can make the crawler retry thread consume CPU and repeatedly hit the DB for a predictable window. The wake duration should be calculated from the actual readiness threshold, e.g. state.after + backoff - now.","status":"open","priority":2,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:00Z","created_by":"dawn","updated_at":"2026-06-27T20:32:00Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"hydrant-6w8","title":"Crawler pause does not stop retry queue processing","description":"Finding: Crawler pause does not stop retry queue processing\nSeverity: MEDIUM\nOriginal Finding ID: 1db4d5a08c8c81918e69516f00833326\nIntroducing Commit: ec1b1027784eb565a4310e955e2688b100408247\nRelevant Paths: src/main.rs | src/crawler/mod.rs\n\nDescription:\nThis commit changes startup so a Crawler is spawned whenever relay hosts exist, even if crawler_enabled is false. Inside Crawler::run, the retry thread is started unconditionally and repeatedly calls process_retry_queue without checking the watch flag. Only the per-relay crawl loop calls wait_enabled(). As a result, setting HYDRANT_ENABLE_CRAWLER=false, starting in filter mode where the crawler defaults to disabled, or PATCHing /ingestion to pause the crawler does not fully stop crawler activity if retry entries already exist in the database. Those entries can trigger batches of up to 1000 signal checks, which perform identity resolution and outbound describeRepo HTTP requests to PDS URLs controlled by DID documents. This defeats the intended runtime/config safety control and can keep causing outbound-network and resource-consumption activity after an operator believes the crawler is paused.","status":"open","priority":2,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:31:59Z","created_by":"dawn","updated_at":"2026-06-27T20:31:59Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"hydrant-p2e","title":"404 describeRepo responses can throttle PDS crawling","description":"Finding: 404 describeRepo responses can throttle PDS crawling\nSeverity: MEDIUM\nOriginal Finding ID: 16f60348cb78819183bd0f88d79d7efd\nIntroducing Commit: f9fea797e5ac3aaff3395015aafcb5a7eaca48a2\nRelevant Paths: src/util/mod.rs | src/crawler/list_repos.rs | src/util/throttle.rs\n\nDescription:\nThe crawler checks untrusted DIDs returned by listRepos by resolving each DID to a PDS and calling com.atproto.repo.describeRepo. Before this commit, 404 responses from describeRepo reached the explicit NOT_FOUND/GONE handling and were treated as NoSignal for that single DID. The new shared is_status_their_fault helper includes 404, and crawler is_throttle_worthy now delegates to it. As a result, a 404 describeRepo response is handled before the NOT_FOUND/GONE branch and calls throttle.record_failure(), which throttles the entire resolved PDS URL for 30 minutes with exponential backoff. A malicious crawler source, or an attacker able to get a DID with a service endpoint pointing at a victim PDS into a crawled listRepos stream, can make the victim PDS return 404 for that DID and suppress signal checks/discovery for other legitimate repos on that PDS in filter mode. The intended 525 addition should not have pulled 404 into this crawler path; 404 should remain endpoint-level throttle-worthy for firehose upgrade failures only, or the crawler helper should exclude 404 so repo-not-found is handled as NoSignal.","status":"open","priority":2,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:31:58Z","created_by":"dawn","updated_at":"2026-06-27T20:31:58Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"hydrant-gvr","title":"Stream replay panics on malformed record paths","description":"Finding: Stream replay panics on malformed record paths\nSeverity: LOW\nOriginal Finding ID: fcef126dfbe48191a5af9f1eecb8fd16\nIntroducing Commit: 6dd9a8cec8525732485c2235ac6dd56ba1937eba\nRelevant Paths: src/control.rs | src/ops.rs | src/api/stream.rs\n\nDescription:\nRecord events are built for /stream subscribers by replaying StoredEvent values from the events keyspace. This commit changed stored_to_event to construct Nsid and Rkey values and to panic with expect() if either value is invalid. However, apply_commit still obtains collection and rkey by only splitting the firehose op.path on '/', then stores those raw components into StoredEvent. If an untrusted or malicious firehose/PDS source supplies an op path such as an empty collection, invalid NSID, or invalid record key, the event can be persisted successfully but later cause stored_to_event to panic. The panic occurs inside the per-subscriber hydrant-stream thread, so it is not a full process abort in the normal panic configuration, but it can disconnect live subscribers and make cursor replay fail at that event.","status":"open","priority":3,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:08Z","created_by":"dawn","updated_at":"2026-06-27T20:32:08Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"hydrant-b0v","title":"Public requestCrawl can poison filter config via PDS status keys","description":"Finding: Public requestCrawl can poison filter config via PDS status keys\nSeverity: LOW\nOriginal Finding ID: 63456bb21eb4819193ca69d1a164aef5\nIntroducing Commit: c4d1302302c501bfecb9a02172c690b7be954179\nRelevant Paths: src/db/pds_meta.rs | src/db/filter.rs | src/state.rs | src/api/xrpc/request_crawl.rs | src/ingest/firehose.rs\n\nDescription:\nThe new PDS metadata layout removed the old `pt|`/`pb|` prefixes and now writes status keys as `{host}|status` into `state.db.filter`. That keyspace is already used for filter configuration where collection filters are stored as `c|{collection}` and signal filters as `s|{signal}`. Because public relay builds expose `com.atproto.sync.requestCrawl`, an unauthenticated client can submit hostname `c`; Hydrant starts a direct PDS ingestor for `wss://c/`, and after repeated connection failures the new code persists `HostStatus::Offline` to key `c|status`. On the next startup, `db::filter::load` scans `c|` keys and treats `c|status` as an operator-configured collection allowlist entry. Since an empty collection list normally means store all collections, this injected single entry changes behavior to store only the `status` collection, effectively dropping normal indexed records. The same un-namespaced layout also creates related collisions such as `s|status` for signal filters and `c|tier` for tier records.","status":"open","priority":3,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:07Z","created_by":"dawn","updated_at":"2026-06-27T20:32:07Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"hydrant-ao2","title":"Malformed commits can poison resync buffers","description":"Finding: Malformed commits can poison resync buffers\nSeverity: LOW\nOriginal Finding ID: fb188b2a29508191a0544ec3a46b8406\nIntroducing Commit: 8c239314f96cdf25a9eb9c1d6b98b4565e2a5b6e\nRelevant Paths: src/ingest/worker.rs | src/ops.rs\n\nDescription:\nThe new IngestError wrapper converts miette::Report values into IngestError::Generic. ops::apply_commit returns miette::Result, and jacquard_repo CommitError values produced while parsing the repository commit are converted to miette reports inside ops::apply_commit. When process_commit uses `?` on apply_commit, those reports become IngestError::Generic, not IngestError::Commit. The new check_if_retriable_failure only matches top-level IngestError variants, so a Generic report containing a CommitError is now considered retriable. The worker then persists the failed commit to the resync_buffer. During later draining, the same malformed commit fails again and is not removed because it is still considered retriable, causing a sticky per-repository ingestion DoS and possible disk/log growth with repeated malformed revisions. Before this commit, check_if_retriable_failure downcasted the miette::Report directly and excluded CommitError from retries.","status":"open","priority":3,"issue_type":"bug","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:06Z","created_by":"dawn","updated_at":"2026-06-27T20:32:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"hydrant-5ci","title":"Legacy DID excludes ignored after key-format change","description":"Finding: Legacy DID excludes ignored after key-format change\nSeverity: MEDIUM\nOriginal Finding ID: 8a58834a31e88191a776fe5ff8ae9c72\nIntroducing Commit: 53ac9dddc4e289bb6eaf5c9fabbf105306e328d1\nRelevant Paths: src/db/filter.rs | src/ingest/firehose.rs | src/crawler/mod.rs\n\nDescription:\nBefore this commit, exclude entries were stored as `x|{raw_did}`. The new `exclude_key` stores excludes as `x|{TrimmedDid}`. Existing databases are not migrated, and the firehose and crawler now only check the new key format. As a result, any DID that was excluded before the upgrade will no longer be skipped, allowing that repo's firehose events to be processed and crawler-discovered repos to be queued despite the operator's denylist. Patch removal also computes only the new key, so legacy entries are not removed unless the operator replaces the entire exclude set.","status":"closed","priority":2,"issue_type":"bug","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:00Z","created_by":"dawn","updated_at":"2026-06-27T22:04:07Z","started_at":"2026-06-27T21:53:53Z","closed_at":"2026-06-27T22:04:07Z","close_reason":"fixed all 12 reported bugs","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"hydrant-m6v","title":"Crawler retry wake can busy-loop on jittered retries","description":"Finding: Crawler retry wake can busy-loop on jittered retries\nSeverity: MEDIUM\nOriginal Finding ID: a84c29ca490881918a976bc3b0231ae7\nIntroducing Commit: e558361eb571e7e019a4a7967bb4ae7e666f3f25\nRelevant Paths: src/crawler/mod.rs\n\nDescription:\nprocess_retry_queue() adds a randomized backoff to each retry entry and correctly treats the entry as not ready when state.after + backoff is still in the future. However, the newly introduced Duration-based wake calculation uses only state.after - now. If now is after state.after but before state.after + backoff, to_std() fails and unwrap_or(Duration::ZERO) is used. The retry thread then calls sleep(Duration::ZERO), immediately re-enters process_retry_queue(), scans the retry keyspace again, and repeats until at least the minimum jitter window has elapsed. A malicious PDS or relay-listed DID that causes retry entries, especially with large Retry-After/throttle durations, can make the crawler retry thread consume CPU and repeatedly hit the DB for a predictable window. The wake duration should be calculated from the actual readiness threshold, e.g. state.after + backoff - now.","status":"closed","priority":2,"issue_type":"bug","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:00Z","created_by":"dawn","updated_at":"2026-06-27T22:04:07Z","started_at":"2026-06-27T21:53:58Z","closed_at":"2026-06-27T22:04:07Z","close_reason":"fixed all 12 reported bugs","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"hydrant-6w8","title":"Crawler pause does not stop retry queue processing","description":"Finding: Crawler pause does not stop retry queue processing\nSeverity: MEDIUM\nOriginal Finding ID: 1db4d5a08c8c81918e69516f00833326\nIntroducing Commit: ec1b1027784eb565a4310e955e2688b100408247\nRelevant Paths: src/main.rs | src/crawler/mod.rs\n\nDescription:\nThis commit changes startup so a Crawler is spawned whenever relay hosts exist, even if crawler_enabled is false. Inside Crawler::run, the retry thread is started unconditionally and repeatedly calls process_retry_queue without checking the watch flag. Only the per-relay crawl loop calls wait_enabled(). As a result, setting HYDRANT_ENABLE_CRAWLER=false, starting in filter mode where the crawler defaults to disabled, or PATCHing /ingestion to pause the crawler does not fully stop crawler activity if retry entries already exist in the database. Those entries can trigger batches of up to 1000 signal checks, which perform identity resolution and outbound describeRepo HTTP requests to PDS URLs controlled by DID documents. This defeats the intended runtime/config safety control and can keep causing outbound-network and resource-consumption activity after an operator believes the crawler is paused.","status":"closed","priority":2,"issue_type":"bug","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T20:31:59Z","created_by":"dawn","updated_at":"2026-06-27T22:04:08Z","started_at":"2026-06-27T21:54:02Z","closed_at":"2026-06-27T22:04:08Z","close_reason":"fixed all 12 reported bugs","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"hydrant-p2e","title":"404 describeRepo responses can throttle PDS crawling","description":"Finding: 404 describeRepo responses can throttle PDS crawling\nSeverity: MEDIUM\nOriginal Finding ID: 16f60348cb78819183bd0f88d79d7efd\nIntroducing Commit: f9fea797e5ac3aaff3395015aafcb5a7eaca48a2\nRelevant Paths: src/util/mod.rs | src/crawler/list_repos.rs | src/util/throttle.rs\n\nDescription:\nThe crawler checks untrusted DIDs returned by listRepos by resolving each DID to a PDS and calling com.atproto.repo.describeRepo. Before this commit, 404 responses from describeRepo reached the explicit NOT_FOUND/GONE handling and were treated as NoSignal for that single DID. The new shared is_status_their_fault helper includes 404, and crawler is_throttle_worthy now delegates to it. As a result, a 404 describeRepo response is handled before the NOT_FOUND/GONE branch and calls throttle.record_failure(), which throttles the entire resolved PDS URL for 30 minutes with exponential backoff. A malicious crawler source, or an attacker able to get a DID with a service endpoint pointing at a victim PDS into a crawled listRepos stream, can make the victim PDS return 404 for that DID and suppress signal checks/discovery for other legitimate repos on that PDS in filter mode. The intended 525 addition should not have pulled 404 into this crawler path; 404 should remain endpoint-level throttle-worthy for firehose upgrade failures only, or the crawler helper should exclude 404 so repo-not-found is handled as NoSignal.","status":"closed","priority":2,"issue_type":"bug","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T20:31:58Z","created_by":"dawn","updated_at":"2026-06-27T22:04:08Z","started_at":"2026-06-27T21:54:06Z","closed_at":"2026-06-27T22:04:08Z","close_reason":"fixed all 12 reported bugs","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"hydrant-gvr","title":"Stream replay panics on malformed record paths","description":"Finding: Stream replay panics on malformed record paths\nSeverity: LOW\nOriginal Finding ID: fcef126dfbe48191a5af9f1eecb8fd16\nIntroducing Commit: 6dd9a8cec8525732485c2235ac6dd56ba1937eba\nRelevant Paths: src/control.rs | src/ops.rs | src/api/stream.rs\n\nDescription:\nRecord events are built for /stream subscribers by replaying StoredEvent values from the events keyspace. This commit changed stored_to_event to construct Nsid and Rkey values and to panic with expect() if either value is invalid. However, apply_commit still obtains collection and rkey by only splitting the firehose op.path on '/', then stores those raw components into StoredEvent. If an untrusted or malicious firehose/PDS source supplies an op path such as an empty collection, invalid NSID, or invalid record key, the event can be persisted successfully but later cause stored_to_event to panic. The panic occurs inside the per-subscriber hydrant-stream thread, so it is not a full process abort in the normal panic configuration, but it can disconnect live subscribers and make cursor replay fail at that event.","status":"closed","priority":3,"issue_type":"bug","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:08Z","created_by":"dawn","updated_at":"2026-06-27T22:04:08Z","started_at":"2026-06-27T21:54:11Z","closed_at":"2026-06-27T22:04:08Z","close_reason":"fixed all 12 reported bugs","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"hydrant-b0v","title":"Public requestCrawl can poison filter config via PDS status keys","description":"Finding: Public requestCrawl can poison filter config via PDS status keys\nSeverity: LOW\nOriginal Finding ID: 63456bb21eb4819193ca69d1a164aef5\nIntroducing Commit: c4d1302302c501bfecb9a02172c690b7be954179\nRelevant Paths: src/db/pds_meta.rs | src/db/filter.rs | src/state.rs | src/api/xrpc/request_crawl.rs | src/ingest/firehose.rs\n\nDescription:\nThe new PDS metadata layout removed the old `pt|`/`pb|` prefixes and now writes status keys as `{host}|status` into `state.db.filter`. That keyspace is already used for filter configuration where collection filters are stored as `c|{collection}` and signal filters as `s|{signal}`. Because public relay builds expose `com.atproto.sync.requestCrawl`, an unauthenticated client can submit hostname `c`; Hydrant starts a direct PDS ingestor for `wss://c/`, and after repeated connection failures the new code persists `HostStatus::Offline` to key `c|status`. On the next startup, `db::filter::load` scans `c|` keys and treats `c|status` as an operator-configured collection allowlist entry. Since an empty collection list normally means store all collections, this injected single entry changes behavior to store only the `status` collection, effectively dropping normal indexed records. The same un-namespaced layout also creates related collisions such as `s|status` for signal filters and `c|tier` for tier records.","status":"closed","priority":3,"issue_type":"bug","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:07Z","created_by":"dawn","updated_at":"2026-06-27T22:04:08Z","started_at":"2026-06-27T21:54:16Z","closed_at":"2026-06-27T22:04:08Z","close_reason":"fixed all 12 reported bugs","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"hydrant-ao2","title":"Malformed commits can poison resync buffers","description":"Finding: Malformed commits can poison resync buffers\nSeverity: LOW\nOriginal Finding ID: fb188b2a29508191a0544ec3a46b8406\nIntroducing Commit: 8c239314f96cdf25a9eb9c1d6b98b4565e2a5b6e\nRelevant Paths: src/ingest/worker.rs | src/ops.rs\n\nDescription:\nThe new IngestError wrapper converts miette::Report values into IngestError::Generic. ops::apply_commit returns miette::Result, and jacquard_repo CommitError values produced while parsing the repository commit are converted to miette reports inside ops::apply_commit. When process_commit uses `?` on apply_commit, those reports become IngestError::Generic, not IngestError::Commit. The new check_if_retriable_failure only matches top-level IngestError variants, so a Generic report containing a CommitError is now considered retriable. The worker then persists the failed commit to the resync_buffer. During later draining, the same malformed commit fails again and is not removed because it is still considered retriable, causing a sticky per-repository ingestion DoS and possible disk/log growth with repeated malformed revisions. Before this commit, check_if_retriable_failure downcasted the miette::Report directly and excluded CommitError from retries.","status":"closed","priority":3,"issue_type":"bug","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:06Z","created_by":"dawn","updated_at":"2026-06-27T22:04:08Z","started_at":"2026-06-27T21:54:20Z","closed_at":"2026-06-27T22:04:08Z","close_reason":"fixed all 12 reported bugs","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"hydrant-ydn","title":"listRepos emits a cursor for an empty next page","description":"Finding: listRepos emits a cursor for an empty next page\nSeverity: INFORMATIONAL\nOriginal Finding ID: bd48a15aa2348191a264838feb43e364\nIntroducing Commit: a85020dfa8913c6db8f49e953e1a0b491d438b9e\nRelevant Paths: src/api/xrpc/list_repos.rs | src/control/mod.rs | src/crawler/list_repos.rs\n\nDescription:\nThe new listRepos handler stops as soon as repos.len() reaches the limit and sets next_cursor to the DID of the last returned repo. If the remaining eligible repository count is exactly equal to the requested limit, the response still contains a cursor even though the next page will be empty. This is inconsistent with normal ATProto pagination semantics and with the existing listHosts implementation, which fetches one extra item to decide whether a next cursor should be returned. The direct impact is correctness: clients may perform unnecessary empty-page fetches, and Hydrant's own listRepos crawler has pre-existing cursor handling that can preserve the last cursor after an empty response, causing later passes to resume after the last DID instead of restarting from the beginning.","status":"closed","priority":4,"issue_type":"task","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:15Z","created_by":"dawn","updated_at":"2026-06-27T21:09:40Z","closed_at":"2026-06-27T21:09:40Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"hydrant-bvr","title":"describeRepo now resolves handle identifiers twice","description":"Finding: describeRepo now resolves handle identifiers twice\nSeverity: INFORMATIONAL\nOriginal Finding ID: c8b82c8db1c481919813fd7089c041d5\nIntroducing Commit: 6dc1a1b2aea872da4c2f283c15eebdd82d2cf12e\nRelevant Paths: src/api/xrpc/describe_repo.rs | src/api/xrpc/resolve_mini_doc.rs\n\nDescription:\nIn the updated describeRepo handler, the new resolve_mini_doc future is created from the original identifier, then the handler separately resolves the same identifier to compute the repository whose collections will be returned. resolve_mini_doc also resolves the identifier internally when it is later awaited. For DID identifiers this is harmless, but for handle identifiers it causes duplicate outbound identity-resolution work. If a malicious or unstable handle resolves to different DIDs across the two lookups, the returned mini-doc fields can describe one DID while the collections field was read from another DID. This appears to be a correctness/resource regression rather than an exploitable security vulnerability under the provided threat model because the affected endpoint is public and the collection metadata is public.","status":"closed","priority":4,"issue_type":"task","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:14Z","created_by":"dawn","updated_at":"2026-06-27T21:09:45Z","closed_at":"2026-06-27T21:09:45Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"hydrant-hn5","title":"listRecords cursor bounds ignore encoded rkey prefixes","description":"Finding: listRecords cursor bounds ignore encoded rkey prefixes\nSeverity: INFORMATIONAL\nOriginal Finding ID: d67e1043888c8191ba1ad05de897300c\nIntroducing Commit: a366d89f3dcf6fa716a7650a490e3ba344d95d7f\nRelevant Paths: src/db/keys.rs | src/api/xrpc.rs\n\nDescription:\nRecord keys are now stored as DID separator plus an encoded rkey where TID keys are prefixed with 't' and string keys are prefixed with 's'. However, handle_list_records still constructs range start/end keys by appending req.cursor.as_bytes() directly to the DID prefix. The response cursor is also converted back to the user-visible rkey string with to_smolstr(), so the next request does not contain the internal 't'/'s' byte needed to seek correctly in the keyspace. As a result, paginated listRecords requests can return empty pages, duplicate earlier results, or otherwise skip records after the first page. This is a public API correctness/regression issue rather than a security vulnerability.","status":"closed","priority":4,"issue_type":"task","owner":"90008@klbr.net","created_at":"2026-06-27T20:32:14Z","created_by":"dawn","updated_at":"2026-06-27T21:09:50Z","closed_at":"2026-06-27T21:09:50Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/src/util/mod.rs b/src/util/mod.rs index aeab0ca..0408cf7 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -290,3 +290,20 @@ pub fn hash(val: &T) -> u64 { val.hash(&mut hasher); hasher.finish() } + +pub async fn read_limited_bytes(resp: reqwest::Response, limit: usize) -> miette::Result { + use bytes::BytesMut; + use futures::StreamExt as _; + use miette::Context as _; + use miette::IntoDiagnostic as _; + let mut stream = resp.bytes_stream(); + let mut buf = BytesMut::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.into_diagnostic().context("failed to read response chunk")?; + if buf.len() + chunk.len() > limit { + miette::bail!("response body too large (exceeds {limit} bytes)"); + } + buf.extend_from_slice(&chunk); + } + Ok(buf.freeze()) +}