From cb3797e918cae1860083acb04c676e4f65dea5e9 Mon Sep 17 00:00:00 2001 From: Trezy Date: Thu, 14 May 2026 11:53:24 -0500 Subject: [PATCH] fix: only count successful PDS resolutions, cap retry-after, update backfill docs Signed-off-by: Trezy --- packages/docs/content/docs/guides/backfill.md | 4 ++-- src/admin/backfill.rs | 6 ++++-- src/http_retry.rs | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/docs/content/docs/guides/backfill.md b/packages/docs/content/docs/guides/backfill.md index da7cc42..642343a 100644 --- a/packages/docs/content/docs/guides/backfill.md +++ b/packages/docs/content/docs/guides/backfill.md @@ -34,7 +34,7 @@ A backfill job has both a `status` (overall state) and a `stage` (current phase) | `running` | Job is actively processing | | `cancelling` | Cancel requested, waiting for the worker to stop | | `cancelled` | Worker has stopped and cleaned up | -| `completed` | All repos processed successfully | +| `completed` | Worker finished processing all resolvable repos | | `failed` | An error occurred | The `stage` field tracks which phase the job is in: `pending`, `discovering_repos`, `resolving_pds`, `fetching_records`, `completed`, `failed`, or `cancelled`. @@ -44,7 +44,7 @@ The `stage` field tracks which phase the job is in: `pending`, `discovering_repo Running jobs can be cancelled via `POST /admin/backfill/{id}/cancel` or the Cancel button in the dashboard. Cancellation is two-phase: 1. The endpoint sets the job status to `cancelling`. -2. The worker checks for cancellation at natural checkpoints (between relay pages, every 100 DIDs during resolution, every 100 repos during fetching). When it detects the `cancelling` status, it stops work and sets the final status to `cancelled`. +2. The worker checks for cancellation at natural checkpoints (between relay pages, every 100 DIDs during resolution, every 10 repos during fetching). When it detects the `cancelling` status, it stops work and sets the final status to `cancelled`. This means there may be a short delay between clicking Cancel and the job fully stopping, depending on what the worker is doing at that moment. diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 238b796..553c3a5 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -346,6 +346,7 @@ async fn run_resolution_phase(state: &AppState, job_id: &str) { let mut resolved_count = already_resolved; + let mut attempted = already_resolved; for (did,) in &unresolved { match profile::resolve_pds_endpoint(&state.http, &state.config.plc_url, did).await { Ok(pds) => { @@ -359,13 +360,14 @@ async fn run_resolution_phase(state: &AppState, job_id: &str) { .bind(did) .execute(&state.db) .await; + resolved_count += 1; } Err(e) => { tracing::warn!(did, error = %e, "failed to resolve PDS endpoint, skipping DID"); } } - resolved_count += 1; - if resolved_count % 100 == 0 { + attempted += 1; + if attempted % 100 == 0 { update_job_counter(state, job_id, "processed_repos", resolved_count).await; if is_cancelled(state, job_id).await { return; diff --git a/src/http_retry.rs b/src/http_retry.rs index 67c9898..c1edb28 100644 --- a/src/http_retry.rs +++ b/src/http_retry.rs @@ -20,4 +20,5 @@ pub fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> u64 { .and_then(|v| v.to_str().ok()) .and_then(|v| v.parse::().ok()) .unwrap_or(5) + .min(120) } -- 2.51.2