diff --git a/src/http_retry.rs b/src/http_retry.rs --- a/src/http_retry.rs +++ b/src/http_retry.rs @@ -20,4 +20,5 @@ .and_then(|v| v.to_str().ok()) .and_then(|v| v.parse::().ok()) .unwrap_or(5) + .min(120) } diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -346,6 +346,7 @@ let mut resolved_count = already_resolved; + let mut attempted = already_resolved; for (did,) in &unresolved { match profile::resolve_pds_endpoint(&state.http, &state.config.plc_url, did).await { Ok(pds) => { @@ -359,13 +360,14 @@ .bind(did) .execute(&state.db) .await; + resolved_count += 1; } Err(e) => { tracing::warn!(did, error = %e, "failed to resolve PDS endpoint, skipping DID"); } } - resolved_count += 1; - if resolved_count % 100 == 0 { + attempted += 1; + if attempted % 100 == 0 { update_job_counter(state, job_id, "processed_repos", resolved_count).await; if is_cancelled(state, job_id).await { return; diff --git a/packages/docs/content/docs/guides/backfill.md b/packages/docs/content/docs/guides/backfill.md --- a/packages/docs/content/docs/guides/backfill.md +++ b/packages/docs/content/docs/guides/backfill.md @@ -34,7 +34,7 @@ | `running` | Job is actively processing | | `cancelling` | Cancel requested, waiting for the worker to stop | | `cancelled` | Worker has stopped and cleaned up | -| `completed` | All repos processed successfully | +| `completed` | Worker finished processing all resolvable repos | | `failed` | An error occurred | The `stage` field tracks which phase the job is in: `pending`, `discovering_repos`, `resolving_pds`, `fetching_records`, `completed`, `failed`, or `cancelled`. @@ -44,7 +44,7 @@ Running jobs can be cancelled via `POST /admin/backfill/{id}/cancel` or the Cancel button in the dashboard. Cancellation is two-phase: 1. The endpoint sets the job status to `cancelling`. -2. The worker checks for cancellation at natural checkpoints (between relay pages, every 100 DIDs during resolution, every 100 repos during fetching). When it detects the `cancelling` status, it stops work and sets the final status to `cancelled`. +2. The worker checks for cancellation at natural checkpoints (between relay pages, every 100 DIDs during resolution, every 10 repos during fetching). When it detects the `cancelling` status, it stops work and sets the final status to `cancelled`. This means there may be a short delay between clicking Cancel and the job fully stopping, depending on what the worker is doing at that moment.