From 8f21474dbdefd550d4a39c2d4afdc9576779d217 Mon Sep 17 00:00:00 2001 From: Trezy Date: Thu, 21 May 2026 09:22:40 -0500 Subject: [PATCH 01/28] feat: parallelize PDS resolution and record retrieval during backfills Signed-off-by: Trezy --- .github/workflows/ci.yml | 176 +++++++ .../20260520000000_add_resolved_repos.sql | 1 + .../20260520000000_add_resolved_repos.sql | 1 + src/admin/backfill.rs | 474 ++++++++++++++++-- src/admin/types.rs | 1 + web/src/app/dashboard/backfill/page.tsx | 24 +- web/src/types/backfill.ts | 1 + 7 files changed, 626 insertions(+), 52 deletions(-) create mode 100644 migrations/postgres/20260520000000_add_resolved_repos.sql create mode 100644 migrations/sqlite/20260520000000_add_resolved_repos.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9b48b2..2ee8050 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,6 +159,182 @@ jobs: - name: Clippy run: cargo clippy --all-targets -- -D warnings + # --------------------------------------------------------------------------- + # PR binaries — build on every PR push so reviewers can test + # --------------------------------------------------------------------------- + pr-binary: + needs: changes + if: >- + github.event_name == 'pull_request' + && needs.changes.outputs.server == 'true' + runs-on: ${{ matrix.runs-on }} + strategy: + fail-fast: false + matrix: + include: + - runs-on: depot-ubuntu-24.04 + arch: amd64 + - runs-on: depot-ubuntu-24.04-arm + arch: arm64 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Cache cargo + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: cargo-release-${{ matrix.arch }}-${{ hashFiles('Cargo.lock') }} + restore-keys: | + cargo-release-${{ matrix.arch }}- + + - name: Build release binary + run: | + docker run --rm \ + -v "$PWD:/app" \ + -v "$HOME/.cargo/registry:/usr/local/cargo/registry" \ + -v "$HOME/.cargo/git:/usr/local/cargo/git" \ + -w /app \ + -e SQLX_OFFLINE=true \ + rust:1.93-bookworm \ + cargo build --release + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: happyview-linux-${{ matrix.arch }} + path: target/release/happyview + + pr-docker: + needs: pr-binary + runs-on: ${{ matrix.runs-on }} + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runs-on: depot-ubuntu-24.04 + arch: amd64 + - platform: linux/arm64 + runs-on: depot-ubuntu-24.04-arm + arch: arm64 + env: + GHCR_IMAGE: ghcr.io/${{ github.repository }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Download pre-built binary + uses: actions/download-artifact@v4 + with: + name: happyview-linux-${{ matrix.arch }} + path: .binary + + - name: Prepare binary override + run: | + mkdir -p .builder-override/app/target/release + cp .binary/happyview .builder-override/app/target/release/happyview + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.GHCR_IMAGE }} + tags: | + type=raw,value=pr-${{ github.event.pull_request.number }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v5 + with: + context: . + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + builder=.builder-override + cache-from: type=gha,scope=pr-${{ matrix.platform }} + cache-to: type=gha,scope=pr-${{ matrix.platform }},mode=max,ignore-error=true + outputs: type=image,"name=${{ env.GHCR_IMAGE }}",push-by-digest=true,name-canonical=true,push=true + + - name: Create manifest + run: | + TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '${{ steps.meta.outputs.json }}') + docker buildx imagetools create --append $TAGS \ + ${{ env.GHCR_IMAGE }}@${{ steps.build.outputs.digest }} 2>/dev/null || \ + docker buildx imagetools create $TAGS \ + ${{ env.GHCR_IMAGE }}@${{ steps.build.outputs.digest }} + + pr-build-comment: + needs: pr-docker + runs-on: depot-ubuntu-24.04 + permissions: + pull-requests: write + steps: + - name: Comment on PR + uses: actions/github-script@v7 + with: + script: | + const sha = context.payload.pull_request.head.sha.substring(0, 7); + const prNumber = context.payload.pull_request.number; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const image = `ghcr.io/${context.repo.owner}/${context.repo.repo}`; + const body = [ + '### PR Build', + '', + `Builds for \`${sha}\`:`, + '', + '**Docker**', + '```sh', + `docker pull ${image}:pr-${prNumber}`, + '```', + '', + '**Binaries**', + '```sh', + `gh run download ${context.runId} -n happyview-linux-amd64`, + `gh run download ${context.runId} -n happyview-linux-arm64`, + '```', + '', + `Or download binaries from the [workflow run](${runUrl}).`, + ].join('\n'); + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const existing = comments.find(c => c.body.startsWith('### PR Build')); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + # --------------------------------------------------------------------------- # Server release # --------------------------------------------------------------------------- diff --git a/migrations/postgres/20260520000000_add_resolved_repos.sql b/migrations/postgres/20260520000000_add_resolved_repos.sql new file mode 100644 index 0000000..da79bf0 --- /dev/null +++ b/migrations/postgres/20260520000000_add_resolved_repos.sql @@ -0,0 +1 @@ +ALTER TABLE backfill_jobs ADD COLUMN resolved_repos INTEGER; diff --git a/migrations/sqlite/20260520000000_add_resolved_repos.sql b/migrations/sqlite/20260520000000_add_resolved_repos.sql new file mode 100644 index 0000000..da79bf0 --- /dev/null +++ b/migrations/sqlite/20260520000000_add_resolved_repos.sql @@ -0,0 +1 @@ +ALTER TABLE backfill_jobs ADD COLUMN resolved_repos INTEGER; diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 553c3a5..26b7253 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -5,9 +5,11 @@ use std::sync::atomic::{AtomicBool, AtomicI32, Ordering}; use axum::Json; use axum::extract::{Path, State}; use axum::http::StatusCode; -use futures_util::stream::{self, StreamExt}; +use futures_util::FutureExt; +use futures_util::stream::{self, FuturesUnordered, StreamExt}; use serde::Deserialize; use serde_json::Value; +use tokio::sync::mpsc; use uuid::Uuid; use crate::AppState; @@ -69,6 +71,7 @@ async fn set_stage(state: &AppState, job_id: &str, stage: &str) { async fn update_job_counter(state: &AppState, job_id: &str, column: &str, value: i32) { let query = match column { "total_repos" => "UPDATE backfill_jobs SET total_repos = ? WHERE id = ?", + "resolved_repos" => "UPDATE backfill_jobs SET resolved_repos = ? WHERE id = ?", "processed_repos" => "UPDATE backfill_jobs SET processed_repos = ? WHERE id = ?", "total_records" => "UPDATE backfill_jobs SET total_records = ? WHERE id = ?", other => { @@ -317,69 +320,448 @@ async fn discover_repos_from_relay( } // --------------------------------------------------------------------------- -// Phase 2: Resolve PDS endpoints +// Pipelined Phase 2+3: Resolve PDS endpoints and fetch records concurrently // --------------------------------------------------------------------------- -async fn run_resolution_phase(state: &AppState, job_id: &str) { - set_stage(state, job_id, "resolving_pds").await; +async fn run_pipelined_resolve_and_fetch( + state: &AppState, + job_id: &str, + collections: &[String], +) -> (i32, i32) { + set_stage(state, job_id, "resolving_and_fetching").await; - let sql = adapt_sql( - "SELECT did FROM backfill_repos WHERE job_id = ? AND pds_endpoint IS NULL", + // Count already-resolved and already-completed repos for accurate progress + let already_resolved: i32 = { + let sql = adapt_sql( + "SELECT COUNT(*) FROM backfill_repos WHERE job_id = ? AND pds_endpoint IS NOT NULL", + state.db_backend, + ); + sqlx::query_as::<_, (i32,)>(&sql) + .bind(job_id) + .fetch_one(&state.db) + .await + .map(|(c,)| c) + .unwrap_or(0) + }; + + let already_completed: i32 = { + let sql = adapt_sql( + "SELECT COUNT(*) FROM backfill_repos WHERE job_id = ? AND status = 'completed'", + state.db_backend, + ); + sqlx::query_as::<_, (i32,)>(&sql) + .bind(job_id) + .fetch_one(&state.db) + .await + .map(|(c,)| c) + .unwrap_or(0) + }; + + update_job_counter(state, job_id, "resolved_repos", already_resolved).await; + update_job_counter(state, job_id, "processed_repos", already_completed).await; + + let existing_records: i32 = { + let sql = adapt_sql( + "SELECT total_records FROM backfill_jobs WHERE id = ?", + state.db_backend, + ); + sqlx::query_as::<_, (Option,)>(&sql) + .bind(job_id) + .fetch_one(&state.db) + .await + .map(|(c,)| c.unwrap_or(0)) + .unwrap_or(0) + }; + + // Shared atomics for lock-free counter updates + let resolved_repos = Arc::new(AtomicI32::new(already_resolved)); + let processed_repos = Arc::new(AtomicI32::new(already_completed)); + let total_records = Arc::new(AtomicI32::new(existing_records)); + let cancelled = Arc::new(AtomicBool::new(false)); + + let (tx, mut rx) = mpsc::channel::<(String, String)>(256); + let tx_resolver = tx.clone(); + let tx_backlog = tx.clone(); + + // --- Resolver task --- + let resolver_state = state.clone(); + let resolver_job_id = job_id.to_string(); + let resolver_resolved = Arc::clone(&resolved_repos); + let resolver_cancelled = Arc::clone(&cancelled); + + let resolver_handle = tokio::spawn(async move { + let sql = adapt_sql( + "SELECT did FROM backfill_repos WHERE job_id = ? AND pds_endpoint IS NULL", + resolver_state.db_backend, + ); + let unresolved: Vec<(String,)> = sqlx::query_as(&sql) + .bind(&resolver_job_id) + .fetch_all(&resolver_state.db) + .await + .unwrap_or_default(); + + let mut attempted: i32 = 0; + for (did,) in &unresolved { + if resolver_cancelled.load(Ordering::Relaxed) { + break; + } + + match profile::resolve_pds_endpoint( + &resolver_state.http, + &resolver_state.config.plc_url, + did, + ) + .await + { + Ok(pds) => { + let sql = adapt_sql( + "UPDATE backfill_repos SET pds_endpoint = ? WHERE job_id = ? AND did = ?", + resolver_state.db_backend, + ); + let _ = sqlx::query(&sql) + .bind(&pds) + .bind(&resolver_job_id) + .bind(did) + .execute(&resolver_state.db) + .await; + + let count = resolver_resolved.fetch_add(1, Ordering::Relaxed) + 1; + if count % 100 == 0 { + update_job_counter( + &resolver_state, + &resolver_job_id, + "resolved_repos", + count, + ) + .await; + } + + if tx_resolver.send((did.clone(), pds)).await.is_err() { + break; + } + } + Err(e) => { + tracing::warn!(did, error = %e, "failed to resolve PDS endpoint, skipping DID"); + } + } + + attempted += 1; + if attempted % 100 == 0 && is_cancelled(&resolver_state, &resolver_job_id).await { + resolver_cancelled.store(true, Ordering::Relaxed); + break; + } + } + + // Persist final resolved count + let final_resolved = resolver_resolved.load(Ordering::Relaxed); + update_job_counter( + &resolver_state, + &resolver_job_id, + "resolved_repos", + final_resolved, + ) + .await; + // tx is dropped here, signalling the fetcher that no more DIDs are coming + }); + + // --- Also send already-resolved-but-unfetched DIDs to the fetcher --- + let pending_sql = adapt_sql( + "SELECT did, pds_endpoint FROM backfill_repos WHERE job_id = ? AND status = 'pending' AND pds_endpoint IS NOT NULL", state.db_backend, ); - let unresolved: Vec<(String,)> = sqlx::query_as(&sql) + let pending_rows: Vec<(String, String)> = sqlx::query_as(&pending_sql) .bind(job_id) .fetch_all(&state.db) .await .unwrap_or_default(); + let backlog_cancelled = Arc::clone(&cancelled); + let backlog_handle = tokio::spawn(async move { + for (did, pds) in pending_rows { + if backlog_cancelled.load(Ordering::Relaxed) { + break; + } + if tx_backlog.send((did, pds)).await.is_err() { + break; + } + } + }); + + // Drop our copy of tx so the channel closes when both senders finish + drop(tx); + + // --- Fetcher: receive (did, pds) pairs and dispatch to PDS workers --- + // Each PDS gets its own worker with a DID channel. Workers acquire a + // semaphore permit before starting, limiting concurrent PDS connections. + // We never hold the workers lock across an `.await` — use `try_send` to + // avoid blocking when a worker's channel is full (overflow goes to a + // retry queue drained on each iteration). + let state = Arc::new(state.clone()); + let collections = Arc::new(collections.to_vec()); + let job_id_arc = Arc::new(job_id.to_string()); + + let pds_semaphore = Arc::new(tokio::sync::Semaphore::new(10)); + let mut pds_workers: HashMap> = HashMap::new(); + let mut worker_handles = FuturesUnordered::new(); + let mut overflow: Vec<(String, String)> = Vec::new(); + + while let Some((did, pds_endpoint)) = rx.recv().await { + // Also drain any overflow from previous iterations + overflow.push((did, pds_endpoint)); + + let mut still_pending = Vec::new(); + for (did, pds_endpoint) in overflow.drain(..) { + if cancelled.load(Ordering::Relaxed) { + break; + } + + // Try to send to an existing PDS worker + if let Some(pds_tx) = pds_workers.get(&pds_endpoint) { + match pds_tx.try_send(did.clone()) { + Ok(()) => continue, + Err(mpsc::error::TrySendError::Full(_)) => { + still_pending.push((did, pds_endpoint)); + continue; + } + Err(mpsc::error::TrySendError::Closed(_)) => { + // Worker finished, will be removed below + } + } + } + + // Remove stale workers whose channels have closed + pds_workers.retain(|_, tx| !tx.is_closed()); + + // Spawn a new PDS worker + let permit = Arc::clone(&pds_semaphore); + let (pds_tx, pds_rx) = mpsc::channel::(64); + let _ = pds_tx.try_send(did); + pds_workers.insert(pds_endpoint.clone(), pds_tx); + + let ctx = FetchContext { + state: Arc::clone(&state), + job_id: Arc::clone(&job_id_arc), + collections: Arc::clone(&collections), + processed_repos: Arc::clone(&processed_repos), + total_records: Arc::clone(&total_records), + cancelled: Arc::clone(&cancelled), + }; + + worker_handles.push(tokio::spawn(async move { + let _permit = permit + .acquire() + .await + .expect("semaphore should not be closed"); + + run_pds_worker(ctx, pds_endpoint, pds_rx).await; + })); + } + overflow = still_pending; + + // Drain any completed worker handles to avoid unbounded accumulation + while let Some(result) = worker_handles.next().now_or_never() { + if let Some(Err(e)) = result { + tracing::warn!(error = %e, "PDS worker task panicked"); + } + } + } + + // Drain remaining overflow after channel closes + for (did, pds_endpoint) in overflow.drain(..) { + if cancelled.load(Ordering::Relaxed) { + break; + } + + // Remove stale workers + pds_workers.retain(|_, tx| !tx.is_closed()); + + if let Some(pds_tx) = pds_workers.get(&pds_endpoint) { + // Channel is bounded; this can block, but all senders are done so it's fine + let _ = pds_tx.send(did).await; + continue; + } + + let permit = Arc::clone(&pds_semaphore); + let (pds_tx, pds_rx) = mpsc::channel::(64); + let _ = pds_tx.try_send(did); + pds_workers.insert(pds_endpoint.clone(), pds_tx); + + let ctx = FetchContext { + state: Arc::clone(&state), + job_id: Arc::clone(&job_id_arc), + collections: Arc::clone(&collections), + processed_repos: Arc::clone(&processed_repos), + total_records: Arc::clone(&total_records), + cancelled: Arc::clone(&cancelled), + }; + + worker_handles.push(tokio::spawn(async move { + let _permit = permit + .acquire() + .await + .expect("semaphore should not be closed"); + + run_pds_worker(ctx, pds_endpoint.clone(), pds_rx).await; + })); + } + + // Drop all PDS senders so workers know no more DIDs are coming + drop(pds_workers); + + // Wait for all PDS workers to finish + while let Some(result) = worker_handles.next().await { + if let Err(e) = result { + tracing::warn!(error = %e, "PDS worker task panicked"); + } + } + + // Wait for resolver and backlog tasks + let _ = resolver_handle.await; + let _ = backlog_handle.await; + + let final_repos = processed_repos.load(Ordering::Relaxed); + let final_records = total_records.load(Ordering::Relaxed); + + // Persist final counts let sql = adapt_sql( - "SELECT COUNT(*) FROM backfill_repos WHERE job_id = ? AND pds_endpoint IS NOT NULL", + "UPDATE backfill_jobs SET processed_repos = ?, total_records = ? WHERE id = ?", state.db_backend, ); - let already_resolved: i32 = sqlx::query_as::<_, (i32,)>(&sql) + let _ = sqlx::query(&sql) + .bind(final_repos) + .bind(final_records) .bind(job_id) - .fetch_one(&state.db) - .await - .map(|(c,)| c) - .unwrap_or(0); + .execute(&state.db) + .await; + + (final_repos, final_records) +} + +struct FetchContext { + state: Arc, + job_id: Arc, + collections: Arc>, + processed_repos: Arc, + total_records: Arc, + cancelled: Arc, +} - let mut resolved_count = already_resolved; +async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::Receiver) { + let FetchContext { + state, + job_id, + collections, + processed_repos, + total_records, + cancelled, + } = ctx; + let mut fetches = FuturesUnordered::new(); + let mut rx_open = true; - let mut attempted = already_resolved; - for (did,) in &unresolved { - match profile::resolve_pds_endpoint(&state.http, &state.config.plc_url, did).await { - Ok(pds) => { + loop { + tokio::select! { + biased; + + Some(result) = fetches.next(), if !fetches.is_empty() => { + let (did, records): (String, i32) = result; + total_records.fetch_add(records, Ordering::Relaxed); + + // Mark DID as completed let sql = adapt_sql( - "UPDATE backfill_repos SET pds_endpoint = ? WHERE job_id = ? AND did = ?", + "UPDATE backfill_repos SET status = 'completed' WHERE job_id = ? AND did = ?", state.db_backend, ); let _ = sqlx::query(&sql) - .bind(&pds) - .bind(job_id) - .bind(did) + .bind(job_id.as_str()) + .bind(&did) .execute(&state.db) .await; - resolved_count += 1; - } - Err(e) => { - tracing::warn!(did, error = %e, "failed to resolve PDS endpoint, skipping DID"); + + let repos = processed_repos.fetch_add(1, Ordering::Relaxed) + 1; + if repos % 10 == 0 { + let records = total_records.load(Ordering::Relaxed); + let sql = adapt_sql( + "UPDATE backfill_jobs SET processed_repos = ?, total_records = ? WHERE id = ?", + state.db_backend, + ); + let _ = sqlx::query(&sql) + .bind(repos) + .bind(records) + .bind(job_id.as_str()) + .execute(&state.db) + .await; + + if is_cancelled(&state, job_id.as_str()).await { + cancelled.store(true, Ordering::Relaxed); + break; + } + } } - } - attempted += 1; - if attempted % 100 == 0 { - update_job_counter(state, job_id, "processed_repos", resolved_count).await; - if is_cancelled(state, job_id).await { - return; + + did = rx.recv(), if rx_open && fetches.len() < 3 => { + match did { + Some(did) if !cancelled.load(Ordering::Relaxed) => { + let state = Arc::clone(&state); + let collections = collections.clone(); + let pds_endpoint = pds_endpoint.clone(); + + fetches.push(async move { + let mut count: i32 = 0; + for collection in collections.iter() { + match fetch_records_from_pds( + &state, + &pds_endpoint, + &did, + collection, + ) + .await + { + Ok(c) => count += c as i32, + Err(e) => { + tracing::warn!( + did, + collection, + pds = %pds_endpoint, + error = %e, + "failed to fetch records from PDS" + ); + } + } + } + (did, count) + }); + } + _ => { + rx_open = false; + } + } } + + else => break, } } - update_job_counter(state, job_id, "processed_repos", resolved_count).await; + // Drain any remaining fetches + while let Some(result) = fetches.next().await { + let (did, records): (String, i32) = result; + total_records.fetch_add(records, Ordering::Relaxed); + + let sql = adapt_sql( + "UPDATE backfill_repos SET status = 'completed' WHERE job_id = ? AND did = ?", + state.db_backend, + ); + let _ = sqlx::query(&sql) + .bind(job_id.as_str()) + .bind(&did) + .execute(&state.db) + .await; + + processed_repos.fetch_add(1, Ordering::Relaxed); + } } // --------------------------------------------------------------------------- -// Phase 3: Fetch records from PDS instances +// Phase 3: Fetch records from PDS instances (legacy, for resumed jobs) // --------------------------------------------------------------------------- async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[String]) -> (i32, i32) { @@ -713,20 +1095,15 @@ async fn run_backfill_job(state: AppState, job_id: String) { } } - if matches!( + let (final_processed, final_records) = if matches!( stage.as_str(), - "pending" | "discovering_repos" | "resolving_pds" + "pending" | "discovering_repos" | "resolving_pds" | "resolving_and_fetching" ) { - run_resolution_phase(&state, &job_id).await; - - if is_cancelled(&state, &job_id).await { - tracing::info!(job_id, "backfill job cancelled"); - finalise_cancel(&state, &job_id).await; - return; - } - } - - let (final_processed, final_records) = run_fetching_phase(&state, &job_id, &collections).await; + run_pipelined_resolve_and_fetch(&state, &job_id, &collections).await + } else { + // stage == "fetching_records": resolution already done (legacy or resumed) + run_fetching_phase(&state, &job_id, &collections).await + }; if is_cancelled(&state, &job_id).await { tracing::info!(job_id, "backfill job cancelled"); @@ -871,7 +1248,7 @@ pub(super) async fn backfill_status( let backend = state.db_backend; let sql = adapt_sql( - "SELECT id, collection, did, status, stage, total_repos, processed_repos, total_records, error, started_at, completed_at, created_at FROM backfill_jobs ORDER BY created_at DESC", + "SELECT id, collection, did, status, stage, total_repos, resolved_repos, processed_repos, total_records, error, started_at, completed_at, created_at FROM backfill_jobs ORDER BY created_at DESC", backend, ); #[allow(clippy::type_complexity)] @@ -884,6 +1261,7 @@ pub(super) async fn backfill_status( Option, Option, Option, + Option, Option, Option, Option, @@ -903,6 +1281,7 @@ pub(super) async fn backfill_status( status, stage, total_repos, + resolved_repos, processed_repos, total_records, error, @@ -917,6 +1296,7 @@ pub(super) async fn backfill_status( status, stage, total_repos, + resolved_repos, processed_repos, total_records, error, diff --git a/src/admin/types.rs b/src/admin/types.rs index 4871b2e..3a5c0d1 100644 --- a/src/admin/types.rs +++ b/src/admin/types.rs @@ -76,6 +76,7 @@ pub(crate) struct BackfillJob { pub(crate) status: String, pub(crate) stage: String, pub(crate) total_repos: Option, + pub(crate) resolved_repos: Option, pub(crate) processed_repos: Option, pub(crate) total_records: Option, pub(crate) error: Option, diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index f2ed0b6..e3f1060 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -90,6 +90,7 @@ function statusBadge(job: BackfillJob) { } function phaseIndex(stage: string): number { + if (stage === "resolving_and_fetching") return 1; const idx = PROGRESS_PHASES.indexOf( stage as (typeof PROGRESS_PHASES)[number], ); @@ -229,6 +230,9 @@ function JobDetail({ function hasReached(phase: (typeof PROGRESS_PHASES)[number]): boolean { if (allDone) return true; + if (job.stage === "resolving_and_fetching") { + return phase === "discovering_repos" || phase === "resolving_pds" || phase === "fetching_records"; + } return current >= phaseIndex(phase); } @@ -307,26 +311,36 @@ function JobDetail({ /> Date: Thu, 21 May 2026 11:32:30 -0500 Subject: [PATCH 02/28] feat: add more detailed information to backfill UI Signed-off-by: Trezy --- Cargo.lock | 39 +- Cargo.toml | 1 + .../20260521000000_backfill_diagnostics.sql | 2 + .../20260521000000_backfill_diagnostics.sql | 2 + src/admin/backfill.rs | 396 +++++++++++- src/admin/mod.rs | 14 + src/admin/settings.rs | 1 + src/admin/types.rs | 71 ++ src/lib.rs | 1 + src/lua/atproto_api.rs | 1 + src/lua/db_api.rs | 1 + src/lua/execute.rs | 1 + src/lua/http_api.rs | 1 + src/lua/xrpc_api.rs | 1 + src/main.rs | 10 + tests/common/app.rs | 1 + tests/lua_atproto_api.rs | 1 + tests/lua_db_api.rs | 1 + web/package-lock.json | 28 + web/package.json | 1 + web/src/app/dashboard/backfill/page.tsx | 610 ++++++++++++++++-- .../app/dashboard/settings/general/page.tsx | 44 ++ web/src/lib/api.ts | 32 +- web/src/types/backfill.ts | 45 ++ 24 files changed, 1231 insertions(+), 74 deletions(-) create mode 100644 migrations/postgres/20260521000000_backfill_diagnostics.sql create mode 100644 migrations/sqlite/20260521000000_backfill_diagnostics.sql diff --git a/Cargo.lock b/Cargo.lock index 29026e6..69eeee1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,6 +157,28 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -483,7 +505,7 @@ dependencies = [ "cap-primitives", "cap-std", "io-lifetimes", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -512,7 +534,7 @@ dependencies = [ "maybe-owned", "rustix 1.1.3", "rustix-linux-procfs", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "winx", ] @@ -1293,7 +1315,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix 1.1.3", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1377,7 +1399,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" dependencies = [ "io-lifetimes", "rustix 1.1.3", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1626,6 +1648,7 @@ dependencies = [ "aes-gcm", "anyhow", "arc-swap", + "async-stream", "atrium-api", "atrium-common", "atrium-identity", @@ -2095,7 +2118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2285ddfe3054097ef4b2fe909ef8c3bcd1ea52a8f0d274416caebeef39f04a65" dependencies = [ "io-lifetimes", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3264,7 +3287,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3975,7 +3998,7 @@ dependencies = [ "fd-lock", "io-lifetimes", "rustix 0.38.44", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "winx", ] @@ -5544,7 +5567,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" dependencies = [ "bitflags", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index aaf858d..40bc34f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ wasmtime = { version = "29", features = ["async"] } wasmtime-wasi = "29" regex = "1.12.3" semver = "1.0" +async-stream = "0.3.6" [[bin]] name = "migrate-lua-sql" diff --git a/migrations/postgres/20260521000000_backfill_diagnostics.sql b/migrations/postgres/20260521000000_backfill_diagnostics.sql new file mode 100644 index 0000000..f80f7c7 --- /dev/null +++ b/migrations/postgres/20260521000000_backfill_diagnostics.sql @@ -0,0 +1,2 @@ +ALTER TABLE backfill_repos ADD COLUMN records_fetched INTEGER NOT NULL DEFAULT 0; +CREATE INDEX IF NOT EXISTS idx_backfill_repos_job_status ON backfill_repos (job_id, status); diff --git a/migrations/sqlite/20260521000000_backfill_diagnostics.sql b/migrations/sqlite/20260521000000_backfill_diagnostics.sql new file mode 100644 index 0000000..f80f7c7 --- /dev/null +++ b/migrations/sqlite/20260521000000_backfill_diagnostics.sql @@ -0,0 +1,2 @@ +ALTER TABLE backfill_repos ADD COLUMN records_fetched INTEGER NOT NULL DEFAULT 0; +CREATE INDEX IF NOT EXISTS idx_backfill_repos_job_status ON backfill_repos (job_id, status); diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 26b7253..c9c3d8c 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -12,6 +12,8 @@ use serde_json::Value; use tokio::sync::mpsc; use uuid::Uuid; +use rand::Rng; + use crate::AppState; use crate::db::{adapt_sql, now_rfc3339}; use crate::error::AppError; @@ -66,6 +68,13 @@ async fn set_stage(state: &AppState, job_id: &str, stage: &str) { .bind(job_id) .execute(&state.db) .await; + publish_event( + state, + super::types::BackfillEvent::JobStageChanged { + job_id: job_id.to_string(), + stage: stage.to_string(), + }, + ); } async fn update_job_counter(state: &AppState, job_id: &str, column: &str, value: i32) { @@ -103,12 +112,13 @@ async fn count_repos(state: &AppState, job_id: &str) -> i32 { .unwrap_or(0) } -async fn cleanup_repos(state: &AppState, job_id: &str) { - let sql = adapt_sql( - "DELETE FROM backfill_repos WHERE job_id = ?", - state.db_backend, - ); - let _ = sqlx::query(&sql).bind(job_id).execute(&state.db).await; +fn publish_event(state: &AppState, event: super::types::BackfillEvent) { + let _ = state.backfill_events_tx.send(event); +} + +fn random_batch_threshold(base: i32) -> i32 { + let low = base - base / 10; + rand::rng().random_range(low..=base) } async fn fail_job(state: &AppState, job_id: &str, error: &str) { @@ -123,7 +133,14 @@ async fn fail_job(state: &AppState, job_id: &str, error: &str) { .bind(job_id) .execute(&state.db) .await; - cleanup_repos(state, job_id).await; + publish_event( + state, + super::types::BackfillEvent::JobCompleted { + job_id: job_id.to_string(), + status: "failed".to_string(), + error: Some(error.to_string()), + }, + ); } async fn is_cancelled(state: &AppState, job_id: &str) -> bool { @@ -159,7 +176,14 @@ async fn finalise_cancel(state: &AppState, job_id: &str) { .bind(job_id) .execute(&state.db) .await; - cleanup_repos(state, job_id).await; + publish_event( + state, + super::types::BackfillEvent::JobCompleted { + job_id: job_id.to_string(), + status: "cancelled".to_string(), + error: None, + }, + ); } async fn complete_job( @@ -182,7 +206,14 @@ async fn complete_job( .bind(job_id) .execute(&state.db) .await; - cleanup_repos(state, job_id).await; + publish_event( + state, + super::types::BackfillEvent::JobCompleted { + job_id: job_id.to_string(), + status: "completed".to_string(), + error: error.map(|e| e.to_string()), + }, + ); } // --------------------------------------------------------------------------- @@ -207,6 +238,13 @@ async fn run_discovery_phase( .bind(did) .execute(&state.db) .await; + publish_event( + state, + super::types::BackfillEvent::RepoDiscovered { + job_id: job_id.to_string(), + did: did.to_string(), + }, + ); } else { for collection in collections { if is_cancelled(state, job_id).await { @@ -301,6 +339,15 @@ async fn discover_repos_from_relay( if let Ok(result) = query.execute(&state.db).await { running_total += result.rows_affected() as i32; } + for repo in chunk { + publish_event( + state, + super::types::BackfillEvent::RepoDiscovered { + job_id: job_id.to_string(), + did: repo.did.clone(), + }, + ); + } } } @@ -401,6 +448,8 @@ async fn run_pipelined_resolve_and_fetch( .unwrap_or_default(); let mut attempted: i32 = 0; + let mut next_flush = random_batch_threshold(100); + let mut next_cancel_check = random_batch_threshold(100); for (did,) in &unresolved { if resolver_cancelled.load(Ordering::Relaxed) { break; @@ -425,8 +474,17 @@ async fn run_pipelined_resolve_and_fetch( .execute(&resolver_state.db) .await; + publish_event( + &resolver_state, + super::types::BackfillEvent::RepoResolved { + job_id: resolver_job_id.clone(), + did: did.clone(), + pds_endpoint: pds.clone(), + }, + ); + let count = resolver_resolved.fetch_add(1, Ordering::Relaxed) + 1; - if count % 100 == 0 { + if count >= next_flush { update_job_counter( &resolver_state, &resolver_job_id, @@ -434,7 +492,18 @@ async fn run_pipelined_resolve_and_fetch( count, ) .await; + next_flush = count + random_batch_threshold(100); } + publish_event( + &resolver_state, + super::types::BackfillEvent::JobCounters { + job_id: resolver_job_id.clone(), + total_repos: None, + resolved_repos: Some(count), + processed_repos: None, + total_records: None, + }, + ); if tx_resolver.send((did.clone(), pds)).await.is_err() { break; @@ -446,9 +515,12 @@ async fn run_pipelined_resolve_and_fetch( } attempted += 1; - if attempted % 100 == 0 && is_cancelled(&resolver_state, &resolver_job_id).await { - resolver_cancelled.store(true, Ordering::Relaxed); - break; + if attempted >= next_cancel_check { + if is_cancelled(&resolver_state, &resolver_job_id).await { + resolver_cancelled.store(true, Ordering::Relaxed); + break; + } + next_cancel_check = attempted + random_batch_threshold(100); } } @@ -657,6 +729,7 @@ async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::R } = ctx; let mut fetches = FuturesUnordered::new(); let mut rx_open = true; + let mut next_flush = random_batch_threshold(10); loop { tokio::select! { @@ -668,18 +741,26 @@ async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::R // Mark DID as completed let sql = adapt_sql( - "UPDATE backfill_repos SET status = 'completed' WHERE job_id = ? AND did = ?", + "UPDATE backfill_repos SET status = 'completed', records_fetched = ? WHERE job_id = ? AND did = ?", state.db_backend, ); let _ = sqlx::query(&sql) + .bind(records) .bind(job_id.as_str()) .bind(&did) .execute(&state.db) .await; + publish_event(&state, super::types::BackfillEvent::RepoFetched { + job_id: job_id.to_string(), + did: did.clone(), + pds_endpoint: pds_endpoint.clone(), + records_fetched: records, + }); + let repos = processed_repos.fetch_add(1, Ordering::Relaxed) + 1; - if repos % 10 == 0 { - let records = total_records.load(Ordering::Relaxed); + let records = total_records.load(Ordering::Relaxed); + if repos >= next_flush { let sql = adapt_sql( "UPDATE backfill_jobs SET processed_repos = ?, total_records = ? WHERE id = ?", state.db_backend, @@ -695,7 +776,15 @@ async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::R cancelled.store(true, Ordering::Relaxed); break; } + next_flush = repos + random_batch_threshold(10); } + publish_event(&state, super::types::BackfillEvent::JobCounters { + job_id: job_id.to_string(), + total_repos: None, + resolved_repos: None, + processed_repos: Some(repos), + total_records: Some(records), + }); } did = rx.recv(), if rx_open && fetches.len() < 3 => { @@ -747,15 +836,26 @@ async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::R total_records.fetch_add(records, Ordering::Relaxed); let sql = adapt_sql( - "UPDATE backfill_repos SET status = 'completed' WHERE job_id = ? AND did = ?", + "UPDATE backfill_repos SET status = 'completed', records_fetched = ? WHERE job_id = ? AND did = ?", state.db_backend, ); let _ = sqlx::query(&sql) + .bind(records) .bind(job_id.as_str()) .bind(&did) .execute(&state.db) .await; + publish_event( + &state, + super::types::BackfillEvent::RepoFetched { + job_id: job_id.to_string(), + did: did.clone(), + pds_endpoint: pds_endpoint.clone(), + records_fetched: records, + }, + ); + processed_repos.fetch_add(1, Ordering::Relaxed); } } @@ -815,6 +915,9 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin let processed_repos = Arc::new(AtomicI32::new(already_completed)); let total_records = Arc::new(AtomicI32::new(existing_records)); let cancelled = Arc::new(AtomicBool::new(false)); + let next_flush = Arc::new(AtomicI32::new( + already_completed + random_batch_threshold(10), + )); let state = Arc::new(state.clone()); let collections = Arc::new(collections.to_vec()); let job_id_arc = Arc::new(job_id.to_string()); @@ -828,6 +931,7 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin let processed_repos = Arc::clone(&processed_repos); let total_records = Arc::clone(&total_records); let cancelled = Arc::clone(&cancelled); + let next_flush = Arc::clone(&next_flush); let job_id = Arc::clone(&job_id_arc); async move { @@ -838,6 +942,7 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin let processed_repos = Arc::clone(&processed_repos); let total_records = Arc::clone(&total_records); let cancelled = Arc::clone(&cancelled); + let next_flush = Arc::clone(&next_flush); let pds_endpoint = pds_endpoint.clone(); let job_id = Arc::clone(&job_id); @@ -846,6 +951,7 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin return; } + let mut did_records: i32 = 0; for collection in collections.iter() { match fetch_records_from_pds( &state, @@ -856,6 +962,7 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin .await { Ok(count) => { + did_records += count as i32; total_records .fetch_add(count as i32, Ordering::Relaxed); } @@ -873,19 +980,23 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin // Mark DID as completed let sql = adapt_sql( - "UPDATE backfill_repos SET status = 'completed' WHERE job_id = ? AND did = ?", + "UPDATE backfill_repos SET status = 'completed', records_fetched = ? WHERE job_id = ? AND did = ?", state.db_backend, ); let _ = sqlx::query(&sql) + .bind(did_records) .bind(job_id.as_str()) .bind(&did) .execute(&state.db) .await; let repos = processed_repos.fetch_add(1, Ordering::Relaxed) + 1; + let records = total_records.load(Ordering::Relaxed); - if repos % 10 == 0 { - let records = total_records.load(Ordering::Relaxed); + let threshold = next_flush.load(Ordering::Relaxed); + if repos >= threshold + && next_flush.compare_exchange(threshold, repos + random_batch_threshold(10), Ordering::Relaxed, Ordering::Relaxed).is_ok() + { let backend = state.db_backend; let sql = adapt_sql( "UPDATE backfill_jobs SET processed_repos = ?, total_records = ? WHERE id = ?", @@ -902,6 +1013,14 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin cancelled.store(true, Ordering::Relaxed); } } + + publish_event(&state, super::types::BackfillEvent::JobCounters { + job_id: job_id.to_string(), + total_repos: None, + resolved_repos: None, + processed_repos: Some(repos), + total_records: Some(records), + }); } }) .await; @@ -1311,6 +1430,243 @@ pub(super) async fn backfill_status( Ok(Json(jobs)) } +// --------------------------------------------------------------------------- +// SSE events endpoint +// --------------------------------------------------------------------------- + +pub(super) async fn backfill_events( + State(state): State, + Path(job_id): Path, + auth: UserAuth, +) -> Result< + axum::response::sse::Sse< + impl futures_util::Stream>, + >, + AppError, +> { + auth.require(Permission::BackfillRead).await?; + + let mut rx = state.backfill_events_tx.subscribe(); + + let stream = async_stream::stream! { + loop { + match rx.recv().await { + Ok(event) => { + let event_job_id = match &event { + super::types::BackfillEvent::RepoDiscovered { job_id, .. } + | super::types::BackfillEvent::RepoResolved { job_id, .. } + | super::types::BackfillEvent::RepoFetched { job_id, .. } + | super::types::BackfillEvent::JobCounters { job_id, .. } + | super::types::BackfillEvent::JobStageChanged { job_id, .. } + | super::types::BackfillEvent::JobCompleted { job_id, .. } => job_id, + }; + if *event_job_id != job_id { + continue; + } + if let Ok(json) = serde_json::to_string(&event) { + yield Ok(axum::response::sse::Event::default().event("event").data(json)); + } + } + Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { + tracing::warn!(job_id, skipped = n, "SSE client lagged behind"); + continue; + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + } + } + }; + + Ok(axum::response::sse::Sse::new(stream).keep_alive(axum::response::sse::KeepAlive::default())) +} + +// --------------------------------------------------------------------------- +// REST detail endpoints +// --------------------------------------------------------------------------- + +#[derive(Deserialize)] +pub(super) struct ReposQuery { + phase: Option, + cursor: Option, + limit: Option, +} + +pub(super) async fn backfill_repos( + State(state): State, + Path(job_id): Path, + auth: UserAuth, + axum::extract::Query(query): axum::extract::Query, +) -> Result, AppError> { + auth.require(Permission::BackfillRead).await?; + + let limit = query.limit.unwrap_or(50).min(100); + let phase_filter = match query.phase.as_deref() { + Some("resolved") => " AND pds_endpoint IS NOT NULL", + Some("fetched") => " AND status = 'completed'", + _ => "", + }; + let cursor_filter = if query.cursor.is_some() { + " AND did > ?" + } else { + "" + }; + + let sql_str = format!( + "SELECT did, pds_endpoint, status, records_fetched FROM backfill_repos WHERE job_id = ?{phase_filter}{cursor_filter} ORDER BY did ASC LIMIT ?", + ); + let sql = adapt_sql(&sql_str, state.db_backend); + + let mut q = sqlx::query_as::<_, (String, Option, String, i32)>(&sql).bind(&job_id); + if let Some(ref cursor) = query.cursor { + q = q.bind(cursor); + } + q = q.bind(limit + 1); + + let rows: Vec<(String, Option, String, i32)> = q + .fetch_all(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to query backfill repos: {e}")))?; + + let has_more = rows.len() > limit as usize; + let repos: Vec = rows + .into_iter() + .take(limit as usize) + .map( + |(did, pds_endpoint, status, records_fetched)| super::types::BackfillRepoEntry { + did, + pds_endpoint, + status, + records_fetched, + }, + ) + .collect(); + + let cursor = if has_more { + repos.last().map(|r| r.did.clone()) + } else { + None + }; + + Ok(Json(super::types::BackfillReposResponse { repos, cursor })) +} + +pub(super) async fn backfill_pds_summary( + State(state): State, + Path(job_id): Path, + auth: UserAuth, +) -> Result, AppError> { + auth.require(Permission::BackfillRead).await?; + + let sql = adapt_sql( + "SELECT pds_endpoint, COUNT(*) as total_repos, SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) as completed_repos, SUM(records_fetched) as total_records FROM backfill_repos WHERE job_id = ? AND pds_endpoint IS NOT NULL GROUP BY pds_endpoint ORDER BY COUNT(*) DESC", + state.db_backend, + ); + + let rows: Vec<(String, i32, i32, i64)> = sqlx::query_as(&sql) + .bind(&job_id) + .fetch_all(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to query PDS summary: {e}")))?; + + let pds_endpoints: Vec = rows + .into_iter() + .map( + |(pds_endpoint, total_repos, completed_repos, total_records)| { + super::types::PdsSummaryEntry { + pds_endpoint, + total_repos, + completed_repos, + total_records: total_records as i32, + } + }, + ) + .collect(); + + Ok(Json(super::types::PdsSummaryResponse { pds_endpoints })) +} + +// --------------------------------------------------------------------------- +// Flush endpoints +// --------------------------------------------------------------------------- + +pub(super) async fn flush_backfill_details( + State(state): State, + Path(job_id): Path, + auth: UserAuth, +) -> Result { + auth.require(Permission::BackfillCreate).await?; + + let sql = adapt_sql( + "DELETE FROM backfill_repos WHERE job_id = ?", + state.db_backend, + ); + let _ = sqlx::query(&sql).bind(&job_id).execute(&state.db).await; + + Ok(StatusCode::NO_CONTENT) +} + +pub(super) async fn flush_all_backfill_details( + State(state): State, + auth: UserAuth, +) -> Result { + auth.require(Permission::BackfillCreate).await?; + + let sql = adapt_sql( + "DELETE FROM backfill_repos WHERE job_id IN (SELECT id FROM backfill_jobs WHERE status IN ('completed', 'cancelled', 'failed'))", + state.db_backend, + ); + let _ = sqlx::query(&sql).execute(&state.db).await; + + Ok(StatusCode::NO_CONTENT) +} + +// --------------------------------------------------------------------------- +// Retention cleanup +// --------------------------------------------------------------------------- + +pub async fn run_backfill_retention_cleanup(state: &AppState) { + use super::settings::get_setting; + + let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(86400)); + interval.tick().await; // first tick is immediate — run once on startup + + loop { + interval.tick().await; + + let retention_days: i64 = + get_setting(&state.db, "backfill_retention_days", state.db_backend) + .await + .and_then(|v| v.parse().ok()) + .unwrap_or(28); + + if retention_days == 0 { + continue; + } + + let cutoff = chrono::Utc::now() - chrono::Duration::days(retention_days); + let cutoff_str = cutoff.to_rfc3339(); + + let sql = adapt_sql( + "DELETE FROM backfill_repos WHERE job_id IN (SELECT id FROM backfill_jobs WHERE completed_at IS NOT NULL AND completed_at < ?)", + state.db_backend, + ); + match sqlx::query(&sql).bind(&cutoff_str).execute(&state.db).await { + Ok(result) => { + let deleted = result.rows_affected(); + if deleted > 0 { + tracing::info!( + deleted, + retention_days, + "cleaned up old backfill detail rows" + ); + } + } + Err(e) => { + tracing::warn!(error = %e, "backfill retention cleanup failed"); + } + } + } +} + // --------------------------------------------------------------------------- // Startup resumption // --------------------------------------------------------------------------- diff --git a/src/admin/mod.rs b/src/admin/mod.rs index fa7722a..5cba8b4 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -37,7 +37,21 @@ pub fn admin_routes(_state: AppState) -> Router { .route("/stats", get(stats::stats)) .route("/backfill", post(backfill::create_backfill)) .route("/backfill/status", get(backfill::backfill_status)) + .route( + "/backfill/details", + delete(backfill::flush_all_backfill_details), + ) .route("/backfill/{id}/cancel", post(backfill::cancel_backfill)) + .route("/backfill/{id}/events", get(backfill::backfill_events)) + .route("/backfill/{id}/repos", get(backfill::backfill_repos)) + .route( + "/backfill/{id}/pds-summary", + get(backfill::backfill_pds_summary), + ) + .route( + "/backfill/{id}/details", + delete(backfill::flush_backfill_details), + ) .route("/events", get(events::list_events)) .route("/users", post(users::create_user).get(users::list_users)) .route("/users/transfer-super", post(users::transfer_super)) diff --git a/src/admin/settings.rs b/src/admin/settings.rs index f86c065..8c6a138 100644 --- a/src/admin/settings.rs +++ b/src/admin/settings.rs @@ -16,6 +16,7 @@ use super::types::{SettingEntry, UpsertSettingBody}; const ENV_FALLBACKS: &[(&str, &str)] = &[ ("app_name", "APP_NAME"), + ("backfill_retention_days", "BACKFILL_RETENTION_DAYS"), ("client_uri", "CLIENT_URI"), ("feature.spaces_enabled", "FEATURE_SPACES_ENABLED"), ("logo_uri", "LOGO_URI"), diff --git a/src/admin/types.rs b/src/admin/types.rs index 3a5c0d1..2e903a8 100644 --- a/src/admin/types.rs +++ b/src/admin/types.rs @@ -85,6 +85,77 @@ pub(crate) struct BackfillJob { pub(crate) created_at: String, } +// --------------------------------------------------------------------------- +// Backfill event types +// --------------------------------------------------------------------------- + +#[derive(Clone, Debug, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum BackfillEvent { + RepoDiscovered { + job_id: String, + did: String, + }, + RepoResolved { + job_id: String, + did: String, + pds_endpoint: String, + }, + RepoFetched { + job_id: String, + did: String, + pds_endpoint: String, + records_fetched: i32, + }, + JobCounters { + job_id: String, + total_repos: Option, + resolved_repos: Option, + processed_repos: Option, + total_records: Option, + }, + JobStageChanged { + job_id: String, + stage: String, + }, + JobCompleted { + job_id: String, + status: String, + error: Option, + }, +} + +// --------------------------------------------------------------------------- +// Backfill detail response types +// --------------------------------------------------------------------------- + +#[derive(Serialize)] +pub(crate) struct BackfillRepoEntry { + pub(crate) did: String, + pub(crate) pds_endpoint: Option, + pub(crate) status: String, + pub(crate) records_fetched: i32, +} + +#[derive(Serialize)] +pub(crate) struct BackfillReposResponse { + pub(crate) repos: Vec, + pub(crate) cursor: Option, +} + +#[derive(Serialize)] +pub(crate) struct PdsSummaryEntry { + pub(crate) pds_endpoint: String, + pub(crate) total_repos: i32, + pub(crate) completed_repos: i32, + pub(crate) total_records: i32, +} + +#[derive(Serialize)] +pub(crate) struct PdsSummaryResponse { + pub(crate) pds_endpoints: Vec, +} + // --------------------------------------------------------------------------- // Network lexicon types // --------------------------------------------------------------------------- diff --git a/src/lib.rs b/src/lib.rs index fe19b38..9ea8e48 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -78,6 +78,7 @@ pub struct AppState { pub official_registry: SharedRegistry, pub official_registry_config: RegistryConfig, pub proxy_config: Arc>, + pub backfill_events_tx: tokio::sync::broadcast::Sender, } impl axum::extract::FromRef for axum_extra::extract::cookie::Key { diff --git a/src/lua/atproto_api.rs b/src/lua/atproto_api.rs index e70e488..560e638 100644 --- a/src/lua/atproto_api.rs +++ b/src/lua/atproto_api.rs @@ -582,6 +582,7 @@ mod tests { proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( crate::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/src/lua/db_api.rs b/src/lua/db_api.rs index 5899bbd..f22725b 100644 --- a/src/lua/db_api.rs +++ b/src/lua/db_api.rs @@ -790,6 +790,7 @@ mod tests { proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( crate::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/src/lua/execute.rs b/src/lua/execute.rs index 2e32cb9..97f5e6b 100644 --- a/src/lua/execute.rs +++ b/src/lua/execute.rs @@ -1180,6 +1180,7 @@ mod tests { proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( crate::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/src/lua/http_api.rs b/src/lua/http_api.rs index 39b7a22..2c20331 100644 --- a/src/lua/http_api.rs +++ b/src/lua/http_api.rs @@ -188,6 +188,7 @@ mod tests { proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( crate::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/src/lua/xrpc_api.rs b/src/lua/xrpc_api.rs index 308be59..6cb7860 100644 --- a/src/lua/xrpc_api.rs +++ b/src/lua/xrpc_api.rs @@ -292,6 +292,7 @@ mod tests { proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( crate::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/src/main.rs b/src/main.rs index 16f92fe..595bb0a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -612,6 +612,8 @@ async fn main() { std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new(config))) }; + let (backfill_events_tx, _) = tokio::sync::broadcast::channel(1024); + let state = AppState { config: config.clone(), http, @@ -631,6 +633,7 @@ async fn main() { official_registry, official_registry_config, proxy_config, + backfill_events_tx, }; jetstream::spawn(state.clone(), collections_rx); @@ -646,6 +649,13 @@ async fn main() { happyview::admin::backfill::resume_backfill_jobs(&state).await; + { + let state = state.clone(); + tokio::spawn(async move { + happyview::admin::backfill::run_backfill_retention_cleanup(&state).await; + }); + } + let app = server::router(state); let addr = config.listen_addr(); diff --git a/tests/common/app.rs b/tests/common/app.rs index f39914b..2331947 100644 --- a/tests/common/app.rs +++ b/tests/common/app.rs @@ -167,6 +167,7 @@ impl TestApp { proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( happyview::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, }; let router = server::router(state.clone()).layer(axum::middleware::from_fn( diff --git a/tests/lua_atproto_api.rs b/tests/lua_atproto_api.rs index 4946ced..459a9b6 100644 --- a/tests/lua_atproto_api.rs +++ b/tests/lua_atproto_api.rs @@ -101,6 +101,7 @@ async fn test_state_with_pool(pool: sqlx::AnyPool, backend: DatabaseBackend) -> proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( happyview::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/tests/lua_db_api.rs b/tests/lua_db_api.rs index 1eb8de3..cd6a621 100644 --- a/tests/lua_db_api.rs +++ b/tests/lua_db_api.rs @@ -104,6 +104,7 @@ async fn test_state_with_pool(pool: sqlx::AnyPool, backend: DatabaseBackend) -> proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( happyview::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/web/package-lock.json b/web/package-lock.json index ed30f1a..df10472 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -13,6 +13,7 @@ "@tabler/icons-react": "^3.36.1", "@tailwindcss/typography": "^0.5.19", "@tanstack/react-table": "^8.21.3", + "@tanstack/react-virtual": "^3.13.25", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", @@ -4143,6 +4144,23 @@ "react-dom": ">=16.8" } }, + "node_modules/@tanstack/react-virtual": { + "version": "3.13.25", + "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.13.25.tgz", + "integrity": "sha512-bmNoqMu6gcAW9JGrKVB0Q1tN1i5RONZF8r1fW0bbE4Oyf3DwEGnzzQJ2OW+Ozg1P4s8PyugkHg2ULZoFQN+cqw==", + "license": "MIT", + "dependencies": { + "@tanstack/virtual-core": "3.15.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/@tanstack/table-core": { "version": "8.21.3", "resolved": "https://registry.npmjs.org/@tanstack/table-core/-/table-core-8.21.3.tgz", @@ -4156,6 +4174,16 @@ "url": "https://github.com/sponsors/tannerlinsley" } }, + "node_modules/@tanstack/virtual-core": { + "version": "3.15.0", + "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.15.0.tgz", + "integrity": "sha512-0AwPGx0I8QxPYjAxShT/+z+ZOe9u8mW5rsXvivCTjRfRmz9a43+3mRyi4wwlyoUqOC56q/jatKa0Bh9M99BEHQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, "node_modules/@ts-morph/common": { "version": "0.27.0", "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.27.0.tgz", diff --git a/web/package.json b/web/package.json index 6286c50..517f9d7 100644 --- a/web/package.json +++ b/web/package.json @@ -14,6 +14,7 @@ "@tabler/icons-react": "^3.36.1", "@tailwindcss/typography": "^0.5.19", "@tanstack/react-table": "^8.21.3", + "@tanstack/react-virtual": "^3.13.25", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index e3f1060..3331075 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -1,18 +1,44 @@ "use client"; -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useCurrentUser } from "@/hooks/use-current-user"; import { cancelBackfillJob, createBackfillJob, getBackfillJobs, + getBackfillRepos, + getBackfillPdsSummary, + flushBackfillDetails, + flushAllBackfillDetails, getLexicons, } from "@/lib/api"; -import type { BackfillJob } from "@/types/backfill"; -import { CheckCircle2, Circle, Loader2 } from "lucide-react"; +import type { + BackfillJob, + BackfillRepoEntry, + PdsSummaryEntry, + BackfillEvent, + BlueskyProfile, +} from "@/types/backfill"; +import { CheckCircle2, ChevronRight, Circle, Loader2 } from "lucide-react"; import { SiteHeader } from "@/components/site-header"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, +} from "@/components/ui/alert-dialog"; import { Badge } from "@/components/ui/badge"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from "@/components/ui/collapsible"; import { Button } from "@/components/ui/button"; import { Combobox, @@ -97,6 +123,257 @@ function phaseIndex(stage: string): number { return idx; } +// SSE hook for backfill events +function useBackfillSSE(jobId: string | null, active: boolean): BackfillEvent[] { + const [events, setEvents] = useState([]); + + useEffect(() => { + if (!jobId || !active) { + setEvents([]); + return; + } + + const basePath = process.env.NEXT_PUBLIC_BASE_PATH || ""; + const es = new EventSource(`${basePath}/admin/backfill/${jobId}/events`, { + withCredentials: true, + }); + + es.addEventListener("event", (e) => { + try { + const event: BackfillEvent = JSON.parse((e as MessageEvent).data); + setEvents((prev) => [...prev, event]); + } catch { /* ignore parse errors */ } + }); + + return () => es.close(); + }, [jobId, active]); + + return events; +} + +// Batch Bluesky profile resolution hook +function useBlueskyProfiles(dids: string[]): Map { + const [profiles, setProfiles] = useState>(new Map()); + const resolvedRef = useRef>(new Set()); + const pendingRef = useRef(false); + + useEffect(() => { + const unresolved = dids.filter((d) => !resolvedRef.current.has(d)); + if (unresolved.length === 0 || pendingRef.current) return; + + pendingRef.current = true; + + const batches: string[][] = []; + for (let i = 0; i < unresolved.length; i += 25) { + batches.push(unresolved.slice(i, i + 25)); + } + + // Mark all as resolved immediately to prevent re-fetching + for (const did of unresolved) { + resolvedRef.current.add(did); + } + + Promise.all( + batches.map(async (batch) => { + const params = batch.map((d) => `actors=${encodeURIComponent(d)}`).join("&"); + try { + const resp = await fetch( + `https://public.api.bsky.app/xrpc/app.bsky.actor.getProfiles?${params}` + ); + if (!resp.ok) return []; + const data = await resp.json(); + return (data.profiles || []) as BlueskyProfile[]; + } catch { + return []; + } + }) + ).then((results) => { + setProfiles((prev) => { + const newProfiles = new Map(prev); + for (const batch of results) { + for (const p of batch) { + newProfiles.set(p.did, p); + } + } + return newProfiles; + }); + pendingRef.current = false; + }); + }, [dids]); + + return profiles; +} + +const BSKY_PDS_SUFFIX = ".bsky.network"; +const BSKY_PDS_HOSTNAMES = ["bsky.social", "staging.bsky.dev"]; +const failedFaviconUrls = new Set(); + +function isBskyPds(pdsEndpoint: string): boolean { + try { + const hostname = new URL(pdsEndpoint).hostname; + return BSKY_PDS_HOSTNAMES.includes(hostname) || hostname.endsWith(BSKY_PDS_SUFFIX); + } catch { + return false; + } +} + +function PdsFavicon({ pdsEndpoint }: { pdsEndpoint: string }) { + let hostname: string; + try { + hostname = new URL(pdsEndpoint).hostname; + } catch { + return ; + } + + if (isBskyPds(pdsEndpoint)) { + return ( + + + + ); + } + + const faviconUrl = `https://twenty-icons.com/${hostname}`; + + if (failedFaviconUrls.has(faviconUrl)) { + return ; + } + + return ; +} + +function PdsFaviconImg({ url }: { url: string }) { + const [failed, setFailed] = useState(false); + + if (failed) return ; + + return ( + { failedFaviconUrls.add(url); setFailed(true); }} + /> + ); +} + +function PdsPlaceholderIcon() { + return ( + + + + + + + + ); +} + +function ScrollSentinel({ onVisible }: { onVisible: () => void }) { + const ref = useRef(null); + const onVisibleRef = useRef(onVisible); + onVisibleRef.current = onVisible; + + useEffect(() => { + const el = ref.current; + if (!el) return; + const observer = new IntersectionObserver( + ([entry]) => { if (entry.isIntersecting) onVisibleRef.current(); }, + { rootMargin: "100px" }, + ); + observer.observe(el); + return () => observer.disconnect(); + }, []); + + return
; +} + +function AnimatedNumber({ value }: { value: number }) { + const targetRef = useRef(value); + const displayedRef = useRef(value); + const [displayed, setDisplayed] = useState(value); + const rafRef = useRef(0); + + targetRef.current = value; + + useEffect(() => { + cancelAnimationFrame(rafRef.current); + + function tick() { + const current = displayedRef.current; + const target = targetRef.current; + const diff = target - current; + + if (Math.abs(diff) < 0.5) { + displayedRef.current = target; + setDisplayed(target); + return; + } + + const next = current + diff * 0.06; + displayedRef.current = next; + setDisplayed(next); + rafRef.current = requestAnimationFrame(tick); + } + + tick(); + return () => cancelAnimationFrame(rafRef.current); + }, [value]); + + return <>{Math.round(displayed).toLocaleString()}; +} + +function CompactRepoRow({ did, profile }: { + did: string; + profile?: BlueskyProfile; +}) { + return ( +
+
+ {profile?.avatar && ( + + )} +
+ + {profile?.handle ? `@${profile.handle}` : {did}} + +
+ ); +} + +function ProfileRow({ did, profile, suffix }: { + did: string; + profile?: BlueskyProfile; + suffix?: React.ReactNode; +}) { + return ( +
+
+ {profile?.avatar && ( + + )} +
+
+

+ {profile?.displayName || profile?.handle || did} +

+ {profile?.handle && ( +

@{profile.handle}

+ )} + {!profile?.handle && ( +

{did}

+ )} +
+ {suffix && ( + {suffix} + )} +
+ ); +} + export default function BackfillPage() { const { hasPermission } = useCurrentUser(); const [jobs, setJobs] = useState([]); @@ -119,6 +396,7 @@ export default function BackfillPage() { }, [load]); const selectedJob = jobs.find((j) => j.id === selectedJobId) ?? null; + const canFlush = hasPermission("backfill:create"); return ( <> @@ -128,9 +406,32 @@ export default function BackfillPage() {

Backfill Jobs

- {hasPermission("backfill:create") && ( - - )} +
+ {canFlush && ( + + + + + + + Clear all job details? + + This will permanently delete per-repo detail data for all backfill jobs. + + + + Cancel + { + await flushAllBackfillDetails(); + }}>Clear + + + + )} + {hasPermission("backfill:create") && ( + + )} +
@@ -201,6 +502,7 @@ export default function BackfillPage() { { await cancelBackfillJob(selectedJob.id); load(); @@ -217,10 +519,12 @@ export default function BackfillPage() { function JobDetail({ job, canCancel, + canFlush, onCancel, }: { job: BackfillJob; canCancel: boolean; + canFlush: boolean; onCancel: () => Promise; }) { const [cancelling, setCancelling] = useState(false); @@ -228,6 +532,21 @@ function JobDetail({ const allDone = job.status === "completed"; const isActive = job.status === "running" || job.status === "cancelling"; + // Detail data state + const [discoveredRepos, setDiscoveredRepos] = useState([]); + const [discoveredCursor, setDiscoveredCursor] = useState(null); + const [discoveredLoaded, setDiscoveredLoaded] = useState(false); + + const [pdsSummary, setPdsSummary] = useState([]); + const [pdsLoaded, setPdsLoaded] = useState(false); + + const [fetchedRepos, setFetchedRepos] = useState([]); + const [fetchedCursor, setFetchedCursor] = useState(null); + const [fetchedLoaded, setFetchedLoaded] = useState(false); + + // SSE events for active jobs + const sseEvents = useBackfillSSE(job.id, isActive); + function hasReached(phase: (typeof PROGRESS_PHASES)[number]): boolean { if (allDone) return true; if (job.stage === "resolving_and_fetching") { @@ -245,6 +564,102 @@ function JobDetail({ } } + // Auto-load detail data when phases are reached + const discoveredReached = hasReached("discovering_repos"); + const pdsReached = hasReached("resolving_pds"); + const fetchedReached = hasReached("fetching_records") || job.stage === "resolving_and_fetching"; + + useEffect(() => { + if (discoveredReached && !discoveredLoaded) { + getBackfillRepos(job.id, { phase: "discovered", limit: 50 }) + .then((resp) => { setDiscoveredRepos(resp.repos); setDiscoveredCursor(resp.cursor); setDiscoveredLoaded(true); }) + .catch(() => {}); + } + }, [job.id, discoveredReached, discoveredLoaded]); + + useEffect(() => { + if (pdsReached && !pdsLoaded) { + getBackfillPdsSummary(job.id) + .then((resp) => { setPdsSummary(resp.pds_endpoints); setPdsLoaded(true); }) + .catch(() => {}); + } + }, [job.id, pdsReached, pdsLoaded]); + + useEffect(() => { + if (fetchedReached && !fetchedLoaded) { + getBackfillRepos(job.id, { phase: "fetched", limit: 50 }) + .then((resp) => { setFetchedRepos(resp.repos); setFetchedCursor(resp.cursor); setFetchedLoaded(true); }) + .catch(() => {}); + } + }, [job.id, fetchedReached, fetchedLoaded]); + + const loadMoreDiscovered = useCallback(async () => { + if (!discoveredCursor) return; + try { + const resp = await getBackfillRepos(job.id, { phase: "discovered", cursor: discoveredCursor, limit: 50 }); + setDiscoveredRepos((prev) => [...prev, ...resp.repos]); + setDiscoveredCursor(resp.cursor); + } catch { /* ignore */ } + }, [job.id, discoveredCursor]); + + const loadMoreFetched = useCallback(async () => { + if (!fetchedCursor) return; + try { + const resp = await getBackfillRepos(job.id, { phase: "fetched", cursor: fetchedCursor, limit: 50 }); + setFetchedRepos((prev) => [...prev, ...resp.repos]); + setFetchedCursor(resp.cursor); + } catch { /* ignore */ } + }, [job.id, fetchedCursor]); + + // Process SSE events + useEffect(() => { + for (const event of sseEvents) { + if (event.type === "repo_discovered" && event.did) { + setDiscoveredRepos((prev) => { + if (prev.some((r) => r.did === event.did)) return prev; + return [{ did: event.did!, pds_endpoint: null, status: "pending", records_fetched: 0 }, ...prev]; + }); + } + if (event.type === "repo_resolved" && event.did && event.pds_endpoint) { + setDiscoveredRepos((prev) => + prev.map((r) => r.did === event.did ? { ...r, pds_endpoint: event.pds_endpoint! } : r) + ); + setPdsSummary((prev) => { + const idx = prev.findIndex((p) => p.pds_endpoint === event.pds_endpoint); + if (idx >= 0) { + const updated = [...prev]; + updated[idx] = { ...updated[idx], total_repos: updated[idx].total_repos + 1 }; + return updated; + } + return [...prev, { pds_endpoint: event.pds_endpoint!, total_repos: 1, completed_repos: 0, total_records: 0 }]; + }); + } + if (event.type === "repo_fetched" && event.did) { + setFetchedRepos((prev) => { + if (prev.some((r) => r.did === event.did)) return prev; + return [{ did: event.did!, pds_endpoint: event.pds_endpoint ?? null, status: "completed", records_fetched: event.records_fetched ?? 0 }, ...prev]; + }); + setPdsSummary((prev) => { + if (!event.pds_endpoint) return prev; + return prev.map((p) => p.pds_endpoint === event.pds_endpoint + ? { ...p, completed_repos: p.completed_repos + 1, total_records: p.total_records + (event.records_fetched ?? 0) } + : p + ); + }); + } + } + }, [sseEvents]); + + // Collect all visible DIDs for profile resolution + const allVisibleDids = useMemo(() => { + const dids = new Set(); + for (const r of discoveredRepos) dids.add(r.did); + for (const r of fetchedRepos) dids.add(r.did); + return Array.from(dids); + }, [discoveredRepos, fetchedRepos]); + + const profiles = useBlueskyProfiles(allVisibleDids); + return ( <> @@ -306,9 +721,27 @@ function JobDetail({ label="Discovering repos" active={isActive && job.stage === "discovering_repos"} reached={hasReached("discovering_repos")} - value={job.total_repos?.toLocaleString()} + value={job.total_repos != null ? : undefined} suffix="repos found" - /> + loading={discoveredReached && !discoveredLoaded} + > + {discoveredRepos.length > 0 ? ( +
+ {discoveredRepos.map((repo) => ( + + ))} + {discoveredCursor && ( + + )} +
+ ) : discoveredLoaded ? ( +

No repos discovered yet.

+ ) : null} + / : undefined } suffix="resolved" - /> + loading={pdsReached && !pdsLoaded} + > + {pdsSummary.length > 0 ? ( +
+ {pdsSummary + .sort((a, b) => b.total_repos - a.total_repos) + .map((pds) => ( +
+ + {new URL(pds.pds_endpoint).hostname} + + / repos · records + +
+ ))} +
+ ) : pdsLoaded ? ( +

No PDS data yet.

+ ) : null} + / repos : undefined } suffix={ hasReached("fetching_records") || job.stage === "resolving_and_fetching" - ? `${job.total_records?.toLocaleString() ?? "0"} records` + ? <> records : undefined } - /> + loading={fetchedReached && !fetchedLoaded} + > + {fetchedRepos.filter((r) => r.records_fetched > 0).length > 0 ? ( +
+ {fetchedRepos.filter((r) => r.records_fetched > 0).map((repo) => ( + records} + /> + ))} + {fetchedCursor && ( + + )} +
+ ) : fetchedLoaded ? ( +

No repos fetched yet.

+ ) : null} +
- {canCancel && isActive && ( - + + {canFlush && !isActive && ( + + + + + + + Clear job details? + + This will permanently delete per-repo detail data for this backfill job. + + + + Cancel + { + await flushBackfillDetails(job.id); + setDiscoveredRepos([]); + setDiscoveredCursor(null); + setDiscoveredLoaded(false); + setPdsSummary([]); + setPdsLoaded(false); + setFetchedRepos([]); + setFetchedCursor(null); + setFetchedLoaded(false); + }}>Clear + + + + )} + {canCancel && isActive && ( - - )} + )} + ); } @@ -370,42 +870,64 @@ function ProgressRow({ reached, value, suffix, + loading, + children, }: { label: string; active: boolean; reached: boolean; - value?: string; - suffix?: string; + value?: React.ReactNode; + suffix?: React.ReactNode; + loading?: boolean; + children?: React.ReactNode; }) { + const [open, setOpen] = useState(false); const done = reached && !active; + const expandable = reached; return ( -
- - {active ? ( - - ) : done ? ( - - ) : ( - - )} - - {label} - {reached && value && ( - - {value} - {suffix ? ` · ${suffix}` : ""} - + + +
+ + {active ? ( + + ) : done ? ( + + ) : ( + + )} + + {label} + {reached && value && ( + + {value} + {suffix && <> · {suffix}} + + )} + {expandable && ( + + )} +
+ + {expandable && ( + +
+ {loading ? ( +
+ +
+ ) : children} +
+ )} -
+ ); } diff --git a/web/src/app/dashboard/settings/general/page.tsx b/web/src/app/dashboard/settings/general/page.tsx index 731ac34..9d04415 100644 --- a/web/src/app/dashboard/settings/general/page.tsx +++ b/web/src/app/dashboard/settings/general/page.tsx @@ -23,6 +23,7 @@ const SETTING_KEYS = [ "logo_uri", "tos_uri", "policy_uri", + "backfill_retention_days", ] as const type FieldKey = (typeof SETTING_KEYS)[number] @@ -80,6 +81,7 @@ export default function GeneralSettingsPage() { logo_uri: "", tos_uri: "", policy_uri: "", + backfill_retention_days: "28", }) const [sources, setSources] = useState>({ app_name: "unset", @@ -87,6 +89,7 @@ export default function GeneralSettingsPage() { logo_uri: "unset", tos_uri: "unset", policy_uri: "unset", + backfill_retention_days: "unset", }) const [logoUploaded, setLogoUploaded] = useState(false) const [error, setError] = useState(null) @@ -104,6 +107,7 @@ export default function GeneralSettingsPage() { logo_uri: byKey.get("logo_uri")?.value ?? "", tos_uri: byKey.get("tos_uri")?.value ?? "", policy_uri: byKey.get("policy_uri")?.value ?? "", + backfill_retention_days: byKey.get("backfill_retention_days")?.value ?? "28", }) setSources({ app_name: (byKey.get("app_name")?.source as "database" | "env" | undefined) ?? "unset", @@ -111,6 +115,7 @@ export default function GeneralSettingsPage() { logo_uri: (byKey.get("logo_uri")?.source as "database" | "env" | undefined) ?? "unset", tos_uri: (byKey.get("tos_uri")?.source as "database" | "env" | undefined) ?? "unset", policy_uri: (byKey.get("policy_uri")?.source as "database" | "env" | undefined) ?? "unset", + backfill_retention_days: (byKey.get("backfill_retention_days")?.source as "database" | "env" | undefined) ?? "unset", }) setLogoUploaded(byKey.has("logo_data")) } catch (e: unknown) { @@ -137,6 +142,14 @@ export default function GeneralSettingsPage() { await upsertSetting(field.key, value) } } + const retentionValue = values["backfill_retention_days"] + if (retentionValue === "") { + if (sources["backfill_retention_days"] === "database") { + await deleteSetting("backfill_retention_days") + } + } else { + await upsertSetting("backfill_retention_days", retentionValue) + } setNotice("Settings saved.") await load() } catch (e: unknown) { @@ -252,6 +265,37 @@ export default function GeneralSettingsPage() { +
+

Data Retention

+

+ Configure how long HappyView retains detailed data from completed backfill jobs. +

+
+ +
+
+ + {sources["backfill_retention_days"] === "env" && ( + + from env var + + )} +
+ + setValues((v) => ({ ...v, backfill_retention_days: e.target.value })) + } + placeholder="28" + disabled={!canManage} + /> +

+ How long to keep per-repo detail data from completed backfill jobs. Set to 0 to keep indefinitely. +

+
+
+
+

Backfill Performance

+

+ Tune concurrency limits for backfill jobs. Changes apply to the next job started. +

+
+ + {([ + { key: "backfill_concurrent_resolution" as const, id: "backfill_concurrent_resolution", label: "Concurrent PLC Resolutions", placeholder: "100", description: "How many DID document lookups to run in parallel during PDS resolution." }, + { key: "backfill_concurrent_pds" as const, id: "backfill_concurrent_pds", label: "Concurrent PDS Hosts", placeholder: "10", description: "How many PDS servers to fetch records from simultaneously." }, + { key: "backfill_concurrent_dids_per_pds" as const, id: "backfill_concurrent_dids_per_pds", label: "Concurrent DIDs per PDS", placeholder: "3", description: "How many repos to fetch concurrently from each PDS host." }, + ]).map((field) => ( +
+
+ + {sources[field.key] === "env" && ( + from env var + )} +
+ setValues((v) => ({ ...v, [field.key]: e.target.value }))} + placeholder={field.placeholder} + disabled={!canManage} + /> +

{field.description}

+
+ ))} +
{([ @@ -355,7 +391,7 @@ export default function GeneralSettingsPage() {
diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 4438ace..1c43284 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -365,6 +365,17 @@ export function getSettings() { return apiFetch("/admin/settings"); } +export type DbInfo = { + backend: "sqlite" | "postgres"; + server_max_connections: number | null; + main_pool_size: number; + backfill_pool_size: number; +}; + +export function getDbInfo() { + return apiFetch("/admin/settings/db-info"); +} + export function upsertSetting(key: string, value: string) { return apiFetch(`/admin/settings/${encodeURIComponent(key)}`, { method: "PUT", -- 2.51.2 From 7ff501afe91b8f45e0c7c34bf016a989ae0d36f5 Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 01:20:23 -0500 Subject: [PATCH 06/28] feat: increase backfill record batch inserts Signed-off-by: Trezy --- src/admin/backfill.rs | 131 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 122 insertions(+), 9 deletions(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 97b4a61..27b2b01 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -20,7 +20,6 @@ use crate::error::AppError; use crate::event_log::{EventLog, Severity, log_event}; use crate::http_retry::parse_retry_after; use crate::profile; -use crate::record_handler::{self, RecordEvent}; use super::auth::UserAuth; use super::permissions::Permission; @@ -1108,6 +1107,95 @@ async fn run_fetching_phase( (final_repos, final_records) } +struct PreparedRecord { + uri: String, + did: String, + collection: String, + rkey: String, + record_json: String, + cid: String, +} + +async fn batch_upsert_records(state: &AppState, batch: &[PreparedRecord]) { + if batch.is_empty() { + return; + } + + let backend = state.db_backend; + let now = now_rfc3339(); + + // Build multi-row INSERT. 8 params per row; ON CONFLICT uses EXCLUDED. + let placeholders: Vec = (0..batch.len()) + .map(|_| "(?, ?, ?, ?, ?, ?, ?, ?)".to_string()) + .collect(); + let raw_sql = format!( + "INSERT INTO records (uri, did, collection, rkey, record, cid, indexed_at, created_at) VALUES {} ON CONFLICT (uri) DO UPDATE SET record = EXCLUDED.record, cid = EXCLUDED.cid, indexed_at = EXCLUDED.indexed_at", + placeholders.join(", ") + ); + let sql = adapt_sql(&raw_sql, backend); + + let mut query = sqlx::query(&sql); + for rec in batch { + query = query + .bind(&rec.uri) + .bind(&rec.did) + .bind(&rec.collection) + .bind(&rec.rkey) + .bind(&rec.record_json) + .bind(&rec.cid) + .bind(&now) + .bind(&now); + } + + if let Err(e) = query.execute(&state.db).await { + tracing::warn!(batch_size = batch.len(), "batch record upsert failed: {e}"); + } + + // Batch sync_refs: delete old refs for all URIs, then insert new ones. + let uris: Vec<&str> = batch.iter().map(|r| r.uri.as_str()).collect(); + let delete_placeholders: Vec<&str> = (0..uris.len()).map(|_| "?").collect(); + let delete_raw = format!( + "DELETE FROM record_refs WHERE source_uri IN ({})", + delete_placeholders.join(", ") + ); + let delete_sql = adapt_sql(&delete_raw, backend); + let mut del_query = sqlx::query(&delete_sql); + for uri in &uris { + del_query = del_query.bind(*uri); + } + let _ = del_query.execute(&state.db).await; + + // Collect all new refs and batch insert them + let mut all_refs: Vec<(&str, String, &str)> = Vec::new(); + for rec in batch { + let record_val: serde_json::Value = + serde_json::from_str(&rec.record_json).unwrap_or_default(); + for target_uri in crate::record_refs::extract_at_uris(&record_val) { + all_refs.push((&rec.uri, target_uri, &rec.collection)); + } + } + + // Insert refs in chunks to stay within SQLite's param limit (3 params per ref) + for chunk in all_refs.chunks(300) { + let ref_placeholders: Vec<&str> = (0..chunk.len()).map(|_| "(?, ?, ?)").collect(); + let ref_raw = format!( + "INSERT INTO record_refs (source_uri, target_uri, collection) VALUES {} ON CONFLICT DO NOTHING", + ref_placeholders.join(", ") + ); + let ref_sql = adapt_sql(&ref_raw, backend); + let mut ref_query = sqlx::query(&ref_sql); + for (source, target, collection) in chunk { + ref_query = ref_query.bind(*source).bind(target).bind(*collection); + } + let _ = ref_query.execute(&state.db).await; + } + + // Queue label backfill for each record + for rec in batch { + crate::labeler::backfill_labels_for_uri(Arc::new(state.clone()), rec.uri.clone()); + } +} + /// Fetch all records for a given DID and collection from a PDS via /// `com.atproto.repo.listRecords`, paginating and handling rate limits. async fn fetch_records_from_pds( @@ -1153,22 +1241,47 @@ async fn fetch_records_from_pds( let page_count = body.records.len(); + let mut batch: Vec = Vec::with_capacity(page_count); for entry in &body.records { let rkey = entry.uri.rsplit('/').next().unwrap_or_default().to_string(); + let uri = format!("at://{did}/{collection}/{rkey}"); + + let rec_to_store = if let Some(script) = state.lexicons.get_index_hook(collection).await + { + let hook_result = crate::lua::execute_hook_script(&crate::lua::HookEvent { + state, + lexicon_id: collection, + script: &script, + action: "create", + uri: &uri, + did, + collection, + rkey: &rkey, + record: Some(&entry.value), + }) + .await; + + match hook_result { + None => continue, + Some(v) => v, + } + } else { + entry.value.clone() + }; - let event = RecordEvent { + batch.push(PreparedRecord { + uri, did: did.to_string(), collection: collection.to_string(), rkey, - action: "create".to_string(), - record: Some(entry.value.clone()), - cid: Some(entry.cid.clone()), - }; - - record_handler::handle_record_event(state, &event).await; - count += 1; + record_json: serde_json::to_string(&rec_to_store).unwrap_or_default(), + cid: entry.cid.clone(), + }); } + count += batch.len() as u32; + batch_upsert_records(state, &batch).await; + match body.cursor { Some(c) if page_count > 0 => cursor = Some(c), _ => break, -- 2.51.2 From 060cba303f392233554f1744ea4714ec09c51fa5 Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 01:23:16 -0500 Subject: [PATCH 07/28] feat: make backfill collection discovery concurrent Signed-off-by: Trezy --- src/admin/backfill.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 27b2b01..ba27f63 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -282,14 +282,16 @@ async fn run_discovery_phase( }, ); } else { - for collection in collections { - if is_cancelled(state, job_id).await { - return; - } - if let Err(e) = discover_repos_from_relay(state, job_id, collection).await { - tracing::warn!(collection, error = %e, "failed to discover repos, skipping"); - } - } + stream::iter(collections.iter()) + .for_each_concurrent(collections.len(), |collection| async move { + if is_cancelled(state, job_id).await { + return; + } + if let Err(e) = discover_repos_from_relay(state, job_id, collection).await { + tracing::warn!(collection, error = %e, "failed to discover repos, skipping"); + } + }) + .await; } let total = count_repos(state, job_id).await; -- 2.51.2 From 66b4804a7db626aabc27e354b5fb6da1c0ed4a9e Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:37:18 -0500 Subject: [PATCH 08/28] fix: get max connections from Postgres instead of current connections Signed-off-by: Trezy --- src/admin/settings.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/admin/settings.rs b/src/admin/settings.rs index de001a3..b91318e 100644 --- a/src/admin/settings.rs +++ b/src/admin/settings.rs @@ -200,8 +200,8 @@ pub(super) async fn db_info( None }; - let main_pool_size = state.db.size() as i64; - let backfill_pool_size = state.backfill_db.size() as i64; + let main_pool_size = state.db.options().get_max_connections() as i64; + let backfill_pool_size = state.backfill_db.options().get_max_connections() as i64; Ok(Json(serde_json::json!({ "backend": match state.db_backend { -- 2.51.2 From 22f3ba92acecba1c84b4355871eb617003f7a7ea Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:39:07 -0500 Subject: [PATCH 09/28] fix: use the correct connection pool for record upserts Signed-off-by: Trezy --- src/admin/backfill.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index ba27f63..73f5955 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -1149,7 +1149,7 @@ async fn batch_upsert_records(state: &AppState, batch: &[PreparedRecord]) { .bind(&now); } - if let Err(e) = query.execute(&state.db).await { + if let Err(e) = query.execute(&state.backfill_db).await { tracing::warn!(batch_size = batch.len(), "batch record upsert failed: {e}"); } @@ -1165,7 +1165,7 @@ async fn batch_upsert_records(state: &AppState, batch: &[PreparedRecord]) { for uri in &uris { del_query = del_query.bind(*uri); } - let _ = del_query.execute(&state.db).await; + let _ = del_query.execute(&state.backfill_db).await; // Collect all new refs and batch insert them let mut all_refs: Vec<(&str, String, &str)> = Vec::new(); @@ -1189,7 +1189,7 @@ async fn batch_upsert_records(state: &AppState, batch: &[PreparedRecord]) { for (source, target, collection) in chunk { ref_query = ref_query.bind(*source).bind(target).bind(*collection); } - let _ = ref_query.execute(&state.db).await; + let _ = ref_query.execute(&state.backfill_db).await; } // Queue label backfill for each record @@ -1700,7 +1700,7 @@ pub(super) async fn backfill_repos( q = q.bind(limit + 1); let rows: Vec<(String, Option, String, i32)> = q - .fetch_all(&state.db) + .fetch_all(&state.backfill_db) .await .map_err(|e| AppError::Internal(format!("failed to query backfill repos: {e}")))?; @@ -1741,7 +1741,7 @@ pub(super) async fn backfill_pds_summary( let rows: Vec<(String, i32, i32, i64)> = sqlx::query_as(&sql) .bind(&job_id) - .fetch_all(&state.db) + .fetch_all(&state.backfill_db) .await .map_err(|e| AppError::Internal(format!("failed to query PDS summary: {e}")))?; @@ -1777,7 +1777,10 @@ pub(super) async fn flush_backfill_details( "DELETE FROM backfill_repos WHERE job_id = ?", state.db_backend, ); - let _ = sqlx::query(&sql).bind(&job_id).execute(&state.db).await; + let _ = sqlx::query(&sql) + .bind(&job_id) + .execute(&state.backfill_db) + .await; Ok(StatusCode::NO_CONTENT) } @@ -1792,7 +1795,7 @@ pub(super) async fn flush_all_backfill_details( "DELETE FROM backfill_repos WHERE job_id IN (SELECT id FROM backfill_jobs WHERE status IN ('completed', 'cancelled', 'failed'))", state.db_backend, ); - let _ = sqlx::query(&sql).execute(&state.db).await; + let _ = sqlx::query(&sql).execute(&state.backfill_db).await; Ok(StatusCode::NO_CONTENT) } -- 2.51.2 From 9c2847ae2e3d3ff80117ee8cdd69341a9a17f471 Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:39:50 -0500 Subject: [PATCH 10/28] fix: clamp backfill concurrency values Signed-off-by: Trezy --- src/admin/backfill.rs | 9 ++++++--- src/db.rs | 5 +++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 73f5955..2e85569 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -134,11 +134,13 @@ async fn load_concurrency(state: &AppState) -> BackfillConcurrency { ) .await .and_then(|v| v.parse().ok()) - .unwrap_or(100); + .unwrap_or(100usize) + .max(1); let pds = super::settings::get_setting(&state.db, "backfill_concurrent_pds", state.db_backend) .await .and_then(|v| v.parse().ok()) - .unwrap_or(10); + .unwrap_or(10usize) + .max(1); let dids_per_pds = super::settings::get_setting( &state.db, "backfill_concurrent_dids_per_pds", @@ -146,7 +148,8 @@ async fn load_concurrency(state: &AppState) -> BackfillConcurrency { ) .await .and_then(|v| v.parse().ok()) - .unwrap_or(3); + .unwrap_or(3usize) + .max(1); BackfillConcurrency { resolution, pds, diff --git a/src/db.rs b/src/db.rs index 638fc02..6eb5cbc 100644 --- a/src/db.rs +++ b/src/db.rs @@ -286,7 +286,7 @@ pub async fn connect(url: &str, backend: DatabaseBackend) -> AnyPool { pub async fn connect_backfill_pool(url: &str, backend: DatabaseBackend) -> AnyPool { let max_connections: u32 = std::env::var("BACKFILL_DATABASE_MAX_CONNECTIONS") .ok() - .and_then(|v| v.parse().ok()) + .and_then(|v| v.parse::().ok()) .unwrap_or_else(|| { let pds: u32 = std::env::var("BACKFILL_CONCURRENT_PDS") .ok() @@ -308,7 +308,8 @@ pub async fn connect_backfill_pool(url: &str, backend: DatabaseBackend) -> AnyPo DatabaseBackend::Postgres => 256, }; needed.min(ceiling) - }); + }) + .max(1); tracing::info!(max_connections, "backfill pool sized"); -- 2.51.2 From bd745ae7d501251c40398065a40c7b9e69698502 Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:41:12 -0500 Subject: [PATCH 11/28] fix: remove cleanup delay caused by tick eater Signed-off-by: Trezy --- src/admin/backfill.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 2e85569..5e8c230 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -1811,7 +1811,6 @@ pub async fn run_backfill_retention_cleanup(state: &AppState) { use super::settings::get_setting; let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(86400)); - interval.tick().await; // first tick is immediate — run once on startup loop { interval.tick().await; -- 2.51.2 From 149d61eda9e81e4f38f9c3c05d6eb1080ef9658a Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:44:15 -0500 Subject: [PATCH 12/28] chore: remove unused module Signed-off-by: Trezy --- web/package-lock.json | 59 ++++++++++++++++++------------------------- web/package.json | 1 - 2 files changed, 25 insertions(+), 35 deletions(-) diff --git a/web/package-lock.json b/web/package-lock.json index df10472..65bd0ba 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -13,7 +13,6 @@ "@tabler/icons-react": "^3.36.1", "@tailwindcss/typography": "^0.5.19", "@tanstack/react-table": "^8.21.3", - "@tanstack/react-virtual": "^3.13.25", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", @@ -117,6 +116,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -774,6 +774,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -2044,6 +2045,7 @@ "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": "^14.21.3 || >=16" }, @@ -4144,23 +4146,6 @@ "react-dom": ">=16.8" } }, - "node_modules/@tanstack/react-virtual": { - "version": "3.13.25", - "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.13.25.tgz", - "integrity": "sha512-bmNoqMu6gcAW9JGrKVB0Q1tN1i5RONZF8r1fW0bbE4Oyf3DwEGnzzQJ2OW+Ozg1P4s8PyugkHg2ULZoFQN+cqw==", - "license": "MIT", - "dependencies": { - "@tanstack/virtual-core": "3.15.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/tannerlinsley" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" - } - }, "node_modules/@tanstack/table-core": { "version": "8.21.3", "resolved": "https://registry.npmjs.org/@tanstack/table-core/-/table-core-8.21.3.tgz", @@ -4174,16 +4159,6 @@ "url": "https://github.com/sponsors/tannerlinsley" } }, - "node_modules/@tanstack/virtual-core": { - "version": "3.15.0", - "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.15.0.tgz", - "integrity": "sha512-0AwPGx0I8QxPYjAxShT/+z+ZOe9u8mW5rsXvivCTjRfRmz9a43+3mRyi4wwlyoUqOC56q/jatKa0Bh9M99BEHQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/tannerlinsley" - } - }, "node_modules/@ts-morph/common": { "version": "0.27.0", "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.27.0.tgz", @@ -4368,6 +4343,7 @@ "integrity": "sha512-oH72nZRfDv9lADUBSo104Aq7gPHpQZc4BTx38r9xf9pg5LfP6EzSyH2n7qFmmxRQXh7YlUXODcYsg6PuTDSxGg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -4377,6 +4353,7 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -4387,6 +4364,7 @@ "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "devOptional": true, "license": "MIT", + "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -4410,8 +4388,7 @@ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", "license": "MIT", - "optional": true, - "peer": true + "optional": true }, "node_modules/@types/unist": { "version": "3.0.3", @@ -4477,6 +4454,7 @@ "integrity": "sha512-IgSWvLobTDOjnaxAfDTIHaECbkNlAlKv2j5SjpB2v7QHKv1FIfjwMy8FsDbVfDX/KjmCmYICcw7uGaXLhtsLNg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.0", "@typescript-eslint/types": "8.56.0", @@ -4990,6 +4968,7 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5376,6 +5355,7 @@ "integrity": "sha512-Ixm8tFfoKKIPYdCCKYTsqv+Fd4IJ0DQqMyEimo+pxUOMUR9cVPlwTrFt9Avu+3cb6Zp3mAzl+t1MrG2fxxKsxw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/types": "^7.26.0" } @@ -5480,6 +5460,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -6441,7 +6422,6 @@ "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.7.tgz", "integrity": "sha512-WhL/YuveyGXJaerVlMYGWhvQswa7myDG17P7Vu65EWC05o8vfeNbvNf4d/BOvH99+ZW+LlQsc1GDKMa1vNK6dw==", "license": "(MPL-2.0 OR Apache-2.0)", - "peer": true, "optionalDependencies": { "@types/trusted-types": "^2.0.7" } @@ -6780,6 +6760,7 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -6920,6 +6901,7 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -7398,6 +7380,7 @@ "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -8196,6 +8179,7 @@ "integrity": "sha512-Eaw2YTGM6WOxA6CXbckaEvslr2Ne4NFsKrvc0v97JD5awbmeBLO5w9Ho9L9kmKonrwF9RJlW6BxT1PVv/agBHQ==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=16.9.0" } @@ -9621,7 +9605,6 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-14.0.0.tgz", "integrity": "sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==", "license": "MIT", - "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -11778,6 +11761,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -11808,6 +11792,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz", "integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -11854,6 +11839,7 @@ "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.2.0.tgz", "integrity": "sha512-ROY9fvHhwOD9ySfrF0wmvu//bKCQ6AeZZq1nJNtbDC+kk5DuSuNX/n6YWYF/SYy7bSba4D4FSz8DJeKY/S/r+g==", "license": "MIT", + "peer": true, "dependencies": { "@types/use-sync-external-store": "^0.0.6", "use-sync-external-store": "^1.4.0" @@ -11992,7 +11978,8 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/redux-thunk": { "version": "3.1.0", @@ -13187,7 +13174,8 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.0.tgz", "integrity": "sha512-yYzTZ4++b7fNYxFfpnberEEKu43w44aqDMNM9MHMmcKuCH7lL8jJ4yJ7LGHv7rSwiqM0nkiobF9I6cLlpS2P7Q==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/tapable": { "version": "2.3.0", @@ -13260,6 +13248,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -13537,6 +13526,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -14286,6 +14276,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/web/package.json b/web/package.json index 517f9d7..6286c50 100644 --- a/web/package.json +++ b/web/package.json @@ -14,7 +14,6 @@ "@tabler/icons-react": "^3.36.1", "@tailwindcss/typography": "^0.5.19", "@tanstack/react-table": "^8.21.3", - "@tanstack/react-virtual": "^3.13.25", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", -- 2.51.2 From ce68c6694b2cc0f5d2789b938133577c5bc1de6b Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:44:56 -0500 Subject: [PATCH 13/28] fix: prevent reprocessing of sse Signed-off-by: Trezy --- web/src/app/dashboard/backfill/page.tsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index 3331075..b3e59ce 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -546,6 +546,7 @@ function JobDetail({ // SSE events for active jobs const sseEvents = useBackfillSSE(job.id, isActive); + const sseProcessedRef = useRef(0); function hasReached(phase: (typeof PROGRESS_PHASES)[number]): boolean { if (allDone) return true; @@ -611,9 +612,14 @@ function JobDetail({ } catch { /* ignore */ } }, [job.id, fetchedCursor]); - // Process SSE events + // Process SSE events — only handle events added since last render useEffect(() => { - for (const event of sseEvents) { + const start = sseProcessedRef.current; + if (start >= sseEvents.length) return; + sseProcessedRef.current = sseEvents.length; + + for (let i = start; i < sseEvents.length; i++) { + const event = sseEvents[i]; if (event.type === "repo_discovered" && event.did) { setDiscoveredRepos((prev) => { if (prev.some((r) => r.did === event.did)) return prev; -- 2.51.2 From 3d31877d207916c2170150357e423d237f793eac Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:45:41 -0500 Subject: [PATCH 14/28] fix: prevent concurrency settings from being set to invalid values Signed-off-by: Trezy --- web/src/app/dashboard/settings/general/page.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/src/app/dashboard/settings/general/page.tsx b/web/src/app/dashboard/settings/general/page.tsx index c7924a8..bc0180e 100644 --- a/web/src/app/dashboard/settings/general/page.tsx +++ b/web/src/app/dashboard/settings/general/page.tsx @@ -336,6 +336,8 @@ export default function GeneralSettingsPage() { setValues((v) => ({ ...v, backfill_retention_days: e.target.value })) @@ -379,6 +381,8 @@ export default function GeneralSettingsPage() { setValues((v) => ({ ...v, [field.key]: e.target.value }))} placeholder={field.placeholder} -- 2.51.2 From f83f15009187196dadb81fbf86f213ec2981ec9b Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:46:39 -0500 Subject: [PATCH 15/28] fix: make backfill progress headings more accessible Signed-off-by: Trezy --- web/src/app/dashboard/backfill/page.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index b3e59ce..c556d9c 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -894,8 +894,9 @@ function ProgressRow({ return ( -
)} -
+ {expandable && ( -- 2.51.2 From bd65caf32eafcc42ef65aae68559bae03b9f22a4 Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 20:14:07 -0500 Subject: [PATCH 16/28] feat: allow backfill jobs to be paused and resumed Signed-off-by: Trezy --- src/admin/backfill.rs | 296 +++++++++++++++--- src/admin/mod.rs | 2 + src/admin/settings.rs | 24 ++ web/package-lock.json | 31 +- web/src/app/dashboard/backfill/page.tsx | 107 ++++++- .../app/dashboard/settings/general/page.tsx | 248 +++++++++------ web/src/lib/api.ts | 14 + 7 files changed, 558 insertions(+), 164 deletions(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 5e8c230..4e1165d 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -179,23 +179,32 @@ async fn fail_job(state: &AppState, job_id: &str, error: &str) { ); } -async fn is_cancelled(state: &AppState, job_id: &str) -> bool { +async fn should_stop(state: &AppState, job_id: &str) -> Option<&'static str> { let sql = adapt_sql( "SELECT status FROM backfill_jobs WHERE id = ?", state.db_backend, ); - sqlx::query_as::<_, (String,)>(&sql) + let status = sqlx::query_as::<_, (String,)>(&sql) .bind(job_id) .fetch_optional(&state.backfill_db) .await .ok() .flatten() - .is_some_and(|(status,)| status == "cancelling") + .map(|(s,)| s); + match status.as_deref() { + Some("cancelling") => Some("cancelling"), + Some("pausing") => Some("pausing"), + _ => None, + } +} + +async fn should_stop_worker(state: &AppState, job_id: &str) -> bool { + should_stop(state, job_id).await.is_some() } async fn request_cancel(state: &AppState, job_id: &str) { let sql = adapt_sql( - "UPDATE backfill_jobs SET status = 'cancelling' WHERE id = ? AND status = 'running'", + "UPDATE backfill_jobs SET status = 'cancelling' WHERE id = ? AND status IN ('running', 'paused')", state.db_backend, ); let _ = sqlx::query(&sql) @@ -225,6 +234,36 @@ async fn finalise_cancel(state: &AppState, job_id: &str) { ); } +async fn request_pause(state: &AppState, job_id: &str) { + let sql = adapt_sql( + "UPDATE backfill_jobs SET status = 'pausing' WHERE id = ? AND status = 'running'", + state.db_backend, + ); + let _ = sqlx::query(&sql) + .bind(job_id) + .execute(&state.backfill_db) + .await; +} + +async fn finalise_pause(state: &AppState, job_id: &str) { + let sql = adapt_sql( + "UPDATE backfill_jobs SET status = 'paused' WHERE id = ?", + state.db_backend, + ); + let _ = sqlx::query(&sql) + .bind(job_id) + .execute(&state.backfill_db) + .await; + publish_event( + state, + super::types::BackfillEvent::JobCompleted { + job_id: job_id.to_string(), + status: "paused".to_string(), + error: None, + }, + ); +} + async fn complete_job( state: &AppState, job_id: &str, @@ -287,7 +326,7 @@ async fn run_discovery_phase( } else { stream::iter(collections.iter()) .for_each_concurrent(collections.len(), |collection| async move { - if is_cancelled(state, job_id).await { + if should_stop_worker(state, job_id).await { return; } if let Err(e) = discover_repos_from_relay(state, job_id, collection).await { @@ -394,7 +433,7 @@ async fn discover_repos_from_relay( update_job_counter(state, job_id, "total_repos", running_total).await; - if is_cancelled(state, job_id).await { + if should_stop_worker(state, job_id).await { return Ok(()); } @@ -492,7 +531,7 @@ async fn run_pipelined_resolve_and_fetch( let mut attempted: i32 = 0; let mut next_flush = random_batch_threshold(100); - let mut next_cancel_check = random_batch_threshold(100); + let mut next_cancel_check = random_batch_threshold(10); let stream_state = resolver_state.clone(); let stream_cancelled = Arc::clone(&resolver_cancelled); @@ -572,11 +611,11 @@ async fn run_pipelined_resolve_and_fetch( attempted += 1; if attempted >= next_cancel_check { - if is_cancelled(&resolver_state, &resolver_job_id).await { + if should_stop_worker(&resolver_state, &resolver_job_id).await { resolver_cancelled.store(true, Ordering::Relaxed); break; } - next_cancel_check = attempted + random_batch_threshold(100); + next_cancel_check = attempted + random_batch_threshold(10); } } @@ -633,7 +672,31 @@ async fn run_pipelined_resolve_and_fetch( let mut worker_handles = FuturesUnordered::new(); let mut overflow: Vec<(String, String)> = Vec::new(); - while let Some((did, pds_endpoint)) = rx.recv().await { + loop { + if cancelled.load(Ordering::Relaxed) { + break; + } + + let poll_state = Arc::clone(&state); + let poll_job_id = Arc::clone(&job_id_arc); + let poll_cancelled = Arc::clone(&cancelled); + let pair = tokio::select! { + biased; + result = rx.recv() => result, + _ = async move { + loop { + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + if poll_cancelled.load(Ordering::Relaxed) || should_stop_worker(&poll_state, &poll_job_id).await { + poll_cancelled.store(true, Ordering::Relaxed); + return; + } + } + } => None, + }; + let Some((did, pds_endpoint)) = pair else { + break; + }; + // Also drain any overflow from previous iterations overflow.push((did, pds_endpoint)); @@ -831,13 +894,12 @@ async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::R .bind(job_id.as_str()) .execute(&state.backfill_db) .await; - - if is_cancelled(&state, job_id.as_str()).await { - cancelled.store(true, Ordering::Relaxed); - break; - } next_flush = repos + random_batch_threshold(10); } + if cancelled.load(Ordering::Relaxed) || should_stop_worker(&state, job_id.as_str()).await { + cancelled.store(true, Ordering::Relaxed); + break; + } publish_event(&state, super::types::BackfillEvent::JobCounters { job_id: job_id.to_string(), total_repos: None, @@ -853,15 +915,20 @@ async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::R let state = Arc::clone(&state); let collections = collections.clone(); let pds_endpoint = pds_endpoint.clone(); + let cancelled = Arc::clone(&cancelled); fetches.push(async move { let mut count: i32 = 0; for collection in collections.iter() { + if cancelled.load(Ordering::Relaxed) { + break; + } match fetch_records_from_pds( &state, &pds_endpoint, &did, collection, + &cancelled, ) .await { @@ -1019,11 +1086,15 @@ async fn run_fetching_phase( let mut did_records: i32 = 0; for collection in collections.iter() { + if cancelled.load(Ordering::Relaxed) { + break; + } match fetch_records_from_pds( &state, &pds_endpoint, &did, collection, + &cancelled, ) .await { @@ -1075,7 +1146,7 @@ async fn run_fetching_phase( .execute(&state.backfill_db) .await; - if is_cancelled(&state, job_id.as_str()).await { + if should_stop_worker(&state, job_id.as_str()).await { cancelled.store(true, Ordering::Relaxed); } } @@ -1208,12 +1279,17 @@ async fn fetch_records_from_pds( pds_endpoint: &str, did: &str, collection: &str, + cancelled: &AtomicBool, ) -> Result { let base = pds_endpoint.trim_end_matches('/'); let mut cursor: Option = None; let mut count: u32 = 0; loop { + if cancelled.load(Ordering::Relaxed) { + break; + } + let mut url = format!( "{base}/xrpc/com.atproto.repo.listRecords?repo={did}&collection={collection}&limit=100" ); @@ -1365,10 +1441,18 @@ async fn run_backfill_job(state: AppState, job_id: String) { if matches!(stage.as_str(), "pending" | "discovering_repos") { run_discovery_phase(&state, &job_id, &collections, did.as_deref()).await; - if is_cancelled(&state, &job_id).await { - tracing::info!(job_id, "backfill job cancelled"); - finalise_cancel(&state, &job_id).await; - return; + match should_stop(&state, &job_id).await { + Some("cancelling") => { + tracing::info!(job_id, "backfill job cancelled"); + finalise_cancel(&state, &job_id).await; + return; + } + Some("pausing") => { + tracing::info!(job_id, "backfill job paused"); + finalise_pause(&state, &job_id).await; + return; + } + _ => {} } let total = count_repos(&state, &job_id).await; @@ -1405,10 +1489,18 @@ async fn run_backfill_job(state: AppState, job_id: String) { run_fetching_phase(&state, &job_id, &collections, &concurrency).await }; - if is_cancelled(&state, &job_id).await { - tracing::info!(job_id, "backfill job cancelled"); - finalise_cancel(&state, &job_id).await; - return; + match should_stop(&state, &job_id).await { + Some("cancelling") => { + tracing::info!(job_id, "backfill job cancelled"); + finalise_cancel(&state, &job_id).await; + return; + } + Some("pausing") => { + tracing::info!(job_id, "backfill job paused"); + finalise_pause(&state, &job_id).await; + return; + } + _ => {} } complete_job(&state, &job_id, final_processed, final_records, None).await; @@ -1515,6 +1607,24 @@ pub(super) async fn cancel_backfill( Some((ref status,)) if status == "cancelling" || status == "cancelled" => { Ok(Json(serde_json::json!({ "id": job_id, "status": status }))) } + Some((ref status,)) if status == "paused" => { + finalise_cancel(&state, &job_id).await; + log_event( + &state.db, + EventLog { + event_type: "backfill.cancelled".to_string(), + severity: Severity::Info, + actor_did: Some(admin.did.clone()), + subject: None, + detail: serde_json::json!({ "job_id": job_id }), + }, + state.db_backend, + ) + .await; + Ok(Json( + serde_json::json!({ "id": job_id, "status": "cancelled" }), + )) + } Some((status,)) if status != "running" => Err(AppError::BadRequest(format!( "job is not running (status: {status})" ))), @@ -1539,6 +1649,111 @@ pub(super) async fn cancel_backfill( } } +/// POST /admin/backfill/{id}/pause — pause a running backfill job. +pub(super) async fn pause_backfill( + State(state): State, + admin: UserAuth, + Path(job_id): Path, +) -> Result, AppError> { + admin.require(Permission::BackfillCreate).await?; + + let sql = adapt_sql( + "SELECT status FROM backfill_jobs WHERE id = ?", + state.db_backend, + ); + let row: Option<(String,)> = sqlx::query_as(&sql) + .bind(&job_id) + .fetch_optional(&state.backfill_db) + .await + .map_err(|e| AppError::Internal(format!("failed to query backfill job: {e}")))?; + + match row { + None => Err(AppError::NotFound("backfill job not found".into())), + Some((ref status,)) if status == "pausing" || status == "paused" => { + Ok(Json(serde_json::json!({ "id": job_id, "status": status }))) + } + Some((status,)) if status != "running" => Err(AppError::BadRequest(format!( + "job is not running (status: {status})" + ))), + Some(_) => { + request_pause(&state, &job_id).await; + log_event( + &state.db, + EventLog { + event_type: "backfill.pausing".to_string(), + severity: Severity::Info, + actor_did: Some(admin.did.clone()), + subject: None, + detail: serde_json::json!({ "job_id": job_id }), + }, + state.db_backend, + ) + .await; + Ok(Json( + serde_json::json!({ "id": job_id, "status": "pausing" }), + )) + } + } +} + +/// POST /admin/backfill/{id}/resume — resume a paused backfill job. +pub(super) async fn resume_backfill( + State(state): State, + admin: UserAuth, + Path(job_id): Path, +) -> Result, AppError> { + admin.require(Permission::BackfillCreate).await?; + + let sql = adapt_sql( + "SELECT status FROM backfill_jobs WHERE id = ?", + state.db_backend, + ); + let row: Option<(String,)> = sqlx::query_as(&sql) + .bind(&job_id) + .fetch_optional(&state.backfill_db) + .await + .map_err(|e| AppError::Internal(format!("failed to query backfill job: {e}")))?; + + match row { + None => Err(AppError::NotFound("backfill job not found".into())), + Some((status,)) if status != "paused" => Err(AppError::BadRequest(format!( + "job is not paused (status: {status})" + ))), + Some(_) => { + let sql = adapt_sql( + "UPDATE backfill_jobs SET status = 'running' WHERE id = ?", + state.db_backend, + ); + let _ = sqlx::query(&sql) + .bind(&job_id) + .execute(&state.backfill_db) + .await; + + let spawn_state = state.clone(); + let spawn_job_id = job_id.clone(); + tokio::spawn(async move { + run_backfill_job(spawn_state, spawn_job_id).await; + }); + + log_event( + &state.db, + EventLog { + event_type: "backfill.resumed".to_string(), + severity: Severity::Info, + actor_did: Some(admin.did.clone()), + subject: None, + detail: serde_json::json!({ "job_id": job_id }), + }, + state.db_backend, + ) + .await; + Ok(Json( + serde_json::json!({ "id": job_id, "status": "running" }), + )) + } + } +} + /// GET /admin/backfill/status — list all backfill jobs. pub(super) async fn backfill_status( State(state): State, @@ -1865,7 +2080,7 @@ pub async fn run_backfill_retention_cleanup(state: &AppState) { /// Jobs stuck in `cancelling` are finalised immediately. pub async fn resume_backfill_jobs(state: &AppState) { let sql = adapt_sql( - "SELECT id, status FROM backfill_jobs WHERE status IN ('running', 'cancelling')", + "SELECT id, status FROM backfill_jobs WHERE status IN ('running', 'cancelling', 'pausing')", state.db_backend, ); let rows: Vec<(String, String)> = sqlx::query_as(&sql) @@ -1874,18 +2089,25 @@ pub async fn resume_backfill_jobs(state: &AppState) { .unwrap_or_default(); for (job_id, status) in rows { - if status == "cancelling" { - tracing::info!( - job_id, - "finalising cancelled backfill job from previous run" - ); - finalise_cancel(state, &job_id).await; - } else { - tracing::info!(job_id, "resuming interrupted backfill job"); - let spawn_state = state.clone(); - tokio::spawn(async move { - run_backfill_job(spawn_state, job_id).await; - }); + match status.as_str() { + "cancelling" => { + tracing::info!( + job_id, + "finalising cancelled backfill job from previous run" + ); + finalise_cancel(state, &job_id).await; + } + "pausing" => { + tracing::info!(job_id, "finalising paused backfill job from previous run"); + finalise_pause(state, &job_id).await; + } + _ => { + tracing::info!(job_id, "resuming interrupted backfill job"); + let spawn_state = state.clone(); + tokio::spawn(async move { + run_backfill_job(spawn_state, job_id).await; + }); + } } } } diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 0bb7d4f..e0f50fa 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -42,6 +42,8 @@ pub fn admin_routes(_state: AppState) -> Router { delete(backfill::flush_all_backfill_details), ) .route("/backfill/{id}/cancel", post(backfill::cancel_backfill)) + .route("/backfill/{id}/pause", post(backfill::pause_backfill)) + .route("/backfill/{id}/resume", post(backfill::resume_backfill)) .route("/backfill/{id}/events", get(backfill::backfill_events)) .route("/backfill/{id}/repos", get(backfill::backfill_repos)) .route( diff --git a/src/admin/settings.rs b/src/admin/settings.rs index b91318e..f730f7b 100644 --- a/src/admin/settings.rs +++ b/src/admin/settings.rs @@ -203,6 +203,29 @@ pub(super) async fn db_info( let main_pool_size = state.db.options().get_max_connections() as i64; let backfill_pool_size = state.backfill_db.options().get_max_connections() as i64; + let pds: i64 = get_setting(&state.db, "backfill_concurrent_pds", state.db_backend) + .await + .and_then(|v| v.parse().ok()) + .unwrap_or(10); + let dids: i64 = get_setting( + &state.db, + "backfill_concurrent_dids_per_pds", + state.db_backend, + ) + .await + .and_then(|v| v.parse().ok()) + .unwrap_or(3); + let resolution: i64 = get_setting( + &state.db, + "backfill_concurrent_resolution", + state.db_backend, + ) + .await + .and_then(|v| v.parse().ok()) + .unwrap_or(100); + let needed_backfill_pool = (pds * dids) + resolution + 4; + let restart_recommended = needed_backfill_pool > backfill_pool_size; + Ok(Json(serde_json::json!({ "backend": match state.db_backend { DatabaseBackend::Sqlite => "sqlite", @@ -211,6 +234,7 @@ pub(super) async fn db_info( "server_max_connections": server_max, "main_pool_size": main_pool_size, "backfill_pool_size": backfill_pool_size, + "restart_recommended": restart_recommended, }))) } diff --git a/web/package-lock.json b/web/package-lock.json index 65bd0ba..ed30f1a 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -116,7 +116,6 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -774,7 +773,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -2045,7 +2043,6 @@ "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": "^14.21.3 || >=16" }, @@ -4343,7 +4340,6 @@ "integrity": "sha512-oH72nZRfDv9lADUBSo104Aq7gPHpQZc4BTx38r9xf9pg5LfP6EzSyH2n7qFmmxRQXh7YlUXODcYsg6PuTDSxGg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -4353,7 +4349,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -4364,7 +4359,6 @@ "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "devOptional": true, "license": "MIT", - "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -4388,7 +4382,8 @@ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", "license": "MIT", - "optional": true + "optional": true, + "peer": true }, "node_modules/@types/unist": { "version": "3.0.3", @@ -4454,7 +4449,6 @@ "integrity": "sha512-IgSWvLobTDOjnaxAfDTIHaECbkNlAlKv2j5SjpB2v7QHKv1FIfjwMy8FsDbVfDX/KjmCmYICcw7uGaXLhtsLNg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.0", "@typescript-eslint/types": "8.56.0", @@ -4968,7 +4962,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5355,7 +5348,6 @@ "integrity": "sha512-Ixm8tFfoKKIPYdCCKYTsqv+Fd4IJ0DQqMyEimo+pxUOMUR9cVPlwTrFt9Avu+3cb6Zp3mAzl+t1MrG2fxxKsxw==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/types": "^7.26.0" } @@ -5460,7 +5452,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -6422,6 +6413,7 @@ "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.7.tgz", "integrity": "sha512-WhL/YuveyGXJaerVlMYGWhvQswa7myDG17P7Vu65EWC05o8vfeNbvNf4d/BOvH99+ZW+LlQsc1GDKMa1vNK6dw==", "license": "(MPL-2.0 OR Apache-2.0)", + "peer": true, "optionalDependencies": { "@types/trusted-types": "^2.0.7" } @@ -6760,7 +6752,6 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -6901,7 +6892,6 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -7380,7 +7370,6 @@ "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -8179,7 +8168,6 @@ "integrity": "sha512-Eaw2YTGM6WOxA6CXbckaEvslr2Ne4NFsKrvc0v97JD5awbmeBLO5w9Ho9L9kmKonrwF9RJlW6BxT1PVv/agBHQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=16.9.0" } @@ -9605,6 +9593,7 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-14.0.0.tgz", "integrity": "sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==", "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -11761,7 +11750,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -11792,7 +11780,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz", "integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==", "license": "MIT", - "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -11839,7 +11826,6 @@ "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.2.0.tgz", "integrity": "sha512-ROY9fvHhwOD9ySfrF0wmvu//bKCQ6AeZZq1nJNtbDC+kk5DuSuNX/n6YWYF/SYy7bSba4D4FSz8DJeKY/S/r+g==", "license": "MIT", - "peer": true, "dependencies": { "@types/use-sync-external-store": "^0.0.6", "use-sync-external-store": "^1.4.0" @@ -11978,8 +11964,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/redux-thunk": { "version": "3.1.0", @@ -13174,8 +13159,7 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.0.tgz", "integrity": "sha512-yYzTZ4++b7fNYxFfpnberEEKu43w44aqDMNM9MHMmcKuCH7lL8jJ4yJ7LGHv7rSwiqM0nkiobF9I6cLlpS2P7Q==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/tapable": { "version": "2.3.0", @@ -13248,7 +13232,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -13526,7 +13509,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -14276,7 +14258,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index c556d9c..d344bcc 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -5,6 +5,8 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useCurrentUser } from "@/hooks/use-current-user"; import { cancelBackfillJob, + pauseBackfillJob, + resumeBackfillJob, createBackfillJob, getBackfillJobs, getBackfillRepos, @@ -20,7 +22,7 @@ import type { BackfillEvent, BlueskyProfile, } from "@/types/backfill"; -import { CheckCircle2, ChevronRight, Circle, Loader2 } from "lucide-react"; +import { CheckCircle2, ChevronRight, Circle, Loader2, PauseCircle } from "lucide-react"; import { SiteHeader } from "@/components/site-header"; import { AlertDialog, @@ -104,6 +106,18 @@ function statusBadge(job: BackfillJob) { cancelling ); + case "pausing": + return ( + + pausing + + ); + case "paused": + return ( + + paused + + ); case "running": return ( @@ -507,6 +521,14 @@ export default function BackfillPage() { await cancelBackfillJob(selectedJob.id); load(); }} + onPause={async () => { + await pauseBackfillJob(selectedJob.id); + load(); + }} + onResume={async () => { + await resumeBackfillJob(selectedJob.id); + load(); + }} /> )} @@ -521,16 +543,23 @@ function JobDetail({ canCancel, canFlush, onCancel, + onPause, + onResume, }: { job: BackfillJob; canCancel: boolean; canFlush: boolean; onCancel: () => Promise; + onPause: () => Promise; + onResume: () => Promise; }) { const [cancelling, setCancelling] = useState(false); + const [pausing, setPausing] = useState(false); + const [resuming, setResuming] = useState(false); const current = phaseIndex(job.stage); const allDone = job.status === "completed"; - const isActive = job.status === "running" || job.status === "cancelling"; + const isActive = job.status === "running" || job.status === "cancelling" || job.status === "pausing"; + const isPaused = job.status === "paused" || job.status === "pausing"; // Detail data state const [discoveredRepos, setDiscoveredRepos] = useState([]); @@ -556,6 +585,23 @@ function JobDetail({ return current >= phaseIndex(phase); } + function isPhasePaused(phase: (typeof PROGRESS_PHASES)[number]): boolean { + if (!isPaused) return false; + if (job.stage === "resolving_and_fetching") { + return phase === "resolving_pds" || phase === "fetching_records"; + } + if (job.stage === "discovering_repos") { + return phase === "discovering_repos"; + } + if (job.stage === "resolving_pds") { + return phase === "resolving_pds"; + } + if (job.stage === "fetching_records") { + return phase === "fetching_records"; + } + return false; + } + async function handleCancel() { setCancelling(true); try { @@ -565,6 +611,24 @@ function JobDetail({ } } + async function handlePause() { + setPausing(true); + try { + await onPause(); + } finally { + setPausing(false); + } + } + + async function handleResume() { + setResuming(true); + try { + await onResume(); + } finally { + setResuming(false); + } + } + // Auto-load detail data when phases are reached const discoveredReached = hasReached("discovering_repos"); const pdsReached = hasReached("resolving_pds"); @@ -727,6 +791,7 @@ function JobDetail({ label="Discovering repos" active={isActive && job.stage === "discovering_repos"} reached={hasReached("discovering_repos")} + paused={isPhasePaused("discovering_repos")} value={job.total_repos != null ? : undefined} suffix="repos found" loading={discoveredReached && !discoveredLoaded} @@ -756,6 +821,7 @@ function JobDetail({ job.stage === "resolving_and_fetching") } reached={hasReached("resolving_pds")} + paused={isPhasePaused("resolving_pds")} value={ hasReached("resolving_pds") ? <> / @@ -790,6 +856,7 @@ function JobDetail({ job.stage === "resolving_and_fetching") } reached={hasReached("fetching_records")} + paused={isPhasePaused("fetching_records")} value={ hasReached("fetching_records") || job.stage === "resolving_and_fetching" @@ -855,6 +922,26 @@ function JobDetail({ )} + {canCancel && (job.status === "running" || job.status === "pausing") && ( + + )} + {canCancel && job.status === "paused" && ( + + )} {canCancel && isActive && ( + )} ); @@ -874,6 +971,7 @@ function ProgressRow({ label, active, reached, + paused, value, suffix, loading, @@ -882,13 +980,14 @@ function ProgressRow({ label: string; active: boolean; reached: boolean; + paused?: boolean; value?: React.ReactNode; suffix?: React.ReactNode; loading?: boolean; children?: React.ReactNode; }) { const [open, setOpen] = useState(false); - const done = reached && !active; + const done = reached && !active && !paused; const expandable = reached; return ( @@ -905,6 +1004,8 @@ function ProgressRow({ {active ? ( + ) : paused ? ( + ) : done ? ( ) : ( diff --git a/web/src/app/dashboard/settings/general/page.tsx b/web/src/app/dashboard/settings/general/page.tsx index bc0180e..a6292ef 100644 --- a/web/src/app/dashboard/settings/general/page.tsx +++ b/web/src/app/dashboard/settings/general/page.tsx @@ -1,9 +1,9 @@ -"use client" +"use client"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react" -import { Upload, Trash2 } from "lucide-react" +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { Upload, Trash2 } from "lucide-react"; -import { useCurrentUser } from "@/hooks/use-current-user" +import { useCurrentUser } from "@/hooks/use-current-user"; import { getSettings, getDbInfo, @@ -13,11 +13,11 @@ import { deleteLogo, type SettingEntry, type DbInfo, -} from "@/lib/api" -import { SiteHeader } from "@/components/site-header" -import { Button } from "@/components/ui/button" -import { Input } from "@/components/ui/input" -import { Label } from "@/components/ui/label" +} from "@/lib/api"; +import { SiteHeader } from "@/components/site-header"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; const SETTING_KEYS = [ "app_name", @@ -29,16 +29,16 @@ const SETTING_KEYS = [ "logo_uri", "tos_uri", "policy_uri", -] as const +] as const; -type FieldKey = (typeof SETTING_KEYS)[number] +type FieldKey = (typeof SETTING_KEYS)[number]; type FieldConfig = { - key: FieldKey - label: string - placeholder: string - description: string -} + key: FieldKey; + label: string; + placeholder: string; + description: string; +}; const FIELDS: FieldConfig[] = [ { @@ -74,11 +74,11 @@ const FIELDS: FieldConfig[] = [ placeholder: "https://example.com/privacy", description: "Link to your privacy policy. Optional.", }, -] +]; export default function GeneralSettingsPage() { - const { hasPermission } = useCurrentUser() - const canManage = hasPermission("settings:manage") + const { hasPermission } = useCurrentUser(); + const canManage = hasPermission("settings:manage"); const [values, setValues] = useState>({ app_name: "", @@ -90,8 +90,10 @@ export default function GeneralSettingsPage() { logo_uri: "", tos_uri: "", policy_uri: "", - }) - const [sources, setSources] = useState>({ + }); + const [sources, setSources] = useState< + Record + >({ app_name: "unset", backfill_concurrent_dids_per_pds: "unset", backfill_concurrent_pds: "unset", @@ -101,34 +103,46 @@ export default function GeneralSettingsPage() { logo_uri: "unset", tos_uri: "unset", policy_uri: "unset", - }) - const [logoUploaded, setLogoUploaded] = useState(false) - const [dbInfo, setDbInfo] = useState(null) - const [error, setError] = useState(null) - const [saving, setSaving] = useState(false) - const [notice, setNotice] = useState(null) - const fileInputRef = useRef(null) + }); + const [logoUploaded, setLogoUploaded] = useState(false); + const [dbInfo, setDbInfo] = useState(null); + const [error, setError] = useState(null); + const [saving, setSaving] = useState(false); + const [notice, setNotice] = useState(null); + const fileInputRef = useRef(null); const load = useCallback(async () => { try { - const entries = await getSettings() - const byKey = new Map(entries.map((e) => [e.key, e])) - const val = (key: string, fallback: string) => byKey.get(key)?.value ?? fallback - const src = (key: string) => (byKey.get(key)?.source as "database" | "env" | undefined) ?? "unset" + const entries = await getSettings(); + const byKey = new Map( + entries.map((e) => [e.key, e]), + ); + const val = (key: string, fallback: string) => + byKey.get(key)?.value ?? fallback; + const src = (key: string) => + (byKey.get(key)?.source as "database" | "env" | undefined) ?? "unset"; setValues({ app_name: val("app_name", ""), - backfill_concurrent_dids_per_pds: val("backfill_concurrent_dids_per_pds", "3"), + backfill_concurrent_dids_per_pds: val( + "backfill_concurrent_dids_per_pds", + "3", + ), backfill_concurrent_pds: val("backfill_concurrent_pds", "10"), - backfill_concurrent_resolution: val("backfill_concurrent_resolution", "100"), + backfill_concurrent_resolution: val( + "backfill_concurrent_resolution", + "100", + ), backfill_retention_days: val("backfill_retention_days", "28"), client_uri: val("client_uri", ""), logo_uri: val("logo_uri", ""), tos_uri: val("tos_uri", ""), policy_uri: val("policy_uri", ""), - }) + }); setSources({ app_name: src("app_name"), - backfill_concurrent_dids_per_pds: src("backfill_concurrent_dids_per_pds"), + backfill_concurrent_dids_per_pds: src( + "backfill_concurrent_dids_per_pds", + ), backfill_concurrent_pds: src("backfill_concurrent_pds"), backfill_concurrent_resolution: src("backfill_concurrent_resolution"), backfill_retention_days: src("backfill_retention_days"), @@ -136,35 +150,35 @@ export default function GeneralSettingsPage() { logo_uri: src("logo_uri"), tos_uri: src("tos_uri"), policy_uri: src("policy_uri"), - }) - setLogoUploaded(byKey.has("logo_data")) + }); + setLogoUploaded(byKey.has("logo_data")); try { - setDbInfo(await getDbInfo()) + setDbInfo(await getDbInfo()); } catch { // non-critical } } catch (e: unknown) { - setError(e instanceof Error ? e.message : String(e)) + setError(e instanceof Error ? e.message : String(e)); } - }, []) + }, []); useEffect(() => { - load() - }, [load]) + load(); + }, [load]); async function handleSave() { - setError(null) - setNotice(null) - setSaving(true) + setError(null); + setNotice(null); + setSaving(true); try { for (const field of FIELDS) { - const value = values[field.key] + const value = values[field.key]; if (value === "") { if (sources[field.key] === "database") { - await deleteSetting(field.key) + await deleteSetting(field.key); } } else { - await upsertSetting(field.key, value) + await upsertSetting(field.key, value); } } const backfillKeys = [ @@ -172,77 +186,79 @@ export default function GeneralSettingsPage() { "backfill_concurrent_pds", "backfill_concurrent_resolution", "backfill_retention_days", - ] as const + ] as const; for (const key of backfillKeys) { - const value = values[key] + const value = values[key]; if (value === "") { if (sources[key] === "database") { - await deleteSetting(key) + await deleteSetting(key); } } else { - await upsertSetting(key, value) + await upsertSetting(key, value); } } - setNotice("Settings saved.") - await load() + setNotice("Settings saved."); + await load(); } catch (e: unknown) { - setError(e instanceof Error ? e.message : String(e)) + setError(e instanceof Error ? e.message : String(e)); } finally { - setSaving(false) + setSaving(false); } } async function handleLogoUpload(e: React.ChangeEvent) { - const file = e.target.files?.[0] - if (!file) return - setError(null) + const file = e.target.files?.[0]; + if (!file) return; + setError(null); try { - await uploadLogo(file) - setNotice("Logo uploaded.") - await load() + await uploadLogo(file); + setNotice("Logo uploaded."); + await load(); } catch (err: unknown) { - setError(err instanceof Error ? err.message : String(err)) + setError(err instanceof Error ? err.message : String(err)); } finally { - if (fileInputRef.current) fileInputRef.current.value = "" + if (fileInputRef.current) fileInputRef.current.value = ""; } } async function handleLogoDelete() { - setError(null) + setError(null); try { - await deleteLogo() - setNotice("Logo removed.") - await load() + await deleteLogo(); + setNotice("Logo removed."); + await load(); } catch (err: unknown) { - setError(err instanceof Error ? err.message : String(err)) + setError(err instanceof Error ? err.message : String(err)); } } const connectionEstimate = useMemo(() => { - const pds = parseInt(values.backfill_concurrent_pds) || 10 - const dids = parseInt(values.backfill_concurrent_dids_per_pds) || 3 - const resolution = parseInt(values.backfill_concurrent_resolution) || 100 - const needed = pds * dids + resolution + 4 - const mainPool = dbInfo?.main_pool_size ?? 32 - const total = needed + mainPool - const serverMax = dbInfo?.server_max_connections ?? null - return { needed, mainPool, total, serverMax } - }, [values, dbInfo]) + const pds = parseInt(values.backfill_concurrent_pds) || 10; + const dids = parseInt(values.backfill_concurrent_dids_per_pds) || 3; + const resolution = parseInt(values.backfill_concurrent_resolution) || 100; + const needed = pds * dids + resolution + 4; + const mainPool = dbInfo?.main_pool_size ?? 32; + const total = needed + mainPool; + const serverMax = dbInfo?.server_max_connections ?? null; + return { needed, mainPool, total, serverMax }; + }, [values, dbInfo]); const connectionWarning = useMemo(() => { - if (!connectionEstimate.serverMax) return null + if (!connectionEstimate.serverMax) return null; if (connectionEstimate.total > connectionEstimate.serverMax) { - return `These settings need ~${connectionEstimate.total} connections (${connectionEstimate.needed} backfill + ${connectionEstimate.mainPool} main), but the database allows ${connectionEstimate.serverMax}. Reduce concurrency or increase the database's max_connections.` + return `These settings need ~${connectionEstimate.total} connections (${connectionEstimate.needed} backfill + ${connectionEstimate.mainPool} main), but the database allows ${connectionEstimate.serverMax}. Reduce concurrency or increase the database's max_connections.`; } - return null - }, [connectionEstimate]) + return null; + }, [connectionEstimate]); return ( <>
{error &&

{error}

} - {notice &&

{notice}

} + {notice && ( +

{notice}

+ )}

Instance Identity

@@ -320,13 +336,16 @@ export default function GeneralSettingsPage() {

Data Retention

- Configure how long HappyView retains detailed data from completed backfill jobs. + Configure how long HappyView retains detailed data from completed + backfill jobs.

- + {sources["backfill_retention_days"] === "env" && ( from env var @@ -340,25 +359,31 @@ export default function GeneralSettingsPage() { step={1} value={values["backfill_retention_days"]} onChange={(e) => - setValues((v) => ({ ...v, backfill_retention_days: e.target.value })) + setValues((v) => ({ + ...v, + backfill_retention_days: e.target.value, + })) } placeholder="28" disabled={!canManage} />

- How long to keep per-repo detail data from completed backfill jobs. Set to 0 to keep indefinitely. + How long to keep per-repo detail data from completed backfill jobs. + Set to 0 to keep indefinitely.

Backfill Performance

- Tune concurrency limits for backfill jobs. Changes apply to the next job started. - The backfill connection pool is auto-sized on startup based on these values. + Tune concurrency limits for backfill jobs. Changes only apply to new + or resumed jobs.

{dbInfo?.server_max_connections && (

- Database limit: {dbInfo.server_max_connections} connections · Main pool: {connectionEstimate.mainPool} · Backfill estimate: {connectionEstimate.needed} + Database limit: {dbInfo.server_max_connections} connections · Main + pool: {connectionEstimate.mainPool} · Backfill estimate:{" "} + {connectionEstimate.needed}

)} {connectionWarning && ( @@ -366,16 +391,39 @@ export default function GeneralSettingsPage() { )}
- {([ - { key: "backfill_concurrent_resolution" as const, id: "backfill_concurrent_resolution", label: "Concurrent PLC Resolutions", placeholder: "100", description: "How many DID document lookups to run in parallel during PDS resolution." }, - { key: "backfill_concurrent_pds" as const, id: "backfill_concurrent_pds", label: "Concurrent PDS Hosts", placeholder: "10", description: "How many PDS servers to fetch records from simultaneously." }, - { key: "backfill_concurrent_dids_per_pds" as const, id: "backfill_concurrent_dids_per_pds", label: "Concurrent DIDs per PDS", placeholder: "3", description: "How many repos to fetch concurrently from each PDS host." }, - ]).map((field) => ( + {[ + { + key: "backfill_concurrent_resolution" as const, + id: "backfill_concurrent_resolution", + label: "Concurrent PLC Resolutions", + placeholder: "100", + description: + "How many DID document lookups to run in parallel during PDS resolution.", + }, + { + key: "backfill_concurrent_pds" as const, + id: "backfill_concurrent_pds", + label: "Concurrent PDS Hosts", + placeholder: "10", + description: + "How many PDS servers to fetch records from simultaneously.", + }, + { + key: "backfill_concurrent_dids_per_pds" as const, + id: "backfill_concurrent_dids_per_pds", + label: "Concurrent DIDs per PDS", + placeholder: "3", + description: + "How many repos to fetch concurrently from each PDS host.", + }, + ].map((field) => (
{sources[field.key] === "env" && ( - from env var + + from env var + )}
setValues((v) => ({ ...v, [field.key]: e.target.value }))} + onChange={(e) => + setValues((v) => ({ ...v, [field.key]: e.target.value })) + } placeholder={field.placeholder} disabled={!canManage} /> @@ -402,5 +452,5 @@ export default function GeneralSettingsPage() {
- ) + ); } diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 1c43284..1d93562 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -189,6 +189,20 @@ export function cancelBackfillJob(id: string) { ); } +export function pauseBackfillJob(id: string) { + return apiFetch<{ id: string; status: string }>( + `/admin/backfill/${id}/pause`, + { method: "POST" }, + ); +} + +export function resumeBackfillJob(id: string) { + return apiFetch<{ id: string; status: string }>( + `/admin/backfill/${id}/resume`, + { method: "POST" }, + ); +} + export function getBackfillRepos( jobId: string, params: { phase?: string; cursor?: string; limit?: number } = {}, -- 2.51.2 From b0d382b2979a2d1f0fbc4976e06db8011bc7bfd8 Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 20:16:18 -0500 Subject: [PATCH 17/28] feat: recommend restart when backfill concurrency settings dont match existing connection pool size Signed-off-by: Trezy --- web/src/app/dashboard/layout.tsx | 25 ++++++----- web/src/components/app-sidebar.tsx | 22 ++++++++++ web/src/components/site-header.tsx | 67 ++++++++++++++++++++++-------- web/src/lib/api.ts | 1 + web/src/lib/restart-context.tsx | 47 +++++++++++++++++++++ 5 files changed, 133 insertions(+), 29 deletions(-) create mode 100644 web/src/lib/restart-context.tsx diff --git a/web/src/app/dashboard/layout.tsx b/web/src/app/dashboard/layout.tsx index 9e5488f..be0b4e3 100644 --- a/web/src/app/dashboard/layout.tsx +++ b/web/src/app/dashboard/layout.tsx @@ -7,6 +7,7 @@ import { useAuth } from "@/lib/auth-context" import { useConfig } from "@/lib/config-context" import { AppSidebar } from "@/components/app-sidebar" import { PluginUpdateProvider } from "@/components/plugin-update-provider" +import { RestartProvider } from "@/lib/restart-context" import { SidebarInset, SidebarProvider } from "@/components/ui/sidebar" import { Toaster } from "@/components/ui/sonner" @@ -33,17 +34,19 @@ export default function DashboardLayout({ return ( - - - {children} - + + + + {children} + + ) diff --git a/web/src/components/app-sidebar.tsx b/web/src/components/app-sidebar.tsx index cc54c77..e9521d8 100644 --- a/web/src/components/app-sidebar.tsx +++ b/web/src/components/app-sidebar.tsx @@ -29,6 +29,8 @@ import { useAuth } from "@/lib/auth-context"; import { useConfig } from "@/lib/config-context"; import { useCurrentUser } from "@/hooks/use-current-user"; import { usePluginUpdates } from "@/components/plugin-update-provider"; +import { useRestart } from "@/lib/restart-context"; +import { getDbInfo } from "@/lib/api"; import { Scroller } from "@/components/ui/scroller"; import { Sidebar, @@ -138,6 +140,7 @@ export function AppSidebar({ ...props }: React.ComponentProps) { const { app_name, logo_url } = useConfig(); const { hasPermission } = useCurrentUser(); const { hasUpdates } = usePluginUpdates(); + const { addReason, removeReason } = useRestart(); const [deadLetterCount, setDeadLetterCount] = useState(0); @@ -160,6 +163,25 @@ export function AppSidebar({ ...props }: React.ComponentProps) { }; }, []); + useEffect(() => { + if (!hasPermission("settings:manage")) return; + let cancelled = false; + getDbInfo() + .then((info) => { + if (!cancelled) { + if (info.restart_recommended) { + addReason("backfill-pool", "Backfill pool is undersized for current concurrency settings."); + } else { + removeReason("backfill-pool"); + } + } + }) + .catch(() => {}); + return () => { + cancelled = true; + }; + }, [hasPermission, addReason, removeReason]); + function filterByPermission(items: NavItem[]) { return items.filter( (item) => diff --git a/web/src/components/site-header.tsx b/web/src/components/site-header.tsx index 741edf2..c5a7eaf 100644 --- a/web/src/components/site-header.tsx +++ b/web/src/components/site-header.tsx @@ -1,8 +1,11 @@ "use client" +import { useState } from "react" import { IconArrowLeft } from "@tabler/icons-react" +import { AlertTriangle, ChevronDown } from "lucide-react" import Link from "next/link" +import { useRestart } from "@/lib/restart-context" import { Button } from "@/components/ui/button" import { Separator } from "@/components/ui/separator" import { SidebarTrigger } from "@/components/ui/sidebar" @@ -15,27 +18,55 @@ export function SiteHeader({ title: string backHref?: string }) { + const { reasons } = useRestart() + const [expanded, setExpanded] = useState(false) + return ( -
-
- - - {backHref && ( - - )} -

{title}

-
- +
+
+
+ + + {backHref && ( + + )} +

{title}

+
+ +
+ {reasons.length > 0 && ( +
+ + {expanded && ( +
    + {reasons.map((reason, i) => ( +
  • + + {reason} +
    • + ))} +
+ )} +
+ )}
) } diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 1d93562..9612788 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -384,6 +384,7 @@ export type DbInfo = { server_max_connections: number | null; main_pool_size: number; backfill_pool_size: number; + restart_recommended: boolean; }; export function getDbInfo() { diff --git a/web/src/lib/restart-context.tsx b/web/src/lib/restart-context.tsx new file mode 100644 index 0000000..a503e61 --- /dev/null +++ b/web/src/lib/restart-context.tsx @@ -0,0 +1,47 @@ +"use client"; + +import { createContext, useCallback, useContext, useMemo, useRef, useState } from "react"; + +interface RestartContextType { + reasons: string[]; + addReason: (key: string, reason: string) => void; + removeReason: (key: string) => void; +} + +const RestartContext = createContext({ + reasons: [], + addReason: () => {}, + removeReason: () => {}, +}); + +export function RestartProvider({ children }: { children: React.ReactNode }) { + const [reasonMap, setReasonMap] = useState>({}); + const reasonMapRef = useRef(reasonMap); + reasonMapRef.current = reasonMap; + + const addReason = useCallback((key: string, reason: string) => { + if (reasonMapRef.current[key] === reason) return; + setReasonMap((prev) => ({ ...prev, [key]: reason })); + }, []); + + const removeReason = useCallback((key: string) => { + setReasonMap((prev) => { + if (!(key in prev)) return prev; + const next = { ...prev }; + delete next[key]; + return next; + }); + }, []); + + const reasons = useMemo(() => Object.values(reasonMap), [reasonMap]); + + return ( + + {children} + + ); +} + +export function useRestart() { + return useContext(RestartContext); +} -- 2.51.2 From 485657997b901f08725385921798b26c253b6e70 Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 12:08:23 -0500 Subject: [PATCH 18/28] feat: make verbose event logs configurable Signed-off-by: Trezy --- src/admin/settings.rs | 1 + src/lib.rs | 1 + src/lua/atproto_api.rs | 1 + src/lua/db_api.rs | 1 + src/lua/execute.rs | 38 ++++---- src/lua/http_api.rs | 1 + src/lua/xrpc_api.rs | 1 + src/main.rs | 34 ++++++- src/record_handler.rs | 96 +++++-------------- tests/common/app.rs | 1 + tests/lua_atproto_api.rs | 1 + tests/lua_db_api.rs | 1 + .../app/dashboard/settings/general/page.tsx | 50 +++++++++- web/src/lib/api.ts | 4 + 14 files changed, 136 insertions(+), 95 deletions(-) diff --git a/src/admin/settings.rs b/src/admin/settings.rs index f730f7b..c3c1a1b 100644 --- a/src/admin/settings.rs +++ b/src/admin/settings.rs @@ -31,6 +31,7 @@ const ENV_FALLBACKS: &[(&str, &str)] = &[ ("logo_uri", "LOGO_URI"), ("tos_uri", "TOS_URI"), ("policy_uri", "POLICY_URI"), + ("verbose_event_logging", "VERBOSE_EVENT_LOGGING"), ]; /// Resolve a setting value: check the DB first, then fall back to env var. diff --git a/src/lib.rs b/src/lib.rs index 3b3a061..39a44ef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -80,6 +80,7 @@ pub struct AppState { pub official_registry_config: RegistryConfig, pub proxy_config: Arc>, pub backfill_events_tx: tokio::sync::broadcast::Sender, + pub verbose_event_logging: std::sync::Arc, } impl axum::extract::FromRef for axum_extra::extract::cookie::Key { diff --git a/src/lua/atproto_api.rs b/src/lua/atproto_api.rs index 40af1b0..aeb047c 100644 --- a/src/lua/atproto_api.rs +++ b/src/lua/atproto_api.rs @@ -584,6 +584,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/lua/db_api.rs b/src/lua/db_api.rs index e8d95b8..5e7f062 100644 --- a/src/lua/db_api.rs +++ b/src/lua/db_api.rs @@ -792,6 +792,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/lua/execute.rs b/src/lua/execute.rs index bf50cf3..0e0037d 100644 --- a/src/lua/execute.rs +++ b/src/lua/execute.rs @@ -4,6 +4,7 @@ use mlua::LuaSerdeExt; use serde_json::Value; use std::collections::HashMap; use std::sync::Arc; +use std::sync::atomic::Ordering; use std::time::Instant; use crate::AppState; @@ -913,23 +914,25 @@ pub async fn execute_hook_script(event: &HookEvent<'_>) -> Option { match run_hook_once(event).await { Ok(hook_result) => { - log_event( - &event.state.db, - EventLog { - event_type: "hook.executed".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(event.uri.to_string()), - detail: serde_json::json!({ - "lexicon_id": event.lexicon_id, - "action": event.action, - "collection": event.collection, - "attempts": attempt + 1, - }), - }, - backend, - ) - .await; + if event.state.verbose_event_logging.load(Ordering::Relaxed) { + log_event( + &event.state.db, + EventLog { + event_type: "hook.executed".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(event.uri.to_string()), + detail: serde_json::json!({ + "lexicon_id": event.lexicon_id, + "action": event.action, + "collection": event.collection, + "attempts": attempt + 1, + }), + }, + backend, + ) + .await; + } return hook_result; } Err(e) => { @@ -1182,6 +1185,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/lua/http_api.rs b/src/lua/http_api.rs index 12b3716..cce9b54 100644 --- a/src/lua/http_api.rs +++ b/src/lua/http_api.rs @@ -190,6 +190,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/lua/xrpc_api.rs b/src/lua/xrpc_api.rs index 82c6ed7..6a5b930 100644 --- a/src/lua/xrpc_api.rs +++ b/src/lua/xrpc_api.rs @@ -294,6 +294,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/main.rs b/src/main.rs index 9a3bf7c..f6a78b2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,8 +28,11 @@ async fn main() { tracing_subscriber::fmt() .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| "happyview=debug,tower_http=debug".parse().unwrap()), + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { + "happyview=debug,tower_http=debug,sqlx=warn" + .parse() + .unwrap() + }), ) .init(); @@ -615,6 +618,15 @@ async fn main() { let (backfill_events_tx, _) = tokio::sync::broadcast::channel(1024); + let verbose_event_logging = { + let enabled = + happyview::admin::settings::get_setting(&db_pool, "verbose_event_logging", db_backend) + .await + .map(|v| v.eq_ignore_ascii_case("true")) + .unwrap_or(false); + std::sync::Arc::new(std::sync::atomic::AtomicBool::new(enabled)) + }; + let state = AppState { config: config.clone(), http, @@ -636,6 +648,7 @@ async fn main() { official_registry_config, proxy_config, backfill_events_tx, + verbose_event_logging, }; jetstream::spawn(state.clone(), collections_rx); @@ -649,6 +662,23 @@ async fn main() { state.db_backend, )); + { + let db = state.db.clone(); + let flag = state.verbose_event_logging.clone(); + let backend = state.db_backend; + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + let enabled = + happyview::admin::settings::get_setting(&db, "verbose_event_logging", backend) + .await + .map(|v| v.eq_ignore_ascii_case("true")) + .unwrap_or(false); + flag.store(enabled, std::sync::atomic::Ordering::Relaxed); + } + }); + } + happyview::admin::backfill::resume_backfill_jobs(&state).await; { diff --git a/src/record_handler.rs b/src/record_handler.rs index e59f75f..34dacee 100644 --- a/src/record_handler.rs +++ b/src/record_handler.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::sync::atomic::Ordering; use serde_json::Value; @@ -74,27 +75,7 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { .await; match hook_result { - None => { - // Hook returned nil — skip indexing this record. - log_event( - db, - EventLog { - event_type: "record.skipped".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - "reason": "hook returned nil", - }), - }, - state.db_backend, - ) - .await; - return; - } + None => return, Some(v) => v, } } else { @@ -138,23 +119,6 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { ) .await; - log_event( - db, - EventLog { - event_type: "record.created".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - }), - }, - backend, - ) - .await; - crate::labeler::backfill_labels_for_uri(Arc::new(state.clone()), uri.clone()); } Err(e) => { @@ -198,48 +162,32 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { .await; if hook_result.is_none() { - // Hook returned nil — skip the delete. - log_event( - db, - EventLog { - event_type: "record.skipped".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - "reason": "hook returned nil", - }), - }, - backend, - ) - .await; + if state.verbose_event_logging.load(Ordering::Relaxed) { + log_event( + db, + EventLog { + event_type: "record.skipped".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(uri.clone()), + detail: serde_json::json!({ + "collection": record.collection, + "did": record.did, + "rkey": record.rkey, + "reason": "hook returned nil", + }), + }, + backend, + ) + .await; + } return; } } let delete_sql = adapt_sql("DELETE FROM records WHERE uri = ?", backend); match sqlx::query(&delete_sql).bind(&uri).execute(db).await { - Ok(_) => { - log_event( - db, - EventLog { - event_type: "record.deleted".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - }), - }, - backend, - ) - .await; - } + Ok(_) => {} Err(e) => { tracing::warn!(uri = %uri, "failed to delete record: {e}"); log_event( diff --git a/tests/common/app.rs b/tests/common/app.rs index bac7ff8..cd08de7 100644 --- a/tests/common/app.rs +++ b/tests/common/app.rs @@ -169,6 +169,7 @@ impl TestApp { ))), backfill_db: pool.clone(), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let router = server::router(state.clone()).layer(axum::middleware::from_fn( diff --git a/tests/lua_atproto_api.rs b/tests/lua_atproto_api.rs index f0a24c1..396a7ba 100644 --- a/tests/lua_atproto_api.rs +++ b/tests/lua_atproto_api.rs @@ -103,6 +103,7 @@ async fn test_state_with_pool(pool: sqlx::AnyPool, backend: DatabaseBackend) -> ))), backfill_db: pool.clone(), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/tests/lua_db_api.rs b/tests/lua_db_api.rs index bbf17f2..d32bb90 100644 --- a/tests/lua_db_api.rs +++ b/tests/lua_db_api.rs @@ -106,6 +106,7 @@ async fn test_state_with_pool(pool: sqlx::AnyPool, backend: DatabaseBackend) -> ))), backfill_db: pool.clone(), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/web/src/app/dashboard/settings/general/page.tsx b/web/src/app/dashboard/settings/general/page.tsx index a6292ef..47ebe58 100644 --- a/web/src/app/dashboard/settings/general/page.tsx +++ b/web/src/app/dashboard/settings/general/page.tsx @@ -18,6 +18,7 @@ import { SiteHeader } from "@/components/site-header"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; const SETTING_KEYS = [ "app_name", @@ -29,6 +30,7 @@ const SETTING_KEYS = [ "logo_uri", "tos_uri", "policy_uri", + "verbose_event_logging", ] as const; type FieldKey = (typeof SETTING_KEYS)[number]; @@ -90,6 +92,7 @@ export default function GeneralSettingsPage() { logo_uri: "", tos_uri: "", policy_uri: "", + verbose_event_logging: "", }); const [sources, setSources] = useState< Record @@ -103,6 +106,7 @@ export default function GeneralSettingsPage() { logo_uri: "unset", tos_uri: "unset", policy_uri: "unset", + verbose_event_logging: "unset", }); const [logoUploaded, setLogoUploaded] = useState(false); const [dbInfo, setDbInfo] = useState(null); @@ -137,6 +141,7 @@ export default function GeneralSettingsPage() { logo_uri: val("logo_uri", ""), tos_uri: val("tos_uri", ""), policy_uri: val("policy_uri", ""), + verbose_event_logging: val("verbose_event_logging", ""), }); setSources({ app_name: src("app_name"), @@ -150,6 +155,7 @@ export default function GeneralSettingsPage() { logo_uri: src("logo_uri"), tos_uri: src("tos_uri"), policy_uri: src("policy_uri"), + verbose_event_logging: src("verbose_event_logging"), }); setLogoUploaded(byKey.has("logo_data")); try { @@ -181,13 +187,14 @@ export default function GeneralSettingsPage() { await upsertSetting(field.key, value); } } - const backfillKeys = [ + const extraKeys = [ "backfill_concurrent_dids_per_pds", "backfill_concurrent_pds", "backfill_concurrent_resolution", "backfill_retention_days", + "verbose_event_logging", ] as const; - for (const key of backfillKeys) { + for (const key of extraKeys) { const value = values[key]; if (value === "") { if (sources[key] === "database") { @@ -442,6 +449,45 @@ export default function GeneralSettingsPage() {
))} +
+

Logging

+

+ Configure event log verbosity. +

+
+ +
+
+
+ + {sources["verbose_event_logging"] === "env" && ( + + from env var + + )} +
+

+ Log every record index, hook execution, and hook skip to the event + log. Generates high write volume and will cause issues if + you're indexing high-traffic collections. Recommended to only use + for debugging. +

+
+ + setValues((v) => ({ + ...v, + verbose_event_logging: checked ? "true" : "", + })) + } + disabled={!canManage} + /> +
+
- - - - - ); - } if (!table.getIsAllPageRowsSelected()) { return ( -- 2.51.2 From 3a0fb9cb18b6afd901e428c3e70976978706e673 Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 12:30:44 -0500 Subject: [PATCH 21/28] fix: better pool size accounting when using sqlite Signed-off-by: Trezy --- src/admin/settings.rs | 11 ++++---- src/db.rs | 60 ++++++++++++++++++++++++++----------------- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/src/admin/settings.rs b/src/admin/settings.rs index c3c1a1b..dd9d3d3 100644 --- a/src/admin/settings.rs +++ b/src/admin/settings.rs @@ -204,11 +204,11 @@ pub(super) async fn db_info( let main_pool_size = state.db.options().get_max_connections() as i64; let backfill_pool_size = state.backfill_db.options().get_max_connections() as i64; - let pds: i64 = get_setting(&state.db, "backfill_concurrent_pds", state.db_backend) + let pds: u32 = get_setting(&state.db, "backfill_concurrent_pds", state.db_backend) .await .and_then(|v| v.parse().ok()) .unwrap_or(10); - let dids: i64 = get_setting( + let dids: u32 = get_setting( &state.db, "backfill_concurrent_dids_per_pds", state.db_backend, @@ -216,7 +216,7 @@ pub(super) async fn db_info( .await .and_then(|v| v.parse().ok()) .unwrap_or(3); - let resolution: i64 = get_setting( + let resolution: u32 = get_setting( &state.db, "backfill_concurrent_resolution", state.db_backend, @@ -224,8 +224,9 @@ pub(super) async fn db_info( .await .and_then(|v| v.parse().ok()) .unwrap_or(100); - let needed_backfill_pool = (pds * dids) + resolution + 4; - let restart_recommended = needed_backfill_pool > backfill_pool_size; + let would_be_pool_size = + crate::db::compute_backfill_pool_size(state.db_backend, pds, dids, resolution); + let restart_recommended = would_be_pool_size as i64 > backfill_pool_size; Ok(Json(serde_json::json!({ "backend": match state.db_backend { diff --git a/src/db.rs b/src/db.rs index 6eb5cbc..9572d69 100644 --- a/src/db.rs +++ b/src/db.rs @@ -283,33 +283,47 @@ pub async fn connect(url: &str, backend: DatabaseBackend) -> AnyPool { pool } -pub async fn connect_backfill_pool(url: &str, backend: DatabaseBackend) -> AnyPool { - let max_connections: u32 = std::env::var("BACKFILL_DATABASE_MAX_CONNECTIONS") +pub fn backfill_pool_ceiling(backend: DatabaseBackend) -> u32 { + match backend { + DatabaseBackend::Sqlite => 64, + DatabaseBackend::Postgres => 256, + } +} + +pub fn needed_backfill_connections(pds: u32, dids_per_pds: u32, resolution: u32) -> u32 { + (pds * dids_per_pds) + resolution + 4 +} + +pub fn compute_backfill_pool_size( + backend: DatabaseBackend, + pds: u32, + dids_per_pds: u32, + resolution: u32, +) -> u32 { + std::env::var("BACKFILL_DATABASE_MAX_CONNECTIONS") .ok() .and_then(|v| v.parse::().ok()) .unwrap_or_else(|| { - let pds: u32 = std::env::var("BACKFILL_CONCURRENT_PDS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(10); - let dids: u32 = std::env::var("BACKFILL_CONCURRENT_DIDS_PER_PDS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(3); - let resolution: u32 = std::env::var("BACKFILL_CONCURRENT_RESOLUTION") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(100); - // Each concurrent worker may need a connection: PDS×DIDs for fetching + - // resolution concurrency + a few for bookkeeping queries. - let needed = (pds * dids) + resolution + 4; - let ceiling = match backend { - DatabaseBackend::Sqlite => 64, - DatabaseBackend::Postgres => 256, - }; - needed.min(ceiling) + needed_backfill_connections(pds, dids_per_pds, resolution) + .min(backfill_pool_ceiling(backend)) }) - .max(1); + .max(1) +} + +pub async fn connect_backfill_pool(url: &str, backend: DatabaseBackend) -> AnyPool { + let pds: u32 = std::env::var("BACKFILL_CONCURRENT_PDS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10); + let dids: u32 = std::env::var("BACKFILL_CONCURRENT_DIDS_PER_PDS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(3); + let resolution: u32 = std::env::var("BACKFILL_CONCURRENT_RESOLUTION") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(100); + let max_connections = compute_backfill_pool_size(backend, pds, dids, resolution); tracing::info!(max_connections, "backfill pool sized"); -- 2.51.2 From d38d05d672e0f692c013565cc49a32e7dd4cbd8f Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 14:53:01 -0500 Subject: [PATCH 22/28] fix: restore missing log events for verbose logging Signed-off-by: Trezy --- src/record_handler.rs | 63 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/src/record_handler.rs b/src/record_handler.rs index 34dacee..9631941 100644 --- a/src/record_handler.rs +++ b/src/record_handler.rs @@ -75,7 +75,28 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { .await; match hook_result { - None => return, + None => { + if state.verbose_event_logging.load(Ordering::Relaxed) { + log_event( + db, + EventLog { + event_type: "record.skipped".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(uri.clone()), + detail: serde_json::json!({ + "collection": record.collection, + "did": record.did, + "rkey": record.rkey, + "reason": "hook returned nil", + }), + }, + state.db_backend, + ) + .await; + } + return; + } Some(v) => v, } } else { @@ -119,6 +140,25 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { ) .await; + if state.verbose_event_logging.load(Ordering::Relaxed) { + log_event( + db, + EventLog { + event_type: "record.created".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(uri.clone()), + detail: serde_json::json!({ + "collection": record.collection, + "did": record.did, + "rkey": record.rkey, + }), + }, + backend, + ) + .await; + } + crate::labeler::backfill_labels_for_uri(Arc::new(state.clone()), uri.clone()); } Err(e) => { @@ -187,7 +227,26 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { let delete_sql = adapt_sql("DELETE FROM records WHERE uri = ?", backend); match sqlx::query(&delete_sql).bind(&uri).execute(db).await { - Ok(_) => {} + Ok(_) => { + if state.verbose_event_logging.load(Ordering::Relaxed) { + log_event( + db, + EventLog { + event_type: "record.deleted".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(uri.clone()), + detail: serde_json::json!({ + "collection": record.collection, + "did": record.did, + "rkey": record.rkey, + }), + }, + backend, + ) + .await; + } + } Err(e) => { tracing::warn!(uri = %uri, "failed to delete record: {e}"); log_event( -- 2.51.2 From 7fb418411f760b6c60db3708e2324fc76b4c758d Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 14:57:36 -0500 Subject: [PATCH 23/28] test: add tests for backfill concurrency Signed-off-by: Trezy --- src/db.rs | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/db.rs b/src/db.rs index 9572d69..a4a6938 100644 --- a/src/db.rs +++ b/src/db.rs @@ -637,6 +637,57 @@ mod tests { // parse_dt // ----------------------------------------------------------------------- + // ----------------------------------------------------------------------- + // backfill pool sizing + // ----------------------------------------------------------------------- + + #[test] + fn needed_backfill_connections_formula() { + assert_eq!(needed_backfill_connections(10, 3, 100), 134); + assert_eq!(needed_backfill_connections(1, 1, 1), 6); + assert_eq!(needed_backfill_connections(0, 0, 0), 4); + } + + #[test] + fn backfill_pool_ceiling_values() { + assert_eq!(backfill_pool_ceiling(DatabaseBackend::Sqlite), 64); + assert_eq!(backfill_pool_ceiling(DatabaseBackend::Postgres), 256); + } + + #[test] + fn needed_connections_capped_by_sqlite_ceiling() { + let needed = needed_backfill_connections(10, 3, 100); + let capped = needed.min(backfill_pool_ceiling(DatabaseBackend::Sqlite)); + assert_eq!(needed, 134); + assert_eq!(capped, 64); + } + + #[test] + fn needed_connections_capped_by_postgres_ceiling() { + let needed = needed_backfill_connections(50, 10, 200); + let capped = needed.min(backfill_pool_ceiling(DatabaseBackend::Postgres)); + assert_eq!(needed, 704); + assert_eq!(capped, 256); + } + + #[test] + fn needed_connections_below_ceiling_unchanged() { + let needed = needed_backfill_connections(2, 2, 10); + let capped = needed.min(backfill_pool_ceiling(DatabaseBackend::Postgres)); + assert_eq!(needed, 18); + assert_eq!(capped, 18); + } + + #[test] + fn needed_connections_minimum_is_overhead() { + let needed = needed_backfill_connections(0, 0, 0); + assert_eq!(needed, 4); + } + + // ----------------------------------------------------------------------- + // parse_dt + // ----------------------------------------------------------------------- + #[test] fn parse_dt_rfc3339() { let dt = parse_dt("2025-03-16T12:34:56Z"); -- 2.51.2 From 62adab05522156e4146fc32e6d36182033908127 Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 15:36:15 -0500 Subject: [PATCH 24/28] fix: prevent backfill jobs from locking locking out pause/cancellation Signed-off-by: Trezy --- .../20260522000000_add_records_collection_indexed_at_index.sql | 2 ++ src/admin/backfill.rs | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 migrations/postgres/20260522000000_add_records_collection_indexed_at_index.sql diff --git a/migrations/postgres/20260522000000_add_records_collection_indexed_at_index.sql b/migrations/postgres/20260522000000_add_records_collection_indexed_at_index.sql new file mode 100644 index 0000000..6ac0175 --- /dev/null +++ b/migrations/postgres/20260522000000_add_records_collection_indexed_at_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_records_collection_indexed_at +ON records (collection, indexed_at DESC); diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 4e1165d..0418ea4 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -681,7 +681,6 @@ async fn run_pipelined_resolve_and_fetch( let poll_job_id = Arc::clone(&job_id_arc); let poll_cancelled = Arc::clone(&cancelled); let pair = tokio::select! { - biased; result = rx.recv() => result, _ = async move { loop { -- 2.51.2 From 1cf8f751eb20365e0a92ad48044d364a4f0d88ea Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 15:37:32 -0500 Subject: [PATCH 25/28] fix: reset backfill ui when flushing details Signed-off-by: Trezy --- web/src/app/dashboard/backfill/page.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index 347feba..8ab0cb2 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -423,6 +423,8 @@ export default function BackfillPage() { Cancel { await flushAllBackfillDetails(); + setSelectedJobId(null); + load(); }}>Clear -- 2.51.2 From 455fbeccfff2bee26b1ac2e1ca51620c87c94e66 Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 15:39:24 -0500 Subject: [PATCH 26/28] docs: update docs to cover backfill updates Signed-off-by: Trezy --- .../docs/api-reference/admin/admin-api.md | 16 +- .../docs/api-reference/admin/backfill.md | 150 +++++++++++++++++- .../docs/api-reference/admin/meta.json | 1 + .../docs/api-reference/admin/records.md | 117 ++++++++++++++ .../docs/api-reference/admin/settings.md | 49 +++++- .../docs/getting-started/configuration.md | 15 +- packages/docs/content/docs/guides/backfill.md | 34 +++- .../content/docs/reference/architecture.md | 57 ++++--- 8 files changed, 409 insertions(+), 30 deletions(-) create mode 100644 packages/docs/content/docs/api-reference/admin/records.md diff --git a/packages/docs/content/docs/api-reference/admin/admin-api.md b/packages/docs/content/docs/api-reference/admin/admin-api.md index 706bf14..409c601 100644 --- a/packages/docs/content/docs/api-reference/admin/admin-api.md +++ b/packages/docs/content/docs/api-reference/admin/admin-api.md @@ -50,7 +50,8 @@ AUTH="Authorization: Bearer $TOKEN" | [API Keys](api-keys.md) | Create, list, and revoke API keys | | [Users](users.md) | Create, list, update, and delete admin users | | [Labelers](labelers.md) | Manage external labeler subscriptions | -| [Instance Settings](settings.md) | Configure app name, logo, and policy URLs | +| [Records](records.md) | List and delete indexed records | +| [Instance Settings](settings.md) | Configure app name, logo, policy URLs, and concurrency settings | | [Domains](domains.md) | Manage domains and their OAuth client identities | | [Script Variables](script-variables.md) | Encrypted key/value pairs for Lua scripts | | [API Clients](api-clients.md) | Register and manage third-party XRPC clients | @@ -72,6 +73,14 @@ Each admin API endpoint requires a specific permission. See the [Permissions gui | `GET /admin/stats` | `stats:read` | | `POST /admin/backfill` | `backfill:create` | | `GET /admin/backfill/status` | `backfill:read` | +| `POST /admin/backfill/{id}/cancel` | `backfill:create` | +| `POST /admin/backfill/{id}/pause` | `backfill:create` | +| `POST /admin/backfill/{id}/resume` | `backfill:create` | +| `GET /admin/backfill/{id}/events` | `backfill:read` | +| `GET /admin/backfill/{id}/repos` | `backfill:read` | +| `GET /admin/backfill/{id}/pds-summary` | `backfill:read` | +| `DELETE /admin/backfill/{id}/details` | `backfill:create` | +| `DELETE /admin/backfill/details` | `backfill:create` | | `GET /admin/events` | `events:read` | | `POST /admin/api-keys` | `api-keys:create` | | `GET /admin/api-keys` | `api-keys:read` | @@ -89,7 +98,12 @@ Each admin API endpoint requires a specific permission. See the [Permissions gui | `GET /admin/labelers` | `labelers:read` | | `PATCH /admin/labelers/{did}` | `labelers:create` | | `DELETE /admin/labelers/{did}` | `labelers:delete` | +| `GET /admin/records` | `records:read` | +| `GET /admin/records/collections` | `records:read` | +| `DELETE /admin/records` | `records:delete` | +| `DELETE /admin/records/collection` | `records:delete-collection`| | `GET /admin/settings` | `settings:manage` | +| `GET /admin/settings/db-info` | `settings:manage` | | `PUT /admin/settings/{key}` | `settings:manage` | | `DELETE /admin/settings/{key}` | `settings:manage` | | `PUT /admin/settings/logo` | `settings:manage` | diff --git a/packages/docs/content/docs/api-reference/admin/backfill.md b/packages/docs/content/docs/api-reference/admin/backfill.md index b4a5a2e..2749488 100644 --- a/packages/docs/content/docs/api-reference/admin/backfill.md +++ b/packages/docs/content/docs/api-reference/admin/backfill.md @@ -151,6 +151,52 @@ curl -X POST "http://127.0.0.1:3000/admin/backfill/$JOB_ID/cancel" -H "$AUTH" Returns `400` if the job is not currently running, or `404` if the job ID is not found. +## Pause a backfill job + +``` +POST /admin/backfill/{id}/pause +``` + +Requests a running backfill job to pause. The job status transitions to `pausing` immediately; the background worker will stop at its next checkpoint and set the status to `paused`. Paused jobs retain all progress and can be resumed later. + +```sh tab="cURL" tab-group="language" +curl -X POST "http://127.0.0.1:3000/admin/backfill/$JOB_ID/pause" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "status": "pausing" +} +``` + +Returns `400` if the job is not currently running, or `404` if the job ID is not found. + +## Resume a backfill job + +``` +POST /admin/backfill/{id}/resume +``` + +Resume a paused backfill job. The job status transitions back to `running` and processing continues from where it left off. + +```sh tab="cURL" tab-group="language" +curl -X POST "http://127.0.0.1:3000/admin/backfill/$JOB_ID/resume" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "status": "running" +} +``` + +Returns `400` if the job is not currently paused, or `404` if the job ID is not found. + ## List backfill jobs ``` @@ -165,6 +211,7 @@ interface BackfillJob { status: string; stage: string; total_repos: number | null; + resolved_repos: number | null; processed_repos: number | null; total_records: number | null; error: string | null; @@ -212,6 +259,7 @@ curl http://127.0.0.1:3000/admin/backfill/status -H "$AUTH" "status": "completed", "stage": "completed", "total_repos": 42, + "resolved_repos": 42, "processed_repos": 42, "total_records": 1000, "error": null, @@ -222,4 +270,104 @@ curl http://127.0.0.1:3000/admin/backfill/status -H "$AUTH" ] ``` -The `status` field tracks the overall job state (`running`, `cancelling`, `cancelled`, `completed`, `failed`). The `stage` field tracks the current processing phase (`pending`, `discovering_repos`, `resolving_pds`, `fetching_records`, `completed`, `failed`, `cancelled`). +The `status` field tracks the overall job state (`running`, `pausing`, `paused`, `cancelling`, `cancelled`, `completed`, `failed`). The `stage` field tracks the current processing phase (`pending`, `discovering_repos`, `resolving_and_fetching`, `completed`, `failed`, `cancelled`). The `resolved_repos` counter tracks PDS resolution progress during the pipelined phase, while `processed_repos` tracks record fetching progress. + +## List repos for a job + +``` +GET /admin/backfill/{id}/repos +``` + +Paginated list of per-DID tracking rows for a backfill job. Requires `BackfillRead`. + +| Param | Type | Required | Description | +| -------- | ------ | -------- | ---------------------------------------------------------------------------------- | +| `phase` | string | no | Filter: `discovered` (all), `resolved` (PDS known), `fetched` (completed) | +| `cursor` | string | no | Keyset cursor (DID) for pagination | +| `limit` | number | no | Max results per page (default 50, max 100) | + +```sh tab="cURL" tab-group="language" +curl "http://127.0.0.1:3000/admin/backfill/$JOB_ID/repos?phase=fetched&limit=10" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "repos": [ + { "did": "did:plc:abc", "pds_endpoint": "https://pds.example.com", "status": "completed", "records_fetched": 42 } + ], + "cursor": "did:plc:def" +} +``` + +`cursor` is `null` when there are no more results. + +## PDS summary for a job + +``` +GET /admin/backfill/{id}/pds-summary +``` + +Aggregated PDS breakdown for a backfill job. Requires `BackfillRead`. No pagination — returns all PDS endpoints in one response, sorted by repo count descending. + +```sh tab="cURL" tab-group="language" +curl "http://127.0.0.1:3000/admin/backfill/$JOB_ID/pds-summary" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "pds_endpoints": [ + { "pds_endpoint": "https://morel.us-east.host.bsky.network", "total_repos": 1200, "completed_repos": 800, "total_records": 5000 } + ] +} +``` + +## Stream backfill events (SSE) + +``` +GET /admin/backfill/{id}/events +``` + +Server-Sent Events stream of real-time backfill progress. Requires `BackfillRead`. The connection stays open until the job completes or the client disconnects. A keepalive comment is sent periodically to prevent timeouts. + +Events are sent with `event: event` and a JSON `data` payload. Each event has a `type` field: + +| Event type | Description | +| ------------------ | ------------------------------------------------- | +| `repo_discovered` | A new DID was found during the discovery phase | +| `repo_resolved` | A DID's PDS endpoint was resolved | +| `repo_fetched` | Record fetching completed for a DID | +| `job_counters` | Updated progress counters | +| `job_stage_changed`| The job moved to a new processing stage | +| `job_completed` | The job finished (completed, failed, or cancelled) | + +## Flush job details + +``` +DELETE /admin/backfill/{id}/details +``` + +Delete all per-repo tracking rows for a single backfill job. Requires `BackfillCreate`. + +```sh tab="cURL" tab-group="language" +curl -X DELETE "http://127.0.0.1:3000/admin/backfill/$JOB_ID/details" -H "$AUTH" +``` + +**Response**: `204 No Content` + +## Flush all job details + +``` +DELETE /admin/backfill/details +``` + +Delete per-repo tracking rows for all completed, cancelled, and failed backfill jobs. Requires `BackfillCreate`. + +```sh tab="cURL" tab-group="language" +curl -X DELETE "http://127.0.0.1:3000/admin/backfill/details" -H "$AUTH" +``` + +**Response**: `204 No Content` diff --git a/packages/docs/content/docs/api-reference/admin/meta.json b/packages/docs/content/docs/api-reference/admin/meta.json index 5582234..d5851f1 100644 --- a/packages/docs/content/docs/api-reference/admin/meta.json +++ b/packages/docs/content/docs/api-reference/admin/meta.json @@ -3,6 +3,7 @@ "pages": [ "admin-api", "lexicons", + "records", "stats", "backfill", "events", diff --git a/packages/docs/content/docs/api-reference/admin/records.md b/packages/docs/content/docs/api-reference/admin/records.md new file mode 100644 index 0000000..1a874aa --- /dev/null +++ b/packages/docs/content/docs/api-reference/admin/records.md @@ -0,0 +1,117 @@ +--- +title: "Records" +--- + +Browse and manage indexed records. All endpoints require the appropriate `records:*` permission. + +```sh tab="cURL" tab-group="language" +# All examples assume $TOKEN is an API key (hv_...) +AUTH="Authorization: Bearer $TOKEN" +``` + +## List records + +``` +GET /admin/records +``` + +Paginated list of records in a collection, ordered by `indexed_at` descending. + +| Param | Type | Required | Description | +| ------------ | ------ | -------- | ----------------------------------------- | +| `collection` | string | yes | Collection NSID to list records from | +| `limit` | number | no | Max results per page (default 20, max 100)| +| `cursor` | string | no | Pagination cursor from a previous response| + +```sh tab="cURL" tab-group="language" +curl "http://127.0.0.1:3000/admin/records?collection=xyz.statusphere.status&limit=10" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "records": [ + { + "uri": "at://did:plc:abc/xyz.statusphere.status/3k...", + "did": "did:plc:abc", + "collection": "xyz.statusphere.status", + "rkey": "3k...", + "cid": "bafyrei...", + "indexed_at": "2025-01-01T00:00:00Z", + "record": { "...": "..." }, + "labels": [] + } + ], + "cursor": "20" +} +``` + +`cursor` is omitted when there are no more results. + +## List collections + +``` +GET /admin/records/collections +``` + +Returns the list of collection NSIDs from registered record-type lexicons. This is a fast lookup (no record counting). + +```sh tab="cURL" tab-group="language" +curl http://127.0.0.1:3000/admin/records/collections -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "collections": [ + "xyz.statusphere.status", + "app.bsky.feed.post" + ] +} +``` + +## Delete a record + +``` +DELETE /admin/records +``` + +Delete a single record by AT URI. + +| Param | Type | Required | Description | +| ----- | ------ | -------- | ------------------------------------ | +| `uri` | string | yes | AT URI of the record to delete | + +```sh tab="cURL" tab-group="language" +curl -X DELETE "http://127.0.0.1:3000/admin/records?uri=at://did:plc:abc/xyz.statusphere.status/3k..." -H "$AUTH" +``` + +**Response**: `204 No Content` + +Returns `404` if the record is not found. + +## Delete all records in a collection + +``` +DELETE /admin/records/collection +``` + +Delete all indexed records for a given collection. Requires both `records:delete` and `records:delete-collection` permissions. + +| Param | Type | Required | Description | +| ------------ | ------ | -------- | ------------------------------------ | +| `collection` | string | yes | Collection NSID to delete from | + +```sh tab="cURL" tab-group="language" +curl -X DELETE "http://127.0.0.1:3000/admin/records/collection?collection=xyz.statusphere.status" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "deleted": 42 +} +``` diff --git a/packages/docs/content/docs/api-reference/admin/settings.md b/packages/docs/content/docs/api-reference/admin/settings.md index 189e40d..d4ecb49 100644 --- a/packages/docs/content/docs/api-reference/admin/settings.md +++ b/packages/docs/content/docs/api-reference/admin/settings.md @@ -58,7 +58,22 @@ resp, err := http.DefaultClient.Do(req) curl http://127.0.0.1:3000/admin/settings -H "$AUTH" ``` -Returns all key/value pairs stored in the `instance_settings` table. +Returns all key/value pairs stored in the `instance_settings` table, plus any env-var fallback values for keys not stored in the database. Each entry includes a `source` field: `"database"` for stored values, `"env"` for env-var fallbacks. + +### Known settings + +| Key | Env var | Default | Description | +|-----|---------|---------|-------------| +| `app_name` | `APP_NAME` | --- | Application name shown in sidebar and OAuth consent screen | +| `client_uri` | `CLIENT_URI` | --- | Public URL for this instance, linked from OAuth consent screen | +| `logo_uri` | `LOGO_URI` | --- | External URL to a logo image | +| `tos_uri` | `TOS_URI` | --- | Link to terms of service | +| `policy_uri` | `POLICY_URI` | --- | Link to privacy policy | +| `backfill_concurrent_pds` | `BACKFILL_CONCURRENT_PDS` | `10` | How many PDS servers to fetch from simultaneously during backfill | +| `backfill_concurrent_dids_per_pds` | `BACKFILL_CONCURRENT_DIDS_PER_PDS` | `3` | How many repos to fetch concurrently from each PDS | +| `backfill_concurrent_resolution` | `BACKFILL_CONCURRENT_RESOLUTION` | `100` | How many DID document lookups to run in parallel during PDS resolution | +| `backfill_retention_days` | `BACKFILL_RETENTION_DAYS` | `28` | Days to keep per-repo detail data from completed backfill jobs. `0` = keep indefinitely | +| `verbose_event_logging` | `VERBOSE_EVENT_LOGGING` | `false` | Log every record index, hook execution, and hook skip to the event log. High write volume — recommended only for debugging | ## Upsert a setting @@ -116,6 +131,38 @@ DELETE /admin/settings/{key} Removes the override; the corresponding environment variable (if any) takes effect again. +## Database info + +``` +GET /admin/settings/db-info +``` + +Returns database backend, connection pool sizes, and whether a server restart is recommended to resize the backfill pool for current concurrency settings. + +```sh tab="cURL" tab-group="language" +curl http://127.0.0.1:3000/admin/settings/db-info -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "backend": "sqlite", + "server_max_connections": null, + "main_pool_size": 32, + "backfill_pool_size": 64, + "restart_recommended": false +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `backend` | string | `"sqlite"` or `"postgres"` | +| `server_max_connections` | number \| null | Postgres `max_connections` setting. `null` for SQLite | +| `main_pool_size` | number | Current main connection pool size | +| `backfill_pool_size` | number | Current backfill connection pool size | +| `restart_recommended` | boolean | `true` if concurrency settings have changed and a restart would resize the pool | + ## Upload / delete logo ``` diff --git a/packages/docs/content/docs/getting-started/configuration.md b/packages/docs/content/docs/getting-started/configuration.md index 5b40060..207a888 100644 --- a/packages/docs/content/docs/getting-started/configuration.md +++ b/packages/docs/content/docs/getting-started/configuration.md @@ -26,7 +26,13 @@ HappyView is configured via environment variables. A `.env` file in the project | `ATTESTATION_PRIVATE_KEY` | no | auto-generated | Hex-encoded 32-byte secp256k1 private key for [attestation signing](../guides/attestation-signing.md). Auto-generated and persisted to database on first run | | `ATTESTATION_KEY_ID` | no | `did:web:{host}#attestation` | Key identifier included in attestation signatures. Derived from `PUBLIC_URL` by default | | `ATTESTATION_SIG_TYPE` | no | app-specific NSID | `$type` value used in attestation signature objects | -| `RUST_LOG` | no | `happyview=debug,tower_http=debug` | Log filter (uses `tracing_subscriber::EnvFilter`) | +| `BACKFILL_CONCURRENT_PDS` | no | `10` | How many PDS servers to fetch from simultaneously during backfill. Overridden by database setting if set via admin API | +| `BACKFILL_CONCURRENT_DIDS_PER_PDS` | no | `3` | How many repos to fetch concurrently from each PDS during backfill. Overridden by database setting if set via admin API | +| `BACKFILL_CONCURRENT_RESOLUTION` | no | `100` | How many DID document lookups to run in parallel during PDS resolution. Overridden by database setting if set via admin API | +| `BACKFILL_RETENTION_DAYS` | no | `28` | Days to keep per-repo detail data from completed backfill jobs. `0` to keep indefinitely. Overridden by database setting if set via admin API | +| `BACKFILL_DATABASE_MAX_CONNECTIONS` | no | auto-calculated | Override the backfill connection pool size. Auto-calculated from concurrency settings if not set | +| `VERBOSE_EVENT_LOGGING` | no | `false` | Log every record index, hook execution, and hook skip to the event log. High write volume — recommended only for debugging. Overridden by database setting if set via admin API | +| `RUST_LOG` | no | `happyview=debug,tower_http=debug,sqlx=warn` | Log filter (uses `tracing_subscriber::EnvFilter`) | | `APP_NAME` | no | --- | Application name shown on OAuth authorization screens. Overridden by database setting if set via admin API | | `LOGO_URI` | no | --- | URL to application logo for OAuth screens. Overridden by database setting or logo upload | | `TOS_URI` | no | --- | URL to terms of service. Overridden by database setting if set via admin API | @@ -59,7 +65,12 @@ SESSION_SECRET=change-me-in-production # TOKEN_ENCRYPTION_KEY=base64-encoded-32-byte-key # DEFAULT_RATE_LIMIT_CAPACITY=100 # DEFAULT_RATE_LIMIT_REFILL_RATE=2.0 -# RUST_LOG=happyview=debug,tower_http=debug +# BACKFILL_CONCURRENT_PDS=10 +# BACKFILL_CONCURRENT_DIDS_PER_PDS=3 +# BACKFILL_CONCURRENT_RESOLUTION=100 +# BACKFILL_RETENTION_DAYS=28 +# VERBOSE_EVENT_LOGGING=false +# RUST_LOG=happyview=debug,tower_http=debug,sqlx=warn # APP_NAME=My App # LOGO_URI=https://example.com/logo.png # TOS_URI=https://example.com/tos diff --git a/packages/docs/content/docs/guides/backfill.md b/packages/docs/content/docs/guides/backfill.md index 642343a..414ebc0 100644 --- a/packages/docs/content/docs/guides/backfill.md +++ b/packages/docs/content/docs/guides/backfill.md @@ -13,13 +13,12 @@ See the [admin API](../api-reference/admin/backfill.md) for endpoint details. ## How it works -A backfill job runs through three sequential phases: +A backfill job starts with a discovery phase and then pipelines resolution and fetching concurrently: 1. **Discovering repos** — HappyView calls the relay's `com.atproto.sync.listReposByCollection` to find repos that contain records for each target collection. Discovered DIDs are stored in a tracking table so progress can be resumed. -2. **Resolving PDS** — For each discovered DID, HappyView resolves the DID document (via PLC directory or `did:web`) to find the user's PDS endpoint. -3. **Fetching records** — HappyView calls `com.atproto.repo.listRecords` on each PDS for the target collection(s), upserting each record into the local database. PDS endpoints are processed concurrently (up to 10 PDS hosts, 3 DIDs per host). +2. **Resolving PDS + Fetching records** (pipelined) — Resolution and fetching run concurrently. As each DID is resolved (via PLC directory or `did:web`), it's immediately handed off for record fetching — there's no need to wait for all DIDs to resolve before fetching begins. HappyView calls `com.atproto.repo.listRecords` on each PDS for the target collection(s), upserting each record into the local database. PDS endpoints are processed concurrently (up to 10 PDS hosts, 3 DIDs per host). -Progress counters (`total_repos`, `processed_repos`, `total_records`) and the current `stage` are updated in real time. The dashboard's Backfill page shows live progress, and clicking a job opens a detail sheet with a stage-by-stage progress log. +Progress counters (`total_repos`, `resolved_repos`, `processed_repos`, `total_records`) and the current `stage` are updated in real time. The dashboard's Backfill page shows live progress, and clicking a job opens a detail sheet with a stage-by-stage progress log. ### Rate limiting @@ -37,7 +36,7 @@ A backfill job has both a `status` (overall state) and a `stage` (current phase) | `completed` | Worker finished processing all resolvable repos | | `failed` | An error occurred | -The `stage` field tracks which phase the job is in: `pending`, `discovering_repos`, `resolving_pds`, `fetching_records`, `completed`, `failed`, or `cancelled`. +The `stage` field tracks which phase the job is in: `pending`, `discovering_repos`, `resolving_and_fetching`, `completed`, `failed`, or `cancelled`. ## Cancelling a job @@ -65,6 +64,31 @@ Re-running a backfill for a collection that's already been backfilled is safe. E Deleting records from HappyView (via the dashboard or API) only removes them from the local database — the records still exist on the atproto network. To restore deleted records, create a backfill job for the affected collection. The backfill will re-discover the repos and re-fetch all records from each PDS, restoring any that were previously deleted. +## Diagnostics + +The dashboard's backfill detail panel includes expandable sections for each processing phase. Clicking a phase row reveals per-repo and per-PDS detail data in real time. + +### Per-repo tracking + +Every DID discovered during a backfill job is tracked in the database with its PDS endpoint, processing status, and record count. This data powers three expandable sections: + +- **Discovering repos** — lists all DIDs discovered for the job, with profile avatars and handles resolved from the Bluesky API. +- **Resolving PDS** — summarises PDS endpoints involved in the job, showing how many repos each PDS is responsible for and how many have been processed. +- **Fetching records** — lists completed repos with their record counts and PDS hostnames. + +All three sections update in real time via SSE (Server-Sent Events) while the job is running. + +### Data retention + +Per-repo detail data is retained after job completion to support post-mortem analysis. A background task runs daily and deletes detail rows for jobs completed more than 28 days ago (configurable via the `backfill_retention_days` setting in **Settings > General**, or the `BACKFILL_RETENTION_DAYS` environment variable). Set to `0` to keep data indefinitely. + +You can also manually clear detail data: + +- **Per-job**: "Clear details" button in the job detail panel footer. +- **All completed jobs**: "Clear all details" button on the Backfill page header. + +Both actions require the `backfill:create` permission. + ## Next steps - [Lexicons](lexicons.md#backfill-flag): Control whether lexicons trigger backfill on upload diff --git a/packages/docs/content/docs/reference/architecture.md b/packages/docs/content/docs/reference/architecture.md index ab58e21..0563cef 100644 --- a/packages/docs/content/docs/reference/architecture.md +++ b/packages/docs/content/docs/reference/architecture.md @@ -154,16 +154,31 @@ sequenceDiagram A->>H: POST /admin/backfill H->>D: Create backfill_jobs record (status = running) + + rect rgb(240, 248, 255) + note over H,Relay: Phase 1: Discovery H->>Relay: listReposByCollection (paginated) Relay-->>H: List of DIDs - loop For each DID - H->>PLC: Resolve DID document - PLC-->>H: PDS endpoint - H->>PDS: listRecords (paginated) - PDS-->>H: Records - H->>D: UPSERT each record - H->>D: Update processed_repos / total_records + H->>D: Insert backfill_repos rows end + + rect rgb(240, 255, 240) + note over H,PDS: Phase 2: Pipelined resolve + fetch (concurrent) + par Resolver task + loop Unresolved DIDs + H->>PLC: Resolve DID document + PLC-->>H: PDS endpoint + H->>D: Update backfill_repos.pds_endpoint + end + and Fetcher task + loop Resolved DIDs (as they arrive) + H->>PDS: listRecords (paginated) + PDS-->>H: Records + H->>D: UPSERT each record + end + end + end + H->>D: Mark job completed (or failed) ``` @@ -274,19 +289,21 @@ sequenceDiagram ### `backfill_jobs` -| Column | Type | Description | -| ----------------- | ----------- | ----------------------------------- | -| `id` | uuid (PK) | | -| `collection` | text | Target collection (null = all) | -| `did` | text | Target DID (null = all) | -| `status` | text | pending, running, completed, failed | -| `total_repos` | integer | | -| `processed_repos` | integer | | -| `total_records` | integer | | -| `error` | text | Error message if failed | -| `started_at` | timestamptz | | -| `completed_at` | timestamptz | | -| `created_at` | timestamptz | | +| Column | Type | Description | +| ----------------- | ----------- | -------------------------------------------------------- | +| `id` | uuid (PK) | | +| `collection` | text | Target collection (null = all) | +| `did` | text | Target DID (null = all) | +| `status` | text | pending, running, pausing, paused, cancelling, cancelled, completed, failed | +| `stage` | text | pending, discovering_repos, resolving_and_fetching, completed, failed, cancelled | +| `total_repos` | integer | Total DIDs discovered | +| `resolved_repos` | integer | DIDs with PDS endpoint resolved | +| `processed_repos` | integer | DIDs with records fetched | +| `total_records` | integer | Total records indexed | +| `error` | text | Error message if failed | +| `started_at` | timestamptz | | +| `completed_at` | timestamptz | | +| `created_at` | timestamptz | | ## Testing -- 2.51.2 From e8268c4038d3724492895253aa0686aa06b8202b Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 18:05:55 -0500 Subject: [PATCH 27/28] fix: allow log verbosity toggle to override env var Signed-off-by: Trezy --- web/src/app/dashboard/settings/general/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/app/dashboard/settings/general/page.tsx b/web/src/app/dashboard/settings/general/page.tsx index 47ebe58..1319841 100644 --- a/web/src/app/dashboard/settings/general/page.tsx +++ b/web/src/app/dashboard/settings/general/page.tsx @@ -481,7 +481,7 @@ export default function GeneralSettingsPage() { onCheckedChange={(checked) => setValues((v) => ({ ...v, - verbose_event_logging: checked ? "true" : "", + verbose_event_logging: checked ? "true" : "false", })) } disabled={!canManage} -- 2.51.2 From 016b0e529445031bc2481285b2c534eb9da4cd2a Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 20:51:50 -0500 Subject: [PATCH 28/28] fix: prevent opening "discovering trepos stage" from locking the ui thread Signed-off-by: Trezy --- src/admin/backfill.rs | 20 +++- web/src/app/dashboard/backfill/page.tsx | 129 ++++++++++++++---------- web/src/workers/backfill-sse.worker.ts | 8 +- 3 files changed, 93 insertions(+), 64 deletions(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 0418ea4..24d48ee 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -325,7 +325,7 @@ async fn run_discovery_phase( ); } else { stream::iter(collections.iter()) - .for_each_concurrent(collections.len(), |collection| async move { + .for_each_concurrent(5, |collection| async move { if should_stop_worker(state, job_id).await { return; } @@ -432,6 +432,16 @@ async fn discover_repos_from_relay( } update_job_counter(state, job_id, "total_repos", running_total).await; + publish_event( + state, + super::types::BackfillEvent::JobCounters { + job_id: job_id.to_string(), + total_repos: Some(running_total), + resolved_repos: None, + processed_repos: None, + total_records: None, + }, + ); if should_stop_worker(state, job_id).await { return Ok(()); @@ -1283,6 +1293,7 @@ async fn fetch_records_from_pds( let base = pds_endpoint.trim_end_matches('/'); let mut cursor: Option = None; let mut count: u32 = 0; + let index_hook = state.lexicons.get_index_hook(collection).await; loop { if cancelled.load(Ordering::Relaxed) { @@ -1326,12 +1337,11 @@ async fn fetch_records_from_pds( let rkey = entry.uri.rsplit('/').next().unwrap_or_default().to_string(); let uri = format!("at://{did}/{collection}/{rkey}"); - let rec_to_store = if let Some(script) = state.lexicons.get_index_hook(collection).await - { + let rec_to_store = if let Some(ref script) = index_hook { let hook_result = crate::lua::execute_hook_script(&crate::lua::HookEvent { state, lexicon_id: collection, - script: &script, + script, action: "create", uri: &uri, did, @@ -1781,7 +1791,7 @@ pub(super) async fn backfill_status( Option, String, )> = sqlx::query_as(&sql) - .fetch_all(&state.db) + .fetch_all(&state.backfill_db) .await .map_err(|e| AppError::Internal(format!("failed to list backfill jobs: {e}")))?; diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index 8ab0cb2..640c532 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -161,7 +161,7 @@ function useBackfillSSE( }); const basePath = process.env.NEXT_PUBLIC_BASE_PATH || ""; - worker.postMessage({ type: "connect", jobId, basePath }); + worker.postMessage({ type: "connect", jobId, baseUrl: `${location.origin}${basePath}` }); return () => { worker.postMessage({ type: "disconnect" }); @@ -390,12 +390,14 @@ export default function BackfillPage() { load(); }, [load]); + const selectedJob = jobs.find((j) => j.id === selectedJobId) ?? null; + const sseActive = selectedJob != null && (selectedJob.status === "running" || selectedJob.status === "cancelling" || selectedJob.status === "pausing"); + useEffect(() => { + if (sseActive) return; const interval = setInterval(load, 5000); return () => clearInterval(interval); - }, [load]); - - const selectedJob = jobs.find((j) => j.id === selectedJobId) ?? null; + }, [load, sseActive]); const canFlush = hasPermission("backfill:create"); return ( @@ -496,7 +498,10 @@ export default function BackfillPage() { { - if (!open) setSelectedJobId(null); + if (!open) { + setSelectedJobId(null); + load(); + } }} > @@ -505,6 +510,13 @@ export default function BackfillPage() { job={selectedJob} canCancel={hasPermission("backfill:create")} canFlush={canFlush} + onJobUpdate={(updater) => { + setJobs((prev) => + prev.map((j) => + j.id === selectedJob.id ? updater(j) : j, + ), + ); + }} onCancel={async () => { await cancelBackfillJob(selectedJob.id); load(); @@ -530,6 +542,7 @@ function JobDetail({ job, canCancel, canFlush, + onJobUpdate, onCancel, onPause, onResume, @@ -537,6 +550,7 @@ function JobDetail({ job: BackfillJob; canCancel: boolean; canFlush: boolean; + onJobUpdate: (updater: (job: BackfillJob) => BackfillJob) => void; onCancel: () => Promise; onPause: () => Promise; onResume: () => Promise; @@ -561,10 +575,11 @@ function JobDetail({ const [fetchedCursor, setFetchedCursor] = useState(null); const [fetchedLoaded, setFetchedLoaded] = useState(false); - // Refs for open state so the SSE callback doesn't need to re-bind on toggle - const discoveredOpenRef = useRef(false); + // Refs for open state and callbacks so the SSE callback doesn't need to re-bind on toggle const pdsOpenRef = useRef(false); const fetchedOpenRef = useRef(false); + const onJobUpdateRef = useRef(onJobUpdate); + onJobUpdateRef.current = onJobUpdate; function hasReached(phase: (typeof PROGRESS_PHASES)[number]): boolean { if (allDone) return true; @@ -626,7 +641,6 @@ function JobDetail({ const [discoveredOpen, setDiscoveredOpen] = useState(false); const [pdsOpen, setPdsOpen] = useState(false); const [fetchedOpen, setFetchedOpen] = useState(false); - discoveredOpenRef.current = discoveredOpen; pdsOpenRef.current = pdsOpen; fetchedOpenRef.current = fetchedOpen; @@ -677,52 +691,33 @@ function JobDetail({ // Uses refs for open state so the callback identity is stable and doesn't // cause the worker to reconnect when sections are toggled. const handleSSEBatch = useCallback((events: BackfillEvent[]) => { - const dOpen = discoveredOpenRef.current; const pOpen = pdsOpenRef.current; const fOpen = fetchedOpenRef.current; - if (dOpen) { - const discovered = events.filter((e) => e.type === "repo_discovered" && e.did); - const resolved = events.filter((e) => e.type === "repo_resolved" && e.did); - if (discovered.length > 0 || resolved.length > 0) { - setDiscoveredRepos((prev) => { - let next = prev; - for (const e of discovered) { - if (!next.some((r) => r.did === e.did)) { - next = [{ did: e.did!, pds_endpoint: null, status: "pending", records_fetched: 0 }, ...next]; - } - } - for (const e of resolved) { - next = next.map((r) => r.did === e.did ? { ...r, pds_endpoint: e.pds_endpoint! } : r); - } - return next; - }); - } - } - if (pOpen) { const pdsEvents = events.filter( (e) => (e.type === "repo_resolved" || e.type === "repo_fetched") && e.pds_endpoint, ); if (pdsEvents.length > 0) { setPdsSummary((prev) => { - let next = [...prev]; + const byEndpoint = new Map(prev.map((p, i) => [p.pds_endpoint, i])); + const next = [...prev]; for (const e of pdsEvents) { - const idx = next.findIndex((p) => p.pds_endpoint === e.pds_endpoint); + const idx = byEndpoint.get(e.pds_endpoint!); if (e.type === "repo_resolved") { - if (idx >= 0) { + if (idx != null) { next[idx] = { ...next[idx], total_repos: next[idx].total_repos + 1 }; } else { + const newIdx = next.length; next.push({ pds_endpoint: e.pds_endpoint!, total_repos: 1, completed_repos: 0, total_records: 0 }); + byEndpoint.set(e.pds_endpoint!, newIdx); } - } else if (e.type === "repo_fetched") { - if (idx >= 0) { - next[idx] = { - ...next[idx], - completed_repos: next[idx].completed_repos + 1, - total_records: next[idx].total_records + (e.records_fetched ?? 0), - }; - } + } else if (e.type === "repo_fetched" && idx != null) { + next[idx] = { + ...next[idx], + completed_repos: next[idx].completed_repos + 1, + total_records: next[idx].total_records + (e.records_fetched ?? 0), + }; } } return next; @@ -734,16 +729,44 @@ function JobDetail({ const fetched = events.filter((e) => e.type === "repo_fetched" && e.did); if (fetched.length > 0) { setFetchedRepos((prev) => { - let next = prev; - for (const e of fetched) { - if (!next.some((r) => r.did === e.did)) { - next = [{ did: e.did!, pds_endpoint: e.pds_endpoint ?? null, status: "completed", records_fetched: e.records_fetched ?? 0 }, ...next]; - } - } - return next; + const existing = new Set(prev.map((r) => r.did)); + const newItems = fetched + .filter((e) => !existing.has(e.did!)) + .map((e) => ({ did: e.did!, pds_endpoint: e.pds_endpoint ?? null, status: "completed" as const, records_fetched: e.records_fetched ?? 0 })); + + if (newItems.length === 0) return prev; + return [...newItems, ...prev]; }); } } + + // Update job counters, stage, and status from SSE + const update = onJobUpdateRef.current; + + // Increment total_repos from repo_discovered events + const discoveredCount = events.filter((e) => e.type === "repo_discovered").length; + const resolvedCount = events.filter((e) => e.type === "repo_resolved").length; + const fetchedEvents = events.filter((e) => e.type === "repo_fetched"); + const fetchedCount = fetchedEvents.length; + const fetchedRecords = fetchedEvents.reduce((sum, e) => sum + (e.records_fetched ?? 0), 0); + + if (discoveredCount > 0 || resolvedCount > 0 || fetchedCount > 0) { + update((j) => ({ + ...j, + total_repos: (j.total_repos ?? 0) + discoveredCount, + resolved_repos: (j.resolved_repos ?? 0) + resolvedCount, + processed_repos: (j.processed_repos ?? 0) + fetchedCount, + total_records: (j.total_records ?? 0) + fetchedRecords, + })); + } + + for (const e of events) { + if (e.type === "job_stage_changed" && e.stage) { + update((j) => ({ ...j, stage: e.stage! })); + } else if (e.type === "job_completed" && e.status) { + update((j) => ({ ...j, status: e.status!, error: e.error ?? null })); + } + } }, []); useBackfillSSE(job.id, isActive, handleSSEBatch); @@ -837,7 +860,6 @@ function JobDetail({ {discoveredRepos.length > 0 ? ( 0 ? ( - {job.status === "pausing" ? "Pausing…" : "Pause Job"} + {pausing || job.status === "pausing" ? "Pausing…" : "Pause Job"} )} {canCancel && job.status === "paused" && ( @@ -1015,7 +1036,6 @@ function JobDetail({ function VirtualRepoList({ repos, - profiles, onVisibleDidsChange, hasMore, onLoadMore, @@ -1023,7 +1043,6 @@ function VirtualRepoList({ renderRow, }: { repos: BackfillRepoEntry[]; - profiles: Map; onVisibleDidsChange: (dids: string[]) => void; hasMore: boolean; onLoadMore: () => void; @@ -1042,10 +1061,10 @@ function VirtualRepoList({ const virtualItems = virtualizer.getVirtualItems(); + const visibleDidsKey = virtualItems.map((item) => repos[item.index]?.did).filter(Boolean).join(","); useEffect(() => { - const dids = virtualItems.map((item) => repos[item.index]?.did).filter(Boolean) as string[]; - onVisibleDidsChange(dids); - }, [virtualItems, repos, onVisibleDidsChange]); + onVisibleDidsChange(visibleDidsKey.split(",").filter(Boolean)); + }, [visibleDidsKey, onVisibleDidsChange]); useEffect(() => { if (!hasMore) return; diff --git a/web/src/workers/backfill-sse.worker.ts b/web/src/workers/backfill-sse.worker.ts index cfdcc82..36d4040 100644 --- a/web/src/workers/backfill-sse.worker.ts +++ b/web/src/workers/backfill-sse.worker.ts @@ -8,7 +8,7 @@ interface BackfillEvent { interface ConnectMessage { type: "connect"; jobId: string; - basePath: string; + baseUrl: string; } interface DisconnectMessage { @@ -30,10 +30,10 @@ function flush() { self.postMessage({ type: "batch", events: batch }); } -function connect(jobId: string, basePath: string) { +function connect(jobId: string, baseUrl: string) { disconnect(); - eventSource = new EventSource(`${basePath}/admin/backfill/${jobId}/events`, { + eventSource = new EventSource(`${baseUrl}/admin/backfill/${jobId}/events`, { withCredentials: true, }); @@ -68,7 +68,7 @@ function disconnect() { self.addEventListener("message", (e: MessageEvent) => { const msg = e.data; if (msg.type === "connect") { - connect(msg.jobId, msg.basePath); + connect(msg.jobId, msg.baseUrl); } else if (msg.type === "disconnect") { disconnect(); } -- 2.51.2