From 455fbeccfff2bee26b1ac2e1ca51620c87c94e66 Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 15:39:24 -0500 Subject: [PATCH] docs: update docs to cover backfill updates Signed-off-by: Trezy --- .../docs/api-reference/admin/admin-api.md | 16 +- .../docs/api-reference/admin/backfill.md | 150 +++++++++++++++++- .../docs/api-reference/admin/meta.json | 1 + .../docs/api-reference/admin/records.md | 117 ++++++++++++++ .../docs/api-reference/admin/settings.md | 49 +++++- .../docs/getting-started/configuration.md | 15 +- packages/docs/content/docs/guides/backfill.md | 34 +++- .../content/docs/reference/architecture.md | 57 ++++--- 8 files changed, 409 insertions(+), 30 deletions(-) create mode 100644 packages/docs/content/docs/api-reference/admin/records.md diff --git a/packages/docs/content/docs/api-reference/admin/admin-api.md b/packages/docs/content/docs/api-reference/admin/admin-api.md index 706bf14..409c601 100644 --- a/packages/docs/content/docs/api-reference/admin/admin-api.md +++ b/packages/docs/content/docs/api-reference/admin/admin-api.md @@ -50,7 +50,8 @@ AUTH="Authorization: Bearer $TOKEN" | [API Keys](api-keys.md) | Create, list, and revoke API keys | | [Users](users.md) | Create, list, update, and delete admin users | | [Labelers](labelers.md) | Manage external labeler subscriptions | -| [Instance Settings](settings.md) | Configure app name, logo, and policy URLs | +| [Records](records.md) | List and delete indexed records | +| [Instance Settings](settings.md) | Configure app name, logo, policy URLs, and concurrency settings | | [Domains](domains.md) | Manage domains and their OAuth client identities | | [Script Variables](script-variables.md) | Encrypted key/value pairs for Lua scripts | | [API Clients](api-clients.md) | Register and manage third-party XRPC clients | @@ -72,6 +73,14 @@ Each admin API endpoint requires a specific permission. See the [Permissions gui | `GET /admin/stats` | `stats:read` | | `POST /admin/backfill` | `backfill:create` | | `GET /admin/backfill/status` | `backfill:read` | +| `POST /admin/backfill/{id}/cancel` | `backfill:create` | +| `POST /admin/backfill/{id}/pause` | `backfill:create` | +| `POST /admin/backfill/{id}/resume` | `backfill:create` | +| `GET /admin/backfill/{id}/events` | `backfill:read` | +| `GET /admin/backfill/{id}/repos` | `backfill:read` | +| `GET /admin/backfill/{id}/pds-summary` | `backfill:read` | +| `DELETE /admin/backfill/{id}/details` | `backfill:create` | +| `DELETE /admin/backfill/details` | `backfill:create` | | `GET /admin/events` | `events:read` | | `POST /admin/api-keys` | `api-keys:create` | | `GET /admin/api-keys` | `api-keys:read` | @@ -89,7 +98,12 @@ Each admin API endpoint requires a specific permission. See the [Permissions gui | `GET /admin/labelers` | `labelers:read` | | `PATCH /admin/labelers/{did}` | `labelers:create` | | `DELETE /admin/labelers/{did}` | `labelers:delete` | +| `GET /admin/records` | `records:read` | +| `GET /admin/records/collections` | `records:read` | +| `DELETE /admin/records` | `records:delete` | +| `DELETE /admin/records/collection` | `records:delete-collection`| | `GET /admin/settings` | `settings:manage` | +| `GET /admin/settings/db-info` | `settings:manage` | | `PUT /admin/settings/{key}` | `settings:manage` | | `DELETE /admin/settings/{key}` | `settings:manage` | | `PUT /admin/settings/logo` | `settings:manage` | diff --git a/packages/docs/content/docs/api-reference/admin/backfill.md b/packages/docs/content/docs/api-reference/admin/backfill.md index b4a5a2e..2749488 100644 --- a/packages/docs/content/docs/api-reference/admin/backfill.md +++ b/packages/docs/content/docs/api-reference/admin/backfill.md @@ -151,6 +151,52 @@ curl -X POST "http://127.0.0.1:3000/admin/backfill/$JOB_ID/cancel" -H "$AUTH" Returns `400` if the job is not currently running, or `404` if the job ID is not found. +## Pause a backfill job + +``` +POST /admin/backfill/{id}/pause +``` + +Requests a running backfill job to pause. The job status transitions to `pausing` immediately; the background worker will stop at its next checkpoint and set the status to `paused`. Paused jobs retain all progress and can be resumed later. + +```sh tab="cURL" tab-group="language" +curl -X POST "http://127.0.0.1:3000/admin/backfill/$JOB_ID/pause" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "status": "pausing" +} +``` + +Returns `400` if the job is not currently running, or `404` if the job ID is not found. + +## Resume a backfill job + +``` +POST /admin/backfill/{id}/resume +``` + +Resume a paused backfill job. The job status transitions back to `running` and processing continues from where it left off. + +```sh tab="cURL" tab-group="language" +curl -X POST "http://127.0.0.1:3000/admin/backfill/$JOB_ID/resume" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "status": "running" +} +``` + +Returns `400` if the job is not currently paused, or `404` if the job ID is not found. + ## List backfill jobs ``` @@ -165,6 +211,7 @@ interface BackfillJob { status: string; stage: string; total_repos: number | null; + resolved_repos: number | null; processed_repos: number | null; total_records: number | null; error: string | null; @@ -212,6 +259,7 @@ curl http://127.0.0.1:3000/admin/backfill/status -H "$AUTH" "status": "completed", "stage": "completed", "total_repos": 42, + "resolved_repos": 42, "processed_repos": 42, "total_records": 1000, "error": null, @@ -222,4 +270,104 @@ curl http://127.0.0.1:3000/admin/backfill/status -H "$AUTH" ] ``` -The `status` field tracks the overall job state (`running`, `cancelling`, `cancelled`, `completed`, `failed`). The `stage` field tracks the current processing phase (`pending`, `discovering_repos`, `resolving_pds`, `fetching_records`, `completed`, `failed`, `cancelled`). +The `status` field tracks the overall job state (`running`, `pausing`, `paused`, `cancelling`, `cancelled`, `completed`, `failed`). The `stage` field tracks the current processing phase (`pending`, `discovering_repos`, `resolving_and_fetching`, `completed`, `failed`, `cancelled`). The `resolved_repos` counter tracks PDS resolution progress during the pipelined phase, while `processed_repos` tracks record fetching progress. + +## List repos for a job + +``` +GET /admin/backfill/{id}/repos +``` + +Paginated list of per-DID tracking rows for a backfill job. Requires `BackfillRead`. + +| Param | Type | Required | Description | +| -------- | ------ | -------- | ---------------------------------------------------------------------------------- | +| `phase` | string | no | Filter: `discovered` (all), `resolved` (PDS known), `fetched` (completed) | +| `cursor` | string | no | Keyset cursor (DID) for pagination | +| `limit` | number | no | Max results per page (default 50, max 100) | + +```sh tab="cURL" tab-group="language" +curl "http://127.0.0.1:3000/admin/backfill/$JOB_ID/repos?phase=fetched&limit=10" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "repos": [ + { "did": "did:plc:abc", "pds_endpoint": "https://pds.example.com", "status": "completed", "records_fetched": 42 } + ], + "cursor": "did:plc:def" +} +``` + +`cursor` is `null` when there are no more results. + +## PDS summary for a job + +``` +GET /admin/backfill/{id}/pds-summary +``` + +Aggregated PDS breakdown for a backfill job. Requires `BackfillRead`. No pagination — returns all PDS endpoints in one response, sorted by repo count descending. + +```sh tab="cURL" tab-group="language" +curl "http://127.0.0.1:3000/admin/backfill/$JOB_ID/pds-summary" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "pds_endpoints": [ + { "pds_endpoint": "https://morel.us-east.host.bsky.network", "total_repos": 1200, "completed_repos": 800, "total_records": 5000 } + ] +} +``` + +## Stream backfill events (SSE) + +``` +GET /admin/backfill/{id}/events +``` + +Server-Sent Events stream of real-time backfill progress. Requires `BackfillRead`. The connection stays open until the job completes or the client disconnects. A keepalive comment is sent periodically to prevent timeouts. + +Events are sent with `event: event` and a JSON `data` payload. Each event has a `type` field: + +| Event type | Description | +| ------------------ | ------------------------------------------------- | +| `repo_discovered` | A new DID was found during the discovery phase | +| `repo_resolved` | A DID's PDS endpoint was resolved | +| `repo_fetched` | Record fetching completed for a DID | +| `job_counters` | Updated progress counters | +| `job_stage_changed`| The job moved to a new processing stage | +| `job_completed` | The job finished (completed, failed, or cancelled) | + +## Flush job details + +``` +DELETE /admin/backfill/{id}/details +``` + +Delete all per-repo tracking rows for a single backfill job. Requires `BackfillCreate`. + +```sh tab="cURL" tab-group="language" +curl -X DELETE "http://127.0.0.1:3000/admin/backfill/$JOB_ID/details" -H "$AUTH" +``` + +**Response**: `204 No Content` + +## Flush all job details + +``` +DELETE /admin/backfill/details +``` + +Delete per-repo tracking rows for all completed, cancelled, and failed backfill jobs. Requires `BackfillCreate`. + +```sh tab="cURL" tab-group="language" +curl -X DELETE "http://127.0.0.1:3000/admin/backfill/details" -H "$AUTH" +``` + +**Response**: `204 No Content` diff --git a/packages/docs/content/docs/api-reference/admin/meta.json b/packages/docs/content/docs/api-reference/admin/meta.json index 5582234..d5851f1 100644 --- a/packages/docs/content/docs/api-reference/admin/meta.json +++ b/packages/docs/content/docs/api-reference/admin/meta.json @@ -3,6 +3,7 @@ "pages": [ "admin-api", "lexicons", + "records", "stats", "backfill", "events", diff --git a/packages/docs/content/docs/api-reference/admin/records.md b/packages/docs/content/docs/api-reference/admin/records.md new file mode 100644 index 0000000..1a874aa --- /dev/null +++ b/packages/docs/content/docs/api-reference/admin/records.md @@ -0,0 +1,117 @@ +--- +title: "Records" +--- + +Browse and manage indexed records. All endpoints require the appropriate `records:*` permission. + +```sh tab="cURL" tab-group="language" +# All examples assume $TOKEN is an API key (hv_...) +AUTH="Authorization: Bearer $TOKEN" +``` + +## List records + +``` +GET /admin/records +``` + +Paginated list of records in a collection, ordered by `indexed_at` descending. + +| Param | Type | Required | Description | +| ------------ | ------ | -------- | ----------------------------------------- | +| `collection` | string | yes | Collection NSID to list records from | +| `limit` | number | no | Max results per page (default 20, max 100)| +| `cursor` | string | no | Pagination cursor from a previous response| + +```sh tab="cURL" tab-group="language" +curl "http://127.0.0.1:3000/admin/records?collection=xyz.statusphere.status&limit=10" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "records": [ + { + "uri": "at://did:plc:abc/xyz.statusphere.status/3k...", + "did": "did:plc:abc", + "collection": "xyz.statusphere.status", + "rkey": "3k...", + "cid": "bafyrei...", + "indexed_at": "2025-01-01T00:00:00Z", + "record": { "...": "..." }, + "labels": [] + } + ], + "cursor": "20" +} +``` + +`cursor` is omitted when there are no more results. + +## List collections + +``` +GET /admin/records/collections +``` + +Returns the list of collection NSIDs from registered record-type lexicons. This is a fast lookup (no record counting). + +```sh tab="cURL" tab-group="language" +curl http://127.0.0.1:3000/admin/records/collections -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "collections": [ + "xyz.statusphere.status", + "app.bsky.feed.post" + ] +} +``` + +## Delete a record + +``` +DELETE /admin/records +``` + +Delete a single record by AT URI. + +| Param | Type | Required | Description | +| ----- | ------ | -------- | ------------------------------------ | +| `uri` | string | yes | AT URI of the record to delete | + +```sh tab="cURL" tab-group="language" +curl -X DELETE "http://127.0.0.1:3000/admin/records?uri=at://did:plc:abc/xyz.statusphere.status/3k..." -H "$AUTH" +``` + +**Response**: `204 No Content` + +Returns `404` if the record is not found. + +## Delete all records in a collection + +``` +DELETE /admin/records/collection +``` + +Delete all indexed records for a given collection. Requires both `records:delete` and `records:delete-collection` permissions. + +| Param | Type | Required | Description | +| ------------ | ------ | -------- | ------------------------------------ | +| `collection` | string | yes | Collection NSID to delete from | + +```sh tab="cURL" tab-group="language" +curl -X DELETE "http://127.0.0.1:3000/admin/records/collection?collection=xyz.statusphere.status" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "deleted": 42 +} +``` diff --git a/packages/docs/content/docs/api-reference/admin/settings.md b/packages/docs/content/docs/api-reference/admin/settings.md index 189e40d..d4ecb49 100644 --- a/packages/docs/content/docs/api-reference/admin/settings.md +++ b/packages/docs/content/docs/api-reference/admin/settings.md @@ -58,7 +58,22 @@ resp, err := http.DefaultClient.Do(req) curl http://127.0.0.1:3000/admin/settings -H "$AUTH" ``` -Returns all key/value pairs stored in the `instance_settings` table. +Returns all key/value pairs stored in the `instance_settings` table, plus any env-var fallback values for keys not stored in the database. Each entry includes a `source` field: `"database"` for stored values, `"env"` for env-var fallbacks. + +### Known settings + +| Key | Env var | Default | Description | +|-----|---------|---------|-------------| +| `app_name` | `APP_NAME` | --- | Application name shown in sidebar and OAuth consent screen | +| `client_uri` | `CLIENT_URI` | --- | Public URL for this instance, linked from OAuth consent screen | +| `logo_uri` | `LOGO_URI` | --- | External URL to a logo image | +| `tos_uri` | `TOS_URI` | --- | Link to terms of service | +| `policy_uri` | `POLICY_URI` | --- | Link to privacy policy | +| `backfill_concurrent_pds` | `BACKFILL_CONCURRENT_PDS` | `10` | How many PDS servers to fetch from simultaneously during backfill | +| `backfill_concurrent_dids_per_pds` | `BACKFILL_CONCURRENT_DIDS_PER_PDS` | `3` | How many repos to fetch concurrently from each PDS | +| `backfill_concurrent_resolution` | `BACKFILL_CONCURRENT_RESOLUTION` | `100` | How many DID document lookups to run in parallel during PDS resolution | +| `backfill_retention_days` | `BACKFILL_RETENTION_DAYS` | `28` | Days to keep per-repo detail data from completed backfill jobs. `0` = keep indefinitely | +| `verbose_event_logging` | `VERBOSE_EVENT_LOGGING` | `false` | Log every record index, hook execution, and hook skip to the event log. High write volume — recommended only for debugging | ## Upsert a setting @@ -116,6 +131,38 @@ DELETE /admin/settings/{key} Removes the override; the corresponding environment variable (if any) takes effect again. +## Database info + +``` +GET /admin/settings/db-info +``` + +Returns database backend, connection pool sizes, and whether a server restart is recommended to resize the backfill pool for current concurrency settings. + +```sh tab="cURL" tab-group="language" +curl http://127.0.0.1:3000/admin/settings/db-info -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "backend": "sqlite", + "server_max_connections": null, + "main_pool_size": 32, + "backfill_pool_size": 64, + "restart_recommended": false +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `backend` | string | `"sqlite"` or `"postgres"` | +| `server_max_connections` | number \| null | Postgres `max_connections` setting. `null` for SQLite | +| `main_pool_size` | number | Current main connection pool size | +| `backfill_pool_size` | number | Current backfill connection pool size | +| `restart_recommended` | boolean | `true` if concurrency settings have changed and a restart would resize the pool | + ## Upload / delete logo ``` diff --git a/packages/docs/content/docs/getting-started/configuration.md b/packages/docs/content/docs/getting-started/configuration.md index 5b40060..207a888 100644 --- a/packages/docs/content/docs/getting-started/configuration.md +++ b/packages/docs/content/docs/getting-started/configuration.md @@ -26,7 +26,13 @@ HappyView is configured via environment variables. A `.env` file in the project | `ATTESTATION_PRIVATE_KEY` | no | auto-generated | Hex-encoded 32-byte secp256k1 private key for [attestation signing](../guides/attestation-signing.md). Auto-generated and persisted to database on first run | | `ATTESTATION_KEY_ID` | no | `did:web:{host}#attestation` | Key identifier included in attestation signatures. Derived from `PUBLIC_URL` by default | | `ATTESTATION_SIG_TYPE` | no | app-specific NSID | `$type` value used in attestation signature objects | -| `RUST_LOG` | no | `happyview=debug,tower_http=debug` | Log filter (uses `tracing_subscriber::EnvFilter`) | +| `BACKFILL_CONCURRENT_PDS` | no | `10` | How many PDS servers to fetch from simultaneously during backfill. Overridden by database setting if set via admin API | +| `BACKFILL_CONCURRENT_DIDS_PER_PDS` | no | `3` | How many repos to fetch concurrently from each PDS during backfill. Overridden by database setting if set via admin API | +| `BACKFILL_CONCURRENT_RESOLUTION` | no | `100` | How many DID document lookups to run in parallel during PDS resolution. Overridden by database setting if set via admin API | +| `BACKFILL_RETENTION_DAYS` | no | `28` | Days to keep per-repo detail data from completed backfill jobs. `0` to keep indefinitely. Overridden by database setting if set via admin API | +| `BACKFILL_DATABASE_MAX_CONNECTIONS` | no | auto-calculated | Override the backfill connection pool size. Auto-calculated from concurrency settings if not set | +| `VERBOSE_EVENT_LOGGING` | no | `false` | Log every record index, hook execution, and hook skip to the event log. High write volume — recommended only for debugging. Overridden by database setting if set via admin API | +| `RUST_LOG` | no | `happyview=debug,tower_http=debug,sqlx=warn` | Log filter (uses `tracing_subscriber::EnvFilter`) | | `APP_NAME` | no | --- | Application name shown on OAuth authorization screens. Overridden by database setting if set via admin API | | `LOGO_URI` | no | --- | URL to application logo for OAuth screens. Overridden by database setting or logo upload | | `TOS_URI` | no | --- | URL to terms of service. Overridden by database setting if set via admin API | @@ -59,7 +65,12 @@ SESSION_SECRET=change-me-in-production # TOKEN_ENCRYPTION_KEY=base64-encoded-32-byte-key # DEFAULT_RATE_LIMIT_CAPACITY=100 # DEFAULT_RATE_LIMIT_REFILL_RATE=2.0 -# RUST_LOG=happyview=debug,tower_http=debug +# BACKFILL_CONCURRENT_PDS=10 +# BACKFILL_CONCURRENT_DIDS_PER_PDS=3 +# BACKFILL_CONCURRENT_RESOLUTION=100 +# BACKFILL_RETENTION_DAYS=28 +# VERBOSE_EVENT_LOGGING=false +# RUST_LOG=happyview=debug,tower_http=debug,sqlx=warn # APP_NAME=My App # LOGO_URI=https://example.com/logo.png # TOS_URI=https://example.com/tos diff --git a/packages/docs/content/docs/guides/backfill.md b/packages/docs/content/docs/guides/backfill.md index 642343a..414ebc0 100644 --- a/packages/docs/content/docs/guides/backfill.md +++ b/packages/docs/content/docs/guides/backfill.md @@ -13,13 +13,12 @@ See the [admin API](../api-reference/admin/backfill.md) for endpoint details. ## How it works -A backfill job runs through three sequential phases: +A backfill job starts with a discovery phase and then pipelines resolution and fetching concurrently: 1. **Discovering repos** — HappyView calls the relay's `com.atproto.sync.listReposByCollection` to find repos that contain records for each target collection. Discovered DIDs are stored in a tracking table so progress can be resumed. -2. **Resolving PDS** — For each discovered DID, HappyView resolves the DID document (via PLC directory or `did:web`) to find the user's PDS endpoint. -3. **Fetching records** — HappyView calls `com.atproto.repo.listRecords` on each PDS for the target collection(s), upserting each record into the local database. PDS endpoints are processed concurrently (up to 10 PDS hosts, 3 DIDs per host). +2. **Resolving PDS + Fetching records** (pipelined) — Resolution and fetching run concurrently. As each DID is resolved (via PLC directory or `did:web`), it's immediately handed off for record fetching — there's no need to wait for all DIDs to resolve before fetching begins. HappyView calls `com.atproto.repo.listRecords` on each PDS for the target collection(s), upserting each record into the local database. PDS endpoints are processed concurrently (up to 10 PDS hosts, 3 DIDs per host). -Progress counters (`total_repos`, `processed_repos`, `total_records`) and the current `stage` are updated in real time. The dashboard's Backfill page shows live progress, and clicking a job opens a detail sheet with a stage-by-stage progress log. +Progress counters (`total_repos`, `resolved_repos`, `processed_repos`, `total_records`) and the current `stage` are updated in real time. The dashboard's Backfill page shows live progress, and clicking a job opens a detail sheet with a stage-by-stage progress log. ### Rate limiting @@ -37,7 +36,7 @@ A backfill job has both a `status` (overall state) and a `stage` (current phase) | `completed` | Worker finished processing all resolvable repos | | `failed` | An error occurred | -The `stage` field tracks which phase the job is in: `pending`, `discovering_repos`, `resolving_pds`, `fetching_records`, `completed`, `failed`, or `cancelled`. +The `stage` field tracks which phase the job is in: `pending`, `discovering_repos`, `resolving_and_fetching`, `completed`, `failed`, or `cancelled`. ## Cancelling a job @@ -65,6 +64,31 @@ Re-running a backfill for a collection that's already been backfilled is safe. E Deleting records from HappyView (via the dashboard or API) only removes them from the local database — the records still exist on the atproto network. To restore deleted records, create a backfill job for the affected collection. The backfill will re-discover the repos and re-fetch all records from each PDS, restoring any that were previously deleted. +## Diagnostics + +The dashboard's backfill detail panel includes expandable sections for each processing phase. Clicking a phase row reveals per-repo and per-PDS detail data in real time. + +### Per-repo tracking + +Every DID discovered during a backfill job is tracked in the database with its PDS endpoint, processing status, and record count. This data powers three expandable sections: + +- **Discovering repos** — lists all DIDs discovered for the job, with profile avatars and handles resolved from the Bluesky API. +- **Resolving PDS** — summarises PDS endpoints involved in the job, showing how many repos each PDS is responsible for and how many have been processed. +- **Fetching records** — lists completed repos with their record counts and PDS hostnames. + +All three sections update in real time via SSE (Server-Sent Events) while the job is running. + +### Data retention + +Per-repo detail data is retained after job completion to support post-mortem analysis. A background task runs daily and deletes detail rows for jobs completed more than 28 days ago (configurable via the `backfill_retention_days` setting in **Settings > General**, or the `BACKFILL_RETENTION_DAYS` environment variable). Set to `0` to keep data indefinitely. + +You can also manually clear detail data: + +- **Per-job**: "Clear details" button in the job detail panel footer. +- **All completed jobs**: "Clear all details" button on the Backfill page header. + +Both actions require the `backfill:create` permission. + ## Next steps - [Lexicons](lexicons.md#backfill-flag): Control whether lexicons trigger backfill on upload diff --git a/packages/docs/content/docs/reference/architecture.md b/packages/docs/content/docs/reference/architecture.md index ab58e21..0563cef 100644 --- a/packages/docs/content/docs/reference/architecture.md +++ b/packages/docs/content/docs/reference/architecture.md @@ -154,16 +154,31 @@ sequenceDiagram A->>H: POST /admin/backfill H->>D: Create backfill_jobs record (status = running) + + rect rgb(240, 248, 255) + note over H,Relay: Phase 1: Discovery H->>Relay: listReposByCollection (paginated) Relay-->>H: List of DIDs - loop For each DID - H->>PLC: Resolve DID document - PLC-->>H: PDS endpoint - H->>PDS: listRecords (paginated) - PDS-->>H: Records - H->>D: UPSERT each record - H->>D: Update processed_repos / total_records + H->>D: Insert backfill_repos rows end + + rect rgb(240, 255, 240) + note over H,PDS: Phase 2: Pipelined resolve + fetch (concurrent) + par Resolver task + loop Unresolved DIDs + H->>PLC: Resolve DID document + PLC-->>H: PDS endpoint + H->>D: Update backfill_repos.pds_endpoint + end + and Fetcher task + loop Resolved DIDs (as they arrive) + H->>PDS: listRecords (paginated) + PDS-->>H: Records + H->>D: UPSERT each record + end + end + end + H->>D: Mark job completed (or failed) ``` @@ -274,19 +289,21 @@ sequenceDiagram ### `backfill_jobs` -| Column | Type | Description | -| ----------------- | ----------- | ----------------------------------- | -| `id` | uuid (PK) | | -| `collection` | text | Target collection (null = all) | -| `did` | text | Target DID (null = all) | -| `status` | text | pending, running, completed, failed | -| `total_repos` | integer | | -| `processed_repos` | integer | | -| `total_records` | integer | | -| `error` | text | Error message if failed | -| `started_at` | timestamptz | | -| `completed_at` | timestamptz | | -| `created_at` | timestamptz | | +| Column | Type | Description | +| ----------------- | ----------- | -------------------------------------------------------- | +| `id` | uuid (PK) | | +| `collection` | text | Target collection (null = all) | +| `did` | text | Target DID (null = all) | +| `status` | text | pending, running, pausing, paused, cancelling, cancelled, completed, failed | +| `stage` | text | pending, discovering_repos, resolving_and_fetching, completed, failed, cancelled | +| `total_repos` | integer | Total DIDs discovered | +| `resolved_repos` | integer | DIDs with PDS endpoint resolved | +| `processed_repos` | integer | DIDs with records fetched | +| `total_records` | integer | Total records indexed | +| `error` | text | Error message if failed | +| `started_at` | timestamptz | | +| `completed_at` | timestamptz | | +| `created_at` | timestamptz | | ## Testing -- 2.51.2