diff --git a/.changeset/lean-contrail.md b/.changeset/lean-contrail.md index b277dd1..51f7c01 100644 --- a/.changeset/lean-contrail.md +++ b/.changeset/lean-contrail.md @@ -2,4 +2,4 @@ "@atmo-dev/contrail": minor --- -Collapse Contrail into one public package and one AppView implementation. Remove the spaces, authority, record-host, community, realtime, sync, and custom Lexicon-tooling products. Route Jetstream, persistent, backfill, and immediate synchronization records through the shared `ingestRecords` admission and projection path. Make materialized relation counts converge when children arrive before parents, and prevent transient PDS failures from being interpreted as authoritative deletions. Keep dependent-subject filtering scoped to dependent collections and restore typed example XRPC clients with Atcute's generator. Preserve `node:sqlite` in the published adapter, make all query and search cursors stable across tied and typed/null rows, use Worker-safe cursor encoding, and bound the complete notify resolution/fetch/body operation. Admit newly discovered actors and their dependent mutations as one batch, and keep subject decisions mutation-local so deletes always pass. Existing pagination cursors from 0.12 are intentionally invalidated by the new stable cursor format; clients should discard persisted cursor tokens when upgrading. +Collapse Contrail into one public package and one AppView implementation. Remove the spaces, authority, record-host, community, realtime, sync, and custom Lexicon-tooling products. Route Jetstream, persistent, backfill, and immediate synchronization records through the shared `ingestRecords` admission and projection path. Make materialized relation counts converge when children arrive before parents, and prevent transient PDS failures from being interpreted as authoritative deletions. Keep dependent-subject filtering scoped to dependent collections and restore typed example XRPC clients with Atcute's generator. Preserve `node:sqlite` in the published adapter, make all query and search cursors stable across tied and typed/null rows, use Worker-safe cursor encoding, and bound the complete notify resolution/fetch/body operation. Admit newly discovered actors and their dependent mutations as one batch, and keep subject decisions mutation-local so deletes always pass. Existing pagination cursors from 0.12 are intentionally invalidated by the new stable cursor format; clients should discard persisted cursor tokens when upgrading. Local development now launches Wrangler through the workspace's pnpm installation instead of invoking `npx`. diff --git a/README.md b/README.md index 94f211b..3e8ec5c 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Add a D1 binding and one-minute cron to `wrangler.jsonc`: Then deploy and backfill: ```bash -npx wrangler d1 create contrail +pnpm wrangler d1 create contrail pnpm wrangler deploy pnpm contrail backfill --remote ``` diff --git a/apps/benchmark/README.md b/apps/benchmark/README.md index 0bcac0c..b824aa7 100644 --- a/apps/benchmark/README.md +++ b/apps/benchmark/README.md @@ -28,7 +28,7 @@ The current defaults are 100 concurrent identity resolutions, 20 active PDS host Before every run the harness recursively deletes its config/concurrency-specific `.cache` directory. It disposes and deletes the local D1 afterward as well; pass `--keep-cache` only for debugging. -Results are written to ignored JSON files under `results/`. Selected reference runs live in `baselines/`: [`calendar-default.json`](baselines/calendar-default.json) is the original 774.49-second global-concurrency run, while [`calendar-host-aware.json`](baselines/calendar-host-aware.json) is the comparable 219.74-second host-aware run with set-based derived projection rebuilds. +Results are written to ignored JSON files under `results/`. Selected reference runs live in `baselines/`: [`calendar-default.json`](baselines/calendar-default.json) is the original 774.49-second global-concurrency run, [`calendar-host-aware.json`](baselines/calendar-host-aware.json) is the 219.74-second host-aware result with set-based derived projection rebuilds, and [`calendar-pipelined.json`](baselines/calendar-pipelined.json) is the comparable 134.98-second result after streaming identity resolution and atomically checkpointing projected pages. Each result includes: diff --git a/apps/benchmark/baselines/calendar-pipelined.json b/apps/benchmark/baselines/calendar-pipelined.json new file mode 100644 index 0000000..3e7a5e6 --- /dev/null +++ b/apps/benchmark/baselines/calendar-pipelined.json @@ -0,0 +1,68 @@ +{ + "format": "contrail.backfill-benchmark-baseline", + "version": 1, + "config": "configs/calendar.config.json", + "implementation_commit": "6406cdc70d43bfb55a8d5d5d3869ed6a2020ccec", + "environment": { + "os": "Darwin 25.4.0 arm64", + "cpu": "Apple M1 Pro", + "memory_bytes": 17179869184, + "node": "26.5.0", + "backend": "wrangler-local-d1" + }, + "options": { + "concurrency": 100, + "pdsConcurrency": 20, + "didsPerPds": 3, + "maxAttempts": 1 + }, + "started_at": "2026-08-05T00:33:54.430Z", + "completed_at": "2026-08-05T00:36:09.411Z", + "timings_ms": { + "binding": 309.11, + "init": 268.88, + "discovery": 1705.78, + "backfill": 132264.39, + "total": 134979.27, + "identity_resolution": 14721.7, + "derived_rebuild": 1032.77 + }, + "throughput": { + "accepted_records_per_second": 659.72, + "indexed_records_per_second": 646.37 + }, + "discovered_accounts": 1633, + "accepted_records": 87257, + "indexed_records": 87246, + "peak_rss_kib": 653536, + "network_max_concurrent": 150, + "accounts": { + "total": 1633, + "complete": 1589, + "pending": 0, + "retrying": 44, + "failed": 0 + }, + "collections": [ + { + "collection": "community.lexicon.calendar.event", + "records": 14610, + "unique_users": 318 + }, + { + "collection": "community.lexicon.calendar.rsvp", + "records": 6272, + "unique_users": 1430 + }, + { + "collection": "app.bsky.actor.profile", + "records": 1399, + "unique_users": 1394 + }, + { + "collection": "app.bsky.graph.follow", + "records": 64965, + "unique_users": 1283 + } + ] +} diff --git a/apps/cloudflare-workers/README.md b/apps/cloudflare-workers/README.md index 41b3966..62b2fb2 100644 --- a/apps/cloudflare-workers/README.md +++ b/apps/cloudflare-workers/README.md @@ -16,7 +16,7 @@ backfills run via the `contrail` cli from the library (see `package.json` script ```bash pnpm install -npx wrangler d1 create contrail # copy database_id into wrangler.jsonc +pnpm wrangler d1 create contrail # copy database_id into wrangler.jsonc pnpm contrail backfill --remote # discover + backfill historical events pnpm deploy # deploy the worker ``` diff --git a/apps/postgres/README.md b/apps/postgres/README.md index f4293f7..94a69e5 100644 --- a/apps/postgres/README.md +++ b/apps/postgres/README.md @@ -10,7 +10,7 @@ cp -r examples/postgres my-contrail-app cd my-contrail-app # Install dependencies -npm install +pnpm install ``` > **Note:** The `contrail` dependency in `package.json` points at `github:flo-bit/contrail`. @@ -18,13 +18,13 @@ npm install > dependency to point at your fork's branch: > > ```bash -> npm install github:your-username/contrail#your-branch +> pnpm add github:your-username/contrail#your-branch > ``` > > Or install from a local checkout: > > ```bash -> npm install /path/to/your/contrail +> pnpm add /path/to/your/contrail > ``` ### Start PostgreSQL @@ -58,7 +58,7 @@ Edit `config.ts` to define your collections, queryable fields, relations, and re ### 1. Discover users and backfill records ```bash -npm run sync +pnpm run sync ``` This finds users from ATProto relays and backfills their existing records from PDS. Safe to interrupt and restart — progress is saved per-DID in the database. @@ -66,7 +66,7 @@ This finds users from ATProto relays and backfills their existing records from P ### 2. Start persistent ingestion ```bash -npm run ingest +pnpm run ingest ``` This opens a long-lived Jetstream connection and continuously indexes new records as they appear on the network. Events are batched and flushed every 5 seconds (or every 50 events, whichever comes first). Handles reconnection automatically. @@ -76,7 +76,7 @@ Press `Ctrl+C` for graceful shutdown — the current batch is flushed and the cu ### 3. Serve the XRPC API ```bash -npm run serve +pnpm run serve ``` Your XRPC API is now available at `http://localhost:3000`: @@ -104,9 +104,9 @@ In production you'd typically run sync once (or periodically), then keep `ingest ```bash # Initial sync (run once, or periodically to discover new users) -npm run sync +pnpm run sync # In separate terminals (or use a process manager) -npm run ingest -npm run serve +pnpm run ingest +pnpm run serve ``` diff --git a/apps/postgres/ingest.ts b/apps/postgres/ingest.ts index 0f0cca4..9118d47 100644 --- a/apps/postgres/ingest.ts +++ b/apps/postgres/ingest.ts @@ -5,7 +5,7 @@ * Events are batched and flushed periodically. Handles reconnection automatically. * * Usage: - * DATABASE_URL="postgresql://contrail:contrail@localhost:5432/contrail" npx tsx ingest.ts + * DATABASE_URL="postgresql://contrail:contrail@localhost:5432/contrail" pnpm exec tsx ingest.ts */ import pg from "pg"; import { Contrail } from "@atmo-dev/contrail"; diff --git a/apps/postgres/serve.ts b/apps/postgres/serve.ts index 27bfcac..77bf87a 100644 --- a/apps/postgres/serve.ts +++ b/apps/postgres/serve.ts @@ -2,7 +2,7 @@ * Serve the Contrail XRPC API over HTTP using PostgreSQL. * * Usage: - * DATABASE_URL="postgresql://contrail:contrail@localhost:5432/contrail" npx tsx serve.ts + * DATABASE_URL="postgresql://contrail:contrail@localhost:5432/contrail" pnpm exec tsx serve.ts */ import pg from "pg"; import { createServer } from "node:http"; diff --git a/apps/postgres/sync.ts b/apps/postgres/sync.ts index 4efe47d..7efafe2 100644 --- a/apps/postgres/sync.ts +++ b/apps/postgres/sync.ts @@ -5,7 +5,7 @@ * per-DID in the database. Restarting resumes from where it left off. * * Usage: - * DATABASE_URL="postgresql://contrail:contrail@localhost:5432/contrail" npx tsx sync.ts + * DATABASE_URL="postgresql://contrail:contrail@localhost:5432/contrail" pnpm exec tsx sync.ts */ import pg from "pg"; import { Contrail } from "@atmo-dev/contrail"; diff --git a/apps/sveltekit-cloudflare-workers/README.md b/apps/sveltekit-cloudflare-workers/README.md index 7b4a716..306b7ae 100644 --- a/apps/sveltekit-cloudflare-workers/README.md +++ b/apps/sveltekit-cloudflare-workers/README.md @@ -45,11 +45,11 @@ Wrangler bindings (`wrangler.jsonc`): ## Deploy ```sh -npx wrangler d1 create statusphere +pnpm wrangler d1 create statusphere # Add database_id to wrangler.jsonc pnpm build -npx wrangler deploy +pnpm wrangler deploy ``` ## How it works diff --git a/apps/sveltekit-cloudflare-workers/package.json b/apps/sveltekit-cloudflare-workers/package.json index b01adb5..8436a7b 100644 --- a/apps/sveltekit-cloudflare-workers/package.json +++ b/apps/sveltekit-cloudflare-workers/package.json @@ -16,10 +16,10 @@ "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", "format": "prettier --write .", "lint": "prettier --check . && eslint .", - "env:generate-key": "npx tsx src/lib/atproto/scripts/generate-key.ts", - "env:generate-secret": "npx tsx src/lib/atproto/scripts/generate-secret.ts", - "env:setup-dev": "npx tsx src/lib/atproto/scripts/setup-dev.ts", - "tunnel": "npx tsx src/lib/atproto/scripts/tunnel.ts" + "env:generate-key": "tsx src/lib/atproto/scripts/generate-key.ts", + "env:generate-secret": "tsx src/lib/atproto/scripts/generate-secret.ts", + "env:setup-dev": "tsx src/lib/atproto/scripts/setup-dev.ts", + "tunnel": "tsx src/lib/atproto/scripts/tunnel.ts" }, "devDependencies": { "@atcute/atproto": "^3.1.10", diff --git a/docs/frameworks/sveltekit-cloudflare.md b/docs/frameworks/sveltekit-cloudflare.md index e2b32a3..dd384dd 100644 --- a/docs/frameworks/sveltekit-cloudflare.md +++ b/docs/frameworks/sveltekit-cloudflare.md @@ -183,9 +183,9 @@ declare global { ## 7. Deploy + backfill ```bash -npx wrangler d1 create yourapp # copy the id into wrangler.jsonc +pnpm wrangler d1 create yourapp # copy the id into wrangler.jsonc pnpm build && pnpm wrangler deploy -npx wrangler secret put CRON_SECRET # paste any random string +pnpm wrangler secret put CRON_SECRET # paste any random string pnpm contrail backfill --remote # one-time historical backfill ``` diff --git a/packages/contrail/src/cli/commands/dev.ts b/packages/contrail/src/cli/commands/dev.ts index 73fcbee..d4b9ead 100644 --- a/packages/contrail/src/cli/commands/dev.ts +++ b/packages/contrail/src/cli/commands/dev.ts @@ -86,7 +86,7 @@ export function registerDev(cli: CAC): void { // production; --test-scheduled enables the manual-trigger endpoint). const cronUrl = `http://localhost:8787/__scheduled?cron=${encodeURIComponent(options.cron)}`; - const wrangler = spawn("npx", ["wrangler", "dev", "--test-scheduled"], { + const wrangler = spawn("pnpm", ["exec", "wrangler", "dev", "--test-scheduled"], { stdio: "inherit", shell: process.platform === "win32", cwd: options.root,