Something went wrong. Try again.
[READ-ONLY] Mirror of https://github.com/flo-bit/contrail. atproto backend in a bottle flo-bit.dev/contrail
Something went wrong. Try again.
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172/** * Discover users and backfill their records into PostgreSQL. * * Safe to kill at any point — discovery and backfill cursors are saved * per-DID in the database. Restarting resumes from where it left off. * * Usage: * DATABASE_URL="postgresql://contrail:contrail@localhost:5432/contrail" pnpm exec tsx sync.ts */import pg from "pg";import { Contrail } from "@atmo-dev/contrail";import { createPostgresDatabase } from "@atmo-dev/contrail/postgres";import { config } from "./contrail.config";
const DATABASE_URL = process.env.DATABASE_URL ?? "postgresql://contrail:contrail@localhost:5433/contrail";
function elapsed(start: number): string { const ms = Date.now() - start; if (ms < 1000) return `${ms}ms`; if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; const mins = Math.floor(ms / 60_000); const secs = ((ms % 60_000) / 1000).toFixed(0); return `${mins}m ${secs}s`;}
async function main() { const pool = new pg.Pool({ connectionString: DATABASE_URL }); const db = createPostgresDatabase(pool); const contrail = new Contrail({ ...config, db }); const syncStart = Date.now();
console.log(`=== Sync (PostgreSQL) ===\n`);
await contrail.init();
console.log("--- Discovery ---"); const discoveryStart = Date.now(); const discovered = await contrail.discover(); console.log( ` Done: ${discovered.length} users in ${elapsed(discoveryStart)}\n`, );
console.log("--- Backfill ---"); const backfillStart = Date.now(); const total = await contrail.backfill({ concurrency: 10, onProgress: ({ records, usersComplete, usersTotal, usersFailed }) => { const secs = (Date.now() - backfillStart) / 1000; const rate = secs > 0 ? Math.round(records / secs) : 0; const failStr = usersFailed > 0 ? ` | ${usersFailed} failed` : ""; process.stdout.write( `\r ${records} records | ${usersComplete}/${usersTotal} users | ${rate}/s | ${elapsed(backfillStart)}${failStr} `, ); }, }); process.stdout.write("\n"); console.log(` Done: ${total} records in ${elapsed(backfillStart)}\n`);
console.log(`=== Finished in ${elapsed(syncStart)} ===`); console.log(` Discovered: ${discovered.length} users`); console.log(` Backfilled: ${total} records`);
await pool.end();}
main().catch((err) => { console.error(err); process.exit(1);});