diff --git a/examples/cloudflare-workers/package.json b/examples/cloudflare-workers/package.json index 01310a2..f0d1cc3 100644 --- a/examples/cloudflare-workers/package.json +++ b/examples/cloudflare-workers/package.json @@ -7,7 +7,7 @@ "dev": "wrangler dev --test-scheduled", "deploy": "wrangler deploy", "generate": "tsx generate.ts", - "generate:pull": "tsx generate.ts && lex-cli pull && tsx generate.ts && lex-cli pull", + "generate:pull": "tsx generate.ts && lex-cli pull && tsx generate.ts && lex-cli pull && lex-cli generate", "sync": "tsx sync.ts", "sync:remote": "tsx sync.ts --remote" }, diff --git a/examples/sveltekit-cloudflare-workers/.gitignore b/examples/sveltekit-cloudflare-workers/.gitignore index 3b462cb..1c43954 100644 --- a/examples/sveltekit-cloudflare-workers/.gitignore +++ b/examples/sveltekit-cloudflare-workers/.gitignore @@ -21,3 +21,6 @@ Thumbs.db # Vite vite.config.js.timestamp-* vite.config.ts.timestamp-* + +# Generated types (regenerated via pnpm generate:pull) +src/lexicon-types/ diff --git a/examples/sveltekit-cloudflare-workers/README.md b/examples/sveltekit-cloudflare-workers/README.md index 0bc33f3..70f9ce0 100644 --- a/examples/sveltekit-cloudflare-workers/README.md +++ b/examples/sveltekit-cloudflare-workers/README.md @@ -1,82 +1,77 @@ -# svelte cloudflare statusphere +# SvelteKit + Contrail + AT Protocol OAuth -> **Work in progress** +A SvelteKit app on Cloudflare Workers with self-hosted AT Protocol record indexing via [contrail](https://github.com/flo-bit/contrail), fully typed queries, and OAuth authentication. ![screenshot](./screenshot.png) +## Setup -**Demo:** https://statusphere.atmo.tools +```sh +pnpm install +pnpm generate:pull # generate lexicons + types +pnpm dev +``` +Dev mode uses a loopback OAuth client — no keys or Cloudflare setup needed. -svelte + cloudflare workers statusphere demo, built with lots of [`@atcute`](https://github.com/mary-ext/atcute) packages, [ufos.microcosm.blue](https://ufos.microcosm.blue/) (for recent status updates without its own backend), jetstream subscription for real-time updates and [@foxui](https://flo-bit.dev/ui-kit) for ui components. +## Config -also doubles as a demo of `@atcute/oauth-node-client` for server-side oauth flows in cloudflare workers, with session storage in KV and HMAC-signed cookies and lots of useful functions. +Define which AT Protocol collections to index in `src/lib/contrail/config.ts`: -## Quick Start +```ts +import type { ContrailConfig } from 'contrail'; -```sh -pnpm install -pnpm dev +export const config: ContrailConfig = { + namespace: 'statusphere.app', + collections: { + 'xyz.statusphere.status': { + queryable: { + status: {}, // equality filter (?status=...) + createdAt: { type: 'range' } // range filter (?createdAtMin=...&createdAtMax=...) + } + } + } +}; ``` -Dev mode uses a loopback oauth client — no keys or cloudflare setup needed. Open the URL shown in the terminal and log in with any Bluesky handle. (The port is randomized per project in case you're running multiple projects at one — set `src/lib/atproto/port.ts`.) +After changing the config, run `pnpm generate:pull` to regenerate lexicons and types. -See [GETTING_STARTED.md](GETTING_STARTED.md) for production deployment, tunnel setup, and configuration. +Wrangler bindings (`wrangler.jsonc`): -## Adding the oauth part to an existing project +- **D1** (`DB`) — contrail's database +- **KV** (`OAUTH_SESSIONS`, `OAUTH_STATES`) — OAuth session storage +- **Vars** — `CRON_SECRET`, `OAUTH_PUBLIC_URL`, `CLIENT_ASSERTION_KEY`, `COOKIE_SECRET` -**With an AI agent** — paste this into Claude Code (or similar) in your existing repo: +## Deploy -``` -add atproto oauth to this project https://raw.githubusercontent.com/flo-bit/svelte-cloudflare-statusphere/main/AGENT_SETUP.md +```sh +npx wrangler d1 create statusphere +# Add database_id to wrangler.jsonc + +pnpm build +npx wrangler deploy ``` -The [agent prompt](AGENT_SETUP.md) asks a few questions and sets everything up. +## How it works -**Manually** — see [SETUP.md](SETUP.md) for a step-by-step guide. +**Contrail** indexes AT Protocol records into D1 via Jetstream (cron, every minute). When a user posts, `contrail.notify()` indexes it immediately. -## Project Structure +**Typed queries** use `@atcute/client` with an in-process handler — full type safety, zero HTTP overhead: +```ts +const client = getClient(platform!.env.DB); +const res = await client.get('xyz.statusphere.status.listRecords', { + params: { limit: 50, profiles: true } // typed params +}); +res.data.records // typed response ``` -src/lib/atproto/ -├── auth.svelte.ts # Client-side auth state & login/logout/signup -├── image-helper.ts # Image compression + upload helpers -├── index.ts # Public exports -├── methods.ts # AT Protocol helpers (read/write/resolve) -├── port.ts # Dev server port (randomized per project) -├── settings.ts # Collections, scope, config constants -├── server/ -│ ├── oauth.ts # OAuthClient factory (loopback vs confidential) -│ ├── oauth.remote.ts # Remote functions: login, logout -│ ├── repo.remote.ts # Remote functions: putRecord, deleteRecord, uploadBlob -│ ├── session.ts # Session restoration from signed cookie -│ ├── profile.ts # Profile loading with optional KV cache -│ ├── kv-store.ts # Cloudflare KV-backed Store -│ └── signed-cookie.ts # HMAC-signed cookie helpers -└── scripts/ - ├── generate-key.ts - ├── generate-secret.ts - ├── setup-dev.ts - └── tunnel.ts - -src/routes/(oauth)/ -├── oauth/callback/+server.ts -├── oauth/jwks.json/+server.ts -└── oauth-client-metadata.json/+server.ts -``` -## How It Works +Types are generated from contrail's config via `pnpm generate:pull`, which produces lexicon JSON and TypeScript types that register with `@atcute/client`. + +**Scheduled ingestion** works around SvelteKit's lack of `scheduled` export support ([sveltejs/kit#4841](https://github.com/sveltejs/kit/issues/4841)) by appending a handler post-build that self-calls `/api/cron`. -- **Auth**: Server-side OAuth via `@atcute/oauth-node-client`. Sessions stored in KV, identified by HMAC-signed `did` cookie. -- **Remote functions**: Write operations and auth actions use SvelteKit remote functions — type-safe server calls without manual API routes. -- **Dev mode**: Loopback client by default. Set `OAUTH_PUBLIC_URL` in `.env` for confidential client via tunnel. -- **Prod mode**: Confidential client with `private_key_jwt`, KV stores, `OAUTH_PUBLIC_URL` from `wrangler.jsonc`. +**OAuth** uses `@atcute/oauth-node-client` with KV-backed sessions and HMAC-signed cookies. See [SETUP.md](SETUP.md) for details. ## License MIT - - -## todo - -- make typesafe (with lexicons) \ No newline at end of file diff --git a/examples/sveltekit-cloudflare-workers/lexicons-generated/statusphere/app/getCursor.json b/examples/sveltekit-cloudflare-workers/lexicons-generated/statusphere/app/getCursor.json new file mode 100644 index 0000000..f85bdbf --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/lexicons-generated/statusphere/app/getCursor.json @@ -0,0 +1,27 @@ +{ + "lexicon": 1, + "id": "statusphere.app.getCursor", + "defs": { + "main": { + "type": "query", + "description": "Get the current cursor position", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "time_us": { + "type": "integer" + }, + "date": { + "type": "string" + }, + "seconds_ago": { + "type": "integer" + } + } + } + } + } + } +} diff --git a/examples/sveltekit-cloudflare-workers/lexicons-generated/statusphere/app/getOverview.json b/examples/sveltekit-cloudflare-workers/lexicons-generated/statusphere/app/getOverview.json new file mode 100644 index 0000000..67dd2f5 --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/lexicons-generated/statusphere/app/getOverview.json @@ -0,0 +1,51 @@ +{ + "lexicon": 1, + "id": "statusphere.app.getOverview", + "defs": { + "main": { + "type": "query", + "description": "Get an overview of all indexed collections", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "total_records", + "collections" + ], + "properties": { + "total_records": { + "type": "integer" + }, + "collections": { + "type": "array", + "items": { + "type": "ref", + "ref": "#collectionStats" + } + } + } + } + } + }, + "collectionStats": { + "type": "object", + "required": [ + "collection", + "records", + "unique_users" + ], + "properties": { + "collection": { + "type": "string" + }, + "records": { + "type": "integer" + }, + "unique_users": { + "type": "integer" + } + } + } + } +} diff --git a/examples/sveltekit-cloudflare-workers/lexicons-generated/statusphere/app/getProfile.json b/examples/sveltekit-cloudflare-workers/lexicons-generated/statusphere/app/getProfile.json new file mode 100644 index 0000000..a226862 --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/lexicons-generated/statusphere/app/getProfile.json @@ -0,0 +1,126 @@ +{ + "lexicon": 1, + "id": "statusphere.app.getProfile", + "defs": { + "main": { + "type": "query", + "description": "Get a user's profile by DID or handle", + "parameters": { + "type": "params", + "required": [ + "actor" + ], + "properties": { + "actor": { + "type": "string", + "format": "at-identifier", + "description": "DID or handle of the user" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "ref", + "ref": "#profileEntry" + } + } + }, + "profileEntry": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string" + }, + "uri": { + "type": "string", + "format": "at-uri" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "cid": { + "type": "string" + }, + "record": { + "type": "ref", + "ref": "#appBskyActorProfile" + } + } + }, + "appBskyActorProfile": { + "type": "object", + "properties": { + "avatar": { + "type": "blob", + "accept": [ + "image/png", + "image/jpeg" + ], + "maxSize": 1000000, + "description": "Small image to be displayed next to posts from account. AKA, 'profile picture'" + }, + "banner": { + "type": "blob", + "accept": [ + "image/png", + "image/jpeg" + ], + "maxSize": 1000000, + "description": "Larger horizontal image to display behind profile view." + }, + "labels": { + "refs": [ + "com.atproto.label.defs#selfLabels" + ], + "type": "union", + "description": "Self-label values, specific to the Bluesky application, on the overall account." + }, + "website": { + "type": "string", + "format": "uri" + }, + "pronouns": { + "type": "string", + "maxLength": 200, + "description": "Free-form pronouns text.", + "maxGraphemes": 20 + }, + "createdAt": { + "type": "string", + "format": "datetime" + }, + "pinnedPost": { + "ref": "com.atproto.repo.strongRef", + "type": "ref" + }, + "description": { + "type": "string", + "maxLength": 2560, + "description": "Free-form profile description text.", + "maxGraphemes": 256 + }, + "displayName": { + "type": "string", + "maxLength": 640, + "maxGraphemes": 64 + }, + "joinedViaStarterPack": { + "ref": "com.atproto.repo.strongRef", + "type": "ref" + } + } + } + } +} diff --git a/examples/sveltekit-cloudflare-workers/lexicons-generated/statusphere/app/notifyOfUpdate.json b/examples/sveltekit-cloudflare-workers/lexicons-generated/statusphere/app/notifyOfUpdate.json new file mode 100644 index 0000000..f34ded3 --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/lexicons-generated/statusphere/app/notifyOfUpdate.json @@ -0,0 +1,59 @@ +{ + "lexicon": 1, + "id": "statusphere.app.notifyOfUpdate", + "defs": { + "main": { + "type": "procedure", + "description": "Notify of a record change for immediate indexing. Fetches the record from the user's PDS and indexes (or deletes) it.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "Single AT URI to fetch and index" + }, + "uris": { + "type": "array", + "items": { + "type": "string", + "format": "at-uri" + }, + "maxLength": 25, + "description": "Batch of AT URIs to fetch and index (max 25)" + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "indexed", + "deleted" + ], + "properties": { + "indexed": { + "type": "integer", + "description": "Number of records created or updated" + }, + "deleted": { + "type": "integer", + "description": "Number of records deleted (not found on PDS)" + }, + "errors": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Errors for individual URIs that could not be processed" + } + } + } + } + } + } +} diff --git a/examples/sveltekit-cloudflare-workers/lexicons-generated/xyz/statusphere/status/getRecord.json b/examples/sveltekit-cloudflare-workers/lexicons-generated/xyz/statusphere/status/getRecord.json new file mode 100644 index 0000000..d13c602 --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/lexicons-generated/xyz/statusphere/status/getRecord.json @@ -0,0 +1,170 @@ +{ + "lexicon": 1, + "id": "xyz.statusphere.status.getRecord", + "defs": { + "main": { + "type": "query", + "description": "Get a single xyz.statusphere.status record by AT URI", + "parameters": { + "type": "params", + "required": [ + "uri" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "AT URI of the record" + }, + "profiles": { + "type": "boolean", + "description": "Include profile + identity info keyed by DID" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "uri", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "cid": { + "type": "string" + }, + "record": { + "type": "ref", + "ref": "xyz.statusphere.status#main" + }, + "time_us": { + "type": "integer" + }, + "profiles": { + "type": "array", + "items": { + "type": "ref", + "ref": "#profileEntry" + } + } + } + } + } + }, + "profileEntry": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string" + }, + "uri": { + "type": "string", + "format": "at-uri" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "cid": { + "type": "string" + }, + "record": { + "type": "ref", + "ref": "#appBskyActorProfile" + } + } + }, + "appBskyActorProfile": { + "type": "object", + "properties": { + "avatar": { + "type": "blob", + "accept": [ + "image/png", + "image/jpeg" + ], + "maxSize": 1000000, + "description": "Small image to be displayed next to posts from account. AKA, 'profile picture'" + }, + "banner": { + "type": "blob", + "accept": [ + "image/png", + "image/jpeg" + ], + "maxSize": 1000000, + "description": "Larger horizontal image to display behind profile view." + }, + "labels": { + "refs": [ + "com.atproto.label.defs#selfLabels" + ], + "type": "union", + "description": "Self-label values, specific to the Bluesky application, on the overall account." + }, + "website": { + "type": "string", + "format": "uri" + }, + "pronouns": { + "type": "string", + "maxLength": 200, + "description": "Free-form pronouns text.", + "maxGraphemes": 20 + }, + "createdAt": { + "type": "string", + "format": "datetime" + }, + "pinnedPost": { + "ref": "com.atproto.repo.strongRef", + "type": "ref" + }, + "description": { + "type": "string", + "maxLength": 2560, + "description": "Free-form profile description text.", + "maxGraphemes": 256 + }, + "displayName": { + "type": "string", + "maxLength": 640, + "maxGraphemes": 64 + }, + "joinedViaStarterPack": { + "ref": "com.atproto.repo.strongRef", + "type": "ref" + } + } + } + } +} diff --git a/examples/sveltekit-cloudflare-workers/lexicons-generated/xyz/statusphere/status/listRecords.json b/examples/sveltekit-cloudflare-workers/lexicons-generated/xyz/statusphere/status/listRecords.json new file mode 100644 index 0000000..69e24d7 --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/lexicons-generated/xyz/statusphere/status/listRecords.json @@ -0,0 +1,222 @@ +{ + "lexicon": 1, + "id": "xyz.statusphere.status.listRecords", + "defs": { + "main": { + "type": "query", + "description": "Query xyz.statusphere.status records with filters", + "parameters": { + "type": "params", + "properties": { + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 200, + "default": 50 + }, + "cursor": { + "type": "string" + }, + "actor": { + "type": "string", + "format": "at-identifier", + "description": "Filter by DID or handle (triggers on-demand backfill)" + }, + "profiles": { + "type": "boolean", + "description": "Include profile + identity info keyed by DID" + }, + "status": { + "type": "string", + "description": "Filter by status" + }, + "createdAtMin": { + "type": "string", + "description": "Minimum value for createdAt" + }, + "createdAtMax": { + "type": "string", + "description": "Maximum value for createdAt" + }, + "sort": { + "type": "string", + "knownValues": [ + "status", + "createdAt" + ], + "description": "Field to sort by (default: time_us)" + }, + "order": { + "type": "string", + "knownValues": [ + "asc", + "desc" + ], + "description": "Sort direction (default: desc for dates/numbers/counts, asc for strings)" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "records" + ], + "properties": { + "records": { + "type": "array", + "items": { + "type": "ref", + "ref": "#record" + } + }, + "cursor": { + "type": "string" + }, + "profiles": { + "type": "array", + "items": { + "type": "ref", + "ref": "#profileEntry" + } + } + } + } + } + }, + "record": { + "type": "object", + "required": [ + "uri", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "cid": { + "type": "string" + }, + "record": { + "type": "ref", + "ref": "xyz.statusphere.status#main" + }, + "time_us": { + "type": "integer" + } + } + }, + "profileEntry": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string" + }, + "uri": { + "type": "string", + "format": "at-uri" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "cid": { + "type": "string" + }, + "record": { + "type": "ref", + "ref": "#appBskyActorProfile" + } + } + }, + "appBskyActorProfile": { + "type": "object", + "properties": { + "avatar": { + "type": "blob", + "accept": [ + "image/png", + "image/jpeg" + ], + "maxSize": 1000000, + "description": "Small image to be displayed next to posts from account. AKA, 'profile picture'" + }, + "banner": { + "type": "blob", + "accept": [ + "image/png", + "image/jpeg" + ], + "maxSize": 1000000, + "description": "Larger horizontal image to display behind profile view." + }, + "labels": { + "refs": [ + "com.atproto.label.defs#selfLabels" + ], + "type": "union", + "description": "Self-label values, specific to the Bluesky application, on the overall account." + }, + "website": { + "type": "string", + "format": "uri" + }, + "pronouns": { + "type": "string", + "maxLength": 200, + "description": "Free-form pronouns text.", + "maxGraphemes": 20 + }, + "createdAt": { + "type": "string", + "format": "datetime" + }, + "pinnedPost": { + "ref": "com.atproto.repo.strongRef", + "type": "ref" + }, + "description": { + "type": "string", + "maxLength": 2560, + "description": "Free-form profile description text.", + "maxGraphemes": 256 + }, + "displayName": { + "type": "string", + "maxLength": 640, + "maxGraphemes": 64 + }, + "joinedViaStarterPack": { + "ref": "com.atproto.repo.strongRef", + "type": "ref" + } + } + } + } +} diff --git a/examples/sveltekit-cloudflare-workers/lexicons-pulled/README.md b/examples/sveltekit-cloudflare-workers/lexicons-pulled/README.md new file mode 100644 index 0000000..812df07 --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/lexicons-pulled/README.md @@ -0,0 +1,5 @@ +# lexicon sources + +this directory contains lexicon documents pulled from the following sources: + +- atproto (nsids: app.bsky.actor.profile, xyz.statusphere.status) diff --git a/examples/sveltekit-cloudflare-workers/lexicons-pulled/app/bsky/actor/profile.json b/examples/sveltekit-cloudflare-workers/lexicons-pulled/app/bsky/actor/profile.json new file mode 100644 index 0000000..1d22cc4 --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/lexicons-pulled/app/bsky/actor/profile.json @@ -0,0 +1,67 @@ +{ + "id": "app.bsky.actor.profile", + "defs": { + "main": { + "key": "literal:self", + "type": "record", + "record": { + "type": "object", + "properties": { + "avatar": { + "type": "blob", + "accept": ["image/png", "image/jpeg"], + "maxSize": 1000000, + "description": "Small image to be displayed next to posts from account. AKA, 'profile picture'" + }, + "banner": { + "type": "blob", + "accept": ["image/png", "image/jpeg"], + "maxSize": 1000000, + "description": "Larger horizontal image to display behind profile view." + }, + "labels": { + "refs": ["com.atproto.label.defs#selfLabels"], + "type": "union", + "description": "Self-label values, specific to the Bluesky application, on the overall account." + }, + "website": { + "type": "string", + "format": "uri" + }, + "pronouns": { + "type": "string", + "maxLength": 200, + "description": "Free-form pronouns text.", + "maxGraphemes": 20 + }, + "createdAt": { + "type": "string", + "format": "datetime" + }, + "pinnedPost": { + "ref": "com.atproto.repo.strongRef", + "type": "ref" + }, + "description": { + "type": "string", + "maxLength": 2560, + "description": "Free-form profile description text.", + "maxGraphemes": 256 + }, + "displayName": { + "type": "string", + "maxLength": 640, + "maxGraphemes": 64 + }, + "joinedViaStarterPack": { + "ref": "com.atproto.repo.strongRef", + "type": "ref" + } + } + }, + "description": "A declaration of a Bluesky account profile." + } + }, + "$type": "com.atproto.lexicon.schema", + "lexicon": 1 +} diff --git a/examples/sveltekit-cloudflare-workers/lexicons-pulled/xyz/statusphere/status.json b/examples/sveltekit-cloudflare-workers/lexicons-pulled/xyz/statusphere/status.json new file mode 100644 index 0000000..c5a314e --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/lexicons-pulled/xyz/statusphere/status.json @@ -0,0 +1,27 @@ +{ + "id": "xyz.statusphere.status", + "defs": { + "main": { + "key": "tid", + "type": "record", + "record": { + "type": "object", + "required": ["status", "createdAt"], + "properties": { + "status": { + "type": "string", + "maxLength": 32, + "minLength": 1, + "maxGraphemes": 1 + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + }, + "$type": "com.atproto.lexicon.schema", + "lexicon": 1 +} diff --git a/examples/sveltekit-cloudflare-workers/package.json b/examples/sveltekit-cloudflare-workers/package.json index 13ee801..97068db 100644 --- a/examples/sveltekit-cloudflare-workers/package.json +++ b/examples/sveltekit-cloudflare-workers/package.json @@ -5,7 +5,11 @@ "type": "module", "scripts": { "dev": "vite dev", - "build": "vite build", + "build": "vite build && tsx scripts/append-scheduled.ts", + "generate": "tsx scripts/generate.ts", + "generate:pull": "tsx scripts/generate.ts && lex-cli pull && tsx scripts/generate.ts && lex-cli pull && lex-cli generate", + "sync": "tsx scripts/sync.ts", + "sync:remote": "tsx scripts/sync.ts --remote", "preview": "vite preview", "prepare": "svelte-kit sync || echo ''", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", @@ -25,6 +29,8 @@ "@atcute/lexicons": "^1.2.9", "@atcute/oauth-node-client": "^1.1.0", "@atcute/tid": "^1.1.2", + "@atcute/lex-cli": "^2.5.3", + "@atcute/lexicon-doc": "^2.1.2", "@cloudflare/workers-types": "^4.20260317.1", "@eslint/compat": "^2.0.3", "@eslint/js": "^10.0.1", @@ -52,6 +58,7 @@ }, "license": "MIT", "dependencies": { + "contrail": "github:flo-bit/contrail", "@atcute/jetstream": "^1.1.2", "@foxui/core": "^0.8.2", "@foxui/social": "^0.8.4", diff --git a/examples/sveltekit-cloudflare-workers/pnpm-lock.yaml b/examples/sveltekit-cloudflare-workers/pnpm-lock.yaml index c4505b3..35f6a60 100644 --- a/examples/sveltekit-cloudflare-workers/pnpm-lock.yaml +++ b/examples/sveltekit-cloudflare-workers/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: '@foxui/time': specifier: ^0.8.2 version: 0.8.2(@internationalized/date@3.10.1)(@sveltejs/kit@2.55.0(@sveltejs/vite-plugin-svelte@7.0.0(svelte@5.55.0)(vite@8.0.3(@emnapi/core@1.9.1)(@emnapi/runtime@1.9.1)(@types/node@25.0.10)(esbuild@0.27.4)(jiti@2.6.1)(tsx@4.21.0)))(svelte@5.55.0)(typescript@6.0.2)(vite@8.0.3(@emnapi/core@1.9.1)(@emnapi/runtime@1.9.1)(@types/node@25.0.10)(esbuild@0.27.4)(jiti@2.6.1)(tsx@4.21.0)))(svelte@5.55.0)(tailwindcss@4.2.2) + contrail: + specifier: github:flo-bit/contrail + version: https://codeload.github.com/flo-bit/contrail/tar.gz/ee24d4faa5d5e1515fa71398bca035b191880dbf(@atcute/identity@1.1.4) valibot: specifier: ^1.3.1 version: 1.3.1(typescript@6.0.2) @@ -36,6 +39,12 @@ importers: '@atcute/identity-resolver': specifier: ^1.2.2 version: 1.2.2(@atcute/identity@1.1.4) + '@atcute/lex-cli': + specifier: ^2.5.3 + version: 2.5.3 + '@atcute/lexicon-doc': + specifier: ^2.1.2 + version: 2.1.2 '@atcute/lexicons': specifier: ^1.2.9 version: 1.2.9 @@ -129,9 +138,21 @@ packages: '@atcute/bluesky@3.3.0': resolution: {integrity: sha512-TrLnlxuD6F/D2ZYzJ3aCiRD0yiFuhmVsd6oULNzzr8V9Xzlufg0yxkRiGmbMiF2iI508y/MFi6vzo625301c5A==, tarball: https://registry.npmjs.org/@atcute/bluesky/-/bluesky-3.3.0.tgz} + '@atcute/car@5.1.1': + resolution: {integrity: sha512-MeRUJNXYgAHrJZw7mMoZJb9xIqv3LZLQw90rRRAVAo8SGNdICwyqe6Bf2LGesX73QM04MBuYO6Kqhvold3TFfg==, tarball: https://registry.npmjs.org/@atcute/car/-/car-5.1.1.tgz} + + '@atcute/cbor@2.3.2': + resolution: {integrity: sha512-xP2SORSau/VVI00x2V4BjwIkHr6EQ7l/MXEOPaa4LGYtePFc4gnD4L1yN10dT5NEuUnvGEuCh6arLB7gz1smVQ==, tarball: https://registry.npmjs.org/@atcute/cbor/-/cbor-2.3.2.tgz} + + '@atcute/cid@2.4.1': + resolution: {integrity: sha512-bwhna69RCv7yetXudtj+2qrMPYvhhIQqvJz6YUpUS98v7OdF3X2dnye9Nig2NDrklZcuyOsu7sQo7GOykJXRLQ==, tarball: https://registry.npmjs.org/@atcute/cid/-/cid-2.4.1.tgz} + '@atcute/client@4.2.1': resolution: {integrity: sha512-ZBFM2pW075JtgGFu5g7HHZBecrClhlcNH8GVP9Zz1aViWR+cjjBsTpeE63rJs+FCOHFYlirUyo5L8SGZ4kMINw==, tarball: https://registry.npmjs.org/@atcute/client/-/client-4.2.1.tgz} + '@atcute/crypto@2.4.1': + resolution: {integrity: sha512-tJ3Pi/XYcAsABKtqSlSOTKfO5YiQ4XdqlTuPS8HiRZSezOPcXBFFzAFWpSIJPURbVPFQL3LLrrK0Ea24wl5qeQ==, tarball: https://registry.npmjs.org/@atcute/crypto/-/crypto-2.4.1.tgz} + '@atcute/identity-resolver@1.2.2': resolution: {integrity: sha512-eUh/UH4bFvuXS0X7epYCeJC/kj4rbBXfSRumLEH4smMVwNOgTo7cL/0Srty+P/qVPoZEyXdfEbS0PHJyzoXmHw==, tarball: https://registry.npmjs.org/@atcute/identity-resolver/-/identity-resolver-1.2.2.tgz} peerDependencies: @@ -143,9 +164,25 @@ packages: '@atcute/jetstream@1.1.2': resolution: {integrity: sha512-u6p/h2xppp7LE6W/9xErAJ6frfN60s8adZuCKtfAaaBBiiYbb1CfpzN8Uc+2qtJZNorqGvuuDb5572Jmh7yHBQ==, tarball: https://registry.npmjs.org/@atcute/jetstream/-/jetstream-1.1.2.tgz} + '@atcute/lex-cli@2.5.3': + resolution: {integrity: sha512-829rvezMOfRkJQRKvupNT8TWT/YYffJ2QsB80D9aPjkXSogrETZA7xZcPaMZBXg+mJaVbLO9S4ThPQmlF0L4UQ==, tarball: https://registry.npmjs.org/@atcute/lex-cli/-/lex-cli-2.5.3.tgz} + hasBin: true + + '@atcute/lexicon-doc@2.1.2': + resolution: {integrity: sha512-jTLcOka7b8BIn2SnIZm2m7l6unlJ0gpgW1MnRpSqNbly/AvyRUR/GREduh/QmjT4SGasDm8vdhrM0kOSPFpDLQ==, tarball: https://registry.npmjs.org/@atcute/lexicon-doc/-/lexicon-doc-2.1.2.tgz} + + '@atcute/lexicon-resolver@0.1.6': + resolution: {integrity: sha512-wJC/ChmpP7k+ywpOd07CMvioXjIGaFpF3bDwXLi/086LYjSWHOvtW6pyC+mqP5wLhjyH2hn4wmi77Buew1l1aw==, tarball: https://registry.npmjs.org/@atcute/lexicon-resolver/-/lexicon-resolver-0.1.6.tgz} + peerDependencies: + '@atcute/identity': ^1.1.0 + '@atcute/identity-resolver': ^1.1.3 + '@atcute/lexicons@1.2.9': resolution: {integrity: sha512-/RRHm2Cw9o8Mcsrq0eo8fjS9okKYLGfuFwrQ0YoP/6sdSDsXshaTLJsvLlcUcaDaSJ1YFOuHIo3zr2Om2F/16g==, tarball: https://registry.npmjs.org/@atcute/lexicons/-/lexicons-1.2.9.tgz} + '@atcute/mst@1.0.0': + resolution: {integrity: sha512-pMce2efib+dmKtnGnIvJZitVncJkpr3AmhyfgfYllni8KzsaDGsJmuGavSVpuojAhQe+6jYwHFtpm/beiiH4uw==, tarball: https://registry.npmjs.org/@atcute/mst/-/mst-1.0.0.tgz} + '@atcute/multibase@1.2.0': resolution: {integrity: sha512-ZK2GRra+qIYq9nNuQB52m2ul0hOmCQEtPobGfTSUxm7pF0OGEkWGkWHugFhNEDVzHzTwPxHp6VGotdZFue4lYQ==, tarball: https://registry.npmjs.org/@atcute/multibase/-/multibase-1.2.0.tgz} @@ -161,6 +198,9 @@ packages: '@atcute/oauth-types@0.1.1': resolution: {integrity: sha512-u+3KMjse3Uc/9hDyilu1QVN7IpcnjVXgRzhddzBB8Uh6wePHNVBDdi9wQvFTVVA3zmxtMJVptXRyLLg6Ou9bqg==, tarball: https://registry.npmjs.org/@atcute/oauth-types/-/oauth-types-0.1.1.tgz} + '@atcute/repo@0.1.3': + resolution: {integrity: sha512-kN4gkrkQgJwI5xkVQ9zSdI8ULR47uH48EdJdjMvKAWsSDu8zKYd8ZQ6n778qa6o3WYEq02v9QVf7pzeU+3fYVA==, tarball: https://registry.npmjs.org/@atcute/repo/-/repo-0.1.3.tgz} + '@atcute/tid@1.1.2': resolution: {integrity: sha512-bmPuOX/TOfcm/vsK9vM98spjkcx2wgd9S2PeK5oLgEr8IbNRPq7iMCAPzOL1nu5XAW3LlkOYQEbYRcw5vcQ37w==, tarball: https://registry.npmjs.org/@atcute/tid/-/tid-1.1.2.tgz} @@ -176,6 +216,9 @@ packages: '@atcute/util-text@1.2.0': resolution: {integrity: sha512-b8WSh+Z7K601eUFFmTFj8QPKDO8Ic0VDDj63sdKzpkm+ySQKsYT5nXekViGqFVKbyKj1V5FyvZvgXad6/aI4QQ==, tarball: https://registry.npmjs.org/@atcute/util-text/-/util-text-1.2.0.tgz} + '@atcute/varint@2.0.0': + resolution: {integrity: sha512-CEY/oVK/nVpL4e5y3sdenLETDL6/Xu5xsE/0TupK+f0Yv8jcD60t2gD8SHROWSvUwYLdkjczLCSA7YrtnjCzWw==, tarball: https://registry.npmjs.org/@atcute/varint/-/varint-2.0.0.tgz} + '@badrap/valita@0.4.6': resolution: {integrity: sha512-4kdqcjyxo/8RQ8ayjms47HCWZIF5981oE5nIenbfThKDxWXtEHKipAOWlflpPJzZx9y/JWYQkp18Awr7VuepFg==, tarball: https://registry.npmjs.org/@badrap/valita/-/valita-0.4.6.tgz} engines: {node: '>= 18'} @@ -819,11 +862,22 @@ packages: '@emnapi/core': ^1.7.1 '@emnapi/runtime': ^1.7.1 + '@noble/secp256k1@3.0.0': + resolution: {integrity: sha512-NJBaR352KyIvj3t6sgT/+7xrNyF9Xk9QlLSIqUGVUYlsnDTAUqY8LOmwpcgEx4AMJXRITQ5XEVHD+mMaPfr3mg==, tarball: https://registry.npmjs.org/@noble/secp256k1/-/secp256k1-3.0.0.tgz} + '@number-flow/svelte@0.4.0': resolution: {integrity: sha512-9tnowrlZlBV3IVe3Gm1V7yXSf4Ugag2k7iW45xqb04HXSa1ApEImopvGWAjJpHDvS849o+UCb0YH461Mtde9lA==, tarball: https://registry.npmjs.org/@number-flow/svelte/-/svelte-0.4.0.tgz} peerDependencies: svelte: ^4 || ^5 + '@optique/core@0.6.11': + resolution: {integrity: sha512-GVLFihzBA1j78NFlkU5N1Lu0jRqET0k6Z66WK8VQKG/a3cxmCInVGSKMIdQG8i6pgC8wD5OizF6Y3QMztmhAxg==, tarball: https://registry.npmjs.org/@optique/core/-/core-0.6.11.tgz} + engines: {bun: '>=1.2.0', deno: '>=2.3.0', node: '>=20.0.0'} + + '@optique/run@0.6.11': + resolution: {integrity: sha512-tsXBEygGSzNpFK2gjsRlXBn7FiScUeLFWIZNpoAZ8iG85Km0/3K9xgqlQAXoQ+uEZBe4XplnzyCDvmEgbyNT8w==, tarball: https://registry.npmjs.org/@optique/run/-/run-0.6.11.tgz} + engines: {bun: '>=1.2.0', deno: '>=2.3.0', node: '>=20.0.0'} + '@oxc-project/types@0.122.0': resolution: {integrity: sha512-oLAl5kBpV4w69UtFZ9xqcmTi+GENWOcPF7FCrczTiBbmC0ibXxCwyvZGbO39rCVEuLGAZM84DH0pUIyyv/YJzA==, tarball: https://registry.npmjs.org/@oxc-project/types/-/types-0.122.0.tgz} @@ -1407,6 +1461,10 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==, tarball: https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz} engines: {node: '>=6'} + contrail@https://codeload.github.com/flo-bit/contrail/tar.gz/ee24d4faa5d5e1515fa71398bca035b191880dbf: + resolution: {tarball: https://codeload.github.com/flo-bit/contrail/tar.gz/ee24d4faa5d5e1515fa71398bca035b191880dbf} + version: 0.0.2 + cookie@0.6.0: resolution: {integrity: sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==, tarball: https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz} engines: {node: '>= 0.6'} @@ -1636,6 +1694,10 @@ packages: hls.js@1.6.15: resolution: {integrity: sha512-E3a5VwgXimGHwpRGV+WxRTKeSp2DW5DI5MWv34ulL3t5UNmyJWCQ1KmLEHbYzcfThfXG8amBL+fCYPneGHC4VA==, tarball: https://registry.npmjs.org/hls.js/-/hls.js-1.6.15.tgz} + hono@4.12.9: + resolution: {integrity: sha512-wy3T8Zm2bsEvxKZM5w21VdHDDcwVS1yUFFY6i8UobSsKfFceT7TOwhbhfKsDyx7tYQlmRM5FLpIuYvNFyjctiA==, tarball: https://registry.npmjs.org/hono/-/hono-4.12.9.tgz} + engines: {node: '>=16.9.0'} + ignore@5.3.2: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==, tarball: https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz} engines: {node: '>= 4'} @@ -2446,11 +2508,35 @@ snapshots: '@atcute/atproto': 3.1.10 '@atcute/lexicons': 1.2.9 + '@atcute/car@5.1.1': + dependencies: + '@atcute/cbor': 2.3.2 + '@atcute/cid': 2.4.1 + '@atcute/uint8array': 1.1.1 + '@atcute/varint': 2.0.0 + + '@atcute/cbor@2.3.2': + dependencies: + '@atcute/cid': 2.4.1 + '@atcute/multibase': 1.2.0 + '@atcute/uint8array': 1.1.1 + + '@atcute/cid@2.4.1': + dependencies: + '@atcute/multibase': 1.2.0 + '@atcute/uint8array': 1.1.1 + '@atcute/client@4.2.1': dependencies: '@atcute/identity': 1.1.4 '@atcute/lexicons': 1.2.9 + '@atcute/crypto@2.4.1': + dependencies: + '@atcute/multibase': 1.2.0 + '@atcute/uint8array': 1.1.1 + '@noble/secp256k1': 3.0.0 + '@atcute/identity-resolver@1.2.2(@atcute/identity@1.1.4)': dependencies: '@atcute/identity': 1.1.4 @@ -2475,6 +2561,38 @@ snapshots: transitivePeerDependencies: - react + '@atcute/lex-cli@2.5.3': + dependencies: + '@atcute/identity': 1.1.4 + '@atcute/identity-resolver': 1.2.2(@atcute/identity@1.1.4) + '@atcute/lexicon-doc': 2.1.2 + '@atcute/lexicon-resolver': 0.1.6(@atcute/identity-resolver@1.2.2(@atcute/identity@1.1.4))(@atcute/identity@1.1.4) + '@atcute/lexicons': 1.2.9 + '@badrap/valita': 0.4.6 + '@optique/core': 0.6.11 + '@optique/run': 0.6.11 + picocolors: 1.1.1 + prettier: 3.8.1 + + '@atcute/lexicon-doc@2.1.2': + dependencies: + '@atcute/identity': 1.1.4 + '@atcute/lexicons': 1.2.9 + '@atcute/uint8array': 1.1.1 + '@atcute/util-text': 1.2.0 + '@badrap/valita': 0.4.6 + + '@atcute/lexicon-resolver@0.1.6(@atcute/identity-resolver@1.2.2(@atcute/identity@1.1.4))(@atcute/identity@1.1.4)': + dependencies: + '@atcute/crypto': 2.4.1 + '@atcute/identity': 1.1.4 + '@atcute/identity-resolver': 1.2.2(@atcute/identity@1.1.4) + '@atcute/lexicon-doc': 2.1.2 + '@atcute/lexicons': 1.2.9 + '@atcute/repo': 0.1.3 + '@atcute/util-fetch': 1.0.5 + '@badrap/valita': 0.4.6 + '@atcute/lexicons@1.2.9': dependencies: '@atcute/uint8array': 1.1.1 @@ -2482,6 +2600,12 @@ snapshots: '@standard-schema/spec': 1.1.0 esm-env: 1.2.2 + '@atcute/mst@1.0.0': + dependencies: + '@atcute/cbor': 2.3.2 + '@atcute/cid': 2.4.1 + '@atcute/uint8array': 1.1.1 + '@atcute/multibase@1.2.0': dependencies: '@atcute/uint8array': 1.1.1 @@ -2517,6 +2641,16 @@ snapshots: '@atcute/oauth-keyset': 0.1.0 '@badrap/valita': 0.4.6 + '@atcute/repo@0.1.3': + dependencies: + '@atcute/car': 5.1.1 + '@atcute/cbor': 2.3.2 + '@atcute/cid': 2.4.1 + '@atcute/crypto': 2.4.1 + '@atcute/lexicons': 1.2.9 + '@atcute/mst': 1.0.0 + '@atcute/uint8array': 1.1.1 + '@atcute/tid@1.1.2': dependencies: '@atcute/time-ms': 1.3.2 @@ -2533,6 +2667,8 @@ snapshots: dependencies: unicode-segmenter: 0.14.5 + '@atcute/varint@2.0.0': {} + '@badrap/valita@0.4.6': {} '@cloudflare/kv-asset-handler@0.4.2': {} @@ -3033,12 +3169,20 @@ snapshots: '@tybys/wasm-util': 0.10.1 optional: true + '@noble/secp256k1@3.0.0': {} + '@number-flow/svelte@0.4.0(svelte@5.55.0)': dependencies: esm-env: 1.2.2 number-flow: 0.6.0 svelte: 5.55.0 + '@optique/core@0.6.11': {} + + '@optique/run@0.6.11': + dependencies: + '@optique/core': 0.6.11 + '@oxc-project/types@0.122.0': {} '@polka/url@1.0.0-next.29': {} @@ -3603,6 +3747,18 @@ snapshots: clsx@2.1.1: {} + contrail@https://codeload.github.com/flo-bit/contrail/tar.gz/ee24d4faa5d5e1515fa71398bca035b191880dbf(@atcute/identity@1.1.4): + dependencies: + '@atcute/atproto': 3.1.10 + '@atcute/client': 4.2.1 + '@atcute/identity-resolver': 1.2.2(@atcute/identity@1.1.4) + '@atcute/jetstream': 1.1.2 + '@atcute/lexicons': 1.2.9 + hono: 4.12.9 + transitivePeerDependencies: + - '@atcute/identity' + - react + cookie@0.6.0: {} cookie@1.1.1: {} @@ -3871,6 +4027,8 @@ snapshots: hls.js@1.6.15: {} + hono@4.12.9: {} + ignore@5.3.2: {} ignore@7.0.5: {} diff --git a/examples/sveltekit-cloudflare-workers/scripts/append-scheduled.ts b/examples/sveltekit-cloudflare-workers/scripts/append-scheduled.ts new file mode 100644 index 0000000..c9bd1cd --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/scripts/append-scheduled.ts @@ -0,0 +1,29 @@ +/** + * Post-build script: appends a `scheduled` handler to the SvelteKit worker output. + * + * SvelteKit's adapter-cloudflare doesn't support the `scheduled` export natively + * (see https://github.com/sveltejs/kit/issues/4841). This script patches the + * generated _worker.js to add one that self-calls the /api/cron endpoint. + */ +import { readFileSync, writeFileSync } from 'fs'; +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; + +const root = join(dirname(fileURLToPath(import.meta.url)), '..'); +const workerPath = join(root, '.svelte-kit', 'cloudflare', '_worker.js'); + +let code = readFileSync(workerPath, 'utf-8'); + +code += ` +// --- Appended by scripts/append-scheduled.ts --- +worker_default.scheduled = async function (event, env, ctx) { + const req = new Request('http://localhost/api/cron', { + method: 'POST', + headers: { 'X-Cron-Secret': env.CRON_SECRET || '' } + }); + ctx.waitUntil(this.fetch(req, env, ctx)); +}; +`; + +writeFileSync(workerPath, code); +console.log('Appended scheduled handler to _worker.js'); diff --git a/examples/sveltekit-cloudflare-workers/scripts/generate.ts b/examples/sveltekit-cloudflare-workers/scripts/generate.ts new file mode 100644 index 0000000..32f8ed8 --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/scripts/generate.ts @@ -0,0 +1,13 @@ +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; +import { config } from '../src/lib/contrail/config'; +import { generateLexicons } from 'contrail/generate'; + +const ROOT_DIR = join(dirname(fileURLToPath(import.meta.url)), '..'); + +generateLexicons({ + config, + rootDir: ROOT_DIR, + outputDir: join(ROOT_DIR, 'lexicons-generated'), + writeRuntimeFiles: true +}); diff --git a/examples/sveltekit-cloudflare-workers/scripts/sync.ts b/examples/sveltekit-cloudflare-workers/scripts/sync.ts new file mode 100644 index 0000000..9847b37 --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/scripts/sync.ts @@ -0,0 +1,68 @@ +/** + * Discover users from relays and backfill their records from PDS. + * + * Usage: + * pnpm sync # local D1 + * pnpm sync:remote # prod D1 + */ +import { Contrail } from 'contrail'; +import { config } from '../src/lib/contrail/config'; +import { getPlatformProxy } from 'wrangler'; + +function elapsed(start: number): string { + const ms = Date.now() - start; + if (ms < 1000) return `${ms}ms`; + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; + const mins = Math.floor(ms / 60_000); + const secs = ((ms % 60_000) / 1000).toFixed(0); + return `${mins}m ${secs}s`; +} + +async function main() { + const remote = process.argv.includes('--remote'); + const syncStart = Date.now(); + + console.log(`=== Sync (${remote ? 'remote/prod' : 'local'} D1) ===\n`); + + const { env, dispose } = await getPlatformProxy<{ DB: D1Database }>({ + environment: remote ? 'production' : undefined + }); + + const contrail = new Contrail({ ...config, db: env.DB }); + + try { + await contrail.init(); + + console.log('--- Discovery ---'); + const discoveryStart = Date.now(); + const discovered = await contrail.discover(); + console.log(` Done: ${discovered.length} users in ${elapsed(discoveryStart)}\n`); + + console.log('--- Backfill ---'); + const backfillStart = Date.now(); + const total = await contrail.backfill({ + concurrency: 100, + onProgress: ({ records, usersComplete, usersTotal, usersFailed }) => { + const secs = (Date.now() - backfillStart) / 1000; + const rate = secs > 0 ? Math.round(records / secs) : 0; + const failStr = usersFailed > 0 ? ` | ${usersFailed} failed` : ''; + process.stdout.write( + `\r ${records} records | ${usersComplete}/${usersTotal} users | ${rate}/s | ${elapsed(backfillStart)}${failStr} ` + ); + } + }); + process.stdout.write('\n'); + console.log(` Done: ${total} records in ${elapsed(backfillStart)}\n`); + + console.log(`=== Finished in ${elapsed(syncStart)} ===`); + console.log(` Discovered: ${discovered.length} users`); + console.log(` Backfilled: ${total} records`); + } finally { + await dispose(); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/examples/sveltekit-cloudflare-workers/src/app.d.ts b/examples/sveltekit-cloudflare-workers/src/app.d.ts index 69b2623..831d778 100644 --- a/examples/sveltekit-cloudflare-workers/src/app.d.ts +++ b/examples/sveltekit-cloudflare-workers/src/app.d.ts @@ -22,6 +22,8 @@ declare global { COOKIE_SECRET: string; OAUTH_PUBLIC_URL: string; PROFILE_CACHE?: KVNamespace; + DB: D1Database; + CRON_SECRET: string; }; } } diff --git a/examples/sveltekit-cloudflare-workers/src/lib/atproto/microcosm.ts b/examples/sveltekit-cloudflare-workers/src/lib/atproto/microcosm.ts deleted file mode 100644 index fca5998..0000000 --- a/examples/sveltekit-cloudflare-workers/src/lib/atproto/microcosm.ts +++ /dev/null @@ -1,19 +0,0 @@ -const BASE_URL = 'https://ufos-api.microcosm.blue'; - -export interface MicrocosmRecord { - did: string; - collection: string; - rkey: string; - record: T; - time_us: number; -} - -export async function recentRecords( - collections: string | string[] -): Promise[]> { - const collection = Array.isArray(collections) ? collections.join(',') : collections; - const url = `${BASE_URL}/records?collection=${encodeURIComponent(collection)}`; - const res = await fetch(url); - if (!res.ok) throw new Error(`Microcosm API error: ${res.status}`); - return res.json(); -} diff --git a/examples/sveltekit-cloudflare-workers/src/lib/atproto/server/repo.remote.ts b/examples/sveltekit-cloudflare-workers/src/lib/atproto/server/repo.remote.ts index 9f2df6b..a9db6fb 100644 --- a/examples/sveltekit-cloudflare-workers/src/lib/atproto/server/repo.remote.ts +++ b/examples/sveltekit-cloudflare-workers/src/lib/atproto/server/repo.remote.ts @@ -2,6 +2,7 @@ import { error } from '@sveltejs/kit'; import { command, getRequestEvent } from '$app/server'; import * as v from 'valibot'; import { collections } from '../settings'; +import { contrail, ensureInit } from '$lib/contrail'; // Validate collection format and check against allowed list from settings const collectionSchema = v.pipe( @@ -32,6 +33,14 @@ export const putRecord = command( } }); + // Immediately index the new/updated record in contrail + const { platform } = getRequestEvent(); + const db = platform?.env?.DB; + if (db) { + await ensureInit(db); + await contrail.notify(response.data.uri, db).catch(() => {}); + } + return response.data; } ); diff --git a/examples/sveltekit-cloudflare-workers/src/lib/contrail/client.ts b/examples/sveltekit-cloudflare-workers/src/lib/contrail/client.ts new file mode 100644 index 0000000..f9d7dc9 --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/src/lib/contrail/client.ts @@ -0,0 +1,32 @@ +import { Client, simpleFetchHandler } from '@atcute/client'; + +export interface Profile { + handle: string; + displayName?: string; + avatar?: string; +} + +/** + * Extract a simple profile from a contrail profile entry. + * Contrail returns { did, handle, record: { displayName, avatar, ... } } + * while components expect { handle, displayName?, avatar? }. + */ +export function extractProfile(entry: { + did: string; + handle?: string; + record?: unknown; +}): Profile { + const record = entry.record as { displayName?: string; avatar?: string } | undefined; + return { + handle: entry.handle ?? entry.did, + displayName: record?.displayName, + avatar: record?.avatar + }; +} + +/** + * Client-side: fully typed @atcute/client that queries the app's own /xrpc/ endpoints. + */ +export function getClient() { + return new Client({ handler: simpleFetchHandler({ service: '' }) }); +} diff --git a/examples/sveltekit-cloudflare-workers/src/lib/contrail/config.ts b/examples/sveltekit-cloudflare-workers/src/lib/contrail/config.ts new file mode 100644 index 0000000..bf287e6 --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/src/lib/contrail/config.ts @@ -0,0 +1,13 @@ +import type { ContrailConfig } from 'contrail'; + +export const config: ContrailConfig = { + namespace: 'statusphere.app', + collections: { + 'xyz.statusphere.status': { + queryable: { + status: {}, + createdAt: { type: 'range' } + } + } + } +}; diff --git a/examples/sveltekit-cloudflare-workers/src/lib/contrail/index.ts b/examples/sveltekit-cloudflare-workers/src/lib/contrail/index.ts new file mode 100644 index 0000000..f8082fc --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/src/lib/contrail/index.ts @@ -0,0 +1,31 @@ +import { Contrail } from 'contrail'; +import { createHandler } from 'contrail/server'; +import { Client } from '@atcute/client'; +import { config } from './config'; + +export const contrail = new Contrail(config); + +let initialized = false; + +export async function ensureInit(db: D1Database) { + if (!initialized) { + await contrail.init(db); + initialized = true; + } +} + +const handle = createHandler(contrail); + +/** + * Server-side: fully typed @atcute/client that routes through contrail in-process. + * No HTTP roundtrip — calls createHandler directly. + */ +export function getServerClient(db: D1Database) { + return new Client({ + handler: async (pathname, init) => { + await ensureInit(db); + const url = new URL(pathname, 'http://localhost'); + return handle(new Request(url, init), db) as Promise; + } + }); +} diff --git a/examples/sveltekit-cloudflare-workers/src/routes/+layout.server.ts b/examples/sveltekit-cloudflare-workers/src/routes/+layout.server.ts index 0fb3c26..756dc10 100644 --- a/examples/sveltekit-cloudflare-workers/src/routes/+layout.server.ts +++ b/examples/sveltekit-cloudflare-workers/src/routes/+layout.server.ts @@ -1,15 +1,25 @@ import type { LayoutServerLoad } from './$types'; -import { loadProfile } from '$lib/atproto/server/profile'; +import { getServerClient } from '$lib/contrail'; +import { extractProfile } from '$lib/contrail/client'; export const load: LayoutServerLoad = async ({ locals, platform }) => { if (!locals.did || !locals.client) { return { did: null, profile: null }; } - const profile = await loadProfile(locals.did, platform?.env?.PROFILE_CACHE); + try { + const client = getServerClient(platform!.env.DB); + const res = await client.get('statusphere.app.getProfile', { + params: { actor: locals.did } + }); - return { - did: locals.did, - profile - }; + if (!res.ok) return { did: locals.did, profile: null }; + + return { + did: locals.did, + profile: extractProfile(res.data) + }; + } catch { + return { did: locals.did, profile: null }; + } }; diff --git a/examples/sveltekit-cloudflare-workers/src/routes/+page.server.ts b/examples/sveltekit-cloudflare-workers/src/routes/+page.server.ts index 7ccf64e..059333a 100644 --- a/examples/sveltekit-cloudflare-workers/src/routes/+page.server.ts +++ b/examples/sveltekit-cloudflare-workers/src/routes/+page.server.ts @@ -1,91 +1,42 @@ -import type { Did } from '@atcute/lexicons'; -import { recentRecords } from '$lib/atproto/microcosm'; -import { loadProfile } from '$lib/atproto/server/profile'; +import { getServerClient } from '$lib/contrail'; +import { extractProfile, type Profile } from '$lib/contrail/client'; import type { PageServerLoad } from './$types'; -export const load: PageServerLoad = async ({ locals, platform }) => { +export const load: PageServerLoad = async ({ platform }) => { try { - const microcosmPromise = recentRecords<{ status: string; createdAt: string }>( - 'xyz.statusphere.status' - ); + const client = getServerClient(platform!.env.DB); - const userPromise = - locals.client && locals.did - ? locals.client - .get('com.atproto.repo.listRecords', { - params: { - repo: locals.did, - collection: 'xyz.statusphere.status', - limit: 20 - } - }) - .catch(() => null) - : null; - - const [microcosmRecords, userResponse] = await Promise.all([microcosmPromise, userPromise]); - - const microcosmStatuses = microcosmRecords.map((r) => ({ - did: r.did, - rkey: r.rkey, - status: r.record.status, - createdAt: r.record.createdAt - })); - - // Find the oldest microcosm timestamp to use as cutoff for user records - const oldestMicrocosm = - microcosmStatuses.length > 0 - ? Math.min(...microcosmStatuses.map((s) => new Date(s.createdAt).getTime())) - : 0; - - const userStatuses = userResponse?.ok - ? userResponse.data.records - .map((r) => ({ - did: locals.did!, - rkey: r.uri.split('/').pop()!, - status: (r.value as { status: string }).status, - createdAt: (r.value as { createdAt: string }).createdAt - })) - .filter((s) => new Date(s.createdAt).getTime() >= oldestMicrocosm) - : []; - - // Merge and deduplicate by did+rkey, then sort by time descending - const seen = new Set(); - const merged = [...userStatuses, ...microcosmStatuses].filter((s) => { - const key = `${s.did}-${s.rkey}`; - if (seen.has(key)) return false; - seen.add(key); - return true; + const res = await client.get('xyz.statusphere.status.listRecords', { + params: { limit: 50, profiles: true, sort: 'createdAt', order: 'desc' } }); - merged.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()); - - // Load profiles for all unique DIDs - const uniqueDids = [...new Set(merged.map((s) => s.did))]; - const profileCache = platform?.env?.PROFILE_CACHE; - const profileEntries = await Promise.all( - uniqueDids.map(async (did) => { - const profile = await loadProfile(did as Did, profileCache); - if (!profile) return null; - return [ - did, - { - handle: profile.handle as string, - displayName: profile.displayName as string | undefined, - avatar: profile.avatar as string | undefined - } - ] as const; + if (!res.ok) return { statuses: [], profiles: {} }; + + const records = res.data.records; + const statuses = records + .map((r) => { + const record = r.record as { status: string; createdAt: string }; + return { + did: r.did, + rkey: r.rkey, + status: record.status, + createdAt: record.createdAt + }; }) - ); - const profiles: Record = {}; - for (const entry of profileEntries) { - if (entry) profiles[entry[0]] = entry[1]; + .filter((s) => !isNaN(new Date(s.createdAt).getTime())); + + const profiles: Record = {}; + if (res.data.profiles) { + for (const p of res.data.profiles) { + profiles[p.did] = extractProfile(p); + } } - return { statuses: merged, profiles }; + return { statuses, profiles }; } catch { return { statuses: [] as { did: string; rkey: string; status: string; createdAt: string }[], - profiles: {} as Record + profiles: {} as Record }; } }; diff --git a/examples/sveltekit-cloudflare-workers/src/routes/+page.svelte b/examples/sveltekit-cloudflare-workers/src/routes/+page.svelte index bb0e283..0225014 100644 --- a/examples/sveltekit-cloudflare-workers/src/routes/+page.svelte +++ b/examples/sveltekit-cloudflare-workers/src/routes/+page.svelte @@ -9,10 +9,11 @@ import { RelativeTime } from '@foxui/time'; import { JetstreamSubscription } from '@atcute/jetstream'; - import { createTID, getDetailedProfile } from '$lib/atproto/methods'; + import { createTID } from '$lib/atproto/methods'; import { putRecord } from '$lib/atproto/server/repo.remote'; import { emojiToNotoAnimatedWebp } from '$lib/emojis'; import { atProtoLoginModalState } from '$lib/atproto/ui/LoginModal.svelte'; + import { getClient, extractProfile } from '$lib/contrail/client'; let { data } = $props(); @@ -32,16 +33,15 @@ ...data.profiles }); + const contrailClient = getClient(); + async function fetchProfile(did: string) { if (profiles[did]) return; try { - const profile = await getDetailedProfile({ did: did as import('@atcute/lexicons').Did }); - if (!profile) return; - profiles[did] = { - handle: profile.handle, - displayName: profile.displayName, - avatar: profile.avatar - }; + const res = await contrailClient.get('statusphere.app.getProfile', { + params: { actor: did } + }); + profiles[did] = extractProfile(res.data); } catch { // ignore fetch errors } diff --git a/examples/sveltekit-cloudflare-workers/src/routes/api/cron/+server.ts b/examples/sveltekit-cloudflare-workers/src/routes/api/cron/+server.ts new file mode 100644 index 0000000..77ff19e --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/src/routes/api/cron/+server.ts @@ -0,0 +1,15 @@ +import { contrail, ensureInit } from '$lib/contrail'; +import type { RequestHandler } from './$types'; + +export const POST: RequestHandler = async ({ request, platform }) => { + const secret = request.headers.get('X-Cron-Secret'); + if (secret !== platform!.env.CRON_SECRET) { + return new Response('Unauthorized', { status: 401 }); + } + + const db = platform!.env.DB; + await ensureInit(db); + await contrail.ingest({}, db); + + return new Response('OK'); +}; diff --git a/examples/sveltekit-cloudflare-workers/src/routes/xrpc/[...path]/+server.ts b/examples/sveltekit-cloudflare-workers/src/routes/xrpc/[...path]/+server.ts new file mode 100644 index 0000000..e592848 --- /dev/null +++ b/examples/sveltekit-cloudflare-workers/src/routes/xrpc/[...path]/+server.ts @@ -0,0 +1,14 @@ +import { createHandler } from 'contrail/server'; +import { contrail, ensureInit } from '$lib/contrail'; +import type { RequestHandler } from './$types'; + +const handle = createHandler(contrail); + +async function handler(request: Request, platform: App.Platform | undefined) { + const db = platform!.env.DB; + await ensureInit(db); + return handle(request, db) as Promise; +} + +export const GET: RequestHandler = async ({ request, platform }) => handler(request, platform); +export const POST: RequestHandler = async ({ request, platform }) => handler(request, platform); diff --git a/examples/sveltekit-cloudflare-workers/wrangler.jsonc b/examples/sveltekit-cloudflare-workers/wrangler.jsonc index 59047a6..1276022 100644 --- a/examples/sveltekit-cloudflare-workers/wrangler.jsonc +++ b/examples/sveltekit-cloudflare-workers/wrangler.jsonc @@ -14,6 +14,16 @@ "vars": { "OAUTH_PUBLIC_URL": "https://statusphere.atmo.tools" }, + "d1_databases": [ + { + "binding": "DB", + "database_name": "statusphere", + "database_id": "REPLACE_WITH_D1_DATABASE_ID" + } + ], + "triggers": { + "crons": ["*/1 * * * *"] + }, "kv_namespaces": [ { "binding": "OAUTH_SESSIONS", diff --git a/package.json b/package.json index 60235c3..29be5e0 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "deploy": "wrangler deploy", "clean": "tsx scripts/clean.ts", "generate": "tsx app/generate.ts", - "generate:pull": "tsx app/generate.ts && lex-cli pull && tsx app/generate.ts && lex-cli pull", + "generate:pull": "tsx app/generate.ts && lex-cli pull && tsx app/generate.ts && lex-cli pull && lex-cli generate", "typecheck": "tsc --noEmit", "ingest": "curl -s http://localhost:8787/__scheduled?cron=*/1+*+*+*+*", "sync": "tsx app/sync.ts", diff --git a/src/core/router/profiles.ts b/src/core/router/profiles.ts index 4551c5b..b66f397 100644 --- a/src/core/router/profiles.ts +++ b/src/core/router/profiles.ts @@ -1,6 +1,8 @@ import type { Database, ContrailConfig, RecordRow } from "../types"; import { recordsTableName } from "../types"; import { resolveIdentities } from "../identity"; +import { getPDS } from "../client"; +import type { Did } from "@atcute/lexicons"; import { batchedInQuery } from "./helpers"; export interface ProfileEntry { @@ -81,6 +83,15 @@ export async function resolveProfiles( // Resolve identities for all DIDs const identities = await resolveIdentities(db, dids); + // Fetch missing profile records from PDS on demand + const missingDids = dids.filter((d) => !result[d]); + if (missingDids.length > 0 && config.profiles && config.profiles.length > 0) { + const fetched = await fetchMissingProfiles(db, config, missingDids); + for (const [did, entry] of Object.entries(fetched)) { + result[did] = entry; + } + } + // Fill in handles and create entries for DIDs without profile records for (const did of dids) { const identity = identities.get(did); @@ -95,3 +106,63 @@ export async function resolveProfiles( return result; } + +/** + * Fetch profile records from PDS for DIDs not yet in the index. + * Fetches in parallel, indexes the results into D1 for future requests. + */ +async function fetchMissingProfiles( + db: Database, + config: ContrailConfig, + dids: string[] +): Promise> { + const result: Record = {}; + const collection = config.profiles![0]; + const table = recordsTableName(collection); + + await Promise.all( + dids.map(async (did) => { + try { + const pds = await getPDS(did as Did, db); + if (!pds) return; + + const url = new URL("/xrpc/com.atproto.repo.getRecord", pds); + url.searchParams.set("repo", did); + url.searchParams.set("collection", collection); + url.searchParams.set("rkey", "self"); + + const res = await fetch(url.toString()); + if (!res.ok) return; + + const data = (await res.json()) as { uri?: string; value?: unknown; cid?: string }; + if (!data.value || !data.cid) return; + + const uri = data.uri ?? `at://${did}/${collection}/self`; + const record = data.value; + const cid = data.cid; + + // Index into D1 for future requests + await db + .prepare( + `INSERT INTO ${table} (uri, did, rkey, cid, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(uri) DO UPDATE SET cid = excluded.cid, record = excluded.record, indexed_at = excluded.indexed_at` + ) + .bind(uri, did, "self", cid, JSON.stringify(record), Date.now() * 1000, Date.now()) + .run(); + + result[did] = { + did, + handle: null, + uri, + collection, + rkey: "self", + cid, + record, + }; + } catch { + // Skip failures silently + } + }) + ); + + return result; +}