diff --git a/.changeset/jetstream-v2-live.md b/.changeset/jetstream-v2-live.md new file mode 100644 index 0000000..fa487ae --- /dev/null +++ b/.changeset/jetstream-v2-live.md @@ -0,0 +1,5 @@ +--- +"@atmo-dev/contrail": minor +--- + +Switch scheduled and persistent live ingestion to Jetstream v2 using the official `@bsky/jetstream` client. Live cursors now use instance-local sequence numbers, one normalized v2 service is durably pinned to each generation, legacy timestamp cursors transition atomically into the seq domain, and existing PDS/Alluvium historical acquisition remains unchanged. Alluvium bootstrap now has a separate `sourceUrl`/`--alluvium-source-url` for its legacy v1 manifest identity. Unscoped `getCursor` responses now return `{ cursor }`; ordered-source responses retain their opaque `{ source, epoch, cursor }` position. The minimum supported Node.js version is now 22.15. diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4ebb3f4..322cbd4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -28,7 +28,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 22.14 + node-version: 22.15 cache: 'pnpm' - name: Install dependencies diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d026ac1..8735e58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 22.14 + node-version: 22.15 cache: 'pnpm' - name: Install dependencies diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c99c117..9ae1749 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,7 +36,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 22.14 + node-version: 22.15 cache: 'pnpm' registry-url: 'https://registry.npmjs.org' diff --git a/apps/atmo-rsvp/src/contrail.config.ts b/apps/atmo-rsvp/src/contrail.config.ts index 37d894e..9e57275 100644 --- a/apps/atmo-rsvp/src/contrail.config.ts +++ b/apps/atmo-rsvp/src/contrail.config.ts @@ -3,10 +3,10 @@ import type { ContrailConfig } from "@atmo-dev/contrail"; export const config: ContrailConfig = { namespace: "rsvp.atmo", profiles: ["app.bsky.actor.profile"], - jetstreams: ["wss://jetstream1.us-east.bsky.network"], + jetstreams: ["https://jetstream.us-east.bsky.network"], orderedSource: { source: "jetstream", - epoch: "api-atmo-rsvp-primary-2026-08", + epoch: "api-atmo-rsvp-primary-v2-2026-08", }, notify: true, serviceAuth: { diff --git a/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json b/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json index 93bb5dd..345596c 100644 --- a/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json +++ b/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json @@ -4,19 +4,13 @@ "defs": { "main": { "type": "query", - "description": "Get the current ingestion observation time", + "description": "Get the current Jetstream live cursor", "output": { "encoding": "application/json", "schema": { "type": "object", "properties": { - "time_us": { - "type": "integer" - }, - "date": { - "type": "string" - }, - "seconds_ago": { + "cursor": { "type": "integer" } } diff --git a/apps/cloudflare-workers/package.json b/apps/cloudflare-workers/package.json index 96bf1e5..8b6926d 100644 --- a/apps/cloudflare-workers/package.json +++ b/apps/cloudflare-workers/package.json @@ -5,7 +5,8 @@ "type": "module", "scripts": { "dev": "contrail dev", - "deploy": "wrangler deploy" + "deploy": "wrangler deploy", + "typecheck": "contrail lexicons check --no-atcute-config && tsc --noEmit" }, "dependencies": { "@atmo-dev/contrail": "workspace:*" diff --git a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json index 32cd295..a7e6ce7 100644 --- a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json +++ b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json @@ -4,19 +4,13 @@ "defs": { "main": { "type": "query", - "description": "Get the current ingestion observation time", + "description": "Get the current Jetstream live cursor", "output": { "encoding": "application/json", "schema": { "type": "object", "properties": { - "time_us": { - "type": "integer" - }, - "date": { - "type": "string" - }, - "seconds_ago": { + "cursor": { "type": "integer" } } diff --git a/apps/sveltekit-cloudflare-workers/package.json b/apps/sveltekit-cloudflare-workers/package.json index 9023e69..dbcacdb 100644 --- a/apps/sveltekit-cloudflare-workers/package.json +++ b/apps/sveltekit-cloudflare-workers/package.json @@ -12,7 +12,7 @@ "preview": "vite preview", "prepare": "svelte-kit sync || echo ''", "check": "pnpm generate:types && svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", - "typecheck": "pnpm check", + "typecheck": "contrail lexicons check --no-atcute-config && pnpm check", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", "format": "prettier --write .", "lint": "prettier --check . && eslint .", @@ -58,8 +58,9 @@ }, "license": "MIT", "dependencies": { - "@atcute/jetstream": "^2.0.2", "@atmo-dev/contrail": "workspace:*", + "@atproto/lex": "^0.3.8", + "@bsky/jetstream": "^1.0.0", "@foxui/core": "^0.9.1", "@foxui/social": "^0.8.10", "@foxui/time": "^0.8.5", diff --git a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts index 05a42fc..4906ad0 100644 --- a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts +++ b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts @@ -9,9 +9,7 @@ const _mainSchema = /*#__PURE__*/ v.query( "output": { "type": "lex", "schema": /*#__PURE__*/ v.object({ - "date": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), - "seconds_ago": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), - "time_us": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), + "cursor": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), }), } } diff --git a/apps/sveltekit-cloudflare-workers/src/routes/+page.svelte b/apps/sveltekit-cloudflare-workers/src/routes/+page.svelte index 422fb87..ed85282 100644 --- a/apps/sveltekit-cloudflare-workers/src/routes/+page.svelte +++ b/apps/sveltekit-cloudflare-workers/src/routes/+page.svelte @@ -7,7 +7,7 @@ import { Button } from '@foxui/core'; import { GithubCorner, PopoverEmojiPicker } from '@foxui/social'; import { RelativeTime } from '@foxui/time'; - import { JetstreamSubscription } from '@atcute/jetstream'; + import { Jetstream } from '@bsky/jetstream'; import { createTID } from '$lib/atproto/methods'; import { putRecord } from '$lib/atproto/server/repo.remote'; @@ -51,34 +51,26 @@ let allStatuses = $derived([...localStatuses, ...liveStatuses, ...data.statuses]); - // Jetstream subscription + // Jetstream v2 live subscription onMount(() => { - const subscription = new JetstreamSubscription({ - url: 'wss://jetstream1.us-east.bsky.network', - wantedCollections: ['xyz.statusphere.status'] - }); - - const iterator = subscription[Symbol.asyncIterator](); + const controller = new AbortController(); + const jetstream = new Jetstream('https://jetstream.us-east.bsky.network'); (async () => { try { - while (true) { - const { value: event, done } = await iterator.next(); - if (done) break; - - if (event.kind !== 'commit') continue; - if (event.commit.operation !== 'create') continue; + for await (const event of jetstream.live({ + raw: true, + kinds: ['commit'], + collections: ['xyz.statusphere.status'], + signal: controller.signal + })) { + if (event.kind !== 'commit' || event.commit.operation !== 'create') continue; const { did } = event; const { rkey, record } = event.commit; - if ( - record === null || - typeof record !== 'object' || - typeof record.status !== 'string' || - typeof record.createdAt !== 'string' - ) { - continue; - } + if (record === null || typeof record !== 'object' || Array.isArray(record)) continue; + const value = record as Record; + if (typeof value.status !== 'string' || typeof value.createdAt !== 'string') continue; const key = `${did}-${rkey}`; if (seenKeys.has(key)) continue; @@ -87,7 +79,7 @@ await fetchProfile(did); liveStatuses = [ - { did, rkey, status: record.status, createdAt: record.createdAt }, + { did, rkey, status: value.status, createdAt: value.createdAt }, ...liveStatuses ]; } @@ -96,9 +88,7 @@ } })(); - return () => { - iterator.return!(); - }; + return () => controller.abort(); }); diff --git a/docs/01-getting-started.md b/docs/01-getting-started.md index fbdb23b..5d90863 100644 --- a/docs/01-getting-started.md +++ b/docs/01-getting-started.md @@ -1,6 +1,6 @@ # Get started locally -Run a local AppView for a public AT Protocol collection with Node.js 22.13 or newer. +Run a local AppView for a public AT Protocol collection with Node.js 22.15 or newer. Create a project with the starter config: diff --git a/docs/04-deploy-cloudflare.md b/docs/04-deploy-cloudflare.md index 5252755..496e039 100644 --- a/docs/04-deploy-cloudflare.md +++ b/docs/04-deploy-cloudflare.md @@ -22,7 +22,7 @@ const config: ContrailConfig = { namespace: "com.example", orderedSource: { source: "jetstream", - epoch: "my-appview-v1", + epoch: "my-appview-jetstream-v2", }, collections: { // ...your existing collections diff --git a/package.json b/package.json index 0252b84..3feede4 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,9 @@ "private": true, "version": "0.0.0", "type": "module", + "engines": { + "node": ">=22.15.0" + }, "scripts": { "build": "turbo run build --filter=./packages/*", "build:apps": "turbo run build", diff --git a/packages/contrail/README.md b/packages/contrail/README.md index e30e3dd..7f7278b 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -79,7 +79,9 @@ await contrail.runPersistent({ After a write to a user's PDS, `contrail.notify(uri)` can fetch the authoritative record immediately. Only an authoritative not-found response deletes local state; rate limits, server errors, timeouts, malformed responses, and network failures leave it unchanged. Authentication and abuse controls for the public HTTP operation remain under design. -Contrail stores source event time, repository revision, source cursor, CID, and local index time separately from record/application time. Scheduled collection drops exact Jetstream observations before admission, counts UTF-8 record bodies plus a fixed 512-byte metadata allowance, and retains the threshold-crossing candidate before stopping. Identity updates have an independent retained-count limit, coalesce by DID, and commit in database batches; excess global identity traffic is accounted but omitted so it cannot starve record progress. Source-filtered observations consume neither candidate nor byte limits, but their yielded cursors remain accounted. Durable actor scope comes from identities, relay discovery, and visible discoverable records, so a failed identity refresh cannot make a restarted isolate filter equal-cursor dependent siblings. Bounded exact-observation hashes at the current microsecond cursor let restarts replay one microsecond of overlap without skipping equal-cursor siblings or recounting earlier ones. Durable tombstones reject stale resurrection, and live Jetstream projection commits its exact accounted cursor in the same transaction. Scheduled mode requires one pinned Jetstream endpoint; use `runPersistent()` for an Atcute failover pool. Persistent ingestion retains its streaming batch lifecycle and does not inherit the scheduled count/byte defaults. A successful PDS `listRecords` page is a current authoritative observation, so it supersedes older durable state without a redundant version read; its version writes and page cursor still commit atomically. Tombstones are retained indefinitely; authoritative rebuild/retention tooling is planned separately. +Contrail stores source event time, repository revision, source cursor, CID, and local index time separately from record/application time. Live ingestion speaks Jetstream v2 and checkpoints its unique monotonic `seq`; the RFC 3339 event time remains display/source metadata and is never used as the resume coordinate. Scheduled collection drops exact observations before admission, counts UTF-8 record bodies plus a fixed 512-byte metadata allowance, and retains the threshold-crossing candidate before stopping. Identity updates have an independent retained-count limit, coalesce by DID, and commit in database batches; excess global identity traffic is accounted but omitted so it cannot starve record progress. Source-filtered observations consume neither candidate nor byte limits, but their yielded seqs remain accounted. Durable actor scope comes from identities, relay discovery, and visible discoverable records. Existing v1 timestamp cursors cross a bounded overlap bridge and atomically transition once to the v2 seq domain. If either a timestamp or seq cursor has already fallen outside the hosted live-retention window, ingestion fails rather than accepting the server's timestamp clamp and silently skipping a gap; rebuild/backfill from a fresh boundary until archive replay is added. Durable tombstones reject stale resurrection, and live projection commits its exact accounted seq in the same transaction. Both scheduled and persistent modes require one pinned Jetstream v2 service because seqs are instance-local. Persistent ingestion retains its streaming batch lifecycle and does not inherit the scheduled count/byte defaults. Historical acquisition still uses the existing PDS or optional Alluvium paths; Jetstream v2 archive replay is not used yet. A successful PDS `listRecords` page is a current authoritative observation, so it supersedes older durable state without a redundant version read; its version writes and page cursor still commit atomically. Tombstones are retained indefinitely; authoritative rebuild/retention tooling is planned separately. + +When upgrading an existing database, point `jetstreams` at one v2 service and change `orderedSource.epoch` to identify the new continuity history. The normalized service origin is durably bound before the timestamp-to-seq transition because seqs are instance-local; changing it later is a hard error. Initialization permits the ordered-source epoch change only while the durable cursor is still in the legacy timestamp domain; the first fully handled v2 event then installs the seq domain one-way. Once a database is in the seq domain, an epoch mismatch remains a hard error. An already-seq-domain database with no durable service binding is also rejected because its originating instance cannot be inferred safely. Projection winner selection is guarded again inside the write transaction. Overlapping cron, persistent, notify, or backfill writers cannot commit a stale canonical row, derived projection, tombstone, or source checkpoint; a changed predecessor rolls the complete attempt back and retries from fresh durable state. @@ -208,10 +210,11 @@ An optional experimental Alluvium adapter consumes existing version-1 collection ```bash contrail backfill --sqlite ./data/contrail.sqlite --alluvium \ --alluvium-epoch my-source-continuity-epoch \ + --alluvium-source-url wss://jetstream1.us-east.bsky.network \ --allow-partial ``` -The epoch is deliberately required because Alluvium protocol v1 does not publish one. `--allow-partial` is needed only when selected manifests explicitly report historical omissions. The command rejects a populated database unless it is resuming durable Alluvium bootstrap state. The equivalent programmatic API is: +The epoch is deliberately required because Alluvium protocol v1 does not publish one. `--alluvium-source-url` names the exact legacy v1 source advertised by the manifests and defaults to the hosted east source shown above; it is intentionally separate from `config.jetstreams`, which selects the v2 live service used after bootstrap. `--allow-partial` is needed only when selected manifests explicitly report historical omissions. The command rejects a populated database unless it is resuming durable Alluvium bootstrap state. The equivalent programmatic API is: ```ts import { DatabaseBootstrapTarget } from "@atmo-dev/contrail"; @@ -236,7 +239,7 @@ const source = createAlluviumBootstrapSources(config, { }); const target = new DatabaseBootstrapTarget(db, config, { deferDerivedProjections: true, - // Commit the archived time_us as the ordinary cron ingestion cursor. + // Commit the archived v1 time_us as the one-time v2 live bridge cursor. liveCursor: createAlluviumLiveCursor(sourceIdentity), }); @@ -247,7 +250,7 @@ await bootstrapFreshProjection({ }); ``` -The offline call loads only the base plus published archive tail. It does not replay direct Jetstream to a newly marked head. The target atomically persists Alluvium's final archived `time_us` as the normal live cursor, so subsequent bounded cron invocations resume there and catch up over as many scheduled windows as necessary. +The offline call loads only the base plus published archive tail. It does not replay direct Jetstream to a newly marked head. The target atomically persists Alluvium's final archived v1 `time_us`; the next v2 live observation transitions that bridge coordinate to the service's seq domain, after which bounded cron invocations resume by seq. The configured live v2 service and continuity epoch remain operator-owned because Alluvium protocol v1 cannot publish a v2 handoff coordinate. This first adapter is generation-only and deliberately strict: selected manifests must expose one shared base and archive boundary, source ID/URL must match configuration, known capture gaps fail, and partial historical coverage requires `allowPartial: true`. It buffers each bounded compressed object so checksum and byte length are verified before any rows are projected. Because version-1 manifests lack global multi-collection delivery steps, it also restores source order with a bounded 100,000-mutation tail buffer; larger tails require the planned bundle protocol. It does not silently fall back to PDS acquisition; omitted-account repair and dependent profile/follow enrichment remain separately accounted PDS work. @@ -290,7 +293,7 @@ const config = { The service description advertises the base service DID, exact fragmented audience, canonical OAuth scope, and each protected query/procedure separately from anonymous methods. `contrail connect` generates `src/contrail/index.ts`; anonymous calls go directly to the configured endpoint, while protected calls lazily discover and validate all service-auth fields. Its exported `contrail.scope` contains one sorted `lxm` parameter per protected method, such as `rpc?aud=did:web:api.example.com%23contrail&lxm=.getFeed&lxm=.notifyOfUpdate`. `contrail.authenticated(authenticatedClient)` returns one Atcute client: advertised provider methods route to Contrail, ordinary methods route to the PDS, and successful tracked `createRecord`, `putRecord`, and `deleteRecord` calls automatically notify Contrail. Handle-form deletes are resolved to canonical DID URIs. Protected methods use cached exact method-bound tokens, while transient discovery failures remain retryable. Feed actors must resolve to the token issuer, and every notified AT URI must belong to the issuer. When the base DID resolves to the public endpoint's `/.well-known/did.json`, the Worker publishes a DID document whose service entry ID is the exact audience. -Public-service mode requires a primary ordered source so `getCursor` can expose its committed position. Existing non-public deployments without one retain the legacy ingestion-time cursor response: +Public-service mode requires a primary ordered source so `getCursor` can expose its committed position. Configure that continuity identity explicitly; non-public deployments without it expose only the unscoped numeric live cursor as `{ cursor }`: ```ts const config = { @@ -302,7 +305,7 @@ const config = { }; ``` -The returned cursor is opaque. Compare the complete `{ source, epoch, cursor }` value for equality; never order cursors from different epochs. Consumers can read the position before and after a query, retry if it changed, then poll it as a refetch/invalidation signal. +The ordered-source cursor is opaque. Compare the complete `{ source, epoch, cursor }` value for equality; never order cursors from different epochs. Consumers can read the position before and after a query, retry if it changed, then poll it as a refetch/invalidation signal. Connect an independent consumer with `contrail connect `. The version-2 provider lock records the deployment and exact Lexicon bundle, but generated clients do not pin the provider's complete method set at runtime. Existing anonymous methods therefore continue working when a provider adds methods. A repeated connection to the same endpoint and provider-owned output root requires `--update`; provider files and the lock are staged and swapped without deleting consumer-owned Lexicons. Version-1 locks must be removed and reconnected. @@ -365,4 +368,4 @@ import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; ``` -The SQLite adapter uses the built-in `node:sqlite` module and therefore requires Node.js 22.13 or newer. D1 implements Contrail's database interface directly. +Contrail requires Node.js 22.15 or newer. The SQLite adapter uses the built-in `node:sqlite` module; D1 implements Contrail's database interface directly. diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 097a403..b02a1d6 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -4,6 +4,9 @@ "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, + "engines": { + "node": ">=22.15.0" + }, "files": [ "dist" ], @@ -91,6 +94,8 @@ "@atcute/tid": "1.1.4", "@atcute/uint8array": "1.1.5", "@atcute/xrpc-server": "^2.0.2", + "@atproto/lex": "^0.3.8", + "@bsky/jetstream": "^1.0.0", "cac": "^7.0.0", "hono": "^4.13.0", "jiti": "^2.7.0", diff --git a/packages/contrail/src/cli/commands/backfill.ts b/packages/contrail/src/cli/commands/backfill.ts index 24fac2a..1414b59 100644 --- a/packages/contrail/src/cli/commands/backfill.ts +++ b/packages/contrail/src/cli/commands/backfill.ts @@ -4,6 +4,7 @@ import { backfillAll, bootstrapAlluvium, bootstrapAlluviumDatabase, + DEFAULT_ALLUVIUM_SOURCE_URL, labelsBackfillAll, labelsBackfillDatabase, } from "../../workers/backfill.js"; @@ -26,6 +27,7 @@ interface BackfillOpts { alluvium?: boolean; alluviumEndpoint: string; alluviumSourceId: string; + alluviumSourceUrl: string; alluviumEpoch?: string; alluviumRetentionHours: number; allowPartial?: boolean; @@ -87,6 +89,11 @@ export function registerBackfill(cli: CAC): void { "Logical source ID expected in Alluvium manifests", { default: "jetstream-us-east" }, ) + .option( + "--alluvium-source-url ", + "Exact legacy Jetstream v1 URL advertised by Alluvium manifests", + { default: DEFAULT_ALLUVIUM_SOURCE_URL }, + ) .option( "--alluvium-epoch ", "Operator-owned continuity epoch (required with --alluvium)", @@ -142,6 +149,7 @@ export function registerBackfill(cli: CAC): void { const alluvium = { endpoint: options.alluviumEndpoint, sourceId: options.alluviumSourceId, + sourceUrl: options.alluviumSourceUrl, sourceEpoch: options.alluviumEpoch, allowPartial: options.allowPartial === true, retentionUs: retentionHours * 60 * 60 * 1_000_000, diff --git a/packages/contrail/src/cli/commands/connect.ts b/packages/contrail/src/cli/commands/connect.ts index 5f92304..2b93b20 100644 --- a/packages/contrail/src/cli/commands/connect.ts +++ b/packages/contrail/src/cli/commands/connect.ts @@ -498,7 +498,7 @@ export async function connectConfigSource(options: { notify: config.notify ?? true, orderedSource: config.orderedSource ?? { source: "jetstream", - epoch: "contrail-local-jetstream-v1", + epoch: "contrail-local-jetstream-v2", }, }; const endpoint = normalizePublicServiceEndpoint( diff --git a/packages/contrail/src/cli/commands/dev.ts b/packages/contrail/src/cli/commands/dev.ts index 1759c89..e312f31 100644 --- a/packages/contrail/src/cli/commands/dev.ts +++ b/packages/contrail/src/cli/commands/dev.ts @@ -14,7 +14,10 @@ import { getBackfillStatus } from "../../core/status.js"; import type { ContrailConfig, Database } from "../../core/types.js"; import { configProjectRoot } from "../../cli-config.js"; import { createHandler } from "../../server.js"; -import { bootstrapAlluviumDatabase } from "../../workers/backfill.js"; +import { + bootstrapAlluviumDatabase, + DEFAULT_ALLUVIUM_SOURCE_URL, +} from "../../workers/backfill.js"; import { defaultConsumerLexiconRoot, prepareDevLexicons, @@ -43,6 +46,7 @@ interface DevOpts { alluvium?: boolean; alluviumEndpoint: string; alluviumSourceId: string; + alluviumSourceUrl: string; alluviumEpoch?: string; alluviumRetentionHours: number; allowPartial?: boolean; @@ -79,7 +83,7 @@ function devConfig(config: ContrailConfig, options: DevOpts): ContrailConfig { source: options.alluviumSourceId, epoch: options.alluviumEpoch ?? "contrail-local-alluvium-v1", } - : { source: "jetstream", epoch: "contrail-local-jetstream-v1" }), + : { source: "jetstream", epoch: "contrail-local-jetstream-v2" }), }; } @@ -278,6 +282,7 @@ async function runSqliteDev( lexicons, endpoint: options.alluviumEndpoint, sourceId: ordered.source, + sourceUrl: options.alluviumSourceUrl, sourceEpoch: ordered.epoch, retentionUs: retentionHours * 60 * 60 * 1_000_000, allowPartial: options.allowPartial === true, @@ -406,6 +411,11 @@ export function registerDev(cli: CAC): void { .option("--alluvium-source-id ", "Alluvium manifest source ID", { default: "jetstream-us-east", }) + .option( + "--alluvium-source-url ", + "Exact legacy Jetstream v1 URL advertised by Alluvium manifests", + { default: DEFAULT_ALLUVIUM_SOURCE_URL }, + ) .option("--alluvium-epoch ", "Operator-owned continuity epoch") .option( "--alluvium-retention-hours ", diff --git a/packages/contrail/src/core/backfill.ts b/packages/contrail/src/core/backfill.ts index e84ad58..1c33bc4 100644 --- a/packages/contrail/src/core/backfill.ts +++ b/packages/contrail/src/core/backfill.ts @@ -35,10 +35,9 @@ const BACKFILL_RETRY_BASE_MS = 15 * 60_000; const BACKFILL_RETRY_MAX_MS = 48 * 60 * 60_000; const DEFAULT_SCHEDULED_MAX_ATTEMPTS = 10; const DERIVED_PROJECTIONS_DIRTY_KEY = "backfill_derived_projections_dirty"; -/** Legacy Jetstream v1 uses wall-clock microseconds as its replay coordinate. - * Keep the same overlap Atcute uses for multi-endpoint clock skew. The future - * source adapter replaces this compatibility marker with a source-owned epoch - * and cursor. */ +/** Historical acquisition still captures a wall-clock boundary. Jetstream v2 + * accepts it through its timestamp compatibility domain; the first delivered + * live event atomically replaces it with the service's seq cursor. */ const INITIAL_CAPTURE_OVERLAP_US = 10_000_000; async function ensureInitialReplayBoundary( diff --git a/packages/contrail/src/core/db/index.ts b/packages/contrail/src/core/db/index.ts index 78b38a2..71a85cf 100644 --- a/packages/contrail/src/core/db/index.ts +++ b/packages/contrail/src/core/db/index.ts @@ -1,6 +1,6 @@ export { initSchema, CONTRAIL_SCHEMA_VERSION } from "./schema"; export { getMeta, setMeta, getMetaNumber } from "./meta"; export { optimizeDatabase } from "./optimize"; -export { assertServingSourceCompatibility, getLastCursor, loadKnownActorDids, getServingSourcePosition, orderedSourcePosition, saveCursor, saveCursorStatement, getCursorObservations, saveCursorObservationStatements, saveOrderedSourcePositionStatement, saveServingSourcePositionStatement, lookupExistingRecords, queryRecords, pruneFeedItems, pruneActorFeed, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor } from "./records"; +export { assertJetstreamServiceCompatibility, assertServingSourceCompatibility, getLastCursor, loadKnownActorDids, getServingSourcePosition, orderedSourcePosition, saveCursor, saveCursorStatement, saveJetstreamCursor, saveJetstreamCursorStatement, getCursorObservations, saveCursorObservationStatements, saveJetstreamCursorObservationStatements, saveOrderedSourcePositionStatement, saveServingSourcePositionStatement, lookupExistingRecords, queryRecords, pruneFeedItems, pruneActorFeed, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor } from "./records"; export type { QueryOptions, SortOption, ExistingRecordInfo, FeedSweepResult, ServingSourcePosition } from "./records"; export type { RecordSource } from "../types"; diff --git a/packages/contrail/src/core/db/records.ts b/packages/contrail/src/core/db/records.ts index 1567b9a..f0a348c 100644 --- a/packages/contrail/src/core/db/records.ts +++ b/packages/contrail/src/core/db/records.ts @@ -1,3 +1,7 @@ +import { + JETSTREAM_V2_SEQ_THRESHOLD, + isJetstreamTimestampCursor, +} from "../jetstream-live"; import type { ContrailConfig, ResolvedContrailConfig, @@ -580,6 +584,48 @@ export function saveServingSourcePositionStatement( .bind(position.source, position.epoch, position.cursor, updatedAt); } +export const JETSTREAM_V2_SERVICE_META_KEY = "jetstream_v2_service"; + +/** Bind instance-local v2 seqs to one normalized service origin. A legacy + * timestamp bridge may acquire its binding before transition; an unbound seq + * cursor is rejected because its originating instance cannot be recovered. */ +export async function assertJetstreamServiceCompatibility( + db: Database, + service: string, +): Promise { + let existing = await db + .prepare("SELECT value FROM _contrail_meta WHERE key = ?") + .bind(JETSTREAM_V2_SERVICE_META_KEY) + .first<{ value: string }>(); + + if (!existing) { + const cursor = await getLastCursor(db); + if (cursor !== null && !isJetstreamTimestampCursor(cursor)) { + throw new Error( + `durable Jetstream v2 seq cursor ${cursor} has no pinned service identity; ` + + "rebuild or explicitly migrate the generation", + ); + } + await db + .prepare( + "INSERT INTO _contrail_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO NOTHING", + ) + .bind(JETSTREAM_V2_SERVICE_META_KEY, service) + .run(); + existing = await db + .prepare("SELECT value FROM _contrail_meta WHERE key = ?") + .bind(JETSTREAM_V2_SERVICE_META_KEY) + .first<{ value: string }>(); + } + + if (existing?.value !== service) { + throw new Error( + `configured Jetstream v2 service ${service} does not match durable service ` + + `${existing?.value ?? ""}; seq cursors are instance-local`, + ); + } +} + export async function assertServingSourceCompatibility( db: Database, orderedSource?: OrderedSourceConfig, @@ -602,6 +648,15 @@ export async function assertServingSourceCompatibility( existing.position.source !== orderedSource.source || existing.position.epoch !== orderedSource.epoch ) { + const sourcePositionCursor = Number(existing.position.cursor); + const liveCursor = await getLastCursor(db); + const pendingV2Transition = + existing.position.source === orderedSource.source && + Number.isSafeInteger(sourcePositionCursor) && + isJetstreamTimestampCursor(sourcePositionCursor) && + liveCursor !== null && + isJetstreamTimestampCursor(liveCursor); + if (pendingV2Transition) return; throw new Error( `configured ordered source ${orderedSource.source}/${orderedSource.epoch} ` + `does not match durable source position ` + @@ -679,20 +734,54 @@ export async function loadKnownActorDids( return known; } +/** Save a legacy timestamp cursor monotonically without allowing an old + * writer to move a database that has transitioned to the v2 seq domain back + * into the timestamp domain. */ export function saveCursorStatement( db: Database, timeUs: number, ): Statement { return db .prepare( - "INSERT INTO cursor (id, time_us) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET time_us = CASE WHEN excluded.time_us > cursor.time_us THEN excluded.time_us ELSE cursor.time_us END", + `INSERT INTO cursor (id, time_us) VALUES (1, ?) + ON CONFLICT(id) DO UPDATE SET time_us = CASE + WHEN cursor.time_us < ${JETSTREAM_V2_SEQ_THRESHOLD} + AND excluded.time_us >= ${JETSTREAM_V2_SEQ_THRESHOLD} + THEN cursor.time_us + WHEN excluded.time_us > cursor.time_us THEN excluded.time_us + ELSE cursor.time_us + END`, ) .bind(timeUs); } -/** Exact source observations already accounted for at the current coarse - * cursor. Scheduled Jetstream resumes one microsecond earlier and uses these - * hashes to avoid both skipping same-cursor siblings and recounting prior ones. */ +/** Save a Jetstream v2 cursor monotonically while permitting the one-way + * legacy timestamp -> v2 seq transition. A stale timestamp-domain writer can + * never move an already-transitioned seq cursor back into the old domain. */ +export function saveJetstreamCursorStatement( + db: Database, + cursor: number, +): Statement { + return db + .prepare( + `INSERT INTO cursor (id, time_us) VALUES (1, ?) + ON CONFLICT(id) DO UPDATE SET time_us = CASE + WHEN cursor.time_us >= ${JETSTREAM_V2_SEQ_THRESHOLD} + AND excluded.time_us < ${JETSTREAM_V2_SEQ_THRESHOLD} + THEN excluded.time_us + WHEN cursor.time_us < ${JETSTREAM_V2_SEQ_THRESHOLD} + AND excluded.time_us >= ${JETSTREAM_V2_SEQ_THRESHOLD} + THEN cursor.time_us + WHEN excluded.time_us > cursor.time_us THEN excluded.time_us + ELSE cursor.time_us + END`, + ) + .bind(cursor); +} + +/** Legacy timestamp-domain observation hashes. V2 seqs are unique and live + * ingestion clears these rows when it next commits; the reader remains for + * schema/API compatibility during the transition. */ export async function getCursorObservations( db: Database, timeUs: number, @@ -729,6 +818,28 @@ export function saveCursorObservationStatements( ]; } +/** V2 may move once from a large timestamp cursor to a smaller seq. Retain + * observations only at the exact resulting cursor so timestamp-domain rows do + * not survive that numeric decrease indefinitely. */ +export function saveJetstreamCursorObservationStatements( + db: Database, + cursor: number, + observations: Iterable, +): Statement[] { + return [ + db.prepare( + "DELETE FROM cursor_observations WHERE time_us <> (SELECT time_us FROM cursor WHERE id = 1)", + ), + ...[...new Set(observations)].map((observation) => + db + .prepare( + "INSERT INTO cursor_observations (time_us, observation) SELECT ?, ? WHERE (SELECT time_us FROM cursor WHERE id = 1) = ? ON CONFLICT(time_us, observation) DO NOTHING", + ) + .bind(cursor, observation, cursor), + ), + ]; +} + export async function saveCursor( db: Database, timeUs: number, @@ -745,6 +856,24 @@ export async function saveCursor( await db.batch(statements); } +export async function saveJetstreamCursor( + db: Database, + cursor: number, + orderedSource?: OrderedSourceConfig, + observations: Iterable = [], +): Promise { + const statements = [saveJetstreamCursorStatement(db, cursor)]; + if (orderedSource) { + statements.push( + saveOrderedSourcePositionStatement(db, orderedSource, cursor), + ); + } + statements.push( + ...saveJetstreamCursorObservationStatements(db, cursor, observations), + ); + await db.batch(statements); +} + // --- Existing record lookup --- export interface ExistingRecordInfo { diff --git a/packages/contrail/src/core/jetstream-live.ts b/packages/contrail/src/core/jetstream-live.ts new file mode 100644 index 0000000..13ae39e --- /dev/null +++ b/packages/contrail/src/core/jetstream-live.ts @@ -0,0 +1,330 @@ +import { + Jetstream, + websocketTransport, + type CollectionFilter, + type LiveOpts, + type LiveTransport, + type RawEvent, + type RawRecordJson, +} from "@bsky/jetstream"; + +/** Jetstream v2 reserves values at and above this boundary for legacy + * unix-microsecond timestamp cursors. Real v2 seq cursors stay below it. */ +export const JETSTREAM_V2_SEQ_THRESHOLD = 1_000_000_000_000_000; + +/** A timestamp bridge intentionally starts slightly before the prior v1 + * position. V1 and v2 are separate instances with separate witness clocks; + * ordinary source ordering makes the overlap idempotent. */ +export const JETSTREAM_V1_BRIDGE_OVERLAP_US = 10_000_000; + +export class JetstreamLiveHistoryExpiredError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "JetstreamLiveHistoryExpiredError"; + } +} + +export function isJetstreamTimestampCursor(cursor: number): boolean { + return cursor >= JETSTREAM_V2_SEQ_THRESHOLD; +} + +export function advanceJetstreamCursor( + current: number | null, + observedSeq: number, +): number { + if ( + !Number.isSafeInteger(observedSeq) || + observedSeq < 0 || + observedSeq >= JETSTREAM_V2_SEQ_THRESHOLD + ) { + throw new TypeError("Jetstream seq must be in the v2 sequence domain"); + } + if (current === null || isJetstreamTimestampCursor(current)) { + return observedSeq; + } + return Math.max(current, observedSeq); +} + +/** Preserve all six microsecond digits from Jetstream's RFC 3339 display time; + * Date.parse alone truncates the final three digits. */ +export function jetstreamDatetimeToMicroseconds(value: string): number { + const match = /^(.*?)(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})$/.exec(value); + if (!match) throw new TypeError(`Invalid Jetstream datetime: ${value}`); + const wholeMilliseconds = Date.parse(`${match[1]}${match[3]}`); + if (!Number.isFinite(wholeMilliseconds)) { + throw new TypeError(`Invalid Jetstream datetime: ${value}`); + } + const fractionUs = Number((match[2] ?? "").slice(0, 6).padEnd(6, "0")); + const result = wholeMilliseconds * 1_000 + fractionUs; + if (!Number.isSafeInteger(result)) { + throw new TypeError(`Jetstream datetime is outside the safe range: ${value}`); + } + return result; +} + +export type JetstreamLiveEvent = + | { + kind: "commit"; + did: string; + seq: number; + time_us: number; + commit: + | { + operation: "create" | "update"; + rev: string; + collection: string; + rkey: string; + cid: string; + record: unknown; + } + | { + operation: "delete"; + rev: string; + collection: string; + rkey: string; + }; + } + | { + kind: "identity"; + did: string; + seq: number; + time_us: number; + identity: { did: string; handle?: string; time?: string }; + } + | { + kind: "account"; + did: string; + seq: number; + time_us: number; + account: { + did: string; + active: boolean; + status?: string; + time?: string; + }; + } + | { + kind: "sync"; + did: string; + seq: number; + time_us: number; + sync: { did: string; rev: string; time?: string }; + }; + +const SUBSCRIBE_EVENTS_NSID = "network.bsky.jetstream.subscribeEvents"; + +/** A normal HTTP request reaches the same pre-upgrade validation as a + * WebSocket dial. Hosted v2 services return 426 for an accepted cursor and a + * structured 400 CursorTooOld before checking Upgrade. Unlike browser + * WebSockets, fetch exposes that status and body in Workers and browsers. */ +async function assertSeqCursorAvailable(options: { + service: string; + cursor: number; + fetchImpl: typeof fetch; + signal?: AbortSignal; +}): Promise { + const service = new URL(options.service); + if (service.protocol === "ws:") service.protocol = "http:"; + if (service.protocol === "wss:") service.protocol = "https:"; + const url = new URL(`/xrpc/${SUBSCRIBE_EVENTS_NSID}`, service.origin); + url.searchParams.set("cursor", String(options.cursor)); + // Keep this probe independent of a deployment's collection count. Cursor + // validation is source-global; one valid kind is enough to reach it. + url.searchParams.set("kinds", "commit"); + + const response = await options.fetchImpl(url, { + cache: "no-store", + redirect: "error", + signal: options.signal, + }); + const body = (await response.text()).slice(0, 2_000); + let errorName: unknown; + let message: unknown; + try { + const parsed = JSON.parse(body) as { error?: unknown; message?: unknown }; + errorName = parsed.error; + message = parsed.message; + } catch { + // Valid cursors return a plain-text 426 because this probe is deliberately + // not a WebSocket upgrade. + } + + if ( + response.status === 400 && + (errorName === "CursorTooOld" || /cursor too old/i.test(body)) + ) { + throw new JetstreamLiveHistoryExpiredError( + `Jetstream seq cursor is older than the live retention window${typeof message === "string" ? `: ${message}` : ""}`, + ); + } + if (response.status !== 426) { + throw new Error( + `Jetstream live cursor preflight failed (${response.status})${body ? `: ${body}` : ""}`, + ); + } +} + +function normalizeLiveEvent( + event: RawEvent, +): JetstreamLiveEvent { + const base = { + did: event.did, + seq: event.seq, + time_us: jetstreamDatetimeToMicroseconds(event.time), + }; + if (event.kind === "commit") { + const commit = event.commit; + return commit.operation === "delete" + ? { + ...base, + kind: "commit", + commit: { + operation: "delete", + rev: commit.rev, + collection: commit.collection, + rkey: commit.rkey, + }, + } + : { + ...base, + kind: "commit", + commit: { + operation: commit.operation, + rev: commit.rev, + collection: commit.collection, + rkey: commit.rkey, + cid: commit.cid, + record: commit.record, + }, + }; + } + if (event.kind === "identity") { + return { ...base, kind: "identity", identity: event.identity }; + } + if (event.kind === "account") { + return { ...base, kind: "account", account: event.account }; + } + return { ...base, kind: "sync", sync: event.sync }; +} + +export interface JetstreamLiveSubscriptionOptions { + /** One pinned Jetstream v2 service. Seq cursors are instance-local. */ + url: string; + cursor?: number; + wantedCollections?: string[]; + signal?: AbortSignal; + /** Override fetch or the live transport for tests/custom runtimes. */ + fetchImpl?: typeof fetch; + liveTransport?: LiveTransport; + onConnectionOpen?: () => void; + onConnectionClose?: (event: { code?: number; reason?: string }) => void; + onConnectionError?: (event: { error: unknown }) => void; + onInfo?: (info: { name: string; message?: string }) => void; +} + +/** Thin live-only adapter over the official v2 client. It deliberately does not + * use snapshot/replay yet; Contrail's existing PDS/Alluvium paths still own + * historical acquisition. */ +export class JetstreamLiveSubscription + implements AsyncIterable +{ + readonly #options: JetstreamLiveSubscriptionOptions; + readonly #wireCursor: number; + cursor: number; + + constructor(options: JetstreamLiveSubscriptionOptions) { + this.#options = options; + const initial = + options.cursor ?? + Math.max(0, Date.now() * 1_000 - JETSTREAM_V1_BRIDGE_OVERLAP_US); + if (!Number.isSafeInteger(initial) || initial < 0) { + throw new TypeError("Jetstream cursor must be a non-negative safe integer"); + } + this.cursor = initial; + this.#wireCursor = isJetstreamTimestampCursor(initial) + ? Math.max(0, initial - JETSTREAM_V1_BRIDGE_OVERLAP_US) + : initial; + } + + async *[Symbol.asyncIterator](): AsyncIterator { + const options = this.#options; + const client = new Jetstream(options.url); + const cursor = { load: async () => this.#wireCursor, async save() {} }; + const transport = + options.liveTransport ?? + websocketTransport({ + // Browser WebSockets hide HTTP handshake status. Let the enclosing + // ingestion loop reconnect from its durable cursor instead of allowing + // ws-client to retry an unclassifiable stale cursor forever. + shouldReconnect: false, + onConnect: () => options.onConnectionOpen?.(), + onClose: (detail) => + options.onConnectionClose?.({ + code: detail.code, + reason: detail.reason, + }), + onError: (error) => options.onConnectionError?.({ error }), + }); + + const liveOptions: LiveOpts & { raw: true } = { + raw: true, + cursor, + collections: options.wantedCollections as CollectionFilter[] | undefined, + // Preserve the legacy live projection surface for now. Account and sync + // semantics remain a separate Contrail feature rather than being silently + // claimed by this transport migration. + kinds: ["commit", "identity"], + signal: options.signal, + liveTransport: transport, + onError: (error) => options.onConnectionError?.({ error }), + onInfo: (info) => { + if (info.name === "OutdatedCursor") { + throw new JetstreamLiveHistoryExpiredError( + `Jetstream timestamp cursor is older than the live retention window${info.message ? `: ${info.message}` : ""}`, + ); + } + options.onInfo?.(info); + }, + }; + let preflightComplete = false; + try { + if (!isJetstreamTimestampCursor(this.#wireCursor)) { + await assertSeqCursorAvailable({ + service: options.url, + cursor: this.#wireCursor, + fetchImpl: options.fetchImpl ?? fetch, + signal: options.signal, + }); + } + preflightComplete = true; + for await (const event of client.live(liveOptions)) { + this.cursor = advanceJetstreamCursor(this.cursor, event.seq); + yield normalizeLiveEvent(event); + } + } catch (error) { + if (error instanceof JetstreamLiveHistoryExpiredError) throw error; + // Close the race where a cursor crosses the retention floor between the + // initial HTTP probe and a browser WebSocket handshake. A second probe + // can promote the otherwise opaque SocketError to the fatal typed error. + if ( + preflightComplete && + !isJetstreamTimestampCursor(this.cursor) && + !options.signal?.aborted + ) { + try { + await assertSeqCursorAvailable({ + service: options.url, + cursor: this.cursor, + fetchImpl: options.fetchImpl ?? fetch, + signal: options.signal, + }); + } catch (probeError) { + if (probeError instanceof JetstreamLiveHistoryExpiredError) { + throw probeError; + } + } + } + throw error; + } + } +} diff --git a/packages/contrail/src/core/jetstream-source.ts b/packages/contrail/src/core/jetstream-source.ts index 3fbadda..30ff0e0 100644 --- a/packages/contrail/src/core/jetstream-source.ts +++ b/packages/contrail/src/core/jetstream-source.ts @@ -141,9 +141,11 @@ function mutationFromEvent( }; } -/** Ordered replay adapter for Jetstream's microsecond cursor protocol. +/** Legacy timestamp-coordinate adapter retained for existing PDS/Alluvium + * bootstrap orchestration. Runtime scheduled and persistent ingestion use the + * v2 seq adapter in `jetstream-live.ts`; archive-backed v2 bootstrap is deferred. * - * Jetstream has no separate head endpoint. Marks therefore wait for a real + * The legacy stream has no separate head endpoint. Marks therefore wait for a real * event from the same filtered stream, and replay waits for an event strictly * beyond that mark. Quiet application collections can add a busy watermark * collection without projecting its records. This avoids claiming catch-up diff --git a/packages/contrail/src/core/jetstream.ts b/packages/contrail/src/core/jetstream.ts index d0f3c07..6b22083 100644 --- a/packages/contrail/src/core/jetstream.ts +++ b/packages/contrail/src/core/jetstream.ts @@ -1,17 +1,21 @@ -import { JetstreamSubscription } from "@atcute/jetstream"; +import { + JetstreamLiveSubscription, + advanceJetstreamCursor, + isJetstreamTimestampCursor, +} from "./jetstream-live"; import type { ContrailConfig, IngestEvent, Database, Logger } from "./types"; import { DEFAULT_JETSTREAMS, getCollectionNsids, getDependentNsids, - jetstreamUrlOption, + jetstreamService, buildFeedTargetCaps, getFeedMutatingNsids, optimizeEnabled, optimizeIntervalMs, optimizeAnalysisLimit, } from "./types"; -import { initSchema, getLastCursor, loadKnownActorDids, saveCursor, saveCursorStatement, getCursorObservations, saveCursorObservationStatements, saveOrderedSourcePositionStatement, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor, getMetaNumber, setMeta, optimizeDatabase } from "./db"; +import { assertJetstreamServiceCompatibility, assertServingSourceCompatibility, initSchema, getLastCursor, loadKnownActorDids, saveJetstreamCursor, saveJetstreamCursorStatement, saveJetstreamCursorObservationStatements, saveOrderedSourcePositionStatement, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor, getMetaNumber, setMeta, optimizeDatabase } from "./db"; import { createIngestEvent, ingestRecords, @@ -292,8 +296,8 @@ function nextWithDeadline( ): Promise | typeof INGEST_TIMEOUT> { let timer: ReturnType; const next = iterator.next(); - // The transport currently has no structural cancellation seam. If the timer - // wins, swallow a later rejection while the iterator is closed best-effort. + // If the timer wins, cancellation is triggered by ingestEvents' cleanup. + // Swallow that later abort rejection because this race has already settled. next.catch(() => {}); const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve(INGEST_TIMEOUT), ms); @@ -339,26 +343,6 @@ function normalizedJson(value: unknown): string { return `{${fields.join(",")}}`; } -async function observationHash(value: string): Promise { - const subtle = ( - globalThis as typeof globalThis & { - crypto?: { - subtle?: { - digest( - algorithm: string, - data: Uint8Array, - ): Promise; - }; - }; - } - ).crypto?.subtle; - if (!subtle) throw new Error("Web Crypto SHA-256 is required for ingestion"); - const digest = await subtle.digest("SHA-256", utf8.encode(value)); - return [...new Uint8Array(digest)] - .map((byte) => byte.toString(16).padStart(2, "0")) - .join(""); -} - interface JetstreamCommitObservation { /** One logical Jetstream source slot, before source ID/epoch prefixing. */ key: string; @@ -372,6 +356,7 @@ interface JetstreamCommitObservation { function jetstreamCommitObservation( event: { did: string; + seq: number; time_us: number; commit: { rev?: string; @@ -386,8 +371,10 @@ function jetstreamCommitObservation( const { commit } = event; const uri = `at://${event.did}/${commit.collection}/${commit.rkey}`; return { - key: JSON.stringify([event.time_us, uri, commit.rev ?? null]), + key: String(event.seq), fingerprint: JSON.stringify([ + uri, + commit.rev ?? null, commit.operation, commit.cid ?? null, commit.operation === "delete" ? null : normalizedJson(commit.record), @@ -402,7 +389,6 @@ export async function ingestEvents( budgetInput: ScheduledIngestOptions | ScheduledIngestBudget | number = DEFAULT_SCHEDULED_INGEST_BUDGET, knownDids?: Set, - startingCursorObservations: ReadonlySet = new Set(), ): Promise<{ events: IngestEvent[]; lastCursor: number | null; @@ -411,7 +397,6 @@ export async function ingestEvents( stats: ScheduledIngestCollectionStats; }> { const budget = resolveScheduledIngestBudget(budgetInput); - const startTimeUs = Date.now() * 1000; const deadline = Date.now() + budget.maxDrainMs; const collected: IngestEvent[] = []; @@ -448,19 +433,13 @@ export async function ingestEvents( diagnosticSamplesOmitted: 0, }; - // A durable timestamp cursor is resumed one microsecond earlier. This works - // with inclusive and exclusive timestamp APIs: all observations at the - // coarse cursor are replayed, while persisted hashes suppress only the exact - // items already accounted for there. It also protects a captured empty-start - // cursor from a later item that happens to share its microsecond. - const requestedCursor = - cursor === null ? null : Math.max(0, cursor - 1); - const subscription = new JetstreamSubscription({ - // A single-instance config is handed over as a string so @atcute skips its - // array-only first-connect cursor rollback (see jetstreamUrlOption). - url: jetstreamUrlOption(urls), + const requestedCursor = cursor; + const subscriptionAbort = new AbortController(); + const subscription = new JetstreamLiveSubscription({ + url: jetstreamService(urls), wantedCollections: collections, ...(requestedCursor !== null ? { cursor: requestedCursor } : {}), + signal: subscriptionAbort.signal, onConnectionOpen() { stats.connections++; }, @@ -473,65 +452,38 @@ export async function ingestEvents( }, }); - // Capture Atcute's constructor cursor before iteration can buffer frames and - // move it ahead. With no durable cursor this is the subscription's effective - // lower bound and must be persisted even when the first drain stays empty. + // Capture the adapter's effective lower bound before iteration can buffer + // frames and move it ahead. With no durable cursor this is a timestamp-domain + // bridge and must be persisted even when the first drain stays empty. const effectiveStartCursor = cursor ?? subscription.cursor ?? null; stats.safeEndingCursor = effectiveStartCursor; - let boundaryCursor = effectiveStartCursor; - let boundaryObservations = new Set(startingCursorObservations); - // Only hashes newly accounted for this cycle are returned for insertion. The - // database unions them with existing boundary rows, keeping each DB batch - // bounded even if many cycles share one coarse cursor. - let cursorObservations = new Set(); - const singleEndpoint = urls.length === 1; + // V2 seqs identify individual events, so there are no equal-cursor siblings + // to retain across cycles. Returning an empty set also retires any legacy + // timestamp-domain observation hashes when the cursor is next committed. + const cursorObservations = new Set(); const iterator = subscription[Symbol.asyncIterator](); type Ev = typeof subscription extends AsyncIterable ? V : never; - const accountSourceItem = async ( - event: Ev, - identity: string, - ): Promise => { - // The scheduled path requires one pinned endpoint. Its one-microsecond - // boundary overlap is known-complete below the durable cursor and must not - // consume candidate/byte budgets on every restart. + const accountSourceItem = (event: Ev): boolean => { + // The v2 wire is inclusive. The official client normally removes this + // overlap itself; keep the boundary guard so custom/test transports cannot + // make the already-committed seq consume a scheduled work budget. if ( - singleEndpoint && effectiveStartCursor !== null && - event.time_us < effectiveStartCursor + !isJetstreamTimestampCursor(effectiveStartCursor) && + event.seq <= effectiveStartCursor ) { stats.resumeOverlapDropped++; return true; } - const hash = await observationHash( - JSON.stringify([sourceId, sourceEpoch, identity]), - ); - if (boundaryCursor === null || event.time_us > boundaryCursor) { - boundaryCursor = event.time_us; - boundaryObservations = new Set([hash]); - cursorObservations = new Set([hash]); - } else if (event.time_us === boundaryCursor) { - const alreadyAccounted = boundaryObservations.has(hash); - boundaryObservations.add(hash); - if (!alreadyAccounted) cursorObservations.add(hash); - } - - if ( - effectiveStartCursor !== null && - event.time_us === effectiveStartCursor && - startingCursorObservations.has(hash) - ) { - stats.cursorBoundaryDuplicatesDropped++; - return true; - } return false; }; - // Fully account for one yielded item before the loop considers any stop - // threshold. Exact boundary observations are suppressed before evolving - // source-scope policy; ordinary cheap filters still precede cycle dedupe. + // Fully account for one yielded seq before the loop considers any stop + // threshold. The resume overlap is suppressed before evolving source-scope + // policy; ordinary cheap filters still precede cycle-local dedupe. const handleEvent = async (event: Ev): Promise => { if (event.kind === "commit") { const { commit } = event; @@ -542,16 +494,7 @@ export async function ingestEvents( sourceEpoch, observation.key, ]); - if ( - await accountSourceItem( - event, - JSON.stringify([ - "commit", - observation.key, - observation.fingerprint, - ]), - ) - ) { + if (accountSourceItem(event)) { return; } @@ -580,7 +523,7 @@ export async function ingestEvents( stats.sourceInconsistencies++; addDiagnosticSample( stats, - `source slot changed payload: ${observation.uri} time_us=${event.time_us}`, + `source slot changed payload: ${observation.uri} seq=${event.seq}`, ); fingerprints.add(observation.fingerprint); } else { @@ -608,7 +551,7 @@ export async function ingestEvents( ...(sourceEpoch === null ? {} : { epoch: sourceEpoch }), time_us: event.time_us, revision: commit.rev, - cursor: String(event.time_us), + cursor: String(event.seq), }, }); collected.push(candidate); @@ -618,7 +561,8 @@ export async function ingestEvents( (candidate.record === null ? 0 : utf8.encode(candidate.record).byteLength); } else if (event.kind === "identity") { stats.identityObservations++; - if (await accountSourceItem(event, normalizedJson(event))) return; + if (accountSourceItem(event)) return; + if (event.identity.handle === undefined) return; if ( identityUpdates.has(event.did) || identityUpdates.size < budget.maxIdentityUpdates @@ -631,75 +575,68 @@ export async function ingestEvents( stats.identityUpdatesOmitted++; } } else { - await accountSourceItem(event, normalizedJson(event)); + accountSourceItem(event); } }; - for (;;) { - // Check the deadline before requesting another item. A hot iterator must not - // get one extra next() after any threshold has been reached. - const remainingMs = deadline - Date.now(); - if (remainingMs <= 0) { - stats.stopReason = "drain-time"; - break; - } - - const step = await nextWithDeadline(iterator, remainingMs); - if (step === INGEST_TIMEOUT) { - stats.stopReason = "drain-time"; - break; - } - if (step.done) { - stats.stopReason = "idle"; - break; - } - const event = step.value; - stats.observedSourceItems++; + try { + for (;;) { + // Check the deadline before requesting another item. A hot iterator must + // not get one extra next() after any threshold has been reached. + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + stats.stopReason = "drain-time"; + break; + } - await handleEvent(event); - // handleEvent either retained, filtered, deduplicated, or deliberately - // handled the item. Only now is its cursor safe to checkpoint. Keep the - // representable checkpoint monotonic across deliberate overlap replay. - stats.lastAccountedCursor = Math.max( - stats.lastAccountedCursor ?? event.time_us, - event.time_us, - ); + const step = await nextWithDeadline(iterator, remainingMs); + if (step === INGEST_TIMEOUT) { + stats.stopReason = "drain-time"; + break; + } + if (step.done) { + stats.stopReason = "idle"; + break; + } + const event = step.value; + stats.observedSourceItems++; + + await handleEvent(event); + // handleEvent either retained, filtered, deduplicated, or deliberately + // handled the item. Only now is its cursor safe to checkpoint. Keep the + // representable checkpoint monotonic across deliberate overlap replay. + stats.lastAccountedCursor = advanceJetstreamCursor( + stats.lastAccountedCursor, + event.seq, + ); - if (stats.retainedCandidates >= budget.maxCandidates) { - stats.stopReason = "count"; - break; - } - if (stats.serializedCandidateBytes >= budget.maxSerializedBytes) { - stats.stopReason = "bytes"; - break; - } - if (event.time_us >= startTimeUs) { - stats.stopReason = "head"; - break; + if (stats.retainedCandidates >= budget.maxCandidates) { + stats.stopReason = "count"; + break; + } + if (stats.serializedCandidateBytes >= budget.maxSerializedBytes) { + stats.stopReason = "bytes"; + break; + } } + } finally { + // Abort first so a pending async-generator pull and the transport's + // reconnect loop settle before return() is queued behind that pull. + subscriptionAbort.abort(); + Promise.resolve(iterator.return?.()).catch(() => {}); } - // Close the subscription's socket, but fire-and-forget: awaiting the - // iterator's return on a quiet stream could itself block (the hang we fix). - Promise.resolve(iterator.return?.()).catch(() => {}); - - // Never read Atcute's cursor again here: it may have moved when a frame was - // buffered but not yielded. The constructor cursor captured above and the - // maximum fully-accounted yielded cursor are the only safe positions. + // Never read the transport cursor again here: it may have moved when a frame + // was buffered but not yielded. The constructor cursor captured above and the + // maximum fully-accounted yielded seq are the only safe positions. const lastCursor = - effectiveStartCursor === null - ? stats.lastAccountedCursor - : Math.max( + stats.lastAccountedCursor === null + ? effectiveStartCursor + : advanceJetstreamCursor( effectiveStartCursor, - stats.lastAccountedCursor ?? effectiveStartCursor, + stats.lastAccountedCursor, ); stats.safeEndingCursor = lastCursor; - if (lastCursor !== boundaryCursor) { - // This occurs only for an older deliberate overlap. Leave the durable - // boundary unchanged rather than attaching old hashes to a newer cursor. - cursorObservations = new Set(); - } - return { events: collected, lastCursor, @@ -767,12 +704,7 @@ export async function runIngestCycle( const log = getLogger(config); const budget = resolveScheduledIngestBudget(budgetInput); const scheduledUrls = config.jetstreams ?? DEFAULT_JETSTREAMS; - if (scheduledUrls.length !== 1) { - throw new TypeError( - "scheduled ingestion requires exactly one pinned Jetstream endpoint; " + - "use runPersistent() for a failover pool", - ); - } + const service = jetstreamService(scheduledUrls); const wallStartedAt = Date.now(); const finishCpuUsage = runtimeCpuUsage(); const s = state ?? createIngestState(); @@ -781,10 +713,10 @@ export async function runIngestCycle( await initSchema(db, config); s.schemaInitialized = true; } + await assertServingSourceCompatibility(db, config.orderedSource); + await assertJetstreamServiceCompatibility(db, service); const cursor = await getLastCursor(db); - const startingCursorObservations = - cursor === null ? new Set() : await getCursorObservations(db, cursor); // Load known DIDs for filtering dependent collections const dependentCollections = getDependentNsids(config); @@ -810,7 +742,6 @@ export async function runIngestCycle( cursor, budget, knownDids, - startingCursorObservations, ); const accepted: IngestEvent[] = []; @@ -834,7 +765,7 @@ export async function runIngestCycle( trailingStatements: isFinalBatch && lastCursor !== null ? [ - saveCursorStatement(db, lastCursor), + saveJetstreamCursorStatement(db, lastCursor), ...(config.orderedSource ? [ saveOrderedSourcePositionStatement( @@ -844,7 +775,7 @@ export async function runIngestCycle( ), ] : []), - ...saveCursorObservationStatements( + ...saveJetstreamCursorObservationStatements( db, lastCursor, cursorObservations, @@ -904,7 +835,7 @@ export async function runIngestCycle( // empty first drain so the next cron cannot silently start later. (stats.lastAccountedCursor !== null || cursor === null) ) { - await saveCursor( + await saveJetstreamCursor( db, lastCursor, config.orderedSource, diff --git a/packages/contrail/src/core/persistent.ts b/packages/contrail/src/core/persistent.ts index 6b19d96..6002848 100644 --- a/packages/contrail/src/core/persistent.ts +++ b/packages/contrail/src/core/persistent.ts @@ -1,14 +1,18 @@ -import type { JetstreamSubscription } from "@atcute/jetstream"; +import { + JetstreamLiveHistoryExpiredError, + JetstreamLiveSubscription, + type JetstreamLiveEvent, +} from "./jetstream-live"; import type { ContrailConfig, IngestEvent, Database, Logger, ResolvedContrailConfig } from "./types"; import { getCollectionNsids, getDependentNsids, buildFeedTargetCaps, getFeedMutatingNsids, - jetstreamUrlOption, + jetstreamService, resolveConfig, } from "./types"; -import { initSchema, getLastCursor, loadKnownActorDids, saveCursorStatement, saveOrderedSourcePositionStatement } from "./db"; +import { assertJetstreamServiceCompatibility, assertServingSourceCompatibility, initSchema, getLastCursor, loadKnownActorDids, saveJetstreamCursorStatement, saveJetstreamCursorObservationStatements, saveOrderedSourcePositionStatement } from "./db"; import { createIngestEvent, ingestRecords, recordTimeUs } from "./ingest"; import { refreshStaleIdentities, applyIdentityEvent } from "./identity"; import { backfillFollowersFromConstellation } from "./constellation"; @@ -25,12 +29,21 @@ import type { IngestState } from "./jetstream"; * the recycling cron isolate). */ const FEED_SWEEP_INTERVAL_MS = 10_000; +export interface PersistentJetstreamSubscription + extends AsyncIterable { + /** Effective initial source coordinate, when the transport exposes one. */ + cursor?: number | null; +} + export interface PersistentIngestOptions { batchSize?: number; flushIntervalMs?: number; signal?: AbortSignal; - /** Override subscription creation for testing */ - createSubscription?: (cursor: number | null) => JetstreamSubscription; + /** Override subscription creation for testing or a custom transport. */ + createSubscription?: ( + cursor: number | null, + signal?: AbortSignal, + ) => PersistentJetstreamSubscription; logger?: Logger; } @@ -55,12 +68,15 @@ export async function runPersistent( const flushIntervalMs = options?.flushIntervalMs ?? 5_000; const signal = options?.signal; const state = createIngestState(); + const service = jetstreamService(config.jetstreams ?? []); // Init schema once if (!state.schemaInitialized) { await initSchema(db, config); state.schemaInitialized = true; } + await assertServingSourceCompatibility(db, config.orderedSource); + await assertJetstreamServiceCompatibility(db, service); // Load known DIDs for dependent collection filtering const dependentCollections: Set = new Set(getDependentNsids(config)); @@ -88,11 +104,13 @@ export async function runPersistent( newlyKnownDids: new Set(), state, log, + service, createSubscription: options?.createSubscription, }); reconnectAttempts = 0; } catch (err) { if (signal?.aborted) break; + if (err instanceof JetstreamLiveHistoryExpiredError) throw err; log.error(`Jetstream connection error: ${err}`); const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), 30_000); reconnectAttempts++; @@ -108,6 +126,7 @@ interface StreamOptions { batchSize: number; flushIntervalMs: number; signal?: AbortSignal; + service: string; collections: string[]; knownDids?: Set; /** DIDs that crossed from unknown→known during this stream's lifetime. @@ -115,7 +134,7 @@ interface StreamOptions { newlyKnownDids?: Set; state: IngestState; log: Logger; - createSubscription?: (cursor: number | null) => any; + createSubscription?: PersistentIngestOptions["createSubscription"]; } async function streamAndFlush( @@ -127,19 +146,47 @@ async function streamAndFlush( const { batchSize, flushIntervalMs, signal, collections, knownDids, state, log } = opts; const subscription = opts.createSubscription - ? opts.createSubscription(cursor) - : new (await import("@atcute/jetstream")).JetstreamSubscription({ - // Single-instance config → string, so @atcute skips its array-only - // first-connect cursor rollback (see jetstreamUrlOption). - url: jetstreamUrlOption(config.jetstreams ?? []), + ? opts.createSubscription(cursor, signal) + : new JetstreamLiveSubscription({ + url: opts.service, wantedCollections: collections, ...(cursor !== null ? { cursor } : {}), - onConnectionOpen() { log.log("Connected to Jetstream"); }, - onConnectionClose(event: any) { log.log(`Disconnected: ${event.code} ${event.reason}`); }, - onConnectionError(event: any) { log.error("Jetstream error:", event.error); }, + signal, + onConnectionOpen() { log.log("Connected to Jetstream v2"); }, + onConnectionClose(event) { log.log(`Disconnected: ${event.code ?? ""} ${event.reason ?? ""}`); }, + onConnectionError(event) { log.error("Jetstream v2 error:", event.error); }, }); + // A fresh v2 subscription begins from a timestamp-domain overlap bridge. + // Persist that exact lower bound before the first pull so failed connection + // attempts cannot create progressively newer bridges and skip the outage. + const initialCursor = cursor === null ? subscription.cursor : null; + if (initialCursor !== null && initialCursor !== undefined) { + if (!Number.isSafeInteger(initialCursor) || initialCursor < 0) { + throw new TypeError( + "persistent Jetstream subscription cursor must be a non-negative safe integer", + ); + } + await db.batch([ + saveJetstreamCursorStatement(db, initialCursor), + ...(config.orderedSource + ? [ + saveOrderedSourcePositionStatement( + db, + config.orderedSource, + initialCursor, + ), + ] + : []), + ...saveJetstreamCursorObservationStatements(db, initialCursor, []), + ]); + } + const buffer: IngestEvent[] = []; + // Highest fully handled v2 seq not yet committed. This includes identity + // events, so a commit-quiet stream still advances and does not replay global + // identity traffic forever after restart. + let pendingCursor: number | null = null; // Guards against overlap between the periodic timer flush and a main-loop // batchSize-driven flush. The main loop only ever awaits flush() sequentially, // but the setInterval callback is a second entry point on another tick. @@ -150,10 +197,13 @@ async function streamAndFlush( flushing = true; try { + const checkpoint = pendingCursor; + if (checkpoint !== null) pendingCursor = null; if (buffer.length > 0) { const batch = buffer.splice(0); - const lastTimeUs = Math.max( - ...batch.map((event) => event.source?.time_us ?? event.time_us), + const lastCursor = Math.max( + checkpoint ?? 0, + ...batch.map((event) => Number(event.source?.cursor)), ); let ingestResult: Awaited>; try { @@ -161,22 +211,24 @@ async function streamAndFlush( phase: "live", knownDids, trailingStatements: [ - saveCursorStatement(db, lastTimeUs), + saveJetstreamCursorStatement(db, lastCursor), ...(config.orderedSource ? [ saveOrderedSourcePositionStatement( db, config.orderedSource, - lastTimeUs, + lastCursor, ), ] : []), + ...saveJetstreamCursorObservationStatements(db, lastCursor, []), ], }); } catch (error) { // The cursor transaction failed, so keep this exact batch at the front // for the timer's next attempt rather than losing it in memory. buffer.unshift(...batch); + pendingCursor = Math.max(pendingCursor ?? 0, lastCursor); throw error; } const { accepted, discoveredDids } = ingestResult; @@ -224,8 +276,27 @@ async function streamAndFlush( await maybeOptimize(db, config, log); log.log( - `Flushed ${accepted.length}/${batch.length} records. Cursor: ${lastTimeUs}`, + `Flushed ${accepted.length}/${batch.length} records. Cursor: ${lastCursor}`, ); + } else if (checkpoint !== null) { + try { + await db.batch([ + saveJetstreamCursorStatement(db, checkpoint), + ...(config.orderedSource + ? [ + saveOrderedSourcePositionStatement( + db, + config.orderedSource, + checkpoint, + ), + ] + : []), + ...saveJetstreamCursorObservationStatements(db, checkpoint, []), + ]); + } catch (error) { + pendingCursor = Math.max(pendingCursor ?? 0, checkpoint); + throw error; + } } // Bounded, cursored feed prune (see sweepFeedItems / runFeedPruneSlice). @@ -328,12 +399,15 @@ async function streamAndFlush( : {}), time_us: event.time_us, revision: commit.rev, - cursor: String(event.time_us), + cursor: String(event.seq), }, }), ); - } else if (event.kind === "identity") { + } else if ( + event.kind === "identity" && + event.identity.handle !== undefined + ) { try { await applyIdentityEvent(db, event.did, event.identity.handle); } catch (err) { @@ -341,6 +415,8 @@ async function streamAndFlush( } } + pendingCursor = Math.max(pendingCursor ?? 0, event.seq); + if (buffer.length >= batchSize) { await flush(); } diff --git a/packages/contrail/src/core/router/diagnostics.ts b/packages/contrail/src/core/router/diagnostics.ts index a080011..2ae8e07 100644 --- a/packages/contrail/src/core/router/diagnostics.ts +++ b/packages/contrail/src/core/router/diagnostics.ts @@ -4,6 +4,7 @@ import { getCollectionShortNames, recordsTableName, nsidForShortName } from "../ import { getLastCursor, getServingSourcePosition } from "../db"; import { getBackfillStatus } from "../status"; import { getRequiredChangeConsumerReadiness } from "../changes"; +import { isJetstreamTimestampCursor } from "../jetstream-live"; export interface CursorStatus { cursor: number | null; @@ -23,6 +24,9 @@ export async function getCursorStatus(db: Database): Promise { return { cursor: null, date: null, seconds_ago: null }; } + if (!isJetstreamTimestampCursor(cursor)) { + return { cursor, date: null, seconds_ago: null }; + } const dateMs = Math.floor(cursor / 1000); return { cursor, @@ -80,13 +84,9 @@ export function registerCursorRoute( app.get(`/xrpc/${ns}.getCursor`, async (c) => { if (!config.orderedSource) { - const legacy = await getCursorStatus(db); - if (legacy.cursor === null) return c.json({}); - return c.json({ - time_us: legacy.cursor, - date: legacy.date, - seconds_ago: legacy.seconds_ago, - }); + const current = await getCursorStatus(db); + if (current.cursor === null) return c.json({}); + return c.json({ cursor: current.cursor }); } const current = await getServingSourcePosition(db); diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index b42bcb2..e1fcef9 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -199,11 +199,46 @@ export function deriveShortName(nsid: string): string { } export const DEFAULT_JETSTREAMS = [ - "wss://jetstream1.us-east.bsky.network", + "https://jetstream.us-east.bsky.network", ]; +/** Canonical source identity used for both the v2 client and its durable + * service binding. The official client addresses XRPC at the origin, so path, + * query, credentials, and fragments are rejected instead of being discarded + * ambiguously. WebSocket and HTTP spellings of the same origin are equivalent. */ +export function normalizeJetstreamService(service: string): string { + const url = new URL(service); + if (url.protocol === "wss:") url.protocol = "https:"; + if (url.protocol === "ws:") url.protocol = "http:"; + if (url.protocol !== "https:" && url.protocol !== "http:") { + throw new TypeError("Jetstream v2 service must use HTTP(S) or WS(S)"); + } + if ( + url.username || + url.password || + (url.pathname !== "" && url.pathname !== "/") || + url.search || + url.hash + ) { + throw new TypeError("Jetstream v2 service must be an origin URL"); + } + return url.origin; +} + +/** Return the one pinned Jetstream v2 service used by live ingestion. + * Sequence cursors are instance-local and cannot fail over between services. */ +export function jetstreamService(jetstreams: string[]): string { + if (jetstreams.length !== 1 || !jetstreams[0]) { + throw new TypeError( + "Jetstream v2 ingestion requires exactly one pinned service", + ); + } + return normalizeJetstreamService(jetstreams[0]); +} + /** - * Shape a configured jetstream list for `@atcute/jetstream`'s `url` option. + * Shape a configured jetstream list for the legacy `@atcute/jetstream` adapter + * still used by v1 archive/bootstrap integrations. * * @atcute distinguishes a string url (one fixed instance) from an array url (a * pool it picks from at random each connect). For an array it seeds @@ -294,14 +329,11 @@ export interface ContrailConfig { validation?: IngestValidationConfig; profiles?: (string | ProfileConfig)[]; relays?: string[]; - /** Jetstream endpoints to ingest from (defaults to {@link DEFAULT_JETSTREAMS}). - * Prefer a single endpoint: one instance has no clock skew, so `@atcute` takes - * no cursor rollback (see {@link jetstreamUrlOption}) — important for the cron - * model, which rebuilds the subscription every cycle. Scheduled ingestion - * requires exactly one pinned endpoint because a pooled connection rolls its - * timestamp cursor back on every new cycle and cannot make bounded progress - * through a dense overlap. Use 2+ only with `runPersistent()`, where the - * connection and its failover cursor remain long-lived. */ + /** Jetstream v2 service used for live ingestion (defaults to + * {@link DEFAULT_JETSTREAMS}). Exactly one service is required: v2 sequence + * cursors are instance-local and cannot fail over between servers. The array + * shape is retained for configuration compatibility and will be simplified in + * a later API cleanup. */ jetstreams?: string[]; /** Identity of the ordered source consumed by live ingestion. Its opaque * cursor is persisted atomically with projected mutations and may be exposed diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index cd07755..1553ecc 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -25,6 +25,10 @@ export * from "./core/generations"; export * from "./core/pds-snapshot"; export * from "./core/jetstream-source"; export * from "./core/jetstream"; +export { + JetstreamLiveHistoryExpiredError, +} from "./core/jetstream-live"; +export type { JetstreamLiveEvent } from "./core/jetstream-live"; export * from "./core/persistent"; export * from "./core/backfill"; export * from "./core/status"; diff --git a/packages/contrail/src/lexicons/generate.ts b/packages/contrail/src/lexicons/generate.ts index e525263..207a439 100644 --- a/packages/contrail/src/lexicons/generate.ts +++ b/packages/contrail/src/lexicons/generate.ts @@ -696,10 +696,8 @@ export function extractXrpcMethods( } function cursorLexicon(config: ContrailConfig): object { - const legacyProperties = { - time_us: { type: "integer" }, - date: { type: "string" }, - seconds_ago: { type: "integer" }, + const unscopedProperties = { + cursor: { type: "integer" }, }; if (!config.orderedSource) { return { @@ -708,10 +706,10 @@ function cursorLexicon(config: ContrailConfig): object { defs: { main: { type: "query", - description: "Get the current ingestion observation time", + description: "Get the current Jetstream live cursor", output: { encoding: "application/json", - schema: { type: "object", properties: legacyProperties }, + schema: { type: "object", properties: unscopedProperties }, }, }, }, diff --git a/packages/contrail/src/workers/backfill.ts b/packages/contrail/src/workers/backfill.ts index 5d2cd40..f2883b4 100644 --- a/packages/contrail/src/workers/backfill.ts +++ b/packages/contrail/src/workers/backfill.ts @@ -106,11 +106,18 @@ export async function backfillAll( ); } +export const DEFAULT_ALLUVIUM_SOURCE_URL = + "wss://jetstream1.us-east.bsky.network"; + export interface AlluviumBootstrapOptions { /** Alluvium HTTP origin. */ endpoint: string | URL; /** Logical source ID advertised by the selected manifests. */ sourceId: string; + /** Exact legacy Jetstream v1 URL advertised by the manifests. This is + * independent of `config.jetstreams`, which selects the v2 live service. + * Default: {@link DEFAULT_ALLUVIUM_SOURCE_URL}. */ + sourceUrl?: string; /** Operator-owned continuity epoch. Alluvium protocol v1 does not publish it. */ sourceEpoch: string; /** Explicitly accept manifests that report historical omissions. */ @@ -171,7 +178,6 @@ async function runAlluviumBootstrap( db: Database, opts: AlluviumBootstrapOptions, ): Promise { - const jetstreams = contrail.config.jetstreams ?? []; if ( contrail.config.orderedSource && (contrail.config.orderedSource.source !== opts.sourceId || @@ -181,16 +187,14 @@ async function runAlluviumBootstrap( "Alluvium source ID/epoch must match the configured orderedSource", ); } - if (jetstreams.length !== 1) { - throw new Error( - "Alluvium bootstrap requires exactly one configured Jetstream endpoint " + - "so scheduled ingestion resumes against the manifest's exact source.", - ); + const sourceUrl = opts.sourceUrl ?? DEFAULT_ALLUVIUM_SOURCE_URL; + if (!sourceUrl.trim()) { + throw new TypeError("Alluvium source URL must not be empty"); } const source = { id: opts.sourceId, epoch: opts.sourceEpoch, - url: jetstreams[0]!, + url: sourceUrl, }; const target = new DatabaseBootstrapTarget(db, contrail.config, { deferDerivedProjections: true, diff --git a/packages/contrail/tests/ingest-hang.test.ts b/packages/contrail/tests/ingest-hang.test.ts index 1dd0782..4201f69 100644 --- a/packages/contrail/tests/ingest-hang.test.ts +++ b/packages/contrail/tests/ingest-hang.test.ts @@ -6,25 +6,37 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; const jetstream = vi.hoisted(() => ({ script: null as | null - | ((self: { cursor: number | null }) => AsyncGenerator), + | ((self: { + cursor: number | null; + signal?: AbortSignal; + }) => AsyncGenerator), abort: false, + lastSignal: null as AbortSignal | null, + pendingPullSettled: false, })); // Replace the real WebSocket-backed subscription with one driven by the test's // `script`. `self.cursor` mirrors the real subscription's progress cursor, // which ingestEvents reads back as `lastCursor`. -vi.mock("@atcute/jetstream", () => { +vi.mock("../src/core/jetstream-live", async (importOriginal) => { + const actual = await importOriginal(); class MockJetstreamSubscription { cursor: number | null = null; - constructor(opts: { cursor?: number }) { + readonly signal?: AbortSignal; + constructor(opts: { cursor?: number; signal?: AbortSignal }) { this.cursor = typeof opts?.cursor === "number" ? opts.cursor : null; + this.signal = opts.signal; + jetstream.lastSignal = opts.signal ?? null; } async *[Symbol.asyncIterator]() { if (!jetstream.script) throw new Error("test did not set jetstream.script"); - yield* jetstream.script(this); + for await (const value of jetstream.script(this)) { + const event = value as { seq?: number; time_us: number }; + yield { ...event, seq: event.seq ?? event.time_us }; + } } } - return { JetstreamSubscription: MockJetstreamSubscription }; + return { ...actual, JetstreamLiveSubscription: MockJetstreamSubscription }; }); import { @@ -126,15 +138,22 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { beforeEach(() => { jetstream.script = null; jetstream.abort = false; + jetstream.lastSignal = null; + jetstream.pendingPullSettled = false; }); it("returns within the safety timeout when the stream replays history then goes quiet", async () => { - jetstream.script = async function* (self: { cursor: number | null }) { + jetstream.script = async function* (self) { // One historical commit (time_us in the past, so never "caught up"). self.cursor = 1_000_000; yield commitEvent("did:plc:author", "community.lexicon.calendar.event", 1_000_000, "evt1"); - // Quiet stream: no further events ever arrive. - await new Promise(() => {}); + // Quiet stream: no further events arrive, but transport cancellation must + // settle the pending pull rather than leave a reconnect loop behind. + await new Promise((resolve) => { + if (self.signal?.aborted) return resolve(); + self.signal?.addEventListener("abort", () => resolve(), { once: true }); + }); + jetstream.pendingPullSettled = true; }; const result = await withTimeout( @@ -156,6 +175,8 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { // The replayed cursor must come back so the caller can persist it. expect(result.lastCursor).toBe(1_000_000); expect(result.stats.stopReason).toBe("drain-time"); + expect(jetstream.lastSignal?.aborted).toBe(true); + await vi.waitFor(() => expect(jetstream.pendingPullSettled).toBe(true)); }); it("never checkpoints past the last yielded event when the subscription buffers ahead", async () => { @@ -225,13 +246,16 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { 1_000_000, "primary", ); - self.cursor = liveUs; - yield commitEvent( - "did:plc:new", - "app.bsky.graph.follow", - liveUs, - "dependent", - ); + self.cursor = 1_000_001; + yield { + ...commitEvent( + "did:plc:new", + "app.bsky.graph.follow", + liveUs, + "dependent", + ), + seq: 1_000_001, + }; }; const result = await ingestEvents( @@ -246,56 +270,48 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { ]); }); - it("breaks via 'caught up to present' and returns the batch + cursor when a kept event reaches the live edge", async () => { - // `live` is >= ingestEvents' startTimeUs (captured at the call below), so the - // 5s safety timeout is irrelevant — the exit must be the caught-up break. - const liveUs = Date.now() * 1000 + 5_000_000; + it("does not infer v2 head progress from the display timestamp", async () => { + // V2 `time` may be operator-imported and is not a resume coordinate. Even a + // future-looking display time must not terminate collection before the next + // seq has been accounted for. + const displayUs = Date.now() * 1000 + 5_000_000; jetstream.script = async function* (self) { self.cursor = 1_000_000; - yield commitEvent("did:plc:author", "community.lexicon.calendar.event", 1_000_000, "hist"); - self.cursor = liveUs; - yield commitEvent("did:plc:author", "community.lexicon.calendar.event", liveUs, "live"); - // Must NOT be reached: the caught-up break fires on `live` before this. - self.cursor = liveUs + 1_000; - yield commitEvent("did:plc:author", "community.lexicon.calendar.event", liveUs + 1_000, "after"); + yield { ...commitEvent("did:plc:author", "community.lexicon.calendar.event", displayUs, "first"), seq: 1_000_000 }; + self.cursor = 1_000_001; + yield { ...commitEvent("did:plc:author", "community.lexicon.calendar.event", displayUs, "second"), seq: 1_000_001 }; await new Promise(() => {}); }; const result = await withTimeout( - ingestEvents(discoverableConfig(), 999_999, 5_000), + ingestEvents(discoverableConfig(), 999_999, 150), 2_000, - "caught-up kept", + "display time is not head", ); - expect(result.events.map((e) => e.rkey)).toEqual(["hist", "live"]); - expect(result.lastCursor).toBe(liveUs); - expect(result.stats.stopReason).toBe("head"); + expect(result.events.map((e) => e.rkey)).toEqual(["first", "second"]); + expect(result.lastCursor).toBe(1_000_001); + expect(result.stats.stopReason).toBe("drain-time"); }); - it("caught-up break fires even on a filtered live event, deferring a following kept event to the next cycle", async () => { - // Behavior change vs the pre-fix loop: filtering now uses early `return`, so - // the caught-up check runs after a filtered event too. A filtered event at - // the live edge breaks the loop BEFORE the kept event that follows it. - // Pre-fix (`continue`) would have skipped the check, collected `evt`, and - // broken on it with cursor liveUs+1000. - const liveUs = Date.now() * 1000 + 5_000_000; - const knownDids = new Set(); // empty -> the follow is filtered (unknown DID) + it("accounts a filtered seq without deferring the following kept seq", async () => { + const knownDids = new Set(); jetstream.script = async function* (self) { - self.cursor = liveUs; - yield commitEvent("did:plc:stranger", "app.bsky.graph.follow", liveUs, "f1"); - self.cursor = liveUs + 1_000; - yield commitEvent("did:plc:author", "community.lexicon.calendar.event", liveUs + 1_000, "evt"); + self.cursor = 1_000_000; + yield commitEvent("did:plc:stranger", "app.bsky.graph.follow", 1_000_000, "f1"); + self.cursor = 1_000_001; + yield commitEvent("did:plc:author", "community.lexicon.calendar.event", 1_000_001, "evt"); await new Promise(() => {}); }; const result = await withTimeout( - ingestEvents(dependentConfig(), 999_999, 5_000, knownDids), + ingestEvents(dependentConfig(), 999_999, 150, knownDids), 2_000, - "caught-up filtered", + "filtered then kept", ); - expect(result.events).toHaveLength(0); // kept `evt` deferred, not collected this cycle - expect(result.lastCursor).toBe(liveUs); // broke at the filtered event, before `evt` + expect(result.events.map((event) => event.rkey)).toEqual(["evt"]); + expect(result.lastCursor).toBe(1_000_001); }); it("collects every event when they flow fast but within the safety timeout (the next()/timeout race drops nothing)", async () => { @@ -613,32 +629,15 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { expect(result.lastCursor).toBe(4_000_001); }); - it("never regresses the safe cursor when a pooled transport replays older overlap", async () => { - const startingCursor = 6_000_000; - jetstream.script = async function* (self) { - self.cursor = startingCursor - 500; - yield commitEvent( - "did:plc:author", - "community.lexicon.calendar.event", - startingCursor - 500, - "rolled-back", - ); - }; + it("rejects pooled v2 services because seq cursors are instance-local", async () => { const config = { ...discoverableConfig(), - jetstreams: ["wss://one.test", "wss://two.test"], + jetstreams: ["https://one.test", "https://two.test"], }; - const result = await ingestEvents( - config, - startingCursor, - budget({ maxCandidates: 1 }), - ); - - expect(result.events).toHaveLength(1); - expect(result.stats.lastAccountedCursor).toBe(startingCursor - 500); - expect(result.lastCursor).toBe(startingCursor); - expect(result.stats.safeEndingCursor).toBe(startingCursor); + await expect( + ingestEvents(config, 6_000_000, budget({ maxCandidates: 1 })), + ).rejects.toThrow("exactly one pinned service"); }); it("keeps source-inconsistency diagnostics and normal logs bounded", async () => { diff --git a/packages/contrail/tests/jetstream-single-url.test.ts b/packages/contrail/tests/jetstream-single-url.test.ts index 23cc167..62e07f2 100644 --- a/packages/contrail/tests/jetstream-single-url.test.ts +++ b/packages/contrail/tests/jetstream-single-url.test.ts @@ -1,31 +1,23 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; -// Records what `url` shape ingestEvents hands to @atcute's JetstreamSubscription. -// @atcute treats a string url and an array url differently: for an ARRAY it seeds -// `#lastUsedUrl = ''`, so the very first connect satisfies its "switched instance" -// check and rolls the cursor back 10s — even when the array holds a single URL and -// nothing switched. Passing a string seeds `#lastUsedUrl = undefined`, so no -// spurious rollback (and thus no re-delivered events / DUPLICATE-in-cycle flood). const recorder = vi.hoisted(() => ({ url: undefined as unknown })); -vi.mock("@atcute/jetstream", () => { +vi.mock("../src/core/jetstream-live", async (importOriginal) => { + const actual = await importOriginal(); class MockJetstreamSubscription { - cursor: number | null = null; + cursor: number | null; constructor(opts: { url?: unknown; cursor?: number }) { - recorder.url = opts?.url; - this.cursor = typeof opts?.cursor === "number" ? opts.cursor : null; + recorder.url = opts.url; + this.cursor = opts.cursor ?? 0; } async *[Symbol.asyncIterator]() { - // Quiet stream: block so ingestEvents exits via its safety timeout. await new Promise(() => {}); } } - return { JetstreamSubscription: MockJetstreamSubscription }; + return { ...actual, JetstreamLiveSubscription: MockJetstreamSubscription }; }); -import { ingestEvents } from "../src/index"; -import { resolveConfig } from "../src/index"; -import type { ContrailConfig } from "../src/index"; +import { ingestEvents, resolveConfig, type ContrailConfig } from "../src/index"; const silentLogger = { log() {}, warn() {}, error() {} }; @@ -42,26 +34,27 @@ function configWithJetstreams(jetstreams: string[]): ContrailConfig { }; } -describe("ingestEvents — jetstream url shape (om-6zdo)", () => { +describe("Jetstream v2 service pinning", () => { beforeEach(() => { recorder.url = undefined; }); - it("passes a single configured jetstream as a string (avoids @atcute's array-only cursor rollback)", async () => { + it("passes the one configured service to the v2 live adapter", async () => { await ingestEvents( - configWithJetstreams(["wss://jetstream.example/subscribe"]), + configWithJetstreams(["https://jetstream.example"]), 999_999, - 50, + 25, ); - expect(recorder.url).toBe("wss://jetstream.example/subscribe"); + expect(recorder.url).toBe("https://jetstream.example"); }); - it("passes multiple configured jetstreams as an array (preserves failover + intentional switch rollback)", async () => { - await ingestEvents( - configWithJetstreams(["wss://a.example", "wss://b.example"]), - 999_999, - 50, - ); - expect(recorder.url).toEqual(["wss://a.example", "wss://b.example"]); + it("rejects a service pool because v2 seq cursors are instance-local", async () => { + await expect( + ingestEvents( + configWithJetstreams(["https://a.example", "https://b.example"]), + 999_999, + 25, + ), + ).rejects.toThrow("exactly one pinned service"); }); }); diff --git a/packages/contrail/tests/jetstream-v2-live.test.ts b/packages/contrail/tests/jetstream-v2-live.test.ts new file mode 100644 index 0000000..f24a821 --- /dev/null +++ b/packages/contrail/tests/jetstream-v2-live.test.ts @@ -0,0 +1,212 @@ +import { describe, expect, it } from "vitest"; +import { + JETSTREAM_V2_SEQ_THRESHOLD, + JetstreamLiveHistoryExpiredError, + JetstreamLiveSubscription, + advanceJetstreamCursor, + isJetstreamTimestampCursor, + jetstreamDatetimeToMicroseconds, +} from "../src/core/jetstream-live"; +import { + assertServingSourceCompatibility, + getLastCursor, + getServingSourcePosition, + saveCursor, +} from "../src/index"; +import { saveJetstreamCursor } from "../src/core/db/records"; +import { createTestDbWithSchema } from "./helpers"; + +describe("Jetstream v2 live cursor semantics", () => { + it("rejects a stale seq through HTTP before an opaque browser transport can retry", async () => { + let transportStarted = false; + const subscription = new JetstreamLiveSubscription({ + url: "https://jetstream.example", + cursor: 42, + fetchImpl: (async (input) => { + expect(String(input)).toContain( + "/xrpc/network.bsky.jetstream.subscribeEvents?cursor=42&kinds=commit", + ); + return new Response( + JSON.stringify({ + error: "CursorTooOld", + message: "cursor 42 below lookback floor 100", + }), + { + status: 400, + headers: { + "Access-Control-Allow-Origin": "*", + "Content-Type": "application/json", + }, + }, + ); + }) as typeof fetch, + liveTransport: { + stream() { + transportStarted = true; + return (async function* () { + await new Promise(() => {}); + })(); + }, + }, + }); + + await expect( + subscription[Symbol.asyncIterator]().next(), + ).rejects.toBeInstanceOf(JetstreamLiveHistoryExpiredError); + expect(transportStarted).toBe(false); + }); + + it("does not misclassify an unverified WebSocket 400 as expired history", async () => { + const handshakeError = new Error("Unexpected server response: 400"); + const subscription = new JetstreamLiveSubscription({ + url: "https://jetstream.example", + cursor: 42, + // Both the initial probe and the post-failure probe prove only that the + // cursor is accepted. The actual filtered WebSocket request may still be + // rejected for an unrelated InvalidRequest. + fetchImpl: (async () => + new Response("WebSocket upgrade required", { status: 426 })) as typeof fetch, + liveTransport: { + stream() { + return (async function* () { + throw handshakeError; + })(); + }, + }, + }); + + const error = await subscription[Symbol.asyncIterator]().next().catch( + (value: unknown) => value, + ); + expect(error).toBe(handshakeError); + expect(error).not.toBeInstanceOf(JetstreamLiveHistoryExpiredError); + }); + + it("preserves microsecond precision from the v2 datetime", () => { + expect( + jetstreamDatetimeToMicroseconds("2026-08-29T13:08:26.865272Z"), + ).toBe(1_788_008_906_865_272); + expect( + jetstreamDatetimeToMicroseconds("2026-08-29T15:08:26.865272+02:00"), + ).toBe(1_788_008_906_865_272); + }); + + it("moves one way from a timestamp bridge to monotonic seqs", () => { + const timestamp = 1_788_008_906_865_272; + expect(isJetstreamTimestampCursor(timestamp)).toBe(true); + expect(isJetstreamTimestampCursor(25_304_406_049)).toBe(false); + expect(advanceJetstreamCursor(timestamp, 25_304_406_049)).toBe( + 25_304_406_049, + ); + expect(advanceJetstreamCursor(25_304_406_049, 25_304_406_048)).toBe( + 25_304_406_049, + ); + expect(JETSTREAM_V2_SEQ_THRESHOLD).toBe(1_000_000_000_000_000); + }); + + it("atomically replaces a legacy timestamp and rejects stale domain rollback", async () => { + const db = await createTestDbWithSchema(); + const timestamp = 1_788_008_906_865_272; + const source = { source: "jetstream", epoch: "v2" }; + + await saveCursor(db, timestamp, { source: "jetstream", epoch: "v1" }, [ + "legacy-observation", + ]); + await assertServingSourceCompatibility(db, source); + + await saveJetstreamCursor(db, 25_304_406_049, source, ["v2-observation"]); + expect(await getLastCursor(db)).toBe(25_304_406_049); + expect(await getServingSourcePosition(db)).toMatchObject({ + position: { + source: "jetstream", + epoch: "v2", + cursor: "25304406049", + }, + }); + + await saveJetstreamCursor(db, timestamp + 1, source, ["stale-timestamp"]); + expect(await getLastCursor(db)).toBe(25_304_406_049); + expect( + ( + await db + .prepare( + "SELECT time_us, observation FROM cursor_observations ORDER BY observation", + ) + .all() + ).results, + ).toEqual([{ time_us: 25_304_406_049, observation: "v2-observation" }]); + }); + + it("prevents a legacy writer from rolling a v2 cursor and epoch back", async () => { + const db = await createTestDbWithSchema(); + const timestamp = 1_788_008_906_865_272; + await saveJetstreamCursor(db, 42, { + source: "jetstream", + epoch: "v2-epoch", + }); + + await saveCursor( + db, + timestamp, + { source: "jetstream", epoch: "stale-v1-epoch" }, + ["stale-v1-observation"], + ); + + expect(await getLastCursor(db)).toBe(42); + expect(await getServingSourcePosition(db)).toMatchObject({ + position: { + source: "jetstream", + epoch: "v2-epoch", + cursor: "42", + }, + }); + expect( + ( + await db + .prepare("SELECT observation FROM cursor_observations") + .all() + ).results, + ).toEqual([]); + }); + + it("still rejects an epoch mismatch after the cursor is in the seq domain", async () => { + const db = await createTestDbWithSchema(); + await saveJetstreamCursor(db, 42, { + source: "jetstream", + epoch: "first-v2-epoch", + }); + await expect( + assertServingSourceCompatibility(db, { + source: "jetstream", + epoch: "different-v2-epoch", + }), + ).rejects.toThrow("does not match durable source position"); + }); + + it("rejects an epoch change when an unscoped writer already moved the live cursor to v2", async () => { + const db = await createTestDbWithSchema(); + const timestamp = 1_788_008_906_865_272; + await saveCursor(db, timestamp, { + source: "jetstream", + epoch: "legacy-epoch", + }); + + // Ingestion can temporarily run without orderedSource. That advances the + // authoritative live cursor while leaving source_position untouched. + await saveJetstreamCursor(db, 42); + expect(await getServingSourcePosition(db)).toMatchObject({ + position: { + source: "jetstream", + epoch: "legacy-epoch", + cursor: String(timestamp), + }, + }); + + await expect( + assertServingSourceCompatibility(db, { + source: "jetstream", + epoch: "new-epoch", + }), + ).rejects.toThrow("does not match durable source position"); + }); +}); diff --git a/packages/contrail/tests/lexicon-generation.test.ts b/packages/contrail/tests/lexicon-generation.test.ts index 2203c59..c081b29 100644 --- a/packages/contrail/tests/lexicon-generation.test.ts +++ b/packages/contrail/tests/lexicon-generation.test.ts @@ -185,7 +185,7 @@ describe("Contrail Lexicon generation", () => { expect(result.generated["example.public.profile.listRecords"]).toBeUndefined(); }); - it("keeps the legacy cursor contract without an ordered source", () => { + it("publishes an unscoped v2 seq cursor without an ordered source", () => { const { root, config } = fixture(); delete config.orderedSource; const result = generateLexicons({ @@ -197,9 +197,7 @@ describe("Contrail Lexicon generation", () => { const cursor = result.generated["example.public.getCursor"] as any; expect(cursor.defs.sourcePosition).toBeUndefined(); expect(cursor.defs.main.output.schema.properties).toEqual({ - time_us: { type: "integer" }, - date: { type: "string" }, - seconds_ago: { type: "integer" }, + cursor: { type: "integer" }, }); }); diff --git a/packages/contrail/tests/network-overrides-appview.test.ts b/packages/contrail/tests/network-overrides-appview.test.ts index 31ee305..4d844d9 100644 --- a/packages/contrail/tests/network-overrides-appview.test.ts +++ b/packages/contrail/tests/network-overrides-appview.test.ts @@ -9,8 +9,8 @@ import type { ContrailConfig } from "../src/index"; // commit without opening a real WebSocket. Everything else in the appview // ingest path runs for real, so this exercises the live-ingest refresh path // end-to-end (the smoking-gun call site `refreshStaleIdentities(db, dids)`). -vi.mock("@atcute/jetstream", async (importOriginal) => { - const actual = await importOriginal(); +vi.mock("../src/core/jetstream-live", async (importOriginal) => { + const actual = await importOriginal(); class MockJetstreamSubscription { cursor: number | null = null; constructor(_opts: unknown) {} @@ -18,6 +18,7 @@ vi.mock("@atcute/jetstream", async (importOriginal) => { yield { kind: "commit", // A past time_us so the ingest loop doesn't treat it as "caught up". + seq: 1_000_000, time_us: 1_000_000, did: "did:plc:ingest", commit: { @@ -30,7 +31,7 @@ vi.mock("@atcute/jetstream", async (importOriginal) => { }; } } - return { ...actual, JetstreamSubscription: MockJetstreamSubscription }; + return { ...actual, JetstreamLiveSubscription: MockJetstreamSubscription }; }); const silentLogger = { log() {}, warn() {}, error() {} }; diff --git a/packages/contrail/tests/persistent.test.ts b/packages/contrail/tests/persistent.test.ts index 47767fd..59332dd 100644 --- a/packages/contrail/tests/persistent.test.ts +++ b/packages/contrail/tests/persistent.test.ts @@ -1,14 +1,20 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; -import type { ContrailConfig, Database } from "../src/index"; +import type { + ContrailConfig, + Database, + PersistentJetstreamSubscription, +} from "../src/index"; import { resolveConfig } from "../src/index"; import { createTestDb, createTestDbWithSchema, TEST_CONFIG } from "./helpers"; import { runPersistent } from "../src/index"; +import { JetstreamLiveHistoryExpiredError } from "../src/core/jetstream-live"; import { getLastCursor, getServingSourcePosition, queryRecords, } from "../src/index"; import { initSchema } from "../src/index"; +import { saveJetstreamCursor } from "../src/core/db/records"; const applyIdentityEventMock = vi.fn().mockResolvedValue(undefined); vi.mock("../src/core/identity", async (importOriginal) => { @@ -27,7 +33,14 @@ beforeEach(async () => { }); // Helper: create a mock async iterable that yields events then hangs until aborted -function mockSubscription(events: Array<{ kind: string; did: string; time_us: number; commit?: any }>) { +function mockSubscription( + events: Array<{ + kind: string; + did: string; + time_us: number; + commit?: any; + }>, +): PersistentJetstreamSubscription { let aborted = false; return { cursor: 0, @@ -37,7 +50,11 @@ function mockSubscription(events: Array<{ kind: string; did: string; time_us: nu next: async () => { if (aborted) return { value: undefined, done: true as const }; if (i < events.length) { - return { value: events[i++], done: false as const }; + const event = events[i++]!; + return { + value: { ...event, seq: event.time_us }, + done: false as const, + }; } // Hang until iterator is returned (abort) return new Promise>(() => {}); @@ -52,6 +69,73 @@ function mockSubscription(events: Array<{ kind: string; did: string; time_us: nu } describe("runPersistent", () => { + it("rejects a raw-entry ordered-source mismatch before opening the stream", async () => { + await saveJetstreamCursor(db, 42, { + source: "jetstream", + epoch: "original-epoch", + }); + let subscriptionCreated = false; + + await expect( + runPersistent( + db, + { + ...TEST_CONFIG, + orderedSource: { source: "jetstream", epoch: "changed-epoch" }, + }, + { + createSubscription: () => { + subscriptionCreated = true; + return mockSubscription([]); + }, + }, + ), + ).rejects.toThrow("does not match durable source position"); + expect(subscriptionCreated).toBe(false); + }); + + it("persists the initial timestamp bridge before its first source pull", async () => { + const bridge = 1_788_008_906_865_272; + const controller = new AbortController(); + let durableCursorDuringPull: number | null = null; + + await runPersistent(db, TEST_CONFIG, { + signal: controller.signal, + createSubscription: (cursor) => { + expect(cursor).toBeNull(); + return { + cursor: bridge, + [Symbol.asyncIterator]() { + return { + next: async () => { + durableCursorDuringPull = await getLastCursor(db); + controller.abort(); + return { value: undefined, done: true as const }; + }, + }; + }, + }; + }, + }); + + expect(durableCursorDuringPull).toBe(bridge); + expect(await getLastCursor(db)).toBe(bridge); + }); + + it("fails instead of retrying forever when live retention has expired", async () => { + const expired = new JetstreamLiveHistoryExpiredError("cursor too old"); + await expect( + runPersistent(db, TEST_CONFIG, { + createSubscription: () => + ({ + async *[Symbol.asyncIterator]() { + throw expired; + }, + }) as any, + }), + ).rejects.toBe(expired); + }); + it("flushes records and the ordered source position atomically", async () => { const config = resolveConfig({ ...TEST_CONFIG, @@ -77,7 +161,10 @@ describe("runPersistent", () => { batchSize: 50, flushIntervalMs: 60_000, // high so only batch size triggers flush signal: controller.signal, - createSubscription: () => mockSubscription(events) as any, + createSubscription: (_cursor, subscriptionSignal) => { + expect(subscriptionSignal).toBe(controller.signal); + return mockSubscription(events); + }, }); // Wait for flush to complete @@ -488,6 +575,33 @@ describe("runPersistent", () => { ); }); + it("checkpoints an identity-only v2 range", async () => { + const events = [ + { + kind: "identity" as const, + did: "did:plc:identity-only", + time_us: 4_100, + identity: { + did: "did:plc:identity-only", + handle: "identity-only.test", + time: "2026-04-01T10:00:00Z", + }, + }, + ]; + const controller = new AbortController(); + const promise = runPersistent(db, TEST_CONFIG, { + batchSize: 100, + flushIntervalMs: 50, + signal: controller.signal, + createSubscription: () => mockSubscription(events) as any, + }); + + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(await getLastCursor(db)).toBe(4_100); + controller.abort(); + await promise; + }); + it("runs the recovery feed sweep on a fully idle stream", async () => { // Regression: the feed sweep used to sit behind flush()'s early return when // the buffer was empty, so over-cap rows (a lowered cap, a bulk import) diff --git a/packages/contrail/tests/scheduled-ingest.test.ts b/packages/contrail/tests/scheduled-ingest.test.ts index 8092753..a148058 100644 --- a/packages/contrail/tests/scheduled-ingest.test.ts +++ b/packages/contrail/tests/scheduled-ingest.test.ts @@ -5,6 +5,7 @@ const source = vi.hoisted(() => ({ requestedCursors: [] as Array, events: [] as Array<{ kind: "commit"; + seq?: number; time_us: number; did: string; commit: { @@ -18,7 +19,8 @@ const source = vi.hoisted(() => ({ }>, })); -vi.mock("@atcute/jetstream", () => { +vi.mock("../src/core/jetstream-live", async (importOriginal) => { + const actual = await importOriginal(); class MockJetstreamSubscription { cursor: number | null; private readonly start: number | null; @@ -31,16 +33,20 @@ vi.mock("@atcute/jetstream", () => { async *[Symbol.asyncIterator]() { for (const event of source.events) { - // Model the supported source's resume coordinate: only events after the - // committed cursor are delivered on the next scheduled connection. - if (this.start !== null && event.time_us <= this.start) continue; - this.cursor = event.time_us; - yield event; + const seq = (event as { seq?: number }).seq ?? event.time_us; + // V2 replay is inclusive at the server and de-duplicated by the client. + if ( + this.start !== null && + this.start < 1_000_000_000_000_000 && + seq <= this.start + ) continue; + this.cursor = seq; + yield { ...event, seq }; } } } - return { JetstreamSubscription: MockJetstreamSubscription }; + return { ...actual, JetstreamLiveSubscription: MockJetstreamSubscription }; }); import { @@ -51,13 +57,15 @@ import { type Database, type Logger, } from "../src/index"; +import { saveJetstreamCursor } from "../src/core/db/records"; import { createTestDb } from "./helpers"; const COLLECTION = "com.example.event"; -function commit(time_us: number, rkey: string) { +function commit(time_us: number, rkey: string, seq = time_us) { return { kind: "commit" as const, + seq, time_us, did: "actor", commit: { @@ -146,54 +154,26 @@ describe("bounded scheduled ingest cycles", () => { ).toEqual(["one", "two", "three"]); }); - it("resumes before a capped coarse cursor without skipping or recounting siblings", async () => { + it("uses unique v2 seqs when display timestamps are equal", async () => { const db = createTestDb(); const output = logger(); const configured = config(output.value); source.events = [ - commit(1_100_000, "same-cursor-a"), - commit(1_100_000, "same-cursor-b"), - commit(1_100_001, "after"), + commit(1_100_000, "same-time-a", 1_100_001), + commit(1_100_000, "same-time-b", 1_100_002), + commit(1_100_001, "after", 1_100_003), ]; const oneCandidate = { ...budget, maxCandidates: 1 }; - await runIngestCycle(db, configured, oneCandidate); - expect(await getLastCursor(db)).toBe(1_100_000); - expect( - (await db.prepare("SELECT rkey FROM records_event ORDER BY rkey").all<{ rkey: string }>()) - .results?.map((row) => row.rkey), - ).toEqual(["same-cursor-a"]); - - await runIngestCycle(db, configured, oneCandidate); - expect(await getLastCursor(db)).toBe(1_100_000); - expect( - (await db.prepare("SELECT rkey FROM records_event ORDER BY rkey").all<{ rkey: string }>()) - .results?.map((row) => row.rkey), - ).toEqual(["same-cursor-a", "same-cursor-b"]); - - await runIngestCycle(db, configured, oneCandidate); - expect(await getLastCursor(db)).toBe(1_100_001); + for (const cursor of [1_100_001, 1_100_002, 1_100_003]) { + await runIngestCycle(db, configured, oneCandidate); + expect(await getLastCursor(db)).toBe(cursor); + } expect( (await db.prepare("SELECT rkey FROM records_event ORDER BY rkey").all<{ rkey: string }>()) .results?.map((row) => row.rkey), - ).toEqual(["after", "same-cursor-a", "same-cursor-b"]); - expect(source.requestedCursors).toEqual([0, 1_099_999, 1_099_999]); - - const summaries = output.lines - .filter((line) => line.text.startsWith("[ingest] cycle summary ")) - .map((line) => - JSON.parse(line.text.slice("[ingest] cycle summary ".length)) as { - retained_candidates: number; - cursor_boundary_duplicates_dropped: number; - }, - ); - expect(summaries.map((summary) => summary.retained_candidates)).toEqual([ - 1, - 1, - 1, - ]); - expect(summaries[1]?.cursor_boundary_duplicates_dropped).toBe(1); - expect(summaries[2]?.cursor_boundary_duplicates_dropped).toBe(2); + ).toEqual(["after", "same-time-a", "same-time-b"]); + expect(source.requestedCursors).toEqual([0, 1_100_001, 1_100_002]); }); it("restores actor scope from a projected record before replaying dependent siblings", async () => { @@ -202,11 +182,11 @@ describe("bounded scheduled ingest cycles", () => { const configured = config(output.value, true); const timeUs = 1_200_000; source.events = [ - commit(timeUs, "discover-actor"), + commit(timeUs, "discover-actor", 1_200_001), { - ...commit(timeUs, "dependent-sibling"), + ...commit(timeUs, "dependent-sibling", 1_200_002), commit: { - ...commit(timeUs, "dependent-sibling").commit, + ...commit(timeUs, "dependent-sibling", 1_200_002).commit, collection: "app.bsky.graph.follow", }, }, @@ -230,25 +210,25 @@ describe("bounded scheduled ingest cycles", () => { ).toEqual({ rkey: "dependent-sibling" }); }); - it("persists Atcute's captured initial cursor after an empty first drain", async () => { + it("persists a timestamp bridge, then transitions to a v2 seq", async () => { const db = createTestDb(); const output = logger(); - source.initialCursor = 7_000_000; + const timestampCursor = 1_788_000_000_000_000; + source.initialCursor = timestampCursor; await runIngestCycle(db, config(output.value), budget); - expect(await getLastCursor(db)).toBe(7_000_000); + expect(await getLastCursor(db)).toBe(timestampCursor); const summary = output.lines.find((line) => line.text.startsWith("[ingest] cycle summary "), ); expect(summary?.text).toContain('"starting_cursor":null'); - expect(summary?.text).toContain('"safe_ending_cursor":7000000'); + expect(summary?.text).toContain(`"safe_ending_cursor":${timestampCursor}`); - // An event sharing the captured microsecond is not skipped by an exclusive - // resume API: the next cycle requests one microsecond earlier. - source.events = [commit(7_000_000, "between-empty-cycles")]; + source.events = [commit(timestampCursor, "between-empty-cycles", 7_000_000)]; await runIngestCycle(db, config(output.value), budget); - expect(source.requestedCursors).toEqual([7_000_000, 6_999_999]); + expect(source.requestedCursors).toEqual([timestampCursor, timestampCursor]); + expect(await getLastCursor(db)).toBe(7_000_000); expect( await db .prepare("SELECT rkey FROM records_event WHERE rkey = ?") @@ -257,6 +237,73 @@ describe("bounded scheduled ingest cycles", () => { ).toEqual({ rkey: "between-empty-cycles" }); }); + it("rejects a raw-entry ordered-source mismatch before opening the stream", async () => { + const db = createTestDb(); + const output = logger(); + const original = { + ...config(output.value), + orderedSource: { source: "jetstream", epoch: "original-epoch" }, + }; + await initSchema(db, original); + await saveJetstreamCursor(db, 42, original.orderedSource); + + await expect( + runIngestCycle( + db, + { + ...original, + orderedSource: { source: "jetstream", epoch: "changed-epoch" }, + }, + budget, + ), + ).rejects.toThrow("does not match durable source position"); + expect(source.requestedCursors).toHaveLength(0); + }); + + it("rejects an existing seq cursor whose originating service was never bound", async () => { + const db = createTestDb(); + const output = logger(); + const configured = config(output.value); + await initSchema(db, configured); + await saveJetstreamCursor(db, 42); + + await expect(runIngestCycle(db, configured, budget)).rejects.toThrow( + "has no pinned service identity", + ); + expect(source.requestedCursors).toHaveLength(0); + }); + + it("binds seq cursors to one normalized durable service origin", async () => { + const db = createTestDb(); + const output = logger(); + const base = config(output.value); + source.initialCursor = 0; + + await runIngestCycle( + db, + { ...base, jetstreams: ["https://pinned.example/"] }, + budget, + ); + expect(await getLastCursor(db)).toBe(0); + + // WS(S) and HTTP(S) spellings normalize to the same client origin. + await runIngestCycle( + db, + { ...base, jetstreams: ["wss://pinned.example"] }, + budget, + ); + const opened = source.requestedCursors.length; + + await expect( + runIngestCycle( + db, + { ...base, jetstreams: ["https://different.example"] }, + budget, + ), + ).rejects.toThrow("does not match durable service"); + expect(source.requestedCursors).toHaveLength(opened); + }); + it("rejects endpoint pools for scheduled ingestion before rollback can starve the cap", async () => { const db = createTestDb(); const output = logger(); @@ -266,7 +313,7 @@ describe("bounded scheduled ingest cycles", () => { }); await expect(runIngestCycle(db, configured, budget)).rejects.toThrow( - "scheduled ingestion requires exactly one pinned Jetstream endpoint", + "Jetstream v2 ingestion requires exactly one pinned service", ); expect(source.requestedCursors).toHaveLength(0); }); diff --git a/packages/contrail/tests/types.test.ts b/packages/contrail/tests/types.test.ts index c0cdec3..2b8a8ee 100644 --- a/packages/contrail/tests/types.test.ts +++ b/packages/contrail/tests/types.test.ts @@ -216,7 +216,9 @@ describe("resolveConfig", () => { it("applies default jetstreams and relays", () => { const resolved = resolveConfig({ namespace: "test", collections: {} }); - expect(resolved.jetstreams).toHaveLength(1); + expect(resolved.jetstreams).toEqual([ + "https://jetstream.us-east.bsky.network", + ]); expect(resolved.relays).toHaveLength(1); }); diff --git a/packages/contrail/tests/worker.test.ts b/packages/contrail/tests/worker.test.ts index 49cedd6..ea7a782 100644 --- a/packages/contrail/tests/worker.test.ts +++ b/packages/contrail/tests/worker.test.ts @@ -387,7 +387,7 @@ describe("createWorker", () => { expect(response.status).toBe(404); }); - it("preserves the legacy cursor response without an ordered source", async () => { + it("returns the unscoped v2 seq without an ordered source", async () => { const config: ContrailConfig = { ...MINIMAL_CONFIG, orderedSource: undefined, @@ -402,10 +402,7 @@ describe("createWorker", () => { new Request("https://api.example.com/xrpc/com.example.getCursor"), env, ); - expect(await response.json()).toMatchObject({ - time_us: 1_234_000, - date: new Date(1234).toISOString(), - }); + expect(await response.json()).toEqual({ cursor: 1_234_000 }); }); it("keeps profiles, feeds, custom queries, and configured notify routes", async () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a049815..f076074 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -118,12 +118,15 @@ importers: apps/sveltekit-cloudflare-workers: dependencies: - '@atcute/jetstream': - specifier: ^2.0.2 - version: 2.0.2(@atcute/lexicons@2.0.3)(typescript@6.0.3) '@atmo-dev/contrail': specifier: workspace:* version: link:../../packages/contrail + '@atproto/lex': + specifier: ^0.3.8 + version: 0.3.8 + '@bsky/jetstream': + specifier: ^1.0.0 + version: 1.0.0(@atproto/lex@0.3.8) '@foxui/core': specifier: ^0.9.1 version: 0.9.1(@internationalized/date@3.12.1)(@sveltejs/kit@2.70.2(@sveltejs/vite-plugin-svelte@7.2.0(svelte@5.56.8(@typescript-eslint/types@8.66.0))(vite@8.2.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.6)))(svelte@5.56.8(@typescript-eslint/types@8.66.0))(typescript@6.0.3)(vite@8.2.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.6)))(svelte@5.56.8(@typescript-eslint/types@8.66.0))(tailwindcss@4.3.3) @@ -281,6 +284,12 @@ importers: '@atcute/xrpc-server': specifier: ^2.0.2 version: 2.0.2(@atcute/cid@2.4.2)(@atcute/lexicons@2.0.3)(typescript@6.0.3) + '@atproto/lex': + specifier: ^0.3.8 + version: 0.3.8 + '@bsky/jetstream': + specifier: ^1.0.0 + version: 1.0.0(@atproto/lex@0.3.8) cac: specifier: ^7.0.0 version: 7.0.0 @@ -464,10 +473,105 @@ packages: peerDependencies: '@atcute/lexicons': ^2.0.0 + '@atproto-labs/did-resolver@0.3.7': + resolution: {integrity: sha512-F3M3U5cU1QSAXX3J5auGEc5N2pgDgpirAjvyDFsytXuyWfrjWfTPd/H+DZrrED7gK+noW0cAWtxxjdX7/cTSVQ==} + engines: {node: '>=22'} + + '@atproto-labs/fetch@0.3.5': + resolution: {integrity: sha512-iDFZEoNqfL17EVKE+uNzdUD7YF8LWhEhxeBtP6x+PNuAxoXv3ip9vLmB8nNzNxFwVn++FipfDtSiPqmziv1bdA==} + engines: {node: '>=22'} + + '@atproto-labs/pipe@0.2.4': + resolution: {integrity: sha512-n67jCcrC+ouAeO10cWkpPzzLMlDi/lDCU30Us+LGqhOPhT6c4t5ASdBLQi9W3jUQtRzQBt3G9zipF+xKWNvVbw==} + engines: {node: '>=22'} + + '@atproto-labs/simple-store-memory@0.2.6': + resolution: {integrity: sha512-DD1v7MEfYF3BAcEpMTTpzfBLWLoI2HuyBhku2YpjHoqPrRrhCtE1alkxfPHjtjJVIjuU/XNU0cF/wB3+5FiKsA==} + engines: {node: '>=22'} + + '@atproto-labs/simple-store@0.5.1': + resolution: {integrity: sha512-vfvoDhu6ds6BT3Pqe+d2/LBU1WpDRtt69y6zltlfMCoucPm82m1j50/Wb6xTvv+oZ+2j0JyZVXsmXQ12SWYQJg==} + engines: {node: '>=22'} + + '@atproto/common-web@0.5.10': + resolution: {integrity: sha512-w4JUdsJ3VXt8ewkavYh5m/u0UbxDtFdCKhNZPEpJ0U1vcWIjDaSInIexteETDgD7fBJAhidIEi30MGz1kk49ug==} + engines: {node: '>=22'} + + '@atproto/common@0.8.1': + resolution: {integrity: sha512-WvgY0CDgodmTS7B1KDVFt868h1b85iQtQZeoDVI+6oOZmg1jINWUl+bZ1rwf59Tk1PqcMz5OzH6OS2J7Mb9/Iw==} + engines: {node: '>=22'} + + '@atproto/crypto@0.5.4': + resolution: {integrity: sha512-UR0BkuYNYuFtw+dA+y/oPPxzX0SWRnGJ+1Cfh/jGP1BvjRUyezK3omjpeLms5fYrXbM9vnfX+ckJFJqkBgLOdw==} + engines: {node: '>=22'} + + '@atproto/did@0.5.4': + resolution: {integrity: sha512-BlnwQ+obL+4ZA71KH/EzZ3TY+cpSxnLiUI85mjBJIQwvDF/oN2sQA18wIj9jpduviIt2b/cMtlJuzHjzBkfXvw==} + engines: {node: '>=22'} + + '@atproto/lex-builder@0.1.13': + resolution: {integrity: sha512-YHSONY88fWdMcN6Xa/Noap98DNW2m5zVZGspQWaROsX2lFx1mLf7g/XKLBcPRVDpE3OWRZ98QkkSZN3SgbOXrg==} + engines: {node: '>=22'} + + '@atproto/lex-cbor@0.1.6': + resolution: {integrity: sha512-PxLQy7jzV8u9zTGURNF8ZczLPk+ZH+WNx/2z/z7RacJbu7sWNpx26U2pC7cIkWtt4jyDaruaLhssw3w3atLotA==} + engines: {node: '>=22'} + + '@atproto/lex-client@0.3.5': + resolution: {integrity: sha512-IOYjgQ2H+pw8QhV0vMMG9PTRg+oh5In9APbGBvotNZlbF4SCZNDVd7dONFY4rU2Dg6BHSkiqbqoNUHNgTC+ITA==} + engines: {node: '>=22'} + + '@atproto/lex-data@0.1.7': + resolution: {integrity: sha512-kW/dPLqo/WgCLV+XESR4JKwV6c1rZWJGOfuPupZGTjEDAKoBbKXdaEzX9/1vKQYbZ9U3j0DS/n7OFFK7wBugyQ==} + engines: {node: '>=22'} + + '@atproto/lex-document@0.1.10': + resolution: {integrity: sha512-7ivrxWttSxInDksR2rtM24xGdntY/3Ew5j08W5SiG45B+T2nyARL5EOk2xMXEm3MLjPrpNYAybA21ZEDYhGG0Q==} + engines: {node: '>=22'} + + '@atproto/lex-installer@0.1.17': + resolution: {integrity: sha512-XSXiFLoDT7YEe+GuUdwczKQmhuLrA2CHVVlL8s5U8FLtWC5Cr3ELmz1t1b/Y8dV+R31hWyvdsk/t992B6+WW3Q==} + engines: {node: '>=22'} + + '@atproto/lex-json@0.1.6': + resolution: {integrity: sha512-mvrAd0lbyuecIHjyld8QN6MN6CBf4j0GCxLzegsvLh0SvDf+GbYWklkcQqmITL44yFQOwmA/QNIQj0Uvh7+R/g==} + engines: {node: '>=22'} + + '@atproto/lex-resolver@0.2.10': + resolution: {integrity: sha512-0Ml37emPqi6zr7hBAVR8fi7XDgG6xWsksBotoMU4Doh/PNFNt/H79Zopt/jnmDRcl8DGj9PNqZOZjzcAr2IuFw==} + engines: {node: '>=22'} + + '@atproto/lex-schema@0.2.6': + resolution: {integrity: sha512-8fe/gmhjMImBsy073jMANkbHHQLe/w4s9XLATpAz+Qd6WrR9RYTqc2DU8uZDL13F5bfdGTj8KECt/Xs4CoPXdQ==} + engines: {node: '>=22'} + + '@atproto/lex@0.3.8': + resolution: {integrity: sha512-8X7f6wxVDm4IVNR80Yg3BV3IXb89sKo/KIZeYb4or/EHP7ljazVU4qU+RPXmlELnWiIcw+PiJE+hPSkGLnMPOA==} + engines: {node: '>=22'} + hasBin: true + + '@atproto/repo@0.10.12': + resolution: {integrity: sha512-SnDSoFi1bRAfN0IcDjSPFcefknDCIIjKgJXgFsd5jvktCkopmzml8BpQEP5t2/mcZ7NvEn5onQ0kaWkXhgL+5g==} + engines: {node: '>=22'} + + '@atproto/syntax@0.7.5': + resolution: {integrity: sha512-6vnLQK8OAzg0dO6z/xnvuXn5zMV0UMI54zbxk7G7BXhGlIlLMb1yo+JVYtAlv8Nxzr+AaFdNZ8AJt+L/QhJMFQ==} + engines: {node: '>=22'} + + '@atproto/ws-client@0.2.1': + resolution: {integrity: sha512-HCGkqH/Ksxpgg/xCyIvH5hK6o8Lg2PxcEA790GCEV9MspCZB/xIesHZVgJN0cEvklGe+6pNCOZE1B5kj9q1ONA==} + engines: {node: '>=22'} + '@babel/runtime@7.29.7': resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} engines: {node: '>=6.9.0'} + '@bsky/jetstream@1.0.0': + resolution: {integrity: sha512-YtQYhG6NEo7GevmVhRAUp8koyjoj+zf7u+Mo5/Of8xmHFVGLhHyQ57jck8JTPjizFW59G3fqM1fifMCuHFzL0w==} + engines: {node: '>=22.15.0'} + peerDependencies: + '@atproto/lex': ^0.3.0 + '@changesets/apply-release-plan@7.1.1': resolution: {integrity: sha512-9qPCm/rLx/xoOFXIHGB229+4GOL76S4MC+7tyOuTsR6+1jYlfFDQORdvwR5hDA6y4FL2BPt3qpbcQIS+dW85LA==} @@ -1568,6 +1672,14 @@ packages: os: [linux] libc: [glibc] + '@noble/curves@1.9.7': + resolution: {integrity: sha512-gbKGcRUYIjA3/zCCNaWDciTMFI0dCkvou3TL8Zmy5Nc7sJ47a0jtOeZoTaMxkuqRo9cRhjOdZJXegxYE5FN/xw==} + engines: {node: ^14.21.3 || >=16} + + '@noble/hashes@1.8.0': + resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} + engines: {node: ^14.21.3 || >=16} + '@noble/secp256k1@3.1.0': resolution: {integrity: sha512-+F7iS7tUMaNGXcc9X3PjmjvuQnXEuSjCRNzVVA2xAcKXgCaP0dHYz4SFyt4FKNHef7sOP//xihowcySSS7PK9g==} @@ -1602,6 +1714,9 @@ packages: '@oxc-project/types@0.142.0': resolution: {integrity: sha512-7W+2q5AKQVU36fkaryontrHn3YDt1RyUYXatw9i5H8ocYe2sPKSFB6eS8WNPeRKiN1qAWWZUPm7gwFzJGrccqQ==} + '@pinojs/redact@0.4.0': + resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + '@polka/url@1.0.0-next.29': resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==} @@ -2177,6 +2292,9 @@ packages: '@tiptap/core': 3.29.2 '@tiptap/pm': 3.29.2 + '@ts-morph/common@0.28.1': + resolution: {integrity: sha512-W74iWf7ILp1ZKNYXY5qbddNaml7e9Sedv5lvU1V8lftlitkc9Pq1A+jlH23ltDgWYeZFFEqGCD1Ies9hqu3O+g==} + '@turbo/darwin-64@2.10.8': resolution: {integrity: sha512-po+7rfJfUnFXjWlcoN2RwhErgzCdRtBc1T26vYPcywHlggmCQiQe1uWaE4j+BibI2uY9/2pDoFzMN0rmSaPFOw==} cpu: [x64] @@ -2358,6 +2476,14 @@ packages: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} + ansi-regex@6.3.0: + resolution: {integrity: sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==} + engines: {node: '>=12'} + + ansi-styles@6.2.3: + resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} + engines: {node: '>=12'} + any-promise@1.3.0: resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} @@ -2379,6 +2505,10 @@ packages: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + atomic-sleep@1.0.0: + resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} + engines: {node: '>=8.0.0'} + axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} @@ -2423,6 +2553,10 @@ packages: resolution: {integrity: sha512-tixWYgm5ZoOD+3g6UTea91eow5z6AAHaho3g0V9CNSNb45gM8SmflpAc+GRd1InC4AqN/07Unrgp56Y94N9hJQ==} engines: {node: '>=20.19.0'} + cborg@4.5.8: + resolution: {integrity: sha512-6/viltD51JklRhq4L7jC3zgy6gryuG5xfZ3kzpE+PravtyeQLeQmCYLREhQH7pWENg5pY4Yu/XCd6a7dKScVlw==} + hasBin: true + chai@6.2.2: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} @@ -2434,10 +2568,17 @@ packages: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} + cliui@9.0.1: + resolution: {integrity: sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==} + engines: {node: '>=20'} + clsx@2.1.1: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + code-block-writer@13.0.3: + resolution: {integrity: sha512-Oofo0pq3IKnsFtuHqSF7TqBfr71aeyZDVJ0HpmqB7FBM2qEigL0iPONSCZSO9pE9dZTAxANe5XHG9Uy0YMv8cg==} + commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} engines: {node: '>= 6'} @@ -2463,6 +2604,9 @@ packages: core-js@3.49.0: resolution: {integrity: sha512-es1U2+YTtzpwkxVLwAFdSpaIMyQaq0PBgm3YD1W3Qpsn1NAmO3KSgZfu+oGSWVu6NvLHoHCV/aYcsE5wiB7ALg==} + core-js@3.50.0: + resolution: {integrity: sha512-BRWgOLKkFeCgRudR6zrs8p9XJZcE14grzKMMssoYrk6krtuEZ7MTKPIY5RzOnqsEKIR9kst7wNzphttraT+Yqw==} + cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -2519,6 +2663,9 @@ packages: emoji-picker-element@1.29.1: resolution: {integrity: sha512-TOiHzu9Dqib3x4MwcAi3wi3RdyT4SoeB4b15AvH1ks4SBwTl7DeebhZ0d3x6dNi4XfNU7IGRZ7NBQllj0RqwrQ==} + emoji-regex@10.6.0: + resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} + enhanced-resolve@5.24.5: resolution: {integrity: sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==} engines: {node: '>=10.13.0'} @@ -2548,6 +2695,10 @@ packages: engines: {node: '>=18'} hasBin: true + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -2715,6 +2866,14 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + + get-east-asian-width@1.6.0: + resolution: {integrity: sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==} + engines: {node: '>=18'} + glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -2801,6 +2960,9 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + iso-datestring-validator@2.2.2: + resolution: {integrity: sha512-yLEMkBbLZTlVQqOnQ4FiMujR6T4DEcCb1xizmvXS+OxuhwcbtynoosRzdMA69zZCShCNAbi+gJ71FxZBBXx1SA==} + jiti@2.7.0: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true @@ -3029,6 +3191,9 @@ packages: lowlight@3.3.0: resolution: {integrity: sha512-0JNhgFoPvP6U6lE/UdVsSq99tn6DhjjpAj5MxG49ewd2mOBVtwWYIT8ClyABhq198aXXODMU6Ox8DrGy/CpTZQ==} + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lz-string@1.5.0: resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==} hasBin: true @@ -3085,6 +3250,9 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + multiformats@13.4.2: + resolution: {integrity: sha512-eh6eHCrRi1+POZ3dA+Dq1C6jhP1GNtr9CRINMb67OKzqW9I5DUuZM/3jLPlzhgpGeiNUlEGEbkCYChXMCc/8DQ==} + mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} @@ -3117,6 +3285,10 @@ packages: resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} engines: {node: '>=12.20.0'} + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -3166,6 +3338,9 @@ packages: react: optional: true + path-browserify@1.0.1: + resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -3233,6 +3408,16 @@ packages: resolution: {integrity: sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==} engines: {node: '>=6'} + pino-abstract-transport@3.0.0: + resolution: {integrity: sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==} + + pino-std-serializers@7.1.0: + resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} + + pino@10.3.1: + resolution: {integrity: sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==} + hasBin: true + pirates@4.0.7: resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} engines: {node: '>= 6'} @@ -3385,6 +3570,9 @@ packages: engines: {node: '>=14'} hasBin: true + process-warning@5.1.0: + resolution: {integrity: sha512-jQSaVHsPgtyw60e1rQ/A+/ArPEj/S8pS/vFnyGa/gYFXrKk/6RuDkoqVDQ5NI5MmS01698ltlAk0NoDBNLujRw==} + prosemirror-changeset@2.4.1: resolution: {integrity: sha512-96WBLhOaYhJ+kPhLg3uW359Tz6I/MfcrQfL4EGv4SrcqKEMC1gmoGrXHecPE8eOwTVCJ4IwgfzM8fFad25wNfw==} @@ -3434,6 +3622,9 @@ packages: queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + rangetouch@2.0.1: resolution: {integrity: sha512-sln+pNSc8NGaHoLzwNBssFSf/rSYkqeBXzX1AtJlkJiUaVSJSbRAWJk+4omsXkN+EJalzkZhWQ3th1m0FpR5xA==} @@ -3445,6 +3636,13 @@ packages: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} + real-require@0.2.0: + resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} + engines: {node: '>= 12.13.0'} + + real-require@1.0.0: + resolution: {integrity: sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==} + regexparam@3.0.0: resolution: {integrity: sha512-RSYAtP31mvYLkAHrOlh25pCNQ5hWnT106VukGaaFfuJrZFkGRX5GhUAdPqpSDXxOhA2c4akmRuplv1mRqnBn6Q==} engines: {node: '>=8'} @@ -3501,6 +3699,10 @@ packages: resolution: {integrity: sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==} engines: {node: '>=6'} + safe-stable-stringify@2.5.0: + resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} + engines: {node: '>=10'} + safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} @@ -3543,6 +3745,9 @@ packages: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} + sonic-boom@4.2.1: + resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -3567,10 +3772,22 @@ packages: std-env@4.2.0: resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} + string-width@7.2.0: + resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} + engines: {node: '>=18'} + + string-width@8.2.2: + resolution: {integrity: sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==} + engines: {node: '>=20'} + strip-ansi@6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} + strip-ansi@7.2.0: + resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} + engines: {node: '>=12'} + strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -3671,6 +3888,10 @@ packages: thenify@3.3.1: resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} + thread-stream@4.2.0: + resolution: {integrity: sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==} + engines: {node: '>=20'} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -3710,6 +3931,9 @@ packages: ts-interface-checker@0.1.13: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} + ts-morph@27.0.2: + resolution: {integrity: sha512-fhUhgeljcrdZ+9DZND1De1029PrE+cMkIP7ooqkLRTrRLTqcki2AstsyJm0vRNbTbVCNJ0idGlbBrfqc7/nA8w==} + tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} @@ -3764,6 +3988,9 @@ packages: ufo@1.6.4: resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} + uint8arrays@5.1.1: + resolution: {integrity: sha512-9muQwa4wZG4dKi9gMAIBtnk2Pw87SRpvWTH6lOGm19V2Uqxr4uomUf2PGqPnWc+qs06sN8owUU4jfcoWOcfwVQ==} + undici-types@8.3.0: resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} @@ -3778,6 +4005,9 @@ packages: unenv@2.0.0-rc.24: resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} + unicode-segmenter@0.14.5: + resolution: {integrity: sha512-jHGmj2LUuqDcX3hqY12Ql+uhUTn8huuxNZGq7GvtF6bSybzH3aFgedYu/KTzQStEgt1Ra2F3HxadNXsNjb3m3g==} + unicode-segmenter@0.17.3: resolution: {integrity: sha512-hKZwqBjJDmqNrq1+LjDxck1qLzJFcLLJM2Xq92ORRHOuau6GSHeELmn+uyk6zNH1CK9mogrUJGP9WJCJUQXMAw==} @@ -3802,6 +4032,9 @@ packages: typescript: optional: true + varint@6.0.0: + resolution: {integrity: sha512-cXEIW6cfr15lFv563k4GuVuW/fiwjknytD37jIOLSdSWuOI6WnO/oKwmP2FQTU2l01LP8/M5TSAJpzUaGe3uWg==} + vite@8.2.0: resolution: {integrity: sha512-pn+CFpM0lwDeKwmOq1ZaBK/9sjorZcgqxki6MbY/jPEVd9vichIlmlD4HmQ5wdP5EgqQCFRaACBxMC7uEGc6lQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -3962,6 +4195,10 @@ packages: '@cloudflare/workers-types': optional: true + wrap-ansi@9.0.2: + resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} + engines: {node: '>=18'} + ws@8.18.0: resolution: {integrity: sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==} engines: {node: '>=10.0.0'} @@ -3990,10 +4227,22 @@ packages: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + yaml@1.10.3: resolution: {integrity: sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA==} engines: {node: '>= 6'} + yargs-parser@22.0.0: + resolution: {integrity: sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + + yargs@18.1.0: + resolution: {integrity: sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -4011,6 +4260,9 @@ packages: zimmerframe@1.1.4: resolution: {integrity: sha512-B58NGBEoc8Y9MWWCQGl/gq9xBCe4IiKM0a2x7GZdQKOW5Exr8S1W24J6OgM1njK8xCRGvAJIL/MxXHf6SkmQKQ==} + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + snapshots: '@atcute/atproto@3.1.12(@atcute/lexicons@2.0.3)': @@ -4237,8 +4489,166 @@ snapshots: - '@atcute/cid' - typescript + '@atproto-labs/did-resolver@0.3.7': + dependencies: + '@atproto-labs/fetch': 0.3.5 + '@atproto-labs/pipe': 0.2.4 + '@atproto-labs/simple-store': 0.5.1 + '@atproto-labs/simple-store-memory': 0.2.6 + '@atproto/did': 0.5.4 + zod: 3.25.76 + + '@atproto-labs/fetch@0.3.5': + dependencies: + '@atproto-labs/pipe': 0.2.4 + + '@atproto-labs/pipe@0.2.4': {} + + '@atproto-labs/simple-store-memory@0.2.6': + dependencies: + '@atproto-labs/simple-store': 0.5.1 + lru-cache: 10.4.3 + + '@atproto-labs/simple-store@0.5.1': {} + + '@atproto/common-web@0.5.10': + dependencies: + '@atproto/lex-data': 0.1.7 + '@atproto/lex-json': 0.1.6 + '@atproto/syntax': 0.7.5 + zod: 3.25.76 + + '@atproto/common@0.8.1': + dependencies: + '@atproto/common-web': 0.5.10 + '@atproto/lex-cbor': 0.1.6 + '@atproto/lex-data': 0.1.7 + multiformats: 13.4.2 + pino: 10.3.1 + + '@atproto/crypto@0.5.4': + dependencies: + '@noble/curves': 1.9.7 + '@noble/hashes': 1.8.0 + uint8arrays: 5.1.1 + + '@atproto/did@0.5.4': + dependencies: + zod: 3.25.76 + + '@atproto/lex-builder@0.1.13': + dependencies: + '@atproto/lex-document': 0.1.10 + '@atproto/lex-schema': 0.2.6 + prettier: 3.9.6 + ts-morph: 27.0.2 + tslib: 2.8.1 + + '@atproto/lex-cbor@0.1.6': + dependencies: + '@atproto/lex-data': 0.1.7 + cborg: 4.5.8 + tslib: 2.8.1 + + '@atproto/lex-client@0.3.5': + dependencies: + '@atproto/lex-data': 0.1.7 + '@atproto/lex-json': 0.1.6 + '@atproto/lex-schema': 0.2.6 + tslib: 2.8.1 + + '@atproto/lex-data@0.1.7': + dependencies: + multiformats: 13.4.2 + tslib: 2.8.1 + unicode-segmenter: 0.14.5 + + '@atproto/lex-document@0.1.10': + dependencies: + '@atproto/lex-schema': 0.2.6 + core-js: 3.50.0 + tslib: 2.8.1 + + '@atproto/lex-installer@0.1.17': + dependencies: + '@atproto/lex-builder': 0.1.13 + '@atproto/lex-cbor': 0.1.6 + '@atproto/lex-data': 0.1.7 + '@atproto/lex-document': 0.1.10 + '@atproto/lex-resolver': 0.2.10 + '@atproto/lex-schema': 0.2.6 + '@atproto/syntax': 0.7.5 + tslib: 2.8.1 + + '@atproto/lex-json@0.1.6': + dependencies: + '@atproto/lex-data': 0.1.7 + tslib: 2.8.1 + + '@atproto/lex-resolver@0.2.10': + dependencies: + '@atproto-labs/did-resolver': 0.3.7 + '@atproto/crypto': 0.5.4 + '@atproto/lex-client': 0.3.5 + '@atproto/lex-data': 0.1.7 + '@atproto/lex-document': 0.1.10 + '@atproto/lex-schema': 0.2.6 + '@atproto/repo': 0.10.12 + '@atproto/syntax': 0.7.5 + tslib: 2.8.1 + + '@atproto/lex-schema@0.2.6': + dependencies: + '@atproto/lex-data': 0.1.7 + '@atproto/syntax': 0.7.5 + '@standard-schema/spec': 1.1.0 + tslib: 2.8.1 + + '@atproto/lex@0.3.8': + dependencies: + '@atproto/lex-builder': 0.1.13 + '@atproto/lex-client': 0.3.5 + '@atproto/lex-data': 0.1.7 + '@atproto/lex-installer': 0.1.17 + '@atproto/lex-json': 0.1.6 + '@atproto/lex-schema': 0.2.6 + tslib: 2.8.1 + yargs: 18.1.0 + + '@atproto/repo@0.10.12': + dependencies: + '@atproto/common': 0.8.1 + '@atproto/common-web': 0.5.10 + '@atproto/crypto': 0.5.4 + '@atproto/lex-cbor': 0.1.6 + '@atproto/lex-data': 0.1.7 + '@atproto/syntax': 0.7.5 + varint: 6.0.0 + zod: 3.25.76 + + '@atproto/syntax@0.7.5': + dependencies: + iso-datestring-validator: 2.2.2 + tslib: 2.8.1 + + '@atproto/ws-client@0.2.1': + dependencies: + ws: 8.21.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + '@babel/runtime@7.29.7': {} + '@bsky/jetstream@1.0.0(@atproto/lex@0.3.8)': + dependencies: + '@atproto/lex': 0.3.8 + '@atproto/lex-cbor': 0.1.6 + '@atproto/ws-client': 0.2.1 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + '@changesets/apply-release-plan@7.1.1': dependencies: '@changesets/config': 3.1.4 @@ -5117,6 +5527,12 @@ snapshots: '@napi-rs/lzma-linux-x64-gnu@1.5.1': optional: true + '@noble/curves@1.9.7': + dependencies: + '@noble/hashes': 1.8.0 + + '@noble/hashes@1.8.0': {} + '@noble/secp256k1@3.1.0': {} '@nodelib/fs.scandir@2.1.5': @@ -5147,6 +5563,8 @@ snapshots: '@oxc-project/types@0.142.0': {} + '@pinojs/redact@0.4.0': {} + '@polka/url@1.0.0-next.29': {} '@poppinss/colors@4.1.6': @@ -5595,6 +6013,12 @@ snapshots: '@tiptap/core': 3.29.2(@tiptap/pm@3.29.2) '@tiptap/pm': 3.29.2 + '@ts-morph/common@0.28.1': + dependencies: + minimatch: 10.2.6 + path-browserify: 1.0.1 + tinyglobby: 0.2.17 + '@turbo/darwin-64@2.10.8': optional: true @@ -5811,6 +6235,10 @@ snapshots: ansi-regex@5.0.1: {} + ansi-regex@6.3.0: {} + + ansi-styles@6.2.3: {} + any-promise@1.3.0: {} argparse@1.0.10: @@ -5825,6 +6253,8 @@ snapshots: assertion-error@2.0.1: {} + atomic-sleep@1.0.0: {} + axobject-query@4.1.0: {} balanced-match@4.0.4: {} @@ -5865,6 +6295,8 @@ snapshots: cac@7.0.0: {} + cborg@4.5.8: {} + chai@6.2.2: {} chardet@2.2.0: {} @@ -5873,8 +6305,16 @@ snapshots: dependencies: readdirp: 4.1.2 + cliui@9.0.1: + dependencies: + string-width: 7.2.0 + strip-ansi: 7.2.0 + wrap-ansi: 9.0.2 + clsx@2.1.1: {} + code-block-writer@13.0.3: {} + commander@4.1.1: {} confbox@0.1.8: {} @@ -5889,6 +6329,8 @@ snapshots: core-js@3.49.0: {} + core-js@3.50.0: {} + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -5929,6 +6371,8 @@ snapshots: emoji-picker-element@1.29.1: {} + emoji-regex@10.6.0: {} + enhanced-resolve@5.24.5: dependencies: graceful-fs: 4.2.11 @@ -6030,6 +6474,8 @@ snapshots: '@esbuild/win32-ia32': 0.28.1 '@esbuild/win32-x64': 0.28.1 + escalade@3.2.0: {} + escape-string-regexp@4.0.0: {} eslint-config-prettier@10.1.8(eslint@10.8.0(jiti@2.7.0)): @@ -6221,6 +6667,10 @@ snapshots: fsevents@2.3.3: optional: true + get-caller-file@2.0.5: {} + + get-east-asian-width@1.6.0: {} + glob-parent@5.1.2: dependencies: is-glob: 4.0.3 @@ -6286,6 +6736,8 @@ snapshots: isexe@2.0.0: {} + iso-datestring-validator@2.2.2: {} + jiti@2.7.0: {} joycon@3.1.1: {} @@ -6450,6 +6902,8 @@ snapshots: devlop: 1.1.0 highlight.js: 11.11.1 + lru-cache@10.4.3: {} + lz-string@1.5.0: {} magic-string@0.30.21: @@ -6514,6 +6968,8 @@ snapshots: ms@2.1.3: {} + multiformats@13.4.2: {} + mz@2.7.0: dependencies: any-promise: 1.3.0 @@ -6536,6 +6992,8 @@ snapshots: obug@2.1.4: {} + on-exit-leak-free@2.1.2: {} + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -6581,6 +7039,8 @@ snapshots: dependencies: event-target-polyfill: 0.0.4 + path-browserify@1.0.1: {} + path-exists@4.0.0: {} path-key@3.1.1: {} @@ -6634,6 +7094,26 @@ snapshots: pify@4.0.1: {} + pino-abstract-transport@3.0.0: + dependencies: + split2: 4.2.0 + + pino-std-serializers@7.1.0: {} + + pino@10.3.1: + dependencies: + '@pinojs/redact': 0.4.0 + atomic-sleep: 1.0.0 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 3.0.0 + pino-std-serializers: 7.1.0 + process-warning: 5.1.0 + quick-format-unescaped: 4.0.4 + real-require: 0.2.0 + safe-stable-stringify: 2.5.0 + sonic-boom: 4.2.1 + thread-stream: 4.2.0 + pirates@4.0.7: {} pkg-types@1.3.1: @@ -6711,6 +7191,8 @@ snapshots: prettier@3.9.6: {} + process-warning@5.1.0: {} + prosemirror-changeset@2.4.1: dependencies: prosemirror-transform: 1.12.0 @@ -6791,6 +7273,8 @@ snapshots: queue-microtask@1.2.3: {} + quick-format-unescaped@4.0.4: {} + rangetouch@2.0.1: {} read-yaml-file@1.1.0: @@ -6802,6 +7286,10 @@ snapshots: readdirp@4.1.2: {} + real-require@0.2.0: {} + + real-require@1.0.0: {} + regexparam@3.0.0: {} resolve-from@5.0.0: {} @@ -6894,6 +7382,8 @@ snapshots: dependencies: mri: 1.2.0 + safe-stable-stringify@2.5.0: {} + safer-buffer@2.1.2: {} semver@7.8.5: {} @@ -6981,6 +7471,10 @@ snapshots: slash@3.0.0: {} + sonic-boom@4.2.1: + dependencies: + atomic-sleep: 1.0.0 + source-map-js@1.2.1: {} source-map@0.7.6: {} @@ -6998,10 +7492,25 @@ snapshots: std-env@4.2.0: {} + string-width@7.2.0: + dependencies: + emoji-regex: 10.6.0 + get-east-asian-width: 1.6.0 + strip-ansi: 7.2.0 + + string-width@8.2.2: + dependencies: + get-east-asian-width: 1.6.0 + strip-ansi: 7.2.0 + strip-ansi@6.0.1: dependencies: ansi-regex: 5.0.1 + strip-ansi@7.2.0: + dependencies: + ansi-regex: 6.3.0 + strip-bom@3.0.0: {} style-to-object@1.0.14: @@ -7119,6 +7628,10 @@ snapshots: dependencies: any-promise: 1.3.0 + thread-stream@4.2.0: + dependencies: + real-require: 1.0.0 + tinybench@2.9.0: {} tinyexec@0.3.2: {} @@ -7146,6 +7659,11 @@ snapshots: ts-interface-checker@0.1.13: {} + ts-morph@27.0.2: + dependencies: + '@ts-morph/common': 0.28.1 + code-block-writer: 13.0.3 + tslib@2.8.1: {} tsup@8.5.1(jiti@2.7.0)(postcss@8.5.25)(tsx@4.23.6)(typescript@6.0.3): @@ -7212,6 +7730,10 @@ snapshots: ufo@1.6.4: {} + uint8arrays@5.1.1: + dependencies: + multiformats: 13.4.2 + undici-types@8.3.0: {} undici@7.24.8: {} @@ -7222,6 +7744,8 @@ snapshots: dependencies: pathe: 2.0.3 + unicode-segmenter@0.14.5: {} + unicode-segmenter@0.17.3: {} universalify@0.1.2: {} @@ -7238,6 +7762,8 @@ snapshots: optionalDependencies: typescript: 6.0.3 + varint@6.0.0: {} + vite@8.2.0(@types/node@26.1.2)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.23.6): dependencies: lightningcss: 1.33.0 @@ -7396,14 +7922,33 @@ snapshots: - bufferutil - utf-8-validate + wrap-ansi@9.0.2: + dependencies: + ansi-styles: 6.2.3 + string-width: 7.2.0 + strip-ansi: 7.2.0 + ws@8.18.0: {} ws@8.21.0: {} xtend@4.0.2: {} + y18n@5.0.8: {} + yaml@1.10.3: {} + yargs-parser@22.0.0: {} + + yargs@18.1.0: + dependencies: + cliui: 9.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + string-width: 8.2.2 + y18n: 5.0.8 + yargs-parser: 22.0.0 + yocto-queue@0.1.0: {} yocto-queue@1.2.2: {} @@ -7422,3 +7967,5 @@ snapshots: youch-core: 0.3.3 zimmerframe@1.1.4: {} + + zod@3.25.76: {} -- 2.54.0 From cd9545e41731039940e00c55194b7871d6e6c5a7 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:04:56 +0200 Subject: [PATCH 2/2] fix clean-install lexicon checks --- apps/cloudflare-workers/package.json | 2 +- apps/sveltekit-cloudflare-workers/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/cloudflare-workers/package.json b/apps/cloudflare-workers/package.json index 8b6926d..5e215fd 100644 --- a/apps/cloudflare-workers/package.json +++ b/apps/cloudflare-workers/package.json @@ -6,7 +6,7 @@ "scripts": { "dev": "contrail dev", "deploy": "wrangler deploy", - "typecheck": "contrail lexicons check --no-atcute-config && tsc --noEmit" + "typecheck": "node ../../packages/contrail/dist/cli.js lexicons check --no-atcute-config && tsc --noEmit" }, "dependencies": { "@atmo-dev/contrail": "workspace:*" diff --git a/apps/sveltekit-cloudflare-workers/package.json b/apps/sveltekit-cloudflare-workers/package.json index dbcacdb..a33e14b 100644 --- a/apps/sveltekit-cloudflare-workers/package.json +++ b/apps/sveltekit-cloudflare-workers/package.json @@ -12,7 +12,7 @@ "preview": "vite preview", "prepare": "svelte-kit sync || echo ''", "check": "pnpm generate:types && svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", - "typecheck": "contrail lexicons check --no-atcute-config && pnpm check", + "typecheck": "node ../../packages/contrail/dist/cli.js lexicons check --no-atcute-config && pnpm check", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", "format": "prettier --write .", "lint": "prettier --check . && eslint .",